Shell_Project (1).ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Shell Project.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "9qSCVq_elmtY"
      },
      "source": [
        "import numpy as np\n",
        "import tensorflow as tf\n",
        "from tensorflow import keras\n",
        "import pandas as pd\n",
        "import seaborn as sns\n",
        "from pylab import rcParams\n",
        "import matplotlib.pyplot as plt\n",
        "from matplotlib import rc\n",
        "from sklearn.model_selection import train_test_split\n",
        "from pandas.plotting import register_matplotlib_converters\n",
        "\n",
        "%matplotlib inline\n",
        "%config InlineBackend.figure_format='retina'\n",
        "\n",
        "register_matplotlib_converters()\n",
        "sns.set(style='whitegrid', palette='muted', font_scale=1.5)\n",
        "\n",
        "rcParams['figure.figsize'] = 22, 10\n",
        "\n",
        "RANDOM_SEED = 42\n",
        "\n",
        "np.random.seed(RANDOM_SEED)\n",
        "tf.random.set_seed(RANDOM_SEED)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Z3-Ln42Rlnl3",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "eff87828-b38b-4917-9ab5-0b8743d80111"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QvslfDWPmELJ"
      },
      "source": [
        "df = pd.read_csv(\"/content/drive/MyDrive/test/train.csv\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "G9xYxV6FojnL",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "f539220d-011f-4fec-f7a5-bbcd155114a0"
      },
      "source": [
        "df.shape"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(527040, 17)"
            ]
          },
          "metadata": {},
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 270
        },
        "id": "FY_MCGFNLwIW",
        "outputId": "f8bbc844-44c6-4ef7-c003-22971721d0a5"
      },
      "source": [
        "df.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>DATE (MM/DD)</th>\n",
              "      <th>MST</th>\n",
              "      <th>Global CMP22 (vent/cor) [W/m^2]</th>\n",
              "      <th>Direct sNIP [W/m^2]</th>\n",
              "      <th>Azimuth Angle [degrees]</th>\n",
              "      <th>Tower Dry Bulb Temp [deg C]</th>\n",
              "      <th>Tower Wet Bulb Temp [deg C]</th>\n",
              "      <th>Tower Dew Point Temp [deg C]</th>\n",
              "      <th>Tower RH [%]</th>\n",
              "      <th>Total Cloud Cover [%]</th>\n",
              "      <th>Peak Wind Speed @ 6ft [m/s]</th>\n",
              "      <th>Avg Wind Direction @ 6ft [deg from N]</th>\n",
              "      <th>Station Pressure [mBar]</th>\n",
              "      <th>Precipitation (Accumulated) [mm]</th>\n",
              "      <th>Snow Depth [cm]</th>\n",
              "      <th>Moisture</th>\n",
              "      <th>Albedo (CMP11)</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1/1</td>\n",
              "      <td>00:00</td>\n",
              "      <td>-0.962276</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>356.85640</td>\n",
              "      <td>7.216</td>\n",
              "      <td>0.988</td>\n",
              "      <td>-7.312</td>\n",
              "      <td>32.33</td>\n",
              "      <td>-1</td>\n",
              "      <td>9.95</td>\n",
              "      <td>271.3</td>\n",
              "      <td>806.779</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.219</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1/1</td>\n",
              "      <td>00:01</td>\n",
              "      <td>-0.937921</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>357.65505</td>\n",
              "      <td>7.251</td>\n",
              "      <td>1.040</td>\n",
              "      <td>-7.260</td>\n",
              "      <td>32.40</td>\n",
              "      <td>-1</td>\n",
              "      <td>8.20</td>\n",
              "      <td>272.9</td>\n",
              "      <td>806.840</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.206</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1/1</td>\n",
              "      <td>00:02</td>\n",
              "      <td>-0.944395</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>358.45438</td>\n",
              "      <td>7.256</td>\n",
              "      <td>1.093</td>\n",
              "      <td>-7.207</td>\n",
              "      <td>32.54</td>\n",
              "      <td>-1</td>\n",
              "      <td>6.70</td>\n",
              "      <td>288.8</td>\n",
              "      <td>806.876</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.148</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>1/1</td>\n",
              "      <td>00:03</td>\n",
              "      <td>-0.951350</td>\n",
              "      <td>-0.029673</td>\n",
              "      <td>359.25416</td>\n",
              "      <td>7.254</td>\n",
              "      <td>1.060</td>\n",
              "      <td>-7.440</td>\n",
              "      <td>31.89</td>\n",
              "      <td>-1</td>\n",
              "      <td>7.70</td>\n",
              "      <td>294.0</td>\n",
              "      <td>806.823</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.235</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>1/1</td>\n",
              "      <td>00:04</td>\n",
              "      <td>-0.934976</td>\n",
              "      <td>-0.054401</td>\n",
              "      <td>0.05415</td>\n",
              "      <td>7.331</td>\n",
              "      <td>1.081</td>\n",
              "      <td>-7.419</td>\n",
              "      <td>31.78</td>\n",
              "      <td>-1</td>\n",
              "      <td>7.20</td>\n",
              "      <td>285.5</td>\n",
              "      <td>806.762</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.182</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "  DATE (MM/DD)    MST  ...  Moisture  Albedo (CMP11)\n",
              "0          1/1  00:00  ...       0.0             0.0\n",
              "1          1/1  00:01  ...       0.0             0.0\n",
              "2          1/1  00:02  ...       0.0             0.0\n",
              "3          1/1  00:03  ...       0.0             0.0\n",
              "4          1/1  00:04  ...       0.0             0.0\n",
              "\n",
              "[5 rows x 17 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pi0RNcp2oDCR"
      },
      "source": [
        "df = df[df['Total Cloud Cover [%]'] >= 0]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "27nbsfZz7hos"
      },
      "source": [
        "#df.corr()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jDby71o9D_9i"
      },
      "source": [
        "df.rename(columns = {'Direct sNIP [W/m^2]':'snip', 'Total Cloud Cover [%]': 'cloud cover', \n",
        "                     'Peak Wind Speed @ 6ft [m/s]': 'wind speed', \n",
        "                     'Avg Wind Direction @ 6ft [deg from N]': 'wind dir',\n",
        "                     'Tower RH [%]': 'rh', 'Global CMP22 (vent/cor) [W/m^2]': 'cmp22',\n",
        "                     'Tower Dew Point Temp [deg C]': 'temp', 'Moisture': 'moisture',\n",
        "                     }, inplace = True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 203
        },
        "id": "lduKA7NCCo-Y",
        "outputId": "f1e97a45-ffa5-47c1-f503-e3f7a5849b5b"
      },
      "source": [
        "req_cols = ['snip', 'wind speed', 'wind dir', 'rh', 'cmp22', 'temp', 'moisture', 'cloud cover']\n",
        "\n",
        "df = df[req_cols]\n",
        "\n",
        "df.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>snip</th>\n",
              "      <th>wind speed</th>\n",
              "      <th>wind dir</th>\n",
              "      <th>rh</th>\n",
              "      <th>cmp22</th>\n",
              "      <th>temp</th>\n",
              "      <th>moisture</th>\n",
              "      <th>cloud cover</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>467</th>\n",
              "      <td>0.079127</td>\n",
              "      <td>0.7</td>\n",
              "      <td>210.3</td>\n",
              "      <td>38.24</td>\n",
              "      <td>14.4935</td>\n",
              "      <td>-5.597</td>\n",
              "      <td>0.0</td>\n",
              "      <td>6</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>468</th>\n",
              "      <td>-0.034618</td>\n",
              "      <td>0.7</td>\n",
              "      <td>232.2</td>\n",
              "      <td>38.58</td>\n",
              "      <td>16.5178</td>\n",
              "      <td>-5.606</td>\n",
              "      <td>0.0</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>469</th>\n",
              "      <td>-0.153309</td>\n",
              "      <td>1.7</td>\n",
              "      <td>263.9</td>\n",
              "      <td>39.12</td>\n",
              "      <td>18.6171</td>\n",
              "      <td>-5.540</td>\n",
              "      <td>0.0</td>\n",
              "      <td>10</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>470</th>\n",
              "      <td>-0.143419</td>\n",
              "      <td>2.2</td>\n",
              "      <td>277.5</td>\n",
              "      <td>39.35</td>\n",
              "      <td>20.5986</td>\n",
              "      <td>-5.494</td>\n",
              "      <td>0.0</td>\n",
              "      <td>13</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>471</th>\n",
              "      <td>-0.079128</td>\n",
              "      <td>2.2</td>\n",
              "      <td>233.4</td>\n",
              "      <td>39.64</td>\n",
              "      <td>22.4362</td>\n",
              "      <td>-5.384</td>\n",
              "      <td>0.0</td>\n",
              "      <td>15</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "         snip  wind speed  wind dir  ...   temp  moisture  cloud cover\n",
              "467  0.079127         0.7     210.3  ... -5.597       0.0            6\n",
              "468 -0.034618         0.7     232.2  ... -5.606       0.0            8\n",
              "469 -0.153309         1.7     263.9  ... -5.540       0.0           10\n",
              "470 -0.143419         2.2     277.5  ... -5.494       0.0           13\n",
              "471 -0.079128         2.2     233.4  ... -5.384       0.0           15\n",
              "\n",
              "[5 rows x 8 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "t4z60-Pt7lmV",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "deec59ab-0619-4dae-f9a3-e825db5556c6"
      },
      "source": [
        "df.shape"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(250731, 8)"
            ]
          },
          "metadata": {},
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RAjPm_lh-2Pq",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "7a76335d-c86e-417d-c9d5-fdac1c493695"
      },
      "source": [
        "train_size = int(len(df) * 0.9)\n",
        "test_size = len(df) - train_size\n",
        "train, test = df.iloc[0:train_size], df.iloc[train_size:len(df)]\n",
        "print(len(train), len(test))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "225657 25074\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Qa2DEC4ccbDM"
      },
      "source": [
        "# Preprocessing"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 203
        },
        "id": "VXmORlMfPMVf",
        "outputId": "4ac6dd6d-f89b-4c3e-db8b-8095c5cc78a1"
      },
      "source": [
        "df.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>snip</th>\n",
              "      <th>wind speed</th>\n",
              "      <th>wind dir</th>\n",
              "      <th>rh</th>\n",
              "      <th>cmp22</th>\n",
              "      <th>temp</th>\n",
              "      <th>moisture</th>\n",
              "      <th>cloud cover</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>467</th>\n",
              "      <td>0.079127</td>\n",
              "      <td>0.7</td>\n",
              "      <td>210.3</td>\n",
              "      <td>38.24</td>\n",
              "      <td>14.4935</td>\n",
              "      <td>-5.597</td>\n",
              "      <td>0.0</td>\n",
              "      <td>6</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>468</th>\n",
              "      <td>-0.034618</td>\n",
              "      <td>0.7</td>\n",
              "      <td>232.2</td>\n",
              "      <td>38.58</td>\n",
              "      <td>16.5178</td>\n",
              "      <td>-5.606</td>\n",
              "      <td>0.0</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>469</th>\n",
              "      <td>-0.153309</td>\n",
              "      <td>1.7</td>\n",
              "      <td>263.9</td>\n",
              "      <td>39.12</td>\n",
              "      <td>18.6171</td>\n",
              "      <td>-5.540</td>\n",
              "      <td>0.0</td>\n",
              "      <td>10</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>470</th>\n",
              "      <td>-0.143419</td>\n",
              "      <td>2.2</td>\n",
              "      <td>277.5</td>\n",
              "      <td>39.35</td>\n",
              "      <td>20.5986</td>\n",
              "      <td>-5.494</td>\n",
              "      <td>0.0</td>\n",
              "      <td>13</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>471</th>\n",
              "      <td>-0.079128</td>\n",
              "      <td>2.2</td>\n",
              "      <td>233.4</td>\n",
              "      <td>39.64</td>\n",
              "      <td>22.4362</td>\n",
              "      <td>-5.384</td>\n",
              "      <td>0.0</td>\n",
              "      <td>15</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "         snip  wind speed  wind dir  ...   temp  moisture  cloud cover\n",
              "467  0.079127         0.7     210.3  ... -5.597       0.0            6\n",
              "468 -0.034618         0.7     232.2  ... -5.606       0.0            8\n",
              "469 -0.153309         1.7     263.9  ... -5.540       0.0           10\n",
              "470 -0.143419         2.2     277.5  ... -5.494       0.0           13\n",
              "471 -0.079128         2.2     233.4  ... -5.384       0.0           15\n",
              "\n",
              "[5 rows x 8 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Mw7dc-GRgIHc",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "392bb4bb-91c6-4a56-97e2-41d2c3c275a6"
      },
      "source": [
        "from sklearn.preprocessing import RobustScaler\n",
        "\n",
        "f_columns = ['snip', 'wind speed', 'wind dir', 'rh', 'cmp22', 'temp', 'moisture']\n",
        "\n",
        "f_transformer = RobustScaler()\n",
        "cnt_transformer = RobustScaler()\n",
        "\n",
        "f_transformer = f_transformer.fit(train[f_columns].to_numpy())\n",
        "cnt_transformer = cnt_transformer.fit(train[['cloud cover']])\n",
        "\n",
        "train.loc[:, f_columns] = f_transformer.transform(train[f_columns].to_numpy())\n",
        "train['cloud cover'] = cnt_transformer.transform(train[['cloud cover']])\n",
        "\n",
        "test.loc[:, f_columns] = f_transformer.transform(test[f_columns].to_numpy())\n",
        "test['cloud cover'] = cnt_transformer.transform(test[['cloud cover']])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/pandas/core/indexing.py:1734: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  isetter(loc, value[:, i].tolist())\n",
            "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:12: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  if sys.path[0] == '':\n",
            "/usr/local/lib/python3.7/dist-packages/pandas/core/indexing.py:1734: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  isetter(loc, value[:, i].tolist())\n",
            "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:15: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  from ipykernel import kernelapp as app\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "iOXU1mwYPlkA",
        "outputId": "8d553aab-2eeb-4577-ce54-4420d0d57409"
      },
      "source": [
        "train['snip']"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "467      -0.620828\n",
              "468      -0.620959\n",
              "469      -0.621095\n",
              "470      -0.621084\n",
              "471      -0.621010\n",
              "            ...   \n",
              "458525   -0.576519\n",
              "458526   -0.553941\n",
              "458527   -0.549056\n",
              "458528   -0.608804\n",
              "458529   -0.609406\n",
              "Name: snip, Length: 225657, dtype: float64"
            ]
          },
          "metadata": {},
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cIUaVnCE-2nm"
      },
      "source": [
        "def create_dataset(X, y, time_steps):\n",
        "    Xs, ys = [], []\n",
        "    for i in range(len(X) - time_steps):\n",
        "        v = X.iloc[i:(i + time_steps)].values\n",
        "        Xs.append(v)        \n",
        "        ys.append(y.iloc[i + time_steps])\n",
        "    return np.array(Xs), np.array(ys)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gywcHGA6-4oW",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "40c3631c-6bf4-40e0-a0b5-7bd2a37fee12"
      },
      "source": [
        "time_steps = 120\n",
        "\n",
        "# reshape to [samples, time_steps, n_features]\n",
        "\n",
        "X_train, y_train = create_dataset(train, train['cloud cover'], time_steps)\n",
        "X_test, y_test = create_dataset(test, test['cloud cover'], time_steps)\n",
        "\n",
        "print(X_train.shape, y_train.shape)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(225537, 120, 8) (225537,)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Tz9KofSH-51-"
      },
      "source": [
        "model = keras.Sequential()\n",
        "model.add(\n",
        "  keras.layers.Bidirectional(\n",
        "    keras.layers.LSTM(\n",
        "      units=64, \n",
        "      input_shape=(X_train.shape[1], X_train.shape[2]),\n",
        "      return_sequences=True\n",
        "    )\n",
        "  )\n",
        ")\n",
        "model.add(keras.layers.Dropout(rate=0.1))\n",
        "\n",
        "model.add(\n",
        "    keras.layers.LSTM(\n",
        "      units=16, \n",
        "      input_shape=(X_train.shape[1], X_train.shape[2])\n",
        "    )\n",
        ")\n",
        "model.add(keras.layers.Dropout(rate=0.1))\n",
        "\n",
        "model.add(keras.layers.Dense(units=1))\n",
        "model.compile(loss='mean_squared_error', optimizer='adam')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uqA3dyPoDcJc"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uufvbrJm_Cry",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "34af9d3a-f6e1-480c-8558-0611434e785f"
      },
      "source": [
        "history = model.fit(\n",
        "    X_train, y_train, \n",
        "    epochs=50, \n",
        "    batch_size=360, \n",
        "    validation_split=0.1,\n",
        "    shuffle=False\n",
        ")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/50\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9dixVkNP2GEN"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "11Dyc1iX_D8X"
      },
      "source": [
        "plt.plot(history.history['loss'], label='train')\n",
        "plt.plot(history.history['val_loss'], label='test')\n",
        "plt.legend();"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FLGKCn9h_Fzg"
      },
      "source": [
        "y_pred = model.predict(X_test)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "c3dHgIxphGPx"
      },
      "source": [
        "y_train_inv = cnt_transformer.inverse_transform(y_train.reshape(1, -1))\n",
        "y_test_inv = cnt_transformer.inverse_transform(y_test.reshape(1, -1))\n",
        "y_pred_inv = cnt_transformer.inverse_transform(y_pred)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zknFRCgtAFVn"
      },
      "source": [
        "'''\n",
        "plt.plot(np.arange(0, len(y_train)), y_train_inv.flatten(), 'g', label=\"history\")\n",
        "plt.plot(np.arange(len(y_train), len(y_train) + len(y_test)), y_test_inv.flatten(), marker='.', label=\"true\")\n",
        "plt.plot(np.arange(len(y_train), len(y_train) + len(y_test)), y_pred_inv.flatten(), 'r', label=\"prediction\")\n",
        "plt.ylabel('cloud cover')\n",
        "plt.xlabel('Time Step')\n",
        "plt.legend()\n",
        "plt.show();\n",
        "'''"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "C0SiahA8BF4y"
      },
      "source": [
        "plt.plot(y_test_inv.flatten(), marker='.', label=\"true\")\n",
        "plt.plot(y_pred_inv.flatten(), 'r', label=\"prediction\")\n",
        "plt.ylabel('cloud cover')\n",
        "plt.xlabel('Time Step')\n",
        "plt.legend()\n",
        "plt.show();"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MGnlvOuCsem2"
      },
      "source": [
        "temp = 300\n",
        "\n",
        "vals = np.zeros(temp)\n",
        "\n",
        "for i in range(1,temp+1):\n",
        "  df_test = pd.read_csv('/content/drive/MyDrive/test/{}/weather_data.csv'.format(i))\n",
        "  df_test = df_test[df_test['Total Cloud Cover [%]'] >= 0]\n",
        "\n",
        "  df_test.rename(columns = {'Direct sNIP [W/m^2]':'snip', 'Total Cloud Cover [%]': 'cloud cover', \n",
        "                     'Peak Wind Speed @ 6ft [m/s]': 'wind speed', \n",
        "                     'Avg Wind Direction @ 6ft [deg from N]': 'wind dir',\n",
        "                     'Tower RH [%]': 'rh', 'Global CMP22 (vent/cor) [W/m^2]': 'cmp22',\n",
        "                     'Tower Dew Point Temp [deg C]': 'temp', 'Moisture': 'moisture',}, inplace = True)\n",
        "  req_cols = ['snip', 'wind speed', 'wind dir', 'rh', 'cmp22', 'temp', 'moisture', 'cloud cover']\n",
        "  df_test = df_test[req_cols]\n",
        "\n",
        "  df_test.loc[:, f_columns] = f_transformer.transform(df_test[f_columns].to_numpy())\n",
        "  df_test['cloud cover'] = cnt_transformer.transform(df_test[['cloud cover']])\n",
        "\n",
        "  X_test, y_test = create_dataset(df_test, df_test['cloud cover'], time_steps)\n",
        "  \n",
        "  y_pred = model.predict(X_test)\n",
        "\n",
        "  y_test_inv = cnt_transformer.inverse_transform(y_test.reshape(1, -1))\n",
        "  y_pred_inv = cnt_transformer.inverse_transform(y_pred)\n",
        "\n",
        "  vals[i-1] = y_pred_inv[-1]\n",
        "  print(i, end=' ')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dYx8QTu74rad"
      },
      "source": [
        "vals.shape"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "C9C5yDyT57Yg"
      },
      "source": [
        "pd.DataFrame(vals).to_csv(\"/content/drive/MyDrive/test/datanew{}.csv\".format(time_steps))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5hkXOVD4klXZ"
      },
      "source": [
        "plt.plot(np.arange(0, y_test_inv.shape[1]) , y_test_inv[0], 'r', np.arange(0, y_test_inv.shape[1]), y_pred_inv, 'b')\n",
        "plt.show()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uSFvSh2Jlbbx"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "maSn_gQg27yE"
      },
      "source": [
        "vals"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bRG47AvL3BPz"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}