covid-19TimeSeries.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "c5fCBlu-BXfI"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "setup complete\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/statsmodels/tools/_testing.py:19: FutureWarning: pandas.util.testing is deprecated. Use the functions in the public API at pandas.testing instead.\n",
      "  import pandas.util.testing as tm\n"
     ]
    }
   ],
   "source": [
    "from datetime import datetime\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from scipy import stats\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import statsmodels.api as sm\n",
    "from statsmodels.graphics.api import qqplot\n",
    "import math\n",
    "import time\n",
    "from sklearn.linear_model import LinearRegression, BayesianRidge\n",
    "from sklearn.model_selection import RandomizedSearchCV, train_test_split\n",
    "from sklearn.preprocessing import PolynomialFeatures\n",
    "from sklearn.svm import SVR\n",
    "from sklearn.metrics import mean_squared_error, mean_absolute_error\n",
    "\n",
    "\n",
    "import matplotlib.pylab as plt\n",
    "%matplotlib inline\n",
    "from matplotlib.pylab import rcParams\n",
    "from statsmodels.tsa.stattools import adfuller\n",
    "from statsmodels.tsa.stattools import acf, pacf\n",
    "rcParams['figure.figsize'] = 30, 10\n",
    "print('setup complete')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "conf_df = pd.read_csv('https://raw.githubusercontent.com/CSSEGISandData/COVID-19/master/csse_covid_19_data/csse_covid_19_time_series/time_series_19-covid-Confirmed.csv')\n",
    "death_df = pd.read_csv('https://raw.githubusercontent.com/CSSEGISandData/COVID-19/master/csse_covid_19_data/csse_covid_19_time_series/time_series_19-covid-Deaths.csv')\n",
    "reco_df = pd.read_csv('https://raw.githubusercontent.com/CSSEGISandData/COVID-19/master/csse_covid_19_data/csse_covid_19_time_series/time_series_19-covid-Recovered.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import datetime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "#getting all the dates\n",
    "cols = conf_df.keys() #it has all the columns\n",
    "\n",
    "conf = conf_df.loc[:, cols[4]:cols[-1]]\n",
    "death = death_df.loc[:, cols[4]:cols[-1]]\n",
    "reco = reco_df.loc[:, cols[4]:cols[-1]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#now we have all the dates\n",
    "dates = conf.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['1/22/20', '1/23/20', '1/24/20', '1/25/20', '1/26/20', '1/27/20',\n",
       "       '1/28/20', '1/29/20', '1/30/20', '1/31/20', '2/1/20', '2/2/20',\n",
       "       '2/3/20', '2/4/20', '2/5/20', '2/6/20', '2/7/20', '2/8/20', '2/9/20',\n",
       "       '2/10/20', '2/11/20', '2/12/20', '2/13/20', '2/14/20', '2/15/20',\n",
       "       '2/16/20', '2/17/20', '2/18/20', '2/19/20', '2/20/20', '2/21/20',\n",
       "       '2/22/20', '2/23/20', '2/24/20', '2/25/20', '2/26/20', '2/27/20',\n",
       "       '2/28/20', '2/29/20', '3/1/20', '3/2/20', '3/3/20', '3/4/20', '3/5/20',\n",
       "       '3/6/20', '3/7/20', '3/8/20', '3/9/20', '3/10/20', '3/11/20', '3/12/20',\n",
       "       '3/13/20', '3/14/20', '3/15/20', '3/16/20', '3/17/20', '3/18/20',\n",
       "       '3/19/20', '3/20/20', '3/21/20', '3/22/20', '3/23/20'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "world_cases = []\n",
    "total_deaths = [] \n",
    "mortality_rate = []\n",
    "recovery_rate = [] \n",
    "total_recovered = [] \n",
    "total_active = [] \n",
    "\n",
    "countrywise_cases_conf = []\n",
    "countrywise_cases_reco = []\n",
    "countrywise_cases_death = []\n",
    "countrywise_cases_active = []\n",
    "\n",
    "#j=0\n",
    "for i in dates:\n",
    "    \n",
    "    confirmed_sum = conf[i].sum()\n",
    "    death_sum = death[i].sum()\n",
    "    recovered_sum = reco[i].sum()\n",
    "    \n",
    "    # confirmed, deaths, recovered, and active\n",
    "    world_cases.append(confirmed_sum)\n",
    "    total_deaths.append(death_sum)\n",
    "    total_recovered.append(recovered_sum)\n",
    "    total_active.append(confirmed_sum-death_sum-recovered_sum)\n",
    "    \n",
    "    # calculate rates\n",
    "    \n",
    "    # case studies \n",
    "    #china_cases.append(conf_df[conf_df['Country/Region']=='China'][i].sum())\n",
    "    #italy_cases.append(conf_df[conf_df['Country/Region']=='Italy'][i].sum())\n",
    "#     india_cases_conf.append(conf_df[conf_df['Country/Region']=='India'][i].sum())\n",
    "#     india_cases_reco.append(reco_df[reco_df['Country/Region']=='India'][i].sum())\n",
    "#     india_cases_death.append(death_df[death_df['Country/Region']=='India'][i].sum())\n",
    "#     india_cases_active.append(india_cases_conf[j] - india_cases_reco[j] - india_cases_death[j])\n",
    "    \n",
    "#    j = j+1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "days_in_future = 10\n",
    "future_forcast = np.array([i for i in range(len(dates)+days_in_future)]).reshape(-1, 1)\n",
    "adjusted_dates = future_forcast[:-10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "days_since_1_22 = np.array([i for i in range(len(dates))]).reshape(-1, 1)\n",
    "world_cases = np.array(world_cases).reshape(-1, 1)\n",
    "total_deaths = np.array(total_deaths).reshape(-1, 1)\n",
    "total_recovered = np.array(total_recovered).reshape(-1, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "start = '1/22/2020'\n",
    "start_date = datetime.datetime.strptime(start, '%m/%d/%Y')\n",
    "future_forcast_dates = []\n",
    "for i in range(len(future_forcast)):\n",
    "    future_forcast_dates.append((start_date + datetime.timedelta(days=i)).strftime('%m/%d/%Y'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#total confirmed cases in the world\n",
    "world_df = pd.DataFrame(index=dates,data=world_cases, columns=['conf_cases'])\n",
    "world_df['daily_new_cases'] = world_df.diff()\n",
    "world_df['daily_new_cases'][0] = world_df['conf_cases'][0]\n",
    "world_df['daily_new_cases'] =world_df['daily_new_cases'].astype('int64')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "total_death_df = pd.DataFrame(index=dates,data=total_deaths, columns=['total_death'])\n",
    "total_death_df['new_death'] = total_death_df.diff()\n",
    "total_death_df['new_death'][0] = total_death_df['total_death'][0] \n",
    "total_death_df['new_death'] = total_death_df['new_death'].astype('int64')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "total_reco_df = pd.DataFrame(index=dates,data=total_recovered, columns=['total_recovered'])\n",
    "total_reco_df['new_reco'] = total_reco_df.diff()\n",
    "total_reco_df['new_reco'][0] = total_reco_df['total_recovered'][0] \n",
    "total_reco_df['new_reco'] = total_reco_df['new_reco'].astype('int64')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "##this is tricky but true as it is daily new active cases being found both way it give same ans\n",
    "total_active_df = pd.DataFrame(index=dates,data=total_active, columns=['total_active'])\n",
    "\n",
    "#total_active_df['new_active111'] = world_df['daily_new_cases'] - total_death_df['new_death'] - total_reco_df['new_reco']\n",
    "total_active_df['new_active'] = total_active_df['total_active'].diff()\n",
    "total_active_df['new_active'][0] = total_active_df['total_active'][0] \n",
    "total_active_df['new_active'] = total_active_df['new_active'].astype('int64')\n",
    "\n",
    "total_active_df=total_active_df.clip(lower =0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "world_full = pd.concat([world_df,total_reco_df,total_death_df,total_active_df], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "#this dataset is for regression for univariate "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>conf_cases</th>\n",
       "      <th>daily_new_cases</th>\n",
       "      <th>total_recovered</th>\n",
       "      <th>new_reco</th>\n",
       "      <th>total_death</th>\n",
       "      <th>new_death</th>\n",
       "      <th>total_active</th>\n",
       "      <th>new_active</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1/22/20</th>\n",
       "      <td>555.0</td>\n",
       "      <td>555</td>\n",
       "      <td>28.0</td>\n",
       "      <td>28</td>\n",
       "      <td>17.0</td>\n",
       "      <td>17</td>\n",
       "      <td>510.0</td>\n",
       "      <td>510</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1/23/20</th>\n",
       "      <td>653.0</td>\n",
       "      <td>98</td>\n",
       "      <td>30.0</td>\n",
       "      <td>2</td>\n",
       "      <td>18.0</td>\n",
       "      <td>1</td>\n",
       "      <td>605.0</td>\n",
       "      <td>95</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1/24/20</th>\n",
       "      <td>941.0</td>\n",
       "      <td>288</td>\n",
       "      <td>36.0</td>\n",
       "      <td>6</td>\n",
       "      <td>26.0</td>\n",
       "      <td>8</td>\n",
       "      <td>879.0</td>\n",
       "      <td>274</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1/25/20</th>\n",
       "      <td>1434.0</td>\n",
       "      <td>493</td>\n",
       "      <td>39.0</td>\n",
       "      <td>3</td>\n",
       "      <td>42.0</td>\n",
       "      <td>16</td>\n",
       "      <td>1353.0</td>\n",
       "      <td>474</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1/26/20</th>\n",
       "      <td>2118.0</td>\n",
       "      <td>684</td>\n",
       "      <td>52.0</td>\n",
       "      <td>13</td>\n",
       "      <td>56.0</td>\n",
       "      <td>14</td>\n",
       "      <td>2010.0</td>\n",
       "      <td>657</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         conf_cases  daily_new_cases  total_recovered  new_reco  total_death  \\\n",
       "1/22/20       555.0              555             28.0        28         17.0   \n",
       "1/23/20       653.0               98             30.0         2         18.0   \n",
       "1/24/20       941.0              288             36.0         6         26.0   \n",
       "1/25/20      1434.0              493             39.0         3         42.0   \n",
       "1/26/20      2118.0              684             52.0        13         56.0   \n",
       "\n",
       "         new_death  total_active  new_active  \n",
       "1/22/20         17         510.0         510  \n",
       "1/23/20          1         605.0          95  \n",
       "1/24/20          8         879.0         274  \n",
       "1/25/20         16        1353.0         474  \n",
       "1/26/20         14        2010.0         657  "
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "world_full.head(5) #datasetprepared"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['1/22/20', '1/23/20', '1/24/20', '1/25/20', '1/26/20', '1/27/20',\n",
       "       '1/28/20', '1/29/20', '1/30/20', '1/31/20', '2/1/20', '2/2/20',\n",
       "       '2/3/20', '2/4/20', '2/5/20', '2/6/20', '2/7/20', '2/8/20', '2/9/20',\n",
       "       '2/10/20', '2/11/20', '2/12/20', '2/13/20', '2/14/20', '2/15/20',\n",
       "       '2/16/20', '2/17/20', '2/18/20', '2/19/20', '2/20/20', '2/21/20',\n",
       "       '2/22/20', '2/23/20', '2/24/20', '2/25/20', '2/26/20', '2/27/20',\n",
       "       '2/28/20', '2/29/20', '3/1/20', '3/2/20', '3/3/20', '3/4/20', '3/5/20',\n",
       "       '3/6/20', '3/7/20', '3/8/20', '3/9/20', '3/10/20', '3/11/20', '3/12/20',\n",
       "       '3/13/20', '3/14/20', '3/15/20', '3/16/20', '3/17/20', '3/18/20',\n",
       "       '3/19/20', '3/20/20', '3/21/20', '3/22/20', '3/23/20'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "world_full.index"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Prediction \n",
    "for next 10 days"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Solving It as Univariate Problem"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train_confirmed, X_test_confirmed, y_train_confirmed, y_test_confirmed = train_test_split(days_since_1_22, world_cases, test_size=0.15, shuffle=False) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "P1fxRgr9BXfc",
    "outputId": "255f5846-8fa2-4f35-ac56-051a2f2ccebb"
   },
   "outputs": [],
   "source": [
    "test_stationarity(tsdata)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " Mean and standard deviation are clearly increasing with time and this is not a stationary series. Also, the test statistic is way more than the critical values. Note that the signed values should be compared and not the absolute values."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Making Time Series stationary"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1. Trend \n",
    "2. Seasonality "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Estimating & Eliminating Trend"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ts_log = np.log(ts)\n",
    "plt.plot(ts_log)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ts_log.index.dtype"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Moving Averages\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#ts_log = pd.DataFrame(ts_log)\n",
    "moving_avg = ts_log.rolling(7).mean()\n",
    "plt.plot(ts_log)\n",
    "plt.plot(moving_avg, color='red')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ts_log_moving_avg_diff = ts_log - moving_avg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ts_log_moving_avg_diff.head(10) #as for first 7 values there is no rolling value\n",
    "#hence dropping it\n",
    "ts_log_moving_avg_diff.dropna(inplace=True)\n",
    "#checking stationarity\n",
    "test_stationarity(ts_log_moving_avg_diff)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This looks like a much better series. The rolling values appear to be varying but there is no specific trend. Also, the test statistic is smaller than the 1% critical values so we can say with 99% confidence that this is a stationary series."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Weighted Moving Average\n",
    "Recent data gets more weightage"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "expwighted_avg = ts_log.ewm(span=7).mean() #span is deacy factor\n",
    "\n",
    "plt.plot(ts_log)\n",
    "plt.plot(expwighted_avg, color='red')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ts_log_ewma_diff = ts_log - expwighted_avg\n",
    "test_stationarity(ts_log_ewma_diff)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Test Statistic  is much lagrger hence previous case was better"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Differencing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ts_log_diff = ts_log - ts_log.shift()\n",
    "plt.plot(ts_log_diff)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ts_log"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ts_log_diff.dropna(inplace=True)\n",
    "test_stationarity(ts_log_diff)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "No success with differencing also"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ts_log_diff2 = ts_log_diff - ts_log_diff.shift()\n",
    "plt.plot(ts_log_diff2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ts_log_diff2.dropna(inplace=True)\n",
    "test_stationarity(ts_log_diff2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Second order differencing gives promising results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Forecasting a Time Series"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "ZuuvOE3bBXfe",
    "outputId": "b495a7ca-5356-47cd-9b9a-1923c241fa8e"
   },
   "outputs": [],
   "source": [
    "#ACF and PACF plots:\n",
    "from statsmodels.tsa.stattools import acf, pacf\n",
    "lag_acf = acf(ts_log_diff2, nlags=5)\n",
    "lag_pacf = pacf(ts_log_diff2, nlags=5, method='ols')\n",
    "#Plot ACF: \n",
    "plt.subplot(121) \n",
    "plt.plot(lag_acf)\n",
    "plt.axhline(y=0,linestyle='--',color='gray')\n",
    "plt.axhline(y=-1.96/np.sqrt(len(ts_log_diff2)),linestyle='--',color='gray')\n",
    "plt.axhline(y=1.96/np.sqrt(len(ts_log_diff2)),linestyle='--',color='gray')\n",
    "plt.title('Autocorrelation Function')\n",
    "#Plot PACF:\n",
    "plt.subplot(122)\n",
    "plt.plot(lag_pacf)\n",
    "plt.axhline(y=0,linestyle='--',color='gray')\n",
    "plt.axhline(y=-1.96/np.sqrt(len(ts_log_diff2)),linestyle='--',color='gray')\n",
    "plt.axhline(y=1.96/np.sqrt(len(ts_log_diff2)),linestyle='--',color='gray')\n",
    "plt.title('Partial Autocorrelation Function')\n",
    "plt.tight_layout()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1. p – The lag value where the PACF chart crosses the upper confidence interval for the first time. If you notice closely, in this case p=1.\n",
    "2. q – The lag value where the ACF chart crosses the upper confidence interval for the first time. If you notice closely, in this case q=1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "UC0YxWwwBXfg"
   },
   "outputs": [],
   "source": [
    "from __future__ import print_function\n",
    "import numpy as np\n",
    "from scipy import stats\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import statsmodels.api as sm\n",
    "\n",
    "from statsmodels.graphics.api import qqplot\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "1ON7kQFfBXfh",
    "outputId": "d2131a8f-4225-4d70-905f-c282fbfea223"
   },
   "outputs": [],
   "source": [
    "# try plotting for various lags 2,3,4,5...20\n",
    "fig = plt.figure(figsize=(12,8))\n",
    "ax1 = fig.add_subplot(211)\n",
    "\n",
    "fig = sm.graphics.tsa.plot_acf(ts_log_diff2.values.squeeze(), lags=3, ax=ax1)\n",
    "ax2 = fig.add_subplot(212)\n",
    "fig = sm.graphics.tsa.plot_pacf(ts_log_diff2, lags=3, ax=ax2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = ts_log_diff2.iloc[0:50]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test = ts_log_diff2.iloc[50:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print (len(train))\n",
    "print (len(test))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### We need to load the ARIMA model first:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "ggtU-FnnBXfj",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "from statsmodels.tsa.arima_model import ARIMA\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The p,d,q values can be specified using the order argument of ARIMA which take a tuple (p,d,q).\n",
    " p is the number of autoregressive terms, d is the number of nonseasonal differences needed for stationarity, and. q is the number of lagged forecast errors in the prediction equation."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### AR model:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = ARIMA(ts_log, order=(2, 2, 0)) #d=2 because 2 oder difference gave best stationarity test  \n",
    "results_AR = model.fit(disp=-1)  \n",
    "plt.plot(ts_log_diff2)\n",
    "plt.plot(results_AR.fittedvalues, color='red')\n",
    "plt.title('RSS: %.4f'% sum((results_AR.fittedvalues-ts_log_diff2)**2))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Moving Average Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = ARIMA(ts_log, order=(0, 2, 2))  \n",
    "results_MA = model.fit(disp=-1)  \n",
    "plt.plot(ts_log_diff2)\n",
    "plt.plot(results_MA.fittedvalues, color='red')\n",
    "plt.title('RSS: %.4f'% sum((results_MA.fittedvalues-ts_log_diff2)**2))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Combined model:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = ARIMA(ts_log, order=(2, 2, 2))  \n",
    "results_ARIMA = model.fit(disp=-1)  \n",
    "plt.plot(ts_log_diff2)\n",
    "plt.plot(results_ARIMA.fittedvalues, color='red')\n",
    "plt.title('RSS: %.4f'% sum((results_ARIMA.fittedvalues-ts_log_diff2)**2))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Combined AR and MA work better "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Using Pyramid ARIMA \n",
    "It has auto arima "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pyramid as pm\n",
    "from pyramid.arima import auto_arima\n",
    "world_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "SeCsWXhZBXf_"
   },
   "outputs": [],
   "source": [
    "#load the data\n",
    "data = world_df['conf_cases'].copy()\n",
    "\n",
    "#divide into train and validation set\n",
    "#train = data[:int(0.7*(len(data)))]\n",
    "#valid = data[int(0.7*(len(data))):]\n",
    "\n",
    "#preprocessing (since arima takes univariate series as input)\n",
    "#train.drop('Month',axis=1,inplace=True)\n",
    "#valid.drop('Month',axis=1,inplace=True)\n",
    "\n",
    "#plotting the data\n",
    "train.plot()\n",
    "valid.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyramid.arima import auto_arima\n",
    "model = auto_arima(data, seasonal=[True,False],seasonal_decompose=[True,False], trace=True, error_action='ignore', suppress_warnings=True)\n",
    "model.fit(train)\n",
    "\n",
    "forecast = model.predict(n_periods=len(future_forcast_dates))\n",
    "forecast = pd.DataFrame(forecast,index = future_forcast_dates.index,columns=['Prediction'])\n",
    "\n",
    "#plot the predictions for validation set\n",
    "plt.plot(train, label='Train')\n",
    "#plt.plot(valid, label='Valid')\n",
    "plt.plot(forecast, label='Prediction')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Hence Its not much useful to use ARIMA on this dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# plt.plot(y_test_confirmed)\n",
    "# plt.plot(test_bayesian_pred)"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "name": "GujaratPred2.ipynb",
   "provenance": [],
   "version": "0.3.2"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}