Covid_OpenLine.Rmd

---
title: "EDA_openList"
author: "Arushi Sharma"
date: "4/10/2020"
output: pdf_document
---

```{r setup, include=FALSE}
knitr::opts_chunk$set(echo = TRUE)
```


```{r message = FALSE}
#Importing all required libraries

library(dplyr)
library(ggplot2)
library(tidyverse)
library(chron)
library(lubridate)
library(caTools)
library(caret)
library(rsample)
library(ROCR)
library(e1071)
library(corrplot)
library(polycor)
library(corrgram)
library(randomForest)
```


```{r}
open_line <-  readr::read_csv("latestdata_10Apr.csv")
open_line
```

```{r}
sum(is.na(open_line$age))
```


```{r message=FALSE}
# Filtering and cleaning data
covid19 <- open_line %>% select(age, sex, country, date_confirmation, symptoms,
                                outcome, travel_history_binary)

summary(covid19)

# Converting age to integer values
covid19$age <- as.integer(covid19$age)

# Converting age into groups
recode_age <- function(x) {
  y <- character(length(x))
  y[x < 10] <- "0_10"
  y[x >= 10] <- "10_20" 
  y[x >= 20] <- "20_30" 
  y[x >= 30] <- "30_40"
  y[x >= 40] <- "40_50" 
  y[x >= 50] <- "50_60" 
  y[x >= 60] <- "60_70" 
  y[x >= 70] <- "70_80" 
  y[x >= 80] <- "80_100"
  y
}
covid19 <- covid19 %>% mutate(age_group = recode_age(age))
covid19 <- mutate(covid19, age_group = factor(age_group,level =c("0_10","10_20","20_30", "30_40","40_50","50_60", "60_70","70_80", "80_100")))


#converting symptoms into categorical variable
covid19$has_covid_symptoms<- str_detect(tolower(covid19$symptoms), 'fever|cough|breath|breathing|throat|pneumonia|respiratory|heart|chest|lung|weak|tired|tiring|fatigue|chill')

#Creating a new column for death
covid19$death <- str_detect(tolower(covid19$outcome), 'death|died|dead|deceased')

#Changing sex column to lowercase
covid19$sex <- tolower(covid19$sex)

#Changing country column to lowercase
covid19$country <- tolower(covid19$country)


#Formatting the date 
covid19$confirmation_date <- (as.Date(covid19$date_confirmation, "%d.%m.%Y"))

#Creating a new column as no of days for each patient by country
covid19$minimum_date <- covid19$confirmation_date
unique_countries <- unique(covid19$country) 

for (ucountry in unique_countries){
  new_date <- covid19 %>% filter(covid19$country == ucountry) %>%
    summarise(min(confirmation_date, na.rm = TRUE))
    earliest_date <- do.call("c", new_date)
    index <- covid19$country == ucountry
    covid19$minimum_date[index] <- earliest_date
}

covid19$days_to_firstoccurence <- as.numeric(as.Date(as.character(covid19$confirmation_date), format="%Y-%m-%d")-
                  as.Date(as.character(covid19$minimum_date), format="%Y-%m-%d"))

```


```{r message=FALSE}

covid <- covid19 %>% select(age, sex, country, travel_history_binary, age_group, has_covid_symptoms,
                            death, confirmation_date, minimum_date, days_to_firstoccurence)

colnames(covid)
summary(covid)
```




```{r}
# Age vs Death rate

age_death <- covid %>% select(age_group, death)
clean_age_death <- na.omit(age_death)
clean_age_death

ggplot(clean_age_death, mapping = aes(x = age_group, fill= as.factor(death))) +
geom_bar(position = "dodge") +
            xlab("Age Group ") +
            ylab("Frequency of Cases") +
            ggtitle("Most affected age groups by CoVid-19")+
   scale_fill_manual(values=c("#CC79A7","#56B4E9"))+
  guides(fill=guide_legend(title="Fatality (TRUE)"))


```


```{r}
#Most affected gender by corona

gender_death <- covid %>% select(sex, death)
clean_gender_death <- na.omit(gender_death)
clean_gender_death

ggplot(clean_gender_death, mapping = aes(x = sex, fill= as.factor(death))) +
geom_bar(position = "dodge") +
            xlab("Gender ") +
            ylab("Frequency of Cases") +
            ggtitle("Most affected gender by CoVid-19")+
  scale_fill_manual(values=c("#CC79A7","#56B4E9"))+
  guides(fill=guide_legend(title="Fatality (TRUE)"))
   

```


```{r}
#Most affected gender by corona

symptoms_death <- covid %>% select(has_covid_symptoms, death)
clean_symptom_death <- na.omit(symptoms_death)
clean_symptom_death

ggplot(clean_symptom_death, mapping = aes(x = symptoms_death, fill= as.factor(death))) +
geom_bar(position = "dodge") +
            xlab("Gender ") +
            ylab("Frequency of Cases") +
            ggtitle("Most affected gender by Corona")

```



```{r}

covid_model <- covid %>% select(age, sex, country, travel_history_binary, has_covid_symptoms,
                            death, days_to_firstoccurence)

summary(covid_model)
sum(!apply(covid_model, 1, anyNA))

```


```{r}

#Handling NA values of columns

#changing all NAs of travel_history_binary to FALSE
covid_model$travel_history_binary[is.na(covid_model$travel_history_binary)] <- FALSE
covid_model$death[is.na(covid_model$death)] <- FALSE
covid_model$has_covid_symptoms[is.na(covid_model$has_covid_symptoms)] <- FALSE
covid_model$country[is.na(covid_model$country)] <- 'other'

covid_model$days_to_firstoccurence[is.na(covid_model$days_to_firstoccurence)] <- mean(covid_model$days_to_firstoccurence, na.rm = TRUE)


covid_model$age[is.na(covid_model$age)] <- mean(covid_model$age, na.rm = TRUE)

summary(covid_model)
sum(!apply(covid_model, 1, anyNA))

covid_model <- na.omit(covid_model)
str(covid_model)

```

# Corelation Matrix

```{r}
#M <- cor(covid_model)
covid_cor <- covid_model %>% select(age, sex, country, travel_history_binary, has_covid_symptoms,
                            death, days_to_firstoccurence)
#corrplot(M, method = "circle")
cordata <- hetcor(covid_cor, use = "complete.obs")

corrplot(cordata$correlations, method="color", na.label = "square", na.label.col = "lightblue")
#sort(unique(covid_cor$country))

```

```{r}
ggplot(data=covid_model)+ geom_boxplot(aes(x=death,y=age))
```

```{r}
ggplot(data=covid_model)+ geom_boxplot(aes(x=death,y=has_covid_symptoms))
```


# Factoring all the columns to be used for modeling

```{r}
covid_model <- covid_model %>%
mutate(
    sex = factor(sex),
    country = factor(country),
    has_covid_symptoms = factor(has_covid_symptoms, label = c(0, 1)),
    death = factor(death, label = c(0, 1)),
    travel_history_binary = factor(travel_history_binary, label = c(0, 1)),
    )

summary(covid_model)
str(covid_model)
```

# Splitting into Training and Testing dataset

```{r}
# Create Training Data

set.seed(100)  # for repeatability of samples

input_ones <- covid_model[which(covid_model$death == 1), ]  # all 1's
input_zeros <- covid_model[which(covid_model$death == 0), ]  # all 0's

input_ones_training_rows <- sample(1:nrow(input_ones), 0.7*nrow(input_ones))  # 1's for training
input_zeros_training_rows <- sample(1:nrow(input_zeros), 0.7*nrow(input_zeros))  # 0's for training. Pick as many 0's as 1's

training_ones <- input_ones[input_ones_training_rows, ]  
training_zeros <- input_zeros[input_zeros_training_rows, ]
trainingData <- rbind(training_ones, training_zeros)  # row bind the 1's and 0's 

# Create Test Data
test_ones <- input_ones[-input_ones_training_rows, ]
test_zeros <- input_zeros[-input_zeros_training_rows, ]
testData <- rbind(test_ones, test_zeros)  # row bind the 1's and 0's 

summary(trainingData)
summary(testData)
```

# Modelling

```{r}

#classifier_types : glm, svm, rf(random forest), svm_oneclass
classifier_type <- "svm"
if (classifier_type=="glm"){
  #classifier <- glm(death ~ age+sex+travel_history_binary+has_covid_symptoms+days_to_firstoccurence,
   #             data=trainingData, family=binomial(link="logit"))
  classifier <- glm(death ~ age+sex+has_covid_symptoms+days_to_firstoccurence,
                data=trainingData, family=binomial(link="logit"))
}

if (classifier_type=="svm"){
  classifier = svm(formula = death ~age+sex+has_covid_symptoms+days_to_firstoccurence,
                 data = predicted_trainData, 
                 type = 'C-classification', kernel = 'radial', gamma=0.25 , nu=0.5) 
}

if (classifier_type=="rf"){
  classifier = randomForest(death ~ age+sex+has_covid_symptoms+days_to_firstoccurence, data = predicted_trainData, importance = TRUE)
}


if (classifier_type=="svm_oneclass"){
  classifier = svm(formula = death ~ age+sex+has_covid_symptoms+days_to_firstoccurence,
                 data = predicted_trainData[predicted_trainData$death == 1,], 
                 type = 'one-classification', kernel = 'radial', gamma=0.25 , nu=0.5 ) 
}


summary(classifier)
classifier
#importance(classifier)        
#varImpPlot(classifier) 

```

# Predicting the trained and test data

```{r}
predicted_train <- predict(classifier, trainingData, type="response")  # predicted scores
head(predicted_train, 5)

predicted_test <- predict(classifier, testData, type="response")
head(predicted_test, 5)

predicted_testData <- testData %>% mutate(pred = predicted_test)
predicted_trainData <- trainingData %>% mutate(pred = predicted_train)

predicted_testData$majority <- 0.0
predicted_trainData$majority <- 0.0

summary(predicted_trainData)
summary(predicted_testData)

#table(predicted_train, predicted_trainData$death)
#table(predicted_test, predicted_testData$death)

```


# Getting results from the Confusion Matrix

```{r message=FALSE}


predict_fatality <- function(predictions, cutoff, baseline=FALSE, classifier_type="glm"){
  if(baseline == FALSE){
    if(classifier_type == "glm"){
      covid_pred <- predictions %>%
      mutate(pred_status =ifelse(pred > cutoff, 1, 0)) %>%
      mutate(pred_status = factor(pred_status, label = c(0, 1))) 
    }
    else if(classifier_type=="svm_oneclass"){
      covid_pred <- predictions
      #covid_pred$pred_status <- ifelse(predictions$pred == TRUE, 1, 0)
      covid_pred$pred_status <- factor(predictions$pred, label = c(0, 1))
    }
    else{
      covid_pred <- predictions
      covid_pred$pred_status <- predictions$pred
    } 
  }
  else{
    covid_pred <- predictions %>%
    mutate(pred_status =ifelse(majority > cutoff, 1, 0)) %>%
    mutate(pred_status = factor(pred_status, label = c(0))) 
  } 
  
    con_matrix <- confusionMatrix(covid_pred$pred_status, covid_pred$death)
    acc <- con_matrix$overall['Accuracy']
    specificity <- con_matrix$byClass["Specificity"]
    sensitivity <- con_matrix$byClass["Sensitivity"]
    f1 <- con_matrix$byClass["F1"]
    print(f1)
    c(acc, specificity, sensitivity)
}

cutoff = seq(0.5, 0.5, 0.1)
train_predicts <- data.frame(Accuracy=double, Specificity=double, Sensitivity=double, stringsAsFactors = FALSE)
test_predicts <- data.frame(Accuracy=double, Specificity=double, Sensitivity=double, stringsAsFactors = FALSE)


#print(train_predicts)
#print(test_predicts)
for (i in cutoff){

  train_predict_row <- predict_fatality(predicted_trainData, i, classifier_type=classifier_type)
  train_predicts <- rbind(train_predicts, train_predict_row)

  test_predict_row <- predict_fatality(predicted_testData, i,  classifier_type=classifier_type)
  test_predicts <- rbind(test_predicts, test_predict_row)
}

#baseline_train <- predict_fatality(predicted_trainData, 0.5, baseline = TRUE)
#baseline_test <- predict_fatality(predicted_testData, 0.5, baseline=TRUE)
#print(baseline_train)
#print(baseline_test)

print(train_predicts)
print(test_predicts)



```

# Plotting the Results

```{r}
training_perf <- readr::read_csv("training_performance.csv")
testing_perf <- readr::read_csv("testing_performance.csv")


training_perf_bkp <- training_perf
testing_perf_bkp <- testing_perf

typeof(training_perf_bkp)

#training_perf[5:5,2:4] <- 0.0
#testing_perf[5:5,2:4]<- 0.0


training_perf %>% gather("Type","Value", -Model) %>%
  ggplot(aes(Model, Value, fill=Type)) + 
  geom_bar(position="dodge", stat="identity")+
  theme_bw()+
  #facet_wrap(~Model, scales="free_x")+
  scale_fill_manual(values=c("#CC79A7","#56B4E9", "#0072B2"))


testing_perf %>% gather("Type","Value", -Model) %>%
  ggplot(aes(Model, Value, fill=Type)) + 
  geom_bar(position="dodge", stat="identity")+
  theme_bw()+
  #facet_wrap(~Model, scales="free_x")+
  scale_fill_manual(values=c("#CC79A7","#56B4E9", "#0072B2"))


```


```{r}
slices <- c(10722, 4597)
lbls <- c("#Training", "#Test")
lbls <- paste(lbls, slices) # add percents to labels
pie(slices, labels = lbls, main="Pie Chart of Data Split")



#pie(slices,labels = lbls, col=rainbow(length(lbls)),
   #main="Pie Chart of Countries") 
```

```{r}
#training dead and alive
slices <- c(10582, 140)

lbls <- c("Recovered", "Dead")
lbls <- paste(lbls, slices) # add percents to labels
pie(slices, labels = lbls, main="Pie Chart of Imbalanced Data")
```

#Rough : plotting confusion matrix graphs

```{r}



names(train_predicts) <- c("Accuracy", "Specificity", "Sensitivity")
names(test_predicts) <- c("Accuracy", "Specificity", "Sensitivity")
#names(baseline_train) <- c("Accuracy", "Specificity", "Sensitivity")

plot(cutoff, train_predicts$Accuracy, type='n', ylim=c(min(train_predicts)-0.005,max(train_predicts))) #dummy
lines(cutoff, train_predicts$Accuracy, col="black")
lines(cutoff, train_predicts$Specificity, col="blue")
lines(cutoff, train_predicts$Sensitivity, col="red")
#lines(cutoff, rep(baseline_train[["Accuracy"]], 9), col="black", lty=4 )
#lines(cutoff, rep(baseline_train[["Specificity"]], 9), col="blue", lty=4)
#lines(cutoff, rep(baseline_train[["Sensitivity"]], 9), col="red", lty=4)

legend("bottomright", legend=c("Accuracy", "Specificity","Sensitivity"),
       col=c("black", "blue", "red"), lty=1, cex=0.8,
       box.lty=2, box.lwd=2, box.col="grey")


plot(cutoff, test_predicts$Accuracy, type='n', ylim=c(min(test_predicts)-0.005, max(test_predicts))) #dummy
lines(cutoff, test_predicts$Accuracy, col="black")
lines(cutoff, test_predicts$Specificity, col="blue")
lines(cutoff, test_predicts$Sensitivity, col="red")
#lines(cutoff, rep(baseline_test[["Accuracy"]], 9), col="black", lty=4 )
#lines(cutoff, rep(baseline_test[["Specificity"]], 9), col="blue", lty=4)
#lines(cutoff, rep(baseline_test[["Sensitivity"]], 9), col="red", lty=4)

legend("bottomright", legend=c("Accuracy", "Specificity","Sensitivity"),
       col=c("black", "blue", "red"), lty=1, cex=0.8,
       box.lty=2, box.lwd=2, box.col="grey")

Accuracy <- train_predicts$Accuracy
Specificity <-  train_predicts$Specificity
Sensitivity <-  train_predicts$Sensitivity
plot_ref_train <- data.frame(Accuracy, Specificity, Sensitivity)

Accuracy <- test_predicts$Accuracy
Specificity <-  test_predicts$Specificity
Sensitivity <-  test_predicts$Sensitivity
plot_ref_test <- data.frame(Accuracy, Specificity, Sensitivity)




```