Introduction

The data investigated in this small workbook goes back to S. Moro, P. Cortez and P. Rita. A Data-Driven Approach to Predict the Success of Bank Telemarketing. Decision Support Systems, Elsevier, 62:22-31, June 2014
The data is available from the UCI Machine Learning Repository.
Several data sets are available; we will use bank-additional-full.csv.
An earlier paper from the authors is Moro, Cortez, Laureano: A data mining approach for bank telemarketing using the rminer package and r tool, which is free for download.

The purpose of this workbook to apply different machine learning algorithms to the data problem. The software R will be used in conjunction with the package caret.

Data Preparation

The data is being loaded (again, we will use the bank-additional-full.csv).
Investigation shows that the data needs to be prepared for analysis. The following steps are performed:

One important change we make is with regards to the the variable duration. It indicates the length of a call. This might be interesting for analysis, but difficult for prediction: In a prediction task we do not know the duration of the call before it actually takes place.
Therefore duration is being excluded from the set of predictors. Researchers who included it found this variable to be higly predictive (which is not surprising).

Also the data set is split into a training set (80%) and a test set (20%).

library(caret)
library(pROC)
library(klaR)
library(dplyr)
bank.raw <- read.csv2(file="bank-additional-full.csv")
str(bank.raw)
bank <- bank.raw
bank$job <- factor(gsub("\\-", "", bank$job))
bank[,16:20] <- apply(bank[,16:20], 2, function (x) as.numeric(as.character(x)))
#bank[bank=="unknown"] <- NA
bank <- droplevels(bank)
bank[bank$pdays==999,"pdays"] <- "unknown"
bank$pdays <- as.factor(bank$pdays)
summary(bank)
names(bank)[names(bank)=="y"] <- "success"
bank$success <- factor(bank$success, levels=c("yes", "no"))

bank <- subset(bank, select=-c(duration))

trainIndex <- createDataPartition(bank$success, p=0.8, list=FALSE, times=1)
bank.train <- bank[trainIndex,]
bank.test <- bank[-trainIndex,]

Modelling

We do not perform any feature engineering resp. feature selction. This, however, would be an important step, as can be seen in [Mores, Cortez, Rita].

We will fit the following models using the caret-package:

  1. CART
  2. C50
  3. Conditional Inference Trees
  4. Logistic Regression
  5. Naive Bayes
  6. Neural Net

10-fold cross-validation is performed on the traing set. The choice of model parameters is optimzed with regards to ROC.

The final evaluation happens based on the “area under the ROC-curve” AUC on the test set.

bank.train.data <- subset(bank.train, select=-c(success), drop=TRUE)
bank.train.class <- subset(bank.train, select=c(success), drop=TRUE)
bank.test.data <- subset(bank.test, select=-c(success), drop=TRUE)
bank.test.class <- subset(bank.test, select=c(success), drop=TRUE)


cvCtrl <- trainControl(method="cv", classProbs=TRUE, summaryFunction=twoClassSummary, verboseIter=TRUE)


fit.rpart <- train(bank.train.data, bank.train.class, method="rpart", trControl=cvCtrl, metric="ROC", tuneLength = 10)
#load("CART_CV_10Iter.RData")
predict.rpart <- predict(fit.rpart)
confusionMatrix(predict.rpart, bank.train.class, positive="yes")
predict.rpart <- predict(fit.rpart, type = "prob")
(auc.rpart <- auc(bank.train.class, predict.rpart$yes))
predict.rpart <- predict(fit.rpart, bank.test.data, type = "prob")
(auc.rpart <- auc(bank.test.class, predict.rpart$yes))



fit.c50 <- train(bank.train.data, bank.train.class, method="C5.0", trControl=cvCtrl, metric="ROC", tuneLength = 10)
#load(file="C50_CV_10Iter.RData")
predict.c50 <- predict(fit.c50)
confusionMatrix(predict.c50, bank.train.class, positive="yes")
predict.c50 <- predict(fit.c50, type = "prob")
(auc.c50 <- auc(bank.train.class, predict.c50$yes))
predict.c50 <- predict(fit.c50, bank.test.data, type="prob")
(auc.c50 <- auc(bank.test.class, predict.c50$yes))


fit.ctree <- train(bank.train.data, bank.train.class, method="ctree", trControl=cvCtrl, metric="ROC")
#load(file = "Ctree_CV_10Iter.RData")
predict.ctree <- predict(fit.ctree)
confusionMatrix(predict.ctree, bank.train.class, positive="yes")
predict.ctree <- predict(fit.ctree, type = "prob")
(auc.ctree <- auc(bank.train.class, predict.ctree$yes))
predict.ctree <- predict(fit.ctree, bank.test.data, type="prob")
(auc.ctree <- auc(bank.test.class, predict.ctree$yes))

bank.train.dummy <- predict(dummyVars(success ~ . ,data=bank.train), newdata=bank.train)
bank.train.dummy <- data.frame(bank.train.dummy, success=factor(bank.train$success))
bank.test.dummy <- predict(dummyVars(success ~ . ,data=bank.test), newdata=bank.test)
bank.test.dummy <- data.frame(bank.test.dummy, success=factor(bank.test$success))


fit.logit <- train(success ~ ., data=bank.train.dummy, method="glm", family=binomial, trControl=cvCtrl, metric="ROC", tuneLength=1)
#load(file = "Logit_CV_10Iter.RData")
predict.logit <- predict.train(fit.logit)
confusionMatrix(predict.logit, bank.train.class, positive="yes")
predict.logit <- predict(fit.logit, type = "prob")
(auc.logit <- auc(bank.train.dummy$success, predict.logit$yes))
predict.logit <- predict(fit.logit, bank.test.dummy, type="prob")
(auc.logit <- auc(bank.test.dummy$success, predict.logit$yes))

fit.nb <- train(bank.train.data, bank.train.class, method="nb", trControl=cvCtrl, metric="ROC", tuneLength=10)
#load(file = "NB_CV_10Iter.RData")
predict.nb <- predict(fit.nb)
confusionMatrix(predict.nb, bank.train.class, positive="yes")
predict.nb <- predict(fit.nb, type = "prob")
(auc.nb <- auc(bank.train.class, predict.nb$yes))
predict.nb <- predict(fit.nb, bank.test.data, type="prob")
(auc.nb <- auc(bank.test.class, predict.nb$yes))


fit.nn <- train(success ~ ., data=bank.train.dummy, method="nnet", trControl=cvCtrl, metric="ROC", tuneLength=10)
#load(file = "NN_CV_10Iter.RData")
predict.nn <- predict(fit.nn)
confusionMatrix(predict.nn, bank.train.class, positive="yes")
predict.nn <- predict(fit.nn, type = "prob")
(auc.nn <- auc(bank.train.class, predict.nn$yes))
predict.nn <- predict(fit.nn, bank.test.dummy, type="prob")
(auc.nn <- auc(bank.test.class, predict.nn$yes))

#rm(list=c("fit.ctree", "fit.logit", "fit.nb", "fit.nn", "fit.rpart"))
load(file="Banking.RData")
library(pROC)
library(caret)
plot.roc(bank.test.class, predict.rpart$yes, col="green")
## 
## Call:
## plot.roc.default(x = bank.test.class, predictor = predict.rpart$yes,     col = "green")
## 
## Data: predict.rpart$yes in 928 controls (bank.test.class yes) > 7309 cases (bank.test.class no).
## Area under the curve: 0.745
lines.roc(bank.test.class, predict.c50$yes, col="blue")
lines.roc(bank.test.class, predict.ctree$yes, col="pink")
lines.roc(bank.test.class, predict.nb$yes, col="red")
lines.roc(bank.test.class, predict.logit$yes, col="orange")
lines.roc(bank.test.class, predict.nn$yes, col="cyan")
legend("bottomright", legend=c("CART", "C50", "Cond. Inference Trees", "Naive Bayes", "Logit", "Neural Net"), col=c("green", "blue", "pink", "red", "orange", "cyan"), lwd=2, cex=0.5)

plot of chunk evaluate

library(dplyr)
#auc <- rbind(C50=auc.c50, CART=auc.rpart, ctree=auc.ctree, Logit=auc.logit, NaiveBayes=auc.nb, NeuralNet=auc.nn)
#auc <- data.frame(Model=row.names(auc), AUC=auc)
(auc <- arrange(auc, desc(AUC)))
##        Model    AUC
## 1        C50 0.8649
## 2      ctree 0.8025
## 3      Logit 0.7894
## 4 NaiveBayes 0.7813
## 5  NeuralNet 0.7558
## 6       CART 0.7511
# plot(varImp(fit.rpart))
plot(varImp(fit.c50, metric="splits"), main="Attribute usage for C50")

plot of chunk evaluate

# plot(varImp(fit.ctree))
# plot(varImp(fit.nb))
# plot(varImp(fit.logit))
# plot(varImp(fit.nn))

It can be seen that C50 performs best, in-sample and out-of-sample. It shows an AUC of 0.86. A plot of variable importance shows that age, the interest rate (3-month EURIBOR), previous campaign success, the job class and education are important variables for the model.