Building predictive models for categorical outcomes
Classification = Predicting categories (Yes/No, Class A/B/C, etc.) REAL EXAMPLES:
WORKFLOW:
KEY ALGORITHMS THIS WEEK: ✓ Logistic Regression ✓ Decision Trees ✓ Random Forests ✓ Naive Bayes ✓ K-Nearest Neighbors (KNN) ✓ Support Vector Machines (SVM) ✓ Ensemble methods EVALUATION METRICS: ✓ Accuracy (% correct) ✓ Precision (of predicted positive) ✓ Recall (of actual positive) ✓ F1 Score (balance precision/recall) ✓ ROC-AUC (overall performance) ✓ Confusion Matrix (detailed breakdown) IMPORTANT: Accuracy alone is NOT enough!
TRAIN-TEST SPLIT - FUNDAMENTAL REQUIREMENT:
Never evaluate on data used for training!
library(caret) library(dplyr) # Load data data <- read_csv("customer_data.csv") # 1000 rows # Split: 80% train, 20% test set.seed(42) index <- createDataPartition(data$churn, p=0.8, list=FALSE) train_data <- data[index, ] test_data <- data[-index, ] print(paste("Train:", nrow(train_data), "rows")) # 800 print(paste("Test:", nrow(test_data), "rows")) # 200 # Alternative simple method train_size <- 0.8 * nrow(data) train_index <- sample(nrow(data), train_size) train_data <- data[train_index, ] test_data <- data[-train_index, ]
LOGISTIC REGRESSION - BASELINE MODEL:
Despite name, used for CLASSIFICATION, not regression
# Example data df <- tibble( age = c(20, 25, 30, 35, 40, 45, 50), approved = c(0, 0, 1, 1, 1, 1, 1) # Binary outcome ) # Logistic regression model <- glm(approved ~ age, family=binomial, data=df) summary(model) # Output: # Coefficients: # Estimate Std. Error z value Pr(>|z|) # (Intercept) -2.7081 2.5054 -1.081 0.280 # age 0.1018 0.0717 1.420 0.156 # Interpretation: # Coefficient = log-odds change # Positive = increases probability # Each year of age → log-odds increase by 0.10 # Predict probabilities predictions <- predict(model, df, type="response") # [1] 0.063 0.104 0.163 0.243 0.337 0.438 0.531 # Interpretation: P(approved=1|age) # Age 20 → 6% approval chance # Age 50 → 53% approval chance # Convert to binary predictions (threshold = 0.5) pred_binary <- ifelse(predictions > 0.5, 1, 0) # [1] 0 0 0 0 0 0 1
DECISION TREES - INTERPRETABLE MODELS:
library(rpart) library(rpart.plot) # Simple tree tree_model <- rpart(Species ~ ., data=iris, method="class") # Plot tree rpart.plot(tree_model) # Predictions predict(tree_model, iris[1:5, ], type="class") # Tree interpretation (very human-readable): # If Petal.Length <= 2.4: Setosa # Else if Petal.Length <= 4.8: Versicolor # Else: Virginica
RANDOM FORESTS - ENSEMBLE METHOD:
library(randomForest) # Train rf_model <- randomForest(Species ~ ., data=iris, ntree=100, mtry=2) # Predictions pred <- predict(rf_model, iris) # Feature importance importance(rf_model) # Shows which variables matter most # Visualize importance plot(importance(rf_model))
MODEL EVALUATION - CRITICAL STEP:
library(caret) # Confusion matrix predictions <- predict(model, test_data, type="response") pred_binary <- ifelse(predictions > 0.5, "Yes", "No") confusion <- table(test_data$churn, pred_binary) # No Yes # No 140 10 # Yes 20 30 # Interpretation: # True Negatives (TN) = 140 # False Positives (FP) = 10 # False Negatives (FN) = 20 # True Positives (TP) = 30 # Calculate metrics accuracy <- (TP + TN) / (TP + TN + FP + FN) # (140+30)/200 = 0.85 precision <- TP / (TP + FP) # 30/40 = 0.75 recall <- TP / (TP + FN) # 30/50 = 0.60 f1 <- 2 * (precision * recall) / (precision + recall) # 0.67 # Automatic calculation confusionMatrix(factor(pred_binary), test_data$churn) # ROC-AUC: Best overall metric library(pROC) roc_obj <- roc(test_data$churn, predictions) auc(roc_obj) # AUC = 0.72 plot(roc_obj, main="ROC Curve")
HYPERPARAMETER TUNING:
# Grid search for best parameters library(caret) # Define parameter grid grid <- expand.grid( mtry = c(2, 3, 4), # Variables per split splitrule = "gini", min.node.size = c(5, 10, 15) ) # Train with cross-validation train_control <- trainControl( method = "cv", # 10-fold cross-validation number = 10, classProbs = TRUE, summaryFunction = twoClassSummary ) # Tune model tune_model <- train( churn ~ ., data = train_data, method = "ranger", trControl = train_control, tuneGrid = grid, metric = "ROC" ) # Best parameters tune_model$bestTune # Final model with best parameters final_model <- tune_model$finalModel
CROSS-VALIDATION - ROBUST EVALUATION:
library(caret) # 5-fold cross-validation folds <- createFolds(data$outcome, k=5) results <- lapply(folds, function(i) { train <- data[-i, ] test <- data[i, ] model <- glm(outcome ~ ., data=train, family=binomial) pred <- predict(model, test, type="response") pred_binary <- ifelse(pred > 0.5, 1, 0) accuracy <- mean(pred_binary == test$outcome) return(accuracy) }) mean(unlist(results)) # Average across folds