library(dplyr) library(rpart) library(rpart.plot) # Classification and Regression Trees (CART) ---- df <- read.csv("https://roualdes.sfo3.digitaloceanspaces.com/data/penguins.csv") ## classification, take 01 df$i <- factor(df$island, labels = c("Biscoe", "Dream", "Torgersen")) ## don't use rpart like this fitr <- rpart(i ~ species + bill_length_mm + bill_depth_mm + flipper_length_mm, data = df, control = rpart.control( cp = 0, minsplit = 2, minbucket = 1, maxdepth = 30, xval = 0 )) rpart.plot(fitr) # more realistic fitr <- rpart(i ~ species + bill_length_mm + bill_depth_mm + flipper_length_mm, data = df) # summary(fitr) rpart.plot(fitr) ## classification, take 02 fitr <- rpart(i ~ species + bill_length_mm + bill_depth_mm + flipper_length_mm, data = df) rpart.plot(fitr) adx <- df$species == "Adelie" phat <- predict(fitr) head(data.frame(Biscoe = phat[,1], Dream = phat[,2], Torgersen = phat[,3], island = df$island, species = df$species )[adx,]) ## regression df <- read.csv("https://roualdes.sfo3.digitaloceanspaces.com/data/abalone.csv") fitr <- rpart(age ~ length + diameter + height + whole_weight + shucked_weight + viscera_weight + shell_weight, data = df) # summary(fitr) rpart.plot(fitr) predict(fitr) # Trees, mathematically ---- # on board # bagging ---- ## bootstrap aggregating # imagine phat \in R^{N, K} # where K is number of groups to be predicted # with probabilities phat # such that each row sums to 1 vote01 <- apply(phat01, 1, which.max) # length N vote02 <- apply(phat02, 1, which.max) ... voteR <- apply(phatR, 1, which.max) for (r in 1:R) { idx <- sample(1:N, N, replace = TRUE) fit <- rpart(y ~ x, data = df[idx,]) phat <- predict(fit) apply(phat, 1, which.max) } # towards boosting ---- ggplot() + geom_point(data = df, aes(bill_length_mm, body_mass_g, color = species)) ggplot() + geom_point(data = df, aes(flipper_length_mm, body_mass_g, color = species)) summary(fitf <- lm(body_mass_g ~ species * flipper_length_mm, data = df)) df$r <- df$body_mass_g - predict(fitf) ggplot() + geom_point(data = df, aes(bill_length_mm, r, color = species)) summary(fitb <- lm(r ~ species * bill_length_mm, data = df)) df$r <- df$r - predict(fitb) ggplot() + geom_point(data = df, aes(bill_depth_mm, r, color = species)) summary(fitd <- lm(r ~ species * bill_depth_mm, data = df)) df$r <- df$r - predict(fitd) ggplot() + geom_point(data = df, aes(sex, r, color = species)) summary(fits <- lm(r ~ sex, data = df)) df$r <- df$r - predict(fits) ggplot() + geom_point(data = df, aes(bill_depth_mm, r, color = species)) # XGBoost ---- # creative solution to split finding + # many tricks to reduce computational costs # Split Finding: # upfront sorting, then # a novel algorithm for quickly calculating quantiles # LightGBM ---- # Gradient-based One-Side Sampling + # Exclusive Feature Bundling + # many tricks to reduce computational costs # Gradient-based One-Sided Sampling # pull up p. 338 - 339 of Elements # focus training on data which incorrectly predict # Exclusive Feature Bundling # reduce the effective number of predictors # by bundling together sparse predictors that # rarely take nonzero values simultaneously # references ---- # OER: Open Educational Resources are both free and publicly accessible # OER 1. https://hastie.su.domains/ElemStatLearn/ chapter Chapters 9 and 10 # OER 2. UC Berkeley's Stat 154/254: Statistical Machine Learning https://stat154.berkeley.edu/spring-2025/lectures/lectures.html # specifically # https://stat154.berkeley.edu/spring-2025/lectures/unit4/unit4_bagging.html # https://stat154.berkeley.edu/spring-2025/lectures/unit4/unit4_boosting.html # https://stat154.berkeley.edu/spring-2025/lectures/unit4/unit4_cart.html # OER 3. https://arxiv.org/abs/1603.02754 XGBoost # OER 4. https://proceedings.neurips.cc/paper_files/paper/2017/file/6449f44a102fde848669bdd9eb6b76fa-Paper.pdf LightGBM