Advanced data manipulation in R
This week builds on Week 1 fundamentals. We dive deep into R's most powerful data structures. KEY TOPICS:
DATA STRUCTURES RECAP: Vector → 1D, single type, atomic Matrix → 2D, single type List → 1D, multiple types Data Frame → 2D, multiple types (Excel-like) This week you'll become proficient with all four. BY END OF WEEK 2: ✓ Create and manipulate vectors efficiently ✓ Master data frame operations ✓ Work with lists fluently ✓ Handle dates and times ✓ Manipulate strings like a pro ✓ Read/write multiple file formats ✓ Clean and validate data ✓ Know how to debug data issues
VECTORS - EVERYTHING IS A VECTOR:
Recall: In R, a single value is a vector of length 1
# Create vectors v1 <- c(1, 2, 3, 4, 5) v2 <- 1:10 v3 <- seq(0, 100, by=10) v4 <- rep(5, times=5) v5 <- c("apple", "banana", "cherry") # Named vectors (elements have names) scores <- c(Alice=85, Bob=90, Charlie=78) scores # Alice Bob Charlie # 85 90 78 scores["Alice"] # [1] 85 scores[c("Alice", "Bob")] # Alice Bob # 85 90 names(scores) # [1] "Alice" "Bob" "Charlie" names(scores) <- c("Student1", "Student2", "Student3")
VECTOR INDEXING - CRITICAL SKILL:
v <- c(10, 20, 30, 40, 50) # Positive indexing (returns elements) v[1] # [1] 10 (first) v[3] # [1] 30 (third) v[c(1, 3, 5)] # [1] 10 30 50 (specific indices) # Negative indexing (removes elements) v[-1] # [1] 20 30 40 50 (exclude first) v[-c(1, 2)] # [1] 30 40 50 (exclude first two) # Logical indexing (conditional) v[v > 25] # [1] 30 40 50 (values > 25) v[v > 25 & v < 50] # [1] 30 40 (between 25-50) # Empty bracket (all elements) v[] # [1] 10 20 30 40 50 # Out of bounds returns NA v[10] # [1] NA
VECTOR ARITHMETIC & RECYCLING:
v1 <- c(1, 2, 3, 4) v2 <- c(10, 20, 30, 40) scalar <- 5 # Element-wise operations v1 + v2 # [1] 11 22 33 44 v1 * scalar # [1] 5 10 15 20 v1 ^ 2 # [1] 1 4 9 16 # RECYCLING - when vectors different lengths! v_short <- c(1, 2) v_long <- c(10, 20, 30, 40) v_short + v_long # [1] 11 22 31 42 # Warning: longer object length is not a multiple of shorter object length # Recycling example c(1, 2, 3) + c(10, 20) # 1+10, 2+20, 3+10 # [1] 11 22 13 # Be careful! Recycling can hide bugs!
VECTOR FUNCTIONS (AGGREGATE OPERATIONS):
v <- c(5, 2, 8, 1, 9, 3) # Summary functions sum(v) # [1] 28 mean(v) # [1] 4.666667 median(v) # [1] 4 sd(v) # Standard deviation var(v) # Variance min(v) # [1] 1 max(v) # [1] 9 range(v) # [1] 1 9 quantile(v) # 0%, 25%, 50%, 75%, 100% # 0% 25% 50% 75% 100% # 1.00 2.75 4.00 7.25 9.00 # Counting length(v) # [1] 6 sum(v > 5) # [1] 3 (count where v > 5) # Sorting and ordering sort(v) # [1] 1 2 3 5 8 9 sort(v, decreasing=TRUE) # [1] 9 8 5 3 2 1 rev(v) # [1] 3 9 1 8 2 5 (reverse) order(v) # [1] 4 2 6 1 5 3 (indices for sorting) # Unique values unique(v) # [1] 5 2 8 1 9 3 (all same here) unique(c(1, 1, 2, 2, 3)) # [1] 1 2 3 # Duplicates duplicated(c(1, 1, 2, 2, 3)) # [1] F T F T F which(duplicated(c(1, 1, 2, 2, 3))) # [1] 2 4
MATRICES - 2D ARRAYS:
# Create matrix m <- matrix(1:12, nrow=3, ncol=4) m # [,1] [,2] [,3] [,4] # [1,] 1 4 7 10 # [2,] 2 5 8 11 # [3,] 3 6 9 12 # By rows m_byrow <- matrix(1:12, nrow=3, ncol=4, byrow=TRUE) m_byrow # [,1] [,2] [,3] [,4] # [1,] 1 2 3 4 # [2,] 5 6 7 8 # [3,] 9 10 11 12 # From vectors (bind) v1 <- c(1, 2, 3) v2 <- c(4, 5, 6) cbind(v1, v2) # Column bind # v1 v2 # [1,] 1 4 # [2,] 2 5 # [3,] 3 6 rbind(v1, v2) # Row bind # [,1] [,2] [,3] # v1 1 2 3 # v2 4 5 6 # Matrix properties dim(m) # [1] 3 4 nrow(m) # [1] 3 ncol(m) # [1] 4 rownames(m) # NULL (none) colnames(m) # NULL # Add names colnames(m) <- c("A", "B", "C", "D") rownames(m) <- c("Row1", "Row2", "Row3") # Indexing m[1, ] # First row m[, 2] # Second column m[1, 2] # Row 1, Column 2 = 4 m["Row1", "B"] # By names
MATRIX OPERATIONS:
m1 <- matrix(1:4, nrow=2) m2 <- matrix(5:8, nrow=2) # Element-wise operations m1 + m2 # Matrix addition m1 * m2 # Element-wise multiplication (not matrix mult!) m1 / m2 # Element-wise division # Matrix multiplication m1 %*% m2 # Matrix multiplication # Matrix functions t(m1) # Transpose det(m1) # Determinant solve(m1) # Inverse diag(m1) # Diagonal elements trace(m1) # Sum of diagonal (require Matrix package) # Apply functions to matrices rowSums(m1) # Sum each row colSums(m1) # Sum each column rowMeans(m1) # Mean of each row colMeans(m1) # Mean of each column apply(m1, 1, sum) # Sum rows (same as rowSums) apply(m1, 2, mean) # Mean columns (same as colMeans)
LOGICAL VECTORS - FUNDAMENTAL:
v <- c(10, 20, 30, 40, 50) # Create logical vector is_large <- v > 25 is_large # [1] FALSE FALSE TRUE TRUE TRUE # Use for filtering (MOST COMMON OPERATION!) v[is_large] # [1] 30 40 50 v[v > 25 & v < 50] # [1] 30 40 v[v %in% c(20, 30, 50)] # [1] 20 30 50 # Count TRUE values sum(is_large) # [1] 3 mean(is_large) # [1] 0.6 (60% are TRUE) any(is_large) # [1] TRUE (at least one) all(is_large) # [1] FALSE (not all) # Find indices of TRUE which(is_large) # [1] 3 4 5 which.min(v) # [1] 1 (index of minimum) which.max(v) # [1] 5 (index of maximum) # Set operations x <- c(1, 2, 3, 4, 5) y <- c(3, 4, 5, 6, 7) intersect(x, y) # [1] 3 4 5 (common) union(x, y) # [1] 1 2 3 4 5 6 7 (all) setdiff(x, y) # [1] 1 2 (in x but not y)
FACTORS - CATEGORICAL DATA:
# Character vector of categories colors <- c("red", "blue", "red", "green", "blue", "red") # Convert to factor colors_factor <- factor(colors) colors_factor # [1] red blue red green blue red # Levels: blue green red # Levels are the unique categories (sorted alphabetically) levels(colors_factor) # [1] "blue" "green" "red" # Can specify order grade_factor <- factor(c("B", "A", "C", "A", "B"), levels = c("A", "B", "C"), ordered = TRUE) grade_factor # [1] B A C A B # Levels: A < B < C # Works with numeric categories too score_cat <- cut(c(45, 78, 92, 56, 88), breaks = c(0, 60, 80, 100), labels = c("F", "C", "A")) score_cat # [1] F C A C A # Levels: F C A # Table (count occurrences) table(colors_factor) # blue green red # 2 1 3
SAMPLING AND SHUFFLING:
# Sample randomly sample(1:10, 5) # 5 random numbers from 1-10 sample(1:10, 5) # Different every time! sample(1:10, 5, replace=TRUE) # With replacement # Shuffle vector v <- 1:5 sample(v) # Random order # Set seed for reproducibility set.seed(42) sample(1:10, 5) # [1] 3 8 4 9 2 set.seed(42) sample(1:10, 5) # [1] 3 8 4 9 2 (same!) # Random numbers from distributions rnorm(10, mean=100, sd=15) # 10 normal values runif(10, min=0, max=1) # 10 uniform [0,1] rbinom(10, size=1, prob=0.5) # 10 coin flips rpois(10, lambda=3) # 10 Poisson values
DATA FRAMES - THE MOST IMPORTANT STRUCTURE:
Think: Data frame = Excel spreadsheet in R
CREATING DATA FRAMES:
# Method 1: data.frame() students <- data.frame( name = c("Alice", "Bob", "Charlie"), age = c(20, 21, 19), score = c(85, 90, 78), grade = c("B", "A", "C") ) students # name age score grade # 1 Alice 20 85 B # 2 Bob 21 90 A # 3 Charlie 19 78 C # Method 2: tibble() (modern data frame) library(tibble) students_tib <- tibble( name = c("Alice", "Bob", "Charlie"), age = c(20, 21, 19), score = c(85, 90, 78) ) # Tibbles print nicer and safer by default # Method 3: From existing data df <- data.frame(v1=1:3, v2=4:6, v3=7:9) # Method 4: Bind vectors v1 <- c("Alice", "Bob", "Charlie") v2 <- c(20, 21, 19) v3 <- c(85, 90, 78) df <- data.frame(name=v1, age=v2, score=v3)
ACCESSING PARTS OF DATA FRAMES:
df <- data.frame( name = c("Alice", "Bob", "Charlie"), age = c(20, 21, 19), score = c(85, 90, 78) ) # Get entire column (multiple methods) df$name # [1] "Alice" "Bob" "Charlie" df[["name"]] # Alternative (returns vector) df["name"] # Returns single-column data frame df[, 1] # By position (returns vector) df[, "name"] # By name (returns vector) # Get specific row df[1, ] # First row (data frame) df[c(1, 3), ] # Rows 1 and 3 df[df$age > 19, ] # Rows where age > 19 # Get specific cell df[1, 2] # Row 1, Column 2 df[1, "age"] # Row 1, column "age" df$age[1] # First age value (also [1] 20) # Dimensions dim(df) # [1] 3 3 nrow(df) # [1] 3 (number rows) ncol(df) # [1] 3 (number columns)
MODIFYING DATA FRAMES:
# Add new column df$gpa <- c(3.5, 3.8, 3.2) # Add multiple columns at once df[c("income", "city")] <- list( income = c(30000, 35000, 28000), city = c("NYC", "LA", "Chicago") ) # Add column with condition df$pass <- df$score >= 80 # Rename column names(df)[1] <- "student_name" colnames(df)[1] <- "student_name" # Rename multiple columns library(dplyr) df <- rename(df, student_name = name, test_score = score ) # Remove column df$gpa <- NULL df[, "gpa"] <- NULL df <- df[, -which(names(df) == "gpa")] # Reorder columns df <- df[, c("name", "score", "age")] # Sort rows df_sorted <- df[order(df$age), ] # By age ascending df_sorted <- df[order(df$age, decreasing=TRUE), ] # Descending df_sorted <- df[order(df$age, -df$score), ] # Age, then score desc # Merge/combine data frames df1 <- data.frame(id=1:3, name=c("A", "B", "C")) df2 <- data.frame(id=1:3, score=c(85, 90, 78)) merged <- merge(df1, df2, by="id") # OR using cbind/rbind df_combined <- cbind(df1, df2)
DATA FRAME MANIPULATION WITH dplyr:
library(dplyr) df <- data.frame( name = c("Alice", "Bob", "Charlie", "David"), age = c(20, 21, 19, 22), score = c(85, 90, 78, 88), city = c("NYC", "LA", "NYC", "Chicago") ) # Select columns (select) select(df, name, score) select(df, -age) # All except age select(df, starts_with("s")) # Columns starting with "s" # Filter rows (filter) filter(df, age > 20) filter(df, score >= 80) filter(df, age > 20 & score >= 85) filter(df, city %in% c("NYC", "LA")) # Add/modify columns (mutate) mutate(df, grade = ifelse(score >= 85, "A", "B"), age_next_year = age + 1 ) # Chaining with pipe |> (or %>%) df |> filter(score >= 80) |> select(name, score) |> arrange(desc(score)) # Easier to read! # Group and summarize df |> group_by(city) |> summarize( n = n(), avg_age = mean(age), avg_score = mean(score) ) # Output: # city n avg_age avg_score # 1 Chicago 1 22 88 # 2 LA 1 21 90 # 3 NYC 2 19.5 81.5