# ====================================================================
# Week 3: The Tidyverse — Data Manipulation
# Kmex Consult - R Data Science Course
# Open in RStudio and run each block with Ctrl+Enter (Cmd+Enter on Mac).
# ====================================================================

# dplyr, tidyr, and the pipe operator

# ====================================================================
# WEEK 3 OVERVIEW
# ====================================================================

The tidyverse is a collection of R packages designed for data analysis.
It's the professional standard for data manipulation in R.

WHAT IS TIDYVERSE?
- Collection of packages that work well together
- dplyr: Data manipulation
- tidyr: Data reshaping
- readr: Reading data
- ggplot2: Visualization (learned in Week 5)
- and more...

PHILOSOPHY:
"Tidy data" = Easy to analyze data
- Each row = observation
- Each column = variable
- Each table = one type of observation

THE PIPE OPERATOR:
%>% (or new |> in R 4.1+)
Reads: "and then"
Makes code more readable

EXAMPLE:
# Without pipe (ugly):
result <- round(mean(sqrt(c(1, 4, 9, 16))), 2)

# With pipe (beautiful):
c(1, 4, 9, 16) |>
sqrt() |>
mean() |>
round(2)

THIS WEEK YOU'LL MASTER:
✓ The 5 dplyr verbs: select, filter, arrange, mutate, summarize
✓ group_by() for grouped operations
✓ tidyr: pivot_wider() and pivot_longer()
✓ The pipe operator |>
✓ Working with character/string data
✓ Dates and times
✓ String manipulation functions
✓ Professional data cleaning workflow


# ====================================================================
# SESSION 1: DPLYR - THE 5 MAIN VERBS
# ====================================================================

# --- Duration: 2 hours ---

# --- 1.1 The 5 Main dplyr Verbs ---
# INTRODUCTION TO DPLYR:
# dplyr provides 5 main "verbs" for data manipulation:
# 1. select() - Choose columns
# 2. filter() - Choose rows
# 3. arrange() - Sort rows
# 4. mutate() - Create/modify columns
# 5. summarize() - Compute summaries
# PLUS two most important:
# 6. group_by() - Group data
# 7. join operations - Combine tables
# EVERYTHING RETURNS A DATA FRAME!
# SETUP:


library(dplyr)

# We'll use this sample data
employees <- tibble(
id = 1:5,
name = c("Alice", "Bob", "Charlie", "David", "Eve"),
department = c("Sales", "IT", "Sales", "HR", "IT"),
salary = c(50000, 60000, 55000, 52000, 65000),
years = c(3, 5, 2, 4, 6)
)

# VERB 1: SELECT() - CHOOSE COLUMNS


# Select specific columns
select(employees, name, salary)
# Output: 2 columns (name, salary)

# Select by position
select(employees, 1, 3, 4) # id, department, salary

# Select with helpers
select(employees, starts_with("s")) # salary
select(employees, ends_with("y")) # salary, years
select(employees, contains("ary")) # salary, years
select(employees, everything()) # All columns

# Remove columns
select(employees, -id, -years)
select(employees, -(1:2))

# Rename while selecting
select(employees,
Employee_Name = name,
Annual_Salary = salary
)

# With pipe
employees |>
select(name, salary)

# VERB 2: FILTER() - CHOOSE ROWS


# Filter rows where condition is TRUE
filter(employees, salary > 55000)
# Output: Rows with salary > 55000

# Multiple conditions (AND)
filter(employees, salary > 55000 & years > 3)

# OR condition
filter(employees, department == "Sales" | department == "IT")

# IN condition
filter(employees, department %in% c("Sales", "HR"))

# NOT condition
filter(employees, !is.na(salary))
filter(employees, !(department == "HR"))

# String matching
filter(employees, name %in% c("Alice", "Bob"))
filter(employees, grepl("^A", name)) # Names starting with A

# With pipe
employees |>
filter(salary > 55000) |>
filter(years > 3)
# Same as: filter(employees, salary > 55000 & years > 3)

# VERB 3: ARRANGE() - SORT ROWS


# Sort ascending (default)
arrange(employees, salary)
# Output: Lowest to highest salary

# Sort descending
arrange(employees, desc(salary))

# Sort by multiple columns
arrange(employees, department, desc(salary))
# First by department (A-Z), then salary (high to low)

# Sort by negative (numeric columns)
arrange(employees, -years) # Descending

# With pipe
employees |>
arrange(desc(salary)) |>
select(name, salary, department)

# VERB 4: MUTATE() - CREATE/MODIFY COLUMNS


# Create new column
mutate(employees, hourly_rate = salary / 2000)

# Create multiple columns
mutate(employees,
hourly_rate = salary / 2000,
bonus = salary * 0.1,
total_comp = salary + bonus
)

# Conditional column (IF statement)
mutate(employees,
senior = if_else(years >= 5, "Yes", "No")
)

# Complex conditional (CASE statement)
mutate(employees,
salary_level = case_when(
salary < 52000 ~ "Entry",
salary < 58000 ~ "Mid",
salary >= 58000 ~ "Senior"
)
)

# Modify existing column
mutate(employees, salary = salary * 1.1) # 10% raise!

# Multiple operations on one row
employees |>
mutate(
salary_level = case_when(
salary < 52000 ~ "Entry",
salary >= 52000 ~ "Mid"
),
eligible_bonus = salary > 55000
) |>
select(name, salary, salary_level, eligible_bonus)

# VERB 5: SUMMARIZE() - COMPUTE SUMMARIES


# Single summary statistic
summarize(employees, avg_salary = mean(salary))
# Output: Single row with avg_salary = 56400

# Multiple summaries
summarize(employees,
avg_salary = mean(salary),
median_salary = median(salary),
min_salary = min(salary),
max_salary = max(salary),
n = n() # Count rows
)

# Other useful aggregate functions
summarize(employees,
total_salary = sum(salary),
count = n(),
distinct_depts = n_distinct(department)
)

# With pipe
employees |>
summarize(
avg_years = mean(years),
total_employees = n()
)

# VERB 6: GROUP_BY() - GROUP OPERATIONS


# Group and then summarize
employees |>
group_by(department) |>
summarize(
avg_salary = mean(salary),
count = n(),
avg_years = mean(years)
)
# Output: One row per department with summaries

# More complex example
employees |>
group_by(department) |>
mutate(dept_avg = mean(salary)) |>
filter(salary > dept_avg) |>
select(name, salary, department, dept_avg)

# Multiple groups
employees |>
group_by(department, years > 3) |>
summarize(count = n())

# THE PIPE OPERATOR |>:


# Without pipe (hard to read):
result <- summarize(
filter(
mutate(
employees,
hourly = salary / 2000
),
hourly > 30
),
avg_hourly = mean(hourly)
)

# With pipe (easy to read):
result <- employees |>
mutate(hourly = salary / 2000) |>
filter(hourly > 30) |>
summarize(avg_hourly = mean(hourly))

# Easier to understand flow!
employees |>
filter(department == "Sales") |>
mutate(bonus = salary * 0.15) |>
arrange(desc(salary)) |>
select(name, salary, bonus)

# KEYS TO SUCCESS:
# - Each pipe starts fresh with the data from previous step
# - Left side of |> is piped into first argument of next function
# - Ctrl+Shift+M creates pipe shortcut (RStudio)

# COMBINING MULTIPLE VERBS:


# Real workflow
employees |>
filter(salary > 50000) |> # Keep high earners
group_by(department) |> # Group by dept
mutate(
dept_avg = mean(salary), # Add dept avg
above_avg = salary > dept_avg # Flag above avg
) |>
arrange(department, desc(salary)) |> # Sort
select(name, department, salary, dept_avg) |> # Keep columns
filter(above_avg) |> # Only above avg
ungroup() # Remove grouping


# ====================================================================
# SESSION 2: TIDYR - RESHAPING DATA
# ====================================================================

# --- Duration: 2 hours ---

# --- 2.1 Wide vs Long Format and Reshaping ---
# TIDY DATA CONCEPT:
# "Happy families are all alike; every unhappy family is unhappy in its own way"
# TIDY DATA REQUIREMENTS:
# 1. Each row = one observation
# 2. Each column = one variable
# 3. Each cell = one value
# WIDE vs LONG FORMAT:
# WIDE FORMAT (often messy):


Name Q1 Q2 Q3 Q4
Alice 85 90 88 92
Bob 78 80 82 85

# - Multiple time periods as columns
# - More compact
# - Harder to analyze
# LONG FORMAT (tidy):


Name Quarter Score
Alice Q1 85
Alice Q2 90
Alice Q3 88
Alice Q4 92
Bob Q1 78
Bob Q2 80
...

# - One observation per row
# - Better for analysis
# - Required by ggplot2 and models
# RESHAPING DATA:
# pivot_wider() - Wide to wide (or long to wide):


# Start with long data
long_df <- tibble(
name = c("Alice", "Alice", "Bob", "Bob"),
quarter = c("Q1", "Q2", "Q1", "Q2"),
score = c(85, 90, 78, 80)
)

# Reshape to wide
wide_df <- pivot_wider(
long_df,
names_from = quarter, # Column names from this
values_from = score # Values from this
)
# Output:
# name Q1 Q2
# Alice 85 90
# Bob 78 80

pivot_longer() - Wide to long:

# # Start with wide data
# wide_df <- tibble(
# name = c("Alice", "Bob"),
# Q1 = c(85, 78),
# Q2 = c(90, 80),
# Q3 = c(88, 82)
# )
# # Reshape to long
# long_df <- pivot_longer(
# wide_df,
# cols = Q1:Q3, # Which columns to pivot
# names_to = "quarter", # New column for names
# values_to = "score" # New column for values
# )
# # Output:
# # name quarter score
# # Alice Q1 85
# # Alice Q2 90
# # Alice Q3 88
# # Bob Q1 78
# # ...



SEPARATING AND UNITING COLUMNS:

# # Start with combined data
# df <- tibble(
# id = 1:3,
# name_age = c("Alice_25", "Bob_30", "Charlie_28")
# )
# # Separate into two columns
# separated <- separate(
# df,
# col = name_age,
# into = c("name", "age"),
# sep = "_"
# )
# # Output:
# # id name age
# # 1 1 Alice 25
# # 2 2 Bob 30
# # 3 3 Charlie 28
# # Unite columns
# df2 <- tibble(
# id = 1:3,
# first = c("Alice", "Bob", "Charlie"),
# last = c("Smith", "Jones", "Brown")
# )
# united <- unite(
# df2,
# col = "full_name",
# first:last,
# sep = " "
# )
# # Output:
# # id full_name
# # 1 1 Alice Smith
# # 2 2 Bob Jones
# # 3 3 Charlie Brown



HANDLING MISSING DATA:

# df <- tibble(
# name = c("Alice", "Bob", "Charlie"),
# age = c(25, NA, 28),
# score = c(85, 90, NA)
# )
# # Check missing
# is.na(df)
# # Find rows with any NA
# df |> filter(!is.complete.cases(df))
# # Remove rows with any NA
# df |> drop_na()
# # Remove rows with NA in specific column
# df |> drop_na(age)
# # Fill missing values
# df |> fill(age, .direction = "down")
# df |> replace_na(list(age = 0, score = 0))