Data Science Fundamentals Course
Data cleaning and preprocessing is arguably the most time-consuming and critical part of any data science project. Often called "data wrangling," this phase can consume 60-80% of a data scientist's time. Clean, well-prepared data leads to better analyses, more accurate models, and more reliable results. This week you will learn essential techniques for:
By the end of Week 4, you will be able to:
Week 4 is divided into three 2-hour sessions:
Missing data (also called null values or NaN - Not a Number) is one of the most common problems in real datasets. Understanding why data is missing and how to handle it is crucial. Types of missing data:
Common causes:
import pandas as pd import numpy as np # Create dataset with missing values df = pd.DataFrame({ 'Name': ['Alice', 'Bob', 'Charlie', 'Diana', 'Eve'], 'Age': [25, np.nan, 28, 32, np.nan], 'City': ['Lagos', 'Accra', np.nan, 'Johannesburg', 'Cape Town'], 'Salary': [60000, 75000, 65000, np.nan, 55000], 'Department': ['Sales', np.nan, 'IT', 'Finance', 'Marketing'] }) print(df) # Output: # Name Age City Salary Department # 0 Alice 25.0 Lagos 60000.0 Sales # 1 Bob NaN Accra 75000.0 NaN # 2 Charlie 28.0 NaN 65000.0 IT # 3 Diana 32.0 Johannesburg NaN Finance # 4 Eve NaN Cape Town 55000.0 Marketing # Detect missing values print(df.isnull()) # Returns boolean DataFrame # Count missing values per column print(df.isnull().sum()) # Output: # Name 0 # Age 2 # City 1 # Salary 1 # Department 1 # dtype: int64 # Count total missing values print(df.isnull().sum().sum()) # Output: 5 # Percentage of missing data print((df.isnull().sum() / len(df) * 100).round(2)) # Output: # Name 0.0 # Age 40.0 # City 20.0 # Salary 20.0 # Department 20.0 # dtype: float64 # Alternative: use .info() to see missing data print(df.info()) # Find rows with any missing values rows_with_missing = df[df.isnull().any(axis=1)] print(rows_with_missing) # Find rows with no missing values complete_rows = df.dropna() print(complete_rows) # Find columns with missing values cols_with_missing = df.columns[df.isnull().any()].tolist() print(f"Columns with missing values: {cols_with_missing}") # Output: Columns with missing values: ['Age', 'City', 'Salary', 'Department']
import pandas as pd import numpy as np df = pd.DataFrame({ 'Name': ['Alice', 'Bob', 'Charlie', 'Diana', 'Eve'], 'Age': [25, np.nan, 28, 32, np.nan], 'City': ['Lagos', 'Accra', np.nan, 'Johannesburg', 'Cape Town'], 'Salary': [60000, 75000, 65000, np.nan, 55000] }) # Remove rows with ANY missing values df_clean = df.dropna() print(df_clean) # Only Charlie and Eve remain - others have missing values # Note: Charlie actually has all values! # Remove rows where ALL values are missing df_clean = df.dropna(how='all') print(df_clean) # Removes entire rows only if all values are NaN # Remove rows with missing values in specific column df_clean = df.dropna(subset=['Age']) print(df_clean) # Removes rows where Age is NaN # Remove rows missing values in specific columns df_clean = df.dropna(subset=['Age', 'Salary']) print(df_clean) # Remove columns with ANY missing values df_clean = df.dropna(axis=1) print(df_clean) # Removes City and Salary columns # Remove columns where percentage of missing > threshold threshold = 0.3 # 30% df_clean = df.dropna(thresh=len(df)*(1-threshold), axis=1) print(df_clean) # When to use deletion: # - Very few missing values (< 5%) # - Missing values are random (MCAR) # - You have plenty of data # When NOT to use: # - Missing data represents important information # - Large percentage of data is missing # - You need to preserve all observations
import pandas as pd import numpy as np df = pd.DataFrame({ 'Product': ['A', 'B', 'C', 'D', 'E'], 'Price': [10.0, np.nan, 15.0, np.nan, 20.0], 'Quantity': [100, 150, np.nan, 200, 250] }) # Fill with constant value df_filled = df.fillna(0) print(df_filled) # Fill with specific value per column df_filled = df.fillna({'Price': 12.5, 'Quantity': 175}) print(df_filled) # Forward fill: use previous value df_filled = df.fillna(method='ffill') print(df_filled) # Backward fill: use next value df_filled = df.fillna(method='bfill') print(df_filled) # Fill with mean (for numerical columns) df['Price'].fillna(df['Price'].mean(), inplace=True) print(df) # Fill with median (robust to outliers) df['Quantity'].fillna(df['Quantity'].median(), inplace=True) print(df) # Fill with mode (most common value - for categorical) df_cat = pd.DataFrame({ 'Department': ['Sales', np.nan, 'IT', 'Sales', np.nan], 'Level': ['Junior', 'Senior', np.nan, 'Junior', 'Senior'] }) df_cat['Department'].fillna(df_cat['Department'].mode()[0], inplace=True) print(df_cat) # Interpolation for time series df_ts = pd.DataFrame({ 'Date': pd.date_range('2023-01-01', periods=5), 'Value': [10, np.nan, 20, np.nan, 30] }) df_ts['Value'] = df_ts['Value'].interpolate() print(df_ts) # Output: Fills NaN values with interpolated values (12, 20, 24, etc.) # When to use imputation: # - Moderate amount of missing data # - Missing data is random (MCAR/MAR) # - Deleting would lose important information # - Methods: # - Mean/Median: for numerical, skewed distributions # - Mode: for categorical # - Forward/Backward fill: for time series # - Interpolation: for continuous sequences
import pandas as pd # Create dataset with duplicates df = pd.DataFrame({ 'ID': [1, 2, 2, 3, 3, 3, 4], 'Name': ['Alice', 'Bob', 'Bob', 'Charlie', 'Charlie', 'Charlie', 'Diana'], 'Score': [85, 92, 92, 78, 78, 78, 95] }) print(df) # Find duplicate rows duplicates = df.duplicated() print(duplicates) # Output: [False False True False True True False] # Count duplicates print(f"Number of duplicates: {duplicates.sum()}") # See duplicate rows print(df[df.duplicated(keep=False)]) # keep=False shows all duplicates # Find duplicates based on specific columns duplicates = df.duplicated(subset=['ID']) print(df[duplicates]) # Remove duplicates: keep first occurrence (default) df_clean = df.drop_duplicates() print(df_clean) # Remove duplicates: keep last occurrence df_clean = df.drop_duplicates(keep='last') print(df_clean) # Remove duplicates based on specific columns df_clean = df.drop_duplicates(subset=['ID']) print(df_clean) # Remove duplicates in specific columns but keep all columns df_clean = df.drop_duplicates(subset=['Name', 'Score'], keep='first') print(df_clean) # Real-world: Check for duplicate entries in customer database customers = pd.DataFrame({ 'Email': ['alice@example.com', 'bob@example.com', 'alice@example.com'], 'Name': ['Alice', 'Bob', 'Alice'], 'Purchase': [100, 200, 150] }) # Find duplicate emails print(customers[customers.duplicated(subset=['Email'], keep=False)]) # Remove duplicate emails, keeping record with highest purchase customers = customers.sort_values('Purchase', ascending=False) customers_clean = customers.drop_duplicates(subset=['Email'], keep='first') print(customers_clean)
import pandas as pd import numpy as np # Create dataset with outliers df = pd.DataFrame({ 'Age': [25, 28, 30, 32, 35, 150, 29, 31], # 150 is outlier 'Salary': [50000, 55000, 60000, 65000, 70000, 75000, 80000, 5000000] # 5000000 is outlier }) print(df) # Method 1: IQR (Interquartile Range) - Most common Q1 = df['Age'].quantile(0.25) Q3 = df['Age'].quantile(0.75) IQR = Q3 - Q1 lower_bound = Q1 - 1.5 * IQR upper_bound = Q3 + 1.5 * IQR outliers = df[(df['Age'] < lower_bound) | (df['Age'] > upper_bound)] print(f"Outliers in Age: {len(outliers)} found") print(outliers) # Remove outliers df_clean = df[(df['Age'] >= lower_bound) & (df['Age'] <= upper_bound)] print(df_clean) # Method 2: Z-score from scipy import stats z_scores = np.abs(stats.zscore(df['Salary'])) outliers = df[z_scores > 3] # Threshold = 3 print(f"\nOutliers (Z-score > 3): {len(outliers)}") print(outliers) # Method 3: Visualization import matplotlib.pyplot as plt df.boxplot(column='Salary') plt.show() # Visual inspection # Real-world: Detect unusual transactions transactions = pd.DataFrame({ 'Amount': [50, 75, 100, 80, 120, 50000, 90, 110, 95], 'Category': ['Food', 'Transport', 'Food', 'Shopping', 'Food', 'Wire', 'Transport', 'Food', 'Shopping'] }) # IQR method for Amount Q1 = transactions['Amount'].quantile(0.25) Q3 = transactions['Amount'].quantile(0.75) IQR = Q3 - Q1 outlier_transactions = transactions[ (transactions['Amount'] < Q1 - 1.5*IQR) | (transactions['Amount'] > Q3 + 1.5*IQR) ] print("Unusual transactions:") print(outlier_transactions) # Handle outliers - Options: # 1. Remove them # 2. Cap them (set to max reasonable value) # 3. Transform them (log transformation) # 4. Keep them (if legitimate data) # Cap outliers at 95th percentile cap_value = transactions['Amount'].quantile(0.95) transactions['Amount_capped'] = transactions['Amount'].clip(upper=cap_value) print(transactions[['Amount', 'Amount_capped']])
import pandas as pd import numpy as np # Create dataset with wrong data types df = pd.DataFrame({ 'ID': ['1', '2', '3', '4'], # Should be int 'Name': ['Alice', 'Bob', 'Charlie', 'Diana'], 'Age': [25, 30, 28, 32], 'Salary': ['50000', '75000', '60000', '80000'], # Should be float 'HireDate': ['2020-01-15', '2019-05-20', '2021-03-10', '2018-11-30'], # Should be datetime 'IsManager': ['True', 'False', 'True', 'False'] # Should be bool }) print(df.dtypes) # Output: # ID object # Name object # Age int64 # Salary object (but should be numeric) # HireDate object (but should be datetime) # IsManager object (but should be bool) # Convert to correct types df['ID'] = df['ID'].astype('int') df['Salary'] = df['Salary'].astype('float') df['HireDate'] = pd.to_datetime(df['HireDate']) df['IsManager'] = df['IsManager'].astype('bool') print(df.dtypes) # Alternative: astype with errors handling df['ID'] = pd.to_numeric(df['ID'], errors='coerce') # NaN if conversion fails # Convert to category (useful for memory and speed) df['Name'] = df['Name'].astype('category') # Working with datetime print(df['HireDate'].dt.year) # Extract year print(df['HireDate'].dt.month) # Extract month print(df['HireDate'].dt.day) # Extract day # Calculate days employed df['DaysEmployed'] = (pd.Timestamp.now() - df['HireDate']).dt.days print(df) # Real-world: Process raw data raw_data = pd.DataFrame({ 'TransactionID': ['A001', 'A002', 'A003'], 'Amount': ['1500.50', '2300.75', 'Invalid'], # One invalid 'Date': ['2023-01-15', '2023-01-16', '2023-01-17'], 'Status': ['Complete', 'Pending', 'Complete'] }) # Safe conversion raw_data['Amount'] = pd.to_numeric(raw_data['Amount'], errors='coerce') raw_data['Date'] = pd.to_datetime(raw_data['Date']) raw_data['TransactionID'] = raw_data['TransactionID'].astype('str') # Remove rows with conversion errors (NaN values) raw_data_clean = raw_data.dropna(subset=['Amount']) print(raw_data_clean)
Scaling is essential because:
Common approaches:
import pandas as pd import numpy as np from sklearn.preprocessing import StandardScaler, MinMaxScaler, RobustScaler # Create dataset with different scales df = pd.DataFrame({ 'Age': [25, 30, 28, 35, 42], 'Salary': [50000, 75000, 60000, 90000, 120000], 'Experience': [2, 5, 3, 8, 15] }) print("Original data:") print(df) print("\nDescriptive stats:") print(df.describe()) # Method 1: Standardization (Z-score) # Formula: (x - mean) / std scaler = StandardScaler() df_standardized = scaler.fit_transform(df) df_standardized = pd.DataFrame(df_standardized, columns=df.columns) print("\nStandardized (Z-score):") print(df_standardized) print(df_standardized.describe()) # Manual standardization df_manual = (df - df.mean()) / df.std() print("\nManual standardization:") print(df_manual) # Method 2: Min-Max Scaling (Normalization) # Formula: (x - min) / (max - min) # Results in range [0, 1] scaler = MinMaxScaler() df_minmax = scaler.fit_transform(df) df_minmax = pd.DataFrame(df_minmax, columns=df.columns) print("\nMin-Max Scaled:") print(df_minmax) # Manual min-max scaling df_minmax_manual = (df - df.min()) / (df.max() - df.min()) print(df_minmax_manual) # Method 3: Robust Scaling (uses median and IQR) # Better for data with outliers scaler = RobustScaler() df_robust = scaler.fit_transform(df) df_robust = pd.DataFrame(df_robust, columns=df.columns) print("\nRobust Scaled:") print(df_robust) # Method 4: Log Scaling # For right-skewed data df_log = np.log(df + 1) # Add 1 to avoid log(0) print("\nLog Scaled:") print(df_log) # When to use each: # - Standardization: Most common, use before ML algorithms # - Min-Max: When you need values in specific range [0,1] # - Robust: When data has outliers # - Log: When data is right-skewed # Real-world: Prepare features for machine learning df = pd.DataFrame({ 'Height_cm': [170, 165, 180, 175, 168], 'Weight_kg': [70, 60, 85, 75, 65], 'Age': [25, 30, 28, 35, 42] }) # Standardize all features scaler = StandardScaler() features_scaled = scaler.fit_transform(df) df_scaled = pd.DataFrame(features_scaled, columns=df.columns) print("Original vs Scaled:") print(pd.DataFrame({ 'Height_original': df['Height_cm'], 'Height_scaled': df_scaled['Height_cm'], 'Weight_original': df['Weight_kg'], 'Weight_scaled': df_scaled['Weight_kg'] }))
import pandas as pd import numpy as np # Create dataset with categorical variables df = pd.DataFrame({ 'Product': ['Laptop', 'Phone', 'Tablet', 'Laptop', 'Phone'], 'Brand': ['Dell', 'Apple', 'Samsung', 'HP', 'Apple'], 'Size': ['Small', 'Medium', 'Large', 'Small', 'Medium'], 'Price': [50000, 70000, 20000, 45000, 65000] }) print("Original data:") print(df) print(df.dtypes) # Method 1: Label Encoding # Assigns integer to each category # Use when: ordinal data (order matters) from sklearn.preprocessing import LabelEncoder le = LabelEncoder() df['Product_encoded'] = le.fit_transform(df['Product']) print("\nLabel Encoding:") print(df[['Product', 'Product_encoded']]) # Laptop -> 0, Phone -> 1, Tablet -> 2 (alphabetical order) # Get mapping mapping = dict(zip(le.classes_, le.transform(le.classes_))) print(f"Mapping: {mapping}") # Method 2: One-Hot Encoding # Creates binary column for each category # Use for: nominal data (no order) df_onehot = pd.get_dummies(df['Product'], prefix='Product') print("\nOne-Hot Encoding:") print(df_onehot) # Drop original and combine df_encoded = pd.concat([df[['Price']], df_onehot], axis=1) print(df_encoded) # Alternative with drop_first (for avoiding multicollinearity) df_onehot_drop = pd.get_dummies(df['Product'], prefix='Product', drop_first=True) print("\nOne-Hot Encoding (drop first):") print(df_onehot_drop) # Method 3: Ordinal Encoding # For ordinal categorical data size_mapping = {'Small': 1, 'Medium': 2, 'Large': 3} df['Size_encoded'] = df['Size'].map(size_mapping) print("\nOrdinal Encoding:") print(df[['Size', 'Size_encoded']]) # Real-world: Prepare mixed data for modeling df_raw = pd.DataFrame({ 'Age': [25, 30, 28, 35, 42], 'Gender': ['M', 'F', 'M', 'F', 'M'], 'Department': ['Sales', 'IT', 'Sales', 'Finance', 'IT'], 'Salary': [50000, 75000, 60000, 90000, 120000] }) # Encode categorical variables df_processed = df_raw.copy() df_processed = pd.get_dummies(df_processed, columns=['Gender', 'Department'], drop_first=True) print("\nProcessed data:") print(df_processed) print(df_processed.dtypes)
import pandas as pd # Create sample datasets customers = pd.DataFrame({ 'CustomerID': [1, 2, 3, 4], 'Name': ['Alice', 'Bob', 'Charlie', 'Diana'], 'City': ['Lagos', 'Accra', 'Nairobi', 'Johannesburg'] }) orders = pd.DataFrame({ 'OrderID': [101, 102, 103, 104], 'CustomerID': [1, 2, 1, 4], 'Amount': [5000, 7500, 3000, 12000] }) print("Customers:") print(customers) print("\nOrders:") print(orders) # Method 1: Inner Merge (only matching rows) merged_inner = pd.merge(customers, orders, on='CustomerID', how='inner') print("\nInner Merge:") print(merged_inner) # Method 2: Left Merge (all from left, matching from right) merged_left = pd.merge(customers, orders, on='CustomerID', how='left') print("\nLeft Merge:") print(merged_left) # Method 3: Right Merge (matching from left, all from right) merged_right = pd.merge(customers, orders, on='CustomerID', how='right') print("\nRight Merge:") print(merged_right) # Method 4: Outer Merge (all from both) merged_outer = pd.merge(customers, orders, on='CustomerID', how='outer') print("\nOuter Merge:") print(merged_outer) # Merge on different column names df1 = pd.DataFrame({'ID': [1, 2, 3], 'Value': ['A', 'B', 'C']}) df2 = pd.DataFrame({'CustomerID': [1, 2, 3], 'Score': [85, 92, 78]}) merged = pd.merge(df1, df2, left_on='ID', right_on='CustomerID') print("\nMerge with different column names:") print(merged) # Concatenate along rows (like UNION in SQL) df_2023 = pd.DataFrame({ 'Product': ['A', 'B', 'C'], 'Sales': [1000, 2000, 1500] }) df_2024 = pd.DataFrame({ 'Product': ['A', 'B', 'D'], 'Sales': [1200, 2300, 1800] }) concat_result = pd.concat([df_2023, df_2024], ignore_index=True) print("\nConcatenated (rows):") print(concat_result) # Concatenate along columns concat_cols = pd.concat([df_2023, df_2024], axis=1, keys=['2023', '2024']) print("\nConcatenated (columns):") print(concat_cols) # Real-world: Combine customer and transaction data customers = pd.DataFrame({ 'CustID': [1, 2, 3, 4], 'Name': ['Alice', 'Bob', 'Charlie', 'Diana'] }) transactions = pd.DataFrame({ 'TransID': [1001, 1002, 1003, 1004, 1005], 'CustID': [1, 1, 2, 3, 1], 'Amount': [500, 300, 800, 1200, 400] }) # Merge and aggregate result = pd.merge(customers, transactions, on='CustID', how='left') customer_totals = result.groupby('Name')['Amount'].sum().reset_index() customer_totals.columns = ['Name', 'TotalSpent'] print("\nCustomer totals:") print(customer_totals)
import pandas as pd import numpy as np # Create base dataset df = pd.DataFrame({ 'Name': ['Alice', 'Bob', 'Charlie', 'Diana'], 'BirthYear': [1998, 1993, 1995, 1990], 'Salary': [50000, 75000, 60000, 90000], 'YearsExp': [2, 10, 8, 15] }) print("Original data:") print(df) # Feature 1: Create derived feature (age from birth year) current_year = 2024 df['Age'] = current_year - df['BirthYear'] print("\nWith Age derived from BirthYear:") print(df) # Feature 2: Create ratio feature df['SalaryPerYear'] = df['Salary'] / df['YearsExp'] print("\nWith SalaryPerYear ratio:") print(df) # Feature 3: Create categorical feature from numerical df['ExperienceLevel'] = pd.cut(df['YearsExp'], bins=[0, 5, 10, 20], labels=['Junior', 'Mid', 'Senior']) print("\nWith ExperienceLevel created from YearsExp:") print(df) # Feature 4: Create boolean feature df['IsHighEarner'] = df['Salary'] > 65000 print("\nWith IsHighEarner boolean:") print(df) # Feature 5: Polynomial features df['Salary_squared'] = df['Salary'] ** 2 df['Salary_sqrt'] = np.sqrt(df['Salary']) print("\nWith polynomial features:") print(df[['Salary', 'Salary_squared', 'Salary_sqrt']]) # Feature 6: Binning/Discretization df['SalaryBand'] = pd.cut(df['Salary'], bins=[0, 55000, 75000, 100000], labels=['Low', 'Medium', 'High']) print("\nWith SalaryBand:") print(df) # Feature 7: Interaction features df['Age_x_Experience'] = df['Age'] * df['YearsExp'] print("\nWith interaction feature:") print(df[['Age', 'YearsExp', 'Age_x_Experience']]) # Real-world: Create features for customer analysis customers = pd.DataFrame({ 'CustomerID': [1, 2, 3, 4, 5], 'JoinDate': pd.date_range('2020-01-01', periods=5, freq='Y'), 'TotalPurchase': [10000, 25000, 5000, 50000, 15000], 'LastPurchaseDate': pd.date_range('2023-01-01', periods=5, freq='M') }) # Feature: Customer tenure (in days) customers['Tenure_days'] = (pd.Timestamp.now() - customers['JoinDate']).dt.days # Feature: Days since last purchase customers['DaysSinceLastPurchase'] = (pd.Timestamp.now() - customers['LastPurchaseDate']).dt.days # Feature: Customer value segment customers['ValueSegment'] = pd.cut(customers['TotalPurchase'], bins=[0, 10000, 30000, float('inf')], labels=['Low', 'Medium', 'High']) # Feature: Churn risk (high days since purchase = high risk) customers['ChurnRisk'] = customers['DaysSinceLastPurchase'] > 180 print("\nCustomer features:") print(customers)
import pandas as pd import numpy as np from sklearn.preprocessing import StandardScaler # Step 1: Load and explore data raw_data = pd.DataFrame({ 'ID': [1, 2, 3, 4, 5, 6, 7, 8], 'Name': ['Alice', 'Bob', None, 'Diana', 'Eve', 'Frank', 'Grace', 'Henry'], 'Age': [25, 30, 28, np.nan, 35, 25, 28, 40], 'Salary': ['50000', '75000', '60000', '80000', 'Unknown', '55000', '70000', '95000'], 'Department': ['Sales', 'IT', 'IT', 'Finance', 'Sales', 'IT', 'Finance', 'Sales'], 'StartDate': ['2020-01-15', '2019-05-20', '2021-03-10', '2018-11-30', '2020-07-10', '2020-01-15', '2019-06-01', '2017-02-15'] }) print("Step 1: Original Data") print(raw_data) print(f"\nShape: {raw_data.shape}") print(f"Missing values: {raw_data.isnull().sum().sum()}") # Step 2: Handle missing values print("\n" + "="*50) print("Step 2: Handle Missing Values") raw_data['Name'].fillna('Unknown', inplace=True) raw_data['Age'].fillna(raw_data['Age'].median(), inplace=True) # Convert salary and handle invalid values raw_data['Salary'] = pd.to_numeric(raw_data['Salary'], errors='coerce') raw_data['Salary'].fillna(raw_data['Salary'].median(), inplace=True) print(raw_data) # Step 3: Handle data types print("\n" + "="*50) print("Step 3: Convert Data Types") raw_data['ID'] = raw_data['ID'].astype('int') raw_data['Age'] = raw_data['Age'].astype('int') raw_data['Salary'] = raw_data['Salary'].astype('float') raw_data['StartDate'] = pd.to_datetime(raw_data['StartDate']) raw_data['Department'] = raw_data['Department'].astype('category') print(raw_data.dtypes) # Step 4: Remove duplicates print("\n" + "="*50) print("Step 4: Remove Duplicates") initial_rows = len(raw_data) raw_data = raw_data.drop_duplicates() print(f"Removed {initial_rows - len(raw_data)} duplicate rows") # Step 5: Handle outliers print("\n" + "="*50) print("Step 5: Handle Outliers") Q1 = raw_data['Age'].quantile(0.25) Q3 = raw_data['Age'].quantile(0.75) IQR = Q3 - Q1 lower_bound = Q1 - 1.5 * IQR upper_bound = Q3 + 1.5 * IQR outliers = raw_data[(raw_data['Age'] < lower_bound) | (raw_data['Age'] > upper_bound)] print(f"Found {len(outliers)} age outliers: {outliers['Age'].tolist()}") # Option: Remove outliers # raw_data = raw_data[(raw_data['Age'] >= lower_bound) & (raw_data['Age'] <= upper_bound)] # Step 6: Create features print("\n" + "="*50) print("Step 6: Create Features") raw_data['Tenure_years'] = (pd.Timestamp.now() - raw_data['StartDate']).dt.days // 365 raw_data['SalaryPerYear'] = raw_data['Salary'] / (raw_data['Tenure_years'] + 1) print(raw_data[['Name', 'StartDate', 'Tenure_years', 'Salary', 'SalaryPerYear']]) # Step 7: Encode categorical variables print("\n" + "="*50) print("Step 7: Encode Categorical Variables") raw_data_encoded = pd.get_dummies(raw_data, columns=['Department'], drop_first=True) print(raw_data_encoded.columns.tolist()) # Step 8: Scale numerical features print("\n" + "="*50) print("Step 8: Scale Numerical Features") scaler = StandardScaler() numerical_cols = ['Age', 'Salary', 'Tenure_years', 'SalaryPerYear'] raw_data_encoded[numerical_cols] = scaler.fit_transform(raw_data_encoded[numerical_cols]) print("\nFinal processed data:") print(raw_data_encoded.head()) # Step 9: Save cleaned data print("\n" + "="*50) print("Step 9: Save Cleaned Data") raw_data_encoded.to_csv('cleaned_data.csv', index=False) print("Cleaned data saved to 'cleaned_data.csv'")
By completing Week 4, you have learned:
Work with a dataset containing missing values:
Prepare features for machine learning:
Build an end-to-end pipeline: