Week 3: Data Manipulation with Pandas

import numpy as np
print(np.__version__)

import numpy as np

# Create array from Python list
arr1 = np.array([1, 2, 3, 4, 5])
print(arr1) # Output: [1 2 3 4 5]
print(type(arr1)) # Output: <class 'numpy.ndarray'>

# Create 2D array (matrix)
arr2d = np.array([[1, 2, 3],
[4, 5, 6],
[7, 8, 9]])
print(arr2d)
# Output:
# [[1 2 3]
# [4 5 6]
# [7 8 9]]

# Create arrays with specific values
zeros = np.zeros(5) # Array of 5 zeros
print(zeros) # Output: [0. 0. 0. 0. 0.]

ones = np.ones((3, 4)) # 3x4 array of ones
print(ones)
# Output:
# [[1. 1. 1. 1.]
# [1. 1. 1. 1.]
# [1. 1. 1. 1.]]

# Array filled with specific value
full = np.full((2, 3), 7) # 2x3 array filled with 7
print(full)
# Output:
# [[7 7 7]
# [7 7 7]]

# Identity matrix (1s on diagonal, 0s elsewhere)
identity = np.eye(3)
print(identity)
# Output:
# [[1. 0. 0.]
# [0. 1. 0.]
# [0. 0. 1.]]

# Create range of values
range_arr = np.arange(0, 10, 2) # start, stop, step
print(range_arr) # Output: [0 2 4 6 8]

# Evenly spaced values
linspace = np.linspace(0, 10, 5) # 5 values from 0 to 10
print(linspace) # Output: [ 0. 2.5 5. 7.5 10. ]

# Random values
random_arr = np.random.rand(3, 3) # 3x3 random array [0, 1)
print(random_arr)

# Random integers
random_int = np.random.randint(1, 10, size=(2, 3)) # Random ints 1-9
print(random_int)

arr = np.array([[1, 2, 3],
[4, 5, 6]])

# Shape: dimensions of array
print(arr.shape) # Output: (2, 3) - 2 rows, 3 columns

# Size: total number of elements
print(arr.size) # Output: 6

# Dtype: data type of elements
print(arr.dtype) # Output: int64

# Ndim: number of dimensions
print(arr.ndim) # Output: 2

# Real-world: Check data dimensions
image = np.random.rand(1920, 1080, 3) # RGB image
print(f"Image shape: {image.shape}") # Output: Image shape: (1920, 1080, 3)
print(f"Total pixels: {image.size}") # Output: Total pixels: 6220800

arr = np.array([10, 20, 30, 40, 50])

# Access single element
print(arr[0]) # Output: 10
print(arr[2]) # Output: 30
print(arr[-1]) # Output: 50

# Slicing
print(arr[1:4]) # Output: [20 30 40]
print(arr[:3]) # Output: [10 20 30]
print(arr[2:]) # Output: [30 40 50]
print(arr[::2]) # Every 2nd element: [10 30 50]

# 2D array indexing
arr2d = np.array([[1, 2, 3],
[4, 5, 6],
[7, 8, 9]])

# Access single element
print(arr2d[0, 0]) # Output: 1
print(arr2d[1, 2]) # Output: 6
print(arr2d[-1, -1]) # Output: 9

# Access entire row
print(arr2d[0]) # Output: [1 2 3]
print(arr2d[1]) # Output: [4 5 6]

# Access entire column
print(arr2d[:, 0]) # Output: [1 4 7]
print(arr2d[:, 2]) # Output: [3 6 9]

# 2D slicing
print(arr2d[0:2, 1:3]) # First 2 rows, columns 1-2
# Output:
# [[2 3]
# [5 6]]

# Modify elements
arr2d[0, 0] = 100
print(arr2d)
# Output:
# [[100 2 3]
# [ 4 5 6]
# [ 7 8 9]]

# Modifying slices affects original array!
arr = np.array([1, 2, 3, 4, 5])
arr[1:3] = [20, 30]
print(arr) # Output: [ 1 20 30 4 5]

arr = np.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10])

# Create boolean mask
mask = arr > 5
print(mask) # Output: [False False False False False True True True True True]

# Use mask to filter
filtered = arr[mask]
print(filtered) # Output: [ 6 7 8 9 10]

# Combine conditions
mask2 = (arr > 3) & (arr < 8)
print(arr[mask2]) # Output: [4 5 6 7]

# Using conditions directly
print(arr[arr > 5]) # Output: [ 6 7 8 9 10]
print(arr[arr % 2 == 0]) # Even numbers: [ 2 4 6 8 10]

# Real-world: Filter data
scores = np.array([45, 52, 78, 95, 88, 62, 91, 55])
passing = scores[scores >= 60]
print(f"Passing scores: {passing}")
# Output: Passing scores: [78 95 88 62 91]

# Count elements meeting condition
count = np.sum(scores >= 70)
print(f"Students with score >= 70: {count}") # Output: Students with score >= 70: 4

import numpy as np

# Element-wise operations
a = np.array([1, 2, 3, 4, 5])
b = np.array([10, 20, 30, 40, 50])

print(a + b) # Output: [11 22 33 44 55]
print(a - b) # Output: [-9 -18 -27 -36 -45]
print(a * b) # Output: [10 40 90 160 250]
print(b / a) # Output: [10. 10. 10. 10. 10.]
print(a ** 2) # Output: [ 1 4 9 16 25]

# Operations with scalars
arr = np.array([1, 2, 3, 4, 5])
print(arr + 10) # Output: [11 12 13 14 15]
print(arr * 2) # Output: [ 2 4 6 8 10]
print(arr / 2) # Output: [0.5 1. 1.5 2. 2.5]

# 2D operations
arr2d = np.array([[1, 2, 3],
[4, 5, 6]])

print(arr2d * 2)
# Output:
# [[ 2 4 6]
# [ 8 10 12]]

# Comparison operations
scores = np.array([45, 78, 92, 55, 88])
print(scores > 70) # Output: [False True True False True]
print(scores == 88) # Output: [False False False False True]
print(scores != 70) # Output: [ True True True True True]

# Broadcasting with scalar
arr = np.array([1, 2, 3, 4])
scalar = 5
print(arr + scalar) # Output: [6 7 8 9]

# Broadcasting 1D with 2D
arr2d = np.array([[1, 2, 3],
[4, 5, 6]])
arr1d = np.array([10, 20, 30])

result = arr2d + arr1d
print(result)
# Output:
# [[11 22 33]
# [14 25 36]]

# Broadcasting visualization:
# arr2d: arr1d: result:
# [[1 2 3] [10 20 30] [[11 22 33]
# [4 5 6]] + (broadcast) [14 25 36]]

# Column broadcasting
col = np.array([[10], [20]]) # Shape (2, 1)
arr2d = np.array([[1, 2, 3],
[4, 5, 6]]) # Shape (2, 3)

result = arr2d + col
print(result)
# Output:
# [[11 12 13]
# [24 25 26]]

# Real-world: Normalize data (subtract mean from each column)
data = np.array([[1, 2, 3],
[4, 5, 6],
[7, 8, 9]], dtype=float)

column_means = np.array([4., 5., 6.]) # Mean of each column
normalized = data - column_means
print(normalized)
# Output:
# [[-3. -3. -3.]
# [ 0. 0. 0.]
# [ 3. 3. 3.]]

import numpy as np

arr = np.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10])

# Basic statistics
print(np.sum(arr)) # Output: 55
print(np.mean(arr)) # Output: 5.5
print(np.median(arr)) # Output: 5.5
print(np.std(arr)) # Standard deviation
print(np.var(arr)) # Variance

# Min and Max
print(np.min(arr)) # Output: 1
print(np.max(arr)) # Output: 10
print(np.argmin(arr)) # Index of minimum: 0
print(np.argmax(arr)) # Index of maximum: 9

# Mathematical functions
print(np.sqrt(arr)) # Square root
print(np.exp(arr)) # e^x
print(np.log(arr)) # Natural logarithm
print(np.abs(np.array([-1, -2, 3, -4]))) # Absolute value

# Rounding
decimals = np.array([1.234, 2.567, 3.891])
print(np.round(decimals, 1)) # Output: [1.2 2.6 3.9]
print(np.floor(decimals)) # Output: [1. 2. 3.]
print(np.ceil(decimals)) # Output: [2. 3. 4.]

# Operations along axis
arr2d = np.array([[1, 2, 3],
[4, 5, 6],
[7, 8, 9]])

print(np.sum(arr2d)) # Total sum: 45
print(np.sum(arr2d, axis=0)) # Sum of each column: [12 15 18]
print(np.sum(arr2d, axis=1)) # Sum of each row: [ 6 15 24]

print(np.mean(arr2d, axis=0)) # Mean of each column
# Output: [4. 5. 6.]

# Real-world: Calculate statistics on test scores
scores = np.array([45, 78, 92, 55, 88, 62, 91, 75])
print(f"Mean score: {np.mean(scores):.2f}")
print(f"Median score: {np.median(scores):.2f}")
print(f"Std dev: {np.std(scores):.2f}")
print(f"Best score: {np.max(scores)}")
print(f"Worst score: {np.min(scores)}")

arr = np.array([3, 1, 4, 1, 5, 9, 2, 6, 5])

# Sorting
sorted_arr = np.sort(arr)
print(sorted_arr) # Output: [1 1 2 3 4 5 5 6 9]

# Get indices that would sort the array
indices = np.argsort(arr)
print(indices) # Output: [1 3 6 0 2 4 8 7 5]

# Reverse sort
reverse_sorted = np.sort(arr)[::-1]
print(reverse_sorted) # Output: [9 6 5 5 4 3 2 1 1]

# Unique values
unique = np.unique(arr)
print(unique) # Output: [1 2 3 4 5 6 9]

# Count occurrences
unique_vals, counts = np.unique(arr, return_counts=True)
for val, count in zip(unique_vals, counts):
print(f"Value {val} appears {count} times")

# Real-world: Find most common score
scores = np.array([78, 92, 78, 55, 88, 78, 91, 75])
unique_scores, counts = np.unique(scores, return_counts=True)
most_common_idx = np.argmax(counts)
most_common_score = unique_scores[most_common_idx]
print(f"Most common score: {most_common_score} (appears {counts[most_common_idx]} times)")

arr = np.array([1, 2, 3, 4, 5, 6])

# Reshape to 2D
arr_2d = arr.reshape(2, 3)
print(arr_2d)
# Output:
# [[1 2 3]
# [4 5 6]]

# Reshape to 3D
arr_3d = arr.reshape(2, 3, 1)
print(arr_3d)

# Flatten: convert to 1D
flattened = arr_2d.flatten()
print(flattened) # Output: [1 2 3 4 5 6]

# Transpose (swap rows and columns)
transposed = arr_2d.T
print(transposed)
# Output:
# [[1 4]
# [2 5]
# [3 6]]

# Concatenate arrays
a = np.array([1, 2, 3])
b = np.array([4, 5, 6])
combined = np.concatenate([a, b])
print(combined) # Output: [1 2 3 4 5 6]

# Stack arrays vertically
arr1 = np.array([1, 2, 3])
arr2 = np.array([4, 5, 6])
stacked_v = np.vstack([arr1, arr2])
print(stacked_v)
# Output:
# [[1 2 3]
# [4 5 6]]

# Stack arrays horizontally
stacked_h = np.hstack([arr1, arr2])
print(stacked_h) # Output: [1 2 3 4 5 6]

# Split array
arr = np.array([1, 2, 3, 4, 5, 6])
split_result = np.split(arr, 3) # Split into 3 equal parts
print(split_result) # Output: [array([1, 2]), array([3, 4]), array([5, 6])]

import pandas as pd
import numpy as np

# Create Series from list
s1 = pd.Series([10, 20, 30, 40, 50])
print(s1)
# Output:
# 0 10
# 1 20
# 2 30
# 3 40
# 4 50
# dtype: int64

# Series with custom index
s2 = pd.Series([10, 20, 30, 40, 50],
index=['a', 'b', 'c', 'd', 'e'])
print(s2)
# Output:
# a 10
# b 20
# c 30
# d 40
# e 50
# dtype: int64

# Access elements by index label
print(s2['a']) # Output: 10
print(s2['c']) # Output: 30

# Create Series from dictionary
student_grades = {'Alice': 85, 'Bob': 92, 'Charlie': 78, 'Diana': 95}
s3 = pd.Series(student_grades)
print(s3)
# Output:
# Alice 85
# Bob 92
# Charlie 78
# Diana 95
# dtype: int64

# Series properties
print(s3.index) # Index(['Alice', 'Bob', 'Charlie', 'Diana'], dtype='object')
print(s3.values) # [85 92 78 95]
print(len(s3)) # Output: 4

# Series operations
print(s3 + 5) # Add 5 to all values
print(s3[s3 > 80]) # Values greater than 80
# Output:
# Alice 85
# Bob 92
# Diana 95
# dtype: int64

# Series methods
print(s3.mean()) # Output: 87.5
print(s3.min()) # Output: 78
print(s3.max()) # Output: 95
print(s3.sum()) # Output: 350

# Real-world: Track daily temperature
dates = pd.Series([20, 22, 18, 25, 23, 19, 21],
index=['Monday', 'Tuesday', 'Wednesday', 'Thursday',
'Friday', 'Saturday', 'Sunday'])
print(dates)
print(f"Average temperature: {dates.mean():.1f}")
print(f"Hottest day: {dates.idxmax()} ({dates.max()}°C)")

import pandas as pd

# Create DataFrame from dictionary of lists
data = {
'Name': ['Alice', 'Bob', 'Charlie', 'Diana'],
'Age': [25, 30, 28, 32],
'City': ['Lagos', 'Accra', 'Nairobi', 'Johannesburg'],
'Salary': [60000, 75000, 65000, 80000]
}

df = pd.DataFrame(data)
print(df)
# Output:
# Name Age City Salary
# 0 Alice 25 Lagos 60000
# 1 Bob 30 Accra 75000
# 2 Charlie 28 Nairobi 65000
# 3 Diana 32 Johannesburg 80000

# DataFrame properties
print(df.shape) # Output: (4, 4) - 4 rows, 4 columns
print(df.columns) # Index(['Name', 'Age', 'City', 'Salary'], dtype='object')
print(df.index) # RangeIndex(start=0, stop=4, step=1)
print(df.dtypes) # Data types of each column
print(df.info()) # Summary information

# View first/last rows
print(df.head()) # First 5 rows (default)
print(df.head(2)) # First 2 rows
print(df.tail()) # Last 5 rows

# Access columns
print(df['Name']) # Get 'Name' column as Series
print(df['Age']) # Get 'Age' column

# Access multiple columns
print(df[['Name', 'City']]) # Get multiple columns as DataFrame

# Access rows by index
print(df.loc[0]) # First row by label
print(df.loc[2]) # Row with index 2

# Access by position
print(df.iloc[0]) # First row by position
print(df.iloc[1, 2]) # Row 1, column 2 (Row 1, 'City')

# Basic statistics
print(df.describe()) # Summary statistics for numeric columns
print(df['Salary'].mean()) # Average salary
print(df['Age'].max()) # Maximum age

import pandas as pd

df = pd.DataFrame({
'Name': ['Alice', 'Bob', 'Charlie'],
'Age': [25, 30, 28],
'Score': [85, 92, 78]
})

# Add new column
df['Grade'] = ['B', 'A', 'C']
print(df)

# Calculate new column from existing
df['Age_in_months'] = df['Age'] * 12
print(df)

# Modify existing column
df['Score'] = df['Score'] + 5 # Add 5 to all scores
print(df)

# Drop column
df = df.drop('Age_in_months', axis=1) # axis=1 for column
print(df)

# Drop row
df = df.drop(1, axis=0) # axis=0 for row
print(df)

# Rename columns
df = df.rename(columns={'Score': 'Test_Score', 'Grade': 'Letter_Grade'})
print(df)

# Sort by column
df = df.sort_values('Test_Score', ascending=False)
print(df)

# Filter rows
high_scores = df[df['Test_Score'] > 85]
print(high_scores)

import pandas as pd

# Read CSV file
df = pd.read_csv('data.csv')

# Read with options
df = pd.read_csv('data.csv',
sep=',', # Delimiter
encoding='utf-8', # File encoding
nrows=100) # Read only first 100 rows

# Read Excel file
df = pd.read_excel('data.xlsx', sheet_name=0) # First sheet

# Read from multiple sheets
xls = pd.ExcelFile('data.xlsx')
print(xls.sheet_names) # List all sheets
df_sheet1 = pd.read_excel('data.xlsx', sheet_name='Sheet1')
df_sheet2 = pd.read_excel('data.xlsx', sheet_name=1)

# Read from other formats
df = pd.read_json('data.json')
df = pd.read_html('webpage.html') # Read tables from HTML

# Example: Create and save CSV
df = pd.DataFrame({
'Product': ['Apple', 'Banana', 'Orange'],
'Price': [0.50, 0.30, 0.60],
'Quantity': [100, 150, 80]
})

# Save to CSV
df.to_csv('products.csv', index=False)

# Save to Excel
df.to_excel('products.xlsx', index=False)

# Save to JSON
df.to_json('products.json')

# Real-world: Load and explore data
df = pd.read_csv('student_data.csv')
print(f"Dataset shape: {df.shape}")
print(f"Columns: {list(df.columns)}")
print(df.head(10))
print(f"\nData types:\n{df.dtypes}")
print(f"\nMissing values:\n{df.isnull().sum()}")
print(f"\nBasic statistics:\n{df.describe()}")

import pandas as pd
import numpy as np

# Create sample dataset
np.random.seed(42)
df = pd.DataFrame({
'Student': ['Alice', 'Bob', 'Charlie', 'Diana', 'Eve'],
'Math': [85, 92, 78, 88, 95],
'English': [90, 85, 88, 92, 89],
'Science': [88, 88, 92, 85, 93]
})

print("Dataset:")
print(df)
print()

# Calculate average score for each student
df['Average'] = df[['Math', 'English', 'Science']].mean(axis=1)
print(df)
print()

# Find best subject
print("Best scores in each subject:")
print(df[['Math', 'English', 'Science']].max())
print()

# Find students with average > 85
excellent = df[df['Average'] > 85]
print("Students with average > 85:")
print(excellent)
print()

# Summary statistics
print("Summary statistics:")
print(df[['Math', 'English', 'Science']].describe())
print()

# Top performer
top_student_idx = df['Average'].idxmax()
top_student = df.loc[top_student_idx, 'Student']
print(f"Top performer: {top_student}")