Handling Missing Values

import pandas as pd
import numpy as np

# Sample dataset with missing values
df = pd.DataFrame({
    'id': [1, 2, 3, 4, 5],
    'name': ['Alice', 'Bob', np.nan, 'David', 'Eve'],
    'age': [25, np.nan, 30, 22, np.nan],
    'salary': [50000, 60000, 75000, np.nan, 45000]
})

print("=== RAW DATA ===")
print(df)

# Check for missing values (returns boolean dataframe)
print("\n=== ISNULL() MATRIX ===")
print(df.isnull())

# Count missing values per column
print("\n=== MISSING COUNTS PER COLUMN ===")
print(df.isnull().sum())

# Calculate percentage of missing data
print("\n=== MISSING PERCENTAGE ===")
print((df.isnull().mean() * 100).round(2), "%")

# Drop ANY row that has at least one NaN
df_drop_any = df.dropna()
# Warning: If you have 50 columns, almost every row might have at least one NaN!

# Drop rows only if ALL columns are NaN
df_drop_all = df.dropna(how='all')

# Drop rows if they are missing data in specific crucial columns
df_drop_subset = df.dropna(subset=['id', 'name'])

# Drop columns that have too many missing values (e.g., > 50% missing)
threshold = len(df) * 0.5
df_drop_cols = df.dropna(axis=1, thresh=threshold)

# 1. Fill with a constant value
df['name_filled'] = df['name'].fillna('Unknown')
df['salary_zero'] = df['salary'].fillna(0)

# 2. Fill with Statistical Measures (Mean/Median)
# Mean is sensitive to outliers. Median is usually safer for skewed data like salary/age.
median_age = df['age'].median()
df['age_imputed'] = df['age'].fillna(median_age)

# 3. Forward Fill and Backward Fill (Great for Time Series)
ts_df = pd.DataFrame({'date': ['Jan', 'Feb', 'Mar', 'Apr'], 'stock_price': [100, np.nan, np.nan, 105]})
# Forward fill: propogates the last valid observation forward
ts_df['ffill'] = ts_df['stock_price'].ffill() 
# Backward fill: uses next valid observation to fill gap
ts_df['bfill'] = ts_df['stock_price'].bfill()

hr_df = pd.DataFrame({
    'dept': ['IT', 'IT', 'IT', 'HR', 'HR', 'HR'],
    'salary': [80000, np.nan, 85000, 50000, 52000, np.nan]
})

# Fill NaN salary with the mean salary of THAT SPECIFIC department
hr_df['salary_imputed'] = hr_df.groupby('dept')['salary'].transform(lambda x: x.fillna(x.mean()))

print("\n=== GROUPBY IMPUTATION ===")
print(hr_df)
# IT gets 82500, HR gets 51000

# Dirty dataset
customers = pd.DataFrame({
    'customer_id': [101, 102, 103, 104, 105],
    'signup_date': ['2023-01-01', '2023-01-02', np.nan, '2023-01-04', '2023-01-05'],
    'country': ['USA', 'UK', np.nan, 'USA', np.nan],
    'lifetime_value': [500, np.nan, 1200, 300, np.nan]
})

# 1. Date missing? Forward fill (assuming sequential signups)
customers['signup_date'] = customers['signup_date'].ffill()

# 2. Country missing? Fill categorical with Mode (most frequent) or 'Unknown'
mode_country = customers['country'].mode()[0]
customers['country'] = customers['country'].fillna(mode_country)

# 3. LTV missing? Fill numeric with Median
customers['lifetime_value'] = customers['lifetime_value'].fillna(customers['lifetime_value'].median())

print("\n=== REPAIRED CUSTOMERS ===")
print(customers)