MCAR (Missing Completely At Random):
  The probability of missing is unrelated to observed or unobserved values.
  Example: sensor failure randomly drops readings.
  Safe to impute without bias.

MAR (Missing At Random):
  The probability of missing depends on observed data, not the missing value itself.
  Example: older patients less likely to record weight (depends on age, not weight).
  Can be imputed using other features.

MNAR (Missing Not At Random):
  The probability of missing depends on the value itself.
  Example: high-income earners skip the income field.
  Hardest to handle — may require domain knowledge or additional data.

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns

df = pd.read_csv('data.csv')

# Summary of missingness
missing_summary = pd.DataFrame({
    'count': df.isnull().sum(),
    'pct': df.isnull().mean() * 100
}).sort_values('pct', ascending=False)
print(missing_summary[missing_summary['count'] > 0])

# Missingness heatmap
plt.figure(figsize=(12, 6))
sns.heatmap(df.isnull(), yticklabels=False, cbar=False, cmap='viridis')
plt.title('Missing Data Heatmap')

# Are missing values correlated? (suggests MAR)
missing_corr = df.isnull().corr()
sns.heatmap(missing_corr, annot=True, cmap='coolwarm')
plt.title('Missing Value Correlation')

from sklearn.impute import SimpleImputer

# Numeric features
num_imputer = SimpleImputer(strategy='median')    # Robust to outliers
# Alternative strategies: 'mean', 'constant', 'most_frequent'

X_num_imputed = num_imputer.fit_transform(X_numeric_train)
X_num_test = num_imputer.transform(X_numeric_test)

# Categorical features
cat_imputer = SimpleImputer(strategy='most_frequent')  # Most common value
# Or use constant: SimpleImputer(strategy='constant', fill_value='Unknown')

X_cat_imputed = cat_imputer.fit_transform(X_categorical_train)

from sklearn.impute import KNNImputer

knn_imputer = KNNImputer(
    n_neighbors=5,
    weights='uniform',   # Or 'distance' — closer neighbors get more weight
    metric='nan_euclidean'  # Handles NaN in distance computation
)

X_imputed = knn_imputer.fit_transform(X_train)

from sklearn.experimental import enable_iterative_imputer
from sklearn.impute import IterativeImputer
from sklearn.ensemble import RandomForestRegressor

# Default: Bayesian Ridge for each column
mice_imputer = IterativeImputer(max_iter=10, random_state=42)

# Custom estimator for each column
mice_rf = IterativeImputer(
    estimator=RandomForestRegressor(n_estimators=10, random_state=42),
    max_iter=10, random_state=42
)

X_imputed = mice_rf.fit_transform(X_train)

from sklearn.impute import MissingIndicator

indicator = MissingIndicator(features='missing-only')  # Only add flags for columns with NaN
missing_flags = indicator.fit_transform(X_train)

import numpy as np
X_with_flags = np.hstack([X_imputed, missing_flags])

# Or use add_indicator parameter in SimpleImputer
imputer_with_flags = SimpleImputer(strategy='median', add_indicator=True)
X_imputed_with_flags = imputer_with_flags.fit_transform(X_train)

# Drop columns with too much missing data
threshold = 0.50  # Drop if >50% missing
cols_to_drop = [col for col in df.columns if df[col].isnull().mean() > threshold]
df_cleaned = df.drop(columns=cols_to_drop)

# Drop rows where target is missing (never impute target)
df_cleaned = df_cleaned.dropna(subset=['target'])

# Keep but impute features with moderate missingness (<50%)
# The missing indicator captures the signal from the missingness pattern

from sklearn.pipeline import Pipeline
from sklearn.model_selection import cross_val_score
from sklearn.ensemble import GradientBoostingClassifier

pipeline = Pipeline([
    ('imputer', SimpleImputer(strategy='median')),
    ('model', GradientBoostingClassifier())
])

# cross_val_score correctly re-fits imputer on each training fold
scores = cross_val_score(pipeline, X, y, cv=5, scoring='roc_auc')