Quick Tip: Use Pandas to Clean Data in Under 45 Seconds

🐼 Master Data Cleaning with Pandas

Data cleaning is 80% of data science work. Learn the most powerful Pandas techniques to clean, transform, and prepare your data for analysis.

📊 Why Data Cleaning Matters

Real-world data is messy. Missing values, duplicates, inconsistent formats - all poison your analysis. Pandas provides powerful tools to tackle these challenges efficiently.

🔧 Top 10 Pandas Cleaning Techniques

1. Handle Missing Values

import pandas as pd

# Check for missing values
df.isnull().sum()

# Drop rows with any missing values
df_clean = df.dropna()

# Fill missing values with mean
df['age'].fillna(df['age'].mean(), inplace=True)

# Forward fill for time series
df['price'].fillna(method='ffill', inplace=True)

# Replace specific values
df.replace({'NA': None, '': None}, inplace=True)

2. Remove Duplicates

# Find duplicates
duplicates = df.duplicated()

# Remove duplicate rows
df_unique = df.drop_duplicates()

# Keep first occurrence
df.drop_duplicates(keep='first', inplace=True)

# Remove duplicates based on specific columns
df.drop_duplicates(subset=['email', 'phone'], inplace=True)

3. Fix Data Types

# Convert to numeric (coerce errors to NaN)
df['price'] = pd.to_numeric(df['price'], errors='coerce')

# Convert to datetime
df['date'] = pd.to_datetime(df['date'], format='%Y-%m-%d')

# Convert to categorical for memory efficiency
df['category'] = df['category'].astype('category')

# Fix mixed types
df['id'] = df['id'].astype(str)

4. Clean String Data

# Strip whitespace
df['name'] = df['name'].str.strip()

# Convert to lowercase
df['email'] = df['email'].str.lower()

# Remove special characters
df['phone'] = df['phone'].str.replace(r'[^0-9]', '', regex=True)

# Extract patterns using regex
df['area_code'] = df['phone'].str.extract(r'(\d{3})')

5. Handle Outliers

# Z-score method
from scipy import stats
z_scores = np.abs(stats.zscore(df['price']))
df_no_outliers = df[z_scores < 3]

# IQR method
Q1 = df['price'].quantile(0.25)
Q3 = df['price'].quantile(0.75)
IQR = Q3 - Q1
df_filtered = df[(df['price'] >= Q1 - 1.5*IQR) & 
                 (df['price'] <= Q3 + 1.5*IQR)]

6. Standardize Formats

# Standardize date formats
df['date'] = pd.to_datetime(df['date']).dt.strftime('%Y-%m-%d')

# Standardize names (Title Case)
df['name'] = df['name'].str.title()

# Standardize phone numbers
df['phone'] = df['phone'].str.replace(r'(\d{3})(\d{3})(\d{4})', 
                                      r'(\1) \2-\3', regex=True)

7. Merge and Join Data

# Merge dataframes
df_merged = pd.merge(df1, df2, on='id', how='left')

# Concatenate vertically
df_combined = pd.concat([df1, df2], ignore_index=True)

# Join on index
df_joined = df1.join(df2, on='key')

8. Handle Encoding Issues

# Read with correct encoding
df = pd.read_csv('data.csv', encoding='utf-8')

# Fix encoding issues
df['text'] = df['text'].str.encode('ascii', 'ignore').str.decode('ascii')

# Handle special characters
df['name'] = df['name'].str.normalize('NFKD')

9. Reshape Data

# Pivot table
df_pivot = df.pivot_table(values='sales', 
                          index='date', 
                          columns='product')

# Melt for tidy data
df_melted = pd.melt(df, id_vars=['id'], 
                    value_vars=['Q1', 'Q2', 'Q3'])

# Stack and unstack
df_stacked = df.stack()
df_unstacked = df_stacked.unstack()

10. Create Data Quality Report

# Quick data quality check
def data_quality_report(df):
    report = {
        'Total Rows': len(df),
        'Total Columns': len(df.columns),
        'Missing Values': df.isnull().sum().sum(),
        'Duplicates': df.duplicated().sum(),
        'Memory Usage': f"{df.memory_usage().sum() / 1024**2:.2f} MB"
    }
    return pd.DataFrame([report]).T

print(data_quality_report(df))

🚀 Putting It All Together

Here's a complete data cleaning pipeline:

def clean_data(df):
    # Make a copy
    df_clean = df.copy()
    
    # Remove duplicates
    df_clean = df_clean.drop_duplicates()
    
    # Fix data types
    df_clean['date'] = pd.to_datetime(df_clean['date'])
    df_clean['price'] = pd.to_numeric(df_clean['price'], errors='coerce')
    
    # Handle missing values
    df_clean['price'].fillna(df_clean['price'].median(), inplace=True)
    
    # Clean strings
    df_clean['name'] = df_clean['name'].str.strip().str.title()
    
    # Remove outliers
    Q1 = df_clean['price'].quantile(0.25)
    Q3 = df_clean['price'].quantile(0.75)
    IQR = Q3 - Q1
    df_clean = df_clean[(df_clean['price'] >= Q1 - 1.5*IQR) & 
                       (df_clean['price'] <= Q3 + 1.5*IQR)]
    
    return df_clean

# Use the pipeline
df_cleaned = clean_data(df)