تنميط مجموعة البيانات

import pandas as pd

df = pd.read_csv("students-performance.csv")

# 1. Shape — how much data do we have?
print(f"Rows: {df.shape[0]}, Columns: {df.shape[1]}")

# 2. Column names and types
print(df.dtypes)

# 3. First and last rows
df.head(3)
df.tail(3)

# 4. Basic statistics for numerical columns
df.describe()

# 5. Categorical value counts
for col in df.select_dtypes(include="object").columns:
    print(f"\n{col}:")
    print(df[col].value_counts())

# Count and percentage of missing values per column
missing = df.isnull().sum()
missing_pct = (missing / len(df) * 100).round(2)
missing_report = pd.DataFrame({
    "missing_count": missing,
    "missing_pct": missing_pct
})
print(missing_report[missing_report["missing_count"] > 0])

# Visualize missing data with a heatmap
import seaborn as sns
import matplotlib.pyplot as plt

plt.figure(figsize=(10, 6))
sns.heatmap(df.isnull(), cbar=True, yticklabels=False, cmap="viridis")
plt.title("Missing Data Pattern")
plt.tight_layout()
plt.show()

# Exact duplicates
n_dupes = df.duplicated().sum()
print(f"Exact duplicate rows: {n_dupes}")

# Near-duplicates on key columns
key_cols = ["gender", "race/ethnicity", "parental level of education"]
n_near = df.duplicated(subset=key_cols).sum()
print(f"Near-duplicates on demographic columns: {n_near}")

# Find constant columns
constant_cols = [col for col in df.columns if df[col].nunique() == 1]
print(f"Constant columns: {constant_cols}")

# Find near-constant columns (>95% same value)
for col in df.columns:
    top_pct = df[col].value_counts(normalize=True).iloc[0]
    if top_pct > 0.95:
        print(f"  Near-constant: {col} — {top_pct:.1%} same value")

# Cardinality for each categorical column
cat_cols = df.select_dtypes(include="object").columns
for col in cat_cols:
    n_unique = df[col].nunique()
    print(f"{col}: {n_unique} unique values")
    if n_unique > 10:
        print(f"  WARNING: High cardinality — consider grouping")

def profile_dataset(df, name="Dataset"):
    """Generate a complete profiling report for a DataFrame."""
    print(f"{'='*60}")
    print(f"  PROFILING REPORT: {name}")
    print(f"{'='*60}")

    # Structure
    print(f"\nSTRUCTURE")
    print(f"  Rows: {df.shape[0]:,}")
    print(f"  Columns: {df.shape[1]}")
    print(f"  Memory usage: {df.memory_usage(deep=True).sum() / 1e6:.2f} MB")

    # Types
    print(f"\nCOLUMN TYPES")
    print(df.dtypes.value_counts().to_string())

    # Missing
    missing = df.isnull().sum()
    if missing.any():
        print(f"\nMISSING VALUES")
        for col in missing[missing > 0].index:
            pct = missing[col] / len(df) * 100
            print(f"  {col}: {missing[col]:,} ({pct:.1f}%)")
    else:
        print(f"\nMISSING VALUES: None detected")

    # Duplicates
    n_dupes = df.duplicated().sum()
    print(f"\nDUPLICATES: {n_dupes} rows ({n_dupes/len(df)*100:.1f}%)")

    # Numerical summary
    num_df = df.select_dtypes(include="number")
    if not num_df.empty:
        print(f"\nNUMERICAL SUMMARY")
        print(num_df.describe().round(2).to_string())

    # Categorical summary
    cat_df = df.select_dtypes(include="object")
    if not cat_df.empty:
        print(f"\nCATEGORICAL SUMMARY")
        for col in cat_df.columns:
            n_unique = df[col].nunique()
            print(f"  {col}: {n_unique} unique values")

    print(f"\n{'='*60}")

# Usage:
# profile_dataset(df, "Students Performance")

import pandas as pd

# students-performance.csv ships with the course — load it from the browser file system.
df = pd.read_csv("students-performance.csv")

# Quick profile
print("Shape:", df.shape)
print("\nDtypes:\n", df.dtypes)
print("\nMissing:\n", df.isnull().sum())
print("\nNumerical stats:\n", df.describe())
print("\nCategorical values:")
for col in df.select_dtypes(include="object").columns:
    print(f"\n{col}:")
    print(df[col].value_counts())