INNER CODE UNIT · Python
__init__
balapriyac/data-science-tutorials · build-with-python/data-cleaning-n-validation-pipeline/main.py:28
def __init__(self):
self.cleaning_stats = {'duplicates_removed': 0, 'nulls_handled': 0, 'validation_errors': 0}
def clean_data(self, df: pd.DataFrame) -> pd.DataFrame:
"""Clean the dataset by handling duplicates and missing values"""
initial_rows = len(df)
# Remove duplicates
df = df.drop_duplicates()
self.cleaning_stats['duplicates_removed'] = initial_rows - len(df)
# Handle missing values
numeric_columns = df.select_dtypes(include=[np.number]).columns
df[numeric_columns] = df[numeric_columns].fillna(df[numeric_columns].median())
string_columns = df.select_dtypes(include=['object']).columns
df[string_columns] = df[string_columns].fillna('Unknown')