INNER CODE UNIT · Python
clean_data
balapriyac/data-science-tutorials · build-with-python/data-cleaning-n-validation-pipeline/main.py:31
def clean_data(self, df: pd.DataFrame) -> pd.DataFrame:
"""Clean the dataset by handling duplicates and missing values"""
initial_rows = len(df)
# Remove duplicates
df = df.drop_duplicates()
self.cleaning_stats['duplicates_removed'] = initial_rows - len(df)
# Handle missing values
numeric_columns = df.select_dtypes(include=[np.number]).columns
df[numeric_columns] = df[numeric_columns].fillna(df[numeric_columns].median())
string_columns = df.select_dtypes(include=['object']).columns
df[string_columns] = df[string_columns].fillna('Unknown')
self.cleaning_stats['nulls_handled'] = df.isnull().sum().sum()
return df