了解数据集转换

import numpy as np
import pandas as pd
from sklearn.model_selection import train_test_split
from sklearn.impute import SimpleImputer
from sklearn.preprocessing import OneHotEncoder, StandardScaler
from sklearn.compose import ColumnTransformer
from sklearn.pipeline import Pipeline
from imblearn.over_sampling import SMOTENC


df = pd.read_csv('my_dataset.csv')

# Remove duplicate rows from the dataset
df.drop_duplicates(inplace=True)

# Separate the features (X) and the target variable (y)
X = df.drop('target', axis=1)
y = df['target']

# Split the data into training and test sets
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=42, stratify=y)

# Check class distribution
print(f"Original training set class distribution: {np.bincount(y_train)}")

# Identify numeric and categorical features
numeric_features = [col for col in X_train.columns if 'num_feature' in col]
categorical_features = [col for col in X_train.columns if 'cat_feature' in col]

# Preprocessing pipeline for numeric missing values and feature scaling
numeric_transformer = Pipeline(steps=[
    ('imputer', SimpleImputer(strategy='mean')),
    ('scaler', StandardScaler())
])
# Preprocessing pipeline for categorical missing values and encoding
categorical_transformer = Pipeline(steps=[
    ('imputer', SimpleImputer(strategy='most_frequent')),
    ('onehot', OneHotEncoder(handle_unknown='ignore'))
])
# Preprocessing
preprocessor = ColumnTransformer(
    transformers=[
        ('num', numeric_transformer, numeric_features),
        ('cat', categorical_transformer, categorical_features)
    ])
X_train_processed = preprocessor.fit_transform(X_train)
X_test_processed = preprocessor.transform(X_test)

# Class balancing
categorical_indices = [X_train.columns.get_loc(col) for col in categorical_features]
smote_nc = SMOTENC(categorical_features=categorical_indices, random_state=42)
X_train_resampled, y_train_resampled = smote_nc.fit_resample(X_train_processed, y_train)

print(f"Resampled training set class distribution: {np.bincount(y_train_resampled)}")