Model Selection Strategies

In machine learning projects, choosing the right algorithm is key to success. Different algorithms are suitable for different types of problems. This chapter will help you build a systematic framework for model selection.

Basic Principles of Model Selection

1. Problem Type Driven Selection

import numpy as np
import pandas as pd
from sklearn.datasets import make_classification, make_regression, make_blobs
from sklearn.model_selection import train_test_split
import matplotlib.pyplot as plt

# Generate different types of datasets for demonstration
def create_sample_datasets():
    # Classification dataset
    X_clf, y_clf = make_classification(
        n_samples=1000, n_features=2, n_redundant=0,
        n_informative=2, n_clusters_per_class=1, random_state=42
    )

    # Regression dataset
    X_reg, y_reg = make_regression(
        n_samples=1000, n_features=1, noise=10, random_state=42
    )

    # Clustering dataset
    X_cluster, y_cluster = make_blobs(
        n_samples=300, centers=4, n_features=2,
        random_state=42, cluster_std=0.60
    )

    return (X_clf, y_clf), (X_reg, y_reg), (X_cluster, y_cluster)

# Create sample data
(X_clf, y_clf), (X_reg, y_reg), (X_cluster, y_cluster) = create_sample_datasets()

# Visualize different types of problems
fig, axes = plt.subplots(1, 3, figsize=(15, 4))

# Classification problem
axes[0].scatter(X_clf[:, 0], X_clf[:, 1], c=y_clf, cmap='viridis')
axes[0].set_title('Classification Problem')
axes[0].set_xlabel('Feature 1')
axes[0].set_ylabel('Feature 2')

# Regression problem
axes[1].scatter(X_reg, y_reg, alpha=0.6)
axes[1].set_title('Regression Problem')
axes[1].set_xlabel('Feature')
axes[1].set_ylabel('Target Value')

# Clustering problem
axes[2].scatter(X_cluster[:, 0], X_cluster[:, 1], c=y_cluster, cmap='viridis')
axes[2].set_title('Clustering Problem')
axes[2].set_xlabel('Feature 1')
axes[2].set_ylabel('Feature 2')

plt.tight_layout()
plt.show()

2. Data Feature Analysis

def analyze_dataset(X, y=None, dataset_name="Dataset"):
    """Analyze dataset features"""
    print(f"\n=== {dataset_name} Analysis ===")
    print(f"Number of samples: {X.shape[0]}")
    print(f"Number of features: {X.shape[1]}")

    if y is not None:
        if len(np.unique(y)) < 20:  # Likely a classification problem
            print(f"Number of classes: {len(np.unique(y))}")
            print(f"Class distribution: {np.bincount(y)}")
        else:  # Likely a regression problem
            print(f"Target value range: [{y.min():.2f}, {y.max():.2f}]")
            print(f"Target value standard deviation: {y.std():.2f}")

    # Feature statistics
    print(f"Feature mean range: [{X.mean(axis=0).min():.2f}, {X.mean(axis=0).max():.2f}]")
    print(f"Feature standard deviation range: [{X.std(axis=0).min():.2f}, {X.std(axis=0).max():.2f}]")

    # Missing value check
    if hasattr(X, 'isnull'):
        missing_count = X.isnull().sum().sum()
        print(f"Number of missing values: {missing_count}")

# Analyze sample datasets
analyze_dataset(X_clf, y_clf, "Classification Dataset")
analyze_dataset(X_reg, y_reg, "Regression Dataset")
analyze_dataset(X_cluster, dataset_name="Clustering Dataset")

Algorithm Selection Guide

Classification Algorithm Selection

from sklearn.linear_model import LogisticRegression
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier
from sklearn.svm import SVC
from sklearn.neighbors import KNeighborsClassifier
from sklearn.naive_bayes import GaussianNB
from sklearn.model_selection import cross_val_score
from sklearn.preprocessing import StandardScaler
from sklearn.pipeline import Pipeline

def compare_classification_algorithms(X, y):
    """Compare performance of different classification algorithms"""

    # Prepare data
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

    # Define algorithms
    algorithms = {
        'Logistic Regression': Pipeline([
            ('scaler', StandardScaler()),
            ('clf', LogisticRegression(random_state=42))
        ]),
        'Decision Tree': DecisionTreeClassifier(random_state=42),
        'Random Forest': RandomForestClassifier(n_estimators=100, random_state=42),
        'Gradient Boosting': GradientBoostingClassifier(random_state=42),
        'SVM': Pipeline([
            ('scaler', StandardScaler()),
            ('clf', SVC(random_state=42))
        ]),
        'K-Nearest Neighbors': Pipeline([
            ('scaler', StandardScaler()),
            ('clf', KNeighborsClassifier())
        ]),
        'Naive Bayes': GaussianNB()
    }

    # Compare performance
    results = {}
    for name, algorithm in algorithms.items():
        scores = cross_val_score(algorithm, X_train, y_train, cv=5, scoring='accuracy')
        results[name] = {
            'mean': scores.mean(),
            'std': scores.std(),
            'scores': scores
        }

    # Display results
    print("Classification Algorithm Performance Comparison:")
    print("-" * 50)
    for name, result in sorted(results.items(), key=lambda x: x[1]['mean'], reverse=True):
        print(f"{name:12s}: {result['mean']:.4f} (+/- {result['std']*2:.4f})")

    return results

# Run comparison
classification_results = compare_classification_algorithms(X_clf, y_clf)

Regression Algorithm Selection

from sklearn.linear_model import LinearRegression, Ridge, Lasso, ElasticNet
from sklearn.tree import DecisionTreeRegressor
from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor
from sklearn.svm import SVR
from sklearn.neighbors import KNeighborsRegressor

def compare_regression_algorithms(X, y):
    """Compare performance of different regression algorithms"""

    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

    algorithms = {
        'Linear Regression': LinearRegression(),
        'Ridge Regression': Ridge(alpha=1.0),
        'Lasso Regression': Lasso(alpha=1.0),
        'Elastic Net': ElasticNet(alpha=1.0),
        'Decision Tree': DecisionTreeRegressor(random_state=42),
        'Random Forest': RandomForestRegressor(n_estimators=100, random_state=42),
        'Gradient Boosting': GradientBoostingRegressor(random_state=42),
        'SVR': Pipeline([
            ('scaler', StandardScaler()),
            ('reg', SVR())
        ]),
        'K-Nearest Neighbors': Pipeline([
            ('scaler', StandardScaler()),
            ('reg', KNeighborsRegressor())
        ])
    }

    results = {}
    for name, algorithm in algorithms.items():
        scores = cross_val_score(algorithm, X_train, y_train, cv=5, scoring='r2')
        results[name] = {
            'mean': scores.mean(),
            'std': scores.std(),
            'scores': scores
        }

    print("Regression Algorithm Performance Comparison (R² Score):")
    print("-" * 50)
    for name, result in sorted(results.items(), key=lambda x: x[1]['mean'], reverse=True):
        print(f"{name:12s}: {result['mean']:.4f} (+/- {result['std']*2:.4f})")

    return results

# Run comparison
regression_results = compare_regression_algorithms(X_reg, y_reg)

Selection Strategies Based on Data Features

1. Dataset Size

def recommend_by_data_size(n_samples, n_features):
    """Recommend algorithms based on dataset size"""

    print(f"Dataset size: {n_samples} samples, {n_features} features")

    if n_samples < 1000:
        print("Small dataset recommendations:")
        print("- Naive Bayes (fast, suitable for small samples)")
        print("- K-Nearest Neighbors (simple, no training required)")
        print("- Linear models (avoid overfitting)")

    elif n_samples < 10000:
        print("Medium dataset recommendations:")
        print("- Random Forest (balances performance and interpretability)")
        print("- Gradient Boosting (usually performs well)")
        print("- SVM (suitable for medium-scale data)")

    else:
        print("Large dataset recommendations:")
        print("- Linear models (fast training)")
        print("- Random Forest (can be trained in parallel)")
        print("- Online learning algorithms")

    if n_features > n_samples:
        print("\nHigh-dimensional data (features > samples):")
        print("- Regularized linear models (Lasso, Ridge)")
        print("- Naive Bayes")
        print("- Consider dimensionality reduction techniques")

# Example recommendations
recommend_by_data_size(1000, 20)
recommend_by_data_size(100000, 50)
recommend_by_data_size(500, 1000)

2. Feature Type Analysis

def analyze_feature_types(X, feature_names=None):
    """Analyze feature types and recommend algorithms"""

    if feature_names is None:
        feature_names = [f"Feature_{i}" for i in range(X.shape[1])]

    # Detect distribution of numerical features
    numerical_features = []
    categorical_features = []

    for i, name in enumerate(feature_names):
        unique_values = len(np.unique(X[:, i]))
        if unique_values < 10:  # Likely a categorical feature
            categorical_features.append(name)
        else:
            numerical_features.append(name)

    print(f"Numerical features: {len(numerical_features)}")
    print(f"Categorical features: {len(categorical_features)}")

    # Recommend algorithms based on feature types
    if len(categorical_features) > len(numerical_features):
        print("\nMore categorical features, recommend:")
        print("- Naive Bayes")
        print("- Decision Tree")
        print("- Random Forest")
    else:
        print("\nMore numerical features, recommend:")
        print("- Linear models")
        print("- SVM")
        print("- K-Nearest Neighbors")

    return numerical_features, categorical_features

# Analyze sample data
numerical_features, categorical_features = analyze_feature_types(X_clf)

Model Complexity vs Performance Trade-off

from sklearn.metrics import accuracy_score, mean_squared_error
import time

def evaluate_complexity_performance(X, y, problem_type='classification'):
    """Evaluate the trade-off between model complexity and performance"""

    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

    if problem_type == 'classification':
        models = {
            'Naive Bayes': GaussianNB(),
            'Logistic Regression': LogisticRegression(random_state=42),
            'Decision Tree': DecisionTreeClassifier(random_state=42),
            'Random Forest': RandomForestClassifier(n_estimators=100, random_state=42),
            'SVM': SVC(random_state=42)
        }
        metric_func = accuracy_score
        metric_name = 'Accuracy'
    else:
        models = {
            'Linear Regression': LinearRegression(),
            'Ridge Regression': Ridge(),
            'Decision Tree': DecisionTreeRegressor(random_state=42),
            'Random Forest': RandomForestRegressor(n_estimators=100, random_state=42),
            'SVR': SVR()
        }
        metric_func = lambda y_true, y_pred: -mean_squared_error(y_true, y_pred)  # Negative MSE, higher is better
        metric_name = 'Negative MSE'

    results = []

    for name, model in models.items():
        # Training time
        start_time = time.time()
        model.fit(X_train, y_train)
        train_time = time.time() - start_time

        # Prediction time
        start_time = time.time()
        y_pred = model.predict(X_test)
        predict_time = time.time() - start_time

        # Performance metric
        performance = metric_func(y_test, y_pred)

        # Model complexity (approximate number of parameters)
        complexity = getattr(model, 'n_features_in_', X.shape[1])
        if hasattr(model, 'coef_'):
            complexity = np.prod(model.coef_.shape)
        elif hasattr(model, 'tree_'):
            complexity = model.tree_.node_count
        elif hasattr(model, 'estimators_'):
            complexity = len(model.estimators_) * 100  # Approximate value

        results.append({
            'model': name,
            'performance': performance,
            'train_time': train_time,
            'predict_time': predict_time,
            'complexity': complexity
        })

    # Display results
    results_df = pd.DataFrame(results)
    results_df = results_df.sort_values('performance', ascending=False)

    print(f"Model Performance and Complexity Comparison ({metric_name}):")
    print("-" * 80)
    print(f"{'Model':<12} {'Performance':<10} {'Train Time':<10} {'Predict Time':<10} {'Complexity':<10}")
    print("-" * 80)

    for _, row in results_df.iterrows():
        print(f"{row['model']:<12} {row['performance']:<10.4f} {row['train_time']:<10.4f} "
              f"{row['predict_time']:<10.4f} {row['complexity']:<10.0f}")

    return results_df

# Evaluate classification problem
print("=== Classification Problem Evaluation ===")
clf_results = evaluate_complexity_performance(X_clf, y_clf, 'classification')

print("\n=== Regression Problem Evaluation ===")
reg_results = evaluate_complexity_performance(X_reg, y_reg, 'regression')

Ensemble Learning Strategies

from sklearn.ensemble import VotingClassifier, VotingRegressor, StackingClassifier
from sklearn.model_selection import cross_val_score

def create_ensemble_models(X, y, problem_type='classification'):
    """Create ensemble models"""

    if problem_type == 'classification':
        # Base classifiers
        base_models = [
            ('lr', LogisticRegression(random_state=42)),
            ('rf', RandomForestClassifier(n_estimators=100, random_state=42)),
            ('svm', SVC(probability=True, random_state=42))
        ]

        # Voting ensemble
        voting_clf = VotingClassifier(
            estimators=base_models,
            voting='soft'  # Use probability voting
        )

        # Stacking ensemble
        stacking_clf = StackingClassifier(
            estimators=base_models,
            final_estimator=LogisticRegression(),
            cv=5
        )

        models = {
            'Voting Ensemble': voting_clf,
            'Stacking Ensemble': stacking_clf
        }

        # Add base models for comparison
        for name, model in base_models:
            models[f'Base Model_{name}'] = model

        scoring = 'accuracy'

    else:
        # Base regressors
        base_models = [
            ('lr', LinearRegression()),
            ('rf', RandomForestRegressor(n_estimators=100, random_state=42)),
            ('svr', SVR())
        ]

        # Voting ensemble
        voting_reg = VotingRegressor(estimators=base_models)

        models = {
            'Voting Ensemble': voting_reg
        }

        # Add base models for comparison
        for name, model in base_models:
            models[f'Base Model_{name}'] = model

        scoring = 'r2'

    # Evaluate all models
    results = {}
    for name, model in models.items():
        scores = cross_val_score(model, X, y, cv=5, scoring=scoring)
        results[name] = {
            'mean': scores.mean(),
            'std': scores.std()
        }

    print(f"Ensemble Learning Effect Comparison ({scoring}):")
    print("-" * 50)
    for name, result in sorted(results.items(), key=lambda x: x[1]['mean'], reverse=True):
        print(f"{name:<15}: {result['mean']:.4f} (+/- {result['std']*2:.4f})")

    return results

# Test ensemble learning
print("=== Classification Ensemble Learning ===")
clf_ensemble_results = create_ensemble_models(X_clf, y_clf, 'classification')

print("\n=== Regression Ensemble Learning ===")
reg_ensemble_results = create_ensemble_models(X_reg, y_reg, 'regression')

Model Selection Decision Tree

def model_selection_guide():
    """Model selection decision guide"""

    guide = """
    Model Selection Decision Tree:

    1. Problem Type?
       ├── Classification Problem
       │   ├── Samples < 1000? → Naive Bayes, K-Nearest Neighbors
       │   ├── Need probability output? → Logistic Regression, Random Forest
       │   ├── Need interpretability? → Decision Tree, Logistic Regression
       │   └── Pursue highest performance? → Random Forest, Gradient Boosting, Ensemble Methods
       │
       ├── Regression Problem
       │   ├── Linear relationship? → Linear Regression, Ridge Regression
       │   ├── Feature selection needed? → Lasso Regression
       │   ├── Non-linear relationship? → Random Forest, Gradient Boosting
       │   └── High-dimensional data? → Regularized linear models
       │
       └── Clustering Problem
           ├── Know number of clusters? → K-Means
           ├── Irregular shapes? → DBSCAN
           └── Hierarchical structure? → Hierarchical Clustering

    2. Data Features?
       ├── High-dimensional sparse? → Linear models, Naive Bayes
       ├── Mixed feature types? → Decision Tree, Random Forest
       ├── Many missing values? → Random Forest, Gradient Boosting
       └── High noise? → Ensemble methods

    3. Performance Requirements?
       ├── Training speed priority? → Naive Bayes, Linear models
       ├── Prediction speed priority? → Linear models, K-Nearest Neighbors
       ├── Memory constraints? → Linear models, Naive Bayes
       └── Highest accuracy? → Ensemble methods, Deep Learning

    4. Interpretability Requirements?
       ├── High interpretability? → Linear models, Decision Tree
       ├── Medium interpretability? → Random Forest (feature importance)
       └── No interpretability requirement? → SVM, Ensemble methods
    """

    print(guide)

# Display decision guide
model_selection_guide()

Automated Model Selection

from sklearn.model_selection import GridSearchCV
from sklearn.metrics import classification_report, regression

def auto_model_selection(X, y, problem_type='auto'):
    """Automated model selection and tuning"""

    # Automatically detect problem type
    if problem_type == 'auto':
        if len(np.unique(y)) < 20 and y.dtype in ['int64', 'int32', 'object']:
            problem_type = 'classification'
        else:
            problem_type = 'regression'

    print(f"Detected problem type: {problem_type}")

    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

    if problem_type == 'classification':
        # Classification algorithms and parameter grids
        models_params = {
            'RandomForest': {
                'model': RandomForestClassifier(random_state=42),
                'params': {
                    'n_estimators': [50, 100, 200],
                    'max_depth': [5, 10, None]
                }
            },
            'GradientBoosting': {
                'model': GradientBoostingClassifier(random_state=42),
                'params': {
                    'n_estimators': [50, 100],
                    'learning_rate': [0.1, 0.2]
                }
            },
            'SVM': {
                'model': Pipeline([
                    ('scaler', StandardScaler()),
                    ('svm', SVC(random_state=42))
                ]),
                'params': {
                    'svm__C': [0.1, 1, 10],
                    'svm__kernel': ['rbf', 'linear']
                }
            }
        }
        scoring = 'accuracy'

    else:
        # Regression algorithms and parameter grids
        models_params = {
            'RandomForest': {
                'model': RandomForestRegressor(random_state=42),
                'params': {
                    'n_estimators': [50, 100, 200],
                    'max_depth': [5, 10, None]
                }
            },
            'GradientBoosting': {
                'model': GradientBoostingRegressor(random_state=42),
                'params': {
                    'n_estimators': [50, 100],
                    'learning_rate': [0.1, 0.2]
                }
            },
            'Ridge': {
                'model': Ridge(),
                'params': {
                    'alpha': [0.1, 1.0, 10.0]
                }
            }
        }
        scoring = 'r2'

    # Automatically search for best model
    best_score = -np.inf
    best_model = None
    best_name = None

    results = {}

    for name, config in models_params.items():
        print(f"\nTesting {name}...")

        grid_search = GridSearchCV(
            config['model'],
            config['params'],
            cv=5,
            scoring=scoring,
            n_jobs=-1
        )

        grid_search.fit(X_train, y_train)

        if grid_search.best_score_ > best_score:
            best_score = grid_search.best_score_
            best_model = grid_search.best_estimator_
            best_name = name

        results[name] = {
            'best_score': grid_search.best_score_,
            'best_params': grid_search.best_params_,
            'model': grid_search.best_estimator_
        }

    # Display results
    print(f"\n=== Automated Model Selection Results ===")
    print(f"Best model: {best_name}")
    print(f"Best cross-validation score: {best_score:.4f}")
    print(f"Best parameters: {results[best_name]['best_params']}")

    # Evaluate on test set
    test_score = best_model.score(X_test, y_test)
    print(f"Test set score: {test_score:.4f}")

    return best_model, results

# Automatically select best model
best_clf_model, clf_auto_results = auto_model_selection(X_clf, y_clf)

Summary

Model selection is a systematic process that requires considering multiple factors:

1. Problem Type: Classification, regression, or clustering
2. Data Features: Sample count, feature count, data types
3. Performance Requirements: Accuracy, speed, memory usage
4. Interpretability Needs: Whether understanding the model's decision process is required
5. Resource Constraints: Computation time, storage space

Selection Recommendations:

• Start with simple models (linear models, Naive Bayes)
• Gradually try complex models (Random Forest, Gradient Boosting)
• Use cross-validation to evaluate performance
• Consider ensemble methods to improve performance
• Balance performance and complexity based on actual needs

In the next chapter, we will learn about Performance Metrics Details to gain a deeper understanding of how to evaluate and compare the performance of different models.