src/enhanced_preprocessing/run_pipeline.py

# ============================================
# RUN 
# ============================================

import pandas as pd
import os

from enhanced_preprocessing.main_pipeline import EnhancedDataPreprocessingPipeline
from src.enhanced_preprocessing.config.config import Config

if __name__ == "__main__":
    """
    Pipeline execution
    """
    
    # Configuration with reasonable parameters
    config = Config(
        data_path='temp_data.csv',
        results_dir='src/enhanced_preprocessing_results',
        target_column='raskhodvoda',
        start_year=1970,
        end_year=1990,
        max_lags=5,
        seasonal_period=365,
        rolling_windows=[7, 30, 90],
        expanding_windows=[30, 90],
        test_size=0.2,
        validation_size=0.1,
        scaling_method='robust',
        feature_selection_method='correlation',
        max_features=20,
        missing_threshold=0.3,
        outlier_method='iqr',
        enable_validation=True
    )

    # Run enhanced pipeline
    pipeline = EnhancedDataPreprocessingPipeline(config)
    processed_data = pipeline.run_full_pipeline(
        use_synthetic=False,
        save_intermediate=True,
        create_reports=True
    )

    if processed_data is not None:
        print("\n" + "="*80)
        print("ENHANCED PIPELINE SUCCESSFULLY COMPLETED!")
        print("="*80)
        print(f"Final data size: {processed_data.shape}")
        print(f"Columns: {list(processed_data.columns)}")
        
        # Get modeling data
        modeling_data = pipeline.get_final_data_for_modelling()
        
        if modeling_data:
            print(f"\nModeling data ready:")
            print(f"  X_train: {modeling_data['X_train'].shape}")
            print(f"  X_val: {modeling_data['X_val'].shape}")
            print(f"  X_test: {modeling_data['X_test'].shape}")
            print(f"  Features: {len(modeling_data['feature_names'])}")
        
        # Save final data using config paths
        # Create subdirectory if it doesn't exist
        output_dir = os.path.join(config.results_dir, 'processed_data')
        os.makedirs(output_dir, exist_ok=True)
        
        output_path = os.path.join(output_dir, 'enhanced_final_processed_data.csv')
        processed_data.to_csv(
            output_path,
            index=True if isinstance(processed_data.index, pd.DatetimeIndex) else False
        )
        print(f"\n✓ Final data saved to '{output_path}'")