update pipeline.
Browse files- README.md +179 -161
- app.py +89 -32
- src/enhanced_preprocessing/__init__.py +0 -0
- main_pipeline.py → src/enhanced_preprocessing/main_pipeline.py +0 -0
- src/enhanced_preprocessing/run_pipeline.py +1 -1
- streamlit/preprocessor_app.py +1 -1
README.md
CHANGED
|
@@ -1,217 +1,225 @@
|
|
| 1 |
---
|
| 2 |
-
title:
|
| 3 |
-
emoji:
|
| 4 |
-
colorFrom:
|
| 5 |
-
colorTo:
|
| 6 |
sdk: docker
|
| 7 |
pinned: true
|
| 8 |
app_file: app.py
|
| 9 |
sdk_version: 1.52.2
|
| 10 |
---
|
| 11 |
|
| 12 |
-
#
|
| 13 |
|
| 14 |
<div align="center">
|
| 15 |
|
| 16 |
-
**
|
| 17 |
|
| 18 |
-
*
|
| 19 |
|
| 20 |
-
[](https://streamlit.io)
|
| 22 |
-
[
|
| 144 |
-
|
| 145 |
-
# Configure pipeline
|
| 146 |
-
config = {
|
| 147 |
-
'target_column': 'target',
|
| 148 |
-
'test_size': 0.2,
|
| 149 |
-
'max_lags': 7,
|
| 150 |
-
'seasonal_period': 30
|
| 151 |
-
}
|
| 152 |
-
|
| 153 |
-
# Create and run pipeline
|
| 154 |
-
pipeline = TimeFlowPipeline(config)
|
| 155 |
-
processed_data = pipeline.run(data)
|
| 156 |
|
| 157 |
-
#
|
| 158 |
-
|
| 159 |
-
X_train, y_train = modeling_data['X_train'], modeling_data['y_train']
|
| 160 |
```
|
| 161 |
|
| 162 |
-
|
|
|
|
|
|
|
| 163 |
|
| 164 |
-
|
| 165 |
-
|--------------|----------------|--------------|-------------------|
|
| 166 |
-
| 10K rows | ~5 seconds | <500 MB | 50-100 features |
|
| 167 |
-
| 100K rows | ~30 seconds | <1 GB | 100-200 features |
|
| 168 |
-
| 1M rows | ~5 minutes | <2 GB | 200-500 features |
|
| 169 |
|
| 170 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 171 |
|
| 172 |
-
##
|
|
|
|
|
|
|
| 173 |
- `missing_threshold`: Threshold for column removal (0.0-0.5)
|
| 174 |
- `outlier_method`: IQR, Z-Score, or Isolation Forest
|
| 175 |
- `scaling_method`: Robust, Standard, MinMax, or None
|
| 176 |
-
|
| 177 |
-
### **Feature Engineering**
|
| 178 |
- `max_lags`: Maximum lag features (1-20)
|
| 179 |
- `seasonal_period`: Seasonal window (7, 30, 90, 365)
|
| 180 |
-
- `rolling_windows`: List of rolling windows [7, 30, 90]
|
| 181 |
|
| 182 |
-
### **
|
|
|
|
|
|
|
|
|
|
|
|
|
| 183 |
- `feature_selection_method`: Correlation, Variance, RF, Mutual Info
|
| 184 |
-
- `max_features`: Maximum features to select (5-100)
|
| 185 |
-
- `split_method`: Time-based or random splitting
|
| 186 |
-
|
| 187 |
-
## 📋 Requirements
|
| 188 |
-
|
| 189 |
-
### **Core Dependencies**
|
| 190 |
-
```txt
|
| 191 |
-
streamlit>=1.28.0
|
| 192 |
-
pandas>=2.0.0
|
| 193 |
-
numpy>=1.24.0
|
| 194 |
-
plotly>=5.17.0
|
| 195 |
-
scikit-learn>=1.3.0
|
| 196 |
-
```
|
| 197 |
|
| 198 |
-
##
|
| 199 |
-
|
| 200 |
-
|
| 201 |
-
|
| 202 |
-
|
| 203 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 204 |
|
| 205 |
## 🤝 Contributing
|
| 206 |
|
| 207 |
We welcome contributions! Here's how you can help:
|
| 208 |
|
| 209 |
### **Areas for Contribution**
|
| 210 |
-
1. **New
|
| 211 |
-
2. **
|
| 212 |
-
3. **
|
| 213 |
-
4. **
|
| 214 |
-
5. **Documentation Improvements**
|
| 215 |
|
| 216 |
### **Development Workflow**
|
| 217 |
```bash
|
|
@@ -227,42 +235,52 @@ git checkout -b feature/new-feature
|
|
| 227 |
|
| 228 |
This project is licensed under the **MIT License** - see the [LICENSE](LICENSE) file for details.
|
| 229 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 230 |
## 🙏 Acknowledgments
|
| 231 |
|
| 232 |
-
### **
|
| 233 |
-
- **
|
| 234 |
-
- **
|
| 235 |
-
|
| 236 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 237 |
|
| 238 |
-
### **
|
| 239 |
-
-
|
| 240 |
-
-
|
| 241 |
-
-
|
| 242 |
-
- 🔧 Scikit-learn
|
| 243 |
-
- 📈 Pandas & NumPy
|
| 244 |
|
| 245 |
## 📞 Support & Contact
|
| 246 |
|
| 247 |
-
### **
|
| 248 |
- 📧 **Email**: cool.araby@gmail.com
|
| 249 |
-
- 💬 **Issues**: [GitHub Issues](https://github.com/
|
| 250 |
-
-
|
| 251 |
|
| 252 |
-
### **Stay
|
| 253 |
-
- ⭐ **Star** the repository
|
| 254 |
-
- 👁️ **Watch** for releases
|
| 255 |
-
- 🔔 **
|
| 256 |
|
| 257 |
---
|
| 258 |
|
| 259 |
<div align="center">
|
| 260 |
|
| 261 |
-
**
|
| 262 |
|
| 263 |
-
*
|
| 264 |
|
| 265 |
-
[](https://huggingface.co/spaces/arabovs-ai-lab/ml-pipeline-pro-dashboard)
|
| 21 |
[](https://streamlit.io)
|
| 22 |
+
[](https://python.org)
|
| 23 |
+
[](https://github.com/arabovs-ai-lab/ml-pipeline-pro-suite)
|
| 24 |
|
| 25 |
</div>
|
| 26 |
|
| 27 |
## 🌟 Overview
|
| 28 |
|
| 29 |
+
**ML Pipeline PRO Suite** is a complete machine learning platform that combines two powerful modules:
|
| 30 |
+
- **TimeFlow Pro** - advanced time series preprocessing and feature engineering
|
| 31 |
+
- **ModelForge Pro** - comprehensive ML training with 30+ algorithms
|
| 32 |
+
|
| 33 |
+
Designed for data scientists, researchers, and ML engineers, it provides an end-to-end workflow from raw data to trained models in a single interface.
|
| 34 |
|
| 35 |
## 🚀 Key Features
|
| 36 |
|
| 37 |
+
### 📊 **TimeFlow Pro - Data Processing Module**
|
| 38 |
- **Interactive Data Exploration**: Real-time preview and statistics
|
| 39 |
- **Missing Value Analysis**: Smart detection and handling strategies
|
| 40 |
- **Outlier Detection**: Multiple methods including IQR, Z-Score, Isolation Forest
|
|
|
|
|
|
|
|
|
|
| 41 |
- **Feature Engineering**: Automatic lag features, rolling statistics, seasonal components
|
| 42 |
- **Stationarity Checking**: ADF tests and transformation suggestions
|
| 43 |
- **Data Scaling**: Robust, Standard, MinMax, and custom scaling methods
|
| 44 |
- **Feature Selection**: Correlation, variance, mutual information, RF importance
|
| 45 |
|
| 46 |
+
### 🤖 **ModelForge Pro - Machine Learning Module**
|
| 47 |
+
- **30+ ML Algorithms**: Linear models (13), tree ensembles (7), gradient boosting (3), and advanced methods
|
| 48 |
+
- **Hyperparameter Tuning**: Grid Search and Random Search optimization
|
| 49 |
+
- **Model Ensembling**: Averaging, Voting, Weighted, and Stacking techniques
|
| 50 |
+
- **Performance Analysis**: Metrics comparison, feature importance analysis
|
| 51 |
+
- **Prediction Generation**: Make predictions on new data with ensemble options
|
| 52 |
+
- **Model Export**: Serialize models in Pickle, Joblib, and Dill formats
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 53 |
|
| 54 |
+
### 🎨 **Unified Interface & Visualization**
|
| 55 |
+
- **Streamlit Web Interface**: Intuitive browser-based workflow
|
| 56 |
+
- **Real-time Monitoring**: Track training progress and compare models
|
| 57 |
+
- **Interactive Charts**: Visualize metrics, residuals, and predictions
|
| 58 |
+
- **Automated Reports**: Generate comprehensive analysis reports
|
| 59 |
|
| 60 |
+
## 🔧 **Complete Pipeline Architecture**
|
| 61 |
|
|
|
|
| 62 |
```
|
| 63 |
+
Data Loading → Validation → Missing Handling → Outlier Treatment → Feature Engineering
|
| 64 |
+
↓
|
| 65 |
+
Stationarity Check → Correlation Analysis → Feature Selection → Data Splitting → Scaling
|
| 66 |
↓
|
| 67 |
+
Algorithm Selection → Model Training → Hyperparameter Tuning → Ensemble Building
|
| 68 |
↓
|
| 69 |
+
Performance Evaluation → Model Comparison → Prediction → Export
|
| 70 |
```
|
| 71 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 72 |
## 📚 Use Cases
|
| 73 |
|
| 74 |
+
### 🔬 **Scientific Research & Education**
|
| 75 |
+
- Time series analysis in hydrometeorology and climate studies
|
| 76 |
+
- Reproducible ML experiments and comparative algorithm studies
|
| 77 |
+
- Educational tool for data science and machine learning courses
|
| 78 |
+
- Feature engineering research and methodology development
|
| 79 |
+
|
| 80 |
+
### 🏢 **Business & Industry Applications**
|
| 81 |
+
- Sales forecasting and demand prediction in retail
|
| 82 |
+
- Predictive maintenance for industrial equipment
|
| 83 |
+
- Financial market analysis and risk assessment
|
| 84 |
+
- Energy consumption forecasting and optimization
|
| 85 |
+
- IoT sensor data analysis and anomaly detection
|
| 86 |
|
| 87 |
+
### 💼 **ML Development & Prototyping**
|
| 88 |
+
- Rapid prototyping of ML solutions
|
| 89 |
+
- Automated model selection and optimization
|
| 90 |
+
- Production-ready pipeline development
|
| 91 |
+
- Team collaboration on ML projects
|
| 92 |
|
| 93 |
+
## 🛠️ Installation & Quick Start
|
|
|
|
|
|
|
|
|
|
|
|
|
| 94 |
|
| 95 |
+
### **1. Download the Project**
|
| 96 |
+
```bash
|
| 97 |
+
# Clone via Git
|
| 98 |
+
git clone https://huggingface.co/spaces/arabovs-ai-lab/ml-pipeline-pro-dashboard
|
| 99 |
+
cd ml-pipeline-pro-dashboard
|
| 100 |
+
```
|
| 101 |
|
| 102 |
+
### **2. Python Installation**
|
| 103 |
```bash
|
| 104 |
+
# Create virtual environment
|
| 105 |
+
python -m venv .venv
|
| 106 |
+
|
| 107 |
+
# Activate (Windows)
|
| 108 |
+
.venv\Scripts\activate
|
| 109 |
+
# Or (Linux/Mac)
|
| 110 |
+
# source .venv/bin/activate
|
| 111 |
|
| 112 |
# Install dependencies
|
| 113 |
pip install -r requirements.txt
|
| 114 |
|
| 115 |
+
# Launch the platform
|
| 116 |
streamlit run app.py
|
| 117 |
```
|
| 118 |
|
| 119 |
+
### **3. Run Individual Modules**
|
| 120 |
```bash
|
| 121 |
+
# Launch only TimeFlow Pro (data preprocessing)
|
| 122 |
+
streamlit run streamlit\preprocessor_app.py
|
| 123 |
|
| 124 |
+
# Launch only ModelForge Pro (machine learning)
|
| 125 |
+
streamlit run streamlit\ml_app.py
|
| 126 |
```
|
| 127 |
|
| 128 |
+
### **4. Docker Deployment**
|
| 129 |
+
```bash
|
| 130 |
+
# Build Docker image
|
| 131 |
+
docker build -t ml-pipeline-pro-suite .
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 132 |
|
| 133 |
+
# Run container
|
| 134 |
+
docker run -p 7860:7860 ml-pipeline-pro-suite
|
|
|
|
| 135 |
```
|
| 136 |
|
| 137 |
+
**After launching**, open your browser at:
|
| 138 |
+
- http://localhost:8501 (Streamlit direct launch)
|
| 139 |
+
- http://localhost:7860 (Docker launch)
|
| 140 |
|
| 141 |
+
## 📊 Performance Benchmarks
|
|
|
|
|
|
|
|
|
|
|
|
|
| 142 |
|
| 143 |
+
| Dataset Size | Processing Time | Memory Usage | Features Generated | Models Trained |
|
| 144 |
+
|--------------|----------------|--------------|-------------------|----------------|
|
| 145 |
+
| 10K rows | 2-5 minutes | 1-2 GB | 50-100 | 15-20 |
|
| 146 |
+
| 50K rows | 10-15 minutes | 3-4 GB | 100-150 | 10-15 |
|
| 147 |
+
| 100K rows | 20-30 minutes | 4-6 GB | 150-200 | 5-10 |
|
| 148 |
|
| 149 |
+
## ⚙️ Configuration Options
|
| 150 |
+
|
| 151 |
+
### **Data Processing (TimeFlow Pro)**
|
| 152 |
- `missing_threshold`: Threshold for column removal (0.0-0.5)
|
| 153 |
- `outlier_method`: IQR, Z-Score, or Isolation Forest
|
| 154 |
- `scaling_method`: Robust, Standard, MinMax, or None
|
|
|
|
|
|
|
| 155 |
- `max_lags`: Maximum lag features (1-20)
|
| 156 |
- `seasonal_period`: Seasonal window (7, 30, 90, 365)
|
|
|
|
| 157 |
|
| 158 |
+
### **Machine Learning (ModelForge Pro)**
|
| 159 |
+
- `algorithms`: Selection from 30+ available models
|
| 160 |
+
- `tuning_method`: Grid Search or Random Search
|
| 161 |
+
- `ensemble_method`: Averaging, Voting, Weighted, Stacking
|
| 162 |
+
- `cv_folds`: Cross-validation folds (2-10)
|
| 163 |
- `feature_selection_method`: Correlation, Variance, RF, Mutual Info
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 164 |
|
| 165 |
+
## 💡 Why Choose ML Pipeline PRO Suite
|
| 166 |
+
|
| 167 |
+
### **For Organizations**
|
| 168 |
+
- **80% faster** ML development through automation
|
| 169 |
+
- **Improved accuracy** with systematic hyperparameter tuning
|
| 170 |
+
- **Reduced expertise barrier** for ML implementation
|
| 171 |
+
- **Complete reproducibility** of workflows
|
| 172 |
+
- **Scalable architecture** for growing data volumes
|
| 173 |
+
|
| 174 |
+
### **For Researchers**
|
| 175 |
+
- **Standardized methodology** for comparative studies
|
| 176 |
+
- **Automated documentation** of experimental parameters
|
| 177 |
+
- **Easy sharing** of complete ML pipelines
|
| 178 |
+
- **Benchmarking capabilities** across algorithms
|
| 179 |
+
- **Transparent** feature importance analysis
|
| 180 |
+
|
| 181 |
+
## 🌍 Target Industries
|
| 182 |
+
|
| 183 |
+
- **Hydrometeorology**: River flow prediction, precipitation forecasting
|
| 184 |
+
- **Finance**: Market trend analysis, risk assessment models
|
| 185 |
+
- **Energy**: Consumption forecasting, renewable generation prediction
|
| 186 |
+
- **Retail**: Demand forecasting, inventory optimization
|
| 187 |
+
- **Healthcare**: Patient monitoring, disease progression modeling
|
| 188 |
+
- **Manufacturing**: Predictive maintenance, quality control
|
| 189 |
+
|
| 190 |
+
## 📋 System Requirements
|
| 191 |
+
|
| 192 |
+
### **Minimum Configuration**
|
| 193 |
+
- **CPU**: 4 cores
|
| 194 |
+
- **RAM**: 8 GB
|
| 195 |
+
- **Storage**: 10 GB free space
|
| 196 |
+
- **OS**: Windows 10+, Ubuntu 18.04+, macOS 10.15+
|
| 197 |
+
- **Python**: 3.10 or higher
|
| 198 |
+
|
| 199 |
+
### **Recommended for Production**
|
| 200 |
+
- **CPU**: 8+ cores
|
| 201 |
+
- **RAM**: 16+ GB
|
| 202 |
+
- **Storage**: 50 GB SSD
|
| 203 |
+
- **GPU**: Optional for accelerated training
|
| 204 |
+
|
| 205 |
+
## 📦 Export Capabilities
|
| 206 |
+
- Trained models in Pickle/Joblib/Dill format
|
| 207 |
+
- Complete preprocessing pipelines
|
| 208 |
+
- Feature importance reports
|
| 209 |
+
- Model performance comparisons
|
| 210 |
+
- Prediction datasets
|
| 211 |
+
- Configuration files for reproducibility
|
| 212 |
|
| 213 |
## 🤝 Contributing
|
| 214 |
|
| 215 |
We welcome contributions! Here's how you can help:
|
| 216 |
|
| 217 |
### **Areas for Contribution**
|
| 218 |
+
1. **New ML Algorithms**: Implement additional machine learning methods
|
| 219 |
+
2. **Visualization Enhancements**: Add new chart types and interactive features
|
| 220 |
+
3. **Performance Optimizations**: Improve speed and memory efficiency
|
| 221 |
+
4. **Export Format Support**: Add support for additional file formats
|
| 222 |
+
5. **Documentation Improvements**: Expand tutorials and examples
|
| 223 |
|
| 224 |
### **Development Workflow**
|
| 225 |
```bash
|
|
|
|
| 235 |
|
| 236 |
This project is licensed under the **MIT License** - see the [LICENSE](LICENSE) file for details.
|
| 237 |
|
| 238 |
+
If you use ML Pipeline PRO Suite in your research, please cite:
|
| 239 |
+
```
|
| 240 |
+
Arabov M. K., Fatkhulloyev N. I. (2024). ML Pipeline PRO Suite: Comprehensive Machine Learning Platform
|
| 241 |
+
with Full Data Processing Cycle. Kazan Federal University.
|
| 242 |
+
```
|
| 243 |
+
|
| 244 |
## 🙏 Acknowledgments
|
| 245 |
|
| 246 |
+
### **Core Development Team**
|
| 247 |
+
- **Mullosharif Arabov** - Platform Architecture & ML Algorithms
|
| 248 |
+
- **Narzullo Fatkhulloyev** - Data Processing & Interface Design
|
| 249 |
+
|
| 250 |
+
### **Built With**
|
| 251 |
+
- 🐍 **Python** - Core programming language
|
| 252 |
+
- 📊 **Streamlit** - Web interface framework
|
| 253 |
+
- 🔧 **Scikit-learn** - Machine learning algorithms
|
| 254 |
+
- 📈 **Pandas & NumPy** - Data manipulation
|
| 255 |
+
- 🎨 **Plotly** - Interactive visualizations
|
| 256 |
+
- ☁️ **Hugging Face** - Deployment platform
|
| 257 |
|
| 258 |
+
### **Special Thanks**
|
| 259 |
+
- **Kazan Federal University** for research support
|
| 260 |
+
- **Open Source Community** for invaluable libraries
|
| 261 |
+
- **Early Testers** for feedback and improvements
|
|
|
|
|
|
|
| 262 |
|
| 263 |
## 📞 Support & Contact
|
| 264 |
|
| 265 |
+
### **Technical Support**
|
| 266 |
- 📧 **Email**: cool.araby@gmail.com
|
| 267 |
+
- 💬 **Issues**: [GitHub Issues](https://github.com/arabovs-ai-lab/ml-pipeline-pro-suite/issues)
|
| 268 |
+
- 🌐 **Live Demo**: [Hugging Face Space](https://huggingface.co/spaces/arabovs-ai-lab/ml-pipeline-pro-dashboard)
|
| 269 |
|
| 270 |
+
### **Stay Connected**
|
| 271 |
+
- ⭐ **Star** the repository on GitHub
|
| 272 |
+
- 👁️ **Watch** for releases and updates
|
| 273 |
+
- 🔔 **Follow** on Hugging Face for new features
|
| 274 |
|
| 275 |
---
|
| 276 |
|
| 277 |
<div align="center">
|
| 278 |
|
| 279 |
+
**From Raw Data to Production Models in One Platform**
|
| 280 |
|
| 281 |
+
*ML Pipeline PRO Suite - Simplifying Machine Learning Workflows*
|
| 282 |
|
| 283 |
+
[](https://huggingface.co/spaces/arabovs-ai-lab/ml-pipeline-pro-dashboard)
|
| 284 |
+
[](https://github.com/arabovs-ai-lab/ml-pipeline-pro-suite/wiki)
|
| 285 |
|
| 286 |
</div>
|
app.py
CHANGED
|
@@ -24,14 +24,14 @@ def check_preprocessing_status():
|
|
| 24 |
|
| 25 |
# Check if the folder exists
|
| 26 |
if not preprocessing_path.exists():
|
| 27 |
-
return False, "
|
| 28 |
|
| 29 |
# Check if the folder is not empty
|
| 30 |
files = list(preprocessing_path.glob("*"))
|
| 31 |
files = [f for f in files if f.is_file()]
|
| 32 |
|
| 33 |
if len(files) == 0:
|
| 34 |
-
return False, "
|
| 35 |
|
| 36 |
# Supported file extensions
|
| 37 |
supported_extensions = {'.csv', '.parquet', '.xlsx', '.json', '.feather'}
|
|
@@ -73,7 +73,7 @@ def check_preprocessing_status():
|
|
| 73 |
return True, f"Folder contains files but no valid data loaded: {file_list}", None
|
| 74 |
|
| 75 |
except Exception as e:
|
| 76 |
-
return False, f"Error checking preprocessing
|
| 77 |
|
| 78 |
# Path configuration
|
| 79 |
current_dir = Path(__file__).parent
|
|
@@ -615,20 +615,9 @@ def show_dashboard():
|
|
| 615 |
• SHAP analysis
|
| 616 |
""")
|
| 617 |
|
| 618 |
-
|
| 619 |
-
current_data = get_current_data()
|
| 620 |
-
has_data = current_data is not None
|
| 621 |
-
|
| 622 |
-
if st.button("Open ML Pipeline", type="primary", width='stretch', disabled=not has_data):
|
| 623 |
st.session_state.current_view = 'ml'
|
| 624 |
st.rerun()
|
| 625 |
-
|
| 626 |
-
if not has_data:
|
| 627 |
-
preprocessing_ready, preprocessing_message, _ = check_preprocessing_status()
|
| 628 |
-
if not preprocessing_ready:
|
| 629 |
-
st.warning(f"⚠️ {preprocessing_message}. First, perform data preprocessing.")
|
| 630 |
-
else:
|
| 631 |
-
st.warning("⚠️ First, preprocessed data is needed")
|
| 632 |
|
| 633 |
st.markdown("---")
|
| 634 |
st.markdown("### 🛠️ Quick Workflow")
|
|
@@ -729,7 +718,7 @@ def show_dashboard():
|
|
| 729 |
|
| 730 |
except Exception as e:
|
| 731 |
st.error(f"File loading error: {str(e)}")
|
| 732 |
-
|
| 733 |
def run_preprocessor():
|
| 734 |
"""Runs the preprocessing module"""
|
| 735 |
st.title("🧹 TimeFlow Pro - Data Preprocessing")
|
|
@@ -945,18 +934,87 @@ def run_ml_pipeline():
|
|
| 945 |
data = get_current_data()
|
| 946 |
|
| 947 |
if data is None:
|
|
|
|
| 948 |
preprocessing_ready, preprocessing_message, _ = check_preprocessing_status()
|
| 949 |
-
st.warning("⚠️ No preprocessed data available!")
|
| 950 |
-
st.info(f"Please perform data preprocessing first. Status: {preprocessing_message}")
|
| 951 |
|
| 952 |
-
|
| 953 |
-
|
| 954 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 955 |
st.session_state.current_view = 'preprocessor'
|
| 956 |
st.rerun()
|
| 957 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 958 |
|
| 959 |
-
#
|
|
|
|
|
|
|
| 960 |
if not isinstance(data, pd.DataFrame) or data.empty:
|
| 961 |
st.error("❌ Invalid data format! Please load valid data.")
|
| 962 |
if st.button("← Go to data preprocessing", type="primary", width='stretch'):
|
|
@@ -999,6 +1057,7 @@ def run_ml_pipeline():
|
|
| 999 |
|
| 1000 |
st.info(f"**Selected target:** `{target_col}` | **Features:** {len(st.session_state.data_info['feature_columns'])} columns")
|
| 1001 |
|
|
|
|
| 1002 |
if 'ml' not in st.session_state.module_cache:
|
| 1003 |
with st.spinner("⏳ Loading ML Pipeline PRO module..."):
|
| 1004 |
module = load_module_dynamically('ml')
|
|
@@ -1036,9 +1095,8 @@ def run_ml_pipeline():
|
|
| 1036 |
st.error(f"❌ Error processing ML results: {str(e)}")
|
| 1037 |
st.code(traceback.format_exc())
|
| 1038 |
|
| 1039 |
-
#
|
| 1040 |
if results:
|
| 1041 |
-
# Try to create a basic structure from raw results
|
| 1042 |
basic_results = {}
|
| 1043 |
if isinstance(results, dict):
|
| 1044 |
for model_name, result in results.items():
|
|
@@ -1069,20 +1127,19 @@ def run_ml_pipeline():
|
|
| 1069 |
st.markdown("### 🚀 ML Pipeline Interface")
|
| 1070 |
|
| 1071 |
try:
|
| 1072 |
-
#
|
| 1073 |
if hasattr(module, 'main'):
|
| 1074 |
-
# Run in safe mode
|
| 1075 |
try:
|
| 1076 |
module.main()
|
| 1077 |
except Exception as e:
|
| 1078 |
st.error(f"❌ Error executing ML module: {str(e)}")
|
| 1079 |
st.code(traceback.format_exc())
|
| 1080 |
|
| 1081 |
-
# Check
|
| 1082 |
if hasattr(module, 'results') and module.results is not None:
|
| 1083 |
ml_results_handler(module.results)
|
| 1084 |
|
| 1085 |
-
# Show
|
| 1086 |
show_alternative_ml(data, target_col)
|
| 1087 |
return
|
| 1088 |
|
|
@@ -1098,7 +1155,7 @@ def run_ml_pipeline():
|
|
| 1098 |
show_alternative_ml(data, target_col)
|
| 1099 |
return
|
| 1100 |
|
| 1101 |
-
# Check for results in
|
| 1102 |
if hasattr(module, 'results') and module.results is not None:
|
| 1103 |
ml_results_handler(module.results)
|
| 1104 |
|
|
@@ -1115,7 +1172,7 @@ def run_ml_pipeline():
|
|
| 1115 |
st.error(f"❌ Error executing ML module: {str(module_error)}")
|
| 1116 |
st.code(traceback.format_exc())
|
| 1117 |
|
| 1118 |
-
# Check
|
| 1119 |
if 'ml_results' in st.session_state and st.session_state.ml_results is not None:
|
| 1120 |
st.info("Found existing results in session_state")
|
| 1121 |
if st.button("📊 Use existing results", width='stretch'):
|
|
@@ -1133,7 +1190,7 @@ def run_ml_pipeline():
|
|
| 1133 |
st.error(f"❌ Error launching ML module: {str(e)}")
|
| 1134 |
st.code(traceback.format_exc())
|
| 1135 |
show_alternative_ml(data, target_col)
|
| 1136 |
-
|
| 1137 |
def show_alternative_ml(data=None, target_col=None):
|
| 1138 |
"""Shows an alternative ML interface with robust metric handling"""
|
| 1139 |
st.markdown("---")
|
|
|
|
| 24 |
|
| 25 |
# Check if the folder exists
|
| 26 |
if not preprocessing_path.exists():
|
| 27 |
+
return False, "Preprocessing folder not found", None
|
| 28 |
|
| 29 |
# Check if the folder is not empty
|
| 30 |
files = list(preprocessing_path.glob("*"))
|
| 31 |
files = [f for f in files if f.is_file()]
|
| 32 |
|
| 33 |
if len(files) == 0:
|
| 34 |
+
return False, "Preprocessing folder is empty", None
|
| 35 |
|
| 36 |
# Supported file extensions
|
| 37 |
supported_extensions = {'.csv', '.parquet', '.xlsx', '.json', '.feather'}
|
|
|
|
| 73 |
return True, f"Folder contains files but no valid data loaded: {file_list}", None
|
| 74 |
|
| 75 |
except Exception as e:
|
| 76 |
+
return False, f"Error checking preprocessing folder: {str(e)}", None
|
| 77 |
|
| 78 |
# Path configuration
|
| 79 |
current_dir = Path(__file__).parent
|
|
|
|
| 615 |
• SHAP analysis
|
| 616 |
""")
|
| 617 |
|
| 618 |
+
if st.button("Open ML Pipeline", type="primary", width='stretch'):
|
|
|
|
|
|
|
|
|
|
|
|
|
| 619 |
st.session_state.current_view = 'ml'
|
| 620 |
st.rerun()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 621 |
|
| 622 |
st.markdown("---")
|
| 623 |
st.markdown("### 🛠️ Quick Workflow")
|
|
|
|
| 718 |
|
| 719 |
except Exception as e:
|
| 720 |
st.error(f"File loading error: {str(e)}")
|
| 721 |
+
|
| 722 |
def run_preprocessor():
|
| 723 |
"""Runs the preprocessing module"""
|
| 724 |
st.title("🧹 TimeFlow Pro - Data Preprocessing")
|
|
|
|
| 934 |
data = get_current_data()
|
| 935 |
|
| 936 |
if data is None:
|
| 937 |
+
# Check preprocessing status
|
| 938 |
preprocessing_ready, preprocessing_message, _ = check_preprocessing_status()
|
|
|
|
|
|
|
| 939 |
|
| 940 |
+
if not preprocessing_ready:
|
| 941 |
+
st.warning("⚠️ **Preprocessing folder is missing or empty**")
|
| 942 |
+
st.info("""
|
| 943 |
+
**To get started:**
|
| 944 |
+
1. **Run TimeFlow Pro** first to preprocess your data
|
| 945 |
+
2. **Or** use data from other sources
|
| 946 |
+
""")
|
| 947 |
+
else:
|
| 948 |
+
st.warning("⚠️ **No data loaded**")
|
| 949 |
+
st.info("Please load data using one of the available options")
|
| 950 |
+
|
| 951 |
+
# Navigation buttons
|
| 952 |
+
st.markdown("---")
|
| 953 |
+
col1, col2 = st.columns(2)
|
| 954 |
+
|
| 955 |
+
with col1:
|
| 956 |
+
if st.button("← Go to TimeFlow Pro", type="secondary", width='stretch'):
|
| 957 |
st.session_state.current_view = 'preprocessor'
|
| 958 |
st.rerun()
|
| 959 |
+
|
| 960 |
+
with col2:
|
| 961 |
+
if st.button("🏠 Go to Dashboard", type="secondary", width='stretch'):
|
| 962 |
+
st.session_state.current_view = 'dashboard'
|
| 963 |
+
st.rerun()
|
| 964 |
+
|
| 965 |
+
# Load ML module even without data for initialization
|
| 966 |
+
if 'ml' not in st.session_state.module_cache:
|
| 967 |
+
with st.spinner("⏳ Loading ML Pipeline PRO module..."):
|
| 968 |
+
module = load_module_dynamically('ml')
|
| 969 |
+
if module:
|
| 970 |
+
st.session_state.module_cache['ml'] = module
|
| 971 |
+
st.success("✅ ML module successfully loaded!")
|
| 972 |
+
else:
|
| 973 |
+
st.error("❌ Failed to load ML module")
|
| 974 |
+
return
|
| 975 |
+
|
| 976 |
+
# Show ML module is loaded and ready
|
| 977 |
+
st.markdown("---")
|
| 978 |
+
st.markdown("### 🚀 ML Pipeline Interface")
|
| 979 |
+
st.info("💡 ML Pipeline PRO is loaded and ready. Load data to start training.")
|
| 980 |
+
|
| 981 |
+
# Load ML module interface with error handling
|
| 982 |
+
try:
|
| 983 |
+
module = st.session_state.module_cache['ml']
|
| 984 |
+
|
| 985 |
+
# Attempt to run ML module in data-less mode
|
| 986 |
+
if hasattr(module, 'main'):
|
| 987 |
+
module.data = None
|
| 988 |
+
module.target_column = None
|
| 989 |
+
|
| 990 |
+
try:
|
| 991 |
+
module.main()
|
| 992 |
+
except Exception as e:
|
| 993 |
+
st.warning(f"ML module requires data to function properly: {str(e)}")
|
| 994 |
+
|
| 995 |
+
elif hasattr(module, 'run'):
|
| 996 |
+
try:
|
| 997 |
+
module.run()
|
| 998 |
+
except Exception as e:
|
| 999 |
+
st.warning(f"ML module requires data: {str(e)}")
|
| 1000 |
+
|
| 1001 |
+
elif hasattr(module, 'StreamlitApp'):
|
| 1002 |
+
try:
|
| 1003 |
+
ml_app = module.StreamlitApp()
|
| 1004 |
+
ml_app.run()
|
| 1005 |
+
except Exception as e:
|
| 1006 |
+
st.warning(f"ML module requires data: {str(e)}")
|
| 1007 |
+
else:
|
| 1008 |
+
st.info("ML module is loaded. Load data to access full functionality.")
|
| 1009 |
+
|
| 1010 |
+
except Exception as e:
|
| 1011 |
+
st.error(f"❌ Error loading ML interface: {str(e)}")
|
| 1012 |
+
|
| 1013 |
+
return # Exit since no data for full operation
|
| 1014 |
|
| 1015 |
+
# ========== DATA AVAILABLE ==========
|
| 1016 |
+
|
| 1017 |
+
# Validate data format
|
| 1018 |
if not isinstance(data, pd.DataFrame) or data.empty:
|
| 1019 |
st.error("❌ Invalid data format! Please load valid data.")
|
| 1020 |
if st.button("← Go to data preprocessing", type="primary", width='stretch'):
|
|
|
|
| 1057 |
|
| 1058 |
st.info(f"**Selected target:** `{target_col}` | **Features:** {len(st.session_state.data_info['feature_columns'])} columns")
|
| 1059 |
|
| 1060 |
+
# Load ML module if not already loaded
|
| 1061 |
if 'ml' not in st.session_state.module_cache:
|
| 1062 |
with st.spinner("⏳ Loading ML Pipeline PRO module..."):
|
| 1063 |
module = load_module_dynamically('ml')
|
|
|
|
| 1095 |
st.error(f"❌ Error processing ML results: {str(e)}")
|
| 1096 |
st.code(traceback.format_exc())
|
| 1097 |
|
| 1098 |
+
# Fallback: save raw results
|
| 1099 |
if results:
|
|
|
|
| 1100 |
basic_results = {}
|
| 1101 |
if isinstance(results, dict):
|
| 1102 |
for model_name, result in results.items():
|
|
|
|
| 1127 |
st.markdown("### 🚀 ML Pipeline Interface")
|
| 1128 |
|
| 1129 |
try:
|
| 1130 |
+
# Execute ML module with safe error handling
|
| 1131 |
if hasattr(module, 'main'):
|
|
|
|
| 1132 |
try:
|
| 1133 |
module.main()
|
| 1134 |
except Exception as e:
|
| 1135 |
st.error(f"❌ Error executing ML module: {str(e)}")
|
| 1136 |
st.code(traceback.format_exc())
|
| 1137 |
|
| 1138 |
+
# Check for results despite error
|
| 1139 |
if hasattr(module, 'results') and module.results is not None:
|
| 1140 |
ml_results_handler(module.results)
|
| 1141 |
|
| 1142 |
+
# Show fallback interface
|
| 1143 |
show_alternative_ml(data, target_col)
|
| 1144 |
return
|
| 1145 |
|
|
|
|
| 1155 |
show_alternative_ml(data, target_col)
|
| 1156 |
return
|
| 1157 |
|
| 1158 |
+
# Check for results in module
|
| 1159 |
if hasattr(module, 'results') and module.results is not None:
|
| 1160 |
ml_results_handler(module.results)
|
| 1161 |
|
|
|
|
| 1172 |
st.error(f"❌ Error executing ML module: {str(module_error)}")
|
| 1173 |
st.code(traceback.format_exc())
|
| 1174 |
|
| 1175 |
+
# Check for existing results
|
| 1176 |
if 'ml_results' in st.session_state and st.session_state.ml_results is not None:
|
| 1177 |
st.info("Found existing results in session_state")
|
| 1178 |
if st.button("📊 Use existing results", width='stretch'):
|
|
|
|
| 1190 |
st.error(f"❌ Error launching ML module: {str(e)}")
|
| 1191 |
st.code(traceback.format_exc())
|
| 1192 |
show_alternative_ml(data, target_col)
|
| 1193 |
+
|
| 1194 |
def show_alternative_ml(data=None, target_col=None):
|
| 1195 |
"""Shows an alternative ML interface with robust metric handling"""
|
| 1196 |
st.markdown("---")
|
src/enhanced_preprocessing/__init__.py
ADDED
|
File without changes
|
main_pipeline.py → src/enhanced_preprocessing/main_pipeline.py
RENAMED
|
File without changes
|
src/enhanced_preprocessing/run_pipeline.py
CHANGED
|
@@ -5,7 +5,7 @@
|
|
| 5 |
import pandas as pd
|
| 6 |
import os
|
| 7 |
|
| 8 |
-
from main_pipeline import EnhancedDataPreprocessingPipeline
|
| 9 |
from src.enhanced_preprocessing.config.config import Config
|
| 10 |
|
| 11 |
if __name__ == "__main__":
|
|
|
|
| 5 |
import pandas as pd
|
| 6 |
import os
|
| 7 |
|
| 8 |
+
from enhanced_preprocessing.main_pipeline import EnhancedDataPreprocessingPipeline
|
| 9 |
from src.enhanced_preprocessing.config.config import Config
|
| 10 |
|
| 11 |
if __name__ == "__main__":
|
streamlit/preprocessor_app.py
CHANGED
|
@@ -6,7 +6,7 @@ import os
|
|
| 6 |
|
| 7 |
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
|
| 8 |
|
| 9 |
-
from main_pipeline import EnhancedDataPreprocessingPipeline
|
| 10 |
from src.enhanced_preprocessing.config.config import Config
|
| 11 |
import streamlit as st
|
| 12 |
import pandas as pd
|
|
|
|
| 6 |
|
| 7 |
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
|
| 8 |
|
| 9 |
+
from src.enhanced_preprocessing.main_pipeline import EnhancedDataPreprocessingPipeline
|
| 10 |
from src.enhanced_preprocessing.config.config import Config
|
| 11 |
import streamlit as st
|
| 12 |
import pandas as pd
|