Ajout du fichier de données nécessaire

app.py

@@ -1,39 +1,53 @@
-import joblib
 import pandas as pd
 import numpy as np
+from xgboost import XGBClassifier
+from lightgbm import LGBMClassifier
+from sklearn.ensemble import RandomForestClassifier
+from sklearn.linear_model import LogisticRegression
+from sklearn.svm import SVC
+from sklearn.preprocessing import StandardScaler, LabelEncoder
+from sklearn.model_selection import StratifiedKFold
+from sklearn.metrics import classification_report, recall_score, f1_score
+from sklearn.impute import SimpleImputer
+from imblearn.over_sampling import SMOTE
+from imblearn.under_sampling import RandomUnderSampler
+from imblearn.pipeline import Pipeline
+import joblib
 from flask import Flask, request, jsonify
 from flask_cors import CORS
 import os
-import logging
-import threading
+import warnings
 import time
 from tqdm import tqdm
+import threading
+import logging
 from tenacity import retry, wait_fixed, stop_after_attempt
-from sklearn.preprocessing import StandardScaler, LabelEncoder
-from sklearn.impute import SimpleImputer
-from sklearn.model_selection import StratifiedKFold
-from sklearn.metrics import f1_score, recall_score
-from imblearn.over_sampling import SMOTE
-from imblearn.under_sampling import RandomUnderSampler
-from imblearn.pipeline import Pipeline
-from xgboost import XGBClassifier
-from lightgbm import LGBMClassifier
-from sklearn.ensemble import RandomForestClassifier
-from sklearn.linear_model import LogisticRegression
-from sklearn.svm import SVC
 
-# Initialiser l'application Flask
+warnings.filterwarnings('ignore', category=UserWarning)
+os.environ["LOKY_MAX_CPU_COUNT"] = "1"
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
 app = Flask(__name__)
 CORS(app)
 
-# Chemins vers les fichiers de modèle
-PRIORITY_MODEL_PATH = 'priority_model.pkl'
-SERVICE_MODEL_PATH = 'service_model.pkl'
-DATASET_PATH = "my_datasheet_80000.csv"
 NEW_DATA_FILE = 'new_data.csv'
+DATASET_PATH = "my_datasheet_80000.csv"
 MIN_NEW_SAMPLES_FOR_RETRAIN = 100
 
-# Variables globales pour les modèles
+# Feature sets for each task
+PRIORITY_FEATURES = [
+    'SpO2', 'Frquce_Rprtr(rpm)', 'Pouls', 'PA', 'Temperature', 'SpO2_Severity', 'Tachypnea', 'Bradypnea',
+    'Tachycardia', 'Bradycardia', 'Critical_Signs', 'SpO2_Temp_Ratio', 'Pouls_PA_Ratio', 'Temp_Pouls_Ratio',
+    'SpO2_PA_Diff', 'SpO2_Temp_Diff', 'PA_Pouls_Diff', 'SpO2_Log', 'Temp_Squared', 'Suggested_Priority'
+]
+
+SERVICE_FEATURES = [
+    'Age', 'Sexe', 'Enceinte', 'SpO2', 'Frquce_Rprtr(rpm)', 'Pouls', 'ECG', 'PA', 'Temperature', 'IMC',
+    'Age_Category', 'Temp_Anomaly', 'PA_High', 'PA_Low', 'Pouls_SpO2_Ratio', 'PA_Temp_Ratio', 'IMC_Temp_Ratio'
+]
+
 priority_model = None
 service_model = None
 priority_scaler = None
@@ -44,32 +58,301 @@ label_encoder_service = LabelEncoder()
 
 model_lock = threading.Lock()
 
-# Initialiser le logger
-logging.basicConfig(level=logging.INFO)
-logger = logging.getLogger(__name__)
+def enhanced_features(df):
+    df['Tachypnea'] = df.apply(lambda row: 1 if (row['Age'] < 1 and row['Frquce_Rprtr(rpm)'] > 40) or 
+                                         (row['Age'] < 12 and row['Frquce_Rprtr(rpm)'] > 30) or 
+                                         (row['Age'] >= 12 and row['Frquce_Rprtr(rpm)'] > 20) else 0, axis=1)
+    df['Bradypnea'] = df.apply(lambda row: 1 if (row['Age'] < 1 and row['Frquce_Rprtr(rpm)'] < 20) or 
+                                         (row['Age'] < 12 and row['Frquce_Rprtr(rpm)'] < 12) or 
+                                         (row['Age'] >= 12 and row['Frquce_Rprtr(rpm)'] < 8) else 0, axis=1)
+    df['Tachycardia'] = df.apply(lambda row: 1 if (row['Age'] < 1 and row['Pouls'] > 160) or 
+                                           (row['Age'] < 12 and row['Pouls'] > 120) or 
+                                           (row['Age'] >= 12 and row['Pouls'] > 100) else 0, axis=1)
+    df['Bradycardia'] = df.apply(lambda row: 1 if (row['Age'] < 1 and row['Pouls'] < 90) or 
+                                           (row['Age'] < 12 and row['Pouls'] < 70) or 
+                                           (row['Age'] >= 12 and row['Pouls'] < 50) else 0, axis=1)
+    df['SpO2_Temp_Ratio'] = df['SpO2'] / (df['Temperature'] + 1e-6)
+    df['Pouls_PA_Ratio'] = df['Pouls'] / (df['PA'] + 1e-6)
+    df['Temp_Pouls_Ratio'] = df['Temperature'] / (df['Pouls'] + 1e-6)
+    df['SpO2_PA_Diff'] = df['SpO2'] - df['PA'] / 10
+    df['SpO2_Temp_Diff'] = df['SpO2'] - df['Temperature']
+    df['PA_Pouls_Diff'] = df['PA'] - df['Pouls']
+    df['IMC_Temp_Ratio'] = df['IMC'] / (df['Temperature'] + 1e-6)
+    df['SpO2_Log'] = np.log1p(df['SpO2'])
+    df['Temp_Squared'] = df['Temperature'] ** 2
+    df['Pouls_SpO2_Ratio'] = df['Pouls'] / (df['SpO2'] + 1e-6)
+    df['PA_Temp_Ratio'] = df['PA'] / (df['Temperature'] + 1e-6)
+    df['Age_Category'] = pd.cut(df['Age'], bins=[0, 1, 12, 45, 65, 120], labels=[0, 1, 2, 3, 4])
+    df['Temp_Anomaly'] = df['Temperature'].apply(lambda x: 1 if x < 35 or x > 38 else 0)
+    df['PA_High'] = df['PA'].apply(lambda x: 1 if x > 160 else 0)
+    df['PA_Low'] = df['PA'].apply(lambda x: 1 if x < 90 else 0)
+    df['SpO2_Severity'] = pd.cut(df['SpO2'], bins=[0, 85, 90, 92, 100], labels=[3, 2, 1, 0])
+    df['Critical_Signs'] = ((df['SpO2'] < 85) | (df['Pouls'] > 150) | (df['Temperature'] > 40) | 
+                            (df['PA'] > 200) | (df['PA'] < 70)).astype(int)
+    return df
+
+def compute_service_and_priority(row):
+    age = row['Age']
+    spO2 = row['SpO2']
+    frq_resp = row['Frquce_Rprtr(rpm)']
+    pouls = row['Pouls']
+    ecg = row['ECG']
+    pa = row['PA']
+    temp = row['Temperature']
+    enceinte = row['Enceinte']
+    imc = row['IMC']
+
+    if age <= 18:
+        service = 'Pédiatriques'
+    elif enceinte:
+        service = 'Gynécologie/Obstétrique'
+    elif ecg == 1 or (pouls < 50 or pouls > 110) or (frq_resp > 20):
+        service = 'Neurologie'
+    elif spO2 < 92 or frq_resp > 18 or pouls > 100 or pa < 90 or pa > 160:
+        service = 'Cardiorespiratoire'
+    elif (imc > 30 and (temp > 38 and temp <= 40) and 70 <= pouls <= 90) or \
+         (70 <= pouls <= 90 and 110 <= pa <= 130 and spO2 >= 97 and temp <= 37.5):
+        service = 'Médecine générale'
+    elif temp > 40:
+        service = 'Radiothérapie'
+    else:
+        service = 'Chirurgie'
 
-# Fonction pour charger les modèles
-def load_models():
+    if spO2 < 85 or temp > 40 or pouls > 150 or pa < 70 or pa > 200:
+        priorite = 1
+    elif spO2 < 88 or temp > 39.5 or pouls > 130 or pa < 80 or pa > 180 or frq_resp > 25:
+        priorite = 2
+    elif spO2 < 90 or temp > 38.5 or pouls > 110 or pa < 90 or pa > 160 or frq_resp > 20:
+        priorite = 3
+    elif spO2 < 92 or temp > 38 or pouls > 100 or pa < 100 or pa > 140 or frq_resp > 18:
+        priorite = 4
+    else:
+        priorite = 5
+
+    return service, priorite
+
+def get_smote_strategy(y, max_samples=1000):
+    class_counts = pd.Series(y).value_counts()
+    strategy = {}
+    for cls, count in class_counts.items():
+        target = min(max_samples, max(count * 2, 100))  # Ensure reasonable class sizes
+    return strategy
+
+def train_priority_model():
+    global priority_model, priority_scaler, priority_imputer
+    try:
+        data = pd.read_csv(DATASET_PATH)
+        data['Sexe'] = data['Sexe'].map({'Masculin': 0, 'Feminin': 1})
+        data['Enceinte'] = data['Enceinte'].astype(int)
+        data['ECG'] = data['ECG'].map({'Normal': 0, 'Anormal': 1})
+        data = enhanced_features(data)
+        data[['Suggested_Service', 'Suggested_Priority']] = data.apply(compute_service_and_priority, axis=1, result_type='expand')
+        data['Suggested_Priority'] = data['Suggested_Priority'].astype(int)
+
+        X = data[PRIORITY_FEATURES]
+        y = data['Priorite'].values - 1  # Shift to 0-based indexing
+
+        priority_imputer = SimpleImputer(strategy='median')
+        X_imputed = priority_imputer.fit_transform(X)
+        priority_scaler = StandardScaler()
+        X_scaled = priority_scaler.fit_transform(X_imputed)
+
+        models = {
+            'XGBoost': XGBClassifier(n_estimators=100, max_depth=4, learning_rate=0.05, n_jobs=-1, random_state=42),
+            'LightGBM': LGBMClassifier(n_estimators=100, max_depth=2, learning_rate=0.05, min_child_samples=5, 
+                                      reg_alpha=0.5, reg_lambda=0.5, n_jobs=-1, random_state=42, verbose=-1),
+            'RandomForest': RandomForestClassifier(n_estimators=100, max_depth=8, n_jobs=-1, random_state=42),
+            'LogisticRegression': LogisticRegression(max_iter=1000, multi_class='multinomial', random_state=42),
+            'SVM': SVC(probability=True, random_state=42)
+        }
+
+        skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)
+        results = {}
+
+        for name, model in models.items():
+            logger.info(f"\nEvaluating {name} for Priority...")
+            scores = {'f1': [], 'recall_p1': [], 'time': []}
+            for train_idx, test_idx in tqdm(skf.split(X_scaled, y), total=5):
+                X_train, X_test = X_scaled[train_idx], X_scaled[test_idx]
+                y_train, y_test = y[train_idx], y[test_idx]
+                
+                min_class_size = pd.Series(y_train).value_counts().min()
+                k_neighbors = min(5, max(1, min_class_size - 1))
+                pipeline = Pipeline([
+                    ('under', RandomUnderSampler(sampling_strategy='majority', random_state=42)),
+                    ('over', SMOTE(sampling_strategy=get_smote_strategy(y_train), random_state=42, k_neighbors=k_neighbors))
+                ])
+                X_train_res, y_train_res = pipeline.fit_resample(X_train, y_train)
+                class_sizes = pd.Series(y_train_res).value_counts().to_dict()
+                logger.info(f"{name} - Resampled class sizes: {class_sizes}")
+                
+                start_time = time.time()
+                model.fit(X_train_res, y_train_res)
+                train_time = time.time() - start_time
+                
+                y_pred = model.predict(X_test)
+                scores['f1'].append(f1_score(y_test, y_pred, average='macro'))
+                scores['recall_p1'].append(recall_score(y_test, y_pred, labels=[0], average=None, zero_division=0)[0])
+                scores['time'].append(train_time)
+                logger.info(f"{name} Fold - F1: {scores['f1'][-1]:.3f}, Recall P1: {scores['recall_p1'][-1]:.3f}")
+
+            results[name] = {
+                'f1': np.mean(scores['f1']),
+                'recall_p1': np.mean(scores['recall_p1']),
+                'time': np.mean(scores['time'])
+            }
+            if name == 'LightGBM':
+                feature_importance = pd.Series(model.feature_importances_, index=PRIORITY_FEATURES).sort_values(ascending=False)
+                logger.info(f"LightGBM Priority Feature Importance:\n{feature_importance}")
+
+        logger.info("\nPriority Model Comparison:")
+        for name, res in results.items():
+            logger.info(f"{name}: F1={res['f1']:.3f}, Recall P1={res['recall_p1']:.3f}, Time={res['time']:.2f}s")
+
+        best_model = max(results, key=lambda k: results[k]['f1'] + results[k]['recall_p1'])
+        logger.info(f"Best Priority Model: {best_model}")
+
+        with model_lock:
+            priority_model = models[best_model]
+            priority_model.fit(X_scaled, y)
+
+        timestamp = int(time.time())
+        joblib.dump(priority_model, f'priority_model_{timestamp}.pkl')
+        joblib.dump(priority_scaler, 'priority_scaler.pkl')
+        joblib.dump(priority_imputer, 'priority_imputer.pkl')
+        logger.info("Priority model saved.")
+    except Exception as e:
+        logger.error(f"Error in priority training: {e}")
+        raise
+
+def train_service_model():
+    global service_model, service_scaler, service_imputer, label_encoder_service
+    try:
+        data = pd.read_csv(DATASET_PATH)
+        data['Sexe'] = data['Sexe'].map({'Masculin': 0, 'Feminin': 1})
+        data['Enceinte'] = data['Enceinte'].astype(int)
+        data['ECG'] = data['ECG'].map({'Normal': 0, 'Anormal': 1})
+        data = enhanced_features(data)
+        data[['Suggested_Service', 'Suggested_Priority']] = data.apply(compute_service_and_priority, axis=1, result_type='expand')
+
+        X = data[SERVICE_FEATURES]
+        y = label_encoder_service.fit_transform(data['Service_Suivant'].fillna('Unknown'))
+
+        service_imputer = SimpleImputer(strategy='median')
+        X_imputed = service_imputer.fit_transform(X)
+        service_scaler = StandardScaler()
+        X_scaled = service_scaler.fit_transform(X_imputed)
+
+        models = {
+            'XGBoost': XGBClassifier(n_estimators=100, max_depth=4, learning_rate=0.05, n_jobs=-1, random_state=42),
+            'LightGBM': LGBMClassifier(n_estimators=100, max_depth=2, learning_rate=0.05, min_child_samples=5, 
+                                      reg_alpha=0.5, reg_lambda=0.5, n_jobs=-1, random_state=42, verbose=-1),
+            'RandomForest': RandomForestClassifier(n_estimators=100, max_depth=8, n_jobs=-1, random_state=42),
+            'LogisticRegression': LogisticRegression(max_iter=1000, multi_class='multinomial', random_state=42),
+            'SVM': SVC(probability=True, random_state=42)
+        }
+
+        skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)
+        results = {}
+
+        for name, model in models.items():
+            logger.info(f"\nEvaluating {name} for Service...")
+            scores = {'f1': [], 'time': []}
+            for train_idx, test_idx in tqdm(skf.split(X_scaled, y), total=5):
+                X_train, X_test = X_scaled[train_idx], X_scaled[test_idx]
+                y_train, y_test = y[train_idx], y[test_idx]
+                
+                min_class_size = pd.Series(y_train).value_counts().min()
+                k_neighbors = min(5, max(1, min_class_size - 1))
+                pipeline = Pipeline([
+                    ('under', RandomUnderSampler(sampling_strategy='majority', random_state=42)),
+                    ('over', SMOTE(sampling_strategy=get_smote_strategy(y_train), random_state=42, k_neighbors=k_neighbors))
+                ])
+                X_train_res, y_train_res = pipeline.fit_resample(X_train, y_train)
+                class_sizes = pd.Series(y_train_res).value_counts().to_dict()
+                logger.info(f"{name} - Resampled class sizes: {class_sizes}")
+                
+                start_time = time.time()
+                model.fit(X_train_res, y_train_res)
+                train_time = time.time() - start_time
+                
+                y_pred = model.predict(X_test)
+                scores['f1'].append(f1_score(y_test, y_pred, average='macro'))
+                scores['time'].append(train_time)
+
+            results[name] = {
+                'f1': np.mean(scores['f1']),
+                'time': np.mean(scores['time'])
+            }
+            if name == 'LightGBM':
+                feature_importance = pd.Series(model.feature_importances_, index=SERVICE_FEATURES).sort_values(ascending=False)
+                logger.info(f"LightGBM Service Feature Importance:\n{feature_importance}")
+
+        logger.info("\nService Model Comparison:")
+        for name, res in results.items():
+            logger.info(f"{name}: F1={res['f1']:.3f}, Time={res['time']:.2f}s")
+
+        best_model = max(results, key=lambda k: results[k]['f1'])
+        logger.info(f"Best Service Model: {best_model}")
+
+        with model_lock:
+            service_model = models[best_model]
+            service_model.fit(X_scaled, y)
+
+        timestamp = int(time.time())
+        joblib.dump(service_model, f'service_model_{timestamp}.pkl')
+        joblib.dump(service_scaler, 'service_scaler.pkl')
+        joblib.dump(service_imputer, 'service_imputer.pkl')
+        joblib.dump(label_encoder_service, 'label_encoder_service.pkl')
+        logger.info("Service model saved.")
+    except Exception as e:
+        logger.error(f"Error in service training: {e}")
+        raise
+
+@retry(wait=wait_fixed(2), stop=stop_after_attempt(3))
+def retrain_models():
     global priority_model, service_model, priority_scaler, service_scaler, priority_imputer, service_imputer, label_encoder_service
-    if os.path.exists(PRIORITY_MODEL_PATH):
-        priority_model = joblib.load(PRIORITY_MODEL_PATH)
-    if os.path.exists(SERVICE_MODEL_PATH):
-        service_model = joblib.load(SERVICE_MODEL_PATH)
-    priority_scaler = joblib.load('priority_scaler.pkl')
-    service_scaler = joblib.load('service_scaler.pkl')
-    priority_imputer = joblib.load('priority_imputer.pkl')
-    service_imputer = joblib.load('service_imputer.pkl')
-    label_encoder_service = joblib.load('label_encoder_service.pkl')
-
-# Charger les modèles au démarrage
-load_models()
-
-# Fonctions et routes Flask
+    while True:
+        time.sleep(3600)
+        if os.path.exists(NEW_DATA_FILE) and os.path.getsize(NEW_DATA_FILE) > 0:
+            try:
+                new_data = pd.read_csv(NEW_DATA_FILE)
+                if len(new_data) >= MIN_NEW_SAMPLES_FOR_RETRAIN:
+                    orig_data = pd.read_csv(DATASET_PATH)
+                    orig_data['Sexe'] = orig_data['Sexe'].map({'Masculin': 0, 'Feminin': 1})
+                    orig_data['Enceinte'] = orig_data['Enceinte'].astype(int)
+                    orig_data['ECG'] = orig_data['ECG'].map({'Normal': 0, 'Anormal': 1})
+                    new_data = enhanced_features(new_data)
+                    combined_data = pd.concat([orig_data, new_data], ignore_index=True)
+
+                    # Priority retraining
+                    X_priority = combined_data[PRIORITY_FEATURES]
+                    y_priority = combined_data['Priorite'].values - 1
+                    X_priority_imputed = priority_imputer.transform(X_priority)
+                    X_priority_scaled = priority_scaler.transform(X_priority_imputed)
+                    with model_lock:
+                        priority_model.fit(X_priority_scaled, y_priority)
+
+                    # Service retraining
+                    X_service = combined_data[SERVICE_FEATURES]
+                    y_service = label_encoder_service.transform(combined_data['Service_Suivant'].fillna('Unknown'))
+                    X_service_imputed = service_imputer.transform(X_service)
+                    X_service_scaled = service_scaler.transform(X_service_imputed)
+                    with model_lock:
+                        service_model.fit(X_service_scaled, y_service)
+
+                    timestamp = int(time.time())
+                    joblib.dump(priority_model, f'priority_model_{timestamp}.pkl')
+                    joblib.dump(service_model, f'service_model_{timestamp}.pkl')
+                    new_data.to_csv(f'archive_new_data_{timestamp}.csv', index=False)
+                    open(NEW_DATA_FILE, 'w').close()
+                    logger.info("Models retrained and saved.")
+            except Exception as e:
+                logger.error(f"Error in retrain: {e}")
+
 @app.route('/predict', methods=['POST'])
 def predict():
-    global priority_model, service_model
-    if priority_model is None or service_model is None:
-        load_models()
+    global priority_model, service_model, priority_scaler, service_scaler, priority_imputer, service_imputer, label_encoder_service
     try:
         data = request.get_json()
         required_fields = ['age', 'sexe', 'enceinte', 'spo2', 'freq_resp', 'pouls', 'ecg', 'pa', 'temperature', 'imc']
@@ -138,4 +421,20 @@ def predict():
         return jsonify({'error': str(e)}), 500
 
 if __name__ == '__main__':
-    app.run(debug=False, host='0.0.0.0', port=5000)
+    FORCE_RETRAIN = True
+    if FORCE_RETRAIN or not (os.path.exists('priority_model.pkl') and os.path.exists('service_model.pkl')):
+        train_priority_model()
+        train_service_model()
+    else:
+        with model_lock:
+            priority_model = joblib.load('priority_model.pkl')
+            service_model = joblib.load('service_model.pkl')
+            priority_scaler = joblib.load('priority_scaler.pkl')
+            service_scaler = joblib.load('service_scaler.pkl')
+            priority_imputer = joblib.load('priority_imputer.pkl')
+            service_imputer = joblib.load('service_imputer.pkl')
+            label_encoder_service = joblib.load('label_encoder_service.pkl')
+
+    retrain_thread = threading.Thread(target=retrain_models, daemon=True)
+    retrain_thread.start()
+    app.run(debug=False, host='0.0.0.0', port=5000)