Ajout du fichier de données nécessaire

app.py

@@ -1,53 +1,39 @@
+import joblib
 import pandas as pd
 import numpy as np
-from xgboost import XGBClassifier
-from lightgbm import LGBMClassifier
-from sklearn.ensemble import RandomForestClassifier
-from sklearn.linear_model import LogisticRegression
-from sklearn.svm import SVC
-from sklearn.preprocessing import StandardScaler, LabelEncoder
-from sklearn.model_selection import StratifiedKFold
-from sklearn.metrics import classification_report, recall_score, f1_score
-from sklearn.impute import SimpleImputer
-from imblearn.over_sampling import SMOTE
-from imblearn.under_sampling import RandomUnderSampler
-from imblearn.pipeline import Pipeline
-import joblib
 from flask import Flask, request, jsonify
 from flask_cors import CORS
 import os
-import warnings
+import logging
+import threading
 import time
 from tqdm import tqdm
-import threading
-import logging
 from tenacity import retry, wait_fixed, stop_after_attempt
+from sklearn.preprocessing import StandardScaler, LabelEncoder
+from sklearn.impute import SimpleImputer
+from sklearn.model_selection import StratifiedKFold
+from sklearn.metrics import f1_score, recall_score
+from imblearn.over_sampling import SMOTE
+from imblearn.under_sampling import RandomUnderSampler
+from imblearn.pipeline import Pipeline
+from xgboost import XGBClassifier
+from lightgbm import LGBMClassifier
+from sklearn.ensemble import RandomForestClassifier
+from sklearn.linear_model import LogisticRegression
+from sklearn.svm import SVC
 
-warnings.filterwarnings('ignore', category=UserWarning)
-os.environ["LOKY_MAX_CPU_COUNT"] = "1"
-
-logging.basicConfig(level=logging.INFO)
-logger = logging.getLogger(__name__)
-
+# Initialiser l'application Flask
 app = Flask(__name__)
 CORS(app)
 
-NEW_DATA_FILE = 'new_data.csv'
+# Chemins vers les fichiers de modèle
+PRIORITY_MODEL_PATH = 'priority_model.pkl'
+SERVICE_MODEL_PATH = 'service_model.pkl'
 DATASET_PATH = "my_datasheet_80000.csv"
+NEW_DATA_FILE = 'new_data.csv'
 MIN_NEW_SAMPLES_FOR_RETRAIN = 100
 
-# Feature sets for each task
-PRIORITY_FEATURES = [
-    'SpO2', 'Frquce_Rprtr(rpm)', 'Pouls', 'PA', 'Temperature', 'SpO2_Severity', 'Tachypnea', 'Bradypnea',
-    'Tachycardia', 'Bradycardia', 'Critical_Signs', 'SpO2_Temp_Ratio', 'Pouls_PA_Ratio', 'Temp_Pouls_Ratio',
-    'SpO2_PA_Diff', 'SpO2_Temp_Diff', 'PA_Pouls_Diff', 'SpO2_Log', 'Temp_Squared', 'Suggested_Priority'
-]
-
-SERVICE_FEATURES = [
-    'Age', 'Sexe', 'Enceinte', 'SpO2', 'Frquce_Rprtr(rpm)', 'Pouls', 'ECG', 'PA', 'Temperature', 'IMC',
-    'Age_Category', 'Temp_Anomaly', 'PA_High', 'PA_Low', 'Pouls_SpO2_Ratio', 'PA_Temp_Ratio', 'IMC_Temp_Ratio'
-]
-
+# Variables globales pour les modèles
 priority_model = None
 service_model = None
 priority_scaler = None
@@ -58,301 +44,32 @@ label_encoder_service = LabelEncoder()
 
 model_lock = threading.Lock()
 
-def enhanced_features(df):
-    df['Tachypnea'] = df.apply(lambda row: 1 if (row['Age'] < 1 and row['Frquce_Rprtr(rpm)'] > 40) or 
-                                         (row['Age'] < 12 and row['Frquce_Rprtr(rpm)'] > 30) or 
-                                         (row['Age'] >= 12 and row['Frquce_Rprtr(rpm)'] > 20) else 0, axis=1)
-    df['Bradypnea'] = df.apply(lambda row: 1 if (row['Age'] < 1 and row['Frquce_Rprtr(rpm)'] < 20) or 
-                                         (row['Age'] < 12 and row['Frquce_Rprtr(rpm)'] < 12) or 
-                                         (row['Age'] >= 12 and row['Frquce_Rprtr(rpm)'] < 8) else 0, axis=1)
-    df['Tachycardia'] = df.apply(lambda row: 1 if (row['Age'] < 1 and row['Pouls'] > 160) or 
-                                           (row['Age'] < 12 and row['Pouls'] > 120) or 
-                                           (row['Age'] >= 12 and row['Pouls'] > 100) else 0, axis=1)
-    df['Bradycardia'] = df.apply(lambda row: 1 if (row['Age'] < 1 and row['Pouls'] < 90) or 
-                                           (row['Age'] < 12 and row['Pouls'] < 70) or 
-                                           (row['Age'] >= 12 and row['Pouls'] < 50) else 0, axis=1)
-    df['SpO2_Temp_Ratio'] = df['SpO2'] / (df['Temperature'] + 1e-6)
-    df['Pouls_PA_Ratio'] = df['Pouls'] / (df['PA'] + 1e-6)
-    df['Temp_Pouls_Ratio'] = df['Temperature'] / (df['Pouls'] + 1e-6)
-    df['SpO2_PA_Diff'] = df['SpO2'] - df['PA'] / 10
-    df['SpO2_Temp_Diff'] = df['SpO2'] - df['Temperature']
-    df['PA_Pouls_Diff'] = df['PA'] - df['Pouls']
-    df['IMC_Temp_Ratio'] = df['IMC'] / (df['Temperature'] + 1e-6)
-    df['SpO2_Log'] = np.log1p(df['SpO2'])
-    df['Temp_Squared'] = df['Temperature'] ** 2
-    df['Pouls_SpO2_Ratio'] = df['Pouls'] / (df['SpO2'] + 1e-6)
-    df['PA_Temp_Ratio'] = df['PA'] / (df['Temperature'] + 1e-6)
-    df['Age_Category'] = pd.cut(df['Age'], bins=[0, 1, 12, 45, 65, 120], labels=[0, 1, 2, 3, 4])
-    df['Temp_Anomaly'] = df['Temperature'].apply(lambda x: 1 if x < 35 or x > 38 else 0)
-    df['PA_High'] = df['PA'].apply(lambda x: 1 if x > 160 else 0)
-    df['PA_Low'] = df['PA'].apply(lambda x: 1 if x < 90 else 0)
-    df['SpO2_Severity'] = pd.cut(df['SpO2'], bins=[0, 85, 90, 92, 100], labels=[3, 2, 1, 0])
-    df['Critical_Signs'] = ((df['SpO2'] < 85) | (df['Pouls'] > 150) | (df['Temperature'] > 40) | 
-                            (df['PA'] > 200) | (df['PA'] < 70)).astype(int)
-    return df
-
-def compute_service_and_priority(row):
-    age = row['Age']
-    spO2 = row['SpO2']
-    frq_resp = row['Frquce_Rprtr(rpm)']
-    pouls = row['Pouls']
-    ecg = row['ECG']
-    pa = row['PA']
-    temp = row['Temperature']
-    enceinte = row['Enceinte']
-    imc = row['IMC']
-
-    if age <= 18:
-        service = 'Pédiatriques'
-    elif enceinte:
-        service = 'Gynécologie/Obstétrique'
-    elif ecg == 1 or (pouls < 50 or pouls > 110) or (frq_resp > 20):
-        service = 'Neurologie'
-    elif spO2 < 92 or frq_resp > 18 or pouls > 100 or pa < 90 or pa > 160:
-        service = 'Cardiorespiratoire'
-    elif (imc > 30 and (temp > 38 and temp <= 40) and 70 <= pouls <= 90) or \
-         (70 <= pouls <= 90 and 110 <= pa <= 130 and spO2 >= 97 and temp <= 37.5):
-        service = 'Médecine générale'
-    elif temp > 40:
-        service = 'Radiothérapie'
-    else:
-        service = 'Chirurgie'
-
-    if spO2 < 85 or temp > 40 or pouls > 150 or pa < 70 or pa > 200:
-        priorite = 1
-    elif spO2 < 88 or temp > 39.5 or pouls > 130 or pa < 80 or pa > 180 or frq_resp > 25:
-        priorite = 2
-    elif spO2 < 90 or temp > 38.5 or pouls > 110 or pa < 90 or pa > 160 or frq_resp > 20:
-        priorite = 3
-    elif spO2 < 92 or temp > 38 or pouls > 100 or pa < 100 or pa > 140 or frq_resp > 18:
-        priorite = 4
-    else:
-        priorite = 5
-
-    return service, priorite
-
-def get_smote_strategy(y, max_samples=1000):
-    class_counts = pd.Series(y).value_counts()
-    strategy = {}
-    for cls, count in class_counts.items():
-        target = min(max_samples, max(count * 2, 100))  # Ensure reasonable class sizes
-    return strategy
-
-def train_priority_model():
-    global priority_model, priority_scaler, priority_imputer
-    try:
-        data = pd.read_csv(DATASET_PATH)
-        data['Sexe'] = data['Sexe'].map({'Masculin': 0, 'Feminin': 1})
-        data['Enceinte'] = data['Enceinte'].astype(int)
-        data['ECG'] = data['ECG'].map({'Normal': 0, 'Anormal': 1})
-        data = enhanced_features(data)
-        data[['Suggested_Service', 'Suggested_Priority']] = data.apply(compute_service_and_priority, axis=1, result_type='expand')
-        data['Suggested_Priority'] = data['Suggested_Priority'].astype(int)
-
-        X = data[PRIORITY_FEATURES]
-        y = data['Priorite'].values - 1  # Shift to 0-based indexing
-
-        priority_imputer = SimpleImputer(strategy='median')
-        X_imputed = priority_imputer.fit_transform(X)
-        priority_scaler = StandardScaler()
-        X_scaled = priority_scaler.fit_transform(X_imputed)
-
-        models = {
-            'XGBoost': XGBClassifier(n_estimators=100, max_depth=4, learning_rate=0.05, n_jobs=-1, random_state=42),
-            'LightGBM': LGBMClassifier(n_estimators=100, max_depth=2, learning_rate=0.05, min_child_samples=5, 
-                                      reg_alpha=0.5, reg_lambda=0.5, n_jobs=-1, random_state=42, verbose=-1),
-            'RandomForest': RandomForestClassifier(n_estimators=100, max_depth=8, n_jobs=-1, random_state=42),
-            'LogisticRegression': LogisticRegression(max_iter=1000, multi_class='multinomial', random_state=42),
-            'SVM': SVC(probability=True, random_state=42)
-        }
-
-        skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)
-        results = {}
-
-        for name, model in models.items():
-            logger.info(f"\nEvaluating {name} for Priority...")
-            scores = {'f1': [], 'recall_p1': [], 'time': []}
-            for train_idx, test_idx in tqdm(skf.split(X_scaled, y), total=5):
-                X_train, X_test = X_scaled[train_idx], X_scaled[test_idx]
-                y_train, y_test = y[train_idx], y[test_idx]
-                
-                min_class_size = pd.Series(y_train).value_counts().min()
-                k_neighbors = min(5, max(1, min_class_size - 1))
-                pipeline = Pipeline([
-                    ('under', RandomUnderSampler(sampling_strategy='majority', random_state=42)),
-                    ('over', SMOTE(sampling_strategy=get_smote_strategy(y_train), random_state=42, k_neighbors=k_neighbors))
-                ])
-                X_train_res, y_train_res = pipeline.fit_resample(X_train, y_train)
-                class_sizes = pd.Series(y_train_res).value_counts().to_dict()
-                logger.info(f"{name} - Resampled class sizes: {class_sizes}")
-                
-                start_time = time.time()
-                model.fit(X_train_res, y_train_res)
-                train_time = time.time() - start_time
-                
-                y_pred = model.predict(X_test)
-                scores['f1'].append(f1_score(y_test, y_pred, average='macro'))
-                scores['recall_p1'].append(recall_score(y_test, y_pred, labels=[0], average=None, zero_division=0)[0])
-                scores['time'].append(train_time)
-                logger.info(f"{name} Fold - F1: {scores['f1'][-1]:.3f}, Recall P1: {scores['recall_p1'][-1]:.3f}")
-
-            results[name] = {
-                'f1': np.mean(scores['f1']),
-                'recall_p1': np.mean(scores['recall_p1']),
-                'time': np.mean(scores['time'])
-            }
-            if name == 'LightGBM':
-                feature_importance = pd.Series(model.feature_importances_, index=PRIORITY_FEATURES).sort_values(ascending=False)
-                logger.info(f"LightGBM Priority Feature Importance:\n{feature_importance}")
-
-        logger.info("\nPriority Model Comparison:")
-        for name, res in results.items():
-            logger.info(f"{name}: F1={res['f1']:.3f}, Recall P1={res['recall_p1']:.3f}, Time={res['time']:.2f}s")
-
-        best_model = max(results, key=lambda k: results[k]['f1'] + results[k]['recall_p1'])
-        logger.info(f"Best Priority Model: {best_model}")
-
-        with model_lock:
-            priority_model = models[best_model]
-            priority_model.fit(X_scaled, y)
-
-        timestamp = int(time.time())
-        joblib.dump(priority_model, f'priority_model_{timestamp}.pkl')
-        joblib.dump(priority_scaler, 'priority_scaler.pkl')
-        joblib.dump(priority_imputer, 'priority_imputer.pkl')
-        logger.info("Priority model saved.")
-    except Exception as e:
-        logger.error(f"Error in priority training: {e}")
-        raise
-
-def train_service_model():
-    global service_model, service_scaler, service_imputer, label_encoder_service
-    try:
-        data = pd.read_csv(DATASET_PATH)
-        data['Sexe'] = data['Sexe'].map({'Masculin': 0, 'Feminin': 1})
-        data['Enceinte'] = data['Enceinte'].astype(int)
-        data['ECG'] = data['ECG'].map({'Normal': 0, 'Anormal': 1})
-        data = enhanced_features(data)
-        data[['Suggested_Service', 'Suggested_Priority']] = data.apply(compute_service_and_priority, axis=1, result_type='expand')
-
-        X = data[SERVICE_FEATURES]
-        y = label_encoder_service.fit_transform(data['Service_Suivant'].fillna('Unknown'))
-
-        service_imputer = SimpleImputer(strategy='median')
-        X_imputed = service_imputer.fit_transform(X)
-        service_scaler = StandardScaler()
-        X_scaled = service_scaler.fit_transform(X_imputed)
-
-        models = {
-            'XGBoost': XGBClassifier(n_estimators=100, max_depth=4, learning_rate=0.05, n_jobs=-1, random_state=42),
-            'LightGBM': LGBMClassifier(n_estimators=100, max_depth=2, learning_rate=0.05, min_child_samples=5, 
-                                      reg_alpha=0.5, reg_lambda=0.5, n_jobs=-1, random_state=42, verbose=-1),
-            'RandomForest': RandomForestClassifier(n_estimators=100, max_depth=8, n_jobs=-1, random_state=42),
-            'LogisticRegression': LogisticRegression(max_iter=1000, multi_class='multinomial', random_state=42),
-            'SVM': SVC(probability=True, random_state=42)
-        }
-
-        skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)
-        results = {}
-
-        for name, model in models.items():
-            logger.info(f"\nEvaluating {name} for Service...")
-            scores = {'f1': [], 'time': []}
-            for train_idx, test_idx in tqdm(skf.split(X_scaled, y), total=5):
-                X_train, X_test = X_scaled[train_idx], X_scaled[test_idx]
-                y_train, y_test = y[train_idx], y[test_idx]
-                
-                min_class_size = pd.Series(y_train).value_counts().min()
-                k_neighbors = min(5, max(1, min_class_size - 1))
-                pipeline = Pipeline([
-                    ('under', RandomUnderSampler(sampling_strategy='majority', random_state=42)),
-                    ('over', SMOTE(sampling_strategy=get_smote_strategy(y_train), random_state=42, k_neighbors=k_neighbors))
-                ])
-                X_train_res, y_train_res = pipeline.fit_resample(X_train, y_train)
-                class_sizes = pd.Series(y_train_res).value_counts().to_dict()
-                logger.info(f"{name} - Resampled class sizes: {class_sizes}")
-                
-                start_time = time.time()
-                model.fit(X_train_res, y_train_res)
-                train_time = time.time() - start_time
-                
-                y_pred = model.predict(X_test)
-                scores['f1'].append(f1_score(y_test, y_pred, average='macro'))
-                scores['time'].append(train_time)
-
-            results[name] = {
-                'f1': np.mean(scores['f1']),
-                'time': np.mean(scores['time'])
-            }
-            if name == 'LightGBM':
-                feature_importance = pd.Series(model.feature_importances_, index=SERVICE_FEATURES).sort_values(ascending=False)
-                logger.info(f"LightGBM Service Feature Importance:\n{feature_importance}")
-
-        logger.info("\nService Model Comparison:")
-        for name, res in results.items():
-            logger.info(f"{name}: F1={res['f1']:.3f}, Time={res['time']:.2f}s")
-
-        best_model = max(results, key=lambda k: results[k]['f1'])
-        logger.info(f"Best Service Model: {best_model}")
-
-        with model_lock:
-            service_model = models[best_model]
-            service_model.fit(X_scaled, y)
-
-        timestamp = int(time.time())
-        joblib.dump(service_model, f'service_model_{timestamp}.pkl')
-        joblib.dump(service_scaler, 'service_scaler.pkl')
-        joblib.dump(service_imputer, 'service_imputer.pkl')
-        joblib.dump(label_encoder_service, 'label_encoder_service.pkl')
-        logger.info("Service model saved.")
-    except Exception as e:
-        logger.error(f"Error in service training: {e}")
-        raise
+# Initialiser le logger
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
 
-@retry(wait=wait_fixed(2), stop=stop_after_attempt(3))
-def retrain_models():
+# Fonction pour charger les modèles
+def load_models():
     global priority_model, service_model, priority_scaler, service_scaler, priority_imputer, service_imputer, label_encoder_service
-    while True:
-        time.sleep(3600)
-        if os.path.exists(NEW_DATA_FILE) and os.path.getsize(NEW_DATA_FILE) > 0:
-            try:
-                new_data = pd.read_csv(NEW_DATA_FILE)
-                if len(new_data) >= MIN_NEW_SAMPLES_FOR_RETRAIN:
-                    orig_data = pd.read_csv(DATASET_PATH)
-                    orig_data['Sexe'] = orig_data['Sexe'].map({'Masculin': 0, 'Feminin': 1})
-                    orig_data['Enceinte'] = orig_data['Enceinte'].astype(int)
-                    orig_data['ECG'] = orig_data['ECG'].map({'Normal': 0, 'Anormal': 1})
-                    new_data = enhanced_features(new_data)
-                    combined_data = pd.concat([orig_data, new_data], ignore_index=True)
-
-                    # Priority retraining
-                    X_priority = combined_data[PRIORITY_FEATURES]
-                    y_priority = combined_data['Priorite'].values - 1
-                    X_priority_imputed = priority_imputer.transform(X_priority)
-                    X_priority_scaled = priority_scaler.transform(X_priority_imputed)
-                    with model_lock:
-                        priority_model.fit(X_priority_scaled, y_priority)
-
-                    # Service retraining
-                    X_service = combined_data[SERVICE_FEATURES]
-                    y_service = label_encoder_service.transform(combined_data['Service_Suivant'].fillna('Unknown'))
-                    X_service_imputed = service_imputer.transform(X_service)
-                    X_service_scaled = service_scaler.transform(X_service_imputed)
-                    with model_lock:
-                        service_model.fit(X_service_scaled, y_service)
-
-                    timestamp = int(time.time())
-                    joblib.dump(priority_model, f'priority_model_{timestamp}.pkl')
-                    joblib.dump(service_model, f'service_model_{timestamp}.pkl')
-                    new_data.to_csv(f'archive_new_data_{timestamp}.csv', index=False)
-                    open(NEW_DATA_FILE, 'w').close()
-                    logger.info("Models retrained and saved.")
-            except Exception as e:
-                logger.error(f"Error in retrain: {e}")
-
+    if os.path.exists(PRIORITY_MODEL_PATH):
+        priority_model = joblib.load(PRIORITY_MODEL_PATH)
+    if os.path.exists(SERVICE_MODEL_PATH):
+        service_model = joblib.load(SERVICE_MODEL_PATH)
+    priority_scaler = joblib.load('priority_scaler.pkl')
+    service_scaler = joblib.load('service_scaler.pkl')
+    priority_imputer = joblib.load('priority_imputer.pkl')
+    service_imputer = joblib.load('service_imputer.pkl')
+    label_encoder_service = joblib.load('label_encoder_service.pkl')
+
+# Charger les modèles au démarrage
+load_models()
+
+# Fonctions et routes Flask
 @app.route('/predict', methods=['POST'])
 def predict():
-    global priority_model, service_model, priority_scaler, service_scaler, priority_imputer, service_imputer, label_encoder_service
+    global priority_model, service_model
+    if priority_model is None or service_model is None:
+        load_models()
     try:
         data = request.get_json()
         required_fields = ['age', 'sexe', 'enceinte', 'spo2', 'freq_resp', 'pouls', 'ecg', 'pa', 'temperature', 'imc']
@@ -421,20 +138,4 @@ def predict():
         return jsonify({'error': str(e)}), 500
 
 if __name__ == '__main__':
-    FORCE_RETRAIN = True
-    if FORCE_RETRAIN or not (os.path.exists('priority_model.pkl') and os.path.exists('service_model.pkl')):
-        train_priority_model()
-        train_service_model()
-    else:
-        with model_lock:
-            priority_model = joblib.load('priority_model.pkl')
-            service_model = joblib.load('service_model.pkl')
-            priority_scaler = joblib.load('priority_scaler.pkl')
-            service_scaler = joblib.load('service_scaler.pkl')
-            priority_imputer = joblib.load('priority_imputer.pkl')
-            service_imputer = joblib.load('service_imputer.pkl')
-            label_encoder_service = joblib.load('label_encoder_service.pkl')
-
-    retrain_thread = threading.Thread(target=retrain_models, daemon=True)
-    retrain_thread.start()
-    app.run(debug=False, host='0.0.0.0', port=5000)
+    app.run(debug=False, host='0.0.0.0', port=5000)