Ajouter des dépendances

app.py

@@ -1,4 +1,3 @@
-<<<<<<< HEAD
 import pandas as pd
 import numpy as np
 from xgboost import XGBClassifier
@@ -438,445 +437,4 @@ if __name__ == '__main__':
 
     retrain_thread = threading.Thread(target=retrain_models, daemon=True)
     retrain_thread.start()
-
-import pandas as pd
-import numpy as np
-from xgboost import XGBClassifier
-from lightgbm import LGBMClassifier
-from sklearn.ensemble import RandomForestClassifier
-from sklearn.linear_model import LogisticRegression
-from sklearn.svm import SVC
-from sklearn.preprocessing import StandardScaler, LabelEncoder
-from sklearn.model_selection import StratifiedKFold
-from sklearn.metrics import classification_report, recall_score, f1_score
-from sklearn.impute import SimpleImputer
-from imblearn.over_sampling import SMOTE
-from imblearn.under_sampling import RandomUnderSampler
-from imblearn.pipeline import Pipeline
-import joblib
-from flask import Flask, request, jsonify
-from flask_cors import CORS
-import os
-import warnings
-import time
-from tqdm import tqdm
-import threading
-import logging
-from tenacity import retry, wait_fixed, stop_after_attempt
-
-warnings.filterwarnings('ignore', category=UserWarning)
-os.environ["LOKY_MAX_CPU_COUNT"] = "1"
-
-logging.basicConfig(level=logging.INFO)
-logger = logging.getLogger(__name__)
-
-app = Flask(__name__)
-CORS(app)
-
-NEW_DATA_FILE = 'new_data.csv'
-DATASET_PATH = "my_datasheet_80000.csv"
-MIN_NEW_SAMPLES_FOR_RETRAIN = 100
-
-# Feature sets for each task
-PRIORITY_FEATURES = [
-    'SpO2', 'Frquce_Rprtr(rpm)', 'Pouls', 'PA', 'Temperature', 'SpO2_Severity', 'Tachypnea', 'Bradypnea',
-    'Tachycardia', 'Bradycardia', 'Critical_Signs', 'SpO2_Temp_Ratio', 'Pouls_PA_Ratio', 'Temp_Pouls_Ratio',
-    'SpO2_PA_Diff', 'SpO2_Temp_Diff', 'PA_Pouls_Diff', 'SpO2_Log', 'Temp_Squared', 'Suggested_Priority'
-]
-
-SERVICE_FEATURES = [
-    'Age', 'Sexe', 'Enceinte', 'SpO2', 'Frquce_Rprtr(rpm)', 'Pouls', 'ECG', 'PA', 'Temperature', 'IMC',
-    'Age_Category', 'Temp_Anomaly', 'PA_High', 'PA_Low', 'Pouls_SpO2_Ratio', 'PA_Temp_Ratio', 'IMC_Temp_Ratio'
-]
-
-priority_model = None
-service_model = None
-priority_scaler = None
-service_scaler = None
-priority_imputer = None
-service_imputer = None
-label_encoder_service = LabelEncoder()
-
-model_lock = threading.Lock()
-
-def enhanced_features(df):
-    df['Tachypnea'] = df.apply(lambda row: 1 if (row['Age'] < 1 and row['Frquce_Rprtr(rpm)'] > 40) or 
-                                         (row['Age'] < 12 and row['Frquce_Rprtr(rpm)'] > 30) or 
-                                         (row['Age'] >= 12 and row['Frquce_Rprtr(rpm)'] > 20) else 0, axis=1)
-    df['Bradypnea'] = df.apply(lambda row: 1 if (row['Age'] < 1 and row['Frquce_Rprtr(rpm)'] < 20) or 
-                                         (row['Age'] < 12 and row['Frquce_Rprtr(rpm)'] < 12) or 
-                                         (row['Age'] >= 12 and row['Frquce_Rprtr(rpm)'] < 8) else 0, axis=1)
-    df['Tachycardia'] = df.apply(lambda row: 1 if (row['Age'] < 1 and row['Pouls'] > 160) or 
-                                           (row['Age'] < 12 and row['Pouls'] > 120) or 
-                                           (row['Age'] >= 12 and row['Pouls'] > 100) else 0, axis=1)
-    df['Bradycardia'] = df.apply(lambda row: 1 if (row['Age'] < 1 and row['Pouls'] < 90) or 
-                                           (row['Age'] < 12 and row['Pouls'] < 70) or 
-                                           (row['Age'] >= 12 and row['Pouls'] < 50) else 0, axis=1)
-    df['SpO2_Temp_Ratio'] = df['SpO2'] / (df['Temperature'] + 1e-6)
-    df['Pouls_PA_Ratio'] = df['Pouls'] / (df['PA'] + 1e-6)
-    df['Temp_Pouls_Ratio'] = df['Temperature'] / (df['Pouls'] + 1e-6)
-    df['SpO2_PA_Diff'] = df['SpO2'] - df['PA'] / 10
-    df['SpO2_Temp_Diff'] = df['SpO2'] - df['Temperature']
-    df['PA_Pouls_Diff'] = df['PA'] - df['Pouls']
-    df['IMC_Temp_Ratio'] = df['IMC'] / (df['Temperature'] + 1e-6)
-    df['SpO2_Log'] = np.log1p(df['SpO2'])
-    df['Temp_Squared'] = df['Temperature'] ** 2
-    df['Pouls_SpO2_Ratio'] = df['Pouls'] / (df['SpO2'] + 1e-6)
-    df['PA_Temp_Ratio'] = df['PA'] / (df['Temperature'] + 1e-6)
-    df['Age_Category'] = pd.cut(df['Age'], bins=[0, 1, 12, 45, 65, 120], labels=[0, 1, 2, 3, 4])
-    df['Temp_Anomaly'] = df['Temperature'].apply(lambda x: 1 if x < 35 or x > 38 else 0)
-    df['PA_High'] = df['PA'].apply(lambda x: 1 if x > 160 else 0)
-    df['PA_Low'] = df['PA'].apply(lambda x: 1 if x < 90 else 0)
-    df['SpO2_Severity'] = pd.cut(df['SpO2'], bins=[0, 85, 90, 92, 100], labels=[3, 2, 1, 0])
-    df['Critical_Signs'] = ((df['SpO2'] < 85) | (df['Pouls'] > 150) | (df['Temperature'] > 40) | 
-                            (df['PA'] > 200) | (df['PA'] < 70)).astype(int)
-    return df
-
-def compute_service_and_priority(row):
-    age = row['Age']
-    spO2 = row['SpO2']
-    frq_resp = row['Frquce_Rprtr(rpm)']
-    pouls = row['Pouls']
-    ecg = row['ECG']
-    pa = row['PA']
-    temp = row['Temperature']
-    enceinte = row['Enceinte']
-    imc = row['IMC']
-
-    if age <= 18:
-        service = 'Pédiatriques'
-    elif enceinte:
-        service = 'Gynécologie/Obstétrique'
-    elif ecg == 1 or (pouls < 50 or pouls > 110) or (frq_resp > 20):
-        service = 'Neurologie'
-    elif spO2 < 92 or frq_resp > 18 or pouls > 100 or pa < 90 or pa > 160:
-        service = 'Cardiorespiratoire'
-    elif (imc > 30 and (temp > 38 and temp <= 40) and 70 <= pouls <= 90) or \
-         (70 <= pouls <= 90 and 110 <= pa <= 130 and spO2 >= 97 and temp <= 37.5):
-        service = 'Médecine générale'
-    elif temp > 40:
-        service = 'Radiothérapie'
-    else:
-        service = 'Chirurgie'
-
-    if spO2 < 85 or temp > 40 or pouls > 150 or pa < 70 or pa > 200:
-        priorite = 1
-    elif spO2 < 88 or temp > 39.5 or pouls > 130 or pa < 80 or pa > 180 or frq_resp > 25:
-        priorite = 2
-    elif spO2 < 90 or temp > 38.5 or pouls > 110 or pa < 90 or pa > 160 or frq_resp > 20:
-        priorite = 3
-    elif spO2 < 92 or temp > 38 or pouls > 100 or pa < 100 or pa > 140 or frq_resp > 18:
-        priorite = 4
-    else:
-        priorite = 5
-
-    return service, priorite
-
-def get_smote_strategy(y, max_samples=1000):
-    class_counts = pd.Series(y).value_counts()
-    strategy = {}
-    for cls, count in class_counts.items():
-        target = min(max_samples, max(count * 2, 100))  # Ensure reasonable class sizes
-    return strategy
-
-def train_priority_model():
-    global priority_model, priority_scaler, priority_imputer
-    try:
-        data = pd.read_csv(DATASET_PATH)
-        data['Sexe'] = data['Sexe'].map({'Masculin': 0, 'Feminin': 1})
-        data['Enceinte'] = data['Enceinte'].astype(int)
-        data['ECG'] = data['ECG'].map({'Normal': 0, 'Anormal': 1})
-        data = enhanced_features(data)
-        data[['Suggested_Service', 'Suggested_Priority']] = data.apply(compute_service_and_priority, axis=1, result_type='expand')
-        data['Suggested_Priority'] = data['Suggested_Priority'].astype(int)
-
-        X = data[PRIORITY_FEATURES]
-        y = data['Priorite'].values - 1  # Shift to 0-based indexing
-
-        priority_imputer = SimpleImputer(strategy='median')
-        X_imputed = priority_imputer.fit_transform(X)
-        priority_scaler = StandardScaler()
-        X_scaled = priority_scaler.fit_transform(X_imputed)
-
-        models = {
-            'XGBoost': XGBClassifier(n_estimators=100, max_depth=4, learning_rate=0.05, n_jobs=-1, random_state=42),
-            'LightGBM': LGBMClassifier(n_estimators=100, max_depth=2, learning_rate=0.05, min_child_samples=5, 
-                                      reg_alpha=0.5, reg_lambda=0.5, n_jobs=-1, random_state=42, verbose=-1),
-            'RandomForest': RandomForestClassifier(n_estimators=100, max_depth=8, n_jobs=-1, random_state=42),
-            'LogisticRegression': LogisticRegression(max_iter=1000, multi_class='multinomial', random_state=42),
-            'SVM': SVC(probability=True, random_state=42)
-        }
-
-        skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)
-        results = {}
-
-        for name, model in models.items():
-            logger.info(f"\nEvaluating {name} for Priority...")
-            scores = {'f1': [], 'recall_p1': [], 'time': []}
-            for train_idx, test_idx in tqdm(skf.split(X_scaled, y), total=5):
-                X_train, X_test = X_scaled[train_idx], X_scaled[test_idx]
-                y_train, y_test = y[train_idx], y[test_idx]
-                
-                min_class_size = pd.Series(y_train).value_counts().min()
-                k_neighbors = min(5, max(1, min_class_size - 1))
-                pipeline = Pipeline([
-                    ('under', RandomUnderSampler(sampling_strategy='majority', random_state=42)),
-                    ('over', SMOTE(sampling_strategy=get_smote_strategy(y_train), random_state=42, k_neighbors=k_neighbors))
-                ])
-                X_train_res, y_train_res = pipeline.fit_resample(X_train, y_train)
-                class_sizes = pd.Series(y_train_res).value_counts().to_dict()
-                logger.info(f"{name} - Resampled class sizes: {class_sizes}")
-                
-                start_time = time.time()
-                model.fit(X_train_res, y_train_res)
-                train_time = time.time() - start_time
-                
-                y_pred = model.predict(X_test)
-                scores['f1'].append(f1_score(y_test, y_pred, average='macro'))
-                scores['recall_p1'].append(recall_score(y_test, y_pred, labels=[0], average=None, zero_division=0)[0])
-                scores['time'].append(train_time)
-                logger.info(f"{name} Fold - F1: {scores['f1'][-1]:.3f}, Recall P1: {scores['recall_p1'][-1]:.3f}")
-
-            results[name] = {
-                'f1': np.mean(scores['f1']),
-                'recall_p1': np.mean(scores['recall_p1']),
-                'time': np.mean(scores['time'])
-            }
-            if name == 'LightGBM':
-                feature_importance = pd.Series(model.feature_importances_, index=PRIORITY_FEATURES).sort_values(ascending=False)
-                logger.info(f"LightGBM Priority Feature Importance:\n{feature_importance}")
-
-        logger.info("\nPriority Model Comparison:")
-        for name, res in results.items():
-            logger.info(f"{name}: F1={res['f1']:.3f}, Recall P1={res['recall_p1']:.3f}, Time={res['time']:.2f}s")
-
-        best_model = max(results, key=lambda k: results[k]['f1'] + results[k]['recall_p1'])
-        logger.info(f"Best Priority Model: {best_model}")
-
-        with model_lock:
-            priority_model = models[best_model]
-            priority_model.fit(X_scaled, y)
-
-        timestamp = int(time.time())
-        joblib.dump(priority_model, f'priority_model_{timestamp}.pkl')
-        joblib.dump(priority_scaler, 'priority_scaler.pkl')
-        joblib.dump(priority_imputer, 'priority_imputer.pkl')
-        logger.info("Priority model saved.")
-    except Exception as e:
-        logger.error(f"Error in priority training: {e}")
-        raise
-
-def train_service_model():
-    global service_model, service_scaler, service_imputer, label_encoder_service
-    try:
-        data = pd.read_csv(DATASET_PATH)
-        data['Sexe'] = data['Sexe'].map({'Masculin': 0, 'Feminin': 1})
-        data['Enceinte'] = data['Enceinte'].astype(int)
-        data['ECG'] = data['ECG'].map({'Normal': 0, 'Anormal': 1})
-        data = enhanced_features(data)
-        data[['Suggested_Service', 'Suggested_Priority']] = data.apply(compute_service_and_priority, axis=1, result_type='expand')
-
-        X = data[SERVICE_FEATURES]
-        y = label_encoder_service.fit_transform(data['Service_Suivant'].fillna('Unknown'))
-
-        service_imputer = SimpleImputer(strategy='median')
-        X_imputed = service_imputer.fit_transform(X)
-        service_scaler = StandardScaler()
-        X_scaled = service_scaler.fit_transform(X_imputed)
-
-        models = {
-            'XGBoost': XGBClassifier(n_estimators=100, max_depth=4, learning_rate=0.05, n_jobs=-1, random_state=42),
-            'LightGBM': LGBMClassifier(n_estimators=100, max_depth=2, learning_rate=0.05, min_child_samples=5, 
-                                      reg_alpha=0.5, reg_lambda=0.5, n_jobs=-1, random_state=42, verbose=-1),
-            'RandomForest': RandomForestClassifier(n_estimators=100, max_depth=8, n_jobs=-1, random_state=42),
-            'LogisticRegression': LogisticRegression(max_iter=1000, multi_class='multinomial', random_state=42),
-            'SVM': SVC(probability=True, random_state=42)
-        }
-
-        skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)
-        results = {}
-
-        for name, model in models.items():
-            logger.info(f"\nEvaluating {name} for Service...")
-            scores = {'f1': [], 'time': []}
-            for train_idx, test_idx in tqdm(skf.split(X_scaled, y), total=5):
-                X_train, X_test = X_scaled[train_idx], X_scaled[test_idx]
-                y_train, y_test = y[train_idx], y[test_idx]
-                
-                min_class_size = pd.Series(y_train).value_counts().min()
-                k_neighbors = min(5, max(1, min_class_size - 1))
-                pipeline = Pipeline([
-                    ('under', RandomUnderSampler(sampling_strategy='majority', random_state=42)),
-                    ('over', SMOTE(sampling_strategy=get_smote_strategy(y_train), random_state=42, k_neighbors=k_neighbors))
-                ])
-                X_train_res, y_train_res = pipeline.fit_resample(X_train, y_train)
-                class_sizes = pd.Series(y_train_res).value_counts().to_dict()
-                logger.info(f"{name} - Resampled class sizes: {class_sizes}")
-                
-                start_time = time.time()
-                model.fit(X_train_res, y_train_res)
-                train_time = time.time() - start_time
-                
-                y_pred = model.predict(X_test)
-                scores['f1'].append(f1_score(y_test, y_pred, average='macro'))
-                scores['time'].append(train_time)
-
-            results[name] = {
-                'f1': np.mean(scores['f1']),
-                'time': np.mean(scores['time'])
-            }
-            if name == 'LightGBM':
-                feature_importance = pd.Series(model.feature_importances_, index=SERVICE_FEATURES).sort_values(ascending=False)
-                logger.info(f"LightGBM Service Feature Importance:\n{feature_importance}")
-
-        logger.info("\nService Model Comparison:")
-        for name, res in results.items():
-            logger.info(f"{name}: F1={res['f1']:.3f}, Time={res['time']:.2f}s")
-
-        best_model = max(results, key=lambda k: results[k]['f1'])
-        logger.info(f"Best Service Model: {best_model}")
-
-        with model_lock:
-            service_model = models[best_model]
-            service_model.fit(X_scaled, y)
-
-        timestamp = int(time.time())
-        joblib.dump(service_model, f'service_model_{timestamp}.pkl')
-        joblib.dump(service_scaler, 'service_scaler.pkl')
-        joblib.dump(service_imputer, 'service_imputer.pkl')
-        joblib.dump(label_encoder_service, 'label_encoder_service.pkl')
-        logger.info("Service model saved.")
-    except Exception as e:
-        logger.error(f"Error in service training: {e}")
-        raise
-
-@retry(wait=wait_fixed(2), stop=stop_after_attempt(3))
-def retrain_models():
-    global priority_model, service_model, priority_scaler, service_scaler, priority_imputer, service_imputer, label_encoder_service
-    while True:
-        time.sleep(3600)
-        if os.path.exists(NEW_DATA_FILE) and os.path.getsize(NEW_DATA_FILE) > 0:
-            try:
-                new_data = pd.read_csv(NEW_DATA_FILE)
-                if len(new_data) >= MIN_NEW_SAMPLES_FOR_RETRAIN:
-                    orig_data = pd.read_csv(DATASET_PATH)
-                    orig_data['Sexe'] = orig_data['Sexe'].map({'Masculin': 0, 'Feminin': 1})
-                    orig_data['Enceinte'] = orig_data['Enceinte'].astype(int)
-                    orig_data['ECG'] = orig_data['ECG'].map({'Normal': 0, 'Anormal': 1})
-                    new_data = enhanced_features(new_data)
-                    combined_data = pd.concat([orig_data, new_data], ignore_index=True)
-
-                    # Priority retraining
-                    X_priority = combined_data[PRIORITY_FEATURES]
-                    y_priority = combined_data['Priorite'].values - 1
-                    X_priority_imputed = priority_imputer.transform(X_priority)
-                    X_priority_scaled = priority_scaler.transform(X_priority_imputed)
-                    with model_lock:
-                        priority_model.fit(X_priority_scaled, y_priority)
-
-                    # Service retraining
-                    X_service = combined_data[SERVICE_FEATURES]
-                    y_service = label_encoder_service.transform(combined_data['Service_Suivant'].fillna('Unknown'))
-                    X_service_imputed = service_imputer.transform(X_service)
-                    X_service_scaled = service_scaler.transform(X_service_imputed)
-                    with model_lock:
-                        service_model.fit(X_service_scaled, y_service)
-
-                    timestamp = int(time.time())
-                    joblib.dump(priority_model, f'priority_model_{timestamp}.pkl')
-                    joblib.dump(service_model, f'service_model_{timestamp}.pkl')
-                    new_data.to_csv(f'archive_new_data_{timestamp}.csv', index=False)
-                    open(NEW_DATA_FILE, 'w').close()
-                    logger.info("Models retrained and saved.")
-            except Exception as e:
-                logger.error(f"Error in retrain: {e}")
-
-@app.route('/predict', methods=['POST'])
-def predict():
-    global priority_model, service_model, priority_scaler, service_scaler, priority_imputer, service_imputer, label_encoder_service
-    try:
-        data = request.get_json()
-        required_fields = ['age', 'sexe', 'enceinte', 'spo2', 'freq_resp', 'pouls', 'ecg', 'pa', 'temperature', 'imc']
-        missing_fields = [field for field in required_fields if field not in data]
-        if missing_fields:
-            return jsonify({'error': f'Missing fields: {", ".join(missing_fields)}'}), 400
-
-        input_data = {
-            'Age': float(data['age']),
-            'Sexe': 0 if data['sexe'].lower() == 'masculin' else 1,
-            'Enceinte': 1 if bool(data['enceinte']) else 0,
-            'SpO2': float(data['spo2']),
-            'Frquce_Rprtr(rpm)': float(data['freq_resp']),
-            'Pouls': float(data['pouls']),
-            'ECG': 0 if data['ecg'].lower() == 'normal' else 1,
-            'PA': float(data['pa']),
-            'Temperature': float(data['temperature']),
-            'IMC': float(data['imc']),
-        }
-
-        input_df = pd.DataFrame([input_data])
-        input_df = enhanced_features(input_df)
-        suggested_service, suggested_priority = compute_service_and_priority(input_df.iloc[0])
-        input_df['Suggested_Priority'] = suggested_priority
-
-        with model_lock:
-            # Priority prediction
-            priority_input = input_df[PRIORITY_FEATURES]
-            priority_imputed = priority_imputer.transform(priority_input)
-            priority_scaled = priority_scaler.transform(priority_imputed)
-            priority_probs = priority_model.predict_proba(priority_scaled)[0]
-            priority_pred = np.argmax(priority_probs) + 1
-            priority_conf = float(max(priority_probs))
-
-            # Service prediction
-            service_input = input_df[SERVICE_FEATURES]
-            service_imputed = service_imputer.transform(service_input)
-            service_scaled = service_scaler.transform(service_imputed)
-            service_probs = service_model.predict_proba(service_scaled)[0]
-            service_pred_idx = np.argmax(service_probs)
-            service_pred = label_encoder_service.inverse_transform([service_pred_idx])[0]
-            service_conf = float(max(service_probs))
-
-            # Fallback to rule-based logic if confidence is low or critical conditions apply
-            if priority_conf < 0.7 or input_df['Critical_Signs'][0] == 1:
-                priority_pred = suggested_priority
-            if service_conf < 0.7 or input_df['Enceinte'][0] == 1:
-                service_pred = suggested_service if input_df['Enceinte'][0] == 0 else 'Gynécologie/Obstétrique'
-
-        input_df['Priorite'] = priority_pred
-        input_df['Service_Suivant'] = service_pred
-        if not os.path.exists(NEW_DATA_FILE):
-            input_df.to_csv(NEW_DATA_FILE, index=False)
-        else:
-            input_df.to_csv(NEW_DATA_FILE, mode='a', header=False, index=False)
-
-        logger.info(f"Predicted: service={service_pred}, priority={priority_pred}, service_conf={service_conf}, priority_conf={priority_conf}")
-        return jsonify({
-            'priority': int(priority_pred),
-            'service_suivant': service_pred,
-            'priority_confidence': priority_conf,
-            'service_confidence': service_conf
-        })
-    except Exception as e:
-        logger.error(f"Prediction error: {str(e)}")
-        return jsonify({'error': str(e)}), 500
-
-if __name__ == '__main__':
-    FORCE_RETRAIN = True
-    if FORCE_RETRAIN or not (os.path.exists('priority_model.pkl') and os.path.exists('service_model.pkl')):
-        train_priority_model()
-        train_service_model()
-    else:
-        with model_lock:
-            priority_model = joblib.load('priority_model.pkl')
-            service_model = joblib.load('service_model.pkl')
-            priority_scaler = joblib.load('priority_scaler.pkl')
-            service_scaler = joblib.load('service_scaler.pkl')
-            priority_imputer = joblib.load('priority_imputer.pkl')
-            service_imputer = joblib.load('service_imputer.pkl')
-            label_encoder_service = joblib.load('label_encoder_service.pkl')
-
-    retrain_thread = threading.Thread(target=retrain_models, daemon=True)
-    retrain_thread.start()
->>>>>>> 12fbcdcf1e034f735bed38d79600e83ccc29f849
     app.run(debug=False, host='0.0.0.0', port=5000)