diff --git a/electron-builder.yml b/electron-builder.yml index 519fea98..2399a4fb 100644 --- a/electron-builder.yml +++ b/electron-builder.yml @@ -37,6 +37,10 @@ extraResources: # Add this section if it doesn't exist, or add the line to it - "**/*" - "!**/.git" - "!**/.git/**" + - from: ./baseFiles # Source directory + to: baseFiles # Destination directory inside Contents/Resources + filter: + - emptyNotebook.ipynb extends: null # Added from package.json mac: # Added from package.json diff --git a/go_server/src/utils.go b/go_server/src/utils.go index 750df4bb..c682c641 100644 --- a/go_server/src/utils.go +++ b/go_server/src/utils.go @@ -163,14 +163,6 @@ func StartPythonScripts(jsonParam string, filename string, id string) (string, e } log.Println("Conda env: " + condaEnv) - // UNCOMMENT TO WRITE JSON PARAM TO FILE FOR DEBUGGING - // jsonParamBytes := []byte(jsonParam) - // err = os.WriteFile("jsonParam.txt", jsonParamBytes, 0644) - // if err != nil { - // log.Println("Error writing jsonParam to file") - // return "", err - // } - Scripts[id] = ScriptInfo{ Cmd: exec.Command(condaEnv, "-u", script, "--json-param", jsonParam, "--id", id), Progress: "", @@ -178,15 +170,13 @@ func StartPythonScripts(jsonParam string, filename string, id string) (string, e stdout, err := Scripts[id].Cmd.StdoutPipe() Mu.Unlock() if err != nil { - log.Println("Error getting stdout pipe") - log.Println(err.Error()) + log.Println("Error getting stdout pipe: " + err.Error()) } Mu.Lock() stderr, err := Scripts[id].Cmd.StderrPipe() Mu.Unlock() if err != nil { - log.Println("Error getting stderr pipe") - log.Println(err.Error()) + log.Println("Error getting stderr pipe: " + err.Error()) } Mu.Lock() err = Scripts[id].Cmd.Start() @@ -195,44 +185,130 @@ func StartPythonScripts(jsonParam string, filename string, id string) (string, e log.Println("Error starting command " + Scripts[id].Cmd.String()) return "", err } + response := "" - go copyOutput(stdout, &response) - go copyOutput(stderr, &response) - err = Scripts[id].Cmd.Wait() - if err != nil { - log.Println("Error waiting for command to finish") - return "", err + // stderrBuf collects every line from stderr so we can surface it on early + // crashes (e.g. ModuleNotFoundError) that never reach send_response(). + var stderrBuf strings.Builder + var stderrMu sync.Mutex + + var wg sync.WaitGroup + wg.Add(2) + go func() { + defer wg.Done() + copyOutput(stdout, &response) + }() + go func() { + defer wg.Done() + copyOutputErr(stderr, &response, &stderrBuf, &stderrMu) + }() + wg.Wait() + + waitErr := Scripts[id].Cmd.Wait() + + // Happy path – the Python layer called send_response() successfully. + if response != "" { + log.Println("Finished running script: " + filename + " with id: " + id) + return response, nil } - log.Println("Finished running script: " + filename + " with id: " + id) - return response, nil + + // The script exited without ever emitting response-ready (early crash). + // Build a structured error JSON that matches get_response_from_error() so + // the frontend receives consistent error objects regardless of where in + // Python the failure happened. + stderrMu.Lock() + capturedStderr := stderrBuf.String() + stderrMu.Unlock() + + exitMsg := "unknown error" + if waitErr != nil { + exitMsg = waitErr.Error() + } + if capturedStderr == "" { + capturedStderr = exitMsg + } + + log.Println("Script crashed before send_response. stderr:\n" + capturedStderr) + + errorPayload := map[string]interface{}{ + "error": map[string]string{ + "message": exitMsg, + "stack_trace": capturedStderr, + "value": capturedStderr, + }, + } + jsonBytes, jsonErr := json.Marshal(errorPayload) + if jsonErr != nil { + // Last-resort fallback – should never happen. + return "", waitErr + } + return string(jsonBytes), nil } -// It is used to transfer stdout and stderr to the terminal +// copyOutput handles stdout: detects response-ready and progress signals. func copyOutput(r io.Reader, response *string) { scanner := bufio.NewScanner(r) - lineText := "" for scanner.Scan() { - lineText = scanner.Text() + lineText := scanner.Text() if strings.Contains(lineText, "response-ready*_*") { path := strings.Split(lineText, "*_*")[1] path = path[:len(path)-1] *response = ReadFile(path) - // delete this file - err := os.Remove(path) - if err != nil { + if err := os.Remove(path); err != nil { log.Println(err) } } else if strings.Contains(lineText, "progress*_*") { - id := strings.Split(lineText, "*_*")[1] - progress := strings.Split(lineText, "*_*")[2] - progress = progress[:len(progress)-1] - log.Println("Progress: " + progress) - Mu.Lock() - Scripts[id] = ScriptInfo{ - Cmd: Scripts[id].Cmd, - Progress: progress, + parts := strings.Split(lineText, "*_*") + if len(parts) >= 3 { + pid := parts[1] + progress := parts[2][:len(parts[2])-1] + log.Println("Progress: " + progress) + Mu.Lock() + Scripts[pid] = ScriptInfo{ + Cmd: Scripts[pid].Cmd, + Progress: progress, + } + Mu.Unlock() + } + } + } +} + +// copyOutputErr mirrors copyOutput for stderr. +// It still watches for response-ready/progress (some Python loggers write to +// stderr) and additionally accumulates every line into stderrBuf so early +// crashes (ImportError, SyntaxError, …) are never silently dropped. +func copyOutputErr(r io.Reader, response *string, stderrBuf *strings.Builder, mu *sync.Mutex) { + scanner := bufio.NewScanner(r) + for scanner.Scan() { + lineText := scanner.Text() + log.Println("[stderr] " + lineText) + + // Accumulate for crash reporting. + mu.Lock() + stderrBuf.WriteString(lineText + "\n") + mu.Unlock() + + // Handle structured signals even if they arrive on stderr. + if strings.Contains(lineText, "response-ready*_*") { + path := strings.Split(lineText, "*_*")[1] + path = path[:len(path)-1] + *response = ReadFile(path) + if err := os.Remove(path); err != nil { + log.Println(err) + } + } else if strings.Contains(lineText, "progress*_*") { + parts := strings.Split(lineText, "*_*") + if len(parts) >= 3 { + pid := parts[1] + progress := parts[2][:len(parts[2])-1] + Mu.Lock() + Scripts[pid] = ScriptInfo{ + Cmd: Scripts[pid].Cmd, + Progress: progress, + } + Mu.Unlock() } - Mu.Unlock() } } } @@ -296,8 +372,8 @@ func KillScript(id string) bool { } }() if script.Cmd != nil { // Check if script.Cmd is not nil - if script.Cmd.ProcessState != nil && script.Cmd.ProcessState.Exited() { - log.Println("Script can be killed") + if script.Cmd.Process != nil && (script.Cmd.ProcessState == nil || !script.Cmd.ProcessState.Exited()) { + log.Println("Script is running, killing it now...") err := script.Cmd.Process.Kill() if err != nil { log.Print("Error killing process: ", err.Error()) diff --git a/pythonCode/med_libs/MEDml/MEDexperiment.py b/pythonCode/med_libs/MEDml/MEDexperiment.py index a2ccfb96..8b95d747 100644 --- a/pythonCode/med_libs/MEDml/MEDexperiment.py +++ b/pythonCode/med_libs/MEDml/MEDexperiment.py @@ -199,7 +199,10 @@ def start(self) -> None: for current_node_id, next_nodes_id_json in self.pipelines_to_execute.items(): node_info = self.pipelines_objects[current_node_id] node: Node = node_info['obj'] - self._progress['currentLabel'] = node.username + if node.username.lower() == 'train model': + self._progress['currentLabel'] = f"Training model (This process may take long)" + else: + self._progress['currentLabel'] = node.username has_been_run = node.has_run() if not has_been_run or 'experiment' not in node_info: node_info['results'] = { @@ -283,7 +286,10 @@ def execute_next_nodes( node = node_info['obj'] experiment = self.copy_experiment(experiment) exp_to_return = experiment - self._progress['currentLabel'] = node.username + if node.username.lower() == 'train model': + self._progress['currentLabel'] = f"Training model (This process may take long)" + else: + self._progress['currentLabel'] = node.username if not node.has_run() or prev_node.has_changed(): if node.type == 'group_models': print("group_models") diff --git a/pythonCode/med_libs/MEDml/nodes/ModelHandler.py b/pythonCode/med_libs/MEDml/nodes/ModelHandler.py index 290453cc..851c569f 100644 --- a/pythonCode/med_libs/MEDml/nodes/ModelHandler.py +++ b/pythonCode/med_libs/MEDml/nodes/ModelHandler.py @@ -69,11 +69,6 @@ class ModelHandler(Node): """ def __init__(self, id_: int, global_config_json: json) -> None: - """ - Args: - id_ (int): The id of the node. - global_config_json (json): The global config json. - """ super().__init__(id_, global_config_json) self.model_name_id = None if self.type == 'train_model': @@ -90,6 +85,17 @@ def __init__(self, id_: int, global_config_json: json) -> None: self.optimize_threshold = self.config_json['data']['internal'].get('optimizeThreshold', False) if self.optimize_threshold: self.threshold_optimization_metric = self.config_json['data']['internal'].get('threshOptimizationMetric', 'Accuracy') + # Normalizing metric names for Pycaret + METRIC_NAME_MAP = { + 'Youden': 'Youden Index', + 'BAC': 'Balanced Accuracy', + 'Specificity': 'Specificity', + 'NPV': 'NPV', + } + self.threshold_optimization_metric = METRIC_NAME_MAP.get( + self.threshold_optimization_metric, + self.threshold_optimization_metric + ) self.model_id = self.config_json['associated_id'] model_obj = self.global_config_json['nodes'][self.model_id] self.model_name_id = model_obj['data']['internal'].get('nameID', None) @@ -99,20 +105,14 @@ def __init__(self, id_: int, global_config_json: json) -> None: } def __calculate_all_metrics(self, y_true, y_pred, y_pred_proba=None): - """ - Calculate comprehensive classification metrics manually - """ metrics = {} try: - # Probability-based metrics (if available) if y_pred_proba is not None: try: - # For binary classification if len(np.unique(y_true)) == 2: metrics['AUC'] = round(roc_auc_score(y_true, y_pred_proba), 3) else: - # For multiclass - use one-vs-rest metrics['AUC'] = round(roc_auc_score(y_true, y_pred_proba, multi_class='ovr', average='weighted'), 3) except Exception as e: print(f"Warning: Could not calculate probability metrics: {e}") @@ -120,36 +120,50 @@ def __calculate_all_metrics(self, y_true, y_pred, y_pred_proba=None): else: metrics['AUC'] = "N/A" - # Basic classification metrics metrics['Sensitivity'] = round(recall_score(y_true, y_pred, zero_division=0), 3) metrics['Specificity'] = round(self.specificity(y_true, y_pred), 3) metrics['PPV'] = round(precision_score(y_true, y_pred, zero_division=0), 3) metrics['NPV'] = round(self.npv(y_true, y_pred), 3) metrics['Accuracy'] = round(accuracy_score(y_true, y_pred), 3) metrics['F1'] = round(f1_score(y_true, y_pred, zero_division=0), 3) - - # Additional metrics metrics['MCC'] = round(matthews_corrcoef(y_true, y_pred), 3) except Exception as e: raise ValueError(f"Error calculating metrics: {e}") - print(f"Error calculating metrics: {e}") - # Set default values for all metrics - default_metrics = ["AUC", "Sensitivity", "Specificity", "PPV", "NPV", "Accuracy", "F1", "MCC"] - for metric in default_metrics: - metrics[metric] = "N/A" return metrics + + def __calculate_all_metrics_to_notebook(self): + self.CodeHandler.add_line("code", "def calculate_all_metrics(y_true, y_pred, y_pred_proba=None):", indent=0) + self.CodeHandler.add_line("code", "metrics = {}", indent=1) + self.CodeHandler.add_line("code", "try:", indent=1) + self.CodeHandler.add_line("code", "if y_pred_proba is not None:", indent=2) + self.CodeHandler.add_line("code", "try:", indent=3) + self.CodeHandler.add_line("code", "if len(np.unique(y_true)) == 2:", indent=4) + self.CodeHandler.add_line("code", "metrics['AUC'] = round(roc_auc_score(y_true, y_pred_proba), 3)", indent=5) + self.CodeHandler.add_line("code", "else:", indent=4) + self.CodeHandler.add_line("code", "metrics['AUC'] = round(roc_auc_score(y_true, y_pred_proba, multi_class='ovr', average='weighted'), 3)", indent=5) + self.CodeHandler.add_line("code", "except Exception as e:", indent=3) + self.CodeHandler.add_line("code", "print(f\"Warning: Could not calculate probability metrics: {e}\")", indent=4) + self.CodeHandler.add_line("code", "metrics['AUC'] = \"N/A\"", indent=4) + self.CodeHandler.add_line("code", "metrics['Sensitivity'] = round(recall_score(y_true, y_pred, zero_division=0), 3)", indent=2) + self.CodeHandler.add_line("code", "metrics['Specificity'] = round(specificity(y_true, y_pred), 3)", indent=2) + self.CodeHandler.add_line("code", "metrics['PPV'] = round(precision_score(y_true, y_pred, zero_division=0), 3)", indent=2) + self.CodeHandler.add_line("code", "metrics['NPV'] = round(npv(y_true, y_pred), 3)", indent=2) + self.CodeHandler.add_line("code", "metrics['Accuracy'] = round(accuracy_score(y_true, y_pred), 3)", indent=2) + self.CodeHandler.add_line("code", "metrics['F1'] = round(f1_score(y_true, y_pred, zero_division=0), 3)", indent=2) + self.CodeHandler.add_line("code", "metrics['MCC'] = round(matthews_corrcoef(y_true, y_pred), 3)", indent=2) + self.CodeHandler.add_line("code", "except Exception as e:", indent=1) + self.CodeHandler.add_line("code", "raise ValueError(f\"Error calculating metrics: {e}\")", indent=2) + self.CodeHandler.add_line("code", "return metrics", indent=1) def __calculate_overall_metrics(self, fold_metrics): - """Calculate mean and std of metrics across all folds""" overall_metrics = {} log_metrics = {} if not fold_metrics: return overall_metrics, log_metrics - # Get all metric names from first fold first_fold_metrics = list(fold_metrics.values())[0] for metric_name in first_fold_metrics.keys(): @@ -170,56 +184,107 @@ def __calculate_overall_metrics(self, fold_metrics): return overall_metrics, log_metrics + def __calculate_overall_metrics_to_notebook(self): + self.CodeHandler.add_line("code", "def calculate_overall_metrics(fold_metrics):", indent=0) + self.CodeHandler.add_line("code", "overall_metrics = {}", indent=1) + self.CodeHandler.add_line("code", "log_metrics = {}", indent=1) + self.CodeHandler.add_line("code", "if not fold_metrics:", indent=1) + self.CodeHandler.add_line("code", "return overall_metrics, log_metrics", indent=2) + self.CodeHandler.add_line("code", "first_fold_metrics = list(fold_metrics.values())[0]", indent=1) + self.CodeHandler.add_line("code", "for metric_name in first_fold_metrics.keys():", indent=1) + self.CodeHandler.add_line("code", "metric_values = []", indent=2) + self.CodeHandler.add_line("code", "for _, metrics in fold_metrics.items():", indent=2) + self.CodeHandler.add_line("code", "if metric_name in list(metrics.keys()) and \"N/A\" not in str(metrics[metric_name]):", indent=3) + self.CodeHandler.add_line("code", "metric_values.append(metrics[metric_name])", indent=4) + self.CodeHandler.add_line("code", "if metric_values:", indent=2) + self.CodeHandler.add_line("code", "overall_metrics[metric_name] = { 'mean': round(float(np.mean(metric_values)), 3), 'median': round(float(np.median(metric_values)), 3), 'std': round(float(np.std(metric_values)), 3), 'min': round(float(np.min(metric_values)), 3), 'max': round(float(np.max(metric_values)), 3) }", indent=3) + self.CodeHandler.add_line("code", "log_metrics[metric_name] = overall_metrics[metric_name]['mean']", indent=3) + self.CodeHandler.add_line("code", "return overall_metrics, log_metrics", indent=1) + + def __recalculate_metrics_with_threshold(self, model, X_test, y_test): + """ + Recalculate all metrics using the model's optimized threshold. + Reads 'probability_threshold' (PyCaret 3.x CustomProbabilityThresholdClassifier) + with fallback to 'threshold' for compatibility with other versions. + Used after optimize_threshold to ensure reported metrics reflect the actual threshold. + """ + threshold = getattr(model, 'probability_threshold', getattr(model, 'threshold', 0.5)) + y_proba = model.predict_proba(X_test)[:, 1] if hasattr(model, 'predict_proba') else None + + if y_proba is not None: + y_pred = (y_proba >= threshold).astype(int) + else: + y_pred = model.predict(X_test) + y_proba = None + + metrics = self.__calculate_all_metrics(y_test, y_pred, y_proba) + + # Wrap in mean/std format to match overall_metrics structure + return { + k: { + 'mean': v, + 'median': v, + 'std': 0.0, + 'min': v, + 'max': v, + } for k, v in metrics.items() if v != "N/A" + } + + def __recalculate_metrics_with_threshold_to_notebook(self): + self.CodeHandler.add_line("code", "def recalculate_metrics_with_threshold(model, X_test, y_test):", indent=0) + self.CodeHandler.add_line("code", "threshold = getattr(model, 'probability_threshold', getattr(model, 'threshold', 0.5))", indent=1) + self.CodeHandler.add_line("code", "y_proba = model.predict_proba(X_test)[:, 1] if hasattr(model, 'predict_proba') else None", indent=1) + self.CodeHandler.add_line("code", "if y_proba is not None:", indent=1) + self.CodeHandler.add_line("code", "y_pred = (y_proba >= threshold).astype(int)", indent=2) + self.CodeHandler.add_line("code", "else:", indent=1) + self.CodeHandler.add_line("code", "y_pred = model.predict(X_test)", indent=2) + self.CodeHandler.add_line("code", "y_proba = None if y_proba is None else y_proba", indent=2) + self.CodeHandler.add_line("code", "metrics = calculate_all_metrics(y_test, y_pred, y_proba)", indent=1) + self.CodeHandler.add_line("code", "return {k: {'mean': v, 'median': v, 'std': 0.0, 'min': v, 'max': v} for k, v in metrics.items() if v != \"N/A\"}", indent=1) + def __custom_train_and_evaluate( self, pycaret_exp, folds: list, X_processed: pd.DataFrame, y_processed: pd.Series, - random_state=42, finalize=False, final_setup_kwargs: dict = {}, **ml_settings ) -> None: - """ - Custom function to train and evaluate models using PyCaret's create_model and tune_model functions. - - Args: - pycaret_exp (object): The PyCaret experiment object. - model_id (str): The model ID to train and evaluate. - folds (list): List of fold data for cross-validation. - X_processed (pd.DataFrame): Processed feature data. - y_processed (pd.Series): Processed target data. - random_state (int): Random state for reproducibility. - ml_settings (dict): Additional settings for model training and evaluation. - - Returns: - None - """ + if folds is None: raise ValueError("Folds should not be None. Check the iteration data.") + # Functions required for code generation + self.CodeHandler.add_import("import copy") + self.CodeHandler.add_import("from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score, roc_auc_score, confusion_matrix, matthews_corrcoef") + self.CodeHandler.add_line("md", "Utility functions for metric calculations") + self.specificity_to_notebook() + self.npv_to_notebook() + self.__calculate_all_metrics_to_notebook() + self.__calculate_overall_metrics_to_notebook() + if self.optimize_threshold: + self.__recalculate_metrics_with_threshold_to_notebook() + # Initialization trained_models = [] fold_performances = [] optimization_metric = 'Accuracy' - - # Store all metrics for each fold all_fold_metrics = {} - # Update code handler with parameters + self.CodeHandler.add_line("md", "Training and evaluating models for each fold") self.CodeHandler.add_line("code", "\n# Initializing model training and evaluation") self.CodeHandler.add_line("code", "trained_models = []") self.CodeHandler.add_line("code", "fold_performances = []") self.CodeHandler.add_line("code", f"optimization_metric = '{optimization_metric}'") + self.CodeHandler.add_line("code", "all_fold_metrics = {}") - # Iterate through each fold and train the model for fold_data in folds: fold_num = fold_data['fold'] train_indices = fold_data['train_indices'] test_indices = fold_data['test_indices'] - # Fold data extraction try: X_train_fold = X_processed.iloc[train_indices] y_train_fold = y_processed.iloc[train_indices] @@ -228,10 +293,9 @@ def __custom_train_and_evaluate( except IndexError as e: raise ValueError(f"Index error during fold data extraction on fold {fold_num}: {e}") - # Model Instantiation + # setup a unique experiment for each fold if 'log_experiment' in list(final_setup_kwargs.keys()): del final_setup_kwargs['log_experiment'] - # Create a new Pycaret experiment for each fold if needed fold_exp = copy.deepcopy(pycaret_exp) fold_exp.setup( data=pd.concat([X_train_fold, y_train_fold], axis=1), @@ -240,24 +304,19 @@ def __custom_train_and_evaluate( index=False, **final_setup_kwargs ) - # Use PyCaret's create_model instead of manual instantiation model = fold_exp.create_model(verbose=False, **ml_settings) - - # For logging purposes + + # This is only for logging purposes _ = pycaret_exp.create_model(verbose=False, **ml_settings) - # Model Tuning if self.isTuningEnabled: - # Check if optimization metric is set if 'optimize' in self.settingsTuning and self.settingsTuning['optimize']: optimization_metric = self.settingsTuning['optimize'] - # Check if a custom grid is provided if self.useTuningGrid and self.model_id in list(self.config_json['data']['internal'].keys()) and 'custom_grid' in list(self.config_json['data']['internal'][self.model_id].keys()): raw_grid = self.config_json['data']['internal'][self.model_id]['custom_grid'] self.settingsTuning['custom_grid'] = sanitize_custom_grid(raw_grid) - # Convert hidden_layer_sizes if it is a string if "hidden_layer_sizes" in self.settingsTuning['custom_grid']: val = self.settingsTuning['custom_grid']["hidden_layer_sizes"] if isinstance(val, list): @@ -273,43 +332,36 @@ def __custom_train_and_evaluate( except Exception as e: raise ValueError(f"Invalid tuple format for hidden_layer_sizes: {val}") from e - # Tune the model model = fold_exp.tune_model(model, **self.settingsTuning) - - # Log models params custom_logger = pycaret_exp.get_config('logging_param').loggers[0] custom_logger.log_params(model.get_params()) - # Model Ensembling try: if self.ensembleEnabled: model = fold_exp.ensemble_model(model, **self.settingsEnsemble) except Exception as e: raise ValueError(f"Failed to ensemble model on fold {fold_num}. Error: {e}") - # Model Calibration try: if self.calibrateEnabled: model = fold_exp.calibrate_model(model, **self.settingsCalibrate) except Exception as e: raise ValueError(f"Failed to calibrate model on fold {fold_num}. Error: {e}") - # Testing on the test set for this fold - # Make predictions on the test set - y_pred = model.predict(X_test_fold) y_proba = model.predict_proba(X_test_fold)[:, 1] if hasattr(model, 'predict_proba') else None - # Calculate all metrics manually + if y_proba is not None: + threshold = getattr(model, 'probability_threshold', getattr(model, 'threshold', 0.5)) + y_pred = (y_proba >= threshold).astype(int) + else: + y_pred = model.predict(X_test_fold) + fold_metric_results = self.__calculate_all_metrics(y_test_fold, y_pred, y_proba) - - # Store metrics for this fold all_fold_metrics[fold_num] = fold_metric_results - # Get predictions for probability-based metrics if available if optimization_metric.lower() == 'auc' and hasattr(model, 'predict_proba'): y_pred = model.predict_proba(X_test_fold)[:, 1] - # Store the model and its performance fold_score = check_metric(y_test_fold, pd.Series(y_pred), metric=optimization_metric) fold_performances.append({ 'fold': fold_num, @@ -318,11 +370,13 @@ def __custom_train_and_evaluate( 'score': fold_score, 'test_indices': test_indices }) - - # Store Results for the fold trained_models.append(model) - # Update code handler with training loop + overall_metrics, log_metrics = self.__calculate_overall_metrics(all_fold_metrics) + custom_logger = pycaret_exp.get_config('logging_param').loggers[0] + custom_logger.log_metrics(log_metrics) + + # Code generation self.CodeHandler.add_line("code", f"\n# Training and evaluating models for {len(folds)} folds") self.CodeHandler.add_line("code", f"for fold_data in folds:") self.CodeHandler.add_line("code", f"fold_num = fold_data['fold']", indent=1) @@ -332,84 +386,91 @@ def __custom_train_and_evaluate( self.CodeHandler.add_line("code", f"y_train_fold = y_processed.iloc[train_indices]", indent=1) self.CodeHandler.add_line("code", f"X_test_fold = X_processed.iloc[test_indices]", indent=1) self.CodeHandler.add_line("code", f"y_test_fold = y_processed.iloc[test_indices]", indent=1) + self.CodeHandler.add_line("code", f"# Setup PyCaret experiment for the fold", indent=1) + self.CodeHandler.add_line("code", f"fold_exp = copy.deepcopy(pycaret_exp)", indent=1) + self.CodeHandler.add_line("code", f"fold_exp.setup(data=pd.concat([X_train_fold, y_train_fold], axis=1), test_data=pd.concat([X_test_fold, y_test_fold]," \ + f" axis=1), log_experiment=False, index=False, **{final_setup_kwargs})", indent=1) self.CodeHandler.add_line("code", f"# Create and fit model", indent=1) - self.CodeHandler.add_line("code", f"model = pycaret_exp.create_model(verbose=False, {self.CodeHandler.convert_dict_to_params(ml_settings)})", indent=1) - self.CodeHandler.add_line("code", f"if hasattr(model, 'random_state'):", indent=1) - self.CodeHandler.add_line("code", f"setattr(model, 'random_state', {random_state})", indent=2) - self.CodeHandler.add_line("code", f"model.fit(X_train_fold, y_train_fold)", indent=1) + self.CodeHandler.add_line("code", f"model = fold_exp.create_model(verbose=False, {self.CodeHandler.convert_dict_to_params(ml_settings)})", indent=1) + self.CodeHandler.add_line("code", f"# This is only for logging purposes", indent=1) + self.CodeHandler.add_line("code", f"_ = pycaret_exp.create_model(verbose=False, {self.CodeHandler.convert_dict_to_params(ml_settings)})", indent=1) + if self.isTuningEnabled: + self.CodeHandler.add_line("code", f"model = fold_exp.tune_model(model, {self.CodeHandler.convert_dict_to_params(self.settingsTuning)})", indent=1) + if self.ensembleEnabled: + self.CodeHandler.add_line("code", f"model = fold_exp.ensemble_model(model, {self.CodeHandler.convert_dict_to_params(self.settingsEnsemble)})", indent=1) + if self.calibrateEnabled: + self.CodeHandler.add_line("code", f"model = fold_exp.calibrate_model(model, {self.CodeHandler.convert_dict_to_params(self.settingsCalibrate)})", indent=1) self.CodeHandler.add_line("code", f"# Making predictions on the test set", indent=1) self.CodeHandler.add_line("code", f"y_pred = model.predict(X_test_fold)", indent=1) - self.CodeHandler.add_line("code", f"# Assess performance", indent=1) - self.CodeHandler.add_line("code", f"if optimization_metric.lower() == 'auc' and hasattr(model, 'predict_proba'):", indent=1) - self.CodeHandler.add_line("code", f"y_pred = model.predict_proba(X_test_fold)[:, 1]", indent=2) - self.CodeHandler.add_import("from pycaret.utils.generic import check_metric") - self.CodeHandler.add_line("code", f"fold_score = check_metric(y_test_fold, pd.Series(y_pred), metric=optimization_metric)", indent=1) - self.CodeHandler.add_line("code", f"fold_performances.append({{'fold': fold_num, 'model': model, 'score': fold_score, 'test_indices': test_indices}})", indent=1) + self.CodeHandler.add_line("code", f"# Calculate fold metrics", indent=1) + self.CodeHandler.add_line("code", f"y_proba = model.predict_proba(X_test_fold)[:, 1] if hasattr(model, 'predict_proba') else None", indent=1) + self.CodeHandler.add_line("code", f"fold_metric_results = calculate_all_metrics(y_test_fold, y_pred, y_proba)", indent=1) + self.CodeHandler.add_line("code", f"all_fold_metrics[fold_num] = fold_metric_results", indent=1) self.CodeHandler.add_line("code", f"trained_models.append(model)", indent=1) + self.CodeHandler.add_line("md", "Final Model Training and Overall Metrics Calculation") + self.CodeHandler.add_line("code", f"\n# Calculating overall metrics across folds") + self.CodeHandler.add_line("code", f"overall_metrics, log_metrics = calculate_overall_metrics(all_fold_metrics)", indent=0) + self.CodeHandler.add_line("code", f"print(f\"Overall Metrics: \", overall_metrics)", indent=0) - # Calculate overall metrics across all folds - overall_metrics, log_metrics = self.__calculate_overall_metrics(all_fold_metrics) - - # Manually log metrics - custom_logger = pycaret_exp.get_config('logging_param').loggers[0] - custom_logger.log_metrics(log_metrics) - - # Select the best model based on performance if fold_performances: - # Sort by score (higher is better) and select the best model - best_model = sorted(fold_performances, key=lambda x: x['score'], reverse=True)[0]['model'] - best_exp = sorted(fold_performances, key=lambda x: x['score'], reverse=True)[0]['experiment'] - - # Update code handler with best model selection - self.CodeHandler.add_line("code", "\n# Selecting the best model based on performance") - self.CodeHandler.add_line("code", f"best_model = sorted(fold_performances, key=lambda x: x['score'], reverse=True)[0]['model']") - - # Final evaluation on the entire dataset if needed try: - # Ensure the final model has the same random state - if hasattr(best_model, 'random_state'): - setattr(best_model, 'random_state', random_state) - - # Update code handler - self.CodeHandler.add_line("code", f"setattr(best_model, 'random_state', {random_state})") - - # Final fit on the entire dataset - best_model.fit(X_processed, y_processed) + # Final model: rebuild from the best config: + best_model = pycaret_exp.create_model(**ml_settings) + self.CodeHandler.add_line("code", f"best_model = pycaret_exp.create_model({self.CodeHandler.convert_dict_to_params(ml_settings)})", indent=0) - # Optimize model's threshold if enabled - if self.optimize_threshold: - # Do not optimize if data is multiclass - if len(pycaret_exp.get_config('y').unique()) != 2: - print("Skipping threshold optimization (multiclass not supported).") - - # Do not optimize if ensemble was applied - elif self.ensembleEnabled: - print("Skipping threshold optimization (ensemble not supported).") - - else: - best_model = best_exp.optimize_threshold(best_model, optimize=self.threshold_optimization_metric) - - # Update code handler with final fit - self.CodeHandler.add_line("code", f"best_model.fit(X_processed, y_processed)") + # Tuning if self.isTuningEnabled: + best_model = pycaret_exp.tune_model(best_model, **self.settingsTuning) self.CodeHandler.add_line("code", f"# Tuning model", indent=0) self.CodeHandler.add_line("code", f"best_model = pycaret_exp.tune_model(best_model, {self.CodeHandler.convert_dict_to_params(self.settingsTuning)})", indent=0) + + # Ensembling if self.ensembleEnabled: + best_model = pycaret_exp.ensemble_model(best_model, **self.settingsEnsemble) self.CodeHandler.add_line("code", f"# Ensembling model", indent=0) self.CodeHandler.add_line("code", f"best_model = pycaret_exp.ensemble_model(best_model, {self.CodeHandler.convert_dict_to_params(self.settingsEnsemble)})", indent=0) + + # Calibration if self.calibrateEnabled: + best_model = pycaret_exp.calibrate_model(best_model, **self.settingsCalibrate) self.CodeHandler.add_line("code", f"# Calibrating model", indent=0) self.CodeHandler.add_line("code", f"best_model = pycaret_exp.calibrate_model(best_model, {self.CodeHandler.convert_dict_to_params(self.settingsCalibrate)})", indent=0) + if self.optimize_threshold: - if len(pycaret_exp.get_config('y').unique()) == 2 and not self.ensembleEnabled: + if len(pycaret_exp.get_config('y').unique()) != 2: + print("Skipping threshold optimization (multiclass not supported).") + elif self.ensembleEnabled: + print("Skipping threshold optimization (ensemble not supported).") + else: + best_model = pycaret_exp.optimize_threshold(best_model, optimize=self.threshold_optimization_metric) + self.CodeHandler.add_line( + "md", + f"Optimized threshold: {getattr(best_model, 'probability_threshold', getattr(best_model, 'threshold', 'not found'))}" + ) + # recalculate metrics using the optimized threshold + X_test_final = pycaret_exp.get_config('X_test_transformed') + y_test_final = pycaret_exp.get_config('y_test_transformed') + overall_metrics = self.__recalculate_metrics_with_threshold(best_model, X_test_final, y_test_final) + log_metrics = {k: v['mean'] for k, v in overall_metrics.items()} + custom_logger.log_metrics(log_metrics) + + # Code generation self.CodeHandler.add_line("code", f"# Optimizing model threshold based on {self.threshold_optimization_metric}", indent=0) - self.CodeHandler.add_line("code", f"best_model = pycaret_exp.optimize_threshold(best_model, optimize='{self.threshold_optimization_metric}')", indent=0) + self.CodeHandler.add_line( + "code", + f"best_model = pycaret_exp.optimize_threshold(best_model, optimize='{self.threshold_optimization_metric}')", + indent=0 + ) + self.CodeHandler.add_line("code", f"# Recalculating metrics with optimized threshold", indent=0) + self.CodeHandler.add_line("code", f"X_test_final = pycaret_exp.get_config('X_test_transformed')", indent=0) + self.CodeHandler.add_line("code", f"y_test_final = pycaret_exp.get_config('y_test_transformed')", indent=0) + self.CodeHandler.add_line("code", f"overall_metrics = recalculate_metrics_with_threshold(best_model, X_test_final, y_test_final)", indent=0) + self.CodeHandler.add_line("code", "print(\"Overall Metrics with optimized threshold: \", overall_metrics)", indent=0) - # Finalize the model if finalize: - best_model = best_exp.finalize_model(best_model) + best_model = pycaret_exp.finalize_model(best_model) + self.CodeHandler.add_line("code", f"best_model = pycaret_exp.finalize_model(best_model)", indent=0) - # Store the final model self.CodeHandler.add_line("code", f"trained_models = [best_model]") return {'model': best_model, 'overall_metrics': overall_metrics} except Exception as e: @@ -418,40 +479,23 @@ def __custom_train_and_evaluate( raise ValueError("No fold performances were recorded. Check the training process.") def __handle_splitted_data(self, experiment: dict, settings: dict, **kwargs) -> None: - """ - Trains and evaluates models using PyCaret's create_model and tune_model functions based on user-defined splits. - - Args: - experiment (dict): The experiment dictionary containing the PyCaret experiment object. - settings (dict): The settings for the model training and evaluation. - **kwargs: Additional arguments including dataset and iteration_result. - Returns: - None - """ - - # Initialize variables final_setup_kwargs = kwargs.get("final_setup_kwargs", {}) iteration_data = kwargs["split_indices"] pycaret_exp = experiment['pycaret_exp'] - random_state = kwargs.get("random_state", 42) finalize = kwargs.get("finalize", False) overall_metrics = {} - # Setup models to train and evaluate try: if self.type != 'train_model': raise ValueError(f"Something went wrong, the type of the node is {self.type}, but it should be 'train_model'.") - # For train_model, we can use the model_id from the configuration settings.update(self.config_json['data']['estimator']['settings']) import ast - # Convert hidden_layer_sizes if it is a string if "hidden_layer_sizes" in settings: val = settings["hidden_layer_sizes"] if isinstance(val, str) and val.startswith("(") and val.endswith(")"): - print("Converting hidden_layer_sizes:", val) try: settings["hidden_layer_sizes"] = ast.literal_eval(val) except Exception as e: @@ -467,29 +511,23 @@ def __handle_splitted_data(self, experiment: dict, settings: dict, **kwargs) -> split_type = iteration_data['type'] folds = iteration_data['folds'] - # Iterate through models to train and evaluate trained_model = None if split_type == "cross_validation": - # Check if estimator is already set in settings if 'estimator' in settings: if model_to_evaluate != settings['estimator']: - raise ValueError(f"Model ID {model_to_evaluate} does not match the estimator in settings. Please check your configuration.") + raise ValueError(f"Model ID {model_to_evaluate} does not match the estimator in settings.") else: settings['estimator'] = model_to_evaluate self.CodeHandler.add_line("code", f"# Training model: {model_to_evaluate}") - # Use PyCaret's create_model instead of manual instantiation trained_model = pycaret_exp.create_model(**settings) self.CodeHandler.add_line("code", f"trained_models = [pycaret_exp.create_model({self.CodeHandler.convert_dict_to_params(settings)})]") - # tune model if enabled if self.isTuningEnabled: - # Check if a custom grid is provided if self.useTuningGrid and self.model_id in list(self.config_json['data']['internal'].keys()) and 'custom_grid' in list(self.config_json['data']['internal'][self.model_id].keys()): raw_grid = self.config_json['data']['internal'][self.model_id]['custom_grid'] self.settingsTuning['custom_grid'] = sanitize_custom_grid(raw_grid) - # Convert hidden_layer_sizes if it is a string if "hidden_layer_sizes" in self.settingsTuning['custom_grid']: val = self.settingsTuning['custom_grid']["hidden_layer_sizes"] if isinstance(val, list): @@ -505,7 +543,6 @@ def __handle_splitted_data(self, experiment: dict, settings: dict, **kwargs) -> except Exception as e: raise ValueError(f"Invalid tuple format for hidden_layer_sizes: {val}") from e - # Tune the model try: trained_model = pycaret_exp.tune_model(trained_model, **self.settingsTuning) except Exception as e: @@ -525,28 +562,19 @@ def __handle_splitted_data(self, experiment: dict, settings: dict, **kwargs) -> f"trained_models = [pycaret_exp.tune_model('{self.config_json['data']['estimator']['type']}', optimize='{self.settingsTuning.get('optimize','Accuracy')}')]" ) - # Ensemble model if enabled if self.ensembleEnabled: trained_model = pycaret_exp.ensemble_model(trained_model, **self.settingsEnsemble) self.CodeHandler.add_line("code", f"trained_models = [pycaret_exp.ensemble_model(trained_models[0], {self.CodeHandler.convert_dict_to_params(self.settingsEnsemble)})]") - # Calibrate model if enabled if self.calibrateEnabled: trained_model = pycaret_exp.calibrate_model(trained_model, **self.settingsCalibrate) self.CodeHandler.add_line("code", f"trained_models = [pycaret_exp.calibrate_model(trained_models[0], {self.CodeHandler.convert_dict_to_params(self.settingsCalibrate)})]") - # Optimize model's threshold if enabled - # Optimize threshold only for binary, non-calibrated, non-ensemble models if self.optimize_threshold: - - # Do not optimize if data is multiclass if len(pycaret_exp.get_config('y').unique()) != 2: print("Skipping threshold optimization (multiclass not supported).") - - # Do not optimize if ensemble was applied elif self.ensembleEnabled: print("Skipping threshold optimization (ensemble not supported).") - else: trained_model = pycaret_exp.optimize_threshold( trained_model, @@ -556,13 +584,18 @@ def __handle_splitted_data(self, experiment: dict, settings: dict, **kwargs) -> "code", f"trained_models = [pycaret_exp.optimize_threshold(trained_models[0], optimize='{self.threshold_optimization_metric}')]" ) + self.CodeHandler.add_line( + "md", + f"Optimized threshold: {getattr(trained_model, 'probability_threshold', getattr(trained_model, 'threshold', 'not found'))}" + ) if finalize: trained_model = pycaret_exp.finalize_model(trained_model) + self.CodeHandler.add_line("code", f"trained_models = [pycaret_exp.finalize_model(trained_models[0])]") - # Get final metrics dict + # Get final metrics from pycaret_exp.pull() (pre-threshold metrics) final_metrics = pycaret_exp.pull().to_dict(orient='records') - for fold in final_metrics[:-2]: # Exclude 'Mean' and 'Std' rows + for fold in final_metrics[:-2]: for metric in list(fold.keys()): if metric not in list(overall_metrics.keys()): overall_metrics[metric] = [] @@ -576,24 +609,34 @@ def __handle_splitted_data(self, experiment: dict, settings: dict, **kwargs) -> 'min': round(float(np.min(overall_metrics[metric])), 3), 'max': round(float(np.max(overall_metrics[metric])), 3), } + + # if threshold was optimized, recalculate metrics using the optimized threshold + # __recalculate_metrics_with_threshold now reads probability_threshold correctly + if self.optimize_threshold and len(pycaret_exp.get_config('y').unique()) == 2 and not self.ensembleEnabled: + X_test_final = pycaret_exp.get_config('X_test_transformed') + y_test_final = pycaret_exp.get_config('y_test_transformed') + overall_metrics = self.__recalculate_metrics_with_threshold(trained_model, X_test_final, y_test_final) + + # Code generation + self.CodeHandler.add_line("code", f"# Recalculating overall metrics with optimized threshold", indent=0) + self.CodeHandler.add_line("code", f"X_test_final = pycaret_exp.get_config('X_test_transformed')", indent=0) + self.CodeHandler.add_line("code", f"y_test_final = pycaret_exp.get_config('y_test_transformed')", indent=0) + self.CodeHandler.add_line("code", f"overall_metrics = recalculate_metrics_with_threshold(trained_models[0], X_test_final, y_test_final)", indent=0) + return {'model': trained_model, 'overall_metrics': overall_metrics} else: - # Retrieve processed data from PyCaret X_processed = pycaret_exp.get_config('X_transformed') y_processed = pycaret_exp.get_config('y_transformed') - # Update code handler self.CodeHandler.add_line("code", "# Retrieve processed data from PyCaret") self.CodeHandler.add_line("code", f"X_processed = pycaret_exp.get_config('X_transformed')") self.CodeHandler.add_line("code", f"y_processed = pycaret_exp.get_config('y_transformed')") - # Custom training and evaluation function results = self.__custom_train_and_evaluate( pycaret_exp, folds, X_processed, y_processed, - random_state, finalize, final_setup_kwargs, **settings @@ -609,6 +652,14 @@ def specificity(self, y_true, y_pred): return tn / (tn + fp) if (tn + fp) > 0 else 0 return 0 + def specificity_to_notebook(self): + self.CodeHandler.add_line("code", "def specificity(y_true, y_pred):", indent=0) + self.CodeHandler.add_line("code", "cm = confusion_matrix(y_true, y_pred)", indent=1) + self.CodeHandler.add_line("code", "if cm.shape == (2, 2):", indent=1) + self.CodeHandler.add_line("code", "tn, fp, fn, tp = cm.ravel()", indent=2) + self.CodeHandler.add_line("code", "return tn / (tn + fp) if (tn + fp) > 0 else 0", indent=2) + self.CodeHandler.add_line("code", "return 0", indent=1) + def balanced_accuracy(self, y_true, y_pred): """Balanced Accuracy""" sensitivity = recall_score(y_true, y_pred, zero_division=0) @@ -623,25 +674,30 @@ def npv(self, y_true, y_pred): return tn / (tn + fn) if (tn + fn) > 0 else 0 return 0 - def youden_index(self, y_true, y_proba): - """Youden's J statistic""" - y_pred = (y_proba >= 0.5).astype(int) - sensitivity = recall_score(y_true, y_pred, zero_division=0) - specificity = self.specificity(y_true, y_pred) - return sensitivity + specificity - 1 - + def npv_to_notebook(self): + self.CodeHandler.add_line("code", "def npv(y_true, y_pred):", indent=0) + self.CodeHandler.add_line("code", "cm = confusion_matrix(y_true, y_pred)", indent=1) + self.CodeHandler.add_line("code", "if cm.shape == (2, 2):", indent=1) + self.CodeHandler.add_line("code", "tn, fp, fn, tp = cm.ravel()", indent=2) + self.CodeHandler.add_line("code", "return tn / (tn + fn) if (tn + fn) > 0 else 0", indent=2) + self.CodeHandler.add_line("code", "return 0", indent=1) + + def youden_index(self, y_true, y_pred): + """ + Youden's J statistic. + PyCaret calls score_func(y_true, y_pred_binary) at each threshold step + during optimize_threshold. Using roc_curve here was incorrect. + """ + return recall_score(y_true, y_pred, zero_division=0) + self.specificity(y_true, y_pred) - 1 + def mcc(self, y_true, y_pred): """Matthews Correlation Coefficient""" return matthews_corrcoef(y_true, y_pred) def _execute(self, experiment: dict = None, **kwargs) -> json: - """ - This function is used to execute the node. - """ print(Fore.BLUE + "=== fit === " + Fore.YELLOW + f"({self.username})" + Fore.RESET) print(Fore.CYAN + f"Using {self.type}" + Fore.RESET) - # Add custom metrics to PyCaret try: experiment['pycaret_exp'].add_metric(id='specificity', name='Specificity', score_func=self.specificity) except Exception as e: @@ -668,7 +724,6 @@ def _execute(self, experiment: dict = None, **kwargs) -> json: else: self.CodeHandler.add_line("md", f"##### *Model ID: {self.username}*") - # Initialization trained_models = None trained_models_json = {} settings = copy.deepcopy(self.settings) @@ -677,14 +732,12 @@ def _execute(self, experiment: dict = None, **kwargs) -> json: splitted = kwargs.get("splitted", None) finalize = kwargs.get("finalize", False) - # If data is splitted, we need to train and evaluate the model using the custom function that handles splitted data if splitted: results = self.__handle_splitted_data(experiment, settings, **kwargs) trained_models = [results['model']] all_metrics = results['overall_metrics'] trained_models_json['overall_metrics'] = all_metrics - # Experimental scene: comparing multiple models with compare_models elif self.type == 'compare_models': models = experiment['pycaret_exp'].compare_models(**settings) self.CodeHandler.add_line("code", f"trained_models = pycaret_exp.compare_models({self.CodeHandler.convert_dict_to_params(settings)})") @@ -692,16 +745,13 @@ def _execute(self, experiment: dict = None, **kwargs) -> json: trained_models = models else: trained_models = [models] - self.CodeHandler.add_line("code", "# pycaret_exp.compare_models() returns a single model, but we want a list of models") self.CodeHandler.add_line("code", "trained_models = [trained_models]") - # Regular scene: training a single model with create_model (with other optional steps like tuning, ensembling, calibrating, threshold optimization) elif self.type == 'train_model': settings.update(self.config_json['data']['estimator']['settings']) import ast - # Convert hidden_layer_sizes if it is a string if "hidden_layer_sizes" in settings: val = settings["hidden_layer_sizes"] if isinstance(val, str) and val.startswith("(") and val.endswith(")"): @@ -714,12 +764,10 @@ def _execute(self, experiment: dict = None, **kwargs) -> json: trained_models = [experiment['pycaret_exp'].create_model(**settings)] self.CodeHandler.add_line("code", f"trained_models = [pycaret_exp.create_model({self.CodeHandler.convert_dict_to_params(settings)})]") if self.isTuningEnabled: - # Check if a custom grid is provided if self.useTuningGrid and self.model_id in list(self.config_json['data']['internal'].keys()) and 'custom_grid' in list(self.config_json['data']['internal'][self.model_id].keys()): raw_grid = self.config_json['data']['internal'][self.model_id]['custom_grid'] self.settingsTuning['custom_grid'] = sanitize_custom_grid(raw_grid) - # Convert hidden_layer_sizes if it is a string if "hidden_layer_sizes" in self.settingsTuning['custom_grid']: val = self.settingsTuning['custom_grid']["hidden_layer_sizes"] if isinstance(val, list): @@ -735,7 +783,6 @@ def _execute(self, experiment: dict = None, **kwargs) -> json: except Exception as e: raise ValueError(f"Invalid tuple format for hidden_layer_sizes: {val}") from e - # Tune the model trained_models = [experiment['pycaret_exp'].tune_model(trained_models[0], **self.settingsTuning)] self.CodeHandler.add_line("code", f"trained_models = [pycaret_exp.tune_model(trained_models[0], {self.CodeHandler.convert_dict_to_params(self.settingsTuning)})]") @@ -748,22 +795,29 @@ def _execute(self, experiment: dict = None, **kwargs) -> json: self.CodeHandler.add_line("code", f"trained_models = [pycaret_exp.calibrate_model(trained_models[0], {self.CodeHandler.convert_dict_to_params(self.settingsCalibrate)})]") if self.optimize_threshold: - # Do not optimize if data is multiclass if len(experiment['pycaret_exp'].get_config('y').unique()) != 2: print("Skipping threshold optimization (multiclass not supported).") - - # Do not optimize if ensemble was applied elif self.ensembleEnabled: print("Skipping threshold optimization (ensemble not supported).") - else: trained_models = [experiment['pycaret_exp'].optimize_threshold(trained_models[0], optimize=self.threshold_optimization_metric)] self.CodeHandler.add_line("code", f"trained_models = [pycaret_exp.optimize_threshold(trained_models[0], optimize='{self.threshold_optimization_metric}')]") + self.CodeHandler.add_line( + "md", + f"Optimized threshold: {getattr(trained_models[0], 'probability_threshold', getattr(trained_models[0], 'threshold', 'not found'))}" + ) + # recalculate metrics using the optimized threshold + X_test_final = experiment['pycaret_exp'].get_config('X_test_transformed') + y_test_final = experiment['pycaret_exp'].get_config('y_test_transformed') + trained_models_json['overall_metrics'] = self.__recalculate_metrics_with_threshold( + trained_models[0], X_test_final, y_test_final + ) if finalize: trained_models = [experiment['pycaret_exp'].finalize_model(model) for model in trained_models] else: raise ValueError(f"Unsupported type: {self.type}. Expected 'compare_models' or 'train_model'.") + trained_models_copy = trained_models.copy() settings_for_next = copy.deepcopy(settings) settings_for_next['fct_type'] = self.type @@ -776,30 +830,3 @@ def _execute(self, experiment: dict = None, **kwargs) -> json: if isinstance(value, np.ndarray): trained_models_json[model_copy.__class__.__name__][key] = value.tolist() return trained_models_json - - def set_model(self, model_id: str) -> None: - """ - This function sets the model configuration for the current node based on the given model_id. - """ - try: - model_obj = self.global_config_json['nodes'][model_id] - self.config_json['data']['estimator'] = { - "type": model_obj['data']['selection'], - "settings": model_obj['data']['settings'] - } - if self.isTuningEnabled: - self.config_json['data']['internal']['settingsTuning'] = model_obj['data']['internal'].get('settingsTuning', {}) - except KeyError as e: - print(f"ERROR: Failed to set model {model_id}. Missing key: {e}") - raise ValueError(f"Model configuration for {model_id} not found in the global config.") - except Exception as e: - print(f"ERROR: An error occurred while setting the model: {e}") - raise ValueError(f"An error occurred while setting model {model_id}.") - - def set_model(self, model_id: str) -> None: - # self.model_id = model_id - model_obj = self.global_config_json['nodes'][model_id] - self.config_json['data']['estimator'] = { - "type": model_obj['data']['selection'], - "settings": model_obj['data']['settings'] - } \ No newline at end of file diff --git a/pythonCode/med_libs/MEDml/nodes/NodeObj.py b/pythonCode/med_libs/MEDml/nodes/NodeObj.py index 184edd64..417d38a5 100644 --- a/pythonCode/med_libs/MEDml/nodes/NodeObj.py +++ b/pythonCode/med_libs/MEDml/nodes/NodeObj.py @@ -201,6 +201,16 @@ def __init__(self, base_code: List[dict] = []) -> None: self.code = base_code self.imports = [] + def __check_if_import_exists(self, import_name: str) -> bool: + """ + Checks if an import already exists in the code + Args: + import_name: name of the import to check + Returns: + True if the import already exists, False otherwise + """ + return import_name in [imp['content'] for imp in self.imports] + def add_import(self, import_name: str): """ Adds an import to the code @@ -210,8 +220,9 @@ def add_import(self, import_name: str): Returns: None """ - self.imports.append( - {"type": "code", "content": import_name, "indent": 0}) + if not self.__check_if_import_exists(import_name): + self.imports.append( + {"type": "code", "content": import_name, "indent": 0}) def add_line(self, line_type: str, line: str, indent: int = 0): """ diff --git a/pythonCode/modules/evaluation/predict_test.py b/pythonCode/modules/evaluation/predict_test.py index 5893da18..f65ece5e 100644 --- a/pythonCode/modules/evaluation/predict_test.py +++ b/pythonCode/modules/evaluation/predict_test.py @@ -108,6 +108,10 @@ def _custom_process(self, json_config: dict) -> dict: self.set_progress(label="Predicting...", now=50) pred_unseen = predict_model(model, data=dataset) self.set_progress(now=70) + + # rename column "prediction_score" to "confidence_score" if it exists + if "prediction_score" in pred_unseen.columns: + pred_unseen = pred_unseen.rename(columns={"prediction_score": "confidence_score"}) # Save predictions prediction_object = MEDDataObject( diff --git a/renderer/components/flow/codeEditor.jsx b/renderer/components/flow/codeEditor.jsx index ec5c097a..79aaa0e2 100644 --- a/renderer/components/flow/codeEditor.jsx +++ b/renderer/components/flow/codeEditor.jsx @@ -120,6 +120,7 @@ const CodeEditor = ({id, path, updateSavedCode}) => { const [content, setContent] = useState("Loading...") const [mode, setMode] = useState("text") const [error, setError] = useState("") + const [stackTrace, setStackTrace] = useState("") const [loading, setLoading] = useState(true) const [loadingSave, setLoadingSave] = useState(false) const { globalData } = useContext(DataContext) @@ -207,9 +208,17 @@ const CodeEditor = ({id, path, updateSavedCode}) => { (response) => { setLoading(false) console.log("Response from backend:", response) - if (response.error){ - setError(response.error) - toast.error("Error opening file: " + response.error) + if (response.error) { + if (typeof response.error === "object") { + setError(response.error.message || "An error occurred while loading the file.") + toast.error("Error loading file: " + (response.error.message || "Unknown error")) + if (response.error.stack_trace) { + setStackTrace(response.error.stack_trace) + } + } else if (typeof response.error === "string") { + setError(response.error) + toast.error("Error opening file: " + response.error) + } } else { setContent(response.content) @@ -253,6 +262,11 @@ const CodeEditor = ({id, path, updateSavedCode}) => {
{error}
+ {stackTrace && ( + 
+              {stackTrace}
+
+ )} ) : ( diff --git a/renderer/components/learning/nodesTypes/datasetNode.jsx b/renderer/components/learning/nodesTypes/datasetNode.jsx index 32d16c43..cab394a4 100644 --- a/renderer/components/learning/nodesTypes/datasetNode.jsx +++ b/renderer/components/learning/nodesTypes/datasetNode.jsx @@ -357,26 +357,34 @@ const DatasetNode = ({ id, data }) => { return ( <> -
- Default inverse = N(negative) / N(positive). We recommand recalibrating through hyperparameters if this fraction exceeds 3. -

- Used for: - - Interpretation + target=".default-inverse-tooltip" + position="right" +> +
+ Class imbalance — general guideline +

+ + Default inverse = N(negative) / N(positive). + We recommend adjusting model settings if this fraction exceeds 3. + +

+ + Used for: +
    +
  • class_weight (Random Forest)
    • +
  • scale_pos_weight (XGBoost)
    • +
  • Calibration & metric selection
    • +
+ + Interpretation:
  • ≤ 3 → Balanced
  • 3 – 6 → Imbalanced
  • > 6 → Highly imbalanced
-
- +
+ + { useEffect(() => { let allModelsData = [] if (models.length > 0) { - models.forEach((model) => { - if (!model.metrics) return - let modifiedRow = { ...model.metrics } - modifiedRow["Parameters"] = model.params - modifiedRow["OverallMetrics"] = selectedResults?.data?.overall_metrics - modifiedRow = Object.assign({ Name: model.name }, modifiedRow) - allModelsData.push(modifiedRow) - }) + models.forEach((model) => { + if (!model.metrics) return + let modifiedRow = { ...model.metrics } + + // FIX: if overall_metrics exists, use these new metrics (optimal threshold) + const overallMetrics = selectedResults?.data?.overall_metrics + if (overallMetrics) { + Object.keys(overallMetrics).forEach((key) => { + modifiedRow[key] = overallMetrics[key].mean + }) + } + + modifiedRow["Parameters"] = model.params + modifiedRow["OverallMetrics"] = overallMetrics + modifiedRow = Object.assign({ Name: model.name }, modifiedRow) + allModelsData.push(modifiedRow) + }) } allModelsData.length > 0 && setAllModelsData(allModelsData) }, [models]) diff --git a/renderer/components/mainPages/settings.jsx b/renderer/components/mainPages/settings.jsx index a02badfe..64bbafd8 100644 --- a/renderer/components/mainPages/settings.jsx +++ b/renderer/components/mainPages/settings.jsx @@ -68,32 +68,35 @@ const SettingsPage = ({pageId = "settings", checkJupyterIsRunning, startJupyterS } /** - * Get the settings from the main process - * if the conda path is defined in the settings, set it - * Check if the server is running and set the state + * Get the settings from the main process on mount */ useEffect(() => { + checkServer() + checkMongoIsRunning() + getJupyterStatus() ipcRenderer.invoke("get-settings").then((receivedSettings) => { console.log("received settings", receivedSettings) setSettings(receivedSettings) - if (pythonEmbedded.pythonEmbedded) { - setCondaPath(pythonEmbedded.pythonEmbedded) - } else if (receivedSettings?.condaPath) { + if (receivedSettings?.condaPath) { setCondaPath(receivedSettings?.condaPath) } if (receivedSettings?.seed) { setSeed(receivedSettings?.seed) } }) - // ipcRenderer.invoke("server-is-running").then((status) => { - // setServerIsRunning(status) - // console.log("server is running", status) - // }) - checkMongoIsRunning() - checkServer() - getJupyterStatus() }, []) + /** + * Get the settings from the main process + * if the conda path is defined in the settings, set it + * Check if the server is running and set the state + */ + useEffect(() => { + if (pythonEmbedded.pythonEmbedded && !condaPath) { + setCondaPath(pythonEmbedded.pythonEmbedded) + } + }, [pythonEmbedded]) + /** * Save the settings in the main process * @param {Object} newSettings - New settings object