diff --git a/docs/curriculum/ml-engineer.compiled.json b/docs/curriculum/ml-engineer.compiled.json new file mode 100644 index 0000000000..1bf950b7fc --- /dev/null +++ b/docs/curriculum/ml-engineer.compiled.json @@ -0,0 +1,762 @@ +{ + "groups": [ + { + "id": "donnees", + "kind": "branches", + "prerequisites": [], + "notebooks": [ + { + "path": "ML/DataScienceWithAgents/01-PythonForDataScience/notebooks/1.2-Manipulation_de_Donnees_avec_NumPy.ipynb", + "title": "1.2 - Manipulation de Données avec NumPy", + "duration_minutes": 45, + "kernel": "Python 3", + "execution_constraints": { + "requires_api": false, + "requires_gpu": false, + "requires_cloud": false, + "requires_wsl": false, + "executable_locally": true + } + }, + { + "path": "ML/DataScienceWithAgents/01-PythonForDataScience/notebooks/1.3-Analyse_de_Donnees_avec_Pandas.ipynb", + "title": "1.3 - Analyse de Données avec Pandas", + "duration_minutes": 30, + "kernel": "Python 3", + "execution_constraints": { + "requires_api": false, + "requires_gpu": false, + "requires_cloud": false, + "requires_wsl": false, + "executable_locally": true + } + } + ] + }, + { + "id": "workflow-ml", + "kind": "branches", + "prerequisites": [ + "donnees" + ], + "notebooks": [ + { + "path": "ML/DataScienceWithAgents/02-ML-Cours/2.1-Workflow-ML.ipynb", + "title": "2.1 — Le workflow d'apprentissage automatique", + "duration_minutes": 45, + "kernel": "Python 3", + "execution_constraints": { + "requires_api": false, + "requires_gpu": false, + "requires_cloud": false, + "requires_wsl": false, + "executable_locally": true + } + }, + { + "path": "ML/DataScienceWithAgents/02-ML-Cours/2.2-Descente-de-gradient.ipynb", + "title": "2.2 — La descente de gradient : comment un modèle apprend", + "duration_minutes": 45, + "kernel": "Python 3", + "execution_constraints": { + "requires_api": false, + "requires_gpu": false, + "requires_cloud": false, + "requires_wsl": false, + "executable_locally": true + } + } + ] + }, + { + "id": "apprentissage-supervise", + "kind": "branches", + "prerequisites": [ + "workflow-ml" + ], + "notebooks": [ + { + "path": "ML/DataScienceWithAgents/02-ML-Cours/2.3-Regression-lineaire-logistique.ipynb", + "title": "2.3 — Régression linéaire et régression logistique", + "duration_minutes": 45, + "kernel": "Python 3", + "execution_constraints": { + "requires_api": false, + "requires_gpu": false, + "requires_cloud": false, + "requires_wsl": false, + "executable_locally": true + } + }, + { + "path": "ML/DataScienceWithAgents/02-ML-Cours/2.3b-Naive-Bayes-Generatif.ipynb", + "title": "Naive Bayes génératif vs régression logistique discriminative", + "duration_minutes": 30, + "kernel": "Python 3", + "execution_constraints": { + "requires_api": false, + "requires_gpu": false, + "requires_cloud": false, + "requires_wsl": false, + "executable_locally": true + } + }, + { + "path": "ML/DataScienceWithAgents/02-ML-Cours/2.3c-Regression-Grande-Dimension.ipynb", + "title": "Régression en grande dimension — quand p >> n : ridge, PCR, PLS", + "duration_minutes": 30, + "kernel": "Python 3", + "execution_constraints": { + "requires_api": false, + "requires_gpu": false, + "requires_cloud": false, + "requires_wsl": false, + "executable_locally": true + } + }, + { + "path": "ML/DataScienceWithAgents/02-ML-Cours/2.3d-Modele-Gaussien-LDA-QDA.ipynb", + "title": "Modèle gaussien, frontière LDA / QDA", + "duration_minutes": 30, + "kernel": "Python 3", + "execution_constraints": { + "requires_api": false, + "requires_gpu": false, + "requires_cloud": false, + "requires_wsl": false, + "executable_locally": true + } + }, + { + "path": "ML/DataScienceWithAgents/02-ML-Cours/2.4-Arbres-Forets-Ensembles.ipynb", + "title": "2.4 — Arbres de décision, forêts aléatoires et boosting", + "duration_minutes": 45, + "kernel": "Python 3", + "execution_constraints": { + "requires_api": false, + "requires_gpu": false, + "requires_cloud": false, + "requires_wsl": false, + "executable_locally": true + } + } + ] + }, + { + "id": "evaluation-validation", + "kind": "branches", + "prerequisites": [ + "apprentissage-supervise" + ], + "notebooks": [ + { + "path": "ML/DataScienceWithAgents/02-ML-Cours/2.5-Biais-Variance-CV-ROC.ipynb", + "title": "2.5 — Biais, variance, validation croisée et courbe ROC", + "duration_minutes": 45, + "kernel": "Python 3", + "execution_constraints": { + "requires_api": false, + "requires_gpu": false, + "requires_cloud": false, + "requires_wsl": false, + "executable_locally": true + } + }, + { + "path": "ML/DataScienceWithAgents/02-ML-Cours/2.10-Optimisation-Hyperparametres.ipynb", + "title": "2.10 — Optimisation d'hyperparamètres : grille, hasard, bayésien — et quand s'arrêter", + "duration_minutes": 30, + "kernel": "Python 3", + "execution_constraints": { + "requires_api": false, + "requires_gpu": true, + "requires_cloud": false, + "requires_wsl": false, + "executable_locally": false + } + }, + { + "path": "ML/DataScienceWithAgents/02-ML-Cours/2.12-Donnees-Desequilibrees.ipynb", + "title": "2.12 — Données déséquilibrées : la courbe PR, les stratégies de rééchantillonnage et le seuil qui coûte", + "duration_minutes": 30, + "kernel": "Python 3", + "execution_constraints": { + "requires_api": false, + "requires_gpu": false, + "requires_cloud": false, + "requires_wsl": false, + "executable_locally": true + } + }, + { + "path": "ML/DataScienceWithAgents/02-ML-Cours/2.13-Analyse-Erreurs.ipynb", + "title": "2.13 — Analyse d'erreurs : diagnostiquer un modèle entraîné", + "duration_minutes": 30, + "kernel": "Python 3", + "execution_constraints": { + "requires_api": false, + "requires_gpu": false, + "requires_cloud": false, + "requires_wsl": false, + "executable_locally": true + } + } + ] + }, + { + "id": "fiabilite", + "kind": "branches", + "prerequisites": [ + "evaluation-validation" + ], + "notebooks": [ + { + "path": "ML/DataScienceWithAgents/02-ML-Cours/2.5b-Calibration-Probabilites.ipynb", + "title": "2.5b — Calibration des probabilités : reliability diagrams, ECE, et pourquoi 0.87 n'est pas 87 % de chances", + "duration_minutes": 30, + "kernel": "Python 3", + "execution_constraints": { + "requires_api": false, + "requires_gpu": false, + "requires_cloud": false, + "requires_wsl": false, + "executable_locally": true + } + }, + { + "path": "ML/DataScienceWithAgents/02-ML-Cours/2.5c-Equite-Sous-Groupes.ipynb", + "title": "2.5c — Equite par sous-groupe : compromis performance-parite, et pourquoi l'accuracy globale ne suffit pas", + "duration_minutes": 30, + "kernel": "Python 3", + "execution_constraints": { + "requires_api": false, + "requires_gpu": false, + "requires_cloud": false, + "requires_wsl": false, + "executable_locally": true + } + }, + { + "path": "ML/DataScienceWithAgents/02-ML-Cours/2.5d-Derive-Distribution-Deploiement.ipynb", + "title": "2.5d — Dérive de distribution au déploiement : détecter, corriger, savoir quand on ne peut pas", + "duration_minutes": 45, + "kernel": "Python 3.11 (coursia-drift)", + "execution_constraints": { + "requires_api": false, + "requires_gpu": false, + "requires_cloud": false, + "requires_wsl": false, + "executable_locally": true + } + }, + { + "path": "ML/DataScienceWithAgents/02-ML-Cours/2.15-Donnee-Comme-Responsabilite-Python.ipynb", + "title": "2.15 — La donnée comme responsabilité : inférence d'appartenance et confidentialité différentielle", + "duration_minutes": 45, + "kernel": "Python 3", + "execution_constraints": { + "requires_api": false, + "requires_gpu": false, + "requires_cloud": false, + "requires_wsl": false, + "executable_locally": true + } + } + ] + }, + { + "id": "interpretabilite", + "kind": "branches", + "prerequisites": [ + "apprentissage-supervise" + ], + "notebooks": [ + { + "path": "ML/DataScienceWithAgents/02-ML-Cours/2.14-Explicabilite-SHAP-LIME-Contrefactuels.ipynb", + "title": "2.14 — Explicabilité (XAI) : SHAP, LIME et contrefactuels", + "duration_minutes": 30, + "kernel": "Python 3", + "execution_constraints": { + "requires_api": false, + "requires_gpu": false, + "requires_cloud": false, + "requires_wsl": false, + "executable_locally": true + } + }, + { + "path": "ML/DataScienceWithAgents/02-ML-Cours/2.14b-XAI-Shap-Attribution-Causal-Bridge.ipynb", + "title": "2.14b — SHAP et do-calculus : la jonction attribution causalité", + "duration_minutes": 30, + "kernel": "Python 3 (coursia-ml-training)", + "execution_constraints": { + "requires_api": false, + "requires_gpu": false, + "requires_cloud": false, + "requires_wsl": false, + "executable_locally": true + } + } + ] + }, + { + "id": "deep-learning-budget", + "kind": "branches", + "prerequisites": [ + "evaluation-validation" + ], + "notebooks": [ + { + "path": "ML/DataScienceWithAgents/03-DeepLearning/3.1-Retropropagation.ipynb", + "title": "3.1 — La rétropropagation : la chaîne des gradients à la main", + "duration_minutes": 30, + "kernel": "Python 3", + "execution_constraints": { + "requires_api": false, + "requires_gpu": false, + "requires_cloud": false, + "requires_wsl": false, + "executable_locally": true + } + }, + { + "path": "ML/DataScienceWithAgents/03-DeepLearning/3.2-Optimisateurs.ipynb", + "title": "3.2 — Les optimisateurs : de SGD à Adam, ce qui change vraiment", + "duration_minutes": 30, + "kernel": "Python 3", + "execution_constraints": { + "requires_api": false, + "requires_gpu": false, + "requires_cloud": false, + "requires_wsl": false, + "executable_locally": true + } + }, + { + "path": "ML/DataScienceWithAgents/03-DeepLearning/3.3-Regularisation.ipynb", + "title": "3.3 — Régularisation : dropout, weight decay, early stopping", + "duration_minutes": 30, + "kernel": "Python 3", + "execution_constraints": { + "requires_api": false, + "requires_gpu": false, + "requires_cloud": false, + "requires_wsl": false, + "executable_locally": true + } + }, + { + "path": "ML/DataScienceWithAgents/03-DeepLearning/3.4-Attention-Transformer-From-Scratch.ipynb", + "title": "3.4 — Attention et Transformer from scratch : jusqu'au mini-GPT", + "duration_minutes": 45, + "kernel": "Python 3 (coursia2)", + "execution_constraints": { + "requires_api": false, + "requires_gpu": false, + "requires_cloud": false, + "requires_wsl": false, + "executable_locally": true + } + }, + { + "path": "ML/DataScienceWithAgents/03-DeepLearning/3.11-Budget-Memoire-Entrainement.ipynb", + "title": "3.11 — Le budget mémoire d'un entraînement : quatre postes, calculés puis mesurés", + "duration_minutes": 60, + "kernel": "Python 3", + "execution_constraints": { + "requires_api": false, + "requires_gpu": true, + "requires_cloud": false, + "requires_wsl": false, + "executable_locally": false + } + }, + { + "path": "ML/DataScienceWithAgents/03-DeepLearning/3.13-Decouper-le-Modele-DDP-ZeRO-FSDP.ipynb", + "title": "3.13 — Découper le modèle : DDP, ZeRO, FSDP — et ce que le pipeline change", + "duration_minutes": 45, + "kernel": "Python 3 (coursia-ml-training)", + "execution_constraints": { + "requires_api": false, + "requires_gpu": true, + "requires_cloud": false, + "requires_wsl": false, + "executable_locally": false + } + } + ] + }, + { + "id": "theorie-generalisation", + "kind": "accretions", + "prerequisites": [ + "deep-learning-budget" + ], + "notebooks": [ + { + "path": "ML/DataScienceWithAgents/02-ML-Cours/2.7-Modeles-Non-Parametriques.ipynb", + "title": "2.7 — Modèles non paramétriques : SVM et k plus proches voisins", + "duration_minutes": 30, + "kernel": "Python 3", + "execution_constraints": { + "requires_api": false, + "requires_gpu": false, + "requires_cloud": false, + "requires_wsl": false, + "executable_locally": true + } + }, + { + "path": "ML/DataScienceWithAgents/02-ML-Cours/2.8-Theorie-PAC.ipynb", + "title": "2.8 — Théorie de l'apprentissage : PAC et dimension de Vapnik-Chervonenkis", + "duration_minutes": 30, + "kernel": "Python 3", + "execution_constraints": { + "requires_api": false, + "requires_gpu": false, + "requires_cloud": false, + "requires_wsl": false, + "executable_locally": true + } + }, + { + "path": "ML/DataScienceWithAgents/02-ML-Cours/2.8b-Theorie-PAC-Lean.ipynb", + "title": "2.8b - Theorie PAC en Lean : l'arc du lake learning_theory_lean", + "duration_minutes": 30, + "kernel": "Lean 4 (WSL)", + "execution_constraints": { + "requires_api": false, + "requires_gpu": false, + "requires_cloud": false, + "requires_wsl": false, + "executable_locally": true + } + }, + { + "path": "ML/DataScienceWithAgents/02-ML-Cours/2.8c-Borne-Temoin-Concentration.ipynb", + "title": "2.8c — Borne + Témoin extrémal + Concentration : ce que learning_theory_lean sait déjà faire", + "duration_minutes": 15, + "kernel": "Python (coursia-ml-training)", + "execution_constraints": { + "requires_api": false, + "requires_gpu": false, + "requires_cloud": false, + "requires_wsl": false, + "executable_locally": true + } + }, + { + "path": "ML/DataScienceWithAgents/02-ML-Cours/2.8d-Lean-Novikoff-Convergence.ipynb", + "title": "Novikoff : la convergence du perceptron, démontrée et rejouée", + "duration_minutes": 30, + "kernel": "Lean 4 (WSL)", + "execution_constraints": { + "requires_api": false, + "requires_gpu": false, + "requires_cloud": false, + "requires_wsl": false, + "executable_locally": true + } + }, + { + "path": "ML/DataScienceWithAgents/02-ML-Cours/2.9-Grokking-Generalisation.ipynb", + "title": "2.9 — Grokking : la généralisation qui arrive en retard", + "duration_minutes": 30, + "kernel": "Python 3", + "execution_constraints": { + "requires_api": false, + "requires_gpu": false, + "requires_cloud": false, + "requires_wsl": false, + "executable_locally": true + } + }, + { + "path": "ML/DataScienceWithAgents/02-ML-Cours/2.9c-Grokking-Diagrammes-Phases.ipynb", + "title": "2.9c — Grokking : le diagramme de phases", + "duration_minutes": 30, + "kernel": "Python 3", + "execution_constraints": { + "requires_api": false, + "requires_gpu": false, + "requires_cloud": false, + "requires_wsl": false, + "executable_locally": true + } + } + ] + }, + { + "id": "compression-mise-en-service", + "kind": "accretions", + "prerequisites": [ + "deep-learning-budget" + ], + "notebooks": [ + { + "path": "ML/DataScienceWithAgents/03-DeepLearning/3.9-Compression-Quantization-FP.ipynb", + "title": "3.9 — Quantization FP : FP32 vers FP16 et BF16 depuis zéro", + "duration_minutes": 45, + "kernel": "coursia-ml-training", + "execution_constraints": { + "requires_api": false, + "requires_gpu": false, + "requires_cloud": false, + "requires_wsl": false, + "executable_locally": true + } + }, + { + "path": "ML/DataScienceWithAgents/03-DeepLearning/3.9a-Compression-Quantization-INT8.ipynb", + "title": "3.9a — Compression par quantification INT8 : le réseau en 4 fois plus petit", + "duration_minutes": 45, + "kernel": "Python 3", + "execution_constraints": { + "requires_api": false, + "requires_gpu": true, + "requires_cloud": false, + "requires_wsl": false, + "executable_locally": false + } + }, + { + "path": "ML/DataScienceWithAgents/03-DeepLearning/3.9b-Compression-Pruning-from-scratch.ipynb", + "title": "3.9b — Compression par élagage : le réseau amputé qui tient", + "duration_minutes": 60, + "kernel": "Python 3", + "execution_constraints": { + "requires_api": false, + "requires_gpu": true, + "requires_cloud": false, + "requires_wsl": false, + "executable_locally": false + } + }, + { + "path": "ML/DataScienceWithAgents/03-DeepLearning/3.9c-Pruning-From-Scratch.ipynb", + "title": "3.9c — Pruning from scratch : magnitude, structured, Lottery Ticket", + "duration_minutes": 45, + "kernel": "Python 3", + "execution_constraints": { + "requires_api": false, + "requires_gpu": true, + "requires_cloud": false, + "requires_wsl": false, + "executable_locally": false + } + }, + { + "path": "ML/DataScienceWithAgents/03-DeepLearning/3.9d-Compression-Distillation-from-scratch.ipynb", + "title": "3.9d — Compression par distillation : transférer le savoir, pas les poids", + "duration_minutes": 45, + "kernel": "Python 3 (coursia-ml-training)", + "execution_constraints": { + "requires_api": false, + "requires_gpu": true, + "requires_cloud": false, + "requires_wsl": false, + "executable_locally": false + } + }, + { + "path": "ML/DataScienceWithAgents/03-DeepLearning/3.9e-Compression-Quantization-SOTA.ipynb", + "title": "3.9e — Quantification SOTA : la même INT8, par l'écosystème torch.ao", + "duration_minutes": 45, + "kernel": "Python 3", + "execution_constraints": { + "requires_api": false, + "requires_gpu": true, + "requires_cloud": false, + "requires_wsl": false, + "executable_locally": false + } + }, + { + "path": "ML/DataScienceWithAgents/03-DeepLearning/3.7-Distillation-Maitre-Eleve.ipynb", + "title": "3.7 — Distillation maître-élève : quand le savoir se transfère par des cibles molles", + "duration_minutes": 30, + "kernel": "coursia-ml-training", + "execution_constraints": { + "requires_api": false, + "requires_gpu": false, + "requires_cloud": false, + "requires_wsl": false, + "executable_locally": true + } + } + ] + }, + { + "id": "modeles-non-supervises", + "kind": "accretions", + "prerequisites": [ + "evaluation-validation" + ], + "notebooks": [ + { + "path": "ML/DataScienceWithAgents/02-ML-Cours/2.6-Clustering-KMeans-PCA.ipynb", + "title": "2.6 — Clustering (KMeans) et réduction de dimension (PCA)", + "duration_minutes": 45, + "kernel": "Python 3", + "execution_constraints": { + "requires_api": false, + "requires_gpu": false, + "requires_cloud": false, + "requires_wsl": false, + "executable_locally": true + } + }, + { + "path": "ML/DataScienceWithAgents/03-DeepLearning/3.8-Representations-Contrastives.ipynb", + "title": "Représentations contrastives modernes — du skip-gram aux vues continues", + "duration_minutes": 30, + "kernel": "Python 3", + "execution_constraints": { + "requires_api": false, + "requires_gpu": false, + "requires_cloud": false, + "requires_wsl": false, + "executable_locally": true + } + }, + { + "path": "ML/DataScienceWithAgents/03-DeepLearning/3.10-Modeles-Generatifs-Diffusion-SOTA.ipynb", + "title": "3.10 — Le pendant SOTA : la bibliothèque diffusers (DDPM et DDIM outillés)", + "duration_minutes": 45, + "kernel": "Python 3", + "execution_constraints": { + "requires_api": false, + "requires_gpu": true, + "requires_cloud": false, + "requires_wsl": false, + "executable_locally": false + } + }, + { + "path": "ML/DataScienceWithAgents/03-DeepLearning/3.6-Modeles-Generatifs.ipynb", + "title": "3.6 — Modèles génératifs : trois objectifs, trois échecs — VAE, GAN et diffusion sur une distribution multimodale bornée", + "duration_minutes": 30, + "kernel": "Python 3", + "execution_constraints": { + "requires_api": false, + "requires_gpu": false, + "requires_cloud": false, + "requires_wsl": false, + "executable_locally": true + } + } + ] + }, + { + "id": "svm-noyautiques", + "kind": "accretions", + "prerequisites": [ + "apprentissage-supervise" + ], + "notebooks": [ + { + "path": "ML/DataScienceWithAgents/02-ML-Cours/2.7b-SMO-From-Scratch.ipynb", + "title": "2.7b — SMO from scratch : SVM soft-margin, boucle de Platt, KKT et gap de dualité", + "duration_minutes": 30, + "kernel": "Python 3", + "execution_constraints": { + "requires_api": false, + "requires_gpu": false, + "requires_cloud": false, + "requires_wsl": false, + "executable_locally": true + } + }, + { + "path": "ML/DataScienceWithAgents/02-ML-Cours/2.7c-SVM-SOTA-Comparison.ipynb", + "title": "2.7c — SVM SOTA : LIBSVM sous le capot de sklearn.svm.SVC", + "duration_minutes": 30, + "kernel": "Python 3", + "execution_constraints": { + "requires_api": false, + "requires_gpu": false, + "requires_cloud": false, + "requires_wsl": false, + "executable_locally": true + } + } + ] + }, + { + "id": "optimisation-convexe-avancee", + "kind": "accretions", + "prerequisites": [ + "deep-learning-budget" + ], + "notebooks": [ + { + "path": "ML/DataScienceWithAgents/02-ML-Cours/2.11b-Proximal-Operators-From-Scratch.ipynb", + "title": "2.11b — Operateurs proximaux : ISTA et FISTA from scratch", + "duration_minutes": 30, + "kernel": "Python 3", + "execution_constraints": { + "requires_api": false, + "requires_gpu": false, + "requires_cloud": false, + "requires_wsl": false, + "executable_locally": true + } + }, + { + "path": "ML/DataScienceWithAgents/02-ML-Cours/2.11c-Lasso-SOTA-Comparison.ipynb", + "title": "2.11c — Lasso SOTA : coordinate descent, LassoCV, et quand le SOTA prend le relais", + "duration_minutes": 30, + "kernel": "Python 3", + "execution_constraints": { + "requires_api": false, + "requires_gpu": false, + "requires_cloud": false, + "requires_wsl": false, + "executable_locally": true + } + }, + { + "path": "ML/DataScienceWithAgents/02-ML-Cours/2.11d-Optimisation-ADMM-From-Scratch.ipynb", + "title": "2.11d — ADMM from scratch", + "duration_minutes": 30, + "kernel": "Python 3", + "execution_constraints": { + "requires_api": false, + "requires_gpu": false, + "requires_cloud": false, + "requires_wsl": false, + "executable_locally": true + } + }, + { + "path": "ML/DataScienceWithAgents/02-ML-Cours/2.11e-CVXPY-Optimisation.ipynb", + "title": "2.11e — CVXPY : le problème déclaré, le solveur choisi", + "duration_minutes": 30, + "kernel": "Python 3", + "execution_constraints": { + "requires_api": false, + "requires_gpu": false, + "requires_cloud": false, + "requires_wsl": false, + "executable_locally": true + } + }, + { + "path": "ML/DataScienceWithAgents/02-ML-Cours/2.11f-Comparaison-Optimisation-Convexe.ipynb", + "title": "2.11f — Comparaison bloc A / bloc B : le tableau croisé de l'optimisation convexe", + "duration_minutes": 30, + "kernel": "Python 3", + "execution_constraints": { + "requires_api": false, + "requires_gpu": false, + "requires_cloud": false, + "requires_wsl": false, + "executable_locally": true + } + } + ] + } + ], + "duration_minutes": 1800, + "known_duration_minutes": 1800 +} diff --git a/docs/curriculum/ml-engineer.json b/docs/curriculum/ml-engineer.json new file mode 100644 index 0000000000..773233163b --- /dev/null +++ b/docs/curriculum/ml-engineer.json @@ -0,0 +1,152 @@ +{ + "description": "Parcours ML Engineer : de la manipulation de donnees (NumPy, Pandas) jusqu'a la mise en service d'un modele, en passant par l'apprentissage supervise, l'evaluation, la fiabilite (calibration, equite, derive), l'interpretabilite, et le deep learning avec budget de calcul. Le fil d'ingenierie est explicite : reproductibilite, surveillance en production, cout de calcul, mise en service (modele comme service). Les trous du parcours sont listes sur l'EPIC parent #19543.", + "branches": [ + { + "id": "donnees", + "description": "Manipulation de donnees en Python : structures numeriques (NumPy) et tabulaires (Pandas). Le ML Engineer passe 80 % de son temps a preparer les donnees ; ce socle est non negociable avant tout modele. Le pre-requis data science du pli 2 (agentification Pandas/LangChain) releve d'un autre grain.", + "notebooks": [ + "ML/DataScienceWithAgents/01-PythonForDataScience/notebooks/1.2-Manipulation_de_Donnees_avec_NumPy.ipynb", + "ML/DataScienceWithAgents/01-PythonForDataScience/notebooks/1.3-Analyse_de_Donnees_avec_Pandas.ipynb" + ] + }, + { + "id": "workflow-ml", + "description": "Vue d'ensemble d'un projet de ML en production : separation train/validation/test, pipeline reproductible, premier modele de bout en bout (2.1-2.2). Avant les modeles lineaires avances, l'ingenieur doit avoir execute un premier pipeline propre.", + "prerequisites": [ + "donnees" + ], + "notebooks": [ + "ML/DataScienceWithAgents/02-ML-Cours/2.1-Workflow-ML.ipynb", + "ML/DataScienceWithAgents/02-ML-Cours/2.2-Descente-de-gradient.ipynb" + ] + }, + { + "id": "apprentissage-supervise", + "description": "Modeles classiques d'apprentissage supervise : regression lineaire et logistique, modeles generatifs (Naive Bayes, LDA/QDA), regression en grande dimension, arbres et ensembles (forets, AdaBoost). Le ML Engineer connait la batterie d'algorithmes avant de choisir ; il ne se jette pas sur un transformer pour 10 000 lignes. Cout d'entrainement, reproductibilite du seed et service en production sont les vrais discriminants entre un modele adapte au contexte et un modele impose par defaut.", + "prerequisites": [ + "workflow-ml" + ], + "notebooks": [ + "ML/DataScienceWithAgents/02-ML-Cours/2.3-Regression-lineaire-logistique.ipynb", + "ML/DataScienceWithAgents/02-ML-Cours/2.3b-Naive-Bayes-Generatif.ipynb", + "ML/DataScienceWithAgents/02-ML-Cours/2.3c-Regression-Grande-Dimension.ipynb", + "ML/DataScienceWithAgents/02-ML-Cours/2.3d-Modele-Gaussien-LDA-QDA.ipynb", + "ML/DataScienceWithAgents/02-ML-Cours/2.4-Arbres-Forets-Ensembles.ipynb" + ] + }, + { + "id": "evaluation-validation", + "description": "Evaluation et validation : biais-variance, validation croisee, courbe ROC, hyperparametres, desequilibre des classes. L'ingenieur distingue erreur d'entrainement et erreur de generalisation ; il sait mesurer avant d'optimiser. L'analyse d'erreurs (2.13) prepare le terrain pour le diagnostic en production.", + "prerequisites": [ + "apprentissage-supervise" + ], + "notebooks": [ + "ML/DataScienceWithAgents/02-ML-Cours/2.5-Biais-Variance-CV-ROC.ipynb", + "ML/DataScienceWithAgents/02-ML-Cours/2.10-Optimisation-Hyperparametres.ipynb", + "ML/DataScienceWithAgents/02-ML-Cours/2.12-Donnees-Desequilibrees.ipynb", + "ML/DataScienceWithAgents/02-ML-Cours/2.13-Analyse-Erreurs.ipynb" + ] + }, + { + "id": "fiabilite", + "description": "Fiabilite d'un modele en production : calibration des probabilites (un classifieur bien calibre dit ce qu'il sait), equite par sous-groupes (le modele ne discrimine pas), detection de derive de distribution en deploiement (les donnees de production ne ressemblent plus aux donnees d'entrainement). Ce que la mise en service exige : le modele reste juste quand le monde change.", + "prerequisites": [ + "evaluation-validation" + ], + "notebooks": [ + "ML/DataScienceWithAgents/02-ML-Cours/2.5b-Calibration-Probabilites.ipynb", + "ML/DataScienceWithAgents/02-ML-Cours/2.5c-Equite-Sous-Groupes.ipynb", + "ML/DataScienceWithAgents/02-ML-Cours/2.5d-Derive-Distribution-Deploiement.ipynb", + "ML/DataScienceWithAgents/02-ML-Cours/2.15-Donnee-Comme-Responsabilite-Python.ipynb" + ] + }, + { + "id": "interpretabilite", + "description": "Interpretabilite et explicabilite : SHAP (attribution par feature), LIME (surrogate local), contrefactuels (que faudrait-il changer pour predire autre chose), pont causal Shap. L'ingenieur doit pouvoir expliquer une prediction a un metier, un juriste ou un regulator ; le 'c'est la magic du modele' ne tient pas en production.", + "prerequisites": [ + "apprentissage-supervise" + ], + "notebooks": [ + "ML/DataScienceWithAgents/02-ML-Cours/2.14-Explicabilite-SHAP-LIME-Contrefactuels.ipynb", + "ML/DataScienceWithAgents/02-ML-Cours/2.14b-XAI-Shap-Attribution-Causal-Bridge.ipynb" + ] + }, + { + "id": "deep-learning-budget", + "description": "Deep learning et budget de calcul : retropropagation, optimiseurs, regularisation, transformer from scratch, budget memoire d'entrainement (3.11), decoupage multi-GPU (DDP/ZeRO/FSDP). Le ML Engineer ne se contente pas d'executer un modele : il sait ce que coute chaque ligne d'entrainement et comment le faire tenir dans la memoire disponible.", + "prerequisites": [ + "evaluation-validation" + ], + "notebooks": [ + "ML/DataScienceWithAgents/03-DeepLearning/3.1-Retropropagation.ipynb", + "ML/DataScienceWithAgents/03-DeepLearning/3.2-Optimisateurs.ipynb", + "ML/DataScienceWithAgents/03-DeepLearning/3.3-Regularisation.ipynb", + "ML/DataScienceWithAgents/03-DeepLearning/3.4-Attention-Transformer-From-Scratch.ipynb", + "ML/DataScienceWithAgents/03-DeepLearning/3.11-Budget-Memoire-Entrainement.ipynb", + "ML/DataScienceWithAgents/03-DeepLearning/3.13-Decouper-le-Modele-DDP-ZeRO-FSDP.ipynb" + ] + } + ], + "accretions": [ + { + "id": "theorie-generalisation", + "description": "Detour theorique : theorie PAC (bornes de generalisation), grokking et phenomenes de generalisation, modeles non-parametriques, bornes de concentration (Lean-temoine pour Novikoff). Ce detour eclaire pourquoi un modele marche ou pas ; il n'est pas un pre-requis pour la mise en service.", + "branch": "deep-learning-budget", + "notebooks": [ + "ML/DataScienceWithAgents/02-ML-Cours/2.7-Modeles-Non-Parametriques.ipynb", + "ML/DataScienceWithAgents/02-ML-Cours/2.8-Theorie-PAC.ipynb", + "ML/DataScienceWithAgents/02-ML-Cours/2.8b-Theorie-PAC-Lean.ipynb", + "ML/DataScienceWithAgents/02-ML-Cours/2.8c-Borne-Temoin-Concentration.ipynb", + "ML/DataScienceWithAgents/02-ML-Cours/2.8d-Lean-Novikoff-Convergence.ipynb", + "ML/DataScienceWithAgents/02-ML-Cours/2.9-Grokking-Generalisation.ipynb", + "ML/DataScienceWithAgents/02-ML-Cours/2.9c-Grokking-Diagrammes-Phases.ipynb" + ] + }, + { + "id": "compression-mise-en-service", + "description": "Detour mise en service : compression (quantization INT8, FP, pruning), distillation maitre-eleve. Le ML Engineer qui doit deployer sur du materiel contraint (mobile, edge) ou reduire le cout d'inference utilise ces techniques ; elles ne sont pas au coeur du parcours.", + "branch": "deep-learning-budget", + "notebooks": [ + "ML/DataScienceWithAgents/03-DeepLearning/3.9-Compression-Quantization-FP.ipynb", + "ML/DataScienceWithAgents/03-DeepLearning/3.9a-Compression-Quantization-INT8.ipynb", + "ML/DataScienceWithAgents/03-DeepLearning/3.9b-Compression-Pruning-from-scratch.ipynb", + "ML/DataScienceWithAgents/03-DeepLearning/3.9c-Pruning-From-Scratch.ipynb", + "ML/DataScienceWithAgents/03-DeepLearning/3.9d-Compression-Distillation-from-scratch.ipynb", + "ML/DataScienceWithAgents/03-DeepLearning/3.9e-Compression-Quantization-SOTA.ipynb", + "ML/DataScienceWithAgents/03-DeepLearning/3.7-Distillation-Maitre-Eleve.ipynb" + ] + }, + { + "id": "modeles-non-supervises", + "description": "Detour methodologique : clustering (KMeans, PCA), representations contrastives, modeles generatifs (diffusion, VAE, GAN). Le ML Engineer peut avoir besoin de clustering ou de representations non-supervisees, mais elles relevent souvent d'un parcours distinct (data scientist, chercheur).", + "branch": "evaluation-validation", + "notebooks": [ + "ML/DataScienceWithAgents/02-ML-Cours/2.6-Clustering-KMeans-PCA.ipynb", + "ML/DataScienceWithAgents/03-DeepLearning/3.8-Representations-Contrastives.ipynb", + "ML/DataScienceWithAgents/03-DeepLearning/3.10-Modeles-Generatifs-Diffusion-SOTA.ipynb", + "ML/DataScienceWithAgents/03-DeepLearning/3.6-Modeles-Generatifs.ipynb" + ] + }, + { + "id": "svm-noyautiques", + "description": "Detour methodologique : SVM (Support Vector Machines), SMO from scratch, comparaison SOTA. Les SVM restent une methode de reference sur petits datasets ; leur mechanique (noyau, marge, dualite) eclaire la pensee des modeles a noyaux en general.", + "branch": "apprentissage-supervise", + "notebooks": [ + "ML/DataScienceWithAgents/02-ML-Cours/2.7b-SMO-From-Scratch.ipynb", + "ML/DataScienceWithAgents/02-ML-Cours/2.7c-SVM-SOTA-Comparison.ipynb" + ] + }, + { + "id": "optimisation-convexe-avancee", + "description": "Detour optimisation : operateurs proximaux, comparaison LASSO SOTA, ADMM from scratch, CVXPY. Ce detour eclaire la mecanique des optimiseurs de la branche deep-learning-budget ; il n'est pas un pre-requis pour les algorithmes classiques.", + "branch": "deep-learning-budget", + "notebooks": [ + "ML/DataScienceWithAgents/02-ML-Cours/2.11b-Proximal-Operators-From-Scratch.ipynb", + "ML/DataScienceWithAgents/02-ML-Cours/2.11c-Lasso-SOTA-Comparison.ipynb", + "ML/DataScienceWithAgents/02-ML-Cours/2.11d-Optimisation-ADMM-From-Scratch.ipynb", + "ML/DataScienceWithAgents/02-ML-Cours/2.11e-CVXPY-Optimisation.ipynb", + "ML/DataScienceWithAgents/02-ML-Cours/2.11f-Comparaison-Optimisation-Convexe.ipynb" + ] + } + ] +} \ No newline at end of file diff --git a/scripts/notebook_tools/tests/test_generate_parcours.py b/scripts/notebook_tools/tests/test_generate_parcours.py index 9f3efdc3a3..360f7f4b90 100644 --- a/scripts/notebook_tools/tests/test_generate_parcours.py +++ b/scripts/notebook_tools/tests/test_generate_parcours.py @@ -401,6 +401,88 @@ def _expected_total_minutes(manifest, catalog, branch_or_accretion_ids, return total +class TestMlEngineerManifest: + """Tests pour le manifeste ML Engineer (#19545, EPIC #19543 pli 1). + + Couvre : (a) tous les notebooks cites existent sur disque ; + (b) le manifeste compile en 7 branches + 5 accretions ; + (c) la duree totale tient en moins de 50 h ; + (d) pas de chemin absent du catalogue (le manifeste n'ajoute pas + de carnet orphelin non catalogue par generate_catalog). + """ + manifest_path = gp.REPO_ROOT / "docs" / "curriculum" / "ml-engineer.json" + + def test_all_selected_notebooks_exist_on_disk(self): + manifest = json.loads(self.manifest_path.read_text(encoding="utf-8")) + selected = [path for group in manifest["branches"] + manifest["accretions"] + for path in group["notebooks"]] + assert len(selected) == len(set(selected)), "duplicate path in manifest" + assert all((gp.REPO_ROOT / "MyIA.AI.Notebooks" / path).is_file() + for path in selected), \ + f"chemins introuvables sur disque: {[p for p in selected if not (gp.REPO_ROOT / 'MyIA.AI.Notebooks' / p).is_file()]}" + + def test_all_paths_are_in_catalog(self): + """Le manifeste ne reference pas un carnet absent du catalogue. + + Si une accretion n'est pas encore cataloguee (generate_catalog + gele), la verification de production compile_parcours echoue. + La regle est explicite : on n'indexe pas un carnet orphelin dans + un manifeste ; le drop ou la regeneration du catalogue relevent + d'un autre grain. + """ + manifest = json.loads(self.manifest_path.read_text(encoding="utf-8")) + catalog = _catalog_with_pending_renames(manifest) + catalog_paths = {entry["path"] for entry in catalog} + selected = [path for group in manifest["branches"] + manifest["accretions"] + for path in group["notebooks"]] + missing = [p for p in selected if p not in catalog_paths] + assert not missing, \ + f"chemins absents du catalogue (regenerer ou drop): {missing}" + + def test_compiles_seven_branches_and_five_accretions(self): + manifest = json.loads(self.manifest_path.read_text(encoding="utf-8")) + catalog = _catalog_with_pending_renames(manifest) + branches = [b["id"] for b in manifest["branches"]] + accretions = [a["id"] for a in manifest["accretions"]] + compiled = gp.compile_parcours(catalog, manifest, branches, accretions) + assert len(compiled["groups"]) == len(branches) + len(accretions) + assert [g["id"] for g in compiled["groups"][:len(branches)]] == branches + + def test_total_duration_under_50_hours(self): + """Le parcours complet tient en moins de 50 h, sinon c'est un autre format qu'un parcours.""" + manifest = json.loads(self.manifest_path.read_text(encoding="utf-8")) + catalog = _catalog_with_pending_renames(manifest) + compiled = gp.compile_parcours( + catalog, manifest, + [b["id"] for b in manifest["branches"]], + [a["id"] for a in manifest["accretions"]], + ) + assert compiled["duration_minutes"] < 50 * 60, ( + f"duree totale {compiled['duration_minutes']} min >= 50 h, " + f"le parcours n'est plus un itineraire mais une encyclopedie" + ) + + def test_ingenierie_threading(self): + """Le fil d'ingenierie est explicite : les descriptions de branche + portent au moins une mention de reproductibilite / production / + mise en service / derive / cout sur les branches profondes. + + Branches superficielles (donnees, workflow-ml) peuvent ne pas + le mentionner : le socle est universel. + """ + manifest = json.loads(self.manifest_path.read_text(encoding="utf-8")) + keywords = ["reproductib", "production", "service", "déploy", + "drift", "drift", "cout", "coût", "calibration", "deploiement"] + for branch in manifest["branches"]: + if branch["id"] in {"donnees", "workflow-ml"}: + continue + desc_lower = branch["description"].lower() + assert any(kw.lower() in desc_lower for kw in keywords), ( + f"branche {branch['id']} sans mention d'ingenierie " + f"(reproductibilite/production/service/deploy/cout/calibration)" + ) + + class TestActuariatManifest: manifest_path = gp.REPO_ROOT / "docs" / "curriculum" / "actuariat.json"