From 5f63163fb18e752dd9b044a2a533623e1339e620 Mon Sep 17 00:00:00 2001 From: Jonas Rembser Date: Sat, 9 May 2026 12:39:44 +0200 Subject: [PATCH] [Python] Stream notebook cell output instead of buffering until done Long-running C++ cells in Jupyter currently produce no output until the cell finishes, because by default they hold the GIL for the entire duration of the call. The StreamCapture polling thread that is supposed to drain captured stdout/stderr to the frontend cannot run until the C++ work is over, so users see nothing while their code runs. This commit suggests to release the GIL on the heavy TInterpreter entry points (`ProcessLine`, `ProcessLineSynch`, `Declare`, `LoadFile`, `LoadMacro`, `ExecuteMacro`) by setting `__release_gil__` on them in the facade. With the GIL free during C++ execution, the polling thread can do its work. To stream the live output correctly, we have the polling thread flush whatever it has captured on each iteration, so output reaches the notebook as it is produced. When output transformers (like HTML pretty-printing) are registered we still accumulate everything and hand the full text to them in `post_execute`, since transformers operate on a complete cell output. The `post_execute` path now flushes any trailing bytes captured between the last poll and EndCapture. While doint these changes, drop the ProcessLineWrapper C++ helper used by processMagicCppCodeImpl. It existed only to return the EErrorCode by value; we can pass a pointer from Python directly via cppyy.ll, which removes a JIT-compiled helper from JupyROOT startup. This avoids the overhead of one interpreter call and removed one more function that we'd have to remember to set `__release_gil__` for. Closes #22170. --- .../pythonizations/python/ROOT/_facade.py | 8 +++ .../python/ROOT/_jupyroot/helpers/utils.py | 51 ++++++++++++------- 2 files changed, 40 insertions(+), 19 deletions(-) diff --git a/bindings/pyroot/pythonizations/python/ROOT/_facade.py b/bindings/pyroot/pythonizations/python/ROOT/_facade.py index 8b37d45f89b4e..643952d9158c8 100644 --- a/bindings/pyroot/pythonizations/python/ROOT/_facade.py +++ b/bindings/pyroot/pythonizations/python/ROOT/_facade.py @@ -300,6 +300,14 @@ def _finalSetup(self): # Make sure the interpreter is initialized once gROOT has been initialized self._cppyy.gbl.TInterpreter.Instance() + # Release the GIL on the heavy TInterpreter functions. This lets + # background Python threads make progress - in particular, JupyROOT's + # StreamCapture polling thread can drain stdout/stderr live to the + # notebook frontend instead of waiting for the cell to finish. + TInterpreter = self._cppyy.gbl.TInterpreter + for name in ("ProcessLine", "ProcessLineSynch", "Declare", "LoadFile", "LoadMacro", "ExecuteMacro"): + getattr(TInterpreter, name).__release_gil__ = True + # Setup interactive usage from Python self.__dict__["app"] = PyROOTApplication(self.PyConfig, self._is_ipython) if not self.gROOT.IsBatch() and self.PyConfig.StartGUIThread: diff --git a/bindings/pyroot/pythonizations/python/ROOT/_jupyroot/helpers/utils.py b/bindings/pyroot/pythonizations/python/ROOT/_jupyroot/helpers/utils.py index 9978a376d4eeb..58c2f195a43e8 100644 --- a/bindings/pyroot/pythonizations/python/ROOT/_jupyroot/helpers/utils.py +++ b/bindings/pyroot/pythonizations/python/ROOT/_jupyroot/helpers/utils.py @@ -15,13 +15,13 @@ from __future__ import print_function +import ctypes import fnmatch import os import re import sys import tempfile import time -import ctypes from contextlib import contextmanager from datetime import datetime from hashlib import sha1 @@ -293,8 +293,12 @@ def processCppCodeImpl(code): def processMagicCppCodeImpl(code): - err = ROOT.ProcessLineWrapper(code) - if err == ROOT.TInterpreter.kProcessing: + import cppyy.ll + + err = ctypes.c_int(0) + err_ptr = cppyy.ll.reinterpret_cast["TInterpreter::EErrorCode*"](ctypes.addressof(err)) + ROOT.gInterpreter.ProcessLine(code, err_ptr) + if err.value == ROOT.TInterpreter.kProcessing: ROOT.gInterpreter.ProcessLine(".@") ROOT.gInterpreter.ProcessLine('cerr << "Unbalanced braces. This cell was not processed." << endl;') @@ -408,6 +412,20 @@ def __init__(self, ip=get_ipython()): self.isFirstPreExecute = True self.isFirstPostExecute = True + def _flushCaptured(self): + # Drain whatever the polling thread has captured so far to the + # notebook's stdout/stderr. + out = self.ioHandler.GetStdout() + err = self.ioHandler.GetStderr() + if out: + sys.stdout.write(out) + sys.stdout.flush() + if err: + sys.stderr.write(err) + sys.stderr.flush() + if out or err: + self.ioHandler.Clear() + def syncCapture(self, defout=""): self.outString = defout self.errString = defout @@ -417,6 +435,11 @@ def syncCapture(self, defout=""): iterIndex = 0 while self.flag: self.ioHandler.Poll() + # Stream ouput live so long-running C++ code shows progress in the + # notebook as it runs. Only when transformers are registered, we + # keep accumulating and get the full output in post_execute. + if not transformers: + self._flushCaptured() if not self.flag: return waitTime = 0.1 if iterIndex >= lenWaitTimes else waitTimes[iterIndex] @@ -442,13 +465,14 @@ def post_execute(self): self.ioHandler.Poll() self.ioHandler.EndCapture() - # Print for the notebook - out = self.ioHandler.GetStdout() - err = self.ioHandler.GetStderr() + # Flush anything that arrived between the polling thread's last + # iteration and EndCapture. With transformers registered nothing has + # been streamed yet, so this hands them the full cell output. if not transformers: - sys.stdout.write(out) - sys.stderr.write(err) + self._flushCaptured() else: + out = self.ioHandler.GetStdout() + err = self.ioHandler.GetStderr() for t in transformers: (out, err, otype) = t(out, err) if otype == "html": @@ -906,16 +930,6 @@ def loadMagicsAndCapturers(): capture.register() -def declareProcessLineWrapper(): - ROOT.gInterpreter.Declare(""" -TInterpreter::EErrorCode ProcessLineWrapper(const char* line) { - TInterpreter::EErrorCode err; - gInterpreter->ProcessLine(line, &err); - return err; -} -""") - - def enhanceROOTModule(): ROOT.enableJSVis = enableJSVis @@ -930,6 +944,5 @@ def iPythonize(): setStyle() initializeJSVis() loadMagicsAndCapturers() - declareProcessLineWrapper() # enableCppHighlighting() enhanceROOTModule()