From 708987fb17cc1a9f2ab8dbee2ed75958d0d6c1d4 Mon Sep 17 00:00:00 2001 From: William Silversmith Date: Wed, 9 Apr 2025 15:28:12 -0400 Subject: [PATCH 01/21] refactor: drop posix_ipc and use native python shared_memory --- cloudvolume/sharedmemory.py | 21 +++++++++------------ setup.py | 1 - 2 files changed, 9 insertions(+), 13 deletions(-) diff --git a/cloudvolume/sharedmemory.py b/cloudvolume/sharedmemory.py index 79aaee56..d082a170 100644 --- a/cloudvolume/sharedmemory.py +++ b/cloudvolume/sharedmemory.py @@ -127,9 +127,7 @@ def allocate_shm_file(filename, nbytes, dbytes, readonly): def ndarray_shm(shape, dtype, location, readonly=False, order='F', **kwargs): """Create a shared memory numpy array. Requires /dev/shm to exist.""" - import posix_ipc - from posix_ipc import O_CREAT - import psutil + from multiprocessing import shared_memory nbytes = Vec(*shape).rectVolume() * np.dtype(dtype).itemsize available = psutil.virtual_memory().available @@ -170,17 +168,15 @@ def ndarray_shm(shape, dtype, location, readonly=False, order='F', **kwargs): # a threading condition where the condition of the shared memory # was adjusted between the check above and now. Better to make sure # that we don't accidently change anything if readonly is set. - flags = 0 if readonly else O_CREAT size = 0 if readonly else int(nbytes) try: - shared = posix_ipc.SharedMemory(location, flags=flags, size=size) - array_like = mmap.mmap(shared.fd, shared.size) - os.close(shared.fd) - renderbuffer = np.ndarray(buffer=array_like, dtype=dtype, shape=shape, order=order, **kwargs) + shm = shared_memory.SharedMemory(name=location, create=(not readonly), size=size, track=(not readonly)) + renderbuffer = np.frombuffer(buffer=shm.buffer, dtype=dtype) + renderbuffer = renderbuffer.reshape(shape, order=order) except OSError as err: if err.errno == errno.ENOMEM: # Out of Memory - posix_ipc.unlink_shared_memory(location) + unlink_shm(location) raise renderbuffer.setflags(write=(not readonly)) @@ -192,10 +188,11 @@ def unlink(location): return unlink_shm(location) def unlink_shm(location): - import posix_ipc + from multiprocessing import shared_memory try: - posix_ipc.unlink_shared_memory(location) - except posix_ipc.ExistentialError: + shm = shared_memory.SharedMemory(name=location, create=False) + shm.unlink() + except FileNotFoundError: return False return True diff --git a/setup.py b/setup.py index 6e23ea6f..ae506f54 100644 --- a/setup.py +++ b/setup.py @@ -56,7 +56,6 @@ def requirements(): "blosc", ], ':sys_platform!="win32"': [ - "posix_ipc>=1.0.4", "psutil>=5.4.3", ], "mesh_viewer": [ 'vtk' ], From 51a972f8ba432eb67645842f98057ab06ecb5717 Mon Sep 17 00:00:00 2001 From: William Silversmith Date: Wed, 9 Apr 2025 18:21:51 -0400 Subject: [PATCH 02/21] fix: missing import --- cloudvolume/sharedmemory.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cloudvolume/sharedmemory.py b/cloudvolume/sharedmemory.py index d082a170..ff800911 100644 --- a/cloudvolume/sharedmemory.py +++ b/cloudvolume/sharedmemory.py @@ -128,6 +128,7 @@ def allocate_shm_file(filename, nbytes, dbytes, readonly): def ndarray_shm(shape, dtype, location, readonly=False, order='F', **kwargs): """Create a shared memory numpy array. Requires /dev/shm to exist.""" from multiprocessing import shared_memory + import psutil nbytes = Vec(*shape).rectVolume() * np.dtype(dtype).itemsize available = psutil.virtual_memory().available From 29e645ca827b53a51efb0bcbcb805e6136ec7965 Mon Sep 17 00:00:00 2001 From: William Silversmith Date: Wed, 9 Apr 2025 18:25:33 -0400 Subject: [PATCH 03/21] fix: track is not well supported --- cloudvolume/sharedmemory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudvolume/sharedmemory.py b/cloudvolume/sharedmemory.py index ff800911..b9c82baf 100644 --- a/cloudvolume/sharedmemory.py +++ b/cloudvolume/sharedmemory.py @@ -172,7 +172,7 @@ def ndarray_shm(shape, dtype, location, readonly=False, order='F', **kwargs): size = 0 if readonly else int(nbytes) try: - shm = shared_memory.SharedMemory(name=location, create=(not readonly), size=size, track=(not readonly)) + shm = shared_memory.SharedMemory(name=location, create=(not readonly), size=size) renderbuffer = np.frombuffer(buffer=shm.buffer, dtype=dtype) renderbuffer = renderbuffer.reshape(shape, order=order) except OSError as err: From 25a60156b50161886018429482938befffa4fbdb Mon Sep 17 00:00:00 2001 From: William Silversmith Date: Wed, 9 Apr 2025 18:26:17 -0400 Subject: [PATCH 04/21] fix: incorrect name for buffer attribute --- cloudvolume/sharedmemory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudvolume/sharedmemory.py b/cloudvolume/sharedmemory.py index b9c82baf..3429c437 100644 --- a/cloudvolume/sharedmemory.py +++ b/cloudvolume/sharedmemory.py @@ -173,7 +173,7 @@ def ndarray_shm(shape, dtype, location, readonly=False, order='F', **kwargs): try: shm = shared_memory.SharedMemory(name=location, create=(not readonly), size=size) - renderbuffer = np.frombuffer(buffer=shm.buffer, dtype=dtype) + renderbuffer = np.frombuffer(buffer=shm.buf, dtype=dtype) renderbuffer = renderbuffer.reshape(shape, order=order) except OSError as err: if err.errno == errno.ENOMEM: # Out of Memory From 0a68c6ae4d507e1ca07bcb0f4c8ea992f6f36d0d Mon Sep 17 00:00:00 2001 From: William Silversmith Date: Wed, 9 Apr 2025 18:27:42 -0400 Subject: [PATCH 05/21] fix: return shm instead of old array_like --- cloudvolume/sharedmemory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudvolume/sharedmemory.py b/cloudvolume/sharedmemory.py index 3429c437..75e6448e 100644 --- a/cloudvolume/sharedmemory.py +++ b/cloudvolume/sharedmemory.py @@ -181,7 +181,7 @@ def ndarray_shm(shape, dtype, location, readonly=False, order='F', **kwargs): raise renderbuffer.setflags(write=(not readonly)) - return array_like, renderbuffer + return shm, renderbuffer def unlink(location): if EMULATE_SHM: From 35b5910c4943513d987db49b4147450eb57d2500 Mon Sep 17 00:00:00 2001 From: William Silversmith Date: Wed, 9 Apr 2025 18:32:43 -0400 Subject: [PATCH 06/21] fix: try deleting shared image first --- cloudvolume/frontends/precomputed.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cloudvolume/frontends/precomputed.py b/cloudvolume/frontends/precomputed.py index 8d3f7bba..eaba007f 100644 --- a/cloudvolume/frontends/precomputed.py +++ b/cloudvolume/frontends/precomputed.py @@ -1127,6 +1127,7 @@ def upload_from_shared_memory(self, location, bbox, order='F', cutout_bbox=None) order=order, use_shared_memory=True, ) + del shared_image mmap_handle.close() def upload_from_file(self, location, bbox, order='F', cutout_bbox=None): @@ -1188,6 +1189,7 @@ def upload_from_file(self, location, bbox, order='F', cutout_bbox=None): order=order, use_file=True, ) + del shared_image mmap_handle.close() def viewer(self, port=1337): From 5030279a7d8ea8c4e0343f0dfe0bd1e4af633267 Mon Sep 17 00:00:00 2001 From: William Silversmith Date: Wed, 9 Apr 2025 18:34:09 -0400 Subject: [PATCH 07/21] fix: try just letting __del__ handle things --- cloudvolume/frontends/precomputed.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cloudvolume/frontends/precomputed.py b/cloudvolume/frontends/precomputed.py index eaba007f..10618b37 100644 --- a/cloudvolume/frontends/precomputed.py +++ b/cloudvolume/frontends/precomputed.py @@ -1189,8 +1189,8 @@ def upload_from_file(self, location, bbox, order='F', cutout_bbox=None): order=order, use_file=True, ) - del shared_image - mmap_handle.close() + # del shared_image + # mmap_handle.close() def viewer(self, port=1337): import cloudvolume.server From c27b9c11082628e73232e686c003836d7ffbafbb Mon Sep 17 00:00:00 2001 From: William Silversmith Date: Wed, 9 Apr 2025 18:35:59 -0400 Subject: [PATCH 08/21] fix: try just deleting the shared array --- cloudvolume/frontends/precomputed.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cloudvolume/frontends/precomputed.py b/cloudvolume/frontends/precomputed.py index 10618b37..22b3ca30 100644 --- a/cloudvolume/frontends/precomputed.py +++ b/cloudvolume/frontends/precomputed.py @@ -1128,7 +1128,7 @@ def upload_from_shared_memory(self, location, bbox, order='F', cutout_bbox=None) use_shared_memory=True, ) del shared_image - mmap_handle.close() + # mmap_handle.close() def upload_from_file(self, location, bbox, order='F', cutout_bbox=None): """ @@ -1189,8 +1189,8 @@ def upload_from_file(self, location, bbox, order='F', cutout_bbox=None): order=order, use_file=True, ) - # del shared_image - # mmap_handle.close() + del shared_image + mmap_handle.close() def viewer(self, port=1337): import cloudvolume.server From f19b5232280a85e2993db9541862d14d5d62e77e Mon Sep 17 00:00:00 2001 From: William Silversmith Date: Wed, 9 Apr 2025 18:38:59 -0400 Subject: [PATCH 09/21] fix: delete renderbuffer before calling close --- cloudvolume/datasource/precomputed/image/tx.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cloudvolume/datasource/precomputed/image/tx.py b/cloudvolume/datasource/precomputed/image/tx.py index 3c361a8d..d412ae57 100644 --- a/cloudvolume/datasource/precomputed/image/tx.py +++ b/cloudvolume/datasource/precomputed/image/tx.py @@ -214,6 +214,7 @@ def upload_aligned( # If manual mode is enabled, it's the # responsibilty of the user to clean up if not use_shared_memory: + del renderbuffer array_like.close() shm.unlink(location) From a1b035d9ca8f69e7ed7171d7e3bebf8ad27513b6 Mon Sep 17 00:00:00 2001 From: William Silversmith Date: Wed, 9 Apr 2025 18:44:50 -0400 Subject: [PATCH 10/21] fix: ensure shm is closed --- .../datasource/precomputed/image/tx.py | 25 ++++++++++--------- 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/cloudvolume/datasource/precomputed/image/tx.py b/cloudvolume/datasource/precomputed/image/tx.py index d412ae57..9c3ab92a 100644 --- a/cloudvolume/datasource/precomputed/image/tx.py +++ b/cloudvolume/datasource/precomputed/image/tx.py @@ -205,18 +205,19 @@ def upload_aligned( secrets=secrets ) - parallel_execution( - cup, chunk_ranges, parallel, - progress, desc="Upload", - cleanup_shm=location - ) - - # If manual mode is enabled, it's the - # responsibilty of the user to clean up - if not use_shared_memory: - del renderbuffer - array_like.close() - shm.unlink(location) + try: + parallel_execution( + cup, chunk_ranges, parallel, + progress, desc="Upload", + cleanup_shm=location + ) + finally: + # If manual mode is enabled, it's the + # responsibilty of the user to clean up + if not use_shared_memory: + del renderbuffer + array_like.close() + shm.unlink(location) def child_upload_process( meta, cache, From 2291fbc8e0ad2c3e306b68272c417936cdb1e5c6 Mon Sep 17 00:00:00 2001 From: William Silversmith Date: Wed, 9 Apr 2025 18:48:41 -0400 Subject: [PATCH 11/21] fix: delete renderbuffer in child process --- cloudvolume/datasource/precomputed/image/tx.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cloudvolume/datasource/precomputed/image/tx.py b/cloudvolume/datasource/precomputed/image/tx.py index 9c3ab92a..f7dca21d 100644 --- a/cloudvolume/datasource/precomputed/image/tx.py +++ b/cloudvolume/datasource/precomputed/image/tx.py @@ -269,6 +269,7 @@ def updatefn(): secrets=secrets, ) finally: + del renderbuffer array_like.close() def threaded_upload_chunks( From 5188cb81a436acca9afc53c45394199c30fee853 Mon Sep 17 00:00:00 2001 From: William Silversmith Date: Mon, 29 Sep 2025 15:09:28 -0400 Subject: [PATCH 12/21] fix: ensure all views are closed --- cloudvolume/frontends/precomputed.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cloudvolume/frontends/precomputed.py b/cloudvolume/frontends/precomputed.py index 22b3ca30..a1cb8a34 100644 --- a/cloudvolume/frontends/precomputed.py +++ b/cloudvolume/frontends/precomputed.py @@ -1127,8 +1127,9 @@ def upload_from_shared_memory(self, location, bbox, order='F', cutout_bbox=None) order=order, use_shared_memory=True, ) + del cutout_image del shared_image - # mmap_handle.close() + mmap_handle.close() def upload_from_file(self, location, bbox, order='F', cutout_bbox=None): """ From d20f3e6f4fb1adb56bc21202eb3fd76b9a07d725 Mon Sep 17 00:00:00 2001 From: William Silversmith Date: Mon, 29 Sep 2025 15:15:46 -0400 Subject: [PATCH 13/21] fix: ensure fs_lock is created in a spawn context --- .../datasource/precomputed/image/common.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/cloudvolume/datasource/precomputed/image/common.py b/cloudvolume/datasource/precomputed/image/common.py index 5250308f..9f9f1360 100644 --- a/cloudvolume/datasource/precomputed/image/common.py +++ b/cloudvolume/datasource/precomputed/image/common.py @@ -63,12 +63,17 @@ def parallel_execution( ): global error_queue - error_queue = mp.Queue() - progress_queue = mp.Queue() - fs_lock = mp.Lock() + # Ensure processes do not accidentally inherit + # locks from forking and deadlock. Instead launch + # new interpreters. + ctx = mp.get_context("spawn") + + error_queue = ctx.Queue() + progress_queue = ctx.Queue() + fs_lock = ctx.Lock() if parallel is True: - parallel = mp.cpu_count() + parallel = ctx.cpu_count() elif parallel <= 0: raise ValueError(f"Parallel must be a positive number or boolean (True: all cpus). Got: {parallel}") @@ -104,11 +109,6 @@ def cleanup(signum, frame): ) proc.start() - # Ensure processes do not accidentally inherit - # locks from forking and deadlock. Instead launch - # new interpreters. - ctx = mp.get_context("spawn") - with concurrent.futures.ProcessPoolExecutor( max_workers=parallel, initializer=initialize_synchronization, From d0d3a10cff957ee4e019d6dd9c3646264c5a667f Mon Sep 17 00:00:00 2001 From: William Silversmith Date: Mon, 29 Sep 2025 15:22:32 -0400 Subject: [PATCH 14/21] fix: use shmloc not bare location --- cloudvolume/sharedmemory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudvolume/sharedmemory.py b/cloudvolume/sharedmemory.py index 75e6448e..03161bd5 100644 --- a/cloudvolume/sharedmemory.py +++ b/cloudvolume/sharedmemory.py @@ -172,7 +172,7 @@ def ndarray_shm(shape, dtype, location, readonly=False, order='F', **kwargs): size = 0 if readonly else int(nbytes) try: - shm = shared_memory.SharedMemory(name=location, create=(not readonly), size=size) + shm = shared_memory.SharedMemory(name=shmloc, create=(not readonly), size=size) renderbuffer = np.frombuffer(buffer=shm.buf, dtype=dtype) renderbuffer = renderbuffer.reshape(shape, order=order) except OSError as err: From 48e4586243228bbc19393a05d74d54f62ac7d630 Mon Sep 17 00:00:00 2001 From: William Silversmith Date: Mon, 29 Sep 2025 15:28:10 -0400 Subject: [PATCH 15/21] Revert "fix: use shmloc not bare location" This reverts commit d0d3a10cff957ee4e019d6dd9c3646264c5a667f. --- cloudvolume/sharedmemory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudvolume/sharedmemory.py b/cloudvolume/sharedmemory.py index 03161bd5..75e6448e 100644 --- a/cloudvolume/sharedmemory.py +++ b/cloudvolume/sharedmemory.py @@ -172,7 +172,7 @@ def ndarray_shm(shape, dtype, location, readonly=False, order='F', **kwargs): size = 0 if readonly else int(nbytes) try: - shm = shared_memory.SharedMemory(name=shmloc, create=(not readonly), size=size) + shm = shared_memory.SharedMemory(name=location, create=(not readonly), size=size) renderbuffer = np.frombuffer(buffer=shm.buf, dtype=dtype) renderbuffer = renderbuffer.reshape(shape, order=order) except OSError as err: From 2df2fbcabd448d1f93744b3a506d8816b2abc1e3 Mon Sep 17 00:00:00 2001 From: William Silversmith Date: Mon, 29 Sep 2025 15:30:30 -0400 Subject: [PATCH 16/21] fix: check for preexisting --- cloudvolume/sharedmemory.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cloudvolume/sharedmemory.py b/cloudvolume/sharedmemory.py index 75e6448e..4f94e1a3 100644 --- a/cloudvolume/sharedmemory.py +++ b/cloudvolume/sharedmemory.py @@ -170,9 +170,10 @@ def ndarray_shm(shape, dtype, location, readonly=False, order='F', **kwargs): # was adjusted between the check above and now. Better to make sure # that we don't accidently change anything if readonly is set. size = 0 if readonly else int(nbytes) + create = (not readonly) and (not preexisting) try: - shm = shared_memory.SharedMemory(name=location, create=(not readonly), size=size) + shm = shared_memory.SharedMemory(name=location, create=create, size=size) renderbuffer = np.frombuffer(buffer=shm.buf, dtype=dtype) renderbuffer = renderbuffer.reshape(shape, order=order) except OSError as err: From 344829572893ed423e95ca3a7fdd316cad1646ab Mon Sep 17 00:00:00 2001 From: William Silversmith Date: Mon, 29 Sep 2025 15:43:44 -0400 Subject: [PATCH 17/21] fixtest: delete hanging reference --- test/test_cloudvolume.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/test_cloudvolume.py b/test/test_cloudvolume.py index 5bfa25ff..78fa327f 100644 --- a/test/test_cloudvolume.py +++ b/test/test_cloudvolume.py @@ -470,6 +470,7 @@ def test_parallel_shared_memory_write(): assert np.all(cv[0,0,:] == 1) assert np.all(cv[1,0,:] == 0) + del shareddata mmapfh.close() shm.unlink(shm_location) From 12b903b26db6308594065aee11af96201090852f Mon Sep 17 00:00:00 2001 From: William Silversmith Date: Mon, 29 Sep 2025 15:50:22 -0400 Subject: [PATCH 18/21] fix: handle file exists --- cloudvolume/sharedmemory.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cloudvolume/sharedmemory.py b/cloudvolume/sharedmemory.py index 4f94e1a3..35efb8d2 100644 --- a/cloudvolume/sharedmemory.py +++ b/cloudvolume/sharedmemory.py @@ -176,6 +176,12 @@ def ndarray_shm(shape, dtype, location, readonly=False, order='F', **kwargs): shm = shared_memory.SharedMemory(name=location, create=create, size=size) renderbuffer = np.frombuffer(buffer=shm.buf, dtype=dtype) renderbuffer = renderbuffer.reshape(shape, order=order) + except FileExistsError: + if not readonly: + raise + shm = shared_memory.SharedMemory(name=location, create=False, size=size) + renderbuffer = np.frombuffer(buffer=shm.buf, dtype=dtype) + renderbuffer = renderbuffer.reshape(shape, order=order) except OSError as err: if err.errno == errno.ENOMEM: # Out of Memory unlink_shm(location) From 42ee492e74133b547e6d94f3d65557f21033d0cd Mon Sep 17 00:00:00 2001 From: William Silversmith Date: Mon, 29 Sep 2025 15:50:33 -0400 Subject: [PATCH 19/21] fixtest: remove dangling references --- test/test_sharedmemory.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/test_sharedmemory.py b/test/test_sharedmemory.py index e1a61a90..500152c6 100644 --- a/test/test_sharedmemory.py +++ b/test/test_sharedmemory.py @@ -109,5 +109,7 @@ def test_ndarray_sh(): except shm.SharedMemoryReadError: pass + del array + array_like.close() assert shm.unlink_shm(location) == True assert shm.unlink_shm(location) == False From 8977a4e9bf8a67696786275520de485c689fd2b9 Mon Sep 17 00:00:00 2001 From: William Silversmith Date: Mon, 29 Sep 2025 15:51:51 -0400 Subject: [PATCH 20/21] fixtest: remove other references to array before closing --- test/test_sharedmemory.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/test_sharedmemory.py b/test/test_sharedmemory.py index 500152c6..3d93cfc1 100644 --- a/test/test_sharedmemory.py +++ b/test/test_sharedmemory.py @@ -68,10 +68,12 @@ def test_ndarray_sh(): array_like, array = shm.ndarray_shm(shape=(2,2,2), dtype=np.uint8, location=location) assert np.all(array == np.zeros(shape=(2,2,2), dtype=np.uint8)) array[:] = 100 + del array array_like.close() array_like, array = shm.ndarray_shm(shape=(2,2,2), dtype=np.uint8, location=location) assert np.all(array[:] == 100) + del array array_like.close() filename = os.path.join(shm.SHM_DIRECTORY, location) @@ -86,7 +88,9 @@ def test_ndarray_sh(): available = psutil.virtual_memory().available array_like, array = shm.ndarray_shm(shape=(available // 10,2,2), dtype=np.uint8, location=location) + del array array_like.close() + try: array_like, array = shm.ndarray_shm(shape=(available,2,2), dtype=np.uint8, location=location) assert False From b6e1c84f3bb6b5097050387759d29cab58e54bd9 Mon Sep 17 00:00:00 2001 From: William Silversmith Date: Mon, 29 Sep 2025 15:57:43 -0400 Subject: [PATCH 21/21] fix: delete dangling reference --- cloudvolume/datasource/precomputed/image/rx.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cloudvolume/datasource/precomputed/image/rx.py b/cloudvolume/datasource/precomputed/image/rx.py index fdd5004a..ba90b0b2 100644 --- a/cloudvolume/datasource/precomputed/image/rx.py +++ b/cloudvolume/datasource/precomputed/image/rx.py @@ -562,6 +562,7 @@ def process(src_img, src_bbox): green=green, secrets=secrets, background_color=background_color ) + del dest_img array_like.close() return len(cloudpaths)