Data Handling Fixes: Out of Bounds & Local

python/compose.yml

@@ -9,8 +9,9 @@ services:
     volumes:
       - ouroboros-volume:/volume
     extra_hosts:
-    - "host.docker.internal:host-gateway"
-
+      - "host.docker.internal:host-gateway"
+    environment:
+      - OUR_ENV=docker
 volumes:
   ouroboros-volume:
     name: ouroboros-volume

python/ouroboros/helpers/volume_cache.py

@@ -1,7 +1,9 @@
+from dataclasses import astuple
+import os
 import sys
 import traceback
 
-from cloudvolume import CloudVolume, VolumeCutout
+from cloudvolume import CloudVolume, VolumeCutout, Bbox
 import numpy as np
 
 from .bounding_boxes import BoundingBox, boxes_dim_range
@@ -157,16 +159,26 @@ def remove_volume(self, volume_index: int, destroy_shared: bool = False):
     def download_volume(
         self, volume_index: int, bounding_box: BoundingBox, parallel=False
     ) -> VolumeCutout:
-        bbox = bounding_box.to_cloudvolume_bbox()
+        bbox = bounding_box.to_cloudvolume_bbox().astype(int)
+        vol_shape = NGOrder(*bbox.size3(), self.cv.cv.num_channels)
+
+        # Limit size of area we are grabbing, in case we go out of bounds.
+        dl_box = Bbox.intersection(self.cv.cv.bounds, bbox)
+        local_min = [int(start) for start in np.subtract(dl_box.minpt, bbox.minpt)]
+
+        local_bounds = np.s_[*[slice(start, stop) for start, stop in
+                               zip(local_min, np.sum([local_min, dl_box.size3()], axis=0))],
+                             :]
 
         # Download the bounding box volume
         if self.use_shared:
-            vol_shape = NGOrder(*bbox.astype(int).size3(), self.cv.cv.num_channels)
             volume = self.shm_host.SharedNPArray(vol_shape, np.float32)
             with volume as volume_data:
-                volume_data[:] = self.cv.cv.download(bbox, mip=self.mip, parallel=parallel)
+                volume_data[:] = 0     # Prob not most efficient but makes math much easier
+                volume_data[local_bounds] = self.cv.cv.download(dl_box, mip=self.mip, parallel=parallel)
         else:
-            volume = self.cv.cv.download(bbox, mip=self.mip, parallel=parallel)
+            volume = np.zeros(astuple(vol_shape))
+            volume[local_bounds] = self.cv.cv.download(dl_box, mip=self.mip, parallel=parallel)
 
         # Store the volume in the cache
         self.volumes[volume_index] = volume
@@ -214,6 +226,9 @@ def flush_local_cache(self):
 class CloudVolumeInterface:
     def __init__(self, source_url: str):
         self.source_url = source_url
+        if os.environ.get('OUR_ENV') == "docker":
+            self.source_url = self.source_url.replace("localhost", "host.docker.internal"
+                                                      ).replace("127.0.0.1", "host.docker.internal")
 
         self.cv = CloudVolume(self.source_url, parallel=True, cache=True)
 

python/ouroboros/pipeline/slice_parallel_pipeline.py

@@ -103,7 +103,7 @@ def _process(self, input_data: tuple[any]) -> None | str:
                     config.slice_width,
                     config.slice_height,
                 ) + ((num_color_channels,) if has_color_channels else ())
-                temp_data = np.zeros(temp_shape, dtype=volume_cache.get_volume_dtype())
+                temp_data = np.zeros(temp_shape, dtype=np.float16)
 
                 imwrite(
                     output_file_path,
@@ -116,7 +116,7 @@ def _process(self, input_data: tuple[any]) -> None | str:
                         if has_color_channels and num_color_channels > 1
                         else "minisblack"
                     ),
-                    metadata=metadata,
+                    metadata=metadata
                 )
             except BaseException as e:
                 return f"Error creating single tif file: {e}"

python/test/helpers/test_volume_cache.py

@@ -1,5 +1,9 @@
 import pytest
 from unittest.mock import MagicMock, patch
+
+from cloudvolume import Bbox
+import numpy as np
+
 from ouroboros.helpers.volume_cache import (
     VolumeCache,
     CloudVolumeInterface,
@@ -19,6 +23,8 @@ def mock_cloud_volume():
         mock_cv.shape = (100, 100, 100, 3)
         mock_cv.cache.flush = MagicMock()
         mock_cv.mip_volume_size = lambda mip: (100, 100, 100)
+        mock_cv.bounds = Bbox((0, 0, 0), (100, 100, 100))
+        mock_cv.download.return_value = np.zeros((0, 10, 0, 1))
         yield mock_cv
 
 
@@ -173,8 +179,8 @@ def test_request_volume_for_slice(volume_cache):
         volume_data, bounding_box = volume_cache.request_volume_for_slice(slice_index)
 
         mock_volume_index.assert_called_once_with(slice_index)
-        assert bounding_box == volume_cache.bounding_boxes[1]
-        assert volume_data == volume_cache.volumes[1]
+        assert np.all(bounding_box == volume_cache.bounding_boxes[1])
+        assert np.all(volume_data == volume_cache.volumes[1])
 
 
 def test_create_processing_data(volume_cache):
@@ -234,7 +240,7 @@ def test_volume_cache_remove_volume(volume_cache):
         volume_data, bounding_box = volume_cache.request_volume_for_slice(slice_index)
 
         mock_volume_index.assert_called_once_with(slice_index)
-        assert volume_data == volume_cache.volumes[1]
+        assert np.all(volume_data == volume_cache.volumes[1])
         volume_cache.remove_volume(1)
         assert volume_cache.volumes[1] is None
 

src/main/servers/main-server.ts

@@ -8,18 +8,14 @@ const PRODUCTION_PATH = join(__dirname, '../../../extra-resources/server/')
 const PRODUCTION_CONFIG = join(PRODUCTION_PATH, 'compose.yml')
 
 export async function startMainServerDevelopment(): Promise<void> {
-	try {
-		await startDockerCompose({
-			cwd: DEVELOPMENT_PATH,
-			config: DEVELOPMENT_CONFIG,
-			build: true,
-			onError: (err) => {
-				console.error('An error occurred while starting the main server:', err)
-			}
-		})
-	} catch (error) {
-		console.error('An error occurred while starting the main server:', error)
-	}
+	await startDockerCompose({
+		cwd: DEVELOPMENT_PATH,
+		config: DEVELOPMENT_CONFIG,
+		build: true,
+		onError: (err) => {
+			console.error('An error occurred while starting the main server:', err)
+		}
+	})
 }
 
 export async function stopMainServerDevelopment(): Promise<void> {
@@ -30,7 +26,7 @@ export async function stopMainServerDevelopment(): Promise<void> {
 }
 
 export async function startMainServerProduction(): Promise<void> {
-	startDockerCompose({
+	await startDockerCompose({
 		cwd: PRODUCTION_PATH,
 		config: PRODUCTION_CONFIG,
 		build: true,