From 49e9c1ea0707ac9b0fadc19c538fd9a7e5e188f7 Mon Sep 17 00:00:00 2001 From: Mate Nagy Date: Wed, 6 Mar 2024 10:00:10 -0800 Subject: [PATCH 01/14] MIB-20 fix with replication due to astype conversion --- mibidata/tiff.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mibidata/tiff.py b/mibidata/tiff.py index 120386a..f9ecdc9 100644 --- a/mibidata/tiff.py +++ b/mibidata/tiff.py @@ -130,7 +130,7 @@ def write(filename, image, sed=None, optical=None, ranges=None, save_dtype = np.uint16 range_dtype = 'I' - to_save = image.data.astype(save_dtype) + to_save = image.data.astype(save_dtype, copy=False) if not np.all(np.equal(to_save, image.data)): raise ValueError('Cannot convert data from ' f'{image.data.dtype} to {save_dtype}') From dfdaafed0b82ec48bb715b3e78bd50aff767db10 Mon Sep 17 00:00:00 2001 From: Mate Nagy Date: Thu, 7 Mar 2024 14:04:41 -0800 Subject: [PATCH 02/14] loosened pandas req --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index 8e66d9d..9aadb75 100644 --- a/environment.yml +++ b/environment.yml @@ -13,7 +13,7 @@ dependencies: - matplotlib==3.6.2 - mock==4.0.3 - numpy==1.23.5 - - pandas==1.2.3 + - pandas - pillow==9.3.0 - pylint==2.15.6 - pytest==7.2.0 From 8db0a9409fa662578bc728121c27532f442803fe Mon Sep 17 00:00:00 2001 From: Mate Nagy Date: Wed, 1 May 2024 16:50:14 -0700 Subject: [PATCH 03/14] MIB-21 script for stitching FOVs - init --- scripts/tiling/stitching.py | 300 ++++++++++++++++++++++++++++++++++++ 1 file changed, 300 insertions(+) create mode 100644 scripts/tiling/stitching.py diff --git a/scripts/tiling/stitching.py b/scripts/tiling/stitching.py new file mode 100644 index 0000000..d80766d --- /dev/null +++ b/scripts/tiling/stitching.py @@ -0,0 +1,300 @@ +""" Script for creating a stitched ROI MIBItiff from a folder of FOVs +""" + +from mibiprism.structure.file_path_templates import * +from mibiprism.structure import file_path_templates +from mibiprism.utils import gcs +import numpy as np +import pandas as pd +import os +from mibidata import mibi_image as mi +from mibitracker.request_helpers import MibiRequests +from collections import OrderedDict +import time +from mibidata import tiff +import json + +MAX_TRIES = 20 + + +def combine_entity_by_name(roi_fov_paths, cols, rows, enforce_square): + min_col = np.min(cols) + min_row = np.min(rows) + cols=[v-min_col+1 for v in cols] + rows=[v-min_row+1 for v in rows] + w = np.max(cols) + h = np.max(rows) + + panel = None + out_img = None + for i,roi_fov_path in enumerate(roi_fov_paths): + print(roi_fov_path) + + fov_img = tiff.read(roi_fov_path) + panel = fov_img.channels + fov_img = fov_img.data + + img_shape = list(np.shape(fov_img)) + ch_count = np.min(img_shape) + if img_shape[-1] == ch_count: + shape_2d = img_shape[:-1] + ch_last = True + elif img_shape[0] == ch_count: + shape_2d = img_shape[1:] + ch_last = False + print("fov shape",np.shape(fov_img)) + + if out_img is None: + if img_shape[-1] == ch_count: + out_img = np.zeros( + (h*shape_2d[0], w*shape_2d[1], ch_count), dtype=fov_img.dtype) + elif img_shape[0] == ch_count: + out_img = np.zeros((ch_count, h*shape_2d[0], w*shape_2d[1]), dtype=fov_img.dtype) + + out_img_shape = list(np.shape(out_img)) + print("out shape",out_img_shape) + + if ch_last: + print("ch last") + for i in range(ch_count): + out_img[((rows[i]-1)*shape_2d[0]):(rows[i]*shape_2d[0]), + ((cols[i]-1)*shape_2d[1]):(cols[i]*shape_2d[1]), i] = fov_img[:,:,i] + else: + print("ch first") + for i in range(ch_count): + out_img[i, + ((rows[i]-1)*shape_2d[0]):(rows[i]*shape_2d[0]), + ((cols[i]-1)*shape_2d[1]):(cols[i]*shape_2d[1])] = fov_img[i,:,:] + + if enforce_square: + if ch_last: + if out_img_shape[0] > out_img_shape[1]: + out_img = np.pad(out_img,((0,0),(0,out_img_shape[0]-out_img_shape[1]),(0,0))) + elif out_img_shape[0] < out_img_shape[1]: + out_img = np.pad(out_img,((0,out_img_shape[1]-out_img_shape[0]),(0,0),(0,0))) + else: + if out_img_shape[-2] > out_img_shape[-1]: + out_img = np.pad(out_img,((0,0),(0,0),(0,out_img_shape[-2]-out_img_shape[-1]))) + elif out_img_shape[-2] < out_img_shape[-1]: + out_img = np.pad(out_img,((0,0),(0,out_img_shape[-1]-out_img_shape[-2]),(0,0))) + + print("padded shape",np.shape(out_img)) + + return out_img.astype(out_img.dtype, copy=False), panel, int(max(w*shape_2d[1], h*shape_2d[0])) + + +def invert_dict(d): + n = OrderedDict({}) + for k, v in d.items(): + if v not in n: + n[v] = [k] + else: + n[v].append(k) + return n + + +def run_task(fov_paths, out_path, session_dict, mt_upload): + "fov-01-ROI01_C01_R03.tiff" + unique_rois = np.unique([os.path.basename(p).split("-")[-1].split(".tiff")[0].split("_")[0] for p in fov_paths]) + roi_path_groups = dict([(u,[p for p in fov_paths if u in p]) for u in unique_rois]) + + mr = None + for t in range(MAX_TRIES): + try: + mr = MibiRequests(**session_dict) + break + except: + if t < MAX_TRIES-1: + time.sleep(0.50) + else: + mr = MibiRequests(**session_dict) + + for roi,roi_fov_paths in roi_path_groups.items(): + print(roi) + + # find the combining params + cols = [] + rows = [] + for fov_path in roi_fov_paths: + fov_name = os.path.basename(fov_path).split("-")[-1].split(".tiff")[0] + _, c, r = fov_name.split("_") + cols.append(int(c[1:].lstrip("0"))) + rows.append(int(r[1:].lstrip("0"))) + + out_img, panel, max_dim = combine_entity_by_name(roi_fov_paths, cols, rows, enforce_square=True) + out_img = out_img.astype(np.uint8, copy=False) + + "fov-04-scan-1.json" + um_min_x, um_min_y = 999999999, 999999999 + for fov_path in roi_fov_paths: + json_file = "-".join(roi_fov_paths.split("-")[:2])+"-scan-1.json" + with open(json_file) as f: + bin_json = json.load(f) + coord = bin_json["coordinates"] + if coord["x"] < um_min_x: + um_min_x = coord["x"] + if coord["y"] < um_min_y: + um_min_y = coord["y"] + + run = os.path.dirname(roi_fov_paths) + fov = "FOV"+os.path.basename(roi_fov_paths).split("-")[1] + + for t in range(MAX_TRIES): + try: + try: + ref_image_id = mr.image_id(run, fov) + except: + ref_image_id = mr.image_id(run, fov.replace("V0", "V")) + break + except: + if t < MAX_TRIES-1: + time.sleep(0.50) + else: + try: + ref_image_id = mr.image_id(run, fov) + except: + ref_image_id = mr.image_id(run, fov.replace("V0", "V")) + + for t in range(MAX_TRIES): + try: + ref_json = mr.get('images/{}/'.format(ref_image_id)).json() + break + except: + if t < MAX_TRIES-1: + time.sleep(0.50) + else: + ref_json = mr.get('images/{}/'.format(ref_image_id)).json() + + sample_id = run+"__"+roi + f_split = ref_json['folder'].split('/') + f_split[0] = sample_id + folder_name = '/'.join(f_split[:3]) + + px_per_u = ref_json["frame"]/ref_json["fov_size"] + + metadata = { + 'run': '{}'.format(ref_json['run']['label']), + 'date': ref_json['run']['run_date'], + 'coordinates': (um_min_x, um_min_y), + 'size': max_dim/px_per_u, # issue: assumption that w==h doens't hold + 'slide': ref_json['section']['slide']['id'], + 'fov_name': sample_id, + 'frame': max_dim, # issue: assumption that w==h doens't hold + 'folder': folder_name, + 'fov_id': f_split[0], + # assumption that all fovs are same + 'dwell': ref_json['dwell_time'], + 'scans': ','.join([str(d) for d in range(ref_json['depths'])]), + # assumption that all fovs are same + 'aperture': ref_json['run']['aperture']['label'], + 'instrument': ref_json['run']['instrument']['name'], + 'tissue': ref_json['formatted_tissue'], + 'panel': ref_json['section']['panel']['name'], + 'version': 'alpha', + # assumption that all fovs are same + 'mass_offset': ref_json['mass_offset'], + # assumption that all fovs are same + 'mass_gain': ref_json['mass_gain'], + # assumption that all fovs are same + 'time_resolution': ref_json['time_bin'], + 'filename': '{}'.format(ref_json['run']['name']) + } + + out_mibi_tiff = mi.MibiImage( + out_img, panel, datetime_format='%Y-%m-%d', **metadata) + print("out_mibi_tiff as mibitiff success") + + f_split = out_mibi_tiff.folder.split('/') + f_split[0] = sample_id + out_mibi_tiff.set_fov_id(f_split[0], '/'.join(f_split)) + + tiff.write(out_path, out_mibi_tiff, dtype=np.float32) + + if mt_upload: + for t in range(MAX_TRIES): + try: + exists = mr.get( + '/images/', params={'run__label': ref_json['run']['label'], 'number': f_split[0]}).json() + break + except: + if t < MAX_TRIES-1: + time.sleep(0.50) + else: + exists = mr.get( + '/images/', params={'run__label': ref_json['run']['label'], 'number': f_split[0]}).json() + + full_id = exists['results'][0]['id'] if exists['count'] else None + + if not full_id: + new_im_metadata = {} + new_im_metadata['run'] = ref_json['run']['id'] + new_im_metadata['point'] = sample_id + new_im_metadata['number'] = out_mibi_tiff.fov_id + new_im_metadata['folder'] = out_mibi_tiff.folder + new_im_metadata['fov_size'] = max_dim/px_per_u + # assumption that all fovs are same + new_im_metadata['dwell_time'] = ref_json['dwell_time'] + # assumption that all fovs are same + new_im_metadata['depths'] = ref_json['depths'] + new_im_metadata['frame'] = max_dim + # assumption that all fovs are same + new_im_metadata['time_bin'] = ref_json['time_bin'] + # assumption that all fovs are same + new_im_metadata['mass_gain'] = ref_json['mass_gain'] + # assumption that all fovs are same + new_im_metadata['mass_offset'] = ref_json['mass_offset'] + new_im_metadata['x_coord'] = int(np.round(um_min_x)) + new_im_metadata['y_coord'] = int(np.round(um_min_y)) + new_im_metadata['tissue'] = \ + ref_json['tissue'] and ref_json['tissue']['id'] + new_im_metadata['section'] = ref_json['section']['id'] + # assumption that all fovs are same + new_im_metadata['aperture'] = ref_json['aperture']['id'] + # assumption that all fovs are same + new_im_metadata['imaging_preset'] = ref_json['imaging_preset'] + # assumption that all fovs are same + new_im_metadata['lens1_voltage'] = ref_json['lens1_voltage'] + + for t in range(MAX_TRIES): + try: + _ = mr.post('/images/', json=new_im_metadata) + break + except: + if t < MAX_TRIES-1: + time.sleep(0.50) + else: + _ = mr.post('/images/', json=new_im_metadata) + + + for t in range(MAX_TRIES): + try: + mr.upload_mibitiff(out_path, run_id=ref_json['run']['id']) + break + except: + if t < MAX_TRIES-1: + time.sleep(0.50) + else: + mr.upload_mibitiff(out_path, run_id=ref_json['run']['id']) + print("mibi_tiff_buf upload to mibitracker success") + + +if __name__ == "__main__": + + fovs_folder = "/Users/mnagy/projects/stitch_script_test/images" + out_path = "/Users/mnagy/projects/stitch_script_test/image.tiff" + cmd = f'ls "{fovs_folder}"' + fov_paths = os.popen(cmd).read().strip().split("\n") + try: + fov_paths.remove("") + except: + pass + + session_dict = { + "url": "", # MIBItracker backend URL + "email": "", # User name + "password": "" # User password + } + mt_upload = False + + run_task(fov_paths, out_path, session_dict, mt_upload) + From 02556e0ea7cf705bb217d2297bed3205d7ad91c9 Mon Sep 17 00:00:00 2001 From: Mate Nagy Date: Wed, 1 May 2024 17:23:43 -0700 Subject: [PATCH 04/14] fixes in script; all seems to work now; just need to test upload once i have access to project i'm testing --- scripts/tiling/stitching.py | 98 ++++++++++--------------------------- 1 file changed, 27 insertions(+), 71 deletions(-) diff --git a/scripts/tiling/stitching.py b/scripts/tiling/stitching.py index d80766d..cd0a098 100644 --- a/scripts/tiling/stitching.py +++ b/scripts/tiling/stitching.py @@ -1,11 +1,7 @@ """ Script for creating a stitched ROI MIBItiff from a folder of FOVs """ -from mibiprism.structure.file_path_templates import * -from mibiprism.structure import file_path_templates -from mibiprism.utils import gcs import numpy as np -import pandas as pd import os from mibidata import mibi_image as mi from mibitracker.request_helpers import MibiRequests @@ -14,7 +10,7 @@ from mibidata import tiff import json -MAX_TRIES = 20 +MAX_TRIES = 10 def combine_entity_by_name(roi_fov_paths, cols, rows, enforce_square): @@ -27,8 +23,8 @@ def combine_entity_by_name(roi_fov_paths, cols, rows, enforce_square): panel = None out_img = None - for i,roi_fov_path in enumerate(roi_fov_paths): - print(roi_fov_path) + for fov_i,roi_fov_path in enumerate(roi_fov_paths): + print(fov_i+1,roi_fov_path) fov_img = tiff.read(roi_fov_path) panel = fov_img.channels @@ -36,68 +32,28 @@ def combine_entity_by_name(roi_fov_paths, cols, rows, enforce_square): img_shape = list(np.shape(fov_img)) ch_count = np.min(img_shape) - if img_shape[-1] == ch_count: - shape_2d = img_shape[:-1] - ch_last = True - elif img_shape[0] == ch_count: - shape_2d = img_shape[1:] - ch_last = False - print("fov shape",np.shape(fov_img)) + shape_2d = img_shape[:-1] if out_img is None: - if img_shape[-1] == ch_count: - out_img = np.zeros( - (h*shape_2d[0], w*shape_2d[1], ch_count), dtype=fov_img.dtype) - elif img_shape[0] == ch_count: - out_img = np.zeros((ch_count, h*shape_2d[0], w*shape_2d[1]), dtype=fov_img.dtype) - + out_img = np.zeros((h*shape_2d[0], w*shape_2d[1], ch_count), dtype=fov_img.dtype) out_img_shape = list(np.shape(out_img)) - print("out shape",out_img_shape) - - if ch_last: - print("ch last") - for i in range(ch_count): - out_img[((rows[i]-1)*shape_2d[0]):(rows[i]*shape_2d[0]), - ((cols[i]-1)*shape_2d[1]):(cols[i]*shape_2d[1]), i] = fov_img[:,:,i] - else: - print("ch first") - for i in range(ch_count): - out_img[i, - ((rows[i]-1)*shape_2d[0]):(rows[i]*shape_2d[0]), - ((cols[i]-1)*shape_2d[1]):(cols[i]*shape_2d[1])] = fov_img[i,:,:] + + for ch_i in range(ch_count): + out_img[((rows[fov_i]-1)*shape_2d[0]):(rows[fov_i]*shape_2d[0]), + ((cols[fov_i]-1)*shape_2d[1]):(cols[fov_i]*shape_2d[1]), ch_i] = fov_img[:,:,ch_i] if enforce_square: - if ch_last: - if out_img_shape[0] > out_img_shape[1]: - out_img = np.pad(out_img,((0,0),(0,out_img_shape[0]-out_img_shape[1]),(0,0))) - elif out_img_shape[0] < out_img_shape[1]: - out_img = np.pad(out_img,((0,out_img_shape[1]-out_img_shape[0]),(0,0),(0,0))) - else: - if out_img_shape[-2] > out_img_shape[-1]: - out_img = np.pad(out_img,((0,0),(0,0),(0,out_img_shape[-2]-out_img_shape[-1]))) - elif out_img_shape[-2] < out_img_shape[-1]: - out_img = np.pad(out_img,((0,0),(0,out_img_shape[-1]-out_img_shape[-2]),(0,0))) + if out_img_shape[0] > out_img_shape[1]: + out_img = np.pad(out_img,((0,0),(0,out_img_shape[0]-out_img_shape[1]),(0,0))) + elif out_img_shape[0] < out_img_shape[1]: + out_img = np.pad(out_img,((0,out_img_shape[1]-out_img_shape[0]),(0,0),(0,0))) - print("padded shape",np.shape(out_img)) - return out_img.astype(out_img.dtype, copy=False), panel, int(max(w*shape_2d[1], h*shape_2d[0])) - -def invert_dict(d): - n = OrderedDict({}) - for k, v in d.items(): - if v not in n: - n[v] = [k] - else: - n[v].append(k) - return n - def run_task(fov_paths, out_path, session_dict, mt_upload): - "fov-01-ROI01_C01_R03.tiff" unique_rois = np.unique([os.path.basename(p).split("-")[-1].split(".tiff")[0].split("_")[0] for p in fov_paths]) roi_path_groups = dict([(u,[p for p in fov_paths if u in p]) for u in unique_rois]) - mr = None for t in range(MAX_TRIES): try: @@ -112,22 +68,21 @@ def run_task(fov_paths, out_path, session_dict, mt_upload): for roi,roi_fov_paths in roi_path_groups.items(): print(roi) - # find the combining params cols = [] rows = [] for fov_path in roi_fov_paths: fov_name = os.path.basename(fov_path).split("-")[-1].split(".tiff")[0] - _, c, r = fov_name.split("_") + c, r = fov_name.split("_")[1:] cols.append(int(c[1:].lstrip("0"))) rows.append(int(r[1:].lstrip("0"))) out_img, panel, max_dim = combine_entity_by_name(roi_fov_paths, cols, rows, enforce_square=True) out_img = out_img.astype(np.uint8, copy=False) - "fov-04-scan-1.json" um_min_x, um_min_y = 999999999, 999999999 for fov_path in roi_fov_paths: - json_file = "-".join(roi_fov_paths.split("-")[:2])+"-scan-1.json" + + json_file = os.path.dirname(fov_path)+"/"+"-".join(os.path.basename(fov_path).split("-")[:2])+"-scan-1.json" with open(json_file) as f: bin_json = json.load(f) coord = bin_json["coordinates"] @@ -136,8 +91,9 @@ def run_task(fov_paths, out_path, session_dict, mt_upload): if coord["y"] < um_min_y: um_min_y = coord["y"] - run = os.path.dirname(roi_fov_paths) - fov = "FOV"+os.path.basename(roi_fov_paths).split("-")[1] + run = os.path.basename(os.path.dirname(roi_fov_paths[0])) + fov = "FOV"+os.path.basename(roi_fov_paths[0]).split("-")[1] + print(run, fov) for t in range(MAX_TRIES): try: @@ -202,13 +158,13 @@ def run_task(fov_paths, out_path, session_dict, mt_upload): out_mibi_tiff = mi.MibiImage( out_img, panel, datetime_format='%Y-%m-%d', **metadata) - print("out_mibi_tiff as mibitiff success") f_split = out_mibi_tiff.folder.split('/') f_split[0] = sample_id out_mibi_tiff.set_fov_id(f_split[0], '/'.join(f_split)) tiff.write(out_path, out_mibi_tiff, dtype=np.float32) + print(f"Stitched MIBItiff saved to {out_path}.") if mt_upload: for t in range(MAX_TRIES): @@ -265,7 +221,6 @@ def run_task(fov_paths, out_path, session_dict, mt_upload): else: _ = mr.post('/images/', json=new_im_metadata) - for t in range(MAX_TRIES): try: mr.upload_mibitiff(out_path, run_id=ref_json['run']['id']) @@ -275,14 +230,15 @@ def run_task(fov_paths, out_path, session_dict, mt_upload): time.sleep(0.50) else: mr.upload_mibitiff(out_path, run_id=ref_json['run']['id']) - print("mibi_tiff_buf upload to mibitracker success") + + print(f"Stitched MIBItiff, {out_path} uploaded to MIBItracker.") if __name__ == "__main__": - fovs_folder = "/Users/mnagy/projects/stitch_script_test/images" - out_path = "/Users/mnagy/projects/stitch_script_test/image.tiff" - cmd = f'ls "{fovs_folder}"' + fovs_folder = "/Users/mnagy/projects/stitch_script_test/2024-04-29T10-07-46_gold_tonsil_3x3_coarse_ROI" + out_path = "/Users/mnagy/projects/stitch_script_test/2024-04-29T10-07-46_gold_tonsil_3x3_coarse_ROI.tiff" + cmd = f'ls {fovs_folder}/*.tiff' fov_paths = os.popen(cmd).read().strip().split("\n") try: fov_paths.remove("") @@ -290,11 +246,11 @@ def run_task(fov_paths, out_path, session_dict, mt_upload): pass session_dict = { - "url": "", # MIBItracker backend URL + "url": "https://mibitracker.api.ionpath.com/", # MIBItracker backend URL "email": "", # User name "password": "" # User password } - mt_upload = False + mt_upload = True run_task(fov_paths, out_path, session_dict, mt_upload) From 8a61f0ed8df92ecea38520095242536bcb0233d0 Mon Sep 17 00:00:00 2001 From: Mate Nagy Date: Fri, 3 May 2024 12:21:10 -0700 Subject: [PATCH 05/14] updated based on PR feedback: reverted pandas version; removed loop from channels --- environment.yml | 2 +- scripts/tiling/stitching.py | 10 ++++------ 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/environment.yml b/environment.yml index 9aadb75..8e66d9d 100644 --- a/environment.yml +++ b/environment.yml @@ -13,7 +13,7 @@ dependencies: - matplotlib==3.6.2 - mock==4.0.3 - numpy==1.23.5 - - pandas + - pandas==1.2.3 - pillow==9.3.0 - pylint==2.15.6 - pytest==7.2.0 diff --git a/scripts/tiling/stitching.py b/scripts/tiling/stitching.py index cd0a098..bfaa5fb 100644 --- a/scripts/tiling/stitching.py +++ b/scripts/tiling/stitching.py @@ -5,7 +5,6 @@ import os from mibidata import mibi_image as mi from mibitracker.request_helpers import MibiRequests -from collections import OrderedDict import time from mibidata import tiff import json @@ -38,9 +37,8 @@ def combine_entity_by_name(roi_fov_paths, cols, rows, enforce_square): out_img = np.zeros((h*shape_2d[0], w*shape_2d[1], ch_count), dtype=fov_img.dtype) out_img_shape = list(np.shape(out_img)) - for ch_i in range(ch_count): - out_img[((rows[fov_i]-1)*shape_2d[0]):(rows[fov_i]*shape_2d[0]), - ((cols[fov_i]-1)*shape_2d[1]):(cols[fov_i]*shape_2d[1]), ch_i] = fov_img[:,:,ch_i] + out_img[((rows[fov_i]-1)*shape_2d[0]):(rows[fov_i]*shape_2d[0]), + ((cols[fov_i]-1)*shape_2d[1]):(cols[fov_i]*shape_2d[1]), :] = fov_img[:,:,:] if enforce_square: if out_img_shape[0] > out_img_shape[1]: @@ -132,10 +130,10 @@ def run_task(fov_paths, out_path, session_dict, mt_upload): 'run': '{}'.format(ref_json['run']['label']), 'date': ref_json['run']['run_date'], 'coordinates': (um_min_x, um_min_y), - 'size': max_dim/px_per_u, # issue: assumption that w==h doens't hold + 'size': max_dim/px_per_u, # issue: assumption that w==h doesn't hold 'slide': ref_json['section']['slide']['id'], 'fov_name': sample_id, - 'frame': max_dim, # issue: assumption that w==h doens't hold + 'frame': max_dim, # issue: assumption that w==h doesn't hold 'folder': folder_name, 'fov_id': f_split[0], # assumption that all fovs are same From 008cc0feefe3e317b4499925e9c109fc9c7a2dba Mon Sep 17 00:00:00 2001 From: Mate Nagy Date: Mon, 6 May 2024 09:11:41 -0700 Subject: [PATCH 06/14] updated based on PR feedbacks --- scripts/stitching/__init__.py | 0 scripts/{tiling => stitching}/stitching.py | 33 +++++++++------------- 2 files changed, 14 insertions(+), 19 deletions(-) create mode 100644 scripts/stitching/__init__.py rename scripts/{tiling => stitching}/stitching.py (91%) diff --git a/scripts/stitching/__init__.py b/scripts/stitching/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/scripts/tiling/stitching.py b/scripts/stitching/stitching.py similarity index 91% rename from scripts/tiling/stitching.py rename to scripts/stitching/stitching.py index bfaa5fb..2da9b49 100644 --- a/scripts/tiling/stitching.py +++ b/scripts/stitching/stitching.py @@ -8,11 +8,12 @@ import time from mibidata import tiff import json +import glob MAX_TRIES = 10 -def combine_entity_by_name(roi_fov_paths, cols, rows, enforce_square): +def combine_entity_by_name(roi_fov_paths, cols, rows, enforce_square=False): min_col = np.min(cols) min_row = np.min(rows) cols=[v-min_col+1 for v in cols] @@ -40,7 +41,7 @@ def combine_entity_by_name(roi_fov_paths, cols, rows, enforce_square): out_img[((rows[fov_i]-1)*shape_2d[0]):(rows[fov_i]*shape_2d[0]), ((cols[fov_i]-1)*shape_2d[1]):(cols[fov_i]*shape_2d[1]), :] = fov_img[:,:,:] - if enforce_square: + if enforce_square: # Set to True if there is a need to pad the image to be a square if out_img_shape[0] > out_img_shape[1]: out_img = np.pad(out_img,((0,0),(0,out_img_shape[0]-out_img_shape[1]),(0,0))) elif out_img_shape[0] < out_img_shape[1]: @@ -49,7 +50,8 @@ def combine_entity_by_name(roi_fov_paths, cols, rows, enforce_square): return out_img.astype(out_img.dtype, copy=False), panel, int(max(w*shape_2d[1], h*shape_2d[0])) -def run_task(fov_paths, out_path, session_dict, mt_upload): +def stitch_fovs(fovs_folder, out_path, session_dict, upload_to_mibitracker): + fov_paths = glob.glob(f'{fovs_folder}/*.tiff') unique_rois = np.unique([os.path.basename(p).split("-")[-1].split(".tiff")[0].split("_")[0] for p in fov_paths]) roi_path_groups = dict([(u,[p for p in fov_paths if u in p]) for u in unique_rois]) mr = None @@ -74,7 +76,7 @@ def run_task(fov_paths, out_path, session_dict, mt_upload): cols.append(int(c[1:].lstrip("0"))) rows.append(int(r[1:].lstrip("0"))) - out_img, panel, max_dim = combine_entity_by_name(roi_fov_paths, cols, rows, enforce_square=True) + out_img, panel, max_dim = combine_entity_by_name(roi_fov_paths, cols, rows, enforce_square=False) out_img = out_img.astype(np.uint8, copy=False) um_min_x, um_min_y = 999999999, 999999999 @@ -164,7 +166,7 @@ def run_task(fov_paths, out_path, session_dict, mt_upload): tiff.write(out_path, out_mibi_tiff, dtype=np.float32) print(f"Stitched MIBItiff saved to {out_path}.") - if mt_upload: + if upload_to_mibitracker: for t in range(MAX_TRIES): try: exists = mr.get( @@ -234,21 +236,14 @@ def run_task(fov_paths, out_path, session_dict, mt_upload): if __name__ == "__main__": - fovs_folder = "/Users/mnagy/projects/stitch_script_test/2024-04-29T10-07-46_gold_tonsil_3x3_coarse_ROI" - out_path = "/Users/mnagy/projects/stitch_script_test/2024-04-29T10-07-46_gold_tonsil_3x3_coarse_ROI.tiff" - cmd = f'ls {fovs_folder}/*.tiff' - fov_paths = os.popen(cmd).read().strip().split("\n") - try: - fov_paths.remove("") - except: - pass - + fovs_folder = "" # Local folder path with the original Run name and contents + out_path = "" # Local file path to save the stitched MIBItiff into session_dict = { - "url": "https://mibitracker.api.ionpath.com/", # MIBItracker backend URL - "email": "", # User name - "password": "" # User password + "url": "", # MIBItracker backend URL (ex: https://mibitracker.api.ionpath.com/) + "email": "", # MIBItracker user name + "password": "" # MIBItrackefr user password } - mt_upload = True + upload_to_mibitracker = False # Set to True to upload the resulting stitched MIBItiff to MIBItracker - run_task(fov_paths, out_path, session_dict, mt_upload) + stitch_fovs(fovs_folder, out_path, session_dict, upload_to_mibitracker) From 588b8545cca54efbdb80e7f171747f8ffb06d5c5 Mon Sep 17 00:00:00 2001 From: Mate Nagy Date: Tue, 14 May 2024 21:49:15 -0700 Subject: [PATCH 07/14] removed need to query mibitracker if not uploading; improved uploaded file name; added margins --- scripts/stitching/stitching.py | 313 +++++++++++++++------------------ 1 file changed, 140 insertions(+), 173 deletions(-) diff --git a/scripts/stitching/stitching.py b/scripts/stitching/stitching.py index 2da9b49..6aeaf73 100644 --- a/scripts/stitching/stitching.py +++ b/scripts/stitching/stitching.py @@ -8,12 +8,11 @@ import time from mibidata import tiff import json -import glob -MAX_TRIES = 10 +MAX_TRIES = 5 -def combine_entity_by_name(roi_fov_paths, cols, rows, enforce_square=False): +def combine_entity_by_name(roi_fov_paths, cols, rows, enforce_square=False, margin=0): min_col = np.min(cols) min_row = np.min(rows) cols=[v-min_col+1 for v in cols] @@ -23,23 +22,27 @@ def combine_entity_by_name(roi_fov_paths, cols, rows, enforce_square=False): panel = None out_img = None + ref_metadata = None for fov_i,roi_fov_path in enumerate(roi_fov_paths): - print(fov_i+1,roi_fov_path) - fov_img = tiff.read(roi_fov_path) panel = fov_img.channels fov_img = fov_img.data + if ref_metadata is None: + ref_metadata = tiff.info(roi_fov_path) img_shape = list(np.shape(fov_img)) ch_count = np.min(img_shape) shape_2d = img_shape[:-1] if out_img is None: - out_img = np.zeros((h*shape_2d[0], w*shape_2d[1], ch_count), dtype=fov_img.dtype) + out_img = np.zeros((h*shape_2d[0]+(h-1)*margin["y"], w*shape_2d[1]+(w-1)*margin["x"], ch_count), + dtype=fov_img.dtype) out_img_shape = list(np.shape(out_img)) - out_img[((rows[fov_i]-1)*shape_2d[0]):(rows[fov_i]*shape_2d[0]), - ((cols[fov_i]-1)*shape_2d[1]):(cols[fov_i]*shape_2d[1]), :] = fov_img[:,:,:] + out_img[((rows[fov_i]-1)*shape_2d[0]+(rows[fov_i]-1)*margin["y"]): + (rows[fov_i]*shape_2d[0]+(rows[fov_i]-1)*margin["y"]), + ((cols[fov_i]-1)*shape_2d[1]+(cols[fov_i]-1)*margin["x"]): + (cols[fov_i]*shape_2d[1]+(cols[fov_i]-1)*margin["x"]), :] = fov_img[:,:,:] if enforce_square: # Set to True if there is a need to pad the image to be a square if out_img_shape[0] > out_img_shape[1]: @@ -47,203 +50,167 @@ def combine_entity_by_name(roi_fov_paths, cols, rows, enforce_square=False): elif out_img_shape[0] < out_img_shape[1]: out_img = np.pad(out_img,((0,out_img_shape[1]-out_img_shape[0]),(0,0),(0,0))) - return out_img.astype(out_img.dtype, copy=False), panel, int(max(w*shape_2d[1], h*shape_2d[0])) - - -def stitch_fovs(fovs_folder, out_path, session_dict, upload_to_mibitracker): - fov_paths = glob.glob(f'{fovs_folder}/*.tiff') - unique_rois = np.unique([os.path.basename(p).split("-")[-1].split(".tiff")[0].split("_")[0] for p in fov_paths]) - roi_path_groups = dict([(u,[p for p in fov_paths if u in p]) for u in unique_rois]) - mr = None - for t in range(MAX_TRIES): - try: - mr = MibiRequests(**session_dict) - break - except: - if t < MAX_TRIES-1: - time.sleep(0.50) - else: - mr = MibiRequests(**session_dict) - - for roi,roi_fov_paths in roi_path_groups.items(): - print(roi) - - cols = [] - rows = [] - for fov_path in roi_fov_paths: - fov_name = os.path.basename(fov_path).split("-")[-1].split(".tiff")[0] - c, r = fov_name.split("_")[1:] - cols.append(int(c[1:].lstrip("0"))) - rows.append(int(r[1:].lstrip("0"))) - - out_img, panel, max_dim = combine_entity_by_name(roi_fov_paths, cols, rows, enforce_square=False) - out_img = out_img.astype(np.uint8, copy=False) - - um_min_x, um_min_y = 999999999, 999999999 - for fov_path in roi_fov_paths: - - json_file = os.path.dirname(fov_path)+"/"+"-".join(os.path.basename(fov_path).split("-")[:2])+"-scan-1.json" - with open(json_file) as f: - bin_json = json.load(f) - coord = bin_json["coordinates"] + return out_img.astype(out_img.dtype, copy=False), panel, int(max(w*shape_2d[1], h*shape_2d[0])), ref_metadata + + +def stitch_fovs(run_folder, out_folder, session_dict, upload_to_mibitracker, fov_margin_size): + run = os.path.basename(run_folder) + run_json_file = os.path.join(run_folder,run)+".json" + with open(run_json_file) as rf: + run_json = json.load(rf) + for r in run_json["rois"]: + if r["standardTarget"] in ["Auto Gain"]: + continue + roi = r["name"] + px_per_u = r["frameSizePixels"]["width"] / r["fovSizeMicrons"] + margin = fov_margin_size + if margin is None: + margin = { + "x":int(np.round(r["xMargin"]*px_per_u)), + "y":int(np.round(r["yMargin"]*px_per_u)) + } + elif type(margin) != dict: + if type(margin) != list: + margin = {"x":margin, "y":margin} + else: + margin = {"x":margin[0], "y":margin[1]} + cols, rows = [], [] + um_min_x, um_min_y = 999999999, 999999999 + roi_fov_paths=[] + for f in r["fovs"]: + fov_name = f["name"] + cols.append(f["gridPosition"]["x"]+1) + rows.append(f["gridPosition"]["y"]+1) + coord = f["centerPointMicrons"] if coord["x"] < um_min_x: um_min_x = coord["x"] if coord["y"] < um_min_y: um_min_y = coord["y"] - - run = os.path.basename(os.path.dirname(roi_fov_paths[0])) - fov = "FOV"+os.path.basename(roi_fov_paths[0]).split("-")[1] - print(run, fov) + roi_fov_paths.append(os.path.join(run_folder,"fov"+"-"+str(f["runOrder"]).zfill(2)+"-"+fov_name)+".tiff") - for t in range(MAX_TRIES): - try: - try: - ref_image_id = mr.image_id(run, fov) - except: - ref_image_id = mr.image_id(run, fov.replace("V0", "V")) - break - except: - if t < MAX_TRIES-1: - time.sleep(0.50) - else: + print(run, roi, f"{len(roi_fov_paths)} FOVs") + out_img, panel, max_dim, ref_metadata = combine_entity_by_name(roi_fov_paths, cols, rows, + enforce_square=False, margin=margin) + out_img = out_img.astype(np.uint8, copy=False) + + metadata = ref_metadata.copy() + metadata["coordinates"] = (um_min_x, um_min_y) + metadata["size"] = max_dim/px_per_u + metadata["fov_name"] = roi + + out_mibi_tiff = mi.MibiImage( + out_img, panel, datetime_format='%Y-%m-%d', **metadata) + + f_split = out_mibi_tiff.folder.split('/') + f_split[0] = roi + out_mibi_tiff.set_fov_id(f_split[0], '/'.join(f_split)) + + out_path = os.path.join(out_folder,roi+".tiff") + tiff.write(out_path, out_mibi_tiff, dtype=np.float32) + print(f"Stitched MIBItiff saved to {out_path}.") + + if upload_to_mibitracker: + mr = None + for t in range(MAX_TRIES): try: - ref_image_id = mr.image_id(run, fov) + mr = MibiRequests(**session_dict) + break except: - ref_image_id = mr.image_id(run, fov.replace("V0", "V")) - - for t in range(MAX_TRIES): - try: - ref_json = mr.get('images/{}/'.format(ref_image_id)).json() - break - except: - if t < MAX_TRIES-1: - time.sleep(0.50) - else: - ref_json = mr.get('images/{}/'.format(ref_image_id)).json() - - sample_id = run+"__"+roi - f_split = ref_json['folder'].split('/') - f_split[0] = sample_id - folder_name = '/'.join(f_split[:3]) - - px_per_u = ref_json["frame"]/ref_json["fov_size"] - - metadata = { - 'run': '{}'.format(ref_json['run']['label']), - 'date': ref_json['run']['run_date'], - 'coordinates': (um_min_x, um_min_y), - 'size': max_dim/px_per_u, # issue: assumption that w==h doesn't hold - 'slide': ref_json['section']['slide']['id'], - 'fov_name': sample_id, - 'frame': max_dim, # issue: assumption that w==h doesn't hold - 'folder': folder_name, - 'fov_id': f_split[0], - # assumption that all fovs are same - 'dwell': ref_json['dwell_time'], - 'scans': ','.join([str(d) for d in range(ref_json['depths'])]), - # assumption that all fovs are same - 'aperture': ref_json['run']['aperture']['label'], - 'instrument': ref_json['run']['instrument']['name'], - 'tissue': ref_json['formatted_tissue'], - 'panel': ref_json['section']['panel']['name'], - 'version': 'alpha', - # assumption that all fovs are same - 'mass_offset': ref_json['mass_offset'], - # assumption that all fovs are same - 'mass_gain': ref_json['mass_gain'], - # assumption that all fovs are same - 'time_resolution': ref_json['time_bin'], - 'filename': '{}'.format(ref_json['run']['name']) - } - - out_mibi_tiff = mi.MibiImage( - out_img, panel, datetime_format='%Y-%m-%d', **metadata) - - f_split = out_mibi_tiff.folder.split('/') - f_split[0] = sample_id - out_mibi_tiff.set_fov_id(f_split[0], '/'.join(f_split)) - - tiff.write(out_path, out_mibi_tiff, dtype=np.float32) - print(f"Stitched MIBItiff saved to {out_path}.") - - if upload_to_mibitracker: - for t in range(MAX_TRIES): - try: - exists = mr.get( - '/images/', params={'run__label': ref_json['run']['label'], 'number': f_split[0]}).json() - break - except: - if t < MAX_TRIES-1: - time.sleep(0.50) - else: + if t < MAX_TRIES-1: + time.sleep(0.50) + else: + mr = MibiRequests(**session_dict) + + for t in range(MAX_TRIES): + try: exists = mr.get( - '/images/', params={'run__label': ref_json['run']['label'], 'number': f_split[0]}).json() + '/images/', params={'run__label': run, 'number': f_split[0]}).json() + break + except: + if t < MAX_TRIES-1: + time.sleep(0.50) + else: + exists = mr.get( + '/images/', params={'run__label': run, 'number': f_split[0]}).json() - full_id = exists['results'][0]['id'] if exists['count'] else None + full_id = exists['results'][0]['id'] if exists['count'] else None + + for t in range(MAX_TRIES): + try: + run_id = mr.get('/runs/', params={'name': run}).json()['results'][0]['id'] + break + except: + if t < MAX_TRIES-1: + time.sleep(0.50) + else: + run_id = mr.get('/runs/', params={'name': run}).json()['results'][0]['id'] - if not full_id: new_im_metadata = {} - new_im_metadata['run'] = ref_json['run']['id'] - new_im_metadata['point'] = sample_id + new_im_metadata['run'] = run_id + new_im_metadata['point'] = roi new_im_metadata['number'] = out_mibi_tiff.fov_id new_im_metadata['folder'] = out_mibi_tiff.folder new_im_metadata['fov_size'] = max_dim/px_per_u - # assumption that all fovs are same - new_im_metadata['dwell_time'] = ref_json['dwell_time'] - # assumption that all fovs are same - new_im_metadata['depths'] = ref_json['depths'] + new_im_metadata['dwell_time'] = metadata['dwell'] + new_im_metadata['depths'] = metadata['scans'] new_im_metadata['frame'] = max_dim - # assumption that all fovs are same - new_im_metadata['time_bin'] = ref_json['time_bin'] - # assumption that all fovs are same - new_im_metadata['mass_gain'] = ref_json['mass_gain'] - # assumption that all fovs are same - new_im_metadata['mass_offset'] = ref_json['mass_offset'] + new_im_metadata['time_bin'] = metadata['time_resolution'] + new_im_metadata['mass_gain'] = metadata['mass_gain'] + new_im_metadata['mass_offset'] = metadata['mass_offset'] new_im_metadata['x_coord'] = int(np.round(um_min_x)) new_im_metadata['y_coord'] = int(np.round(um_min_y)) - new_im_metadata['tissue'] = \ - ref_json['tissue'] and ref_json['tissue']['id'] - new_im_metadata['section'] = ref_json['section']['id'] - # assumption that all fovs are same - new_im_metadata['aperture'] = ref_json['aperture']['id'] - # assumption that all fovs are same - new_im_metadata['imaging_preset'] = ref_json['imaging_preset'] - # assumption that all fovs are same - new_im_metadata['lens1_voltage'] = ref_json['lens1_voltage'] + new_im_metadata['tissue'] = metadata['raw_description']['fov']['section']['tissue'] \ + and metadata['raw_description']['fov']['section']['tissue']['id'] + new_im_metadata['section'] = metadata['raw_description']['fov']['section']['id'] + new_im_metadata['aperture'] = metadata['aperture'] + new_im_metadata['imaging_preset'] = metadata['imaging_preset'] + new_im_metadata['lens1_voltage'] = metadata['lens1_voltage'] + + if not full_id: + for t in range(MAX_TRIES): + try: + _ = mr.post('/images/', json=new_im_metadata) + break + except: + if t < MAX_TRIES-1: + time.sleep(0.50) + else: + _ = mr.post('/images/', json=new_im_metadata) + else: + for t in range(MAX_TRIES): + try: + _ = mr.put(f'/images/{full_id}', json=new_im_metadata) + break + except: + if t < MAX_TRIES-1: + time.sleep(0.50) + else: + _ = mr.put(f'/images/{full_id}', json=new_im_metadata) for t in range(MAX_TRIES): try: - _ = mr.post('/images/', json=new_im_metadata) + mr.upload_mibitiff(out_path, run_id=run_id) break except: if t < MAX_TRIES-1: time.sleep(0.50) else: - _ = mr.post('/images/', json=new_im_metadata) - - for t in range(MAX_TRIES): - try: - mr.upload_mibitiff(out_path, run_id=ref_json['run']['id']) - break - except: - if t < MAX_TRIES-1: - time.sleep(0.50) - else: - mr.upload_mibitiff(out_path, run_id=ref_json['run']['id']) - - print(f"Stitched MIBItiff, {out_path} uploaded to MIBItracker.") + mr.upload_mibitiff(out_path, run_id=run_id) + + print(f"Stitched MIBItiff, {out_path}, uploaded to MIBItracker.") if __name__ == "__main__": - fovs_folder = "" # Local folder path with the original Run name and contents - out_path = "" # Local file path to save the stitched MIBItiff into - session_dict = { + run_folder = "" # Local folder path with the original Run name and contents + out_folder = "" # Local folder path to save the stitched MIBItiffs into + upload_to_mibitracker = True # Set to True to upload the resulting stitched MIBItiff to MIBItracker + session_dict = { # Only needed if uploading to MIBItracker "url": "", # MIBItracker backend URL (ex: https://mibitracker.api.ionpath.com/) "email": "", # MIBItracker user name "password": "" # MIBItrackefr user password } - upload_to_mibitracker = False # Set to True to upload the resulting stitched MIBItiff to MIBItracker + fov_margin_size = None # Sets the margin size in pixels when defined (int, list [x,y], or dict {x:0,y:0}); Leave as None to use as defined during imaging Run - stitch_fovs(fovs_folder, out_path, session_dict, upload_to_mibitracker) + stitch_fovs(run_folder, out_folder, session_dict, upload_to_mibitracker, fov_margin_size) From 9ad84fdebe2e8163665711425cfc4850cee3caab Mon Sep 17 00:00:00 2001 From: Mate Nagy Date: Tue, 4 Jun 2024 20:22:45 -0700 Subject: [PATCH 08/14] added nonsquare to stitch --- scripts/stitching/stitching.py | 32 +++++++++++++++++++++++--------- 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/scripts/stitching/stitching.py b/scripts/stitching/stitching.py index 6aeaf73..6483e76 100644 --- a/scripts/stitching/stitching.py +++ b/scripts/stitching/stitching.py @@ -50,10 +50,10 @@ def combine_entity_by_name(roi_fov_paths, cols, rows, enforce_square=False, marg elif out_img_shape[0] < out_img_shape[1]: out_img = np.pad(out_img,((0,out_img_shape[1]-out_img_shape[0]),(0,0),(0,0))) - return out_img.astype(out_img.dtype, copy=False), panel, int(max(w*shape_2d[1], h*shape_2d[0])), ref_metadata + return out_img.astype(out_img.dtype, copy=False), panel, (w*shape_2d[1], h*shape_2d[0]), ref_metadata -def stitch_fovs(run_folder, out_folder, session_dict, upload_to_mibitracker, fov_margin_size): +def stitch_fovs(run_folder, out_folder, session_dict, upload_to_mibitracker, fov_margin_size, enforce_square): run = os.path.basename(run_folder) run_json_file = os.path.join(run_folder,run)+".json" with open(run_json_file) as rf: @@ -62,7 +62,10 @@ def stitch_fovs(run_folder, out_folder, session_dict, upload_to_mibitracker, fov if r["standardTarget"] in ["Auto Gain"]: continue roi = r["name"] - px_per_u = r["frameSizePixels"]["width"] / r["fovSizeMicrons"] + if isinstance(r["fovSizeMicrons"],(tuple,list)): # image is non-square + px_per_u = r["frameSizePixels"]["width"] / r["fovSizeMicrons"][0] # 0th is width + else: # image is square + px_per_u = r["frameSizePixels"]["width"] / r["fovSizeMicrons"] margin = fov_margin_size if margin is None: margin = { @@ -89,13 +92,17 @@ def stitch_fovs(run_folder, out_folder, session_dict, upload_to_mibitracker, fov roi_fov_paths.append(os.path.join(run_folder,"fov"+"-"+str(f["runOrder"]).zfill(2)+"-"+fov_name)+".tiff") print(run, roi, f"{len(roi_fov_paths)} FOVs") - out_img, panel, max_dim, ref_metadata = combine_entity_by_name(roi_fov_paths, cols, rows, - enforce_square=False, margin=margin) + out_img, panel, dims, ref_metadata = combine_entity_by_name(roi_fov_paths, cols, rows, + enforce_square=enforce_square, + margin=margin) out_img = out_img.astype(np.uint8, copy=False) metadata = ref_metadata.copy() metadata["coordinates"] = (um_min_x, um_min_y) - metadata["size"] = max_dim/px_per_u + if dims[0] != dims[1]: # non-square + metadata["size"] = (dims[0]/px_per_u, dims[1]/px_per_u) + else: # square + metadata["size"] = dims[0]/px_per_u metadata["fov_name"] = roi out_mibi_tiff = mi.MibiImage( @@ -150,10 +157,16 @@ def stitch_fovs(run_folder, out_folder, session_dict, upload_to_mibitracker, fov new_im_metadata['point'] = roi new_im_metadata['number'] = out_mibi_tiff.fov_id new_im_metadata['folder'] = out_mibi_tiff.folder - new_im_metadata['fov_size'] = max_dim/px_per_u + if dims[0] != dims[1]: # non-square + new_im_metadata["fov_size"] = (dims[0]/px_per_u, dims[1]/px_per_u) + else: # square + new_im_metadata["fov_size"] = dims[0]/px_per_u new_im_metadata['dwell_time'] = metadata['dwell'] new_im_metadata['depths'] = metadata['scans'] - new_im_metadata['frame'] = max_dim + if dims[0] != dims[1]: # non-square + new_im_metadata["frame"] = (dims[0], dims[1]) + else: # square + new_im_metadata["frame"] = dims[0] new_im_metadata['time_bin'] = metadata['time_resolution'] new_im_metadata['mass_gain'] = metadata['mass_gain'] new_im_metadata['mass_offset'] = metadata['mass_offset'] @@ -211,6 +224,7 @@ def stitch_fovs(run_folder, out_folder, session_dict, upload_to_mibitracker, fov "password": "" # MIBItrackefr user password } fov_margin_size = None # Sets the margin size in pixels when defined (int, list [x,y], or dict {x:0,y:0}); Leave as None to use as defined during imaging Run + enforce_square = False # Set to True to pad the stitched image to be padded with blank space to make it square - stitch_fovs(run_folder, out_folder, session_dict, upload_to_mibitracker, fov_margin_size) + stitch_fovs(run_folder, out_folder, session_dict, upload_to_mibitracker, fov_margin_size, enforce_square) From 64d5bf8b4c96abd7327874a5ef51ca775c033793 Mon Sep 17 00:00:00 2001 From: Mate Nagy Date: Tue, 4 Jun 2024 20:34:57 -0700 Subject: [PATCH 09/14] added use of set_image_size function --- scripts/stitching/stitching.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/stitching/stitching.py b/scripts/stitching/stitching.py index 6483e76..3dfbbf0 100644 --- a/scripts/stitching/stitching.py +++ b/scripts/stitching/stitching.py @@ -107,6 +107,8 @@ def stitch_fovs(run_folder, out_folder, session_dict, upload_to_mibitracker, fov out_mibi_tiff = mi.MibiImage( out_img, panel, datetime_format='%Y-%m-%d', **metadata) + if dims[0] != dims[1]: # non-square + out_mibi_tiff.set_image_size((dims[0], dims[1])) f_split = out_mibi_tiff.folder.split('/') f_split[0] = roi From f1d1dc65ccef804a5d5d5a2fd5f233b9f5941092 Mon Sep 17 00:00:00 2001 From: Mate Nagy Date: Wed, 5 Jun 2024 14:18:53 -0700 Subject: [PATCH 10/14] fixes with nonsquare stitch --- scripts/stitching/stitching.py | 46 ++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/scripts/stitching/stitching.py b/scripts/stitching/stitching.py index 3dfbbf0..3b3d7d2 100644 --- a/scripts/stitching/stitching.py +++ b/scripts/stitching/stitching.py @@ -17,8 +17,8 @@ def combine_entity_by_name(roi_fov_paths, cols, rows, enforce_square=False, marg min_row = np.min(rows) cols=[v-min_col+1 for v in cols] rows=[v-min_row+1 for v in rows] - w = np.max(cols) - h = np.max(rows) + w = int(np.max(cols)) + h = int(np.max(rows)) panel = None out_img = None @@ -32,7 +32,7 @@ def combine_entity_by_name(roi_fov_paths, cols, rows, enforce_square=False, marg img_shape = list(np.shape(fov_img)) ch_count = np.min(img_shape) - shape_2d = img_shape[:-1] + shape_2d = [int(v) for v in img_shape[:-1]] if out_img is None: out_img = np.zeros((h*shape_2d[0]+(h-1)*margin["y"], w*shape_2d[1]+(w-1)*margin["x"], ch_count), @@ -59,7 +59,7 @@ def stitch_fovs(run_folder, out_folder, session_dict, upload_to_mibitracker, fov with open(run_json_file) as rf: run_json = json.load(rf) for r in run_json["rois"]: - if r["standardTarget"] in ["Auto Gain"]: + if r["standardTarget"] in ["Auto Gain", "Molybdenum Foil"]: continue roi = r["name"] if isinstance(r["fovSizeMicrons"],(tuple,list)): # image is non-square @@ -81,6 +81,8 @@ def stitch_fovs(run_folder, out_folder, session_dict, upload_to_mibitracker, fov um_min_x, um_min_y = 999999999, 999999999 roi_fov_paths=[] for f in r["fovs"]: + if not f["enabled"]: + continue fov_name = f["name"] cols.append(f["gridPosition"]["x"]+1) rows.append(f["gridPosition"]["y"]+1) @@ -92,23 +94,24 @@ def stitch_fovs(run_folder, out_folder, session_dict, upload_to_mibitracker, fov roi_fov_paths.append(os.path.join(run_folder,"fov"+"-"+str(f["runOrder"]).zfill(2)+"-"+fov_name)+".tiff") print(run, roi, f"{len(roi_fov_paths)} FOVs") - out_img, panel, dims, ref_metadata = combine_entity_by_name(roi_fov_paths, cols, rows, - enforce_square=enforce_square, - margin=margin) + try: + out_img, panel, dims, ref_metadata = combine_entity_by_name(roi_fov_paths, cols, rows, + enforce_square=enforce_square, + margin=margin) + except: + continue out_img = out_img.astype(np.uint8, copy=False) metadata = ref_metadata.copy() metadata["coordinates"] = (um_min_x, um_min_y) - if dims[0] != dims[1]: # non-square - metadata["size"] = (dims[0]/px_per_u, dims[1]/px_per_u) - else: # square - metadata["size"] = dims[0]/px_per_u metadata["fov_name"] = roi out_mibi_tiff = mi.MibiImage( out_img, panel, datetime_format='%Y-%m-%d', **metadata) if dims[0] != dims[1]: # non-square - out_mibi_tiff.set_image_size((dims[0], dims[1])) + out_mibi_tiff.set_image_size((dims[0]/px_per_u, dims[1]/px_per_u)) + else: # square + out_mibi_tiff.set_image_size(dims[0]/px_per_u) f_split = out_mibi_tiff.folder.split('/') f_split[0] = roi @@ -159,16 +162,16 @@ def stitch_fovs(run_folder, out_folder, session_dict, upload_to_mibitracker, fov new_im_metadata['point'] = roi new_im_metadata['number'] = out_mibi_tiff.fov_id new_im_metadata['folder'] = out_mibi_tiff.folder - if dims[0] != dims[1]: # non-square - new_im_metadata["fov_size"] = (dims[0]/px_per_u, dims[1]/px_per_u) - else: # square - new_im_metadata["fov_size"] = dims[0]/px_per_u + # if dims[0] != dims[1]: # non-square: to be used in MT 2.0 + # new_im_metadata["fov_size"] = (dims[0]/px_per_u, dims[1]/px_per_u) + # else: # square + new_im_metadata["fov_size"] = dims[0]/px_per_u new_im_metadata['dwell_time'] = metadata['dwell'] new_im_metadata['depths'] = metadata['scans'] - if dims[0] != dims[1]: # non-square - new_im_metadata["frame"] = (dims[0], dims[1]) - else: # square - new_im_metadata["frame"] = dims[0] + # if dims[0] != dims[1]: # non-square: to be used in MT 2.0 + # new_im_metadata["frame"] = (dims[0], dims[1]) + # else: # square + new_im_metadata["frame"] = dims[0] new_im_metadata['time_bin'] = metadata['time_resolution'] new_im_metadata['mass_gain'] = metadata['mass_gain'] new_im_metadata['mass_offset'] = metadata['mass_offset'] @@ -223,10 +226,9 @@ def stitch_fovs(run_folder, out_folder, session_dict, upload_to_mibitracker, fov session_dict = { # Only needed if uploading to MIBItracker "url": "", # MIBItracker backend URL (ex: https://mibitracker.api.ionpath.com/) "email": "", # MIBItracker user name - "password": "" # MIBItrackefr user password + "password": "" # MIBItracker user password } fov_margin_size = None # Sets the margin size in pixels when defined (int, list [x,y], or dict {x:0,y:0}); Leave as None to use as defined during imaging Run enforce_square = False # Set to True to pad the stitched image to be padded with blank space to make it square stitch_fovs(run_folder, out_folder, session_dict, upload_to_mibitracker, fov_margin_size, enforce_square) - From 015b35b2267a6973c49331c44619c8cf537d1025 Mon Sep 17 00:00:00 2001 From: Jay Tarolli Date: Tue, 2 Jul 2024 08:00:18 -0700 Subject: [PATCH 11/14] Add argparse for script parameters --- scripts/stitching/stitching.py | 205 ++++++++++++++++++++++----------- 1 file changed, 139 insertions(+), 66 deletions(-) diff --git a/scripts/stitching/stitching.py b/scripts/stitching/stitching.py index 3b3d7d2..e920c14 100644 --- a/scripts/stitching/stitching.py +++ b/scripts/stitching/stitching.py @@ -1,24 +1,27 @@ -""" Script for creating a stitched ROI MIBItiff from a folder of FOVs +""" Script for creating a stitched ROI MIBItiff from a folder of FOVs. """ -import numpy as np +import argparse +import json import os -from mibidata import mibi_image as mi -from mibitracker.request_helpers import MibiRequests +import sys import time + +import numpy as np + +from mibidata import mibi_image as mi from mibidata import tiff -import json +from mibitracker.request_helpers import MibiRequests MAX_TRIES = 5 - -def combine_entity_by_name(roi_fov_paths, cols, rows, enforce_square=False, margin=0): +def combine_entity_by_name(roi_fov_paths, cols, rows, enforce_square, margin): min_col = np.min(cols) min_row = np.min(rows) cols=[v-min_col+1 for v in cols] rows=[v-min_row+1 for v in rows] - w = int(np.max(cols)) - h = int(np.max(rows)) + w = np.max(cols) + h = np.max(rows) panel = None out_img = None @@ -32,40 +35,44 @@ def combine_entity_by_name(roi_fov_paths, cols, rows, enforce_square=False, marg img_shape = list(np.shape(fov_img)) ch_count = np.min(img_shape) - shape_2d = [int(v) for v in img_shape[:-1]] + shape_2d = img_shape[:-1] if out_img is None: - out_img = np.zeros((h*shape_2d[0]+(h-1)*margin["y"], w*shape_2d[1]+(w-1)*margin["x"], ch_count), - dtype=fov_img.dtype) + out_img = np.zeros( + (h*shape_2d[0]+(h-1)*margin["y"], + w*shape_2d[1]+(w-1)*margin["x"], ch_count), + dtype=fov_img.dtype) out_img_shape = list(np.shape(out_img)) out_img[((rows[fov_i]-1)*shape_2d[0]+(rows[fov_i]-1)*margin["y"]): (rows[fov_i]*shape_2d[0]+(rows[fov_i]-1)*margin["y"]), ((cols[fov_i]-1)*shape_2d[1]+(cols[fov_i]-1)*margin["x"]): - (cols[fov_i]*shape_2d[1]+(cols[fov_i]-1)*margin["x"]), :] = fov_img[:,:,:] + (cols[fov_i]*shape_2d[1]+(cols[fov_i]-1)*margin["x"]), :] = \ + fov_img[:,:,:] - if enforce_square: # Set to True if there is a need to pad the image to be a square + if enforce_square: if out_img_shape[0] > out_img_shape[1]: - out_img = np.pad(out_img,((0,0),(0,out_img_shape[0]-out_img_shape[1]),(0,0))) + out_img = np.pad( + out_img,((0,0),(0,out_img_shape[0]-out_img_shape[1]),(0,0))) elif out_img_shape[0] < out_img_shape[1]: - out_img = np.pad(out_img,((0,out_img_shape[1]-out_img_shape[0]),(0,0),(0,0))) + out_img = np.pad( + out_img,((0,out_img_shape[1]-out_img_shape[0]),(0,0),(0,0))) - return out_img.astype(out_img.dtype, copy=False), panel, (w*shape_2d[1], h*shape_2d[0]), ref_metadata + return out_img.astype(out_img.dtype, copy=False), panel, \ + int(max(w*shape_2d[1], h*shape_2d[0])), ref_metadata -def stitch_fovs(run_folder, out_folder, session_dict, upload_to_mibitracker, fov_margin_size, enforce_square): +def stitch_fovs(run_folder, out_folder, session_dict, upload_to_mibitracker, + fov_margin_size, enforce_square): run = os.path.basename(run_folder) run_json_file = os.path.join(run_folder,run)+".json" with open(run_json_file) as rf: run_json = json.load(rf) for r in run_json["rois"]: - if r["standardTarget"] in ["Auto Gain", "Molybdenum Foil"]: + if r["standardTarget"] in ["Auto Gain"]: continue roi = r["name"] - if isinstance(r["fovSizeMicrons"],(tuple,list)): # image is non-square - px_per_u = r["frameSizePixels"]["width"] / r["fovSizeMicrons"][0] # 0th is width - else: # image is square - px_per_u = r["frameSizePixels"]["width"] / r["fovSizeMicrons"] + px_per_u = r["frameSizePixels"]["width"] / r["fovSizeMicrons"] margin = fov_margin_size if margin is None: margin = { @@ -81,8 +88,6 @@ def stitch_fovs(run_folder, out_folder, session_dict, upload_to_mibitracker, fov um_min_x, um_min_y = 999999999, 999999999 roi_fov_paths=[] for f in r["fovs"]: - if not f["enabled"]: - continue fov_name = f["name"] cols.append(f["gridPosition"]["x"]+1) rows.append(f["gridPosition"]["y"]+1) @@ -91,27 +96,25 @@ def stitch_fovs(run_folder, out_folder, session_dict, upload_to_mibitracker, fov um_min_x = coord["x"] if coord["y"] < um_min_y: um_min_y = coord["y"] - roi_fov_paths.append(os.path.join(run_folder,"fov"+"-"+str(f["runOrder"]).zfill(2)+"-"+fov_name)+".tiff") + roi_fov_paths.append( + os.path.join( + run_folder, + "fov" + "-" + str( + f["runOrder"]).zfill(2) + "-" + fov_name) + ".tiff") print(run, roi, f"{len(roi_fov_paths)} FOVs") - try: - out_img, panel, dims, ref_metadata = combine_entity_by_name(roi_fov_paths, cols, rows, - enforce_square=enforce_square, - margin=margin) - except: - continue + out_img, panel, max_dim, ref_metadata = combine_entity_by_name( + roi_fov_paths, cols, rows, enforce_square=enforce_square, + margin=margin) out_img = out_img.astype(np.uint8, copy=False) metadata = ref_metadata.copy() metadata["coordinates"] = (um_min_x, um_min_y) + metadata["size"] = max_dim/px_per_u metadata["fov_name"] = roi out_mibi_tiff = mi.MibiImage( out_img, panel, datetime_format='%Y-%m-%d', **metadata) - if dims[0] != dims[1]: # non-square - out_mibi_tiff.set_image_size((dims[0]/px_per_u, dims[1]/px_per_u)) - else: # square - out_mibi_tiff.set_image_size(dims[0]/px_per_u) f_split = out_mibi_tiff.folder.split('/') f_split[0] = roi @@ -136,50 +139,57 @@ def stitch_fovs(run_folder, out_folder, session_dict, upload_to_mibitracker, fov for t in range(MAX_TRIES): try: exists = mr.get( - '/images/', params={'run__label': run, 'number': f_split[0]}).json() + '/images/', + params={ + 'run__label': run, + 'number': f_split[0]}).json() break except: if t < MAX_TRIES-1: time.sleep(0.50) else: exists = mr.get( - '/images/', params={'run__label': run, 'number': f_split[0]}).json() + '/images/', + params={ + 'run__label': run, + 'number': f_split[0]}).json() - full_id = exists['results'][0]['id'] if exists['count'] else None + full_id = exists['results'][0]['id'] if exists['count'] \ + else None for t in range(MAX_TRIES): try: - run_id = mr.get('/runs/', params={'name': run}).json()['results'][0]['id'] + run_id = mr.get( + '/runs/', + params={'name': run}).json()['results'][0]['id'] break except: if t < MAX_TRIES-1: time.sleep(0.50) else: - run_id = mr.get('/runs/', params={'name': run}).json()['results'][0]['id'] + run_id = mr.get( + '/runs/', + params={'name': run}).json()['results'][0]['id'] new_im_metadata = {} new_im_metadata['run'] = run_id new_im_metadata['point'] = roi new_im_metadata['number'] = out_mibi_tiff.fov_id new_im_metadata['folder'] = out_mibi_tiff.folder - # if dims[0] != dims[1]: # non-square: to be used in MT 2.0 - # new_im_metadata["fov_size"] = (dims[0]/px_per_u, dims[1]/px_per_u) - # else: # square - new_im_metadata["fov_size"] = dims[0]/px_per_u + new_im_metadata['fov_size'] = max_dim/px_per_u new_im_metadata['dwell_time'] = metadata['dwell'] new_im_metadata['depths'] = metadata['scans'] - # if dims[0] != dims[1]: # non-square: to be used in MT 2.0 - # new_im_metadata["frame"] = (dims[0], dims[1]) - # else: # square - new_im_metadata["frame"] = dims[0] + new_im_metadata['frame'] = max_dim new_im_metadata['time_bin'] = metadata['time_resolution'] new_im_metadata['mass_gain'] = metadata['mass_gain'] new_im_metadata['mass_offset'] = metadata['mass_offset'] new_im_metadata['x_coord'] = int(np.round(um_min_x)) new_im_metadata['y_coord'] = int(np.round(um_min_y)) - new_im_metadata['tissue'] = metadata['raw_description']['fov']['section']['tissue'] \ - and metadata['raw_description']['fov']['section']['tissue']['id'] - new_im_metadata['section'] = metadata['raw_description']['fov']['section']['id'] + new_im_metadata['tissue'] = metadata['raw_description']['fov'] \ + ['section']['tissue'] and metadata['raw_description'] \ + ['fov']['section']['tissue']['id'] + new_im_metadata['section'] = metadata['raw_description'] \ + ['fov']['section']['id'] new_im_metadata['aperture'] = metadata['aperture'] new_im_metadata['imaging_preset'] = metadata['imaging_preset'] new_im_metadata['lens1_voltage'] = metadata['lens1_voltage'] @@ -197,13 +207,14 @@ def stitch_fovs(run_folder, out_folder, session_dict, upload_to_mibitracker, fov else: for t in range(MAX_TRIES): try: - _ = mr.put(f'/images/{full_id}', json=new_im_metadata) + mr.put(f'/images/{full_id}', json=new_im_metadata) break except: if t < MAX_TRIES-1: time.sleep(0.50) else: - _ = mr.put(f'/images/{full_id}', json=new_im_metadata) + mr.put( + f'/images/{full_id}', json=new_im_metadata) for t in range(MAX_TRIES): try: @@ -215,20 +226,82 @@ def stitch_fovs(run_folder, out_folder, session_dict, upload_to_mibitracker, fov else: mr.upload_mibitiff(out_path, run_id=run_id) - print(f"Stitched MIBItiff, {out_path}, uploaded to MIBItracker.") - + print( + f"Stitched MIBItiff, {out_path}, uploaded to MIBItracker.") + +def parse_args(args): + ''' Argument parsing helper function. + ''' + parser = argparse.ArgumentParser( + description='Script for creating a stitched ROI MIBItiff from a folder ' + 'of FOVs.', + formatter_class=argparse.RawTextHelpFormatter) + parser.add_argument( + '--run_folder', + help='Local folder path with the original Run name and contents.') + parser.add_argument( + '--out_folder', + help='Local folder path to save the stitched MIBItiffs into.') + parser.add_argument( + '--fov_margin_size_x', type=int, required=False, default=0, + help='(optional) Sets the margin size in pixels for the x-dimension. ' + 'Defaults to 0 for no overlap or spacing. Use a positive integer ' + 'to add spacing between adjacent tiles and a negative integer to ' + 'overlap adjacent tiles.') + parser.add_argument( + '--fov_margin_size_y', type=int, required=False, default=0, + help='(optional) Sets the margin size in pixels for the y-dimension. ' + 'Defaults to 0 for no overlap or spacing. Use a positive integer ' + 'to add spacing between adjacent tiles and a negative integer to ' + 'overlap adjacent tiles.') + parser.add_argument( + '--upload_to_mibitracker', default=True, type=bool, + help='(optional) Set to True to upload the resulting stitched MIBItiff to ' + 'MIBItracker. Defaults to True.') + parser.add_argument( + '--mibitracker_url', required=False, + help='(optional) MIBItracker backend URL if --upload_to_mibitracker if ' + 'True (e.g. https://mibitracker.api.ionpath.com/).') + parser.add_argument( + '--mibitracker_email', required=False, + help='(optional) MIBItracker user name if --upload_to_mibitracker is ' + 'True.') + parser.add_argument( + '--mibitracker_password', required=False, + help='(optional) MIBItracker password if --upload_to_mibitracker is ' + 'True.') + parser.add_argument( + '--enforce_square', default=False, type=bool, + help='(optional) Set to True to pad the stitched image with zeros so ' + 'the resulting MIBItiff image is square. Defaults to False.') + + args = parser.parse_args(args) + if args.upload_to_mibitracker and not all( + [args.mibitracker_url, args.mibitracker_email, + args.mibitracker_password]): + raise ValueError( + '--mibitracker_url, --mibitracker_email, and ' + '--mibitracker_password must be specified if ' + '--upload_to_mibitracker is `True`.') + if not args.fov_margin_size_x and args.fov_margin_size_y: + args.fov_margin_size = None + else: + args.fov_margin_size = { + 'x': args.fov_margin_size_x if args.fov_margin_size_x else 0, + 'y': args.fov_margin_size_y if args.fov_margin_size_y else 0, + } + + return args if __name__ == "__main__": + args = parse_args(sys.argv[1:]) - run_folder = "" # Local folder path with the original Run name and contents - out_folder = "" # Local folder path to save the stitched MIBItiffs into - upload_to_mibitracker = True # Set to True to upload the resulting stitched MIBItiff to MIBItracker - session_dict = { # Only needed if uploading to MIBItracker - "url": "", # MIBItracker backend URL (ex: https://mibitracker.api.ionpath.com/) - "email": "", # MIBItracker user name - "password": "" # MIBItracker user password + session_dict = { + 'url': args.mibitracker_url, + 'email': args.mibitracker_email, + 'password': args.mibitracker_password } - fov_margin_size = None # Sets the margin size in pixels when defined (int, list [x,y], or dict {x:0,y:0}); Leave as None to use as defined during imaging Run - enforce_square = False # Set to True to pad the stitched image to be padded with blank space to make it square - stitch_fovs(run_folder, out_folder, session_dict, upload_to_mibitracker, fov_margin_size, enforce_square) + stitch_fovs(args.run_folder, args.out_folder, session_dict, + args.upload_to_mibitracker, args.fov_margin_size, + args.enforce_square) From 13cfd063fd1a58b06b4b702b6af24a42e8403059 Mon Sep 17 00:00:00 2001 From: Jay Tarolli Date: Wed, 3 Jul 2024 15:49:40 -0700 Subject: [PATCH 12/14] Verify args and small changes --- scripts/stitching/stitching.py | 300 +++++++++++++++++---------------- 1 file changed, 159 insertions(+), 141 deletions(-) diff --git a/scripts/stitching/stitching.py b/scripts/stitching/stitching.py index e920c14..e875b5f 100644 --- a/scripts/stitching/stitching.py +++ b/scripts/stitching/stitching.py @@ -68,179 +68,189 @@ def stitch_fovs(run_folder, out_folder, session_dict, upload_to_mibitracker, run_json_file = os.path.join(run_folder,run)+".json" with open(run_json_file) as rf: run_json = json.load(rf) - for r in run_json["rois"]: - if r["standardTarget"] in ["Auto Gain"]: - continue - roi = r["name"] - px_per_u = r["frameSizePixels"]["width"] / r["fovSizeMicrons"] - margin = fov_margin_size - if margin is None: - margin = { - "x":int(np.round(r["xMargin"]*px_per_u)), - "y":int(np.round(r["yMargin"]*px_per_u)) - } - elif type(margin) != dict: - if type(margin) != list: - margin = {"x":margin, "y":margin} - else: - margin = {"x":margin[0], "y":margin[1]} - cols, rows = [], [] - um_min_x, um_min_y = 999999999, 999999999 - roi_fov_paths=[] - for f in r["fovs"]: - fov_name = f["name"] - cols.append(f["gridPosition"]["x"]+1) - rows.append(f["gridPosition"]["y"]+1) - coord = f["centerPointMicrons"] - if coord["x"] < um_min_x: - um_min_x = coord["x"] - if coord["y"] < um_min_y: - um_min_y = coord["y"] - roi_fov_paths.append( - os.path.join( - run_folder, - "fov" + "-" + str( - f["runOrder"]).zfill(2) + "-" + fov_name) + ".tiff") - - print(run, roi, f"{len(roi_fov_paths)} FOVs") - out_img, panel, max_dim, ref_metadata = combine_entity_by_name( - roi_fov_paths, cols, rows, enforce_square=enforce_square, - margin=margin) - out_img = out_img.astype(np.uint8, copy=False) + for r in run_json["rois"]: + if r["standardTarget"] in ["Auto Gain"]: + continue + roi = r["name"] + px_per_u = r["frameSizePixels"]["width"] / r["fovSizeMicrons"] + margin = fov_margin_size + if margin is None: + margin = { + "x":int(np.round(r["xMargin"]*px_per_u)), + "y":int(np.round(r["yMargin"]*px_per_u)) + } + elif type(margin) != dict: + if type(margin) != list: + margin = {"x":margin, "y":margin} + else: + margin = {"x":margin[0], "y":margin[1]} + cols, rows = [], [] + um_min_x, um_min_y = 999999999, 999999999 + roi_fov_paths=[] + for f in r["fovs"]: + fov_name = f["name"] + cols.append(f["gridPosition"]["x"]+1) + rows.append(f["gridPosition"]["y"]+1) + coord = f["centerPointMicrons"] + if coord["x"] < um_min_x: + um_min_x = coord["x"] + if coord["y"] < um_min_y: + um_min_y = coord["y"] + roi_fov_paths.append( + os.path.join( + run_folder, + "fov" + "-" + str( + f["runOrder"]).zfill(2) + "-" + fov_name) + ".tiff") + + print(f'Stitching {run} - {roi}: {len(roi_fov_paths)} FOVs') + out_img, panel, max_dim, ref_metadata = combine_entity_by_name( + roi_fov_paths, cols, rows, enforce_square=enforce_square, + margin=margin) + out_img = out_img.astype(np.uint8, copy=False) - metadata = ref_metadata.copy() - metadata["coordinates"] = (um_min_x, um_min_y) - metadata["size"] = max_dim/px_per_u - metadata["fov_name"] = roi + metadata = ref_metadata.copy() + metadata["coordinates"] = (um_min_x, um_min_y) + metadata["size"] = max_dim/px_per_u + metadata["fov_name"] = roi - out_mibi_tiff = mi.MibiImage( - out_img, panel, datetime_format='%Y-%m-%d', **metadata) - - f_split = out_mibi_tiff.folder.split('/') - f_split[0] = roi - out_mibi_tiff.set_fov_id(f_split[0], '/'.join(f_split)) + out_mibi_tiff = mi.MibiImage( + out_img, panel, datetime_format='%Y-%m-%d', **metadata) + + f_split = out_mibi_tiff.folder.split('/') + f_split[0] = roi + out_mibi_tiff.set_fov_id(f_split[0], '/'.join(f_split)) + if not out_folder: + out_path = os.path.join(run_folder, roi + ".tiff") + else: out_path = os.path.join(out_folder,roi+".tiff") - tiff.write(out_path, out_mibi_tiff, dtype=np.float32) - print(f"Stitched MIBItiff saved to {out_path}.") + tiff.write(out_path, out_mibi_tiff, dtype=np.float32) + print(f"Stitched MIBItiff saved to {out_path}.") - if upload_to_mibitracker: - mr = None - for t in range(MAX_TRIES): - try: + if upload_to_mibitracker: + mr = None + for t in range(MAX_TRIES): + try: + mr = MibiRequests(**session_dict) + break + except: + if t < MAX_TRIES-1: + time.sleep(0.50) + else: mr = MibiRequests(**session_dict) - break - except: - if t < MAX_TRIES-1: - time.sleep(0.50) - else: - mr = MibiRequests(**session_dict) - - for t in range(MAX_TRIES): - try: + for t in range(MAX_TRIES): + try: + exists = mr.get( + '/images/', + params={ + 'run__label': run, + 'number': f_split[0]}).json() + break + except: + if t < MAX_TRIES-1: + time.sleep(0.50) + else: exists = mr.get( '/images/', params={ 'run__label': run, 'number': f_split[0]}).json() - break - except: - if t < MAX_TRIES-1: - time.sleep(0.50) - else: - exists = mr.get( - '/images/', - params={ - 'run__label': run, - 'number': f_split[0]}).json() - full_id = exists['results'][0]['id'] if exists['count'] \ - else None + full_id = exists['results'][0]['id'] if exists['count'] \ + else None - for t in range(MAX_TRIES): - try: + for t in range(MAX_TRIES): + try: + run_id = mr.get( + '/runs/', + params={'name': run}).json()['results'][0]['id'] + break + except: + if t < MAX_TRIES-1: + time.sleep(0.50) + else: run_id = mr.get( '/runs/', params={'name': run}).json()['results'][0]['id'] + + new_im_metadata = {} + new_im_metadata['run'] = run_id + new_im_metadata['point'] = roi + new_im_metadata['number'] = out_mibi_tiff.fov_id + new_im_metadata['folder'] = out_mibi_tiff.folder + new_im_metadata['fov_size'] = max_dim/px_per_u + new_im_metadata['dwell_time'] = metadata['dwell'] + new_im_metadata['depths'] = metadata['scans'] + new_im_metadata['frame'] = max_dim + new_im_metadata['time_bin'] = metadata['time_resolution'] + new_im_metadata['mass_gain'] = metadata['mass_gain'] + new_im_metadata['mass_offset'] = metadata['mass_offset'] + new_im_metadata['x_coord'] = int(np.round(um_min_x)) + new_im_metadata['y_coord'] = int(np.round(um_min_y)) + new_im_metadata['tissue'] = metadata['raw_description']['fov'] \ + ['section']['tissue'] and metadata['raw_description'] \ + ['fov']['section']['tissue']['id'] + new_im_metadata['section'] = metadata['raw_description'] \ + ['fov']['section']['id'] + new_im_metadata['aperture'] = metadata['aperture'] + new_im_metadata['imaging_preset'] = metadata['imaging_preset'] + new_im_metadata['lens1_voltage'] = metadata['lens1_voltage'] + + if not full_id: + for t in range(MAX_TRIES): + try: + _ = mr.post('/images/', json=new_im_metadata) break except: if t < MAX_TRIES-1: time.sleep(0.50) else: - run_id = mr.get( - '/runs/', - params={'name': run}).json()['results'][0]['id'] - - new_im_metadata = {} - new_im_metadata['run'] = run_id - new_im_metadata['point'] = roi - new_im_metadata['number'] = out_mibi_tiff.fov_id - new_im_metadata['folder'] = out_mibi_tiff.folder - new_im_metadata['fov_size'] = max_dim/px_per_u - new_im_metadata['dwell_time'] = metadata['dwell'] - new_im_metadata['depths'] = metadata['scans'] - new_im_metadata['frame'] = max_dim - new_im_metadata['time_bin'] = metadata['time_resolution'] - new_im_metadata['mass_gain'] = metadata['mass_gain'] - new_im_metadata['mass_offset'] = metadata['mass_offset'] - new_im_metadata['x_coord'] = int(np.round(um_min_x)) - new_im_metadata['y_coord'] = int(np.round(um_min_y)) - new_im_metadata['tissue'] = metadata['raw_description']['fov'] \ - ['section']['tissue'] and metadata['raw_description'] \ - ['fov']['section']['tissue']['id'] - new_im_metadata['section'] = metadata['raw_description'] \ - ['fov']['section']['id'] - new_im_metadata['aperture'] = metadata['aperture'] - new_im_metadata['imaging_preset'] = metadata['imaging_preset'] - new_im_metadata['lens1_voltage'] = metadata['lens1_voltage'] - - if not full_id: - for t in range(MAX_TRIES): - try: _ = mr.post('/images/', json=new_im_metadata) - break - except: - if t < MAX_TRIES-1: - time.sleep(0.50) - else: - _ = mr.post('/images/', json=new_im_metadata) - else: - for t in range(MAX_TRIES): - try: - mr.put(f'/images/{full_id}', json=new_im_metadata) - break - except: - if t < MAX_TRIES-1: - time.sleep(0.50) - else: - mr.put( - f'/images/{full_id}', json=new_im_metadata) - + else: for t in range(MAX_TRIES): try: - mr.upload_mibitiff(out_path, run_id=run_id) + mr.put(f'/images/{full_id}', json=new_im_metadata) break except: if t < MAX_TRIES-1: time.sleep(0.50) else: - mr.upload_mibitiff(out_path, run_id=run_id) - - print( - f"Stitched MIBItiff, {out_path}, uploaded to MIBItracker.") + mr.put( + f'/images/{full_id}', json=new_im_metadata) + + for t in range(MAX_TRIES): + try: + mr.upload_mibitiff(out_path, run_id=run_id) + break + except: + if t < MAX_TRIES-1: + time.sleep(0.50) + else: + mr.upload_mibitiff(out_path, run_id=run_id) + + print( + f"Stitched MIBItiff, {out_path}, uploaded to MIBItracker.") def parse_args(args): ''' Argument parsing helper function. ''' parser = argparse.ArgumentParser( description='Script for creating a stitched ROI MIBItiff from a folder ' - 'of FOVs.', + 'of FOVs. This script will take all tiled FOVs contained ' + 'in --run_folder and stitch them into a single MIBItiff ' + 'file. The output file will have the name [ROI_name].tiff ' + 'and by default will be saved in the same folder as the ' + 'individual FOV MIBItiffs. The output folder can be ' + 'specified with the --out_folder argument. Additional ' + 'arguments can be used to upload the resulting MIBItiff to ' + 'MIBItracker as well as control the size and shape of the ' + 'reconstructed image.', formatter_class=argparse.RawTextHelpFormatter) parser.add_argument( '--run_folder', help='Local folder path with the original Run name and contents.') parser.add_argument( - '--out_folder', + '--out_folder', required=False, help='Local folder path to save the stitched MIBItiffs into.') parser.add_argument( '--fov_margin_size_x', type=int, required=False, default=0, @@ -255,13 +265,13 @@ def parse_args(args): 'to add spacing between adjacent tiles and a negative integer to ' 'overlap adjacent tiles.') parser.add_argument( - '--upload_to_mibitracker', default=True, type=bool, - help='(optional) Set to True to upload the resulting stitched MIBItiff to ' - 'MIBItracker. Defaults to True.') + '--upload_to_mibitracker', action='store_true', + help='Pass this flag to upload the resulting stitched MIBItiff to ' + 'MIBItracker.') parser.add_argument( '--mibitracker_url', required=False, help='(optional) MIBItracker backend URL if --upload_to_mibitracker if ' - 'True (e.g. https://mibitracker.api.ionpath.com/).') + 'True (e.g. https://sitename.api.ionpath.com/).') parser.add_argument( '--mibitracker_email', required=False, help='(optional) MIBItracker user name if --upload_to_mibitracker is ' @@ -271,11 +281,18 @@ def parse_args(args): help='(optional) MIBItracker password if --upload_to_mibitracker is ' 'True.') parser.add_argument( - '--enforce_square', default=False, type=bool, - help='(optional) Set to True to pad the stitched image with zeros so ' - 'the resulting MIBItiff image is square. Defaults to False.') - + '--enforce_square', action='store_true', + help='Pass this flag to pad the stitched image with zeros so the ' + 'resulting MIBItiff image is square.') + args = parser.parse_args(args) + + # Check that required arguments are included + if not args.run_folder: + raise ValueError( + '--run_folder is a required argument and must be specified') + # If uploading to MIBItracker, check that URL, email, and password are + # included. if args.upload_to_mibitracker and not all( [args.mibitracker_url, args.mibitracker_email, args.mibitracker_password]): @@ -283,6 +300,7 @@ def parse_args(args): '--mibitracker_url, --mibitracker_email, and ' '--mibitracker_password must be specified if ' '--upload_to_mibitracker is `True`.') + # Properly set the dict with the x- and y-margins if not args.fov_margin_size_x and args.fov_margin_size_y: args.fov_margin_size = None else: From 9e4c1795d7c4e04af90341534b6beb18c7560ba3 Mon Sep 17 00:00:00 2001 From: Jay Tarolli Date: Wed, 3 Jul 2024 15:57:51 -0700 Subject: [PATCH 13/14] Remove unused variable --- scripts/stitching/stitching.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/stitching/stitching.py b/scripts/stitching/stitching.py index e875b5f..ea2bcb0 100644 --- a/scripts/stitching/stitching.py +++ b/scripts/stitching/stitching.py @@ -199,7 +199,7 @@ def stitch_fovs(run_folder, out_folder, session_dict, upload_to_mibitracker, if not full_id: for t in range(MAX_TRIES): try: - _ = mr.post('/images/', json=new_im_metadata) + mr.post('/images/', json=new_im_metadata) break except: if t < MAX_TRIES-1: From 07d097e0d320ae351f1a58f46b8dfcd850d7535d Mon Sep 17 00:00:00 2001 From: Jay Tarolli Date: Tue, 6 Aug 2024 11:16:09 -0700 Subject: [PATCH 14/14] Add README for the script --- scripts/stitching/README.md | 78 ++++++++++++++++++++++++++++++++++ scripts/stitching/stitching.py | 2 + 2 files changed, 80 insertions(+) create mode 100644 scripts/stitching/README.md diff --git a/scripts/stitching/README.md b/scripts/stitching/README.md new file mode 100644 index 0000000..3ffa5b6 --- /dev/null +++ b/scripts/stitching/README.md @@ -0,0 +1,78 @@ +# Stitiching +The [stitching.py](stitching.py) script is provided as a working example for +reconstructing multiple acquired FOVs into a single ROI, saving the combined +image as a MIBItiff file and optionally uploading the resulting image file to +MIBItracker. + +## Requirements +- The `mibilib` conda environment has been created and activated or the required +packages have been installed to a Python environment. For details on installing +the `mibilib` conda environment, see the [README](../../README.md). +- The FOVs to stitch into a single ROI must have been acquired in the same run +and the run must have been acquired with the ROI acquisition feature introduced +with MIBIcontrol v1.9.1. + +## Stitching a ROI +For a complete list of required and optional script arguments, please refer to +the [Usage](#usage) section. + +The only required argument is the `--run_folder` which specifies the path to a +local folder which contains one or more ROIs to stitch. Note that all ROIs +contained in the folder will be reconstructed. When reconstructing a ROI from a +series of FOVs with this script, the `fov_name` metadata field of the MIBItiff +will be the name of the ROI, which is parsed from the file names in the local +folder. + +`python .\stitching.py --run_folder /path/to/local/folder/` + + +Optionally, you can specify that the MIBItiff of the reconstructed ROI be saved +to a different location with the `--out_folder` argument. + +`python .\stitching.py --run_folder /path/to/local/folder/ --out_folder /path/to/another/folder` + +To upload the reconstructed ROI MIBItiff to MIBItracker, you must include +`--upload_to_mibitracker` and specify all of `--mibitracker_url`, +`--mibitracker_email`, and `--mibitracker_password`. Often, the URL, email, and +password are saved as environment variables. + +`python .\stitching.py --run_folder /path/to/local/folder/ --upload_to_mibitracker --mibitracker_url https://sitename.api.ionpath.com --mibitracker_email you@email.com --mibitracker_password yourmibitrackerpassword` + +When uploading to MIBItracker, the MIBItiff will be uploaded to the same run as +the run which acquired the ROI. The `fov_name` will be the name of the ROI, +which is parsed from the file names in the local folder. + +If the ROI was acquired with overlap of adjacent FOVs or padding between +adjacent FOVs, use the `--fov_margin_size_x` and/or `--fov_margin_size_y` +arguments. + +`python .\stitching.py --run_folder /path/to/local/folder/ --fov_margin_size_x 20 --fov_margin_size_y 10` + +## Usage +Run `python stitching.py --help` to print out a list of required and optional +script arguments. + +usage: `stitching.py [-h] [--run_folder RUN_FOLDER] [--out_folder OUT_FOLDER] + [--fov_margin_size_x FOV_MARGIN_SIZE_X] + [--fov_margin_size_y FOV_MARGIN_SIZE_Y] + [--upload_to_mibitracker] + [--mibitracker_url MIBITRACKER_URL] + [--mibitracker_email MIBITRACKER_EMAIL] + [--mibitracker_password MIBITRACKER_PASSWORD] + [--enforce_square]` + +Script for creating a stitched ROI MIBItiff from a folder of FOVs. This script will take all tiled FOVs contained in --run_folder and stitch them into a single MIBItiff file. The output file will have the name [ROI_name].tiff and by default will be saved in the same folder as the individual FOV MIBItiffs. The output folder can be specified with the `--out_folder` argument. Additional arguments can be used to upload the resulting MIBItiff to MIBItracker as well as control the size and shape of the reconstructed image. + +||| +|-|-| +|`-h`, `--help`|show this help message and exit| +|`--run_folder`|Local folder path with the original Run name and contents.| +|`--out_folder`|Local folder path to save the stitched MIBItiffs into.| +|`--fov_margin_size_x`|(optional) Sets the margin size in pixels for the x-dimension. Defaults to 0 for no overlap or spacing. Use a positive integer to add spacing between adjacent tiles and a negative integer to overlap adjacent tiles.| +|`--fov_margin_size_y`|(optional) Sets the margin size in pixels for the y-dimension. Defaults to 0 for no overlap or spacing. Use a positive integer to add spacing between adjacent tiles and a negative integer to overlap adjacent tiles.| +|`--upload_to_mibitracker`|Pass this flag to upload the resulting stitched MIBItiff to MIBItracker.| +|`--mibitracker_url`|(optional) MIBItracker backend URL if --upload_to_mibitracker if True (e.g. https://sitename.api.ionpath.com/).| +|`--mibitracker_email`|(optional) MIBItracker user name if --upload_to_mibitracker is True.| +|`--mibitracker_password`|(optional) MIBItracker password if --upload_to_mibitracker is True.| +|`--enforce_square`|Pass this flag to pad the stitched image with zeros so the resulting MIBItiff image is square.| + diff --git a/scripts/stitching/stitching.py b/scripts/stitching/stitching.py index ea2bcb0..6d112ba 100644 --- a/scripts/stitching/stitching.py +++ b/scripts/stitching/stitching.py @@ -9,6 +9,8 @@ import numpy as np +sys.path.append('../..') + from mibidata import mibi_image as mi from mibidata import tiff from mibitracker.request_helpers import MibiRequests