From 20e87628b63ef083d9b34a598f2a062ddf785e64 Mon Sep 17 00:00:00 2001 From: emjoyce <=> Date: Thu, 13 Oct 2022 12:55:21 -0700 Subject: [PATCH 1/7] added task function for layer histogram creation --- skeleton_keys/tasks.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/skeleton_keys/tasks.py b/skeleton_keys/tasks.py index a970808..f936448 100644 --- a/skeleton_keys/tasks.py +++ b/skeleton_keys/tasks.py @@ -12,9 +12,15 @@ ProcessMorphologyFeaturesParameters, main as morph_feat_main, ) +from skeleton_keys.cmds.depth_profiles_from_aligned_swcs import ( + ProfilesFromAlignedSwcsParameters, + main as depth_profile_main +) + import argschema + @queueable def layer_align_cell( specimen_id, @@ -115,3 +121,28 @@ def extract_morphology_features( schema_type=ProcessMorphologyFeaturesParameters, input_data=input_data, args=[] ) morph_feat_main(module.args) + +@queueable +def create_layer_histograms( + specimen_id_file, + swc_dir, + layer_depths_file, + output_hist_file, + output_soma_file, + bin_size=5.0, + below_wm=200.0,): + input_data = { + "specimen_id_file": specimen_id_file, + "swc_dir": swc_dir, + "layer_depths_file": layer_depths_file, + "output_hist_file": output_hist_file, + "output_soma_file": output_soma_file, + "bin_size": bin_size, + "below_wm": below_wm, + } + + input_data = {k: v for k, v in input_data.items() if v is not None} + module = argschema.ArgSchemaParser( + schema_type=ProfilesFromAlignedSwcsParameters, input_data=input_data, args=[] + ) + depth_profile_main(module.args) \ No newline at end of file From 2692c711bc3480bf130426f3a1abab7f461daacd Mon Sep 17 00:00:00 2001 From: emjoyce <=> Date: Fri, 14 Oct 2022 15:57:01 -0700 Subject: [PATCH 2/7] more changes for hist tasks --- Dockerfile | 4 ++-- skeleton_keys/cmds/depth_profiles_from_aligned_swcs.py | 7 +++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 6cfddb0..131ea0e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,8 @@ FROM continuumio/miniconda3:4.10.3 RUN apt-get update && apt-get install -y build-essential && rm -rf /var/lib/apt/lists/* RUN conda config --set channel_priority strict -RUN conda install -y -c conda-forge rtree==0.9.7 fenics==2019.1.0 python==3.8 mshr==2019.1.0 hdf5==1.10.6 h5py==2.10.0 -RUN pip install git+https://github.com/fcollman/AllenSDK.git +RUN conda install -y -c conda-forge rtree==0.9.7 fenics==2019.1.0 python==3.8 mshr==2019.1.0 hdf5==1.10.6 h5py==2.10.0 Jinja2==2.11.3 +RUN pip install git+https://github.com/AllenInstitute/AllenSDK.git RUN pip install git+https://github.com/AllenInstitute/neuron_morphology@cloudfiles ARG GITHUB_TOKEN WORKDIR /usr/local/src diff --git a/skeleton_keys/cmds/depth_profiles_from_aligned_swcs.py b/skeleton_keys/cmds/depth_profiles_from_aligned_swcs.py index 427476a..db7ad4b 100644 --- a/skeleton_keys/cmds/depth_profiles_from_aligned_swcs.py +++ b/skeleton_keys/cmds/depth_profiles_from_aligned_swcs.py @@ -33,8 +33,11 @@ class ProfilesFromAlignedSwcsParameters(ags.ArgSchema): ) -def main(): - module = ags.ArgSchemaParser(schema_type=ProfilesFromAlignedSwcsParameters) +def main(args = None): + if args == None: + module = ags.ArgSchemaParser(schema_type=ProfilesFromAlignedSwcsParameters) + else: + module = args # Load the specimen IDs specimen_id_file = module.args["specimen_id_file"] From 864b3e32c6c053ca8c766736b187368455111c33 Mon Sep 17 00:00:00 2001 From: emjoyce <=> Date: Fri, 14 Oct 2022 17:28:58 -0700 Subject: [PATCH 3/7] changes for hist tasks --- skeleton_keys/cmds/depth_profiles_from_aligned_swcs.py | 6 ++---- skeleton_keys/tasks.py | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/skeleton_keys/cmds/depth_profiles_from_aligned_swcs.py b/skeleton_keys/cmds/depth_profiles_from_aligned_swcs.py index db7ad4b..2f1d66d 100644 --- a/skeleton_keys/cmds/depth_profiles_from_aligned_swcs.py +++ b/skeleton_keys/cmds/depth_profiles_from_aligned_swcs.py @@ -33,11 +33,9 @@ class ProfilesFromAlignedSwcsParameters(ags.ArgSchema): ) -def main(args = None): - if args == None: +def main(module = None): + if module == None: module = ags.ArgSchemaParser(schema_type=ProfilesFromAlignedSwcsParameters) - else: - module = args # Load the specimen IDs specimen_id_file = module.args["specimen_id_file"] diff --git a/skeleton_keys/tasks.py b/skeleton_keys/tasks.py index f936448..63898fd 100644 --- a/skeleton_keys/tasks.py +++ b/skeleton_keys/tasks.py @@ -145,4 +145,4 @@ def create_layer_histograms( module = argschema.ArgSchemaParser( schema_type=ProfilesFromAlignedSwcsParameters, input_data=input_data, args=[] ) - depth_profile_main(module.args) \ No newline at end of file + depth_profile_main(module) \ No newline at end of file From ae1cfd720b1a0b7a4e1c027b3becdc7dada3c90f Mon Sep 17 00:00:00 2001 From: emjoyce <=> Date: Thu, 20 Oct 2022 11:30:39 -0700 Subject: [PATCH 4/7] separating histogram tasks into separate file --- skeleton_keys/hist_tasks.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 skeleton_keys/hist_tasks.py diff --git a/skeleton_keys/hist_tasks.py b/skeleton_keys/hist_tasks.py new file mode 100644 index 0000000..23fd40a --- /dev/null +++ b/skeleton_keys/hist_tasks.py @@ -0,0 +1,33 @@ +from skeleton_keys.cmds.depth_profiles_from_aligned_swcs import ( + ProfilesFromAlignedSwcsParameters, + main as depth_profile_main +) + +import argschema +from taskqueue import queueable + + +@queueable +def create_layer_histograms( + specimen_id_file, + swc_dir, + layer_depths_file, + output_hist_file, + output_soma_file, + bin_size=5.0, + below_wm=200.0,): + input_data = { + "specimen_id_file": specimen_id_file, + "swc_dir": swc_dir, + "layer_depths_file": layer_depths_file, + "output_hist_file": output_hist_file, + "output_soma_file": output_soma_file, + "bin_size": bin_size, + "below_wm": below_wm, + } + + input_data = {k: v for k, v in input_data.items() if v is not None} + module = argschema.ArgSchemaParser( + schema_type=ProfilesFromAlignedSwcsParameters, input_data=input_data, args=[] + ) + depth_profile_main(module) \ No newline at end of file From a7ea6e6c4356f6e8a75b91c23dac19040202b9af Mon Sep 17 00:00:00 2001 From: emjoyce <=> Date: Tue, 25 Oct 2022 16:43:54 -0700 Subject: [PATCH 5/7] adding mshr workaround for feature creation tasks --- Dockerfile | 7 +++--- skeleton_keys/hist_tasks.py | 47 ++++++++++++++++++++++++++++++++++++- task_worker.py | 2 +- 3 files changed, 50 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index 131ea0e..0dea9f8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,12 +1,11 @@ FROM continuumio/miniconda3:4.10.3 RUN apt-get update && apt-get install -y build-essential && rm -rf /var/lib/apt/lists/* RUN conda config --set channel_priority strict -RUN conda install -y -c conda-forge rtree==0.9.7 fenics==2019.1.0 python==3.8 mshr==2019.1.0 hdf5==1.10.6 h5py==2.10.0 Jinja2==2.11.3 +RUN conda install -y -c conda-forge rtree==0.9.7 fenics==2019.1.0 python==3.8 hdf5==1.10.6 h5py==2.10.0 Jinja2==2.11.3 RUN pip install git+https://github.com/AllenInstitute/AllenSDK.git -RUN pip install git+https://github.com/AllenInstitute/neuron_morphology@cloudfiles -ARG GITHUB_TOKEN +RUN pip install git+https://github.com/AllenInstitute/neuron_morphology@science_staging WORKDIR /usr/local/src -RUN git clone https://${GITHUB_TOKEN}@github.com/AllenInstitute/ccf_streamlines.git &&\ +RUN git clone https://github.com/AllenInstitute/ccf_streamlines.git &&\ pip install ./ccf_streamlines &&\ rm -rf /usr/local/src/ccf_streamlines WORKDIR /usr/local/src/skeleton_keys diff --git a/skeleton_keys/hist_tasks.py b/skeleton_keys/hist_tasks.py index 23fd40a..cf6cdaa 100644 --- a/skeleton_keys/hist_tasks.py +++ b/skeleton_keys/hist_tasks.py @@ -3,6 +3,11 @@ main as depth_profile_main ) +from skeleton_keys.cmds.process_morphology_features import ( + ProcessMorphologyFeaturesParameters, + main as morph_feat_main, +) + import argschema from taskqueue import queueable @@ -30,4 +35,44 @@ def create_layer_histograms( module = argschema.ArgSchemaParser( schema_type=ProfilesFromAlignedSwcsParameters, input_data=input_data, args=[] ) - depth_profile_main(module) \ No newline at end of file + depth_profile_main(module) + +@queueable +def extract_morphology_features( + specimen_id_file, + aligned_depth_profile_file, + aligned_soma_file, + output_file, + swc_dir=None, + swc_paths_file=None, + layer_list=None, + analyze_axon=False, + analyze_basal_dendrite=False, + analyze_apical_dendrite=False, + analyze_basal_dendrite_depth=False, + axon_depth_profile_loadings_file=None, + basal_dendrite_depth_profile_loadings_file=None, + apical_dendrite_depth_profile_loadings_file=None, +): + input_data = { + "specimen_id_file": specimen_id_file, + "aligned_depth_profile_file": aligned_depth_profile_file, + "aligned_soma_file": aligned_soma_file, + "swc_dir": swc_dir, + "swc_paths_file": swc_paths_file, + "layer_list": layer_list, + "analyze_axon": analyze_axon, + "analyze_basal_dendrite": analyze_basal_dendrite, + "analyze_apical_dendrite": analyze_apical_dendrite, + "analyze_basal_dendrite_depth": analyze_basal_dendrite_depth, + "axon_depth_profile_loadings_file": axon_depth_profile_loadings_file, + "basal_dendrite_depth_profile_loadings_file": basal_dendrite_depth_profile_loadings_file, + "apical_dendrite_depth_profile_loadings_file": apical_dendrite_depth_profile_loadings_file, + "output_file": output_file, + } + + input_data = {k: v for k, v in input_data.items() if v is not None} + module = argschema.ArgSchemaParser( + schema_type=ProcessMorphologyFeaturesParameters, input_data=input_data, args=[] + ) + morph_feat_main(module.args) \ No newline at end of file diff --git a/task_worker.py b/task_worker.py index 024eeac..f39fdb9 100644 --- a/task_worker.py +++ b/task_worker.py @@ -1,4 +1,4 @@ -from skeleton_keys.tasks import * +from skeleton_keys.hist_tasks import create_layer_histograms from taskqueue import TaskQueue import sys From 3206534a4b7bd89228caf4215df66d976c62182d Mon Sep 17 00:00:00 2001 From: emjoyce <=> Date: Thu, 27 Oct 2022 11:11:24 -0700 Subject: [PATCH 6/7] dockerfile updates for feature processing --- Dockerfile | 3 ++- .../cmds/process_morphology_features.py | 21 +++++++++++++++++-- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 0dea9f8..f2bd921 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ FROM continuumio/miniconda3:4.10.3 RUN apt-get update && apt-get install -y build-essential && rm -rf /var/lib/apt/lists/* RUN conda config --set channel_priority strict -RUN conda install -y -c conda-forge rtree==0.9.7 fenics==2019.1.0 python==3.8 hdf5==1.10.6 h5py==2.10.0 Jinja2==2.11.3 +RUN conda install -y -c conda-forge rtree==0.9.7 fenics==2019.1.0 python==3.9 gmsh==4.10.5 hdf5==1.10.6 h5py==2.10.0 Jinja2==2.11.3 RUN pip install git+https://github.com/AllenInstitute/AllenSDK.git RUN pip install git+https://github.com/AllenInstitute/neuron_morphology@science_staging WORKDIR /usr/local/src @@ -13,3 +13,4 @@ COPY setup.cfg /usr/local/src/skeleton_keys RUN python3 -c "import configparser; c = configparser.ConfigParser(); c.read('setup.cfg'); print(c['options']['install_requires'])" | xargs pip install COPY . /usr/local/src/skeleton_keys RUN python setup.py install + diff --git a/skeleton_keys/cmds/process_morphology_features.py b/skeleton_keys/cmds/process_morphology_features.py index 2b71856..16cdd80 100644 --- a/skeleton_keys/cmds/process_morphology_features.py +++ b/skeleton_keys/cmds/process_morphology_features.py @@ -1,7 +1,9 @@ import json +from re import A import argschema as ags import numpy as np import pandas as pd +import logging from multiprocessing import Pool from skeleton_keys.database_queries import ( swc_paths_from_database @@ -365,6 +367,11 @@ def main(args): if analyze_axon_flag: axon_depth_df = select_and_convert_depth_columns(depth_profile_df, "2_") available_ids = axon_depth_df.index.intersection(specimen_ids) + if len(available_ids) != len(specimen_ids): + missing_apical_num = len(specimen_ids) - len(available_ids) + logging.warning(f'{missing_apical_num} out of {specimen_ids} neurons passed do no have axons') + if len(available_ids) == 0: + raise Exception(f'None of the neurons passed have axon identified nodes (label 2, in columns beginning with 2_) in the depth profiles file at {aligned_depth_profile_file}') transformed = analyze_depth_profiles( axon_depth_df.loc[available_ids, :], args["axon_depth_profile_loadings_file"], @@ -383,14 +390,19 @@ def main(args): ) if analyze_apical_flag: - apical_depth_df = select_and_convert_depth_columns(depth_profile_df, "4_") + apical_depth_df = select_and_convert_depth_columns(depth_profile_df, "4_") available_ids = apical_depth_df.index.intersection(specimen_ids) + if len(available_ids) != len(specimen_ids): + missing_apical_num = len(specimen_ids) - len(available_ids) + logging.warning(f'{missing_apical_num} out of {specimen_ids} neurons do no have apicals') + if len(available_ids) == 0: + raise Exception(f'None of the neurons passed have apical identified nodes (label 4, in columns beginning with 4_) in the depth profiles file at {aligned_depth_profile_file}') transformed = analyze_depth_profiles( apical_depth_df.loc[available_ids, :], args["apical_dendrite_depth_profile_loadings_file"], args["save_apical_dendrite_depth_profile_loadings_file"], ) - for i, sp_id in enumerate(specimen_ids): + for i, sp_id in enumerate(available_ids): for j in range(transformed.shape[1]): depth_result.append( { @@ -410,6 +422,11 @@ def main(args): # other analyses if analyze_basal_dendrite_depth_flag: available_ids = basal_depth_df.index.intersection(specimen_ids) + if len(available_ids) != len(specimen_ids): + missing_apical_num = len(specimen_ids) - len(available_ids) + logging.warning(f'{missing_apical_num} out of {specimen_ids} neurons passed do no have basal-identified nodes') + if len(available_ids) == 0: + raise Exception(f'None of the neurons passed have basal identified nodes (label 3, in columns beginning with 3_) in the depth profiles file at {aligned_depth_profile_file}') transformed = analyze_depth_profiles( basal_depth_df.loc[available_ids, :], args["basal_dendrite_depth_profile_loadings_file"], From f071cc456fe6c488f6874eb7065d8d606c422e49 Mon Sep 17 00:00:00 2001 From: emjoyce <=> Date: Thu, 27 Oct 2022 11:22:24 -0700 Subject: [PATCH 7/7] updating axon feature extraction iteration --- skeleton_keys/cmds/process_morphology_features.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/skeleton_keys/cmds/process_morphology_features.py b/skeleton_keys/cmds/process_morphology_features.py index 16cdd80..163ba6b 100644 --- a/skeleton_keys/cmds/process_morphology_features.py +++ b/skeleton_keys/cmds/process_morphology_features.py @@ -377,7 +377,7 @@ def main(args): args["axon_depth_profile_loadings_file"], args["save_axon_depth_profile_loadings_file"], ) - for i, sp_id in enumerate(specimen_ids): + for i, sp_id in enumerate(available_ids): for j in range(transformed.shape[1]): depth_result.append( {