issue with bluepyopt_build function

[danilobenozzo]

Exception in thread Thread-1:
Traceback (most recent call last):
File "/usr/lib/python3.9/threading.py", line 954, in _bootstrap_inner
self.run()
File "/usr/lib/python3.9/threading.py", line 892, in run
self._target(*self._args, **self._kwargs)
File "/home/benozzo/dbbs_lab/test_opt/venv/lib/python3.9/site-packages/pebble/pool/process.py", line 169, in task_scheduler_loop
pool_manager.schedule(task)
File "/home/benozzo/dbbs_lab/test_opt/venv/lib/python3.9/site-packages/pebble/pool/process.py", line 218, in schedule
self.worker_manager.dispatch(task)
File "/home/benozzo/dbbs_lab/test_opt/venv/lib/python3.9/site-packages/pebble/pool/process.py", line 351, in dispatch
self.pool_channel.send(WorkerTask(task.id, task.payload))
File "/home/benozzo/dbbs_lab/test_opt/venv/lib/python3.9/site-packages/pebble/pool/channel.py", line 70, in send
return self.writer.send(obj)
File "/usr/lib/python3.9/multiprocessing/connection.py", line 211, in send
self._send_bytes(_ForkingPickler.dumps(obj))
File "/usr/lib/python3.9/multiprocessing/reduction.py", line 51, in dumps
cls(buf, protocol).dump(obj)
AttributeError: Can't pickle local object 'bluepyopt_build..ArborizeMorphology'

when running the optimization
moving back to what was in /origin/feature/optimization works

[Helveg]

Can you post the code that led to this error, so I can reproduce it?

[danilobenozzo]

import logging

import bluepyopt as bpop
import bluepyopt.ephys as ephys
#import dbbs_models
from bsb.morphologies import parse_morphology_file

from arborize import define_constraints
#from arborize.optimizers._bluepyopt import get_bpo_cell
from arborize.builders._bluepyopt import bluepyopt_build
from arborize.schematics import bsb_schematic


logging.getLogger().setLevel(logging.DEBUG)


def create_schema(constraints, tags, file_morphology):
    _m = parse_morphology_file(file_morphology, tags=tags)
    return bsb_schematic(_m, constraints)


schema = create_schema(
    define_constraints(
        {
            "synapse_types": {
                ("AMPA", "granule"): {
                    "tau_facil": 5,
                    "tau_rec": 8,
                    "tau_1": 1,
                    "gmax": 140000,
                    "U": 0.43,
                },
                ("NMDA", "granule"): {
                    "tau_facil": 5,
                    "tau_rec": 8,
                    "tau_1": 1,
                    "gmax": 23500,
                    "U": 0.43,
                },
                ("GABA", "granule"): {"U": 0.35},
            },
            "cable_types": {
                "soma": {
                    "cable": {"Ra": 100, "cm": 2},
                    "ions": {"k": {"rev_pot": -80.993}, "ca": {"rev_pot": 137.5}},
                    "mechanisms": {
                        "Leak": {
                            "e": -60,
                            "gmax": [0.00029038073716, 0.00029038073717],
                        },
                        "Kv3_4": {"gkbar": 0.00076192450952},
                        "Kv4_3": {"gkbar": 0.00281496839061},
                        "Kir2_3": {"gkbar": 0.00074725514702},
                        "Ca": {"gcabar": 0.00060938071784},
                        "Kv1_1": {"gbar": 0.00569738264555},
                        "Kv1_5": {"gKur": 0.00083407556714},
                        "Kv2_2": {"gKv2_2bar": 1.203410852e-05},
                        ("cdp5", "CR"): {},
                    },
                },
                "dendrites": {
                    "cable": {"Ra": 100, "cm": 2.5},
                    "ions": {"k": {"rev_pot": -80.993}, "ca": {"rev_pot": 137.5}},
                    "mechanisms": {
                        "Leak": {"e": -60, "gmax": 0.00025029700737},
                        ("Leak", "GABA"): {},
                        "Ca": {"gcabar": 0.00500128008459},
                        "Kca1_1": {"gbar": 0.01001807454651},
                        "Kv1_1": {"gbar": 0.00381819207934},
                        ("cdp5", "CR"): {},
                    },
                },
                "axon": {"cable": {}, "ions": {}, "mechanisms": {}},
                "ascending_axon": {
                    "cable": {"Ra": 100, "cm": 1},
                    "ions": {
                        "na": {"rev_pot": 87.39},
                        "k": {"rev_pot": -80.993},
                        "ca": {"rev_pot": 137.5},
                    },
                    "mechanisms": {
                        ("Na", "granule_cell"): {"gnabar": 0.02630163681502},
                        "Kv3_4": {"gkbar": 0.00237386061632},
                        "Leak": {"e": -60, "gmax": 9.364092125e-05},
                        "Ca": {"gcabar": 0.00068197420273},
                        ("cdp5", "CR"): {},
                    },
                },
                "parallel_fiber": {
                    "cable": {"Ra": 100, "cm": 1},
                    "ions": {
                        "na": {"rev_pot": 87.39},
                        "k": {"rev_pot": -80.993},
                        "ca": {"rev_pot": 137.5},
                    },
                    "mechanisms": {
                        ("Na", "granule_cell"): {"gnabar": 0.01771848449261},
                        "Kv3_4": {"gkbar": 0.00817568047037},
                        "Leak": {"e": -60, "gmax": 3.5301616e-07},
                        "Ca": {"gcabar": 0.0002085683353},
                        ("cdp5", "CR"): {},
                    },
                },
                "axon_initial_segment": {
                    "cable": {"Ra": 100, "cm": 1},
                    "ions": {
                        "na": {"rev_pot": 87.39},
                        "k": {"rev_pot": -80.993},
                        "ca": {"rev_pot": 137.5},
                    },
                    "mechanisms": {
                        ("Na", "granule_cell_FHF"): {"gnabar": 1.28725006737226},
                        "Kv3_4": {"gkbar": 0.00649595340654},
                        "Leak": {"e": -60, "gmax": 0.00029276697557},
                        "Ca": {"gcabar": 0.00031198539472},
                        "Km": {"gkbar": 0.00056671971737},
                        ("cdp5", "CR"): {},
                    },
                },
                "axon_hillock": {
                    "cable": {"Ra": 100, "cm": 2},
                    "ions": {
                        "na": {"rev_pot": 87.39},
                        "k": {"rev_pot": -80.993},
                        "ca": {"rev_pot": 137.5},
                    },
                    "mechanisms": {
                        "Leak": {"e": -60, "gmax": 0.0003695818972},
                        ("Na", "granule_cell_FHF"): {"gnabar": 0.00928805851462},
                        "Kv3_4": {"gkbar": 0.02037346310915},
                        "Ca": {"gcabar": 0.00057726155447},
                        ("cdp5", "CR"): {},
                    },
                },
            },
        },
        use_defaults=True,
    ),
    {
        16: ["axon", "axon_hillock"],
        17: ["axon", "axon_initial_segment"],
        18: ["axon", "ascending_axon"],
        19: ["axon", "parallel_fiber"],
    },
    r"/home/benozzo/dbbs_lab/repos/models/dbbs_models/morphologies/GranuleCell.swc",
)

#cell = get_bpo_cell(schema)
cell = bluepyopt_build(schema)

soma_loc = ephys.locations.NrnSeclistCompLocation(
    name="soma", seclist_name="soma", sec_index=0, comp_x=0.5
)

NUMBER_INDIVIDUALS = 1  # Number of individuals in offspring
NUMBER_GENERATIONS = 1  # Maximum number of generations

sweep_protocols = []
for protocol_name, amplitude in [
    # ("step1", 0.01),
    ("step2", 0.016),
    # ("step3", 0.022),
]:
    stim = ephys.stimuli.NrnSquarePulse(
        step_amplitude=amplitude,
        step_delay=100,
        step_duration=100,
        location=soma_loc,
        total_duration=200,
    )
    rec = ephys.recordings.CompRecording(
        name="%s.soma.v" % protocol_name, location=soma_loc, variable="v"
    )
    protocol = ephys.protocols.SweepProtocol(protocol_name, [stim], [rec])
    sweep_protocols.append(protocol)
threestep_protocol = ephys.protocols.SequenceProtocol(
    "twostep", protocols=sweep_protocols
)

# NEURON sim
nrn = ephys.simulators.NrnSimulator(dt=0.025, cvode_active=False)

# feature of obj function
efel_feature_means = {
    "step1": {
        "AP_height": 20.93,
        "ISI_CV": 0.261,
        "AHP_depth_abs_slow": -52.69,
        "AP_width": 0.665,
        "voltage_base": -68.5,
        "AHP_depth_abs": -59.21,
        "time_to_first_spike": 31.9,
        "adaptation_index2": 0.1062,
        "mean_frequency": 25,
    },
    "step2": {
        "AP_height": 19.255,
        "ISI_CV": 0.14,
        "AHP_depth_abs_slow": -48.935,
        "AP_width": 0.695,
        "voltage_base": -68.77,
        "AHP_depth_abs": -58.3,
        "time_to_first_spike": 19.0,
        "adaptation_index2": 0.034,
        "mean_frequency": 40,
    },
    "step3": {
        "AP_height": 17.645,
        "ISI_CV": 0.148,
        "AHP_depth_abs_slow": -32.67,
        "AP_width": 0.7135,
        "voltage_base": -69.125,
        "AHP_depth_abs": -57.191,
        "time_to_first_spike": 14.65,
        "adaptation_index2": 0.029,
        "mean_frequency": 50,
    },
}

# obj function
objectives = []
for protocol in sweep_protocols:
    stim_start = protocol.stimuli[0].step_delay
    stim_end = stim_start + protocol.stimuli[0].step_duration
    for efel_feature_name, mean in efel_feature_means[protocol.name].items():
        feature_name = "%s.%s" % (protocol.name, efel_feature_name)
        feature = ephys.efeatures.eFELFeature(
            feature_name,
            efel_feature_name=efel_feature_name,
            recording_names={"": "%s.soma.v" % protocol.name},
            stim_start=stim_start,
            stim_end=stim_end,
            exp_mean=mean,
            exp_std=0.1 * mean,
        )
        objective = ephys.objectives.SingletonObjective(feature_name, feature)
        objectives.append(objective)
score_calc = ephys.objectivescalculators.ObjectivesCalculator(objectives)

print([(p.name, p.frozen) for p in cell.params.values()])
# cell evaluator
cell_evaluator = ephys.evaluators.CellEvaluator(
    cell_model=cell,
    param_names=[p.name for p in cell.params.values() if not p.frozen],
    fitness_protocols={threestep_protocol.name: threestep_protocol},
    fitness_calculator=score_calc,
    sim=nrn,
)

# opt
optimisation = bpop.optimisations.DEAPOptimisation(
    evaluator=cell_evaluator,
    offspring_size=NUMBER_INDIVIDUALS,
    seed=333,
)

print("run opt")
final_pop, hall_of_fame, logs, hist = optimisation.run(max_ngen=NUMBER_GENERATIONS)

# best individual
best_ind = hall_of_fame[0]
print("Best individual: ", best_ind)
print("Fitness values: ", best_ind.fitness.values)

best_ind_dict = cell_evaluator.param_dict(best_ind)
print("Ind dict", best_ind_dict)

# run protocol
responses = threestep_protocol.run(
    cell_model=cell,
    param_values={"gmax_glia__dbbs__Leak__0_soma": 0.00029038073716554835},
    sim=nrn,
    isolate=False,
)

import plotly.graph_objs as go

go.Figure(
    [
        go.Scatter(x=resp["time"], y=resp["voltage"], name=name)
        for name, resp in responses.items()
    ]
).write_html("gpop.html")

[Helveg]

Please try again on latest master, or pip install arborize==4.0.0b6. Reopen this issue if the problem persists :)

[danilobenozzo]

Traceback (most recent call last):
File "/home/benozzo/dbbs_lab/test_opt/venv/lib/python3.9/site-packages/scoop/_control.py", line 150, in runFuture
future.resultValue = future.callable(*future.args, **future.kargs)
File "/usr/lib/python3.9/runpy.py", line 268, in run_path
return _run_module_code(code, init_globals, run_name,
File "/usr/lib/python3.9/runpy.py", line 97, in _run_module_code
_run_code(code, mod_globals, init_globals,
File "/usr/lib/python3.9/runpy.py", line 87, in _run_code
exec(code, run_globals)
File "granule_BluePyOpt.py", line 256, in
final_pop, hall_of_fame, logs, hist = optimisation.run(max_ngen=NUMBER_GENERATIONS)
File "/home/benozzo/dbbs_lab/test_opt/venv/lib/python3.9/site-packages/bluepyopt/deapext/optimisations.py", line 309, in run
pop, hof, log, history = algorithms.eaAlphaMuPlusLambdaCheckpoint(
File "/home/benozzo/dbbs_lab/test_opt/venv/lib/python3.9/site-packages/bluepyopt/deapext/algorithms.py", line 154, in eaAlphaMuPlusLambdaCheckpoint
invalid_count = _evaluate_invalid_fitness(toolbox, population)
File "/home/benozzo/dbbs_lab/test_opt/venv/lib/python3.9/site-packages/bluepyopt/deapext/algorithms.py", line 61, in _evaluate_invalid_fitness
fitnesses = toolbox.map(toolbox.evaluate, invalid_ind)
File "/home/benozzo/dbbs_lab/test_opt/venv/lib/python3.9/site-packages/scoop/fallbacks.py", line 50, in wrapper
return func(*args, **kwargs)
File "/home/benozzo/dbbs_lab/test_opt/venv/lib/python3.9/site-packages/scoop/futures.py", line 135, in map
futures = _mapFuture(func, *iterables)
File "/home/benozzo/dbbs_lab/test_opt/venv/lib/python3.9/site-packages/scoop/futures.py", line 100, in mapFuture
childrenList.append(submit(callable, *args))
File "/home/benozzo/dbbs_lab/test_opt/venv/lib/python3.9/site-packages/scoop/fallbacks.py", line 66, in wrapper
return func(*args, **kwargs)
File "/home/benozzo/dbbs_lab/test_opt/venv/lib/python3.9/site-packages/scoop/futures.py", line 310, in submit
control.execQueue.append(child)
File "/home/benozzo/dbbs_lab/test_opt/venv/lib/python3.9/site-packages/scoop/_types.py", line 316, in append
self.socket.sendFuture(sending_future)
File "/home/benozzo/dbbs_lab/test_opt/venv/lib/python3.9/site-packages/scoop/_comm/scoopzmq.py", line 369, in sendFuture
pickle.dumps(future, pickle.HIGHEST_PROTOCOL),
AttributeError: Can't pickle local object 'bluepyopt_build..ArborizeMorphology'

the issue was solved when running the code above which I used for testing that has only 1 generation and 1 individual,
but now it happens again with NUMBER_GENERATIONS > 1
to reproduce:

import logging

import bluepyopt as bpop
import bluepyopt.ephys as ephys
#import dbbs_models
from bsb.morphologies import parse_morphology_file

from arborize import define_constraints
#from arborize.optimizers._bluepyopt import get_bpo_cell
from arborize.builders._bluepyopt import bluepyopt_build
from arborize.schematics import bsb_schematic


logging.getLogger().setLevel(logging.DEBUG)


def create_schema(constraints, tags, file_morphology):
    _m = parse_morphology_file(file_morphology, tags=tags)
    return bsb_schematic(_m, constraints)


schema = create_schema(
    define_constraints(
        {
            "synapse_types": {
                ("AMPA", "granule"): {
                    "tau_facil": 5,
                    "tau_rec": 8,
                    "tau_1": 1,
                    "gmax": 140000,
                    "U": 0.43,
                },
                ("NMDA", "granule"): {
                    "tau_facil": 5,
                    "tau_rec": 8,
                    "tau_1": 1,
                    "gmax": 23500,
                    "U": 0.43,
                },
                ("GABA", "granule"): {"U": 0.35},
            },
            "cable_types": {
                "soma": {
                    "cable": {"Ra": 100, "cm": 2},
                    "ions": {"k": {"rev_pot": -80.993}, "ca": {"rev_pot": 137.5}},
                    "mechanisms": {
                        "Leak": {
                            "e": -60,
                            "gmax": [0.00029038073716, 0.00029038073717],
                        },
                        "Kv3_4": {"gkbar": 0.00076192450952},
                        "Kv4_3": {"gkbar": 0.00281496839061},
                        "Kir2_3": {"gkbar": 0.00074725514702},
                        "Ca": {"gcabar": 0.00060938071784},
                        "Kv1_1": {"gbar": 0.00569738264555},
                        "Kv1_5": {"gKur": 0.00083407556714},
                        "Kv2_2": {"gKv2_2bar": 1.203410852e-05},
                        ("cdp5", "CR"): {},
                    },
                },
                "dendrites": {
                    "cable": {"Ra": 100, "cm": 2.5},
                    "ions": {"k": {"rev_pot": -80.993}, "ca": {"rev_pot": 137.5}},
                    "mechanisms": {
                        "Leak": {"e": -60, "gmax": 0.00025029700737},
                        ("Leak", "GABA"): {},
                        "Ca": {"gcabar": 0.00500128008459},
                        "Kca1_1": {"gbar": 0.01001807454651},
                        "Kv1_1": {"gbar": 0.00381819207934},
                        ("cdp5", "CR"): {},
                    },
                },
                "axon": {"cable": {}, "ions": {}, "mechanisms": {}},
                "ascending_axon": {
                    "cable": {"Ra": 100, "cm": 1},
                    "ions": {
                        "na": {"rev_pot": 87.39},
                        "k": {"rev_pot": -80.993},
                        "ca": {"rev_pot": 137.5},
                    },
                    "mechanisms": {
                        ("Na", "granule_cell"): {"gnabar": 0.02630163681502},
                        "Kv3_4": {"gkbar": 0.00237386061632},
                        "Leak": {"e": -60, "gmax": 9.364092125e-05},
                        "Ca": {"gcabar": 0.00068197420273},
                        ("cdp5", "CR"): {},
                    },
                },
                "parallel_fiber": {
                    "cable": {"Ra": 100, "cm": 1},
                    "ions": {
                        "na": {"rev_pot": 87.39},
                        "k": {"rev_pot": -80.993},
                        "ca": {"rev_pot": 137.5},
                    },
                    "mechanisms": {
                        ("Na", "granule_cell"): {"gnabar": 0.01771848449261},
                        "Kv3_4": {"gkbar": 0.00817568047037},
                        "Leak": {"e": -60, "gmax": 3.5301616e-07},
                        "Ca": {"gcabar": 0.0002085683353},
                        ("cdp5", "CR"): {},
                    },
                },
                "axon_initial_segment": {
                    "cable": {"Ra": 100, "cm": 1},
                    "ions": {
                        "na": {"rev_pot": 87.39},
                        "k": {"rev_pot": -80.993},
                        "ca": {"rev_pot": 137.5},
                    },
                    "mechanisms": {
                        ("Na", "granule_cell_FHF"): {"gnabar": 1.28725006737226},
                        "Kv3_4": {"gkbar": 0.00649595340654},
                        "Leak": {"e": -60, "gmax": 0.00029276697557},
                        "Ca": {"gcabar": 0.00031198539472},
                        "Km": {"gkbar": 0.00056671971737},
                        ("cdp5", "CR"): {},
                    },
                },
                "axon_hillock": {
                    "cable": {"Ra": 100, "cm": 2},
                    "ions": {
                        "na": {"rev_pot": 87.39},
                        "k": {"rev_pot": -80.993},
                        "ca": {"rev_pot": 137.5},
                    },
                    "mechanisms": {
                        "Leak": {"e": -60, "gmax": 0.0003695818972},
                        ("Na", "granule_cell_FHF"): {"gnabar": 0.00928805851462},
                        "Kv3_4": {"gkbar": 0.02037346310915},
                        "Ca": {"gcabar": 0.00057726155447},
                        ("cdp5", "CR"): {},
                    },
                },
            },
        },
        use_defaults=True,
    ),
    {
        16: ["axon", "axon_hillock"],
        17: ["axon", "axon_initial_segment"],
        18: ["axon", "ascending_axon"],
        19: ["axon", "parallel_fiber"],
    },
    r"/home/benozzo/dbbs_lab/repos/models/dbbs_models/morphologies/GranuleCell.swc",
)

#cell = get_bpo_cell(schema)
cell = bluepyopt_build(schema)

soma_loc = ephys.locations.NrnSeclistCompLocation(
    name="soma", seclist_name="soma", sec_index=0, comp_x=0.5
)

NUMBER_INDIVIDUALS = 2  # Number of individuals in offspring
NUMBER_GENERATIONS = 1  # Maximum number of generations

sweep_protocols = []
for protocol_name, amplitude in [
    # ("step1", 0.01),
    ("step2", 0.016),
    # ("step3", 0.022),
]:
    stim = ephys.stimuli.NrnSquarePulse(
        step_amplitude=amplitude,
        step_delay=100,
        step_duration=100,
        location=soma_loc,
        total_duration=200,
    )
    rec = ephys.recordings.CompRecording(
        name="%s.soma.v" % protocol_name, location=soma_loc, variable="v"
    )
    protocol = ephys.protocols.SweepProtocol(protocol_name, [stim], [rec])
    sweep_protocols.append(protocol)
threestep_protocol = ephys.protocols.SequenceProtocol(
    "twostep", protocols=sweep_protocols
)

# NEURON sim
nrn = ephys.simulators.NrnSimulator(dt=0.025, cvode_active=False)

# feature of obj function
efel_feature_means = {
    "step1": {
        "AP_height": 20.93,
        "ISI_CV": 0.261,
        "AHP_depth_abs_slow": -52.69,
        "AP_width": 0.665,
        "voltage_base": -68.5,
        "AHP_depth_abs": -59.21,
        "time_to_first_spike": 31.9,
        "adaptation_index2": 0.1062,
        "mean_frequency": 25,
    },
    "step2": {
        "AP_height": 19.255,
        "ISI_CV": 0.14,
        "AHP_depth_abs_slow": -48.935,
        "AP_width": 0.695,
        "voltage_base": -68.77,
        "AHP_depth_abs": -58.3,
        "time_to_first_spike": 19.0,
        "adaptation_index2": 0.034,
        "mean_frequency": 40,
    },
    "step3": {
        "AP_height": 17.645,
        "ISI_CV": 0.148,
        "AHP_depth_abs_slow": -32.67,
        "AP_width": 0.7135,
        "voltage_base": -69.125,
        "AHP_depth_abs": -57.191,
        "time_to_first_spike": 14.65,
        "adaptation_index2": 0.029,
        "mean_frequency": 50,
    },
}

# obj function
objectives = []
for protocol in sweep_protocols:
    stim_start = protocol.stimuli[0].step_delay
    stim_end = stim_start + protocol.stimuli[0].step_duration
    for efel_feature_name, mean in efel_feature_means[protocol.name].items():
        feature_name = "%s.%s" % (protocol.name, efel_feature_name)
        feature = ephys.efeatures.eFELFeature(
            feature_name,
            efel_feature_name=efel_feature_name,
            recording_names={"": "%s.soma.v" % protocol.name},
            stim_start=stim_start,
            stim_end=stim_end,
            exp_mean=mean,
            exp_std=0.1 * mean,
        )
        objective = ephys.objectives.SingletonObjective(feature_name, feature)
        objectives.append(objective)
score_calc = ephys.objectivescalculators.ObjectivesCalculator(objectives)

print([(p.name, p.frozen) for p in cell.params.values()])
# cell evaluator
cell_evaluator = ephys.evaluators.CellEvaluator(
    cell_model=cell,
    param_names=[p.name for p in cell.params.values() if not p.frozen],
    fitness_protocols={threestep_protocol.name: threestep_protocol},
    fitness_calculator=score_calc,
    sim=nrn,
)

# opt
optimisation = bpop.optimisations.DEAPOptimisation(
    evaluator=cell_evaluator,
    offspring_size=NUMBER_INDIVIDUALS,
    seed=333,
)

print("run opt")
final_pop, hall_of_fame, logs, hist = optimisation.run(max_ngen=NUMBER_GENERATIONS)

# best individual
best_ind = hall_of_fame[0]
print("Best individual: ", best_ind)
print("Fitness values: ", best_ind.fitness.values)

best_ind_dict = cell_evaluator.param_dict(best_ind)
print("Ind dict", best_ind_dict)

# run protocol
responses = threestep_protocol.run(
    cell_model=cell,
    param_values={"gmax_glia__dbbs__Leak__0_soma": 0.00029038073716554835},
    sim=nrn,
    isolate=False,
)

import plotly.graph_objs as go

go.Figure(
    [
        go.Scatter(x=resp["time"], y=resp["voltage"], name=name)
        for name, resp in responses.items()
    ]
).write_html("gpop.html")