diff --git a/CHANGELOG.md b/CHANGELOG.md index 36c234e7..80c00d15 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ ## lifebit-ai/cloudos-cli: changelog +## v2.80.0 (2026-02-16) + +### Feat + +- Adds new parameter `--params-file` for `job run` that supports .json/.yaml/.yml extensions + ## v2.79.0 (2026-01-28) ### Patch diff --git a/cloudos_cli/_version.py b/cloudos_cli/_version.py index 3126b4f0..42294c8e 100644 --- a/cloudos_cli/_version.py +++ b/cloudos_cli/_version.py @@ -1 +1 @@ -__version__ = '2.79.0' +__version__ = '2.80.0' diff --git a/cloudos_cli/jobs/cli.py b/cloudos_cli/jobs/cli.py index b94bfd48..fd65b415 100644 --- a/cloudos_cli/jobs/cli.py +++ b/cloudos_cli/jobs/cli.py @@ -64,6 +64,10 @@ def job(): @click.option('--job-config', help=('A config file similar to a nextflow.config file, ' + 'but only with the parameters to use with your job.')) +@click.option('--params-file', + help=('A file containing the parameters to pass to the job call. ' + + 'It should be a .json or .yaml file with a dictionary structure ' + + 'where keys are parameter names and values are parameter values.')) @click.option('-p', '--parameter', multiple=True, @@ -199,6 +203,7 @@ def run(ctx, workflow_name, last, job_config, + params_file, parameter, git_commit, git_tag, @@ -400,6 +405,7 @@ def run(ctx, if workflow_type == 'nextflow': print(f'\tNextflow version: {nextflow_version}') j_id = j.send_job(job_config=job_config, + params_file=params_file, parameter=parameter, is_module=is_module, git_commit=git_commit, diff --git a/cloudos_cli/jobs/job.py b/cloudos_cli/jobs/job.py index 2ac729be..981ae63a 100644 --- a/cloudos_cli/jobs/job.py +++ b/cloudos_cli/jobs/job.py @@ -9,6 +9,7 @@ from cloudos_cli.utils.errors import BadRequestException from cloudos_cli.utils.requests import retry_requests_post, retry_requests_get, retry_requests_delete from pathlib import Path +from urllib.parse import urlparse import base64 from cloudos_cli.utils.array_job import classify_pattern, get_file_or_folder_id, extract_project import os @@ -174,8 +175,73 @@ def fetch_cloudos_id(self, else: raise ValueError(f'No {name} element in {resource} was found') + def build_parameters_file_payload(self, params_file): + """Build the parametersFile payload for a params file path.""" + if params_file is None: + return None + if isinstance(params_file, (list, tuple)): + if len(params_file) != 1: + raise ValueError('Please, provide a single file for --params-file.') + params_file = params_file[0] + + ext = os.path.splitext(params_file)[1].lower() + allowed_ext = {'.json', '.yaml', '.yml'} + if ext not in allowed_ext: + raise ValueError('Please, provide a .json or .yaml file for --params-file.') + + if params_file.startswith('s3://'): + parsed = urlparse(params_file) + bucket = parsed.netloc + s3_key = parsed.path.lstrip('/') + if not bucket or not s3_key: + raise ValueError('Invalid S3 URL. Please, provide a full s3://bucket/key path.') + name = s3_key.rstrip('/').split('/')[-1] + return { + "parametersFile": { + "dataItemEmbedded": { + "data": { + "name": name, + "s3BucketName": bucket, + "s3ObjectKey": s3_key + }, + "type": "S3File" + } + } + } + + if not self.project_name: + raise ValueError('Please, provide --project-name to resolve --params-file paths.') + + normalized_path = params_file.lstrip('/') + allowed_prefixes = ('Data', 'Analyses Results', 'Cohorts') + if not normalized_path.startswith(allowed_prefixes): + raise ValueError('Params file path must start with Data, Analyses Results, or Cohorts.') + + command_path = Path(normalized_path) + command_dir = str(command_path.parent) + command_name = command_path.name + file_id = get_file_or_folder_id( + self.cloudos_url, + self.apikey, + self.workspace_id, + self.project_name, + self.verify, + command_dir, + command_name, + is_file=True + ) + return { + "parametersFile": { + "dataItem": { + "kind": "File", + "item": f"{file_id}" + } + } + } + def convert_nextflow_to_json(self, job_config, + params_file, parameter, array_parameter, array_file_header, @@ -217,6 +283,8 @@ def convert_nextflow_to_json(self, ---------- job_config : string Path to a nextflow.config file with parameters scope. + params_file : string + S3 or File Explorer path to a JSON/YAML file with Nextflow parameters. parameter : tuple Tuple of strings indicating the parameters to pass to the pipeline call. They are in the following form: ('param1=param1val', 'param2=param2val', ...) @@ -424,6 +492,9 @@ def convert_nextflow_to_json(self, "usesFusionFileSystem": use_mountpoints, "accelerateSavingResults": accelerate_saving_results } + params_file_payload = self.build_parameters_file_payload(params_file) + if params_file_payload is not None: + params.update(params_file_payload) if workflow_type != 'docker': params["nextflowVersion"] = nextflow_version if execution_platform != 'hpc': @@ -473,6 +544,7 @@ def convert_nextflow_to_json(self, def send_job(self, job_config=None, + params_file=None, parameter=(), array_parameter=(), array_file_header=None, @@ -513,6 +585,8 @@ def send_job(self, ---------- job_config : string Path to a nextflow.config file with parameters scope. + params_file : string + S3 or File Explorer path to a JSON/YAML file with Nextflow parameters. parameter : tuple Tuple of strings indicating the parameters to pass to the pipeline call. They are in the following form: ('param1=param1val', 'param2=param2val', ...) @@ -609,6 +683,7 @@ def send_job(self, "apikey": apikey } params = self.convert_nextflow_to_json(job_config, + params_file, parameter, array_parameter, array_file_header, diff --git a/tests/test_bash/test_get_array_file_cols.py b/tests/test_bash/test_get_array_file_cols.py index c4738c46..a2373d35 100644 --- a/tests/test_bash/test_get_array_file_cols.py +++ b/tests/test_bash/test_get_array_file_cols.py @@ -100,7 +100,7 @@ def test_get_array_file_columns(): s3_object_key_b64 = base64.b64encode(s3_object_key.encode()).decode() break else: - raise ValueError(f'File "{file_name}" not found in the "Data" folder of the project "{project_name}".') + raise ValueError(f'File "{file_name}" not found in the "Data" folder of the project "{PROJECT_NAME}".') url = ( f"{CLOUDOS_URL}/api/v1/jobs/array-file/metadata" diff --git a/tests/test_jobs/test_convert_nextflow_to_json.py b/tests/test_jobs/test_convert_nextflow_to_json.py index 1789e6b0..dfd8c7c2 100644 --- a/tests/test_jobs/test_convert_nextflow_to_json.py +++ b/tests/test_jobs/test_convert_nextflow_to_json.py @@ -8,6 +8,7 @@ param_dict = { "config": "cloudos_cli/examples/rnatoy.config", "parameter": (), + "params_file": None, "array_parameter": (), "array_file_header": None, "is_module": False, @@ -46,8 +47,19 @@ def test_convert_nextflow_to_json_output_correct(): - job_json = Job.convert_nextflow_to_json( - 1, param_dict["config"], + job = Job( + "https://cloudos.example", + "test_api_key", + None, + "workspace_id", + "project", + "workflow", + project_id=param_dict["project_id"], + workflow_id=param_dict["workflow_id"] + ) + job_json = job.convert_nextflow_to_json( + param_dict["config"], + param_dict["params_file"], parameter=param_dict["parameter"], array_parameter=param_dict["array_parameter"], array_file_header=param_dict["array_file_header"], @@ -92,8 +104,19 @@ def test_convert_nextflow_to_json_output_correct(): def test_convert_nextflow_to_json_badly_formed_config(): no_equals_config = "tests/test_data/wrong_params.config" with pytest.raises(ValueError) as excinfo: - Job.convert_nextflow_to_json( - 1, no_equals_config, + job = Job( + "https://cloudos.example", + "test_api_key", + None, + "workspace_id", + "project", + "workflow", + project_id=param_dict["project_id"], + workflow_id=param_dict["workflow_id"] + ) + job.convert_nextflow_to_json( + no_equals_config, + param_dict["params_file"], parameter=param_dict["parameter"], array_parameter=param_dict["array_parameter"], array_file_header=param_dict["array_file_header"], @@ -135,3 +158,59 @@ def test_convert_nextflow_to_json_badly_formed_config(): tests/test_data/wrong_params.config\ using the \'=\' as spacer.\ E.g: name = my_name".replace(" ", "") in str(excinfo.value) + + +def test_params_file_payload_s3(): + job = Job( + "https://cloudos.example", + "test_api_key", + None, + "workspace_id", + "project", + "workflow", + project_id=param_dict["project_id"], + workflow_id=param_dict["workflow_id"] + ) + payload = job.build_parameters_file_payload("s3://my-bucket/path/to/params.json") + assert payload == { + "parametersFile": { + "dataItemEmbedded": { + "data": { + "name": "params.json", + "s3BucketName": "my-bucket", + "s3ObjectKey": "path/to/params.json" + }, + "type": "S3File" + } + } + } + + +def test_params_file_payload_file_explorer(monkeypatch): + job = Job( + "https://cloudos.example", + "test_api_key", + None, + "workspace_id", + "project", + "workflow", + project_id=param_dict["project_id"], + workflow_id=param_dict["workflow_id"] + ) + + def fake_get_file_or_folder_id(*args, **kwargs): + return "file-123" + + monkeypatch.setattr( + "cloudos_cli.jobs.job.get_file_or_folder_id", + fake_get_file_or_folder_id + ) + payload = job.build_parameters_file_payload("Data/params-files/run160226.json") + assert payload == { + "parametersFile": { + "dataItem": { + "kind": "File", + "item": "file-123" + } + } + }