From 896c380170fb573f6dd16c419cc6134a869291ad Mon Sep 17 00:00:00 2001 From: M Bernt Date: Sat, 6 Jul 2019 14:47:51 +0200 Subject: [PATCH 001/121] test data: initial commit --- test-data/empty.ctd | 12 ++++ test-data/empty.xml | 25 +++++++ test-data/float.ctd | 47 +++++++++++++ test-data/float.xml | 158 ++++++++++++++++++++++++++++++++++++++++++ test-data/integer.ctd | 47 +++++++++++++ test-data/integer.xml | 158 ++++++++++++++++++++++++++++++++++++++++++ test-data/macros.xml | 31 +++++++++ test-data/select.ctd | 20 ++++++ test-data/select.xml | 134 +++++++++++++++++++++++++++++++++++ test-data/string.ctd | 20 ++++++ test-data/string.xml | 118 +++++++++++++++++++++++++++++++ 11 files changed, 770 insertions(+) create mode 100644 test-data/empty.ctd create mode 100644 test-data/empty.xml create mode 100644 test-data/float.ctd create mode 100644 test-data/float.xml create mode 100644 test-data/integer.ctd create mode 100644 test-data/integer.xml create mode 100644 test-data/macros.xml create mode 100644 test-data/select.ctd create mode 100644 test-data/select.xml create mode 100644 test-data/string.ctd create mode 100644 test-data/string.xml diff --git a/test-data/empty.ctd b/test-data/empty.ctd new file mode 100644 index 00000000..316502e3 --- /dev/null +++ b/test-data/empty.ctd @@ -0,0 +1,12 @@ + + + + + + + + + + + + diff --git a/test-data/empty.xml b/test-data/empty.xml new file mode 100644 index 00000000..07e534e7 --- /dev/null +++ b/test-data/empty.xml @@ -0,0 +1,25 @@ + + + + + String parameter tests + + echo StringTest + macros.xml + + + + + <![CDATA[echo StringTest + +> $param_stdout +]]> + + + + + <![CDATA[String parameter tests. + + +For more information, visit http://docs.are/unavailable.html]]> + diff --git a/test-data/float.ctd b/test-data/float.ctd new file mode 100644 index 00000000..2ab6c54d --- /dev/null +++ b/test-data/float.ctd @@ -0,0 +1,47 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/test-data/float.xml b/test-data/float.xml new file mode 100644 index 00000000..f4fa7405 --- /dev/null +++ b/test-data/float.xml @@ -0,0 +1,158 @@ + + + + + Floateger parameter tests + + echo FloatTest + macros.xml + + + + + <![CDATA[echo FloatTest + +#if str($param_param_wodefault_mandatory_unrestricted): + -param_wodefault_mandatory_unrestricted $param_param_wodefault_mandatory_unrestricted +#end if +#if str($param_param_default_mandatory_unrestricted): + -param_default_mandatory_unrestricted $param_param_default_mandatory_unrestricted +#end if +#if str($param_param_wodefault_optional_unrestricted): + -param_wodefault_optional_unrestricted $param_param_wodefault_optional_unrestricted +#end if +#if str($param_param_default_optional_unrestricted): + -param_default_optional_unrestricted $param_param_default_optional_unrestricted +#end if +#if str($param_param_wodefault_mandatory_unrestricted_min): + -param_wodefault_mandatory_unrestricted_min $param_param_wodefault_mandatory_unrestricted_min +#end if +#if str($param_param_default_mandatory_unrestricted_min): + -param_default_mandatory_unrestricted_min $param_param_default_mandatory_unrestricted_min +#end if +#if str($param_param_wodefault_optional_unrestricted_min): + -param_wodefault_optional_unrestricted_min $param_param_wodefault_optional_unrestricted_min +#end if +#if str($param_param_default_optional_unrestricted_min): + -param_default_optional_unrestricted_min $param_param_default_optional_unrestricted_min +#end if +#if str($param_param_wodefault_mandatory_unrestricted_max): + -param_wodefault_mandatory_unrestricted_max $param_param_wodefault_mandatory_unrestricted_max +#end if +#if str($param_param_default_mandatory_unrestricted_max): + -param_default_mandatory_unrestricted_max $param_param_default_mandatory_unrestricted_max +#end if +#if str($param_param_wodefault_optional_unrestricted_max): + -param_wodefault_optional_unrestricted_max $param_param_wodefault_optional_unrestricted_max +#end if +#if str($param_param_default_optional_unrestricted_max): + -param_default_optional_unrestricted_max $param_param_default_optional_unrestricted_max +#end if +#if str($param_param_wodefault_mandatory_unrestricted_minmax): + -param_wodefault_mandatory_unrestricted_minmax $param_param_wodefault_mandatory_unrestricted_minmax +#end if +#if str($param_param_default_mandatory_unrestricted_minmax): + -param_default_mandatory_unrestricted_minmax $param_param_default_mandatory_unrestricted_minmax +#end if +#if str($param_param_wodefault_optional_unrestricted_minmax): + -param_wodefault_optional_unrestricted_minmax $param_param_wodefault_optional_unrestricted_minmax +#end if +#if str($param_param_default_optional_unrestricted_minmax): + -param_default_optional_unrestricted_minmax $param_param_default_optional_unrestricted_minmax +#end if +#if $adv_opts.adv_opts_selector=='advanced': + #if str($adv_opts.param_param_wodefault_mandatory_unrestricted_adv): + -param_wodefault_mandatory_unrestricted_adv $adv_opts.param_param_wodefault_mandatory_unrestricted_adv +#end if + #if str($adv_opts.param_param_default_mandatory_unrestricted_adv): + -param_default_mandatory_unrestricted_adv $adv_opts.param_param_default_mandatory_unrestricted_adv +#end if + #if str($adv_opts.param_param_wodefault_optional_unrestricted_adv): + -param_wodefault_optional_unrestricted_adv $adv_opts.param_param_wodefault_optional_unrestricted_adv +#end if + #if str($adv_opts.param_param_default_optional_unrestricted_adv): + -param_default_optional_unrestricted_adv $adv_opts.param_param_default_optional_unrestricted_adv +#end if + #if str($adv_opts.param_param_wodefault_mandatory_unrestricted_adv_min): + -param_wodefault_mandatory_unrestricted_adv_min $adv_opts.param_param_wodefault_mandatory_unrestricted_adv_min +#end if + #if str($adv_opts.param_param_default_mandatory_unrestricted_adv_min): + -param_default_mandatory_unrestricted_adv_min $adv_opts.param_param_default_mandatory_unrestricted_adv_min +#end if + #if str($adv_opts.param_param_wodefault_optional_unrestricted_adv_min): + -param_wodefault_optional_unrestricted_adv_min $adv_opts.param_param_wodefault_optional_unrestricted_adv_min +#end if + #if str($adv_opts.param_param_default_optional_unrestricted_adv_min): + -param_default_optional_unrestricted_adv_min $adv_opts.param_param_default_optional_unrestricted_adv_min +#end if + #if str($adv_opts.param_param_wodefault_mandatory_unrestricted_adv_max): + -param_wodefault_mandatory_unrestricted_adv_max $adv_opts.param_param_wodefault_mandatory_unrestricted_adv_max +#end if + #if str($adv_opts.param_param_default_mandatory_unrestricted_adv_max): + -param_default_mandatory_unrestricted_adv_max $adv_opts.param_param_default_mandatory_unrestricted_adv_max +#end if + #if str($adv_opts.param_param_wodefault_optional_unrestricted_adv_max): + -param_wodefault_optional_unrestricted_adv_max $adv_opts.param_param_wodefault_optional_unrestricted_adv_max +#end if + #if str($adv_opts.param_param_default_optional_unrestricted_adv_max): + -param_default_optional_unrestricted_adv_max $adv_opts.param_param_default_optional_unrestricted_adv_max +#end if + #if str($adv_opts.param_param_wodefault_mandatory_unrestricted_adv_minmax): + -param_wodefault_mandatory_unrestricted_adv_minmax $adv_opts.param_param_wodefault_mandatory_unrestricted_adv_minmax +#end if + #if str($adv_opts.param_param_default_mandatory_unrestricted_adv_minmax): + -param_default_mandatory_unrestricted_adv_minmax $adv_opts.param_param_default_mandatory_unrestricted_adv_minmax +#end if + #if str($adv_opts.param_param_wodefault_optional_unrestricted_adv_minmax): + -param_wodefault_optional_unrestricted_adv_minmax $adv_opts.param_param_wodefault_optional_unrestricted_adv_minmax +#end if + #if str($adv_opts.param_param_default_optional_unrestricted_adv_minmax): + -param_default_optional_unrestricted_adv_minmax $adv_opts.param_param_default_optional_unrestricted_adv_minmax +#end if +#end if +> $param_stdout +]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + <![CDATA[Floateger parameter tests. + + +For more information, visit http://docs.are/unavailable.html]]> + diff --git a/test-data/integer.ctd b/test-data/integer.ctd new file mode 100644 index 00000000..356b58da --- /dev/null +++ b/test-data/integer.ctd @@ -0,0 +1,47 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/test-data/integer.xml b/test-data/integer.xml new file mode 100644 index 00000000..4ae33636 --- /dev/null +++ b/test-data/integer.xml @@ -0,0 +1,158 @@ + + + + + Integer parameter tests + + echo IntTest + macros.xml + + + + + <![CDATA[echo IntTest + +#if str($param_param_wodefault_mandatory_unrestricted): + -param_wodefault_mandatory_unrestricted $param_param_wodefault_mandatory_unrestricted +#end if +#if str($param_param_default_mandatory_unrestricted): + -param_default_mandatory_unrestricted $param_param_default_mandatory_unrestricted +#end if +#if str($param_param_wodefault_optional_unrestricted): + -param_wodefault_optional_unrestricted $param_param_wodefault_optional_unrestricted +#end if +#if str($param_param_default_optional_unrestricted): + -param_default_optional_unrestricted $param_param_default_optional_unrestricted +#end if +#if str($param_param_wodefault_mandatory_unrestricted_min): + -param_wodefault_mandatory_unrestricted_min $param_param_wodefault_mandatory_unrestricted_min +#end if +#if str($param_param_default_mandatory_unrestricted_min): + -param_default_mandatory_unrestricted_min $param_param_default_mandatory_unrestricted_min +#end if +#if str($param_param_wodefault_optional_unrestricted_min): + -param_wodefault_optional_unrestricted_min $param_param_wodefault_optional_unrestricted_min +#end if +#if str($param_param_default_optional_unrestricted_min): + -param_default_optional_unrestricted_min $param_param_default_optional_unrestricted_min +#end if +#if str($param_param_wodefault_mandatory_unrestricted_max): + -param_wodefault_mandatory_unrestricted_max $param_param_wodefault_mandatory_unrestricted_max +#end if +#if str($param_param_default_mandatory_unrestricted_max): + -param_default_mandatory_unrestricted_max $param_param_default_mandatory_unrestricted_max +#end if +#if str($param_param_wodefault_optional_unrestricted_max): + -param_wodefault_optional_unrestricted_max $param_param_wodefault_optional_unrestricted_max +#end if +#if str($param_param_default_optional_unrestricted_max): + -param_default_optional_unrestricted_max $param_param_default_optional_unrestricted_max +#end if +#if str($param_param_wodefault_mandatory_unrestricted_minmax): + -param_wodefault_mandatory_unrestricted_minmax $param_param_wodefault_mandatory_unrestricted_minmax +#end if +#if str($param_param_default_mandatory_unrestricted_minmax): + -param_default_mandatory_unrestricted_minmax $param_param_default_mandatory_unrestricted_minmax +#end if +#if str($param_param_wodefault_optional_unrestricted_minmax): + -param_wodefault_optional_unrestricted_minmax $param_param_wodefault_optional_unrestricted_minmax +#end if +#if str($param_param_default_optional_unrestricted_minmax): + -param_default_optional_unrestricted_minmax $param_param_default_optional_unrestricted_minmax +#end if +#if $adv_opts.adv_opts_selector=='advanced': + #if str($adv_opts.param_param_wodefault_mandatory_unrestricted_adv): + -param_wodefault_mandatory_unrestricted_adv $adv_opts.param_param_wodefault_mandatory_unrestricted_adv +#end if + #if str($adv_opts.param_param_default_mandatory_unrestricted_adv): + -param_default_mandatory_unrestricted_adv $adv_opts.param_param_default_mandatory_unrestricted_adv +#end if + #if str($adv_opts.param_param_wodefault_optional_unrestricted_adv): + -param_wodefault_optional_unrestricted_adv $adv_opts.param_param_wodefault_optional_unrestricted_adv +#end if + #if str($adv_opts.param_param_default_optional_unrestricted_adv): + -param_default_optional_unrestricted_adv $adv_opts.param_param_default_optional_unrestricted_adv +#end if + #if str($adv_opts.param_param_wodefault_mandatory_unrestricted_adv_min): + -param_wodefault_mandatory_unrestricted_adv_min $adv_opts.param_param_wodefault_mandatory_unrestricted_adv_min +#end if + #if str($adv_opts.param_param_default_mandatory_unrestricted_adv_min): + -param_default_mandatory_unrestricted_adv_min $adv_opts.param_param_default_mandatory_unrestricted_adv_min +#end if + #if str($adv_opts.param_param_wodefault_optional_unrestricted_adv_min): + -param_wodefault_optional_unrestricted_adv_min $adv_opts.param_param_wodefault_optional_unrestricted_adv_min +#end if + #if str($adv_opts.param_param_default_optional_unrestricted_adv_min): + -param_default_optional_unrestricted_adv_min $adv_opts.param_param_default_optional_unrestricted_adv_min +#end if + #if str($adv_opts.param_param_wodefault_mandatory_unrestricted_adv_max): + -param_wodefault_mandatory_unrestricted_adv_max $adv_opts.param_param_wodefault_mandatory_unrestricted_adv_max +#end if + #if str($adv_opts.param_param_default_mandatory_unrestricted_adv_max): + -param_default_mandatory_unrestricted_adv_max $adv_opts.param_param_default_mandatory_unrestricted_adv_max +#end if + #if str($adv_opts.param_param_wodefault_optional_unrestricted_adv_max): + -param_wodefault_optional_unrestricted_adv_max $adv_opts.param_param_wodefault_optional_unrestricted_adv_max +#end if + #if str($adv_opts.param_param_default_optional_unrestricted_adv_max): + -param_default_optional_unrestricted_adv_max $adv_opts.param_param_default_optional_unrestricted_adv_max +#end if + #if str($adv_opts.param_param_wodefault_mandatory_unrestricted_adv_minmax): + -param_wodefault_mandatory_unrestricted_adv_minmax $adv_opts.param_param_wodefault_mandatory_unrestricted_adv_minmax +#end if + #if str($adv_opts.param_param_default_mandatory_unrestricted_adv_minmax): + -param_default_mandatory_unrestricted_adv_minmax $adv_opts.param_param_default_mandatory_unrestricted_adv_minmax +#end if + #if str($adv_opts.param_param_wodefault_optional_unrestricted_adv_minmax): + -param_wodefault_optional_unrestricted_adv_minmax $adv_opts.param_param_wodefault_optional_unrestricted_adv_minmax +#end if + #if str($adv_opts.param_param_default_optional_unrestricted_adv_minmax): + -param_default_optional_unrestricted_adv_minmax $adv_opts.param_param_default_optional_unrestricted_adv_minmax +#end if +#end if +> $param_stdout +]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + <![CDATA[Integer parameter tests. + + +For more information, visit http://docs.are/unavailable.html]]> + diff --git a/test-data/macros.xml b/test-data/macros.xml new file mode 100644 index 00000000..82907e42 --- /dev/null +++ b/test-data/macros.xml @@ -0,0 +1,31 @@ + + + + + requirement + + + + + + + + + + + doi:10.1186/1471-2105-9-163 + + + + + + + + + + + + + + + diff --git a/test-data/select.ctd b/test-data/select.ctd new file mode 100644 index 00000000..f141dd12 --- /dev/null +++ b/test-data/select.ctd @@ -0,0 +1,20 @@ + + + + + + + + + + + + + + + + + + + + diff --git a/test-data/select.xml b/test-data/select.xml new file mode 100644 index 00000000..d79bf0f2 --- /dev/null +++ b/test-data/select.xml @@ -0,0 +1,134 @@ + + + + + String parameter tests + + echo SelectTest + macros.xml + + + + + <![CDATA[echo SelectTest + +#if $param_param_wodefault_mandatory_restricted: + -param_wodefault_mandatory_restricted + #if " " in str($param_param_wodefault_mandatory_restricted): + "$param_param_wodefault_mandatory_restricted" + #else + $param_param_wodefault_mandatory_restricted + #end if +#end if +#if $param_param_default_mandatory_restricted: + -param_default_mandatory_restricted + #if " " in str($param_param_default_mandatory_restricted): + "$param_param_default_mandatory_restricted" + #else + $param_param_default_mandatory_restricted + #end if +#end if +#if $param_param_wodefault_optional_restricted: + -param_wodefault_optional_restricted + #if " " in str($param_param_wodefault_optional_restricted): + "$param_param_wodefault_optional_restricted" + #else + $param_param_wodefault_optional_restricted + #end if +#end if +#if $param_param_default_optional_restricted: + -param_default_optional_restricted + #if " " in str($param_param_default_optional_restricted): + "$param_param_default_optional_restricted" + #else + $param_param_default_optional_restricted + #end if +#end if +#if $adv_opts.adv_opts_selector=='advanced': + #if $adv_opts.param_param_wodefault_mandatory_restricted_adv: + -param_wodefault_mandatory_restricted_adv + #if " " in str($adv_opts.param_param_wodefault_mandatory_restricted_adv): + "$adv_opts.param_param_wodefault_mandatory_restricted_adv" + #else + $adv_opts.param_param_wodefault_mandatory_restricted_adv + #end if +#end if + #if $adv_opts.param_param_default_mandatory_restricted_adv: + -param_default_mandatory_restricted_adv + #if " " in str($adv_opts.param_param_default_mandatory_restricted_adv): + "$adv_opts.param_param_default_mandatory_restricted_adv" + #else + $adv_opts.param_param_default_mandatory_restricted_adv + #end if +#end if + #if $adv_opts.param_param_wodefault_optional_restricted_adv: + -param_wodefault_optional_restricted_adv + #if " " in str($adv_opts.param_param_wodefault_optional_restricted_adv): + "$adv_opts.param_param_wodefault_optional_restricted_adv" + #else + $adv_opts.param_param_wodefault_optional_restricted_adv + #end if +#end if + #if $adv_opts.param_param_default_optional_restricted_adv: + -param_default_optional_restricted_adv + #if " " in str($adv_opts.param_param_default_optional_restricted_adv): + "$adv_opts.param_param_default_optional_restricted_adv" + #else + $adv_opts.param_param_default_optional_restricted_adv + #end if +#end if +#end if +> $param_stdout +]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + <![CDATA[String parameter tests. + + +For more information, visit http://docs.are/unavailable.html]]> + diff --git a/test-data/string.ctd b/test-data/string.ctd new file mode 100644 index 00000000..11517f36 --- /dev/null +++ b/test-data/string.ctd @@ -0,0 +1,20 @@ + + + + + + + + + + + + + + + + + + + + diff --git a/test-data/string.xml b/test-data/string.xml new file mode 100644 index 00000000..67109ad6 --- /dev/null +++ b/test-data/string.xml @@ -0,0 +1,118 @@ + + + + + String parameter tests + + echo StringTest + macros.xml + + + + + <![CDATA[echo StringTest + +#if str($param_param_wodefault_mandatory_unrestricted): + -param_wodefault_mandatory_unrestricted "$param_param_wodefault_mandatory_unrestricted" +#end if +#if str($param_param_default_mandatory_unrestricted): + -param_default_mandatory_unrestricted "$param_param_default_mandatory_unrestricted" +#end if +#if str($param_param_wodefault_optional_unrestricted): + -param_wodefault_optional_unrestricted "$param_param_wodefault_optional_unrestricted" +#end if +#if str($param_param_default_optional_unrestricted): + -param_default_optional_unrestricted "$param_param_default_optional_unrestricted" +#end if +#if $adv_opts.adv_opts_selector=='advanced': + #if str($adv_opts.param_param_wodefault_mandatory_unrestricted_adv): + -param_wodefault_mandatory_unrestricted_adv "$adv_opts.param_param_wodefault_mandatory_unrestricted_adv" +#end if + #if str($adv_opts.param_param_default_mandatory_unrestricted_adv): + -param_default_mandatory_unrestricted_adv "$adv_opts.param_param_default_mandatory_unrestricted_adv" +#end if + #if str($adv_opts.param_param_wodefault_optional_unrestricted_adv): + -param_wodefault_optional_unrestricted_adv "$adv_opts.param_param_wodefault_optional_unrestricted_adv" +#end if + #if str($adv_opts.param_param_default_optional_unrestricted_adv): + -param_default_optional_unrestricted_adv "$adv_opts.param_param_default_optional_unrestricted_adv" +#end if +#end if +> $param_stdout +]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + <![CDATA[String parameter tests. + + +For more information, visit http://docs.are/unavailable.html]]> + From c549147105fcbec25e5cc4ea29babf6da6149dcd Mon Sep 17 00:00:00 2001 From: Tomnl Date: Tue, 9 Jul 2019 16:21:09 +0100 Subject: [PATCH 002/121] add unit tests --- .travis.yml | 41 +++++++ tests/__init__.py | 0 tests/test-data/AccurateMassSearch.ctd | 44 +++++++ tests/test-data/AccurateMassSearch.xml | 144 ++++++++++++++++++++++ tests/test-data/empty.ctd | 12 ++ tests/test-data/empty.xml | 25 ++++ tests/test-data/float.ctd | 47 ++++++++ tests/test-data/float.xml | 158 +++++++++++++++++++++++++ tests/test-data/integer.ctd | 47 ++++++++ tests/test-data/integer.xml | 158 +++++++++++++++++++++++++ tests/test-data/macros.xml | 31 +++++ tests/test-data/select.ctd | 20 ++++ tests/test-data/select.xml | 134 +++++++++++++++++++++ tests/test-data/string.ctd | 20 ++++ tests/test-data/string.xml | 118 ++++++++++++++++++ tests/test_galaxy_cli.py | 64 ++++++++++ 16 files changed, 1063 insertions(+) create mode 100644 .travis.yml create mode 100644 tests/__init__.py create mode 100644 tests/test-data/AccurateMassSearch.ctd create mode 100644 tests/test-data/AccurateMassSearch.xml create mode 100644 tests/test-data/empty.ctd create mode 100644 tests/test-data/empty.xml create mode 100644 tests/test-data/float.ctd create mode 100644 tests/test-data/float.xml create mode 100644 tests/test-data/integer.ctd create mode 100644 tests/test-data/integer.xml create mode 100644 tests/test-data/macros.xml create mode 100644 tests/test-data/select.ctd create mode 100644 tests/test-data/select.xml create mode 100644 tests/test-data/string.ctd create mode 100644 tests/test-data/string.xml create mode 100644 tests/test_galaxy_cli.py diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 00000000..340840cf --- /dev/null +++ b/.travis.yml @@ -0,0 +1,41 @@ +language: python +python: + - "2.7" + +install: + - sudo apt-get update + - if [[ "$TRAVIS_PYTHON_VERSION" == "2.7" ]]; then + wget https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh -O miniconda.sh; + else + wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh; + fi + - bash miniconda.sh -b -p $HOME/miniconda + - export PATH="$HOME/miniconda/bin:$PATH" + - hash -r + - conda config --set always_yes yes --set changeps1 no + - conda update -q conda + - conda info -a + + - conda create -q -n test-environment python=$TRAVIS_PYTHON_VERSION + - source activate test-environment + - conda config --add channels conda-forge + - conda config --add channels bioconda + - conda config --add channels anaconda + - conda config --add channels workflowconversion + + - git clone https://github.com/WorkflowConversion/CTDopts $HOME/CTDopts + - cd $HOME/CTDopts + - python setup.py install + - conda install -c conda-forge lxml + - conda install -c conda-forge ruamel.yaml + - conda install libxml2 + - cd $TRAVIS_BUILD_DIR + - python setup.py install + + - conda install coverage green codecov + +script: + - python setup.py test + +after_script: +- python -m codecov diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/test-data/AccurateMassSearch.ctd b/tests/test-data/AccurateMassSearch.ctd new file mode 100644 index 00000000..6092d44e --- /dev/null +++ b/tests/test-data/AccurateMassSearch.ctd @@ -0,0 +1,44 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/test-data/AccurateMassSearch.xml b/tests/test-data/AccurateMassSearch.xml new file mode 100644 index 00000000..14bff665 --- /dev/null +++ b/tests/test-data/AccurateMassSearch.xml @@ -0,0 +1,144 @@ + + + + + Match MS signals to molecules from a database by mass. + + AccurateMassSearch + macros.xml + + + + + AccurateMassSearch + +#if str($param_in): + -in $param_in +#end if +#if str($param_out): + -out $param_out +#end if +#if str($param_out_annotation): + -out_annotation $param_out_annotation +#end if +#if str($param_positive_adducts): + -positive_adducts $param_positive_adducts +#end if +#if str($param_negative_adducts): + -negative_adducts $param_negative_adducts +#end if +#if str($param_threads): + -threads $param_threads +#end if +-db:mapping + #for token in $param_db_mapping: + $token + #end for +-db:struct + #for token in $param_db_struct: + $token + #end for +#if str($param_algorithm_mass_error_value): + -algorithm:mass_error_value $param_algorithm_mass_error_value +#end if +#if $param_algorithm_mass_error_unit: + -algorithm:mass_error_unit + #if " " in str($param_algorithm_mass_error_unit): + "$param_algorithm_mass_error_unit" + #else + $param_algorithm_mass_error_unit + #end if +#end if +#if $param_algorithm_ionization_mode: + -algorithm:ionization_mode + #if " " in str($param_algorithm_ionization_mode): + "$param_algorithm_ionization_mode" + #else + $param_algorithm_ionization_mode + #end if +#end if +#if $param_algorithm_isotopic_similarity: + -algorithm:isotopic_similarity +#end if +#if $param_algorithm_keep_unidentified_masses: + -algorithm:keep_unidentified_masses +#end if +#if str($param_algorithm_mzTab_exportIsotopeIntensities): + -algorithm:mzTab:exportIsotopeIntensities $param_algorithm_mzTab_exportIsotopeIntensities +#end if +#if $adv_opts.adv_opts_selector=='advanced': + #if str($adv_opts.param_log): + -log "$adv_opts.param_log" +#end if + #if str($adv_opts.param_debug): + -debug $adv_opts.param_debug +#end if + #if $adv_opts.param_no_progress: + -no_progress +#end if + #if $adv_opts.param_force: + -force +#end if + #if $adv_opts.param_test: + -test +#end if +#end if + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Match MS signals to molecules from a database by mass. + + +For more information, visit http://ftp.mi.fu-berlin.de/OpenMS/release-documentation/html/UTILS_AccurateMassSearch.html + diff --git a/tests/test-data/empty.ctd b/tests/test-data/empty.ctd new file mode 100644 index 00000000..316502e3 --- /dev/null +++ b/tests/test-data/empty.ctd @@ -0,0 +1,12 @@ + + + + + + + + + + + + diff --git a/tests/test-data/empty.xml b/tests/test-data/empty.xml new file mode 100644 index 00000000..07e534e7 --- /dev/null +++ b/tests/test-data/empty.xml @@ -0,0 +1,25 @@ + + + + + String parameter tests + + echo StringTest + macros.xml + + + + + <![CDATA[echo StringTest + +> $param_stdout +]]> + + + + + <![CDATA[String parameter tests. + + +For more information, visit http://docs.are/unavailable.html]]> + diff --git a/tests/test-data/float.ctd b/tests/test-data/float.ctd new file mode 100644 index 00000000..2ab6c54d --- /dev/null +++ b/tests/test-data/float.ctd @@ -0,0 +1,47 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/test-data/float.xml b/tests/test-data/float.xml new file mode 100644 index 00000000..f4fa7405 --- /dev/null +++ b/tests/test-data/float.xml @@ -0,0 +1,158 @@ + + + + + Floateger parameter tests + + echo FloatTest + macros.xml + + + + + <![CDATA[echo FloatTest + +#if str($param_param_wodefault_mandatory_unrestricted): + -param_wodefault_mandatory_unrestricted $param_param_wodefault_mandatory_unrestricted +#end if +#if str($param_param_default_mandatory_unrestricted): + -param_default_mandatory_unrestricted $param_param_default_mandatory_unrestricted +#end if +#if str($param_param_wodefault_optional_unrestricted): + -param_wodefault_optional_unrestricted $param_param_wodefault_optional_unrestricted +#end if +#if str($param_param_default_optional_unrestricted): + -param_default_optional_unrestricted $param_param_default_optional_unrestricted +#end if +#if str($param_param_wodefault_mandatory_unrestricted_min): + -param_wodefault_mandatory_unrestricted_min $param_param_wodefault_mandatory_unrestricted_min +#end if +#if str($param_param_default_mandatory_unrestricted_min): + -param_default_mandatory_unrestricted_min $param_param_default_mandatory_unrestricted_min +#end if +#if str($param_param_wodefault_optional_unrestricted_min): + -param_wodefault_optional_unrestricted_min $param_param_wodefault_optional_unrestricted_min +#end if +#if str($param_param_default_optional_unrestricted_min): + -param_default_optional_unrestricted_min $param_param_default_optional_unrestricted_min +#end if +#if str($param_param_wodefault_mandatory_unrestricted_max): + -param_wodefault_mandatory_unrestricted_max $param_param_wodefault_mandatory_unrestricted_max +#end if +#if str($param_param_default_mandatory_unrestricted_max): + -param_default_mandatory_unrestricted_max $param_param_default_mandatory_unrestricted_max +#end if +#if str($param_param_wodefault_optional_unrestricted_max): + -param_wodefault_optional_unrestricted_max $param_param_wodefault_optional_unrestricted_max +#end if +#if str($param_param_default_optional_unrestricted_max): + -param_default_optional_unrestricted_max $param_param_default_optional_unrestricted_max +#end if +#if str($param_param_wodefault_mandatory_unrestricted_minmax): + -param_wodefault_mandatory_unrestricted_minmax $param_param_wodefault_mandatory_unrestricted_minmax +#end if +#if str($param_param_default_mandatory_unrestricted_minmax): + -param_default_mandatory_unrestricted_minmax $param_param_default_mandatory_unrestricted_minmax +#end if +#if str($param_param_wodefault_optional_unrestricted_minmax): + -param_wodefault_optional_unrestricted_minmax $param_param_wodefault_optional_unrestricted_minmax +#end if +#if str($param_param_default_optional_unrestricted_minmax): + -param_default_optional_unrestricted_minmax $param_param_default_optional_unrestricted_minmax +#end if +#if $adv_opts.adv_opts_selector=='advanced': + #if str($adv_opts.param_param_wodefault_mandatory_unrestricted_adv): + -param_wodefault_mandatory_unrestricted_adv $adv_opts.param_param_wodefault_mandatory_unrestricted_adv +#end if + #if str($adv_opts.param_param_default_mandatory_unrestricted_adv): + -param_default_mandatory_unrestricted_adv $adv_opts.param_param_default_mandatory_unrestricted_adv +#end if + #if str($adv_opts.param_param_wodefault_optional_unrestricted_adv): + -param_wodefault_optional_unrestricted_adv $adv_opts.param_param_wodefault_optional_unrestricted_adv +#end if + #if str($adv_opts.param_param_default_optional_unrestricted_adv): + -param_default_optional_unrestricted_adv $adv_opts.param_param_default_optional_unrestricted_adv +#end if + #if str($adv_opts.param_param_wodefault_mandatory_unrestricted_adv_min): + -param_wodefault_mandatory_unrestricted_adv_min $adv_opts.param_param_wodefault_mandatory_unrestricted_adv_min +#end if + #if str($adv_opts.param_param_default_mandatory_unrestricted_adv_min): + -param_default_mandatory_unrestricted_adv_min $adv_opts.param_param_default_mandatory_unrestricted_adv_min +#end if + #if str($adv_opts.param_param_wodefault_optional_unrestricted_adv_min): + -param_wodefault_optional_unrestricted_adv_min $adv_opts.param_param_wodefault_optional_unrestricted_adv_min +#end if + #if str($adv_opts.param_param_default_optional_unrestricted_adv_min): + -param_default_optional_unrestricted_adv_min $adv_opts.param_param_default_optional_unrestricted_adv_min +#end if + #if str($adv_opts.param_param_wodefault_mandatory_unrestricted_adv_max): + -param_wodefault_mandatory_unrestricted_adv_max $adv_opts.param_param_wodefault_mandatory_unrestricted_adv_max +#end if + #if str($adv_opts.param_param_default_mandatory_unrestricted_adv_max): + -param_default_mandatory_unrestricted_adv_max $adv_opts.param_param_default_mandatory_unrestricted_adv_max +#end if + #if str($adv_opts.param_param_wodefault_optional_unrestricted_adv_max): + -param_wodefault_optional_unrestricted_adv_max $adv_opts.param_param_wodefault_optional_unrestricted_adv_max +#end if + #if str($adv_opts.param_param_default_optional_unrestricted_adv_max): + -param_default_optional_unrestricted_adv_max $adv_opts.param_param_default_optional_unrestricted_adv_max +#end if + #if str($adv_opts.param_param_wodefault_mandatory_unrestricted_adv_minmax): + -param_wodefault_mandatory_unrestricted_adv_minmax $adv_opts.param_param_wodefault_mandatory_unrestricted_adv_minmax +#end if + #if str($adv_opts.param_param_default_mandatory_unrestricted_adv_minmax): + -param_default_mandatory_unrestricted_adv_minmax $adv_opts.param_param_default_mandatory_unrestricted_adv_minmax +#end if + #if str($adv_opts.param_param_wodefault_optional_unrestricted_adv_minmax): + -param_wodefault_optional_unrestricted_adv_minmax $adv_opts.param_param_wodefault_optional_unrestricted_adv_minmax +#end if + #if str($adv_opts.param_param_default_optional_unrestricted_adv_minmax): + -param_default_optional_unrestricted_adv_minmax $adv_opts.param_param_default_optional_unrestricted_adv_minmax +#end if +#end if +> $param_stdout +]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + <![CDATA[Floateger parameter tests. + + +For more information, visit http://docs.are/unavailable.html]]> + diff --git a/tests/test-data/integer.ctd b/tests/test-data/integer.ctd new file mode 100644 index 00000000..356b58da --- /dev/null +++ b/tests/test-data/integer.ctd @@ -0,0 +1,47 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/test-data/integer.xml b/tests/test-data/integer.xml new file mode 100644 index 00000000..4ae33636 --- /dev/null +++ b/tests/test-data/integer.xml @@ -0,0 +1,158 @@ + + + + + Integer parameter tests + + echo IntTest + macros.xml + + + + + <![CDATA[echo IntTest + +#if str($param_param_wodefault_mandatory_unrestricted): + -param_wodefault_mandatory_unrestricted $param_param_wodefault_mandatory_unrestricted +#end if +#if str($param_param_default_mandatory_unrestricted): + -param_default_mandatory_unrestricted $param_param_default_mandatory_unrestricted +#end if +#if str($param_param_wodefault_optional_unrestricted): + -param_wodefault_optional_unrestricted $param_param_wodefault_optional_unrestricted +#end if +#if str($param_param_default_optional_unrestricted): + -param_default_optional_unrestricted $param_param_default_optional_unrestricted +#end if +#if str($param_param_wodefault_mandatory_unrestricted_min): + -param_wodefault_mandatory_unrestricted_min $param_param_wodefault_mandatory_unrestricted_min +#end if +#if str($param_param_default_mandatory_unrestricted_min): + -param_default_mandatory_unrestricted_min $param_param_default_mandatory_unrestricted_min +#end if +#if str($param_param_wodefault_optional_unrestricted_min): + -param_wodefault_optional_unrestricted_min $param_param_wodefault_optional_unrestricted_min +#end if +#if str($param_param_default_optional_unrestricted_min): + -param_default_optional_unrestricted_min $param_param_default_optional_unrestricted_min +#end if +#if str($param_param_wodefault_mandatory_unrestricted_max): + -param_wodefault_mandatory_unrestricted_max $param_param_wodefault_mandatory_unrestricted_max +#end if +#if str($param_param_default_mandatory_unrestricted_max): + -param_default_mandatory_unrestricted_max $param_param_default_mandatory_unrestricted_max +#end if +#if str($param_param_wodefault_optional_unrestricted_max): + -param_wodefault_optional_unrestricted_max $param_param_wodefault_optional_unrestricted_max +#end if +#if str($param_param_default_optional_unrestricted_max): + -param_default_optional_unrestricted_max $param_param_default_optional_unrestricted_max +#end if +#if str($param_param_wodefault_mandatory_unrestricted_minmax): + -param_wodefault_mandatory_unrestricted_minmax $param_param_wodefault_mandatory_unrestricted_minmax +#end if +#if str($param_param_default_mandatory_unrestricted_minmax): + -param_default_mandatory_unrestricted_minmax $param_param_default_mandatory_unrestricted_minmax +#end if +#if str($param_param_wodefault_optional_unrestricted_minmax): + -param_wodefault_optional_unrestricted_minmax $param_param_wodefault_optional_unrestricted_minmax +#end if +#if str($param_param_default_optional_unrestricted_minmax): + -param_default_optional_unrestricted_minmax $param_param_default_optional_unrestricted_minmax +#end if +#if $adv_opts.adv_opts_selector=='advanced': + #if str($adv_opts.param_param_wodefault_mandatory_unrestricted_adv): + -param_wodefault_mandatory_unrestricted_adv $adv_opts.param_param_wodefault_mandatory_unrestricted_adv +#end if + #if str($adv_opts.param_param_default_mandatory_unrestricted_adv): + -param_default_mandatory_unrestricted_adv $adv_opts.param_param_default_mandatory_unrestricted_adv +#end if + #if str($adv_opts.param_param_wodefault_optional_unrestricted_adv): + -param_wodefault_optional_unrestricted_adv $adv_opts.param_param_wodefault_optional_unrestricted_adv +#end if + #if str($adv_opts.param_param_default_optional_unrestricted_adv): + -param_default_optional_unrestricted_adv $adv_opts.param_param_default_optional_unrestricted_adv +#end if + #if str($adv_opts.param_param_wodefault_mandatory_unrestricted_adv_min): + -param_wodefault_mandatory_unrestricted_adv_min $adv_opts.param_param_wodefault_mandatory_unrestricted_adv_min +#end if + #if str($adv_opts.param_param_default_mandatory_unrestricted_adv_min): + -param_default_mandatory_unrestricted_adv_min $adv_opts.param_param_default_mandatory_unrestricted_adv_min +#end if + #if str($adv_opts.param_param_wodefault_optional_unrestricted_adv_min): + -param_wodefault_optional_unrestricted_adv_min $adv_opts.param_param_wodefault_optional_unrestricted_adv_min +#end if + #if str($adv_opts.param_param_default_optional_unrestricted_adv_min): + -param_default_optional_unrestricted_adv_min $adv_opts.param_param_default_optional_unrestricted_adv_min +#end if + #if str($adv_opts.param_param_wodefault_mandatory_unrestricted_adv_max): + -param_wodefault_mandatory_unrestricted_adv_max $adv_opts.param_param_wodefault_mandatory_unrestricted_adv_max +#end if + #if str($adv_opts.param_param_default_mandatory_unrestricted_adv_max): + -param_default_mandatory_unrestricted_adv_max $adv_opts.param_param_default_mandatory_unrestricted_adv_max +#end if + #if str($adv_opts.param_param_wodefault_optional_unrestricted_adv_max): + -param_wodefault_optional_unrestricted_adv_max $adv_opts.param_param_wodefault_optional_unrestricted_adv_max +#end if + #if str($adv_opts.param_param_default_optional_unrestricted_adv_max): + -param_default_optional_unrestricted_adv_max $adv_opts.param_param_default_optional_unrestricted_adv_max +#end if + #if str($adv_opts.param_param_wodefault_mandatory_unrestricted_adv_minmax): + -param_wodefault_mandatory_unrestricted_adv_minmax $adv_opts.param_param_wodefault_mandatory_unrestricted_adv_minmax +#end if + #if str($adv_opts.param_param_default_mandatory_unrestricted_adv_minmax): + -param_default_mandatory_unrestricted_adv_minmax $adv_opts.param_param_default_mandatory_unrestricted_adv_minmax +#end if + #if str($adv_opts.param_param_wodefault_optional_unrestricted_adv_minmax): + -param_wodefault_optional_unrestricted_adv_minmax $adv_opts.param_param_wodefault_optional_unrestricted_adv_minmax +#end if + #if str($adv_opts.param_param_default_optional_unrestricted_adv_minmax): + -param_default_optional_unrestricted_adv_minmax $adv_opts.param_param_default_optional_unrestricted_adv_minmax +#end if +#end if +> $param_stdout +]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + <![CDATA[Integer parameter tests. + + +For more information, visit http://docs.are/unavailable.html]]> + diff --git a/tests/test-data/macros.xml b/tests/test-data/macros.xml new file mode 100644 index 00000000..82907e42 --- /dev/null +++ b/tests/test-data/macros.xml @@ -0,0 +1,31 @@ + + + + + requirement + + + + + + + + + + + doi:10.1186/1471-2105-9-163 + + + + + + + + + + + + + + + diff --git a/tests/test-data/select.ctd b/tests/test-data/select.ctd new file mode 100644 index 00000000..f141dd12 --- /dev/null +++ b/tests/test-data/select.ctd @@ -0,0 +1,20 @@ + + + + + + + + + + + + + + + + + + + + diff --git a/tests/test-data/select.xml b/tests/test-data/select.xml new file mode 100644 index 00000000..d79bf0f2 --- /dev/null +++ b/tests/test-data/select.xml @@ -0,0 +1,134 @@ + + + + + String parameter tests + + echo SelectTest + macros.xml + + + + + <![CDATA[echo SelectTest + +#if $param_param_wodefault_mandatory_restricted: + -param_wodefault_mandatory_restricted + #if " " in str($param_param_wodefault_mandatory_restricted): + "$param_param_wodefault_mandatory_restricted" + #else + $param_param_wodefault_mandatory_restricted + #end if +#end if +#if $param_param_default_mandatory_restricted: + -param_default_mandatory_restricted + #if " " in str($param_param_default_mandatory_restricted): + "$param_param_default_mandatory_restricted" + #else + $param_param_default_mandatory_restricted + #end if +#end if +#if $param_param_wodefault_optional_restricted: + -param_wodefault_optional_restricted + #if " " in str($param_param_wodefault_optional_restricted): + "$param_param_wodefault_optional_restricted" + #else + $param_param_wodefault_optional_restricted + #end if +#end if +#if $param_param_default_optional_restricted: + -param_default_optional_restricted + #if " " in str($param_param_default_optional_restricted): + "$param_param_default_optional_restricted" + #else + $param_param_default_optional_restricted + #end if +#end if +#if $adv_opts.adv_opts_selector=='advanced': + #if $adv_opts.param_param_wodefault_mandatory_restricted_adv: + -param_wodefault_mandatory_restricted_adv + #if " " in str($adv_opts.param_param_wodefault_mandatory_restricted_adv): + "$adv_opts.param_param_wodefault_mandatory_restricted_adv" + #else + $adv_opts.param_param_wodefault_mandatory_restricted_adv + #end if +#end if + #if $adv_opts.param_param_default_mandatory_restricted_adv: + -param_default_mandatory_restricted_adv + #if " " in str($adv_opts.param_param_default_mandatory_restricted_adv): + "$adv_opts.param_param_default_mandatory_restricted_adv" + #else + $adv_opts.param_param_default_mandatory_restricted_adv + #end if +#end if + #if $adv_opts.param_param_wodefault_optional_restricted_adv: + -param_wodefault_optional_restricted_adv + #if " " in str($adv_opts.param_param_wodefault_optional_restricted_adv): + "$adv_opts.param_param_wodefault_optional_restricted_adv" + #else + $adv_opts.param_param_wodefault_optional_restricted_adv + #end if +#end if + #if $adv_opts.param_param_default_optional_restricted_adv: + -param_default_optional_restricted_adv + #if " " in str($adv_opts.param_param_default_optional_restricted_adv): + "$adv_opts.param_param_default_optional_restricted_adv" + #else + $adv_opts.param_param_default_optional_restricted_adv + #end if +#end if +#end if +> $param_stdout +]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + <![CDATA[String parameter tests. + + +For more information, visit http://docs.are/unavailable.html]]> + diff --git a/tests/test-data/string.ctd b/tests/test-data/string.ctd new file mode 100644 index 00000000..11517f36 --- /dev/null +++ b/tests/test-data/string.ctd @@ -0,0 +1,20 @@ + + + + + + + + + + + + + + + + + + + + diff --git a/tests/test-data/string.xml b/tests/test-data/string.xml new file mode 100644 index 00000000..67109ad6 --- /dev/null +++ b/tests/test-data/string.xml @@ -0,0 +1,118 @@ + + + + + String parameter tests + + echo StringTest + macros.xml + + + + + <![CDATA[echo StringTest + +#if str($param_param_wodefault_mandatory_unrestricted): + -param_wodefault_mandatory_unrestricted "$param_param_wodefault_mandatory_unrestricted" +#end if +#if str($param_param_default_mandatory_unrestricted): + -param_default_mandatory_unrestricted "$param_param_default_mandatory_unrestricted" +#end if +#if str($param_param_wodefault_optional_unrestricted): + -param_wodefault_optional_unrestricted "$param_param_wodefault_optional_unrestricted" +#end if +#if str($param_param_default_optional_unrestricted): + -param_default_optional_unrestricted "$param_param_default_optional_unrestricted" +#end if +#if $adv_opts.adv_opts_selector=='advanced': + #if str($adv_opts.param_param_wodefault_mandatory_unrestricted_adv): + -param_wodefault_mandatory_unrestricted_adv "$adv_opts.param_param_wodefault_mandatory_unrestricted_adv" +#end if + #if str($adv_opts.param_param_default_mandatory_unrestricted_adv): + -param_default_mandatory_unrestricted_adv "$adv_opts.param_param_default_mandatory_unrestricted_adv" +#end if + #if str($adv_opts.param_param_wodefault_optional_unrestricted_adv): + -param_wodefault_optional_unrestricted_adv "$adv_opts.param_param_wodefault_optional_unrestricted_adv" +#end if + #if str($adv_opts.param_param_default_optional_unrestricted_adv): + -param_default_optional_unrestricted_adv "$adv_opts.param_param_default_optional_unrestricted_adv" +#end if +#end if +> $param_stdout +]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + <![CDATA[String parameter tests. + + +For more information, visit http://docs.are/unavailable.html]]> + diff --git a/tests/test_galaxy_cli.py b/tests/test_galaxy_cli.py new file mode 100644 index 00000000..51de457a --- /dev/null +++ b/tests/test_galaxy_cli.py @@ -0,0 +1,64 @@ +import unittest +import os +import subprocess +import tempfile + +def to_test_data(*args): + return os.path.abspath(os.path.join(os.path.dirname(os.path.realpath(__file__)), "test-data", *args)) + +def file2list(pth): + with open(pth) as f: + lines = f.readlines() + return lines + +class GalaxyCliTestCase(unittest.TestCase): + + def _compare_cli_output(self, fileprefix): + in_pth = to_test_data('{}.ctd'.format(fileprefix)) + macro_pth = to_test_data('macros.xml') + tmp = tempfile.mkdtemp() + + out_file = os.path.join(tmp, '{}.xml'.format(fileprefix)) + #out_file = to_test_data('{}.xml'.format(fileprefix)) + + cmd = ['CTDConverter', 'galaxy', '-i', in_pth, '-o', out_file, '-m', macro_pth, '-b', 'version'] + print(cmd) + + popen = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + output, err = popen.communicate() + print(output) + print(err) + + old_file_pth = to_test_data('{}.xml'.format(fileprefix)) + + print(out_file) + print(old_file_pth) + + new_l = file2list(out_file) + old_l = file2list(old_file_pth) + + for i in range(0, len(new_l)): + self.assertEqual(new_l[i].rstrip(), old_l[i].rstrip()) + + def test_galaxy_cli_full_file_ctd(self): + self._compare_cli_output('AccurateMassSearch') + + def test_galaxy_cli_empty_ctd(self): + self._compare_cli_output('empty') + + def test_galaxy_cli_float_ctd(self): + self._compare_cli_output('float') + + def test_galaxy_cli_integer_ctd(self): + self._compare_cli_output('integer') + + def test_galaxy_cli_select_ctd(self): + self._compare_cli_output('select') + + def test_galaxy_cli_string_ctd(self): + self._compare_cli_output('string') + + + +if __name__ == '__main__': + unittest.main() From 2a4ae8df2c587df5d3c646819cba74c360aed2e0 Mon Sep 17 00:00:00 2001 From: M Bernt Date: Mon, 15 Jul 2019 14:14:16 +0200 Subject: [PATCH 003/121] switch CTDopts branch --- .travis.yml | 3 +- test-data/empty.ctd | 12 ---- test-data/empty.xml | 25 ------- test-data/float.ctd | 47 ------------- test-data/float.xml | 158 ------------------------------------------ test-data/integer.ctd | 47 ------------- test-data/integer.xml | 158 ------------------------------------------ test-data/macros.xml | 31 --------- test-data/select.ctd | 20 ------ test-data/select.xml | 134 ----------------------------------- test-data/string.ctd | 20 ------ test-data/string.xml | 118 ------------------------------- 12 files changed, 2 insertions(+), 771 deletions(-) delete mode 100644 test-data/empty.ctd delete mode 100644 test-data/empty.xml delete mode 100644 test-data/float.ctd delete mode 100644 test-data/float.xml delete mode 100644 test-data/integer.ctd delete mode 100644 test-data/integer.xml delete mode 100644 test-data/macros.xml delete mode 100644 test-data/select.ctd delete mode 100644 test-data/select.xml delete mode 100644 test-data/string.ctd delete mode 100644 test-data/string.xml diff --git a/.travis.yml b/.travis.yml index 340840cf..ef5aea5c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,7 +23,8 @@ install: - conda config --add channels anaconda - conda config --add channels workflowconversion - - git clone https://github.com/WorkflowConversion/CTDopts $HOME/CTDopts +# - git clone https://github.com/WorkflowConversion/CTDopts $HOME/CTDopts + - git clone -b empty_numeric_defaults https://github.com/bernt-matthias/CTDopts $HOME/CTDopts - cd $HOME/CTDopts - python setup.py install - conda install -c conda-forge lxml diff --git a/test-data/empty.ctd b/test-data/empty.ctd deleted file mode 100644 index 316502e3..00000000 --- a/test-data/empty.ctd +++ /dev/null @@ -1,12 +0,0 @@ - - - - - - - - - - - - diff --git a/test-data/empty.xml b/test-data/empty.xml deleted file mode 100644 index 07e534e7..00000000 --- a/test-data/empty.xml +++ /dev/null @@ -1,25 +0,0 @@ - - - - - String parameter tests - - echo StringTest - macros.xml - - - - - <![CDATA[echo StringTest - -> $param_stdout -]]> - - - - - <![CDATA[String parameter tests. - - -For more information, visit http://docs.are/unavailable.html]]> - diff --git a/test-data/float.ctd b/test-data/float.ctd deleted file mode 100644 index 2ab6c54d..00000000 --- a/test-data/float.ctd +++ /dev/null @@ -1,47 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/test-data/float.xml b/test-data/float.xml deleted file mode 100644 index f4fa7405..00000000 --- a/test-data/float.xml +++ /dev/null @@ -1,158 +0,0 @@ - - - - - Floateger parameter tests - - echo FloatTest - macros.xml - - - - - <![CDATA[echo FloatTest - -#if str($param_param_wodefault_mandatory_unrestricted): - -param_wodefault_mandatory_unrestricted $param_param_wodefault_mandatory_unrestricted -#end if -#if str($param_param_default_mandatory_unrestricted): - -param_default_mandatory_unrestricted $param_param_default_mandatory_unrestricted -#end if -#if str($param_param_wodefault_optional_unrestricted): - -param_wodefault_optional_unrestricted $param_param_wodefault_optional_unrestricted -#end if -#if str($param_param_default_optional_unrestricted): - -param_default_optional_unrestricted $param_param_default_optional_unrestricted -#end if -#if str($param_param_wodefault_mandatory_unrestricted_min): - -param_wodefault_mandatory_unrestricted_min $param_param_wodefault_mandatory_unrestricted_min -#end if -#if str($param_param_default_mandatory_unrestricted_min): - -param_default_mandatory_unrestricted_min $param_param_default_mandatory_unrestricted_min -#end if -#if str($param_param_wodefault_optional_unrestricted_min): - -param_wodefault_optional_unrestricted_min $param_param_wodefault_optional_unrestricted_min -#end if -#if str($param_param_default_optional_unrestricted_min): - -param_default_optional_unrestricted_min $param_param_default_optional_unrestricted_min -#end if -#if str($param_param_wodefault_mandatory_unrestricted_max): - -param_wodefault_mandatory_unrestricted_max $param_param_wodefault_mandatory_unrestricted_max -#end if -#if str($param_param_default_mandatory_unrestricted_max): - -param_default_mandatory_unrestricted_max $param_param_default_mandatory_unrestricted_max -#end if -#if str($param_param_wodefault_optional_unrestricted_max): - -param_wodefault_optional_unrestricted_max $param_param_wodefault_optional_unrestricted_max -#end if -#if str($param_param_default_optional_unrestricted_max): - -param_default_optional_unrestricted_max $param_param_default_optional_unrestricted_max -#end if -#if str($param_param_wodefault_mandatory_unrestricted_minmax): - -param_wodefault_mandatory_unrestricted_minmax $param_param_wodefault_mandatory_unrestricted_minmax -#end if -#if str($param_param_default_mandatory_unrestricted_minmax): - -param_default_mandatory_unrestricted_minmax $param_param_default_mandatory_unrestricted_minmax -#end if -#if str($param_param_wodefault_optional_unrestricted_minmax): - -param_wodefault_optional_unrestricted_minmax $param_param_wodefault_optional_unrestricted_minmax -#end if -#if str($param_param_default_optional_unrestricted_minmax): - -param_default_optional_unrestricted_minmax $param_param_default_optional_unrestricted_minmax -#end if -#if $adv_opts.adv_opts_selector=='advanced': - #if str($adv_opts.param_param_wodefault_mandatory_unrestricted_adv): - -param_wodefault_mandatory_unrestricted_adv $adv_opts.param_param_wodefault_mandatory_unrestricted_adv -#end if - #if str($adv_opts.param_param_default_mandatory_unrestricted_adv): - -param_default_mandatory_unrestricted_adv $adv_opts.param_param_default_mandatory_unrestricted_adv -#end if - #if str($adv_opts.param_param_wodefault_optional_unrestricted_adv): - -param_wodefault_optional_unrestricted_adv $adv_opts.param_param_wodefault_optional_unrestricted_adv -#end if - #if str($adv_opts.param_param_default_optional_unrestricted_adv): - -param_default_optional_unrestricted_adv $adv_opts.param_param_default_optional_unrestricted_adv -#end if - #if str($adv_opts.param_param_wodefault_mandatory_unrestricted_adv_min): - -param_wodefault_mandatory_unrestricted_adv_min $adv_opts.param_param_wodefault_mandatory_unrestricted_adv_min -#end if - #if str($adv_opts.param_param_default_mandatory_unrestricted_adv_min): - -param_default_mandatory_unrestricted_adv_min $adv_opts.param_param_default_mandatory_unrestricted_adv_min -#end if - #if str($adv_opts.param_param_wodefault_optional_unrestricted_adv_min): - -param_wodefault_optional_unrestricted_adv_min $adv_opts.param_param_wodefault_optional_unrestricted_adv_min -#end if - #if str($adv_opts.param_param_default_optional_unrestricted_adv_min): - -param_default_optional_unrestricted_adv_min $adv_opts.param_param_default_optional_unrestricted_adv_min -#end if - #if str($adv_opts.param_param_wodefault_mandatory_unrestricted_adv_max): - -param_wodefault_mandatory_unrestricted_adv_max $adv_opts.param_param_wodefault_mandatory_unrestricted_adv_max -#end if - #if str($adv_opts.param_param_default_mandatory_unrestricted_adv_max): - -param_default_mandatory_unrestricted_adv_max $adv_opts.param_param_default_mandatory_unrestricted_adv_max -#end if - #if str($adv_opts.param_param_wodefault_optional_unrestricted_adv_max): - -param_wodefault_optional_unrestricted_adv_max $adv_opts.param_param_wodefault_optional_unrestricted_adv_max -#end if - #if str($adv_opts.param_param_default_optional_unrestricted_adv_max): - -param_default_optional_unrestricted_adv_max $adv_opts.param_param_default_optional_unrestricted_adv_max -#end if - #if str($adv_opts.param_param_wodefault_mandatory_unrestricted_adv_minmax): - -param_wodefault_mandatory_unrestricted_adv_minmax $adv_opts.param_param_wodefault_mandatory_unrestricted_adv_minmax -#end if - #if str($adv_opts.param_param_default_mandatory_unrestricted_adv_minmax): - -param_default_mandatory_unrestricted_adv_minmax $adv_opts.param_param_default_mandatory_unrestricted_adv_minmax -#end if - #if str($adv_opts.param_param_wodefault_optional_unrestricted_adv_minmax): - -param_wodefault_optional_unrestricted_adv_minmax $adv_opts.param_param_wodefault_optional_unrestricted_adv_minmax -#end if - #if str($adv_opts.param_param_default_optional_unrestricted_adv_minmax): - -param_default_optional_unrestricted_adv_minmax $adv_opts.param_param_default_optional_unrestricted_adv_minmax -#end if -#end if -> $param_stdout -]]> - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - <![CDATA[Floateger parameter tests. - - -For more information, visit http://docs.are/unavailable.html]]> - diff --git a/test-data/integer.ctd b/test-data/integer.ctd deleted file mode 100644 index 356b58da..00000000 --- a/test-data/integer.ctd +++ /dev/null @@ -1,47 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/test-data/integer.xml b/test-data/integer.xml deleted file mode 100644 index 4ae33636..00000000 --- a/test-data/integer.xml +++ /dev/null @@ -1,158 +0,0 @@ - - - - - Integer parameter tests - - echo IntTest - macros.xml - - - - - <![CDATA[echo IntTest - -#if str($param_param_wodefault_mandatory_unrestricted): - -param_wodefault_mandatory_unrestricted $param_param_wodefault_mandatory_unrestricted -#end if -#if str($param_param_default_mandatory_unrestricted): - -param_default_mandatory_unrestricted $param_param_default_mandatory_unrestricted -#end if -#if str($param_param_wodefault_optional_unrestricted): - -param_wodefault_optional_unrestricted $param_param_wodefault_optional_unrestricted -#end if -#if str($param_param_default_optional_unrestricted): - -param_default_optional_unrestricted $param_param_default_optional_unrestricted -#end if -#if str($param_param_wodefault_mandatory_unrestricted_min): - -param_wodefault_mandatory_unrestricted_min $param_param_wodefault_mandatory_unrestricted_min -#end if -#if str($param_param_default_mandatory_unrestricted_min): - -param_default_mandatory_unrestricted_min $param_param_default_mandatory_unrestricted_min -#end if -#if str($param_param_wodefault_optional_unrestricted_min): - -param_wodefault_optional_unrestricted_min $param_param_wodefault_optional_unrestricted_min -#end if -#if str($param_param_default_optional_unrestricted_min): - -param_default_optional_unrestricted_min $param_param_default_optional_unrestricted_min -#end if -#if str($param_param_wodefault_mandatory_unrestricted_max): - -param_wodefault_mandatory_unrestricted_max $param_param_wodefault_mandatory_unrestricted_max -#end if -#if str($param_param_default_mandatory_unrestricted_max): - -param_default_mandatory_unrestricted_max $param_param_default_mandatory_unrestricted_max -#end if -#if str($param_param_wodefault_optional_unrestricted_max): - -param_wodefault_optional_unrestricted_max $param_param_wodefault_optional_unrestricted_max -#end if -#if str($param_param_default_optional_unrestricted_max): - -param_default_optional_unrestricted_max $param_param_default_optional_unrestricted_max -#end if -#if str($param_param_wodefault_mandatory_unrestricted_minmax): - -param_wodefault_mandatory_unrestricted_minmax $param_param_wodefault_mandatory_unrestricted_minmax -#end if -#if str($param_param_default_mandatory_unrestricted_minmax): - -param_default_mandatory_unrestricted_minmax $param_param_default_mandatory_unrestricted_minmax -#end if -#if str($param_param_wodefault_optional_unrestricted_minmax): - -param_wodefault_optional_unrestricted_minmax $param_param_wodefault_optional_unrestricted_minmax -#end if -#if str($param_param_default_optional_unrestricted_minmax): - -param_default_optional_unrestricted_minmax $param_param_default_optional_unrestricted_minmax -#end if -#if $adv_opts.adv_opts_selector=='advanced': - #if str($adv_opts.param_param_wodefault_mandatory_unrestricted_adv): - -param_wodefault_mandatory_unrestricted_adv $adv_opts.param_param_wodefault_mandatory_unrestricted_adv -#end if - #if str($adv_opts.param_param_default_mandatory_unrestricted_adv): - -param_default_mandatory_unrestricted_adv $adv_opts.param_param_default_mandatory_unrestricted_adv -#end if - #if str($adv_opts.param_param_wodefault_optional_unrestricted_adv): - -param_wodefault_optional_unrestricted_adv $adv_opts.param_param_wodefault_optional_unrestricted_adv -#end if - #if str($adv_opts.param_param_default_optional_unrestricted_adv): - -param_default_optional_unrestricted_adv $adv_opts.param_param_default_optional_unrestricted_adv -#end if - #if str($adv_opts.param_param_wodefault_mandatory_unrestricted_adv_min): - -param_wodefault_mandatory_unrestricted_adv_min $adv_opts.param_param_wodefault_mandatory_unrestricted_adv_min -#end if - #if str($adv_opts.param_param_default_mandatory_unrestricted_adv_min): - -param_default_mandatory_unrestricted_adv_min $adv_opts.param_param_default_mandatory_unrestricted_adv_min -#end if - #if str($adv_opts.param_param_wodefault_optional_unrestricted_adv_min): - -param_wodefault_optional_unrestricted_adv_min $adv_opts.param_param_wodefault_optional_unrestricted_adv_min -#end if - #if str($adv_opts.param_param_default_optional_unrestricted_adv_min): - -param_default_optional_unrestricted_adv_min $adv_opts.param_param_default_optional_unrestricted_adv_min -#end if - #if str($adv_opts.param_param_wodefault_mandatory_unrestricted_adv_max): - -param_wodefault_mandatory_unrestricted_adv_max $adv_opts.param_param_wodefault_mandatory_unrestricted_adv_max -#end if - #if str($adv_opts.param_param_default_mandatory_unrestricted_adv_max): - -param_default_mandatory_unrestricted_adv_max $adv_opts.param_param_default_mandatory_unrestricted_adv_max -#end if - #if str($adv_opts.param_param_wodefault_optional_unrestricted_adv_max): - -param_wodefault_optional_unrestricted_adv_max $adv_opts.param_param_wodefault_optional_unrestricted_adv_max -#end if - #if str($adv_opts.param_param_default_optional_unrestricted_adv_max): - -param_default_optional_unrestricted_adv_max $adv_opts.param_param_default_optional_unrestricted_adv_max -#end if - #if str($adv_opts.param_param_wodefault_mandatory_unrestricted_adv_minmax): - -param_wodefault_mandatory_unrestricted_adv_minmax $adv_opts.param_param_wodefault_mandatory_unrestricted_adv_minmax -#end if - #if str($adv_opts.param_param_default_mandatory_unrestricted_adv_minmax): - -param_default_mandatory_unrestricted_adv_minmax $adv_opts.param_param_default_mandatory_unrestricted_adv_minmax -#end if - #if str($adv_opts.param_param_wodefault_optional_unrestricted_adv_minmax): - -param_wodefault_optional_unrestricted_adv_minmax $adv_opts.param_param_wodefault_optional_unrestricted_adv_minmax -#end if - #if str($adv_opts.param_param_default_optional_unrestricted_adv_minmax): - -param_default_optional_unrestricted_adv_minmax $adv_opts.param_param_default_optional_unrestricted_adv_minmax -#end if -#end if -> $param_stdout -]]> - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - <![CDATA[Integer parameter tests. - - -For more information, visit http://docs.are/unavailable.html]]> - diff --git a/test-data/macros.xml b/test-data/macros.xml deleted file mode 100644 index 82907e42..00000000 --- a/test-data/macros.xml +++ /dev/null @@ -1,31 +0,0 @@ - - - - - requirement - - - - - - - - - - - doi:10.1186/1471-2105-9-163 - - - - - - - - - - - - - - - diff --git a/test-data/select.ctd b/test-data/select.ctd deleted file mode 100644 index f141dd12..00000000 --- a/test-data/select.ctd +++ /dev/null @@ -1,20 +0,0 @@ - - - - - - - - - - - - - - - - - - - - diff --git a/test-data/select.xml b/test-data/select.xml deleted file mode 100644 index d79bf0f2..00000000 --- a/test-data/select.xml +++ /dev/null @@ -1,134 +0,0 @@ - - - - - String parameter tests - - echo SelectTest - macros.xml - - - - - <![CDATA[echo SelectTest - -#if $param_param_wodefault_mandatory_restricted: - -param_wodefault_mandatory_restricted - #if " " in str($param_param_wodefault_mandatory_restricted): - "$param_param_wodefault_mandatory_restricted" - #else - $param_param_wodefault_mandatory_restricted - #end if -#end if -#if $param_param_default_mandatory_restricted: - -param_default_mandatory_restricted - #if " " in str($param_param_default_mandatory_restricted): - "$param_param_default_mandatory_restricted" - #else - $param_param_default_mandatory_restricted - #end if -#end if -#if $param_param_wodefault_optional_restricted: - -param_wodefault_optional_restricted - #if " " in str($param_param_wodefault_optional_restricted): - "$param_param_wodefault_optional_restricted" - #else - $param_param_wodefault_optional_restricted - #end if -#end if -#if $param_param_default_optional_restricted: - -param_default_optional_restricted - #if " " in str($param_param_default_optional_restricted): - "$param_param_default_optional_restricted" - #else - $param_param_default_optional_restricted - #end if -#end if -#if $adv_opts.adv_opts_selector=='advanced': - #if $adv_opts.param_param_wodefault_mandatory_restricted_adv: - -param_wodefault_mandatory_restricted_adv - #if " " in str($adv_opts.param_param_wodefault_mandatory_restricted_adv): - "$adv_opts.param_param_wodefault_mandatory_restricted_adv" - #else - $adv_opts.param_param_wodefault_mandatory_restricted_adv - #end if -#end if - #if $adv_opts.param_param_default_mandatory_restricted_adv: - -param_default_mandatory_restricted_adv - #if " " in str($adv_opts.param_param_default_mandatory_restricted_adv): - "$adv_opts.param_param_default_mandatory_restricted_adv" - #else - $adv_opts.param_param_default_mandatory_restricted_adv - #end if -#end if - #if $adv_opts.param_param_wodefault_optional_restricted_adv: - -param_wodefault_optional_restricted_adv - #if " " in str($adv_opts.param_param_wodefault_optional_restricted_adv): - "$adv_opts.param_param_wodefault_optional_restricted_adv" - #else - $adv_opts.param_param_wodefault_optional_restricted_adv - #end if -#end if - #if $adv_opts.param_param_default_optional_restricted_adv: - -param_default_optional_restricted_adv - #if " " in str($adv_opts.param_param_default_optional_restricted_adv): - "$adv_opts.param_param_default_optional_restricted_adv" - #else - $adv_opts.param_param_default_optional_restricted_adv - #end if -#end if -#end if -> $param_stdout -]]> - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - <![CDATA[String parameter tests. - - -For more information, visit http://docs.are/unavailable.html]]> - diff --git a/test-data/string.ctd b/test-data/string.ctd deleted file mode 100644 index 11517f36..00000000 --- a/test-data/string.ctd +++ /dev/null @@ -1,20 +0,0 @@ - - - - - - - - - - - - - - - - - - - - diff --git a/test-data/string.xml b/test-data/string.xml deleted file mode 100644 index 67109ad6..00000000 --- a/test-data/string.xml +++ /dev/null @@ -1,118 +0,0 @@ - - - - - String parameter tests - - echo StringTest - macros.xml - - - - - <![CDATA[echo StringTest - -#if str($param_param_wodefault_mandatory_unrestricted): - -param_wodefault_mandatory_unrestricted "$param_param_wodefault_mandatory_unrestricted" -#end if -#if str($param_param_default_mandatory_unrestricted): - -param_default_mandatory_unrestricted "$param_param_default_mandatory_unrestricted" -#end if -#if str($param_param_wodefault_optional_unrestricted): - -param_wodefault_optional_unrestricted "$param_param_wodefault_optional_unrestricted" -#end if -#if str($param_param_default_optional_unrestricted): - -param_default_optional_unrestricted "$param_param_default_optional_unrestricted" -#end if -#if $adv_opts.adv_opts_selector=='advanced': - #if str($adv_opts.param_param_wodefault_mandatory_unrestricted_adv): - -param_wodefault_mandatory_unrestricted_adv "$adv_opts.param_param_wodefault_mandatory_unrestricted_adv" -#end if - #if str($adv_opts.param_param_default_mandatory_unrestricted_adv): - -param_default_mandatory_unrestricted_adv "$adv_opts.param_param_default_mandatory_unrestricted_adv" -#end if - #if str($adv_opts.param_param_wodefault_optional_unrestricted_adv): - -param_wodefault_optional_unrestricted_adv "$adv_opts.param_param_wodefault_optional_unrestricted_adv" -#end if - #if str($adv_opts.param_param_default_optional_unrestricted_adv): - -param_default_optional_unrestricted_adv "$adv_opts.param_param_default_optional_unrestricted_adv" -#end if -#end if -> $param_stdout -]]> - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - <![CDATA[String parameter tests. - - -For more information, visit http://docs.are/unavailable.html]]> - From 81f54654f84b5542b32b4a14dbcf5fd937f12d1e Mon Sep 17 00:00:00 2001 From: M Bernt Date: Mon, 15 Jul 2019 14:17:51 +0200 Subject: [PATCH 004/121] fixed CTDopts branch name --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index ef5aea5c..be252d41 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,7 +24,7 @@ install: - conda config --add channels workflowconversion # - git clone https://github.com/WorkflowConversion/CTDopts $HOME/CTDopts - - git clone -b empty_numeric_defaults https://github.com/bernt-matthias/CTDopts $HOME/CTDopts + - git clone -b topic/empty_numeric_defaults https://github.com/bernt-matthias/CTDopts $HOME/CTDopts - cd $HOME/CTDopts - python setup.py install - conda install -c conda-forge lxml From 1ebc562f26b632502231da4995ff41f7eb6e84f3 Mon Sep 17 00:00:00 2001 From: M Bernt Date: Mon, 15 Jul 2019 14:30:34 +0200 Subject: [PATCH 005/121] improved indentation of float.ctd --- tests/test-data/float.ctd | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/tests/test-data/float.ctd b/tests/test-data/float.ctd index 2ab6c54d..ac578bda 100644 --- a/tests/test-data/float.ctd +++ b/tests/test-data/float.ctd @@ -3,45 +3,45 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - - + + - + - + - + - + From 79e516d5b36ded76dffc076e3337f6264e4a0f91 Mon Sep 17 00:00:00 2001 From: M Bernt Date: Mon, 15 Jul 2019 15:38:15 +0200 Subject: [PATCH 006/121] update test results --- tests/test-data/empty.xml | 8 ++--- tests/test-data/float.xml | 72 ++++++++++++++++++------------------- tests/test-data/integer.xml | 72 ++++++++++++++++++------------------- tests/test-data/select.xml | 24 ++++++------- tests/test-data/string.xml | 24 ++++++------- 5 files changed, 100 insertions(+), 100 deletions(-) diff --git a/tests/test-data/empty.xml b/tests/test-data/empty.xml index 07e534e7..e7daf119 100644 --- a/tests/test-data/empty.xml +++ b/tests/test-data/empty.xml @@ -10,16 +10,16 @@ - <![CDATA[echo StringTest + echo StringTest > $param_stdout -]]> + - <![CDATA[String parameter tests. + String parameter tests. -For more information, visit http://docs.are/unavailable.html]]> +For more information, visit http://docs.are/unavailable.html diff --git a/tests/test-data/float.xml b/tests/test-data/float.xml index f4fa7405..1fe902f6 100644 --- a/tests/test-data/float.xml +++ b/tests/test-data/float.xml @@ -10,7 +10,7 @@ - <![CDATA[echo FloatTest + echo FloatTest #if str($param_param_wodefault_mandatory_unrestricted): -param_wodefault_mandatory_unrestricted $param_param_wodefault_mandatory_unrestricted @@ -111,48 +111,48 @@ #end if #end if > $param_stdout -]]> + - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + - <![CDATA[Floateger parameter tests. + Floateger parameter tests. -For more information, visit http://docs.are/unavailable.html]]> +For more information, visit http://docs.are/unavailable.html diff --git a/tests/test-data/integer.xml b/tests/test-data/integer.xml index 4ae33636..e423d597 100644 --- a/tests/test-data/integer.xml +++ b/tests/test-data/integer.xml @@ -10,7 +10,7 @@ - <![CDATA[echo IntTest + echo IntTest #if str($param_param_wodefault_mandatory_unrestricted): -param_wodefault_mandatory_unrestricted $param_param_wodefault_mandatory_unrestricted @@ -111,48 +111,48 @@ #end if #end if > $param_stdout -]]> + - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + - <![CDATA[Integer parameter tests. + Integer parameter tests. -For more information, visit http://docs.are/unavailable.html]]> +For more information, visit http://docs.are/unavailable.html diff --git a/tests/test-data/select.xml b/tests/test-data/select.xml index d79bf0f2..437c68ba 100644 --- a/tests/test-data/select.xml +++ b/tests/test-data/select.xml @@ -10,7 +10,7 @@ - <![CDATA[echo SelectTest + echo SelectTest #if $param_param_wodefault_mandatory_restricted: -param_wodefault_mandatory_restricted @@ -79,45 +79,45 @@ #end if #end if > $param_stdout -]]> + - + - + - + - + - + - + - + - + @@ -127,8 +127,8 @@ - <![CDATA[String parameter tests. + String parameter tests. -For more information, visit http://docs.are/unavailable.html]]> +For more information, visit http://docs.are/unavailable.html diff --git a/tests/test-data/string.xml b/tests/test-data/string.xml index 67109ad6..90896ddb 100644 --- a/tests/test-data/string.xml +++ b/tests/test-data/string.xml @@ -10,7 +10,7 @@ - <![CDATA[echo StringTest + echo StringTest #if str($param_param_wodefault_mandatory_unrestricted): -param_wodefault_mandatory_unrestricted "$param_param_wodefault_mandatory_unrestricted" @@ -39,9 +39,9 @@ #end if #end if > $param_stdout -]]> + - + @@ -49,7 +49,7 @@ - + @@ -57,7 +57,7 @@ - + @@ -65,7 +65,7 @@ - + @@ -74,7 +74,7 @@ - + @@ -82,7 +82,7 @@ - + @@ -90,7 +90,7 @@ - + @@ -98,7 +98,7 @@ - + @@ -111,8 +111,8 @@ - <![CDATA[String parameter tests. + String parameter tests. -For more information, visit http://docs.are/unavailable.html]]> +For more information, visit http://docs.are/unavailable.html From 8d58f90617894df48ffa229c98e35161126ab48b Mon Sep 17 00:00:00 2001 From: M Bernt Date: Mon, 15 Jul 2019 16:39:25 +0200 Subject: [PATCH 007/121] added test files for bool, in/out files --- tests/test-data/bool.ctd | 17 +++++++ tests/test-data/bool.xml | 62 ++++++++++++++++++++++++++ tests/test-data/ifile.ctd | 28 ++++++++++++ tests/test-data/ifile.xml | 94 +++++++++++++++++++++++++++++++++++++++ tests/test-data/ofile.ctd | 28 ++++++++++++ tests/test-data/ofile.xml | 87 ++++++++++++++++++++++++++++++++++++ 6 files changed, 316 insertions(+) create mode 100644 tests/test-data/bool.ctd create mode 100644 tests/test-data/bool.xml create mode 100644 tests/test-data/ifile.ctd create mode 100644 tests/test-data/ifile.xml create mode 100644 tests/test-data/ofile.ctd create mode 100644 tests/test-data/ofile.xml diff --git a/tests/test-data/bool.ctd b/tests/test-data/bool.ctd new file mode 100644 index 00000000..a9f46f32 --- /dev/null +++ b/tests/test-data/bool.ctd @@ -0,0 +1,17 @@ + + + + + + + + + + + + + + + + + diff --git a/tests/test-data/bool.xml b/tests/test-data/bool.xml new file mode 100644 index 00000000..4139fbcc --- /dev/null +++ b/tests/test-data/bool.xml @@ -0,0 +1,62 @@ + + + + + String parameter tests + + echo StringTest + macros.xml + + + + + echo StringTest + +#if $param_param_true_optional_unrestricted: + -param_true_optional_unrestricted + #if " " in str($param_param_true_optional_unrestricted): + "$param_param_true_optional_unrestricted" + #else + $param_param_true_optional_unrestricted + #end if +#end if +#if $param_param_false_optional_unrestricted: + -param_false_optional_unrestricted +#end if +#if $adv_opts.adv_opts_selector=='advanced': + #if $adv_opts.param_param_true_optional_unrestricted_adv: + -param_true_optional_unrestricted_adv + #if " " in str($adv_opts.param_param_true_optional_unrestricted_adv): + "$adv_opts.param_param_true_optional_unrestricted_adv" + #else + $adv_opts.param_param_true_optional_unrestricted_adv + #end if +#end if + #if $adv_opts.param_param_false_optional_unrestricted_adv: + -param_false_optional_unrestricted_adv +#end if +#end if +> $param_stdout + + + + + + + + + + + + + + + + + + + String parameter tests. + + +For more information, visit http://docs.are/unavailable.html + diff --git a/tests/test-data/ifile.ctd b/tests/test-data/ifile.ctd new file mode 100644 index 00000000..3ff8d7d6 --- /dev/null +++ b/tests/test-data/ifile.ctd @@ -0,0 +1,28 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/test-data/ifile.xml b/tests/test-data/ifile.xml new file mode 100644 index 00000000..f1b379dc --- /dev/null +++ b/tests/test-data/ifile.xml @@ -0,0 +1,94 @@ + + + + + Input File parameter tests + + echo inFileTest + macros.xml + + + + + echo inFileTest + +#if str($param_param_wodefault_mandatory_unrestricted_noformat): + -param_wodefault_mandatory_unrestricted_noformat $param_param_wodefault_mandatory_unrestricted_noformat +#end if +#if str($param_param_default_mandatory_unrestricted_noformat): + -param_default_mandatory_unrestricted_noformat $param_param_default_mandatory_unrestricted_noformat +#end if +#if str($param_param_wodefault_optional_unrestricted_noformat): + -param_wodefault_optional_unrestricted_noformat $param_param_wodefault_optional_unrestricted_noformat +#end if +#if str($param_param_default_optional_unrestricted_noformat): + -param_default_optional_unrestricted_noformat $param_param_default_optional_unrestricted_noformat +#end if +#if str($param_param_wodefault_mandatory_unrestricted_multformat): + -param_wodefault_mandatory_unrestricted_multformat $param_param_wodefault_mandatory_unrestricted_multformat +#end if +#if str($param_param_default_mandatory_unrestricted_multformat): + -param_default_mandatory_unrestricted_multformat $param_param_default_mandatory_unrestricted_multformat +#end if +#if str($param_param_wodefault_optional_unrestricted_multformat): + -param_wodefault_optional_unrestricted_multformat $param_param_wodefault_optional_unrestricted_multformat +#end if +#if str($param_param_default_optional_unrestricted_multformat): + -param_default_optional_unrestricted_multformat $param_param_default_optional_unrestricted_multformat +#end if +#if $adv_opts.adv_opts_selector=='advanced': + #if str($adv_opts.param_param_wodefault_mandatory_unrestricted_adv_noformat): + -param_wodefault_mandatory_unrestricted_adv_noformat $adv_opts.param_param_wodefault_mandatory_unrestricted_adv_noformat +#end if + #if str($adv_opts.param_param_default_mandatory_unrestricted_adv_noformat): + -param_default_mandatory_unrestricted_adv_noformat $adv_opts.param_param_default_mandatory_unrestricted_adv_noformat +#end if + #if str($adv_opts.param_param_wodefault_optional_unrestricted_adv_noformat): + -param_wodefault_optional_unrestricted_adv_noformat $adv_opts.param_param_wodefault_optional_unrestricted_adv_noformat +#end if + #if str($adv_opts.param_param_default_optional_unrestricted_adv_noformat): + -param_default_optional_unrestricted_adv_noformat $adv_opts.param_param_default_optional_unrestricted_adv_noformat +#end if + #if str($adv_opts.param_param_wodefault_mandatory_unrestricted_adv_multformat): + -param_wodefault_mandatory_unrestricted_adv_multformat $adv_opts.param_param_wodefault_mandatory_unrestricted_adv_multformat +#end if + #if str($adv_opts.param_param_default_mandatory_unrestricted_adv_multformat): + -param_default_mandatory_unrestricted_adv_multformat $adv_opts.param_param_default_mandatory_unrestricted_adv_multformat +#end if + #if str($adv_opts.param_param_wodefault_optional_unrestricted_adv_multformat): + -param_wodefault_optional_unrestricted_adv_multformat $adv_opts.param_param_wodefault_optional_unrestricted_adv_multformat +#end if + #if str($adv_opts.param_param_default_optional_unrestricted_adv_multformat): + -param_default_optional_unrestricted_adv_multformat $adv_opts.param_param_default_optional_unrestricted_adv_multformat +#end if +#end if +> $param_stdout + + + + + + + + + + + + + + + + + + + + + + + + + Input File parameter tests. + + +For more information, visit http://docs.are/unavailable.html + diff --git a/tests/test-data/ofile.ctd b/tests/test-data/ofile.ctd new file mode 100644 index 00000000..f7fff8fa --- /dev/null +++ b/tests/test-data/ofile.ctd @@ -0,0 +1,28 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/test-data/ofile.xml b/tests/test-data/ofile.xml new file mode 100644 index 00000000..f2ec74e7 --- /dev/null +++ b/tests/test-data/ofile.xml @@ -0,0 +1,87 @@ + + + + + Input File parameter tests + + echo outFileTest + macros.xml + + + + + echo outFileTest + +#if str($param_param_wodefault_mandatory_unrestricted_snglformat): + -param_wodefault_mandatory_unrestricted_snglformat $param_param_wodefault_mandatory_unrestricted_snglformat +#end if +#if str($param_param_default_mandatory_unrestricted_snglformat): + -param_default_mandatory_unrestricted_snglformat $param_param_default_mandatory_unrestricted_snglformat +#end if +#if str($param_param_wodefault_optional_unrestricted_snglformat): + -param_wodefault_optional_unrestricted_snglformat $param_param_wodefault_optional_unrestricted_snglformat +#end if +#if str($param_param_default_optional_unrestricted_snglformat): + -param_default_optional_unrestricted_snglformat $param_param_default_optional_unrestricted_snglformat +#end if +#if str($param_param_wodefault_mandatory_unrestricted_adv_snglformat): + -param_wodefault_mandatory_unrestricted_adv_snglformat $param_param_wodefault_mandatory_unrestricted_adv_snglformat +#end if +#if str($param_param_default_mandatory_unrestricted_adv_snglformat): + -param_default_mandatory_unrestricted_adv_snglformat $param_param_default_mandatory_unrestricted_adv_snglformat +#end if +#if str($param_param_wodefault_optional_unrestricted_adv_snglformat): + -param_wodefault_optional_unrestricted_adv_snglformat $param_param_wodefault_optional_unrestricted_adv_snglformat +#end if +#if str($param_param_default_optional_unrestricted_adv_snglformat): + -param_default_optional_unrestricted_adv_snglformat $param_param_default_optional_unrestricted_adv_snglformat +#end if +#if str($param_param_wodefault_mandatory_unrestricted_multformat): + -param_wodefault_mandatory_unrestricted_multformat $param_param_wodefault_mandatory_unrestricted_multformat +#end if +#if str($param_param_default_mandatory_unrestricted_multformat): + -param_default_mandatory_unrestricted_multformat $param_param_default_mandatory_unrestricted_multformat +#end if +#if str($param_param_wodefault_optional_unrestricted_multformat): + -param_wodefault_optional_unrestricted_multformat $param_param_wodefault_optional_unrestricted_multformat +#end if +#if str($param_param_default_optional_unrestricted_multformat): + -param_default_optional_unrestricted_multformat $param_param_default_optional_unrestricted_multformat +#end if +#if str($param_param_wodefault_mandatory_unrestricted_adv_multformat): + -param_wodefault_mandatory_unrestricted_adv_multformat $param_param_wodefault_mandatory_unrestricted_adv_multformat +#end if +#if str($param_param_default_mandatory_unrestricted_adv_multformat): + -param_default_mandatory_unrestricted_adv_multformat $param_param_default_mandatory_unrestricted_adv_multformat +#end if +#if str($param_param_wodefault_optional_unrestricted_adv_multformat): + -param_wodefault_optional_unrestricted_adv_multformat $param_param_wodefault_optional_unrestricted_adv_multformat +#end if +#if str($param_param_default_optional_unrestricted_adv_multformat): + -param_default_optional_unrestricted_adv_multformat $param_param_default_optional_unrestricted_adv_multformat +#end if + + + + + + + + + + + + + + + + + + + + + Input File parameter tests. + + +For more information, visit http://docs.are/unavailable.html + From c8ec983a1adda0459579d89f7dbb757f1dae959f Mon Sep 17 00:00:00 2001 From: M Bernt Date: Tue, 16 Jul 2019 15:21:52 +0200 Subject: [PATCH 008/121] - added filetypes to test call - removed test file from OpenMS - fixed xsd link - extended select, i/o file test data --- tests/test-data/AccurateMassSearch.ctd | 44 -- tests/test-data/AccurateMassSearch.xml | 144 ---- tests/test-data/bool.ctd | 20 +- tests/test-data/bool.xml | 48 +- tests/test-data/empty.ctd | 2 +- tests/test-data/float.ctd | 2 +- tests/test-data/ifile.ctd | 62 +- tests/test-data/ifile.xml | 272 ++++++-- tests/test-data/integer.ctd | 66 +- tests/test-data/integer.xml | 192 +++--- tests/test-data/ofile.ctd | 50 +- tests/test-data/ofile.xml | 178 +++-- tests/test-data/repeat.ctd | 125 ++++ tests/test-data/repeat.xml | 866 +++++++++++++++++++++++++ tests/test-data/select.ctd | 51 +- tests/test-data/select.xml | 244 +++++-- tests/test-data/string.ctd | 2 +- tests/test_galaxy_cli.py | 3 +- 18 files changed, 1820 insertions(+), 551 deletions(-) delete mode 100644 tests/test-data/AccurateMassSearch.ctd delete mode 100644 tests/test-data/AccurateMassSearch.xml create mode 100644 tests/test-data/repeat.ctd create mode 100644 tests/test-data/repeat.xml diff --git a/tests/test-data/AccurateMassSearch.ctd b/tests/test-data/AccurateMassSearch.ctd deleted file mode 100644 index 6092d44e..00000000 --- a/tests/test-data/AccurateMassSearch.ctd +++ /dev/null @@ -1,44 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/tests/test-data/AccurateMassSearch.xml b/tests/test-data/AccurateMassSearch.xml deleted file mode 100644 index 14bff665..00000000 --- a/tests/test-data/AccurateMassSearch.xml +++ /dev/null @@ -1,144 +0,0 @@ - - - - - Match MS signals to molecules from a database by mass. - - AccurateMassSearch - macros.xml - - - - - AccurateMassSearch - -#if str($param_in): - -in $param_in -#end if -#if str($param_out): - -out $param_out -#end if -#if str($param_out_annotation): - -out_annotation $param_out_annotation -#end if -#if str($param_positive_adducts): - -positive_adducts $param_positive_adducts -#end if -#if str($param_negative_adducts): - -negative_adducts $param_negative_adducts -#end if -#if str($param_threads): - -threads $param_threads -#end if --db:mapping - #for token in $param_db_mapping: - $token - #end for --db:struct - #for token in $param_db_struct: - $token - #end for -#if str($param_algorithm_mass_error_value): - -algorithm:mass_error_value $param_algorithm_mass_error_value -#end if -#if $param_algorithm_mass_error_unit: - -algorithm:mass_error_unit - #if " " in str($param_algorithm_mass_error_unit): - "$param_algorithm_mass_error_unit" - #else - $param_algorithm_mass_error_unit - #end if -#end if -#if $param_algorithm_ionization_mode: - -algorithm:ionization_mode - #if " " in str($param_algorithm_ionization_mode): - "$param_algorithm_ionization_mode" - #else - $param_algorithm_ionization_mode - #end if -#end if -#if $param_algorithm_isotopic_similarity: - -algorithm:isotopic_similarity -#end if -#if $param_algorithm_keep_unidentified_masses: - -algorithm:keep_unidentified_masses -#end if -#if str($param_algorithm_mzTab_exportIsotopeIntensities): - -algorithm:mzTab:exportIsotopeIntensities $param_algorithm_mzTab_exportIsotopeIntensities -#end if -#if $adv_opts.adv_opts_selector=='advanced': - #if str($adv_opts.param_log): - -log "$adv_opts.param_log" -#end if - #if str($adv_opts.param_debug): - -debug $adv_opts.param_debug -#end if - #if $adv_opts.param_no_progress: - -no_progress -#end if - #if $adv_opts.param_force: - -force -#end if - #if $adv_opts.param_test: - -test -#end if -#end if - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Match MS signals to molecules from a database by mass. - - -For more information, visit http://ftp.mi.fu-berlin.de/OpenMS/release-documentation/html/UTILS_AccurateMassSearch.html - diff --git a/tests/test-data/bool.ctd b/tests/test-data/bool.ctd index a9f46f32..d59641e2 100644 --- a/tests/test-data/bool.ctd +++ b/tests/test-data/bool.ctd @@ -1,16 +1,16 @@ - - - - - + + + + + - + - - - - + + + + diff --git a/tests/test-data/bool.xml b/tests/test-data/bool.xml index 4139fbcc..30674b85 100644 --- a/tests/test-data/bool.xml +++ b/tests/test-data/bool.xml @@ -1,61 +1,61 @@ - - - String parameter tests + + + Bool parameter tests - echo StringTest + echo BoolTest macros.xml - echo StringTest + echo BoolTest -#if $param_param_true_optional_unrestricted: - -param_true_optional_unrestricted - #if " " in str($param_param_true_optional_unrestricted): - "$param_param_true_optional_unrestricted" +#if $param_true_optional_unrestricted: + -true_optional_unrestricted + #if " " in str($param_true_optional_unrestricted): + "$param_true_optional_unrestricted" #else - $param_param_true_optional_unrestricted + $param_true_optional_unrestricted #end if #end if -#if $param_param_false_optional_unrestricted: - -param_false_optional_unrestricted +#if $param_false_optional_unrestricted: + -false_optional_unrestricted #end if #if $adv_opts.adv_opts_selector=='advanced': - #if $adv_opts.param_param_true_optional_unrestricted_adv: - -param_true_optional_unrestricted_adv - #if " " in str($adv_opts.param_param_true_optional_unrestricted_adv): - "$adv_opts.param_param_true_optional_unrestricted_adv" + #if $adv_opts.param_true_optional_unrestricted_adv: + -true_optional_unrestricted_adv + #if " " in str($adv_opts.param_true_optional_unrestricted_adv): + "$adv_opts.param_true_optional_unrestricted_adv" #else - $adv_opts.param_param_true_optional_unrestricted_adv + $adv_opts.param_true_optional_unrestricted_adv #end if #end if - #if $adv_opts.param_param_false_optional_unrestricted_adv: - -param_false_optional_unrestricted_adv + #if $adv_opts.param_false_optional_unrestricted_adv: + -false_optional_unrestricted_adv #end if #end if > $param_stdout - + - + - + - + - String parameter tests. + Bool parameter tests. For more information, visit http://docs.are/unavailable.html diff --git a/tests/test-data/empty.ctd b/tests/test-data/empty.ctd index 316502e3..1ffc19de 100644 --- a/tests/test-data/empty.ctd +++ b/tests/test-data/empty.ctd @@ -2,7 +2,7 @@ - + diff --git a/tests/test-data/float.ctd b/tests/test-data/float.ctd index ac578bda..af537d83 100644 --- a/tests/test-data/float.ctd +++ b/tests/test-data/float.ctd @@ -2,7 +2,7 @@ - + diff --git a/tests/test-data/ifile.ctd b/tests/test-data/ifile.ctd index 3ff8d7d6..be957cb9 100644 --- a/tests/test-data/ifile.ctd +++ b/tests/test-data/ifile.ctd @@ -2,26 +2,54 @@ - + - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/test-data/ifile.xml b/tests/test-data/ifile.xml index f1b379dc..232b9fa0 100644 --- a/tests/test-data/ifile.xml +++ b/tests/test-data/ifile.xml @@ -12,76 +12,252 @@ echo inFileTest -#if str($param_param_wodefault_mandatory_unrestricted_noformat): - -param_wodefault_mandatory_unrestricted_noformat $param_param_wodefault_mandatory_unrestricted_noformat +#if str($param_mandatory_noformat): + -mandatory_noformat $param_mandatory_noformat #end if -#if str($param_param_default_mandatory_unrestricted_noformat): - -param_default_mandatory_unrestricted_noformat $param_param_default_mandatory_unrestricted_noformat +#if str($param_optional_noformat): + -optional_noformat $param_optional_noformat #end if -#if str($param_param_wodefault_optional_unrestricted_noformat): - -param_wodefault_optional_unrestricted_noformat $param_param_wodefault_optional_unrestricted_noformat +#if str($param_mandatory_oneformat): + -mandatory_oneformat $param_mandatory_oneformat #end if -#if str($param_param_default_optional_unrestricted_noformat): - -param_default_optional_unrestricted_noformat $param_param_default_optional_unrestricted_noformat +#if str($param_optional_oneformat): + -optional_oneformat $param_optional_oneformat #end if -#if str($param_param_wodefault_mandatory_unrestricted_multformat): - -param_wodefault_mandatory_unrestricted_multformat $param_param_wodefault_mandatory_unrestricted_multformat +#if str($param_mandatory_multformat): + -mandatory_multformat $param_mandatory_multformat #end if -#if str($param_param_default_mandatory_unrestricted_multformat): - -param_default_mandatory_unrestricted_multformat $param_param_default_mandatory_unrestricted_multformat +#if str($param_optional_multformat): + -optional_multformat $param_optional_multformat #end if -#if str($param_param_wodefault_optional_unrestricted_multformat): - -param_wodefault_optional_unrestricted_multformat $param_param_wodefault_optional_unrestricted_multformat +#if str($param_default_mandatory_noformat): + -default_mandatory_noformat $param_default_mandatory_noformat #end if -#if str($param_param_default_optional_unrestricted_multformat): - -param_default_optional_unrestricted_multformat $param_param_default_optional_unrestricted_multformat +#if str($param_default_optional_noformat): + -default_optional_noformat $param_default_optional_noformat #end if +#if str($param_default_mandatory_oneformat): + -default_mandatory_oneformat $param_default_mandatory_oneformat +#end if +#if str($param_default_optional_oneformat): + -default_optional_oneformat $param_default_optional_oneformat +#end if +#if str($param_default_mandatory_multformat): + -default_mandatory_multformat $param_default_mandatory_multformat +#end if +#if str($param_default_optional_multformat): + -default_optional_multformat $param_default_optional_multformat +#end if +-list_mandatory_noformat + #for token in $param_list_mandatory_noformat: + $token + #end for +-list_optional_noformat + #for token in $param_list_optional_noformat: + $token + #end for +-list_mandatory_oneformat + #for token in $param_list_mandatory_oneformat: + $token + #end for +-list_optional_oneformat + #for token in $param_list_optional_oneformat: + $token + #end for +-list_mandatory_multformat + #for token in $param_list_mandatory_multformat: + $token + #end for +-list_optional_multformat + #for token in $param_list_optional_multformat: + $token + #end for #if $adv_opts.adv_opts_selector=='advanced': - #if str($adv_opts.param_param_wodefault_mandatory_unrestricted_adv_noformat): - -param_wodefault_mandatory_unrestricted_adv_noformat $adv_opts.param_param_wodefault_mandatory_unrestricted_adv_noformat + #if str($adv_opts.param_mandatory_noformat_adv): + -mandatory_noformat_adv $adv_opts.param_mandatory_noformat_adv +#end if + #if str($adv_opts.param_optional_noformat_adv): + -optional_noformat_adv $adv_opts.param_optional_noformat_adv +#end if + #if str($adv_opts.param_mandatory_oneformat_adv): + -mandatory_oneformat_adv $adv_opts.param_mandatory_oneformat_adv +#end if + #if str($adv_opts.param_optional_oneformat_adv): + -optional_oneformat_adv $adv_opts.param_optional_oneformat_adv +#end if + #if str($adv_opts.param_mandatory_multformat_adv): + -mandatory_multformat_adv $adv_opts.param_mandatory_multformat_adv #end if - #if str($adv_opts.param_param_default_mandatory_unrestricted_adv_noformat): - -param_default_mandatory_unrestricted_adv_noformat $adv_opts.param_param_default_mandatory_unrestricted_adv_noformat + #if str($adv_opts.param_optional_multformat_adv): + -optional_multformat_adv $adv_opts.param_optional_multformat_adv #end if - #if str($adv_opts.param_param_wodefault_optional_unrestricted_adv_noformat): - -param_wodefault_optional_unrestricted_adv_noformat $adv_opts.param_param_wodefault_optional_unrestricted_adv_noformat + #if str($adv_opts.param_default_mandatory_noformat_adv): + -default_mandatory_noformat_adv $adv_opts.param_default_mandatory_noformat_adv #end if - #if str($adv_opts.param_param_default_optional_unrestricted_adv_noformat): - -param_default_optional_unrestricted_adv_noformat $adv_opts.param_param_default_optional_unrestricted_adv_noformat + #if str($adv_opts.param_default_optional_noformat_adv): + -default_optional_noformat_adv $adv_opts.param_default_optional_noformat_adv #end if - #if str($adv_opts.param_param_wodefault_mandatory_unrestricted_adv_multformat): - -param_wodefault_mandatory_unrestricted_adv_multformat $adv_opts.param_param_wodefault_mandatory_unrestricted_adv_multformat + #if str($adv_opts.param_default_mandatory_oneformat_adv): + -default_mandatory_oneformat_adv $adv_opts.param_default_mandatory_oneformat_adv #end if - #if str($adv_opts.param_param_default_mandatory_unrestricted_adv_multformat): - -param_default_mandatory_unrestricted_adv_multformat $adv_opts.param_param_default_mandatory_unrestricted_adv_multformat + #if str($adv_opts.param_default_optional_oneformat_adv): + -default_optional_oneformat_adv $adv_opts.param_default_optional_oneformat_adv #end if - #if str($adv_opts.param_param_wodefault_optional_unrestricted_adv_multformat): - -param_wodefault_optional_unrestricted_adv_multformat $adv_opts.param_param_wodefault_optional_unrestricted_adv_multformat + #if str($adv_opts.param_default_mandatory_multformat_adv): + -default_mandatory_multformat_adv $adv_opts.param_default_mandatory_multformat_adv #end if - #if str($adv_opts.param_param_default_optional_unrestricted_adv_multformat): - -param_default_optional_unrestricted_adv_multformat $adv_opts.param_param_default_optional_unrestricted_adv_multformat + #if str($adv_opts.param_default_optional_multformat_adv): + -default_optional_multformat_adv $adv_opts.param_default_optional_multformat_adv #end if + -list_mandatory_noformat_adv + #for token in $param_list_mandatory_noformat_adv: + $token + #end for + -list_optional_noformat_adv + #for token in $param_list_optional_noformat_adv: + $token + #end for + -list_mandatory_oneformat_adv + #for token in $param_list_mandatory_oneformat_adv: + $token + #end for + -list_optional_oneformat_adv + #for token in $param_list_optional_oneformat_adv: + $token + #end for + -list_mandatory_multformat_adv + #for token in $param_list_mandatory_multformat_adv: + $token + #end for + -list_optional_multformat_adv + #for token in $param_list_optional_multformat_adv: + $token + #end for #end if > $param_stdout - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/test-data/integer.ctd b/tests/test-data/integer.ctd index 356b58da..a358e3c6 100644 --- a/tests/test-data/integer.ctd +++ b/tests/test-data/integer.ctd @@ -2,45 +2,45 @@ - + - - - - - - - - + + + + + + + + - - - - - - - - + + + + + + + + - - - - - - - - + + + + + + + + - - - - - - - - + + + + + + + + diff --git a/tests/test-data/integer.xml b/tests/test-data/integer.xml index e423d597..bf20e9b2 100644 --- a/tests/test-data/integer.xml +++ b/tests/test-data/integer.xml @@ -12,140 +12,140 @@ echo IntTest -#if str($param_param_wodefault_mandatory_unrestricted): - -param_wodefault_mandatory_unrestricted $param_param_wodefault_mandatory_unrestricted +#if str($param_wodefault_mandatory_unrestricted): + -wodefault_mandatory_unrestricted $param_wodefault_mandatory_unrestricted #end if -#if str($param_param_default_mandatory_unrestricted): - -param_default_mandatory_unrestricted $param_param_default_mandatory_unrestricted +#if str($param_default_mandatory_unrestricted): + -default_mandatory_unrestricted $param_default_mandatory_unrestricted #end if -#if str($param_param_wodefault_optional_unrestricted): - -param_wodefault_optional_unrestricted $param_param_wodefault_optional_unrestricted +#if str($param_wodefault_optional_unrestricted): + -wodefault_optional_unrestricted $param_wodefault_optional_unrestricted #end if -#if str($param_param_default_optional_unrestricted): - -param_default_optional_unrestricted $param_param_default_optional_unrestricted +#if str($param_default_optional_unrestricted): + -default_optional_unrestricted $param_default_optional_unrestricted #end if -#if str($param_param_wodefault_mandatory_unrestricted_min): - -param_wodefault_mandatory_unrestricted_min $param_param_wodefault_mandatory_unrestricted_min +#if str($param_wodefault_mandatory_unrestricted_min): + -wodefault_mandatory_unrestricted_min $param_wodefault_mandatory_unrestricted_min #end if -#if str($param_param_default_mandatory_unrestricted_min): - -param_default_mandatory_unrestricted_min $param_param_default_mandatory_unrestricted_min +#if str($param_default_mandatory_unrestricted_min): + -default_mandatory_unrestricted_min $param_default_mandatory_unrestricted_min #end if -#if str($param_param_wodefault_optional_unrestricted_min): - -param_wodefault_optional_unrestricted_min $param_param_wodefault_optional_unrestricted_min +#if str($param_wodefault_optional_unrestricted_min): + -wodefault_optional_unrestricted_min $param_wodefault_optional_unrestricted_min #end if -#if str($param_param_default_optional_unrestricted_min): - -param_default_optional_unrestricted_min $param_param_default_optional_unrestricted_min +#if str($param_default_optional_unrestricted_min): + -default_optional_unrestricted_min $param_default_optional_unrestricted_min #end if -#if str($param_param_wodefault_mandatory_unrestricted_max): - -param_wodefault_mandatory_unrestricted_max $param_param_wodefault_mandatory_unrestricted_max +#if str($param_wodefault_mandatory_unrestricted_max): + -wodefault_mandatory_unrestricted_max $param_wodefault_mandatory_unrestricted_max #end if -#if str($param_param_default_mandatory_unrestricted_max): - -param_default_mandatory_unrestricted_max $param_param_default_mandatory_unrestricted_max +#if str($param_default_mandatory_unrestricted_max): + -default_mandatory_unrestricted_max $param_default_mandatory_unrestricted_max #end if -#if str($param_param_wodefault_optional_unrestricted_max): - -param_wodefault_optional_unrestricted_max $param_param_wodefault_optional_unrestricted_max +#if str($param_wodefault_optional_unrestricted_max): + -wodefault_optional_unrestricted_max $param_wodefault_optional_unrestricted_max #end if -#if str($param_param_default_optional_unrestricted_max): - -param_default_optional_unrestricted_max $param_param_default_optional_unrestricted_max +#if str($param_default_optional_unrestricted_max): + -default_optional_unrestricted_max $param_default_optional_unrestricted_max #end if -#if str($param_param_wodefault_mandatory_unrestricted_minmax): - -param_wodefault_mandatory_unrestricted_minmax $param_param_wodefault_mandatory_unrestricted_minmax +#if str($param_wodefault_mandatory_unrestricted_minmax): + -wodefault_mandatory_unrestricted_minmax $param_wodefault_mandatory_unrestricted_minmax #end if -#if str($param_param_default_mandatory_unrestricted_minmax): - -param_default_mandatory_unrestricted_minmax $param_param_default_mandatory_unrestricted_minmax +#if str($param_default_mandatory_unrestricted_minmax): + -default_mandatory_unrestricted_minmax $param_default_mandatory_unrestricted_minmax #end if -#if str($param_param_wodefault_optional_unrestricted_minmax): - -param_wodefault_optional_unrestricted_minmax $param_param_wodefault_optional_unrestricted_minmax +#if str($param_wodefault_optional_unrestricted_minmax): + -wodefault_optional_unrestricted_minmax $param_wodefault_optional_unrestricted_minmax #end if -#if str($param_param_default_optional_unrestricted_minmax): - -param_default_optional_unrestricted_minmax $param_param_default_optional_unrestricted_minmax +#if str($param_default_optional_unrestricted_minmax): + -default_optional_unrestricted_minmax $param_default_optional_unrestricted_minmax #end if #if $adv_opts.adv_opts_selector=='advanced': - #if str($adv_opts.param_param_wodefault_mandatory_unrestricted_adv): - -param_wodefault_mandatory_unrestricted_adv $adv_opts.param_param_wodefault_mandatory_unrestricted_adv + #if str($adv_opts.param_wodefault_mandatory_unrestricted_adv): + -wodefault_mandatory_unrestricted_adv $adv_opts.param_wodefault_mandatory_unrestricted_adv #end if - #if str($adv_opts.param_param_default_mandatory_unrestricted_adv): - -param_default_mandatory_unrestricted_adv $adv_opts.param_param_default_mandatory_unrestricted_adv + #if str($adv_opts.param_default_mandatory_unrestricted_adv): + -default_mandatory_unrestricted_adv $adv_opts.param_default_mandatory_unrestricted_adv #end if - #if str($adv_opts.param_param_wodefault_optional_unrestricted_adv): - -param_wodefault_optional_unrestricted_adv $adv_opts.param_param_wodefault_optional_unrestricted_adv + #if str($adv_opts.param_wodefault_optional_unrestricted_adv): + -wodefault_optional_unrestricted_adv $adv_opts.param_wodefault_optional_unrestricted_adv #end if - #if str($adv_opts.param_param_default_optional_unrestricted_adv): - -param_default_optional_unrestricted_adv $adv_opts.param_param_default_optional_unrestricted_adv + #if str($adv_opts.param_default_optional_unrestricted_adv): + -default_optional_unrestricted_adv $adv_opts.param_default_optional_unrestricted_adv #end if - #if str($adv_opts.param_param_wodefault_mandatory_unrestricted_adv_min): - -param_wodefault_mandatory_unrestricted_adv_min $adv_opts.param_param_wodefault_mandatory_unrestricted_adv_min + #if str($adv_opts.param_wodefault_mandatory_unrestricted_adv_min): + -wodefault_mandatory_unrestricted_adv_min $adv_opts.param_wodefault_mandatory_unrestricted_adv_min #end if - #if str($adv_opts.param_param_default_mandatory_unrestricted_adv_min): - -param_default_mandatory_unrestricted_adv_min $adv_opts.param_param_default_mandatory_unrestricted_adv_min + #if str($adv_opts.param_default_mandatory_unrestricted_adv_min): + -default_mandatory_unrestricted_adv_min $adv_opts.param_default_mandatory_unrestricted_adv_min #end if - #if str($adv_opts.param_param_wodefault_optional_unrestricted_adv_min): - -param_wodefault_optional_unrestricted_adv_min $adv_opts.param_param_wodefault_optional_unrestricted_adv_min + #if str($adv_opts.param_wodefault_optional_unrestricted_adv_min): + -wodefault_optional_unrestricted_adv_min $adv_opts.param_wodefault_optional_unrestricted_adv_min #end if - #if str($adv_opts.param_param_default_optional_unrestricted_adv_min): - -param_default_optional_unrestricted_adv_min $adv_opts.param_param_default_optional_unrestricted_adv_min + #if str($adv_opts.param_default_optional_unrestricted_adv_min): + -default_optional_unrestricted_adv_min $adv_opts.param_default_optional_unrestricted_adv_min #end if - #if str($adv_opts.param_param_wodefault_mandatory_unrestricted_adv_max): - -param_wodefault_mandatory_unrestricted_adv_max $adv_opts.param_param_wodefault_mandatory_unrestricted_adv_max + #if str($adv_opts.param_wodefault_mandatory_unrestricted_adv_max): + -wodefault_mandatory_unrestricted_adv_max $adv_opts.param_wodefault_mandatory_unrestricted_adv_max #end if - #if str($adv_opts.param_param_default_mandatory_unrestricted_adv_max): - -param_default_mandatory_unrestricted_adv_max $adv_opts.param_param_default_mandatory_unrestricted_adv_max + #if str($adv_opts.param_default_mandatory_unrestricted_adv_max): + -default_mandatory_unrestricted_adv_max $adv_opts.param_default_mandatory_unrestricted_adv_max #end if - #if str($adv_opts.param_param_wodefault_optional_unrestricted_adv_max): - -param_wodefault_optional_unrestricted_adv_max $adv_opts.param_param_wodefault_optional_unrestricted_adv_max + #if str($adv_opts.param_wodefault_optional_unrestricted_adv_max): + -wodefault_optional_unrestricted_adv_max $adv_opts.param_wodefault_optional_unrestricted_adv_max #end if - #if str($adv_opts.param_param_default_optional_unrestricted_adv_max): - -param_default_optional_unrestricted_adv_max $adv_opts.param_param_default_optional_unrestricted_adv_max + #if str($adv_opts.param_default_optional_unrestricted_adv_max): + -default_optional_unrestricted_adv_max $adv_opts.param_default_optional_unrestricted_adv_max #end if - #if str($adv_opts.param_param_wodefault_mandatory_unrestricted_adv_minmax): - -param_wodefault_mandatory_unrestricted_adv_minmax $adv_opts.param_param_wodefault_mandatory_unrestricted_adv_minmax + #if str($adv_opts.param_wodefault_mandatory_unrestricted_adv_minmax): + -wodefault_mandatory_unrestricted_adv_minmax $adv_opts.param_wodefault_mandatory_unrestricted_adv_minmax #end if - #if str($adv_opts.param_param_default_mandatory_unrestricted_adv_minmax): - -param_default_mandatory_unrestricted_adv_minmax $adv_opts.param_param_default_mandatory_unrestricted_adv_minmax + #if str($adv_opts.param_default_mandatory_unrestricted_adv_minmax): + -default_mandatory_unrestricted_adv_minmax $adv_opts.param_default_mandatory_unrestricted_adv_minmax #end if - #if str($adv_opts.param_param_wodefault_optional_unrestricted_adv_minmax): - -param_wodefault_optional_unrestricted_adv_minmax $adv_opts.param_param_wodefault_optional_unrestricted_adv_minmax + #if str($adv_opts.param_wodefault_optional_unrestricted_adv_minmax): + -wodefault_optional_unrestricted_adv_minmax $adv_opts.param_wodefault_optional_unrestricted_adv_minmax #end if - #if str($adv_opts.param_param_default_optional_unrestricted_adv_minmax): - -param_default_optional_unrestricted_adv_minmax $adv_opts.param_param_default_optional_unrestricted_adv_minmax + #if str($adv_opts.param_default_optional_unrestricted_adv_minmax): + -default_optional_unrestricted_adv_minmax $adv_opts.param_default_optional_unrestricted_adv_minmax #end if #end if > $param_stdout - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + diff --git a/tests/test-data/ofile.ctd b/tests/test-data/ofile.ctd index f7fff8fa..bed6a853 100644 --- a/tests/test-data/ofile.ctd +++ b/tests/test-data/ofile.ctd @@ -1,28 +1,40 @@ - + - + - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/test-data/ofile.xml b/tests/test-data/ofile.xml index f2ec74e7..f3678f7a 100644 --- a/tests/test-data/ofile.xml +++ b/tests/test-data/ofile.xml @@ -1,6 +1,6 @@ - + Input File parameter tests @@ -12,73 +12,153 @@ echo outFileTest -#if str($param_param_wodefault_mandatory_unrestricted_snglformat): - -param_wodefault_mandatory_unrestricted_snglformat $param_param_wodefault_mandatory_unrestricted_snglformat +#if str($param_wodefault_mandatory_noformat): + -wodefault_mandatory_noformat $param_wodefault_mandatory_noformat #end if -#if str($param_param_default_mandatory_unrestricted_snglformat): - -param_default_mandatory_unrestricted_snglformat $param_param_default_mandatory_unrestricted_snglformat +#if str($param_default_mandatory_noformat): + -default_mandatory_noformat $param_default_mandatory_noformat #end if -#if str($param_param_wodefault_optional_unrestricted_snglformat): - -param_wodefault_optional_unrestricted_snglformat $param_param_wodefault_optional_unrestricted_snglformat +#if str($param_wodefault_optional_noformat): + -wodefault_optional_noformat $param_wodefault_optional_noformat #end if -#if str($param_param_default_optional_unrestricted_snglformat): - -param_default_optional_unrestricted_snglformat $param_param_default_optional_unrestricted_snglformat +#if str($param_default_optional_noformat): + -default_optional_noformat $param_default_optional_noformat #end if -#if str($param_param_wodefault_mandatory_unrestricted_adv_snglformat): - -param_wodefault_mandatory_unrestricted_adv_snglformat $param_param_wodefault_mandatory_unrestricted_adv_snglformat +#if str($param_wodefault_mandatory_oneformat): + -wodefault_mandatory_oneformat $param_wodefault_mandatory_oneformat #end if -#if str($param_param_default_mandatory_unrestricted_adv_snglformat): - -param_default_mandatory_unrestricted_adv_snglformat $param_param_default_mandatory_unrestricted_adv_snglformat +#if str($param_default_mandatory_oneformat): + -default_mandatory_oneformat $param_default_mandatory_oneformat #end if -#if str($param_param_wodefault_optional_unrestricted_adv_snglformat): - -param_wodefault_optional_unrestricted_adv_snglformat $param_param_wodefault_optional_unrestricted_adv_snglformat +#if str($param_wodefault_optional_oneformat): + -wodefault_optional_oneformat $param_wodefault_optional_oneformat #end if -#if str($param_param_default_optional_unrestricted_adv_snglformat): - -param_default_optional_unrestricted_adv_snglformat $param_param_default_optional_unrestricted_adv_snglformat +#if str($param_default_optional_oneformat): + -default_optional_oneformat $param_default_optional_oneformat #end if -#if str($param_param_wodefault_mandatory_unrestricted_multformat): - -param_wodefault_mandatory_unrestricted_multformat $param_param_wodefault_mandatory_unrestricted_multformat +#if str($param_wodefault_mandatory_multformat): + -wodefault_mandatory_multformat $param_wodefault_mandatory_multformat #end if -#if str($param_param_default_mandatory_unrestricted_multformat): - -param_default_mandatory_unrestricted_multformat $param_param_default_mandatory_unrestricted_multformat +#if str($param_default_mandatory_multformat): + -default_mandatory_multformat $param_default_mandatory_multformat #end if -#if str($param_param_wodefault_optional_unrestricted_multformat): - -param_wodefault_optional_unrestricted_multformat $param_param_wodefault_optional_unrestricted_multformat +#if str($param_wodefault_optional_multformat): + -wodefault_optional_multformat $param_wodefault_optional_multformat #end if -#if str($param_param_default_optional_unrestricted_multformat): - -param_default_optional_unrestricted_multformat $param_param_default_optional_unrestricted_multformat +#if str($param_default_optional_multformat): + -default_optional_multformat $param_default_optional_multformat #end if -#if str($param_param_wodefault_mandatory_unrestricted_adv_multformat): - -param_wodefault_mandatory_unrestricted_adv_multformat $param_param_wodefault_mandatory_unrestricted_adv_multformat + +#if $rep_param_list_mandatory_noformat: +-list_mandatory_noformat + #for token in $rep_param_list_mandatory_noformat: + #if " " in str(token): + "$token.param_list_mandatory_noformat" + #else + $token.param_list_mandatory_noformat + #end if + #end for +#end if + +#if $rep_param_list_optional_noformat: +-list_optional_noformat + #for token in $rep_param_list_optional_noformat: + #if " " in str(token): + "$token.param_list_optional_noformat" + #else + $token.param_list_optional_noformat + #end if + #end for +#end if + +#if $rep_param_list_mandatory_oneformat: +-list_mandatory_oneformat + #for token in $rep_param_list_mandatory_oneformat: + #if " " in str(token): + "$token.param_list_mandatory_oneformat" + #else + $token.param_list_mandatory_oneformat + #end if + #end for +#end if + +#if $rep_param_list_optional_oneformat: +-list_optional_oneformat + #for token in $rep_param_list_optional_oneformat: + #if " " in str(token): + "$token.param_list_optional_oneformat" + #else + $token.param_list_optional_oneformat + #end if + #end for +#end if + +#if $rep_param_list_mandatory_multformat: +-list_mandatory_multformat + #for token in $rep_param_list_mandatory_multformat: + #if " " in str(token): + "$token.param_list_mandatory_multformat" + #else + $token.param_list_mandatory_multformat + #end if + #end for +#end if + +#if $rep_param_list_optional_multformat: +-list_optional_multformat + #for token in $rep_param_list_optional_multformat: + #if " " in str(token): + "$token.param_list_optional_multformat" + #else + $token.param_list_optional_multformat + #end if + #end for +#end if +#if str($param_mandatory_noformat_adv): + -mandatory_noformat_adv $param_mandatory_noformat_adv +#end if +#if str($param_optional_noformat_adv): + -optional_noformat_adv $param_optional_noformat_adv +#end if +#if str($param_mandatory_oneformat_adv): + -mandatory_oneformat_adv $param_mandatory_oneformat_adv #end if -#if str($param_param_default_mandatory_unrestricted_adv_multformat): - -param_default_mandatory_unrestricted_adv_multformat $param_param_default_mandatory_unrestricted_adv_multformat +#if str($param_optional_oneformat_adv): + -optional_oneformat_adv $param_optional_oneformat_adv #end if -#if str($param_param_wodefault_optional_unrestricted_adv_multformat): - -param_wodefault_optional_unrestricted_adv_multformat $param_param_wodefault_optional_unrestricted_adv_multformat +#if str($param_mandatory_multformat_adv): + -mandatory_multformat_adv $param_mandatory_multformat_adv #end if -#if str($param_param_default_optional_unrestricted_adv_multformat): - -param_default_optional_unrestricted_adv_multformat $param_param_default_optional_unrestricted_adv_multformat +#if str($param_optional_multformat_adv): + -optional_multformat_adv $param_optional_multformat_adv #end if - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + Input File parameter tests. diff --git a/tests/test-data/repeat.ctd b/tests/test-data/repeat.ctd new file mode 100644 index 00000000..225db4c3 --- /dev/null +++ b/tests/test-data/repeat.ctd @@ -0,0 +1,125 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/test-data/repeat.xml b/tests/test-data/repeat.xml new file mode 100644 index 00000000..9134942e --- /dev/null +++ b/tests/test-data/repeat.xml @@ -0,0 +1,866 @@ + + + + + Repeat parameter tests + + echo RepeatTest + macros.xml + + + + + echo RepeatTest + + +#if $rep_param_string_wodefault_optional: +-string_wodefault_optional + #for token in $rep_param_string_wodefault_optional: + #if " " in str(token): + "$token.param_string_wodefault_optional" + #else + $token.param_string_wodefault_optional + #end if + #end for +#end if + +#if $rep_param_string_wodefault_mandatory: +-string_wodefault_mandatory + #for token in $rep_param_string_wodefault_mandatory: + #if " " in str(token): + "$token.param_string_wodefault_mandatory" + #else + $token.param_string_wodefault_mandatory + #end if + #end for +#end if + +#if $rep_param_string_default_optional: +-string_default_optional + #for token in $rep_param_string_default_optional: + #if " " in str(token): + "$token.param_string_default_optional" + #else + $token.param_string_default_optional + #end if + #end for +#end if + +#if $rep_param_string_default_mandatory: +-string_default_mandatory + #for token in $rep_param_string_default_mandatory: + #if " " in str(token): + "$token.param_string_default_mandatory" + #else + $token.param_string_default_mandatory + #end if + #end for +#end if + +#if $rep_param_int_wodefault_optional: +-int_wodefault_optional + #for token in $rep_param_int_wodefault_optional: + #if " " in str(token): + "$token.param_int_wodefault_optional" + #else + $token.param_int_wodefault_optional + #end if + #end for +#end if + +#if $rep_param_int_wodefault_mandatory: +-int_wodefault_mandatory + #for token in $rep_param_int_wodefault_mandatory: + #if " " in str(token): + "$token.param_int_wodefault_mandatory" + #else + $token.param_int_wodefault_mandatory + #end if + #end for +#end if + +#if $rep_param_int_default_optional: +-int_default_optional + #for token in $rep_param_int_default_optional: + #if " " in str(token): + "$token.param_int_default_optional" + #else + $token.param_int_default_optional + #end if + #end for +#end if + +#if $rep_param_int_default_mandatory: +-int_default_mandatory + #for token in $rep_param_int_default_mandatory: + #if " " in str(token): + "$token.param_int_default_mandatory" + #else + $token.param_int_default_mandatory + #end if + #end for +#end if + +#if $rep_param_int_resticted_wodefault_optional: +-int_resticted_wodefault_optional + #for token in $rep_param_int_resticted_wodefault_optional: + #if " " in str(token): + "$token.param_int_resticted_wodefault_optional" + #else + $token.param_int_resticted_wodefault_optional + #end if + #end for +#end if + +#if $rep_param_int_resticted_wodefault_mandatory: +-int_resticted_wodefault_mandatory + #for token in $rep_param_int_resticted_wodefault_mandatory: + #if " " in str(token): + "$token.param_int_resticted_wodefault_mandatory" + #else + $token.param_int_resticted_wodefault_mandatory + #end if + #end for +#end if + +#if $rep_param_int_resticted_default_optional: +-int_resticted_default_optional + #for token in $rep_param_int_resticted_default_optional: + #if " " in str(token): + "$token.param_int_resticted_default_optional" + #else + $token.param_int_resticted_default_optional + #end if + #end for +#end if + +#if $rep_param_int_resticted_default_mandatory: +-int_resticted_default_mandatory + #for token in $rep_param_int_resticted_default_mandatory: + #if " " in str(token): + "$token.param_int_resticted_default_mandatory" + #else + $token.param_int_resticted_default_mandatory + #end if + #end for +#end if + +#if $rep_param_double_wodefault_optional: +-double_wodefault_optional + #for token in $rep_param_double_wodefault_optional: + #if " " in str(token): + "$token.param_double_wodefault_optional" + #else + $token.param_double_wodefault_optional + #end if + #end for +#end if + +#if $rep_param_double_wodefault_mandatory: +-double_wodefault_mandatory + #for token in $rep_param_double_wodefault_mandatory: + #if " " in str(token): + "$token.param_double_wodefault_mandatory" + #else + $token.param_double_wodefault_mandatory + #end if + #end for +#end if + +#if $rep_param_double_default_optional: +-double_default_optional + #for token in $rep_param_double_default_optional: + #if " " in str(token): + "$token.param_double_default_optional" + #else + $token.param_double_default_optional + #end if + #end for +#end if + +#if $rep_param_double_default_mandatory: +-double_default_mandatory + #for token in $rep_param_double_default_mandatory: + #if " " in str(token): + "$token.param_double_default_mandatory" + #else + $token.param_double_default_mandatory + #end if + #end for +#end if + +#if $rep_param_double_resticted_wodefault_optional: +-double_resticted_wodefault_optional + #for token in $rep_param_double_resticted_wodefault_optional: + #if " " in str(token): + "$token.param_double_resticted_wodefault_optional" + #else + $token.param_double_resticted_wodefault_optional + #end if + #end for +#end if + +#if $rep_param_double_resticted_wodefault_mandatory: +-double_resticted_wodefault_mandatory + #for token in $rep_param_double_resticted_wodefault_mandatory: + #if " " in str(token): + "$token.param_double_resticted_wodefault_mandatory" + #else + $token.param_double_resticted_wodefault_mandatory + #end if + #end for +#end if + +#if $rep_param_double_resticted_default_optional: +-double_resticted_default_optional + #for token in $rep_param_double_resticted_default_optional: + #if " " in str(token): + "$token.param_double_resticted_default_optional" + #else + $token.param_double_resticted_default_optional + #end if + #end for +#end if + +#if $rep_param_double_resticted_default_mandatory: +-double_resticted_default_mandatory + #for token in $rep_param_double_resticted_default_mandatory: + #if " " in str(token): + "$token.param_double_resticted_default_mandatory" + #else + $token.param_double_resticted_default_mandatory + #end if + #end for +#end if + +#if $rep_param_string_wodefault_optional_adv: +-string_wodefault_optional_adv + #for token in $rep_param_string_wodefault_optional_adv: + #if " " in str(token): + "$token.param_string_wodefault_optional_adv" + #else + $token.param_string_wodefault_optional_adv + #end if + #end for +#end if + +#if $rep_param_string_wodefault_mandatory_adv: +-string_wodefault_mandatory_adv + #for token in $rep_param_string_wodefault_mandatory_adv: + #if " " in str(token): + "$token.param_string_wodefault_mandatory_adv" + #else + $token.param_string_wodefault_mandatory_adv + #end if + #end for +#end if + +#if $rep_param_string_default_optional_adv: +-string_default_optional_adv + #for token in $rep_param_string_default_optional_adv: + #if " " in str(token): + "$token.param_string_default_optional_adv" + #else + $token.param_string_default_optional_adv + #end if + #end for +#end if + +#if $rep_param_string_default_mandatory_adv: +-string_default_mandatory_adv + #for token in $rep_param_string_default_mandatory_adv: + #if " " in str(token): + "$token.param_string_default_mandatory_adv" + #else + $token.param_string_default_mandatory_adv + #end if + #end for +#end if + +#if $rep_param_int_wodefault_optional_adv: +-int_wodefault_optional_adv + #for token in $rep_param_int_wodefault_optional_adv: + #if " " in str(token): + "$token.param_int_wodefault_optional_adv" + #else + $token.param_int_wodefault_optional_adv + #end if + #end for +#end if + +#if $rep_param_int_wodefault_mandatory_adv: +-int_wodefault_mandatory_adv + #for token in $rep_param_int_wodefault_mandatory_adv: + #if " " in str(token): + "$token.param_int_wodefault_mandatory_adv" + #else + $token.param_int_wodefault_mandatory_adv + #end if + #end for +#end if + +#if $rep_param_int_default_optional_adv: +-int_default_optional_adv + #for token in $rep_param_int_default_optional_adv: + #if " " in str(token): + "$token.param_int_default_optional_adv" + #else + $token.param_int_default_optional_adv + #end if + #end for +#end if + +#if $rep_param_int_default_mandatory_adv: +-int_default_mandatory_adv + #for token in $rep_param_int_default_mandatory_adv: + #if " " in str(token): + "$token.param_int_default_mandatory_adv" + #else + $token.param_int_default_mandatory_adv + #end if + #end for +#end if + +#if $rep_param_int_resticted_wodefault_optional_adv: +-int_resticted_wodefault_optional_adv + #for token in $rep_param_int_resticted_wodefault_optional_adv: + #if " " in str(token): + "$token.param_int_resticted_wodefault_optional_adv" + #else + $token.param_int_resticted_wodefault_optional_adv + #end if + #end for +#end if + +#if $rep_param_int_resticted_wodefault_mandatory_adv: +-int_resticted_wodefault_mandatory_adv + #for token in $rep_param_int_resticted_wodefault_mandatory_adv: + #if " " in str(token): + "$token.param_int_resticted_wodefault_mandatory_adv" + #else + $token.param_int_resticted_wodefault_mandatory_adv + #end if + #end for +#end if + +#if $rep_param_int_resticted_default_optional_adv: +-int_resticted_default_optional_adv + #for token in $rep_param_int_resticted_default_optional_adv: + #if " " in str(token): + "$token.param_int_resticted_default_optional_adv" + #else + $token.param_int_resticted_default_optional_adv + #end if + #end for +#end if + +#if $rep_param_int_resticted_default_mandatory_adv: +-int_resticted_default_mandatory_adv + #for token in $rep_param_int_resticted_default_mandatory_adv: + #if " " in str(token): + "$token.param_int_resticted_default_mandatory_adv" + #else + $token.param_int_resticted_default_mandatory_adv + #end if + #end for +#end if + +#if $rep_param_double_wodefault_optional_adv: +-double_wodefault_optional_adv + #for token in $rep_param_double_wodefault_optional_adv: + #if " " in str(token): + "$token.param_double_wodefault_optional_adv" + #else + $token.param_double_wodefault_optional_adv + #end if + #end for +#end if + +#if $rep_param_double_wodefault_mandatory_adv: +-double_wodefault_mandatory_adv + #for token in $rep_param_double_wodefault_mandatory_adv: + #if " " in str(token): + "$token.param_double_wodefault_mandatory_adv" + #else + $token.param_double_wodefault_mandatory_adv + #end if + #end for +#end if + +#if $rep_param_double_default_optional_adv: +-double_default_optional_adv + #for token in $rep_param_double_default_optional_adv: + #if " " in str(token): + "$token.param_double_default_optional_adv" + #else + $token.param_double_default_optional_adv + #end if + #end for +#end if + +#if $rep_param_double_default_mandatory_adv: +-double_default_mandatory_adv + #for token in $rep_param_double_default_mandatory_adv: + #if " " in str(token): + "$token.param_double_default_mandatory_adv" + #else + $token.param_double_default_mandatory_adv + #end if + #end for +#end if + +#if $rep_param_double_resticted_wodefault_optional_adv: +-double_resticted_wodefault_optional_adv + #for token in $rep_param_double_resticted_wodefault_optional_adv: + #if " " in str(token): + "$token.param_double_resticted_wodefault_optional_adv" + #else + $token.param_double_resticted_wodefault_optional_adv + #end if + #end for +#end if + +#if $rep_param_double_resticted_wodefault_mandatory_adv: +-double_resticted_wodefault_mandatory_adv + #for token in $rep_param_double_resticted_wodefault_mandatory_adv: + #if " " in str(token): + "$token.param_double_resticted_wodefault_mandatory_adv" + #else + $token.param_double_resticted_wodefault_mandatory_adv + #end if + #end for +#end if + +#if $rep_param_double_resticted_default_optional_adv: +-double_resticted_default_optional_adv + #for token in $rep_param_double_resticted_default_optional_adv: + #if " " in str(token): + "$token.param_double_resticted_default_optional_adv" + #else + $token.param_double_resticted_default_optional_adv + #end if + #end for +#end if + +#if $rep_param_double_resticted_default_mandatory_adv: +-double_resticted_default_mandatory_adv + #for token in $rep_param_double_resticted_default_mandatory_adv: + #if " " in str(token): + "$token.param_double_resticted_default_mandatory_adv" + #else + $token.param_double_resticted_default_mandatory_adv + #end if + #end for +#end if +> $param_stdout + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Repeat parameter tests. + + +For more information, visit http://docs.are/unavailable.html + diff --git a/tests/test-data/select.ctd b/tests/test-data/select.ctd index f141dd12..77048022 100644 --- a/tests/test-data/select.ctd +++ b/tests/test-data/select.ctd @@ -1,19 +1,44 @@ - - - - + + + + - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/test-data/select.xml b/tests/test-data/select.xml index 437c68ba..7e243ccd 100644 --- a/tests/test-data/select.xml +++ b/tests/test-data/select.xml @@ -2,7 +2,7 @@ - String parameter tests + Select parameter tests echo SelectTest macros.xml @@ -12,112 +12,256 @@ echo SelectTest -#if $param_param_wodefault_mandatory_restricted: - -param_wodefault_mandatory_restricted - #if " " in str($param_param_wodefault_mandatory_restricted): - "$param_param_wodefault_mandatory_restricted" +#if $param_wodefault_mandatory: + -wodefault_mandatory + #if " " in str($param_wodefault_mandatory): + "$param_wodefault_mandatory" #else - $param_param_wodefault_mandatory_restricted + $param_wodefault_mandatory #end if #end if -#if $param_param_default_mandatory_restricted: - -param_default_mandatory_restricted - #if " " in str($param_param_default_mandatory_restricted): - "$param_param_default_mandatory_restricted" +#if $param_default_mandatory: + -default_mandatory + #if " " in str($param_default_mandatory): + "$param_default_mandatory" #else - $param_param_default_mandatory_restricted + $param_default_mandatory #end if #end if -#if $param_param_wodefault_optional_restricted: - -param_wodefault_optional_restricted - #if " " in str($param_param_wodefault_optional_restricted): - "$param_param_wodefault_optional_restricted" +#if $param_wodefault_optional: + -wodefault_optional + #if " " in str($param_wodefault_optional): + "$param_wodefault_optional" #else - $param_param_wodefault_optional_restricted + $param_wodefault_optional #end if #end if -#if $param_param_default_optional_restricted: - -param_default_optional_restricted - #if " " in str($param_param_default_optional_restricted): - "$param_param_default_optional_restricted" +#if $param_default_optional: + -default_optional + #if " " in str($param_default_optional): + "$param_default_optional" #else - $param_param_default_optional_restricted + $param_default_optional #end if #end if + +#if $rep_param_list_wodefault_optional: +-list_wodefault_optional + #for token in $rep_param_list_wodefault_optional: + #if " " in str(token): + "$token.param_list_wodefault_optional" + #else + $token.param_list_wodefault_optional + #end if + #end for +#end if + +#if $rep_param_list_wodefault_mandatory: +-list_wodefault_mandatory + #for token in $rep_param_list_wodefault_mandatory: + #if " " in str(token): + "$token.param_list_wodefault_mandatory" + #else + $token.param_list_wodefault_mandatory + #end if + #end for +#end if + +#if $rep_param_list_default_optional: +-list_default_optional + #for token in $rep_param_list_default_optional: + #if " " in str(token): + "$token.param_list_default_optional" + #else + $token.param_list_default_optional + #end if + #end for +#end if + +#if $rep_param_list_default_mandatory: +-list_default_mandatory + #for token in $rep_param_list_default_mandatory: + #if " " in str(token): + "$token.param_list_default_mandatory" + #else + $token.param_list_default_mandatory + #end if + #end for +#end if + +#if $rep_param_list_wodefault_optional_adv: +-list_wodefault_optional_adv + #for token in $rep_param_list_wodefault_optional_adv: + #if " " in str(token): + "$token.param_list_wodefault_optional_adv" + #else + $token.param_list_wodefault_optional_adv + #end if + #end for +#end if + +#if $rep_param_list_wodefault_mandatory_adv: +-list_wodefault_mandatory_adv + #for token in $rep_param_list_wodefault_mandatory_adv: + #if " " in str(token): + "$token.param_list_wodefault_mandatory_adv" + #else + $token.param_list_wodefault_mandatory_adv + #end if + #end for +#end if + +#if $rep_param_list_default_optional_adv: +-list_default_optional_adv + #for token in $rep_param_list_default_optional_adv: + #if " " in str(token): + "$token.param_list_default_optional_adv" + #else + $token.param_list_default_optional_adv + #end if + #end for +#end if + +#if $rep_param_list_default_mandatory_adv: +-list_default_mandatory_adv + #for token in $rep_param_list_default_mandatory_adv: + #if " " in str(token): + "$token.param_list_default_mandatory_adv" + #else + $token.param_list_default_mandatory_adv + #end if + #end for +#end if #if $adv_opts.adv_opts_selector=='advanced': - #if $adv_opts.param_param_wodefault_mandatory_restricted_adv: - -param_wodefault_mandatory_restricted_adv - #if " " in str($adv_opts.param_param_wodefault_mandatory_restricted_adv): - "$adv_opts.param_param_wodefault_mandatory_restricted_adv" + #if $adv_opts.param_wodefault_mandatory_adv: + -wodefault_mandatory_adv + #if " " in str($adv_opts.param_wodefault_mandatory_adv): + "$adv_opts.param_wodefault_mandatory_adv" #else - $adv_opts.param_param_wodefault_mandatory_restricted_adv + $adv_opts.param_wodefault_mandatory_adv #end if #end if - #if $adv_opts.param_param_default_mandatory_restricted_adv: - -param_default_mandatory_restricted_adv - #if " " in str($adv_opts.param_param_default_mandatory_restricted_adv): - "$adv_opts.param_param_default_mandatory_restricted_adv" + #if $adv_opts.param_default_mandatory_adv: + -default_mandatory_adv + #if " " in str($adv_opts.param_default_mandatory_adv): + "$adv_opts.param_default_mandatory_adv" #else - $adv_opts.param_param_default_mandatory_restricted_adv + $adv_opts.param_default_mandatory_adv #end if #end if - #if $adv_opts.param_param_wodefault_optional_restricted_adv: - -param_wodefault_optional_restricted_adv - #if " " in str($adv_opts.param_param_wodefault_optional_restricted_adv): - "$adv_opts.param_param_wodefault_optional_restricted_adv" + #if $adv_opts.param_wodefault_optional_adv: + -wodefault_optional_adv + #if " " in str($adv_opts.param_wodefault_optional_adv): + "$adv_opts.param_wodefault_optional_adv" #else - $adv_opts.param_param_wodefault_optional_restricted_adv + $adv_opts.param_wodefault_optional_adv #end if #end if - #if $adv_opts.param_param_default_optional_restricted_adv: - -param_default_optional_restricted_adv - #if " " in str($adv_opts.param_param_default_optional_restricted_adv): - "$adv_opts.param_param_default_optional_restricted_adv" + #if $adv_opts.param_default_optional_adv: + -default_optional_adv + #if " " in str($adv_opts.param_default_optional_adv): + "$adv_opts.param_default_optional_adv" #else - $adv_opts.param_param_default_optional_restricted_adv + $adv_opts.param_default_optional_adv #end if #end if #end if > $param_stdout - + - + - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - + - + - + @@ -127,7 +271,7 @@ - String parameter tests. + Select parameter tests. For more information, visit http://docs.are/unavailable.html diff --git a/tests/test-data/string.ctd b/tests/test-data/string.ctd index 11517f36..5c0e5fc7 100644 --- a/tests/test-data/string.ctd +++ b/tests/test-data/string.ctd @@ -2,7 +2,7 @@ - + diff --git a/tests/test_galaxy_cli.py b/tests/test_galaxy_cli.py index 51de457a..9659931f 100644 --- a/tests/test_galaxy_cli.py +++ b/tests/test_galaxy_cli.py @@ -16,12 +16,13 @@ class GalaxyCliTestCase(unittest.TestCase): def _compare_cli_output(self, fileprefix): in_pth = to_test_data('{}.ctd'.format(fileprefix)) macro_pth = to_test_data('macros.xml') + ftypes_pth = to_test_data('filetypes.txt') tmp = tempfile.mkdtemp() out_file = os.path.join(tmp, '{}.xml'.format(fileprefix)) #out_file = to_test_data('{}.xml'.format(fileprefix)) - cmd = ['CTDConverter', 'galaxy', '-i', in_pth, '-o', out_file, '-m', macro_pth, '-b', 'version'] + cmd = ['CTDConverter', 'galaxy', '-i', in_pth, '-o', out_file, '-f', ftypes_pth, '-m', macro_pth, '-b', 'version'] print(cmd) popen = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) From a2efdb18874678e33cb02a9d12a2e99c087d94f1 Mon Sep 17 00:00:00 2001 From: M Bernt Date: Tue, 16 Jul 2019 15:31:26 +0200 Subject: [PATCH 009/121] added filetypes.txt --- tests/test-data/filetypes.txt | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 tests/test-data/filetypes.txt diff --git a/tests/test-data/filetypes.txt b/tests/test-data/filetypes.txt new file mode 100644 index 00000000..ae3d61a2 --- /dev/null +++ b/tests/test-data/filetypes.txt @@ -0,0 +1,4 @@ +# CTD type # Galaxy type # Long Galaxy data type # Mimetype +ext txt galaxy.datatypes.data:Text +eyt txt galaxy.datatypes.data:Text +ezt txt galaxy.datatypes.data:Text From c3947af71357f570a6bd3351550542f4f4577440 Mon Sep 17 00:00:00 2001 From: M Bernt Date: Tue, 16 Jul 2019 16:06:03 +0200 Subject: [PATCH 010/121] advanced repeats were not advanced --- tests/test-data/repeat.ctd | 40 ++-- tests/test-data/repeat.xml | 444 +++++++++++++++++++------------------ 2 files changed, 244 insertions(+), 240 deletions(-) diff --git a/tests/test-data/repeat.ctd b/tests/test-data/repeat.ctd index 225db4c3..75d90af5 100644 --- a/tests/test-data/repeat.ctd +++ b/tests/test-data/repeat.ctd @@ -63,58 +63,58 @@ - - - + + + - + - - - + + + - + - - - + + + - + - - - + + + - + - - - + + + - + diff --git a/tests/test-data/repeat.xml b/tests/test-data/repeat.xml index 9134942e..028177a3 100644 --- a/tests/test-data/repeat.xml +++ b/tests/test-data/repeat.xml @@ -232,7 +232,8 @@ #end if #end for #end if - +#if $adv_opts.adv_opts_selector=='advanced': + #if $rep_param_string_wodefault_optional_adv: -string_wodefault_optional_adv #for token in $rep_param_string_wodefault_optional_adv: @@ -243,7 +244,7 @@ #end if #end for #end if - + #if $rep_param_string_wodefault_mandatory_adv: -string_wodefault_mandatory_adv #for token in $rep_param_string_wodefault_mandatory_adv: @@ -254,7 +255,7 @@ #end if #end for #end if - + #if $rep_param_string_default_optional_adv: -string_default_optional_adv #for token in $rep_param_string_default_optional_adv: @@ -265,7 +266,7 @@ #end if #end for #end if - + #if $rep_param_string_default_mandatory_adv: -string_default_mandatory_adv #for token in $rep_param_string_default_mandatory_adv: @@ -276,7 +277,7 @@ #end if #end for #end if - + #if $rep_param_int_wodefault_optional_adv: -int_wodefault_optional_adv #for token in $rep_param_int_wodefault_optional_adv: @@ -287,7 +288,7 @@ #end if #end for #end if - + #if $rep_param_int_wodefault_mandatory_adv: -int_wodefault_mandatory_adv #for token in $rep_param_int_wodefault_mandatory_adv: @@ -298,7 +299,7 @@ #end if #end for #end if - + #if $rep_param_int_default_optional_adv: -int_default_optional_adv #for token in $rep_param_int_default_optional_adv: @@ -309,7 +310,7 @@ #end if #end for #end if - + #if $rep_param_int_default_mandatory_adv: -int_default_mandatory_adv #for token in $rep_param_int_default_mandatory_adv: @@ -320,7 +321,7 @@ #end if #end for #end if - + #if $rep_param_int_resticted_wodefault_optional_adv: -int_resticted_wodefault_optional_adv #for token in $rep_param_int_resticted_wodefault_optional_adv: @@ -331,7 +332,7 @@ #end if #end for #end if - + #if $rep_param_int_resticted_wodefault_mandatory_adv: -int_resticted_wodefault_mandatory_adv #for token in $rep_param_int_resticted_wodefault_mandatory_adv: @@ -342,7 +343,7 @@ #end if #end for #end if - + #if $rep_param_int_resticted_default_optional_adv: -int_resticted_default_optional_adv #for token in $rep_param_int_resticted_default_optional_adv: @@ -353,7 +354,7 @@ #end if #end for #end if - + #if $rep_param_int_resticted_default_mandatory_adv: -int_resticted_default_mandatory_adv #for token in $rep_param_int_resticted_default_mandatory_adv: @@ -364,7 +365,7 @@ #end if #end for #end if - + #if $rep_param_double_wodefault_optional_adv: -double_wodefault_optional_adv #for token in $rep_param_double_wodefault_optional_adv: @@ -375,7 +376,7 @@ #end if #end for #end if - + #if $rep_param_double_wodefault_mandatory_adv: -double_wodefault_mandatory_adv #for token in $rep_param_double_wodefault_mandatory_adv: @@ -386,7 +387,7 @@ #end if #end for #end if - + #if $rep_param_double_default_optional_adv: -double_default_optional_adv #for token in $rep_param_double_default_optional_adv: @@ -397,7 +398,7 @@ #end if #end for #end if - + #if $rep_param_double_default_mandatory_adv: -double_default_mandatory_adv #for token in $rep_param_double_default_mandatory_adv: @@ -408,7 +409,7 @@ #end if #end for #end if - + #if $rep_param_double_resticted_wodefault_optional_adv: -double_resticted_wodefault_optional_adv #for token in $rep_param_double_resticted_wodefault_optional_adv: @@ -419,7 +420,7 @@ #end if #end for #end if - + #if $rep_param_double_resticted_wodefault_mandatory_adv: -double_resticted_wodefault_mandatory_adv #for token in $rep_param_double_resticted_wodefault_mandatory_adv: @@ -430,7 +431,7 @@ #end if #end for #end if - + #if $rep_param_double_resticted_default_optional_adv: -double_resticted_default_optional_adv #for token in $rep_param_double_resticted_default_optional_adv: @@ -441,7 +442,7 @@ #end if #end for #end if - + #if $rep_param_double_resticted_default_mandatory_adv: -double_resticted_default_mandatory_adv #for token in $rep_param_double_resticted_default_mandatory_adv: @@ -452,6 +453,7 @@ #end if #end for #end if +#end if > $param_stdout @@ -655,206 +657,208 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From 1376b84d80f8240ea7639ef1da25025d22f57ce8 Mon Sep 17 00:00:00 2001 From: M Bernt Date: Tue, 16 Jul 2019 16:14:59 +0200 Subject: [PATCH 011/121] add missing tests --- tests/test_galaxy_cli.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/tests/test_galaxy_cli.py b/tests/test_galaxy_cli.py index 9659931f..d87c9ede 100644 --- a/tests/test_galaxy_cli.py +++ b/tests/test_galaxy_cli.py @@ -41,8 +41,8 @@ def _compare_cli_output(self, fileprefix): for i in range(0, len(new_l)): self.assertEqual(new_l[i].rstrip(), old_l[i].rstrip()) - def test_galaxy_cli_full_file_ctd(self): - self._compare_cli_output('AccurateMassSearch') + def test_galaxy_cli_bool_ctd(self): + self._compare_cli_output('bool') def test_galaxy_cli_empty_ctd(self): self._compare_cli_output('empty') @@ -53,12 +53,20 @@ def test_galaxy_cli_float_ctd(self): def test_galaxy_cli_integer_ctd(self): self._compare_cli_output('integer') + def test_galaxy_cli_repeat_ctd(self): + self._compare_cli_output('repeat') + def test_galaxy_cli_select_ctd(self): self._compare_cli_output('select') def test_galaxy_cli_string_ctd(self): self._compare_cli_output('string') + def test_galaxy_cli_ifile_ctd(self): + self._compare_cli_output('ifile') + + def test_galaxy_cli_ofile_ctd(self): + self._compare_cli_output('ofile') if __name__ == '__main__': From 616074dc5b65cdc90a70ef6ef1bcd30a80a398d9 Mon Sep 17 00:00:00 2001 From: M Bernt Date: Sat, 6 Jul 2019 15:47:29 +0200 Subject: [PATCH 012/121] make help and command CDATA --- galaxy/converter.py | 160 +++++++++++++++++++++++++++--------- tests/test-data/bool.xml | 10 +-- tests/test-data/empty.xml | 10 +-- tests/test-data/float.xml | 10 +-- tests/test-data/ifile.xml | 10 +-- tests/test-data/integer.xml | 10 +-- tests/test-data/ofile.xml | 8 +- tests/test-data/repeat.xml | 10 +-- tests/test-data/select.xml | 10 +-- tests/test-data/string.xml | 10 +-- 10 files changed, 167 insertions(+), 81 deletions(-) diff --git a/galaxy/converter.py b/galaxy/converter.py index 08c2f422..1cb5dcee 100755 --- a/galaxy/converter.py +++ b/galaxy/converter.py @@ -6,7 +6,7 @@ from collections import OrderedDict from string import strip from lxml import etree -from lxml.etree import SubElement, Element, ElementTree, ParseError, parse +from lxml.etree import CDATA, SubElement, Element, ElementTree, ParseError, parse from common import utils, logger from common.exceptions import ApplicationException, InvalidModelException @@ -39,6 +39,10 @@ def __init__(self, extension, galaxy_extension=None, galaxy_type=None, mimetype= def add_specific_args(parser): + """ + add command line arguments specific for galaxy tool generation + @param parser an instance of ArgumentParser + """ parser.add_argument("-f", "--formats-file", dest="formats_file", help="File containing the supported file formats. Run with '-h' or '--help' to see a " "brief example on the layout of this file.", default=None, required=False) @@ -70,41 +74,49 @@ def add_specific_args(parser): def convert_models(args, parsed_ctds): - # validate and prepare the passed arguments - validate_and_prepare_args(args) + """ + main conversion function - # extract the names of the macros and check that we have found the ones we need - macros_to_expand = parse_macros_files(args.macros_files) + @param args command line arguments + @param parsed_ctds the ctds + """ + # validate and prepare the passed arguments + validate_and_prepare_args(args) - # parse the given supported file-formats file - supported_file_formats = parse_file_formats(args.formats_file) + # extract the names of the macros and check that we have found the ones we need + macros_to_expand = parse_macros_files(args.macros_files) - # parse the skip/required tools files - skip_tools = parse_tools_list_file(args.skip_tools_file) - required_tools = parse_tools_list_file(args.required_tools_file) - - _convert_internal(parsed_ctds, - supported_file_formats=supported_file_formats, - default_executable_path=args.default_executable_path, - add_to_command_line=args.add_to_command_line, - blacklisted_parameters=args.blacklisted_parameters, - required_tools=required_tools, - skip_tools=skip_tools, - macros_file_names=args.macros_files, - macros_to_expand=macros_to_expand, - parameter_hardcoder=args.parameter_hardcoder) - - # generation of galaxy stubs is ready... now, let's see if we need to generate a tool_conf.xml - if args.tool_conf_destination is not None: - generate_tool_conf(parsed_ctds, args.tool_conf_destination, - args.galaxy_tool_path, args.default_category) - - # generate datatypes_conf.xml - if args.data_types_destination is not None: - generate_data_type_conf(supported_file_formats, args.data_types_destination) + # parse the given supported file-formats file + supported_file_formats = parse_file_formats(args.formats_file) + + # parse the skip/required tools files + skip_tools = parse_tools_list_file(args.skip_tools_file) + required_tools = parse_tools_list_file(args.required_tools_file) + + _convert_internal(parsed_ctds, + supported_file_formats=supported_file_formats, + default_executable_path=args.default_executable_path, + add_to_command_line=args.add_to_command_line, + blacklisted_parameters=args.blacklisted_parameters, + required_tools=required_tools, + skip_tools=skip_tools, + macros_file_names=args.macros_files, + macros_to_expand=macros_to_expand, + parameter_hardcoder=args.parameter_hardcoder) + + # generation of galaxy stubs is ready... now, let's see if we need to generate a tool_conf.xml + if args.tool_conf_destination is not None: + generate_tool_conf(parsed_ctds, args.tool_conf_destination, + args.galaxy_tool_path, args.default_category) + + # generate datatypes_conf.xml + if args.data_types_destination is not None: + generate_data_type_conf(supported_file_formats, args.data_types_destination) def parse_tools_list_file(tools_list_file): + """ + """ tools_list = None if tools_list_file is not None: tools_list = [] @@ -119,6 +131,8 @@ def parse_tools_list_file(tools_list_file): def parse_macros_files(macros_file_names): + """ + """ macros_to_expand = list() for macros_file_name in macros_file_names: @@ -159,6 +173,8 @@ def parse_macros_files(macros_file_names): def parse_file_formats(formats_file): + """ + """ supported_formats = {} if formats_file is not None: line_number = 0 @@ -193,6 +209,11 @@ def parse_file_formats(formats_file): def validate_and_prepare_args(args): + """ + check command line arguments + @param args command line arguments + @return None + """ # check that only one of skip_tools_file and required_tools_file has been provided if args.skip_tools_file is not None and args.required_tools_file is not None: raise ApplicationException( @@ -221,12 +242,22 @@ def validate_and_prepare_args(args): def get_preferred_file_extension(): + """ + get the file extension for the output files + @return "xml" + """ return "xml" def _convert_internal(parsed_ctds, **kwargs): - # parse all input files into models using CTDopts (via utils) - # the output is a tuple containing the model, output destination, origin file + """ + parse all input files into models using CTDopts (via utils) + + @param parsed_ctds the ctds + @param kwargs skip_tools, required_tools, and additional parameters for + expand_macros, create_command, create_inputs, create_outputs + @return a tuple containing the model, output destination, origin file + """ for parsed_ctd in parsed_ctds: model = parsed_ctd.ctd_model origin_file = parsed_ctd.input_file @@ -256,6 +287,11 @@ def _convert_internal(parsed_ctds, **kwargs): def write_header(tool, model): + """ + add comments to the tool header + @param tool the tool xml + @param model the ctd model + """ tool.addprevious(etree.Comment( "This is a configuration file for the integration of a tools into Galaxy (https://galaxyproject.org/). " "This file was automatically generated using CTDConverter.")) @@ -318,16 +354,30 @@ def generate_data_type_conf(supported_file_formats, data_types_destination): def create_tool(model): + """ + initialize the tool + @param model the ctd model + """ return Element("tool", OrderedDict([("id", model.name), ("name", model.name), ("version", model.version)])) def create_description(tool, model): + """ + add description to the tool + @param tool the Galaxy tool + @param model the ctd model + """ if "description" in model.opt_attribs.keys() and model.opt_attribs["description"] is not None: description = SubElement(tool,"description") description.text = model.opt_attribs["description"] def create_command(tool, model, **kwargs): + """ + @param tool the Galaxy tool + @param model the ctd model + @param kwargs + """ final_command = utils.extract_tool_executable_path(model, kwargs["default_executable_path"]) + '\n' final_command += kwargs["add_to_command_line"] + '\n' advanced_command_start = "#if $adv_opts.adv_opts_selector=='advanced':\n" @@ -417,12 +467,17 @@ def create_command(tool, model, **kwargs): final_command += "> $param_stdout\n" command_node = add_child_node(tool, "command") - command_node.text = final_command + command_node.text = CDATA(final_command) # creates the xml elements needed to import the needed macros files # and to "expand" the macros def expand_macros(tool, model, **kwargs): + """ + @param tool the Galaxy tool + @param model the ctd model + @param kwargs + """ macros_node = add_child_node(tool, "macros") token_node = add_child_node(macros_node, "token") token_node.attrib["name"] = "@EXECUTABLE@" @@ -455,13 +510,19 @@ def get_input_with_same_restrictions(out_param, model, supported_file_formats): def create_inputs(tool, model, **kwargs): + """ + create input section of the Galaxy tool + @param tool the Galaxy tool + @param model the ctd model + @param kwargs + """ inputs_node = SubElement(tool, "inputs") # some suites (such as OpenMS) need some advanced options when handling inputs expand_advanced_node = None parameter_hardcoder = kwargs["parameter_hardcoder"] - # treat all non output-file parameters as inputs + # treat all non output-file/advanced/blacklisted/hardcoded parameters as inputs for param in utils.extract_and_flatten_parameters(model): # no need to show hardcoded parameters hardcoded_value = parameter_hardcoder.get_hardcoded_value(param.name, model.name) @@ -484,6 +545,8 @@ def create_inputs(tool, model, **kwargs): param_node = add_child_node(parent_node, "param") create_param_attribute_list(param_node, param, kwargs["supported_file_formats"]) + # tread advanced parameters as inputs + # TODO: merge with previous loop to avoid code duplication? for param in utils.extract_and_flatten_parameters(model): # no need to show hardcoded parameters hardcoded_value = parameter_hardcoder.get_hardcoded_value(param.name, model.name) @@ -526,6 +589,12 @@ def create_repeat_attribute_list(rep_node, param): def create_param_attribute_list(param_node, param, supported_file_formats): + """ + get the attributes of input parameters + @param param_node the galaxy tool param node + @param param the ctd parameter + @param supported_file_formats + """ param_node.attrib["name"] = get_galaxy_parameter_name(param) param_type = TYPE_TO_GALAXY_TYPE[param.type] @@ -778,6 +847,12 @@ def create_boolean_parameter(param_node, param): def create_outputs(parent, model, **kwargs): + """ + create outputs section of the Galaxy tool + @param tool the Galaxy tool + @param model the ctd model + @param kwargs + """ outputs_node = add_child_node(parent, "outputs") parameter_hardcoder = kwargs["parameter_hardcoder"] @@ -867,12 +942,23 @@ def create_change_format_node(parent, data_formats, input_ref): # Shows basic information about the file, such as data ranges and file type. def create_help(tool, model): + """ + create help section of the Galaxy tool + @param tool the Galaxy tool + @param model the ctd model + @param kwargs + """ help_node = add_child_node(tool, "help") - # TODO: do we need CDATA Section here? - help_node.text = utils.extract_tool_help_text(model) + help_node.text = CDATA(utils.extract_tool_help_text(model)) -# adds and returns a child node using the given name to the given parent node def add_child_node(parent_node, child_node_name, attributes=OrderedDict([])): + """ + helper function to add a child node using the given name to the given parent node + @param parent_node the parent + @param child_node_name the desired name of the child + @param attributes desired attributes of the child + @return the created child node + """ child_node = SubElement(parent_node, child_node_name, attributes) return child_node diff --git a/tests/test-data/bool.xml b/tests/test-data/bool.xml index 30674b85..5b5a6a39 100644 --- a/tests/test-data/bool.xml +++ b/tests/test-data/bool.xml @@ -10,7 +10,7 @@ - echo BoolTest + +> $param_stdout +]]> @@ -55,8 +55,8 @@ - Bool parameter tests. + +For more information, visit http://docs.are/unavailable.html]]> diff --git a/tests/test-data/empty.xml b/tests/test-data/empty.xml index e7daf119..b66b175c 100644 --- a/tests/test-data/empty.xml +++ b/tests/test-data/empty.xml @@ -10,16 +10,16 @@ - echo StringTest + +> $param_stdout +]]> - String parameter tests. + +For more information, visit http://docs.are/unavailable.html]]> diff --git a/tests/test-data/float.xml b/tests/test-data/float.xml index 1fe902f6..e2626295 100644 --- a/tests/test-data/float.xml +++ b/tests/test-data/float.xml @@ -10,7 +10,7 @@ - echo FloatTest + +> $param_stdout +]]> @@ -151,8 +151,8 @@ - Floateger parameter tests. + +For more information, visit http://docs.are/unavailable.html]]> diff --git a/tests/test-data/ifile.xml b/tests/test-data/ifile.xml index 232b9fa0..b16380ce 100644 --- a/tests/test-data/ifile.xml +++ b/tests/test-data/ifile.xml @@ -10,7 +10,7 @@ - echo inFileTest + +> $param_stdout +]]> @@ -263,8 +263,8 @@ - Input File parameter tests. + +For more information, visit http://docs.are/unavailable.html]]> diff --git a/tests/test-data/integer.xml b/tests/test-data/integer.xml index bf20e9b2..f0883908 100644 --- a/tests/test-data/integer.xml +++ b/tests/test-data/integer.xml @@ -10,7 +10,7 @@ - echo IntTest + +> $param_stdout +]]> @@ -151,8 +151,8 @@ - Integer parameter tests. + +For more information, visit http://docs.are/unavailable.html]]> diff --git a/tests/test-data/ofile.xml b/tests/test-data/ofile.xml index f3678f7a..295d71af 100644 --- a/tests/test-data/ofile.xml +++ b/tests/test-data/ofile.xml @@ -10,7 +10,7 @@ - echo outFileTest + +]]> @@ -160,8 +160,8 @@ - Input File parameter tests. + +For more information, visit http://docs.are/unavailable.html]]> diff --git a/tests/test-data/repeat.xml b/tests/test-data/repeat.xml index 028177a3..fbae2070 100644 --- a/tests/test-data/repeat.xml +++ b/tests/test-data/repeat.xml @@ -10,7 +10,7 @@ - echo RepeatTest + +> $param_stdout +]]> @@ -863,8 +863,8 @@ - Repeat parameter tests. + +For more information, visit http://docs.are/unavailable.html]]> diff --git a/tests/test-data/select.xml b/tests/test-data/select.xml index 7e243ccd..5c572ee5 100644 --- a/tests/test-data/select.xml +++ b/tests/test-data/select.xml @@ -10,7 +10,7 @@ - echo SelectTest + +> $param_stdout +]]> @@ -271,8 +271,8 @@ - Select parameter tests. + +For more information, visit http://docs.are/unavailable.html]]> diff --git a/tests/test-data/string.xml b/tests/test-data/string.xml index 90896ddb..2c6cca80 100644 --- a/tests/test-data/string.xml +++ b/tests/test-data/string.xml @@ -10,7 +10,7 @@ - echo StringTest + +> $param_stdout +]]> @@ -111,8 +111,8 @@ - String parameter tests. + +For more information, visit http://docs.are/unavailable.html]]> From b94253fcd330e0b95019d9a9b01c1a0b5cd96492 Mon Sep 17 00:00:00 2001 From: M Bernt Date: Wed, 17 Jul 2019 13:24:55 +0200 Subject: [PATCH 013/121] render all bools are rendered as booleans OpenMS (ctd 1.6.2) uses string with restriction true/false for booleans - those with default false are flags, ie `-param` - otherwise `-param false` needs to be given for simplification the neccessary command line is now stored directly in the true/falsevalue attributes also added more docs --- galaxy/converter.py | 51 ++++++++++++++++++++-------------------- tests/test-data/bool.ctd | 10 ++++---- tests/test-data/bool.xml | 40 +++++++------------------------ 3 files changed, 38 insertions(+), 63 deletions(-) diff --git a/galaxy/converter.py b/galaxy/converter.py index 1cb5dcee..a20ed73c 100755 --- a/galaxy/converter.py +++ b/galaxy/converter.py @@ -441,9 +441,7 @@ def create_command(tool, model, **kwargs): command += " #end if\n" command += "#end if\n" elif is_boolean_parameter(param): - command += "#if " + actual_parameter + ":\n" - command += " %s\n" % command_line_prefix - command += "#end if\n" + command += "%s\n" % actual_parameter elif TYPE_TO_GALAXY_TYPE[param.type] is 'text': command += "#if str(" + actual_parameter + "):\n" command += " %s " % command_line_prefix @@ -596,7 +594,7 @@ def create_param_attribute_list(param_node, param, supported_file_formats): @param supported_file_formats """ param_node.attrib["name"] = get_galaxy_parameter_name(param) - + print param_node.attrib["name"] param_type = TYPE_TO_GALAXY_TYPE[param.type] if param_type is None: raise ModelError("Unrecognized parameter type %(type)s for parameter %(name)s" @@ -607,7 +605,7 @@ def create_param_attribute_list(param_node, param, supported_file_formats): if is_selection_parameter(param): param_type = "select" - if len(param.restrictions.choices) < 5: + if 2 < len(param.restrictions.choices) < 5: param_node.attrib["display"] = "radio" if is_boolean_parameter(param): @@ -670,7 +668,7 @@ def create_param_attribute_list(param_node, param, supported_file_formats): raise InvalidModelException("Unrecognized restriction type [%(type)s] for parameter [%(name)s]" % {"type": type(param.restrictions), "name": param.name}) - if param_type == "select" and param.default in param.restrictions.choices: + if param_type == "select" and param_type != "boolean" and param.default in param.restrictions.choices: param_node.attrib["optional"] = "False" else: param_node.attrib["optional"] = str(not param.required) @@ -793,16 +791,15 @@ def generate_label_and_help(desc): return label, help_text -# determines if the given choices are boolean (basically, if the possible values are yes/no, true/false) def is_boolean_parameter(param): + """ + determines if the given choices are boolean (basically, if the possible values are true/false) + @param param the ctd parameter + @return True iff a boolean parameter + """ # detect boolean selects of OpenMS if is_selection_parameter(param): - if len(param.restrictions.choices) == 2: - # check that default value is false to make sure it is an actual flag - if "false" in param.restrictions.choices and \ - "true" in param.restrictions.choices and \ - param.default == "false": - return True + return set(param.restrictions.choices) == set(["true", "false"]) else: return param.type is bool @@ -819,23 +816,25 @@ def get_lowercase_list(some_list): return lowercase_list -# creates a galaxy boolean parameter type -# this method assumes that param has restrictions, and that only two restictions are present -# (either yes/no or true/false) def create_boolean_parameter(param_node, param): - # first, determine the 'truevalue' and the 'falsevalue' - """TODO: true and false values can be way more than 'true' and 'false' + """ + creates a galaxy boolean parameter type + this method assumes that param has restrictions, and that only two restictions are present + (either yes/no or true/false) # first, determine the 'truevalue' and the 'falsevalue' + + TODO: true and false values can be way more than 'true' and 'false' but for that we need CTD support """ - # by default, 'true' and 'false' are handled as flags, like the verbose flag (i.e., -v) - true_value = "-%s" % utils.extract_param_name(param) - false_value = "" + # in ctd (1.6.2) bools are strings with restriction true,false + # - if the default is false then they are flags + # - otherwise the true or false value needs to be added (where the true case is unnecessary) choices = get_lowercase_list(param.restrictions.choices) - if "yes" in choices: - true_value = "yes" - false_value = "no" - param_node.attrib["truevalue"] = true_value - param_node.attrib["falsevalue"] = false_value + if set(choices) == set(["true","false"]) and param.default == "true": + param_node.attrib["truevalue"] = "" + param_node.attrib["falsevalue"] = "-%s false" % utils.extract_param_name(param) + else: + param_node.attrib["truevalue"] = "-%s" % utils.extract_param_name(param) + param_node.attrib["falsevalue"] = "" # set the checked attribute if param.default is not None: diff --git a/tests/test-data/bool.ctd b/tests/test-data/bool.ctd index d59641e2..bec0cc7d 100644 --- a/tests/test-data/bool.ctd +++ b/tests/test-data/bool.ctd @@ -6,11 +6,11 @@ - - - - - + + + + + diff --git a/tests/test-data/bool.xml b/tests/test-data/bool.xml index 5b5a6a39..cb1b9ec5 100644 --- a/tests/test-data/bool.xml +++ b/tests/test-data/bool.xml @@ -12,44 +12,20 @@ $param_stdout ]]> - - - - - + + - - - - - + + From d4e1965ea08f13fa09cda344d998f0dc5576480c Mon Sep 17 00:00:00 2001 From: M Bernt Date: Thu, 18 Jul 2019 14:45:09 +0200 Subject: [PATCH 014/121] improved handling of selects & always render optional selects: - add possibility of selects with multiple=true from ITEMLIST - just add single quotes around all option values optional: - optional was only set for params with restrictions now its always set (one might remove it for bools...) also add more documentation --- galaxy/converter.py | 97 +++++--- tests/test-data/bool.xml | 8 +- tests/test-data/float.xml | 64 +++--- tests/test-data/ifile.xml | 24 +- tests/test-data/integer.xml | 64 +++--- tests/test-data/macros.xml | 2 +- tests/test-data/ofile.xml | 36 +-- tests/test-data/repeat.xml | 400 ++++++++++---------------------- tests/test-data/select.ctd | 88 ++++--- tests/test-data/select.xml | 442 +++++++++++++++++++++++------------- tests/test-data/string.xml | 32 +-- 11 files changed, 628 insertions(+), 629 deletions(-) diff --git a/galaxy/converter.py b/galaxy/converter.py index a20ed73c..f6709b6b 100755 --- a/galaxy/converter.py +++ b/galaxy/converter.py @@ -403,6 +403,12 @@ def create_command(tool, model, **kwargs): # parameter is neither blacklisted nor hardcoded... galaxy_parameter_name = get_galaxy_parameter_name(param) repeat_galaxy_parameter_name = get_repeat_galaxy_parameter_name(param) + if param.advanced and param.type is not _OutFile: + actual_parameter = "$adv_opts.%s" % galaxy_parameter_name + actual_repeat = "$adv_opts.%s" % repeat_galaxy_parameter_name + else: + actual_parameter = "$%s" % galaxy_parameter_name + actual_repeat = "$%s" % repeat_galaxy_parameter_name # logic for ITEMLISTs if param.is_list: @@ -410,42 +416,36 @@ def create_command(tool, model, **kwargs): command += command_line_prefix + "\n" command += " #for token in $" + galaxy_parameter_name + ":\n" command += " $token\n" - command += " #end for\n" + command += " #end for\n" + elif is_selection_parameter(param): + command += "#if " + actual_parameter + ":\n" + command += " #set tmp=' '.join([\"'%s'\"%str(_) for _ in " + actual_parameter + "])\n" + command += " %s\n" % command_line_prefix + command += " $tmp\n" + command += "#end if\n" else: - command += "\n#if $" + repeat_galaxy_parameter_name + ":\n" + command += "\n#if " + actual_repeat + ":\n" command += command_line_prefix + "\n" - command += " #for token in $" + repeat_galaxy_parameter_name + ":\n" - command += " #if \" \" in str(token):\n" - command += " \"$token." + galaxy_parameter_name + "\"\n" - command += " #else\n" - command += " $token." + galaxy_parameter_name + "\n" - command += " #end if\n" + command += " #for token in " + actual_repeat + ":\n" + command += " '$token." + galaxy_parameter_name + "'\n" command += " #end for\n" command += "#end if\n" # logic for other ITEMs else: - if param.advanced and param.type is not _OutFile: - actual_parameter = "$adv_opts.%s" % galaxy_parameter_name - else: - actual_parameter = "$%s" % galaxy_parameter_name # TODO only useful for text fields, integers or floats # not useful for choices, input fields ... - if not is_boolean_parameter(param) and type(param.restrictions) is _Choices : + if is_selection_parameter(param): command += "#if " + actual_parameter + ":\n" command += " %s\n" % command_line_prefix - command += " #if \" \" in str(" + actual_parameter + "):\n" - command += " \"" + actual_parameter + "\"\n" - command += " #else\n" - command += " " + actual_parameter + "\n" - command += " #end if\n" + command += " '" + actual_parameter + "'\n" command += "#end if\n" elif is_boolean_parameter(param): command += "%s\n" % actual_parameter elif TYPE_TO_GALAXY_TYPE[param.type] is 'text': command += "#if str(" + actual_parameter + "):\n" command += " %s " % command_line_prefix - command += " \"" + actual_parameter + "\"\n" + command += " '" + actual_parameter + "'\n" command += "#end if\n" else: command += "#if str(" + actual_parameter + "):\n" @@ -494,6 +494,13 @@ def expand_macros(tool, model, **kwargs): def get_galaxy_parameter_name(param): + """ + get the name of the parameter used in the galaxy tool + - prepend param_ + - replace : and - by _ + @param param the parameter + @return the name used for the parameter in the tool form + """ return "param_%s" % utils.extract_param_name(param).replace(":", "_").replace("-", "_") @@ -505,7 +512,7 @@ def get_input_with_same_restrictions(out_param, model, supported_file_formats): out_param_formats = get_supported_file_types(out_param.restrictions.formats, supported_file_formats) if in_param_formats == out_param_formats: return param - + def create_inputs(tool, model, **kwargs): """ @@ -534,8 +541,12 @@ def create_inputs(tool, model, **kwargs): parent_node = inputs_node - # for lists we need a repeat tag - if param.is_list and param.type is not _InFile: + # for lists we need a repeat tag, execept if + # - list of strings with a fixed set of options -> select w multiple=true + # - InFiles -> data w multiple=true + if param.is_list and\ + not (param.type is str and param.restrictions is not None) and\ + param.type is not _InFile: rep_node = add_child_node(parent_node, "repeat") create_repeat_attribute_list(rep_node, param) parent_node = rep_node @@ -560,7 +571,9 @@ def create_inputs(tool, model, **kwargs): parent_node = expand_advanced_node # for lists we need a repeat tag - if param.is_list and param.type is not _InFile: + if param.is_list and\ + not (param.type is str and param.restrictions is not None) and\ + param.type is not _InFile: rep_node = add_child_node(parent_node, "repeat") create_repeat_attribute_list(rep_node, param) parent_node = rep_node @@ -570,6 +583,12 @@ def create_inputs(tool, model, **kwargs): def get_repeat_galaxy_parameter_name(param): + """ + get the name of the repeat that contains the parameter + - prepend rep_ + @param param the parameter + @return the name of the repeat + """ return "rep_" + get_galaxy_parameter_name(param) @@ -594,7 +613,6 @@ def create_param_attribute_list(param_node, param, supported_file_formats): @param supported_file_formats """ param_node.attrib["name"] = get_galaxy_parameter_name(param) - print param_node.attrib["name"] param_type = TYPE_TO_GALAXY_TYPE[param.type] if param_type is None: raise ModelError("Unrecognized parameter type %(type)s for parameter %(name)s" @@ -605,8 +623,10 @@ def create_param_attribute_list(param_node, param, supported_file_formats): if is_selection_parameter(param): param_type = "select" - if 2 < len(param.restrictions.choices) < 5: + if len(param.restrictions.choices) < 5 and not param.is_list: param_node.attrib["display"] = "radio" + if param.is_list: + param_node.attrib["multiple"] = "true" if is_boolean_parameter(param): param_type = "boolean" @@ -634,20 +654,32 @@ def create_param_attribute_list(param_node, param, supported_file_formats): else: param_node.attrib["type"] = param_type + if param_type == "select" and param.default in param.restrictions.choices: + param_node.attrib["optional"] = "False" + else: + param_node.attrib["optional"] = str(not param.required) + # check for parameters with restricted values (which will correspond to a "select" in galaxy) if param.restrictions is not None: # it could be either _Choices or _NumericRange, with special case for boolean types if param_type == "boolean": create_boolean_parameter(param_node, param) elif type(param.restrictions) is _Choices: + # add a nothing selected option to mandatory options w/o default + if param_node.attrib["optional"] == "False" and (param.default is None or param.default is _Null): + param.restrictions.choices.insert(0, "select a value") # create as many - - + + - - + + diff --git a/tests/test-data/float.xml b/tests/test-data/float.xml index e2626295..3a5b2bed 100644 --- a/tests/test-data/float.xml +++ b/tests/test-data/float.xml @@ -113,39 +113,39 @@ > $param_stdout ]]> - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + diff --git a/tests/test-data/ifile.xml b/tests/test-data/ifile.xml index b16380ce..feddc933 100644 --- a/tests/test-data/ifile.xml +++ b/tests/test-data/ifile.xml @@ -137,19 +137,19 @@ > $param_stdout ]]> - - + + - - + + - + @@ -157,7 +157,7 @@ - + @@ -198,19 +198,19 @@ - - + + - - + + - + @@ -218,7 +218,7 @@ - + diff --git a/tests/test-data/integer.xml b/tests/test-data/integer.xml index f0883908..1a8bbdc5 100644 --- a/tests/test-data/integer.xml +++ b/tests/test-data/integer.xml @@ -113,39 +113,39 @@ > $param_stdout ]]> - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + diff --git a/tests/test-data/macros.xml b/tests/test-data/macros.xml index 82907e42..acf44cb0 100644 --- a/tests/test-data/macros.xml +++ b/tests/test-data/macros.xml @@ -2,7 +2,7 @@ - requirement + bash diff --git a/tests/test-data/ofile.xml b/tests/test-data/ofile.xml index 295d71af..99d4d2f4 100644 --- a/tests/test-data/ofile.xml +++ b/tests/test-data/ofile.xml @@ -52,66 +52,42 @@ #if $rep_param_list_mandatory_noformat: -list_mandatory_noformat #for token in $rep_param_list_mandatory_noformat: - #if " " in str(token): - "$token.param_list_mandatory_noformat" - #else - $token.param_list_mandatory_noformat - #end if + '$token.param_list_mandatory_noformat' #end for #end if #if $rep_param_list_optional_noformat: -list_optional_noformat #for token in $rep_param_list_optional_noformat: - #if " " in str(token): - "$token.param_list_optional_noformat" - #else - $token.param_list_optional_noformat - #end if + '$token.param_list_optional_noformat' #end for #end if #if $rep_param_list_mandatory_oneformat: -list_mandatory_oneformat #for token in $rep_param_list_mandatory_oneformat: - #if " " in str(token): - "$token.param_list_mandatory_oneformat" - #else - $token.param_list_mandatory_oneformat - #end if + '$token.param_list_mandatory_oneformat' #end for #end if #if $rep_param_list_optional_oneformat: -list_optional_oneformat #for token in $rep_param_list_optional_oneformat: - #if " " in str(token): - "$token.param_list_optional_oneformat" - #else - $token.param_list_optional_oneformat - #end if + '$token.param_list_optional_oneformat' #end for #end if #if $rep_param_list_mandatory_multformat: -list_mandatory_multformat #for token in $rep_param_list_mandatory_multformat: - #if " " in str(token): - "$token.param_list_mandatory_multformat" - #else - $token.param_list_mandatory_multformat - #end if + '$token.param_list_mandatory_multformat' #end for #end if #if $rep_param_list_optional_multformat: -list_optional_multformat #for token in $rep_param_list_optional_multformat: - #if " " in str(token): - "$token.param_list_optional_multformat" - #else - $token.param_list_optional_multformat - #end if + '$token.param_list_optional_multformat' #end for #end if #if str($param_mandatory_noformat_adv): diff --git a/tests/test-data/repeat.xml b/tests/test-data/repeat.xml index fbae2070..e7d09790 100644 --- a/tests/test-data/repeat.xml +++ b/tests/test-data/repeat.xml @@ -16,441 +16,281 @@ #if $rep_param_string_wodefault_optional: -string_wodefault_optional #for token in $rep_param_string_wodefault_optional: - #if " " in str(token): - "$token.param_string_wodefault_optional" - #else - $token.param_string_wodefault_optional - #end if + '$token.param_string_wodefault_optional' #end for #end if #if $rep_param_string_wodefault_mandatory: -string_wodefault_mandatory #for token in $rep_param_string_wodefault_mandatory: - #if " " in str(token): - "$token.param_string_wodefault_mandatory" - #else - $token.param_string_wodefault_mandatory - #end if + '$token.param_string_wodefault_mandatory' #end for #end if #if $rep_param_string_default_optional: -string_default_optional #for token in $rep_param_string_default_optional: - #if " " in str(token): - "$token.param_string_default_optional" - #else - $token.param_string_default_optional - #end if + '$token.param_string_default_optional' #end for #end if #if $rep_param_string_default_mandatory: -string_default_mandatory #for token in $rep_param_string_default_mandatory: - #if " " in str(token): - "$token.param_string_default_mandatory" - #else - $token.param_string_default_mandatory - #end if + '$token.param_string_default_mandatory' #end for #end if #if $rep_param_int_wodefault_optional: -int_wodefault_optional #for token in $rep_param_int_wodefault_optional: - #if " " in str(token): - "$token.param_int_wodefault_optional" - #else - $token.param_int_wodefault_optional - #end if + '$token.param_int_wodefault_optional' #end for #end if #if $rep_param_int_wodefault_mandatory: -int_wodefault_mandatory #for token in $rep_param_int_wodefault_mandatory: - #if " " in str(token): - "$token.param_int_wodefault_mandatory" - #else - $token.param_int_wodefault_mandatory - #end if + '$token.param_int_wodefault_mandatory' #end for #end if #if $rep_param_int_default_optional: -int_default_optional #for token in $rep_param_int_default_optional: - #if " " in str(token): - "$token.param_int_default_optional" - #else - $token.param_int_default_optional - #end if + '$token.param_int_default_optional' #end for #end if #if $rep_param_int_default_mandatory: -int_default_mandatory #for token in $rep_param_int_default_mandatory: - #if " " in str(token): - "$token.param_int_default_mandatory" - #else - $token.param_int_default_mandatory - #end if + '$token.param_int_default_mandatory' #end for #end if #if $rep_param_int_resticted_wodefault_optional: -int_resticted_wodefault_optional #for token in $rep_param_int_resticted_wodefault_optional: - #if " " in str(token): - "$token.param_int_resticted_wodefault_optional" - #else - $token.param_int_resticted_wodefault_optional - #end if + '$token.param_int_resticted_wodefault_optional' #end for #end if #if $rep_param_int_resticted_wodefault_mandatory: -int_resticted_wodefault_mandatory #for token in $rep_param_int_resticted_wodefault_mandatory: - #if " " in str(token): - "$token.param_int_resticted_wodefault_mandatory" - #else - $token.param_int_resticted_wodefault_mandatory - #end if + '$token.param_int_resticted_wodefault_mandatory' #end for #end if #if $rep_param_int_resticted_default_optional: -int_resticted_default_optional #for token in $rep_param_int_resticted_default_optional: - #if " " in str(token): - "$token.param_int_resticted_default_optional" - #else - $token.param_int_resticted_default_optional - #end if + '$token.param_int_resticted_default_optional' #end for #end if #if $rep_param_int_resticted_default_mandatory: -int_resticted_default_mandatory #for token in $rep_param_int_resticted_default_mandatory: - #if " " in str(token): - "$token.param_int_resticted_default_mandatory" - #else - $token.param_int_resticted_default_mandatory - #end if + '$token.param_int_resticted_default_mandatory' #end for #end if #if $rep_param_double_wodefault_optional: -double_wodefault_optional #for token in $rep_param_double_wodefault_optional: - #if " " in str(token): - "$token.param_double_wodefault_optional" - #else - $token.param_double_wodefault_optional - #end if + '$token.param_double_wodefault_optional' #end for #end if #if $rep_param_double_wodefault_mandatory: -double_wodefault_mandatory #for token in $rep_param_double_wodefault_mandatory: - #if " " in str(token): - "$token.param_double_wodefault_mandatory" - #else - $token.param_double_wodefault_mandatory - #end if + '$token.param_double_wodefault_mandatory' #end for #end if #if $rep_param_double_default_optional: -double_default_optional #for token in $rep_param_double_default_optional: - #if " " in str(token): - "$token.param_double_default_optional" - #else - $token.param_double_default_optional - #end if + '$token.param_double_default_optional' #end for #end if #if $rep_param_double_default_mandatory: -double_default_mandatory #for token in $rep_param_double_default_mandatory: - #if " " in str(token): - "$token.param_double_default_mandatory" - #else - $token.param_double_default_mandatory - #end if + '$token.param_double_default_mandatory' #end for #end if #if $rep_param_double_resticted_wodefault_optional: -double_resticted_wodefault_optional #for token in $rep_param_double_resticted_wodefault_optional: - #if " " in str(token): - "$token.param_double_resticted_wodefault_optional" - #else - $token.param_double_resticted_wodefault_optional - #end if + '$token.param_double_resticted_wodefault_optional' #end for #end if #if $rep_param_double_resticted_wodefault_mandatory: -double_resticted_wodefault_mandatory #for token in $rep_param_double_resticted_wodefault_mandatory: - #if " " in str(token): - "$token.param_double_resticted_wodefault_mandatory" - #else - $token.param_double_resticted_wodefault_mandatory - #end if + '$token.param_double_resticted_wodefault_mandatory' #end for #end if #if $rep_param_double_resticted_default_optional: -double_resticted_default_optional #for token in $rep_param_double_resticted_default_optional: - #if " " in str(token): - "$token.param_double_resticted_default_optional" - #else - $token.param_double_resticted_default_optional - #end if + '$token.param_double_resticted_default_optional' #end for #end if #if $rep_param_double_resticted_default_mandatory: -double_resticted_default_mandatory #for token in $rep_param_double_resticted_default_mandatory: - #if " " in str(token): - "$token.param_double_resticted_default_mandatory" - #else - $token.param_double_resticted_default_mandatory - #end if + '$token.param_double_resticted_default_mandatory' #end for #end if #if $adv_opts.adv_opts_selector=='advanced': -#if $rep_param_string_wodefault_optional_adv: +#if $adv_opts.rep_param_string_wodefault_optional_adv: -string_wodefault_optional_adv - #for token in $rep_param_string_wodefault_optional_adv: - #if " " in str(token): - "$token.param_string_wodefault_optional_adv" - #else - $token.param_string_wodefault_optional_adv - #end if + #for token in $adv_opts.rep_param_string_wodefault_optional_adv: + '$token.param_string_wodefault_optional_adv' #end for #end if -#if $rep_param_string_wodefault_mandatory_adv: +#if $adv_opts.rep_param_string_wodefault_mandatory_adv: -string_wodefault_mandatory_adv - #for token in $rep_param_string_wodefault_mandatory_adv: - #if " " in str(token): - "$token.param_string_wodefault_mandatory_adv" - #else - $token.param_string_wodefault_mandatory_adv - #end if + #for token in $adv_opts.rep_param_string_wodefault_mandatory_adv: + '$token.param_string_wodefault_mandatory_adv' #end for #end if -#if $rep_param_string_default_optional_adv: +#if $adv_opts.rep_param_string_default_optional_adv: -string_default_optional_adv - #for token in $rep_param_string_default_optional_adv: - #if " " in str(token): - "$token.param_string_default_optional_adv" - #else - $token.param_string_default_optional_adv - #end if + #for token in $adv_opts.rep_param_string_default_optional_adv: + '$token.param_string_default_optional_adv' #end for #end if -#if $rep_param_string_default_mandatory_adv: +#if $adv_opts.rep_param_string_default_mandatory_adv: -string_default_mandatory_adv - #for token in $rep_param_string_default_mandatory_adv: - #if " " in str(token): - "$token.param_string_default_mandatory_adv" - #else - $token.param_string_default_mandatory_adv - #end if + #for token in $adv_opts.rep_param_string_default_mandatory_adv: + '$token.param_string_default_mandatory_adv' #end for #end if -#if $rep_param_int_wodefault_optional_adv: +#if $adv_opts.rep_param_int_wodefault_optional_adv: -int_wodefault_optional_adv - #for token in $rep_param_int_wodefault_optional_adv: - #if " " in str(token): - "$token.param_int_wodefault_optional_adv" - #else - $token.param_int_wodefault_optional_adv - #end if + #for token in $adv_opts.rep_param_int_wodefault_optional_adv: + '$token.param_int_wodefault_optional_adv' #end for #end if -#if $rep_param_int_wodefault_mandatory_adv: +#if $adv_opts.rep_param_int_wodefault_mandatory_adv: -int_wodefault_mandatory_adv - #for token in $rep_param_int_wodefault_mandatory_adv: - #if " " in str(token): - "$token.param_int_wodefault_mandatory_adv" - #else - $token.param_int_wodefault_mandatory_adv - #end if + #for token in $adv_opts.rep_param_int_wodefault_mandatory_adv: + '$token.param_int_wodefault_mandatory_adv' #end for #end if -#if $rep_param_int_default_optional_adv: +#if $adv_opts.rep_param_int_default_optional_adv: -int_default_optional_adv - #for token in $rep_param_int_default_optional_adv: - #if " " in str(token): - "$token.param_int_default_optional_adv" - #else - $token.param_int_default_optional_adv - #end if + #for token in $adv_opts.rep_param_int_default_optional_adv: + '$token.param_int_default_optional_adv' #end for #end if -#if $rep_param_int_default_mandatory_adv: +#if $adv_opts.rep_param_int_default_mandatory_adv: -int_default_mandatory_adv - #for token in $rep_param_int_default_mandatory_adv: - #if " " in str(token): - "$token.param_int_default_mandatory_adv" - #else - $token.param_int_default_mandatory_adv - #end if + #for token in $adv_opts.rep_param_int_default_mandatory_adv: + '$token.param_int_default_mandatory_adv' #end for #end if -#if $rep_param_int_resticted_wodefault_optional_adv: +#if $adv_opts.rep_param_int_resticted_wodefault_optional_adv: -int_resticted_wodefault_optional_adv - #for token in $rep_param_int_resticted_wodefault_optional_adv: - #if " " in str(token): - "$token.param_int_resticted_wodefault_optional_adv" - #else - $token.param_int_resticted_wodefault_optional_adv - #end if + #for token in $adv_opts.rep_param_int_resticted_wodefault_optional_adv: + '$token.param_int_resticted_wodefault_optional_adv' #end for #end if -#if $rep_param_int_resticted_wodefault_mandatory_adv: +#if $adv_opts.rep_param_int_resticted_wodefault_mandatory_adv: -int_resticted_wodefault_mandatory_adv - #for token in $rep_param_int_resticted_wodefault_mandatory_adv: - #if " " in str(token): - "$token.param_int_resticted_wodefault_mandatory_adv" - #else - $token.param_int_resticted_wodefault_mandatory_adv - #end if + #for token in $adv_opts.rep_param_int_resticted_wodefault_mandatory_adv: + '$token.param_int_resticted_wodefault_mandatory_adv' #end for #end if -#if $rep_param_int_resticted_default_optional_adv: +#if $adv_opts.rep_param_int_resticted_default_optional_adv: -int_resticted_default_optional_adv - #for token in $rep_param_int_resticted_default_optional_adv: - #if " " in str(token): - "$token.param_int_resticted_default_optional_adv" - #else - $token.param_int_resticted_default_optional_adv - #end if + #for token in $adv_opts.rep_param_int_resticted_default_optional_adv: + '$token.param_int_resticted_default_optional_adv' #end for #end if -#if $rep_param_int_resticted_default_mandatory_adv: +#if $adv_opts.rep_param_int_resticted_default_mandatory_adv: -int_resticted_default_mandatory_adv - #for token in $rep_param_int_resticted_default_mandatory_adv: - #if " " in str(token): - "$token.param_int_resticted_default_mandatory_adv" - #else - $token.param_int_resticted_default_mandatory_adv - #end if + #for token in $adv_opts.rep_param_int_resticted_default_mandatory_adv: + '$token.param_int_resticted_default_mandatory_adv' #end for #end if -#if $rep_param_double_wodefault_optional_adv: +#if $adv_opts.rep_param_double_wodefault_optional_adv: -double_wodefault_optional_adv - #for token in $rep_param_double_wodefault_optional_adv: - #if " " in str(token): - "$token.param_double_wodefault_optional_adv" - #else - $token.param_double_wodefault_optional_adv - #end if + #for token in $adv_opts.rep_param_double_wodefault_optional_adv: + '$token.param_double_wodefault_optional_adv' #end for #end if -#if $rep_param_double_wodefault_mandatory_adv: +#if $adv_opts.rep_param_double_wodefault_mandatory_adv: -double_wodefault_mandatory_adv - #for token in $rep_param_double_wodefault_mandatory_adv: - #if " " in str(token): - "$token.param_double_wodefault_mandatory_adv" - #else - $token.param_double_wodefault_mandatory_adv - #end if + #for token in $adv_opts.rep_param_double_wodefault_mandatory_adv: + '$token.param_double_wodefault_mandatory_adv' #end for #end if -#if $rep_param_double_default_optional_adv: +#if $adv_opts.rep_param_double_default_optional_adv: -double_default_optional_adv - #for token in $rep_param_double_default_optional_adv: - #if " " in str(token): - "$token.param_double_default_optional_adv" - #else - $token.param_double_default_optional_adv - #end if + #for token in $adv_opts.rep_param_double_default_optional_adv: + '$token.param_double_default_optional_adv' #end for #end if -#if $rep_param_double_default_mandatory_adv: +#if $adv_opts.rep_param_double_default_mandatory_adv: -double_default_mandatory_adv - #for token in $rep_param_double_default_mandatory_adv: - #if " " in str(token): - "$token.param_double_default_mandatory_adv" - #else - $token.param_double_default_mandatory_adv - #end if + #for token in $adv_opts.rep_param_double_default_mandatory_adv: + '$token.param_double_default_mandatory_adv' #end for #end if -#if $rep_param_double_resticted_wodefault_optional_adv: +#if $adv_opts.rep_param_double_resticted_wodefault_optional_adv: -double_resticted_wodefault_optional_adv - #for token in $rep_param_double_resticted_wodefault_optional_adv: - #if " " in str(token): - "$token.param_double_resticted_wodefault_optional_adv" - #else - $token.param_double_resticted_wodefault_optional_adv - #end if + #for token in $adv_opts.rep_param_double_resticted_wodefault_optional_adv: + '$token.param_double_resticted_wodefault_optional_adv' #end for #end if -#if $rep_param_double_resticted_wodefault_mandatory_adv: +#if $adv_opts.rep_param_double_resticted_wodefault_mandatory_adv: -double_resticted_wodefault_mandatory_adv - #for token in $rep_param_double_resticted_wodefault_mandatory_adv: - #if " " in str(token): - "$token.param_double_resticted_wodefault_mandatory_adv" - #else - $token.param_double_resticted_wodefault_mandatory_adv - #end if + #for token in $adv_opts.rep_param_double_resticted_wodefault_mandatory_adv: + '$token.param_double_resticted_wodefault_mandatory_adv' #end for #end if -#if $rep_param_double_resticted_default_optional_adv: +#if $adv_opts.rep_param_double_resticted_default_optional_adv: -double_resticted_default_optional_adv - #for token in $rep_param_double_resticted_default_optional_adv: - #if " " in str(token): - "$token.param_double_resticted_default_optional_adv" - #else - $token.param_double_resticted_default_optional_adv - #end if + #for token in $adv_opts.rep_param_double_resticted_default_optional_adv: + '$token.param_double_resticted_default_optional_adv' #end for #end if -#if $rep_param_double_resticted_default_mandatory_adv: +#if $adv_opts.rep_param_double_resticted_default_mandatory_adv: -double_resticted_default_mandatory_adv - #for token in $rep_param_double_resticted_default_mandatory_adv: - #if " " in str(token): - "$token.param_double_resticted_default_mandatory_adv" - #else - $token.param_double_resticted_default_mandatory_adv - #end if + #for token in $adv_opts.rep_param_double_resticted_default_mandatory_adv: + '$token.param_double_resticted_default_mandatory_adv' #end for #end if #end if @@ -458,7 +298,7 @@ ]]> - + @@ -468,7 +308,7 @@ - + @@ -478,7 +318,7 @@ - + @@ -488,7 +328,7 @@ - + @@ -498,7 +338,7 @@ - + @@ -508,7 +348,7 @@ - + @@ -518,7 +358,7 @@ - + @@ -528,7 +368,7 @@ - + @@ -538,7 +378,7 @@ - + @@ -548,7 +388,7 @@ - + @@ -558,7 +398,7 @@ - + @@ -568,7 +408,7 @@ - + @@ -578,7 +418,7 @@ - + @@ -588,7 +428,7 @@ - + @@ -598,7 +438,7 @@ - + @@ -608,7 +448,7 @@ - + @@ -618,7 +458,7 @@ - + @@ -628,7 +468,7 @@ - + @@ -638,7 +478,7 @@ - + @@ -648,7 +488,7 @@ - + @@ -659,7 +499,7 @@ - + @@ -669,7 +509,7 @@ - + @@ -679,7 +519,7 @@ - + @@ -689,7 +529,7 @@ - + @@ -699,7 +539,7 @@ - + @@ -709,7 +549,7 @@ - + @@ -719,7 +559,7 @@ - + @@ -729,7 +569,7 @@ - + @@ -739,7 +579,7 @@ - + @@ -749,7 +589,7 @@ - + @@ -759,7 +599,7 @@ - + @@ -769,7 +609,7 @@ - + @@ -779,7 +619,7 @@ - + @@ -789,7 +629,7 @@ - + @@ -799,7 +639,7 @@ - + @@ -809,7 +649,7 @@ - + @@ -819,7 +659,7 @@ - + @@ -829,7 +669,7 @@ - + @@ -839,7 +679,7 @@ - + @@ -849,7 +689,7 @@ - + diff --git a/tests/test-data/select.ctd b/tests/test-data/select.ctd index 77048022..2abc52de 100644 --- a/tests/test-data/select.ctd +++ b/tests/test-data/select.ctd @@ -7,37 +7,69 @@ - - - - - - - - - + + + + + + + + + + - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/test-data/select.xml b/tests/test-data/select.xml index 5c572ee5..624721f3 100644 --- a/tests/test-data/select.xml +++ b/tests/test-data/select.xml @@ -14,257 +14,377 @@ #if $param_wodefault_mandatory: -wodefault_mandatory - #if " " in str($param_wodefault_mandatory): - "$param_wodefault_mandatory" - #else - $param_wodefault_mandatory - #end if + '$param_wodefault_mandatory' #end if #if $param_default_mandatory: -default_mandatory - #if " " in str($param_default_mandatory): - "$param_default_mandatory" - #else - $param_default_mandatory - #end if + '$param_default_mandatory' #end if #if $param_wodefault_optional: -wodefault_optional - #if " " in str($param_wodefault_optional): - "$param_wodefault_optional" - #else - $param_wodefault_optional - #end if + '$param_wodefault_optional' #end if #if $param_default_optional: -default_optional - #if " " in str($param_default_optional): - "$param_default_optional" - #else - $param_default_optional - #end if + '$param_default_optional' #end if - -#if $rep_param_list_wodefault_optional: --list_wodefault_optional - #for token in $rep_param_list_wodefault_optional: - #if " " in str(token): - "$token.param_list_wodefault_optional" - #else - $token.param_list_wodefault_optional - #end if - #end for +#if $param_wodefault_mandatory_many: + -wodefault_mandatory_many + '$param_wodefault_mandatory_many' #end if - -#if $rep_param_list_wodefault_mandatory: --list_wodefault_mandatory - #for token in $rep_param_list_wodefault_mandatory: - #if " " in str(token): - "$token.param_list_wodefault_mandatory" - #else - $token.param_list_wodefault_mandatory - #end if - #end for +#if $param_default_mandatory_many: + -default_mandatory_many + '$param_default_mandatory_many' #end if - -#if $rep_param_list_default_optional: --list_default_optional - #for token in $rep_param_list_default_optional: - #if " " in str(token): - "$token.param_list_default_optional" - #else - $token.param_list_default_optional - #end if - #end for +#if $param_wodefault_optional_many: + -wodefault_optional_many + '$param_wodefault_optional_many' #end if - -#if $rep_param_list_default_mandatory: --list_default_mandatory - #for token in $rep_param_list_default_mandatory: - #if " " in str(token): - "$token.param_list_default_mandatory" - #else - $token.param_list_default_mandatory - #end if - #end for +#if $param_default_optional_many: + -default_optional_many + '$param_default_optional_many' #end if - -#if $rep_param_list_wodefault_optional_adv: --list_wodefault_optional_adv - #for token in $rep_param_list_wodefault_optional_adv: - #if " " in str(token): - "$token.param_list_wodefault_optional_adv" - #else - $token.param_list_wodefault_optional_adv - #end if - #end for +#if $param_list_wodefault_optional: + #set tmp=' '.join(["'%s'"%str(_) for _ in $param_list_wodefault_optional]) + -list_wodefault_optional + $tmp #end if - -#if $rep_param_list_wodefault_mandatory_adv: --list_wodefault_mandatory_adv - #for token in $rep_param_list_wodefault_mandatory_adv: - #if " " in str(token): - "$token.param_list_wodefault_mandatory_adv" - #else - $token.param_list_wodefault_mandatory_adv - #end if - #end for +#if $param_list_wodefault_mandatory: + #set tmp=' '.join(["'%s'"%str(_) for _ in $param_list_wodefault_mandatory]) + -list_wodefault_mandatory + $tmp #end if - -#if $rep_param_list_default_optional_adv: --list_default_optional_adv - #for token in $rep_param_list_default_optional_adv: - #if " " in str(token): - "$token.param_list_default_optional_adv" - #else - $token.param_list_default_optional_adv - #end if - #end for +#if $param_list_default_optional: + #set tmp=' '.join(["'%s'"%str(_) for _ in $param_list_default_optional]) + -list_default_optional + $tmp #end if - -#if $rep_param_list_default_mandatory_adv: --list_default_mandatory_adv - #for token in $rep_param_list_default_mandatory_adv: - #if " " in str(token): - "$token.param_list_default_mandatory_adv" - #else - $token.param_list_default_mandatory_adv - #end if - #end for +#if $param_list_default_mandatory: + #set tmp=' '.join(["'%s'"%str(_) for _ in $param_list_default_mandatory]) + -list_default_mandatory + $tmp +#end if +#if $param_list_wodefault_optional_many: + #set tmp=' '.join(["'%s'"%str(_) for _ in $param_list_wodefault_optional_many]) + -list_wodefault_optional_many + $tmp +#end if +#if $param_list_wodefault_mandatory_many: + #set tmp=' '.join(["'%s'"%str(_) for _ in $param_list_wodefault_mandatory_many]) + -list_wodefault_mandatory_many + $tmp +#end if +#if $param_list_default_optional_many: + #set tmp=' '.join(["'%s'"%str(_) for _ in $param_list_default_optional_many]) + -list_default_optional_many + $tmp +#end if +#if $param_list_default_mandatory_many: + #set tmp=' '.join(["'%s'"%str(_) for _ in $param_list_default_mandatory_many]) + -list_default_mandatory_many + $tmp #end if #if $adv_opts.adv_opts_selector=='advanced': #if $adv_opts.param_wodefault_mandatory_adv: -wodefault_mandatory_adv - #if " " in str($adv_opts.param_wodefault_mandatory_adv): - "$adv_opts.param_wodefault_mandatory_adv" - #else - $adv_opts.param_wodefault_mandatory_adv - #end if + '$adv_opts.param_wodefault_mandatory_adv' #end if #if $adv_opts.param_default_mandatory_adv: -default_mandatory_adv - #if " " in str($adv_opts.param_default_mandatory_adv): - "$adv_opts.param_default_mandatory_adv" - #else - $adv_opts.param_default_mandatory_adv - #end if + '$adv_opts.param_default_mandatory_adv' #end if #if $adv_opts.param_wodefault_optional_adv: -wodefault_optional_adv - #if " " in str($adv_opts.param_wodefault_optional_adv): - "$adv_opts.param_wodefault_optional_adv" - #else - $adv_opts.param_wodefault_optional_adv - #end if + '$adv_opts.param_wodefault_optional_adv' #end if #if $adv_opts.param_default_optional_adv: -default_optional_adv - #if " " in str($adv_opts.param_default_optional_adv): - "$adv_opts.param_default_optional_adv" - #else - $adv_opts.param_default_optional_adv - #end if + '$adv_opts.param_default_optional_adv' +#end if + #if $adv_opts.param_wodefault_mandatory_many_adv: + -wodefault_mandatory_many_adv + '$adv_opts.param_wodefault_mandatory_many_adv' +#end if + #if $adv_opts.param_default_mandatory_many_adv: + -default_mandatory_many_adv + '$adv_opts.param_default_mandatory_many_adv' +#end if + #if $adv_opts.param_wodefault_optional_many_adv: + -wodefault_optional_many_adv + '$adv_opts.param_wodefault_optional_many_adv' +#end if + #if $adv_opts.param_default_optional_many_adv: + -default_optional_many_adv + '$adv_opts.param_default_optional_many_adv' +#end if + #if $adv_opts.param_list_wodefault_optional_adv: + #set tmp=' '.join(["'%s'"%str(_) for _ in $adv_opts.param_list_wodefault_optional_adv]) + -list_wodefault_optional_adv + $tmp +#end if + #if $adv_opts.param_list_wodefault_mandatory_adv: + #set tmp=' '.join(["'%s'"%str(_) for _ in $adv_opts.param_list_wodefault_mandatory_adv]) + -list_wodefault_mandatory_adv + $tmp +#end if + #if $adv_opts.param_list_default_optional_adv: + #set tmp=' '.join(["'%s'"%str(_) for _ in $adv_opts.param_list_default_optional_adv]) + -list_default_optional_adv + $tmp +#end if + #if $adv_opts.param_list_default_mandatory_adv: + #set tmp=' '.join(["'%s'"%str(_) for _ in $adv_opts.param_list_default_mandatory_adv]) + -list_default_mandatory_adv + $tmp +#end if + #if $adv_opts.param_list_wodefault_optional_many_adv: + #set tmp=' '.join(["'%s'"%str(_) for _ in $adv_opts.param_list_wodefault_optional_many_adv]) + -list_wodefault_optional_many_adv + $tmp +#end if + #if $adv_opts.param_list_wodefault_mandatory_many_adv: + #set tmp=' '.join(["'%s'"%str(_) for _ in $adv_opts.param_list_wodefault_mandatory_many_adv]) + -list_wodefault_mandatory_many_adv + $tmp +#end if + #if $adv_opts.param_list_default_optional_many_adv: + #set tmp=' '.join(["'%s'"%str(_) for _ in $adv_opts.param_list_default_optional_many_adv]) + -list_default_optional_many_adv + $tmp +#end if + #if $adv_opts.param_list_default_mandatory_many_adv: + #set tmp=' '.join(["'%s'"%str(_) for _ in $adv_opts.param_list_default_mandatory_many_adv]) + -list_default_mandatory_many_adv + $tmp #end if #end if > $param_stdout ]]> + - + + + value != "select a value" - + - + + - + + + + + + + + + + + + + + + + + value != "select a value" + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + value != "select a value" - + - + + - - + + + + + + + + + + + + + + + + + + + + + value != "select a value" + + + + + + + + + + + + + + + + + + + + value != "select a value" - - - + - + + - - - + + - - - + - + + - - - + + + + + value != "select a value" + + + + + + + - - - + + + - - - + + + + + + + + + - - - + + + + value != "select a value" - - - + - + + - + - + + - + + + + + + + + + + + + + + + value != "select a value" + + - + + + - + - + + + diff --git a/tests/test-data/string.xml b/tests/test-data/string.xml index 2c6cca80..5b41dbd3 100644 --- a/tests/test-data/string.xml +++ b/tests/test-data/string.xml @@ -13,35 +13,35 @@ $param_stdout ]]> - + @@ -49,7 +49,7 @@ - + @@ -57,7 +57,7 @@ - + @@ -65,7 +65,7 @@ - + @@ -74,7 +74,7 @@ - + @@ -82,7 +82,7 @@ - + @@ -90,7 +90,7 @@ - + @@ -98,7 +98,7 @@ - + From 84cd70fd88b9413adbf46d9adb1f8ad278516d3f Mon Sep 17 00:00:00 2001 From: M Bernt Date: Thu, 18 Jul 2019 15:35:51 +0200 Subject: [PATCH 015/121] fix specification of select default value dont set it via the param's value attribute, but the selected attribute of the options (the latter was already the case) --- galaxy/converter.py | 8 ++++++-- tests/test-data/select.xml | 32 ++++++++++++++++---------------- 2 files changed, 22 insertions(+), 18 deletions(-) diff --git a/galaxy/converter.py b/galaxy/converter.py index f6709b6b..79df6bc6 100755 --- a/galaxy/converter.py +++ b/galaxy/converter.py @@ -714,7 +714,10 @@ def create_param_attribute_list(param_node, param, supported_file_formats): # check for default value if param.default is not None and param.default is not _Null: - if type(param.default) is list: + # defaults of selects are set via the selected attribute of the options + if param_type == "select": + pass + elif type(param.default) is list: # we ASSUME that a list of parameters looks like: # $ tool -ignore He Ar Xe # meaning, that, for example, Helium, Argon and Xenon will be ignored @@ -722,7 +725,6 @@ def create_param_attribute_list(param_node, param, supported_file_formats): elif param_type != "boolean": param_node.attrib["value"] = str(param.default) - else: # simple boolean with a default if param.default is True: @@ -835,6 +837,8 @@ def is_boolean_parameter(param): def is_selection_parameter(param): """ determines if there are choices for the parameter and its not bool + @param param the ctd parameter + @return True iff a selection parameter """ if type(param.restrictions) is _Choices: return set(param.restrictions.choices) != set(["true", "false"]) diff --git a/tests/test-data/select.xml b/tests/test-data/select.xml index 624721f3..00f83850 100644 --- a/tests/test-data/select.xml +++ b/tests/test-data/select.xml @@ -169,7 +169,7 @@ value != "select a value" - + @@ -181,7 +181,7 @@ - + @@ -196,7 +196,7 @@ value != "select a value" - + @@ -210,7 +210,7 @@ - + @@ -231,13 +231,13 @@ value != "select a value" - + - + @@ -259,14 +259,14 @@ value != "select a value" - + - + @@ -282,7 +282,7 @@ value != "select a value" - + @@ -294,7 +294,7 @@ - + @@ -309,7 +309,7 @@ value != "select a value" - + @@ -323,7 +323,7 @@ - + @@ -344,13 +344,13 @@ value != "select a value" - + - + @@ -372,14 +372,14 @@ value != "select a value" - + - + From c966a9df44e558ff407fc914cd7b2937d9922b7c Mon Sep 17 00:00:00 2001 From: M Bernt Date: Thu, 18 Jul 2019 16:11:16 +0200 Subject: [PATCH 016/121] fix indentation of advanced parameters in command block --- common/utils.py | 11 ++ galaxy/converter.py | 4 +- tests/test-data/bool.xml | 4 +- tests/test-data/float.xml | 96 ++++++------- tests/test-data/ifile.xml | 120 ++++++++-------- tests/test-data/integer.xml | 96 ++++++------- tests/test-data/repeat.xml | 280 ++++++++++++++++++------------------ tests/test-data/select.xml | 144 +++++++++---------- tests/test-data/string.xml | 24 ++-- 9 files changed, 395 insertions(+), 384 deletions(-) diff --git a/common/utils.py b/common/utils.py index b5b0b1f3..c69ba1b2 100644 --- a/common/utils.py +++ b/common/utils.py @@ -316,3 +316,14 @@ def extract_command_line_prefix(param, ctd_model): # there was no mapping, so for the cli name we will use a '-' in the prefix param_cli_name = "-" + param_name return param_cli_name + + +def indent(s, indentation=" "): + """ + helper function to indent text + @param s the text (a string) + @param indentation the desired indentation + @return indented text + """ + return "\n".join([indentation+_ for _ in s.splitlines()])+"\n" + diff --git a/galaxy/converter.py b/galaxy/converter.py index 79df6bc6..84ea4508 100755 --- a/galaxy/converter.py +++ b/galaxy/converter.py @@ -454,7 +454,7 @@ def create_command(tool, model, **kwargs): command += "#end if\n" if param.advanced and param.type is not _OutFile: - advanced_command += " %s" % command + advanced_command += "%s" % utils.indent(command) else: final_command += command @@ -462,7 +462,7 @@ def create_command(tool, model, **kwargs): final_command += "%s%s%s\n" % (advanced_command_start, advanced_command, advanced_command_end) if not found_output_parameter: - final_command += "> $param_stdout\n" + final_command += "> $param_stdout\n" command_node = add_child_node(tool, "command") command_node.text = CDATA(final_command) diff --git a/tests/test-data/bool.xml b/tests/test-data/bool.xml index 65d0eaad..0e331fa1 100644 --- a/tests/test-data/bool.xml +++ b/tests/test-data/bool.xml @@ -15,8 +15,8 @@ $param_true $param_false #if $adv_opts.adv_opts_selector=='advanced': - $adv_opts.param_true_adv - $adv_opts.param_false_adv + $adv_opts.param_true_adv + $adv_opts.param_false_adv #end if > $param_stdout ]]> diff --git a/tests/test-data/float.xml b/tests/test-data/float.xml index 3a5b2bed..09ec5150 100644 --- a/tests/test-data/float.xml +++ b/tests/test-data/float.xml @@ -61,54 +61,54 @@ -param_default_optional_unrestricted_minmax $param_param_default_optional_unrestricted_minmax #end if #if $adv_opts.adv_opts_selector=='advanced': - #if str($adv_opts.param_param_wodefault_mandatory_unrestricted_adv): - -param_wodefault_mandatory_unrestricted_adv $adv_opts.param_param_wodefault_mandatory_unrestricted_adv -#end if - #if str($adv_opts.param_param_default_mandatory_unrestricted_adv): - -param_default_mandatory_unrestricted_adv $adv_opts.param_param_default_mandatory_unrestricted_adv -#end if - #if str($adv_opts.param_param_wodefault_optional_unrestricted_adv): - -param_wodefault_optional_unrestricted_adv $adv_opts.param_param_wodefault_optional_unrestricted_adv -#end if - #if str($adv_opts.param_param_default_optional_unrestricted_adv): - -param_default_optional_unrestricted_adv $adv_opts.param_param_default_optional_unrestricted_adv -#end if - #if str($adv_opts.param_param_wodefault_mandatory_unrestricted_adv_min): - -param_wodefault_mandatory_unrestricted_adv_min $adv_opts.param_param_wodefault_mandatory_unrestricted_adv_min -#end if - #if str($adv_opts.param_param_default_mandatory_unrestricted_adv_min): - -param_default_mandatory_unrestricted_adv_min $adv_opts.param_param_default_mandatory_unrestricted_adv_min -#end if - #if str($adv_opts.param_param_wodefault_optional_unrestricted_adv_min): - -param_wodefault_optional_unrestricted_adv_min $adv_opts.param_param_wodefault_optional_unrestricted_adv_min -#end if - #if str($adv_opts.param_param_default_optional_unrestricted_adv_min): - -param_default_optional_unrestricted_adv_min $adv_opts.param_param_default_optional_unrestricted_adv_min -#end if - #if str($adv_opts.param_param_wodefault_mandatory_unrestricted_adv_max): - -param_wodefault_mandatory_unrestricted_adv_max $adv_opts.param_param_wodefault_mandatory_unrestricted_adv_max -#end if - #if str($adv_opts.param_param_default_mandatory_unrestricted_adv_max): - -param_default_mandatory_unrestricted_adv_max $adv_opts.param_param_default_mandatory_unrestricted_adv_max -#end if - #if str($adv_opts.param_param_wodefault_optional_unrestricted_adv_max): - -param_wodefault_optional_unrestricted_adv_max $adv_opts.param_param_wodefault_optional_unrestricted_adv_max -#end if - #if str($adv_opts.param_param_default_optional_unrestricted_adv_max): - -param_default_optional_unrestricted_adv_max $adv_opts.param_param_default_optional_unrestricted_adv_max -#end if - #if str($adv_opts.param_param_wodefault_mandatory_unrestricted_adv_minmax): - -param_wodefault_mandatory_unrestricted_adv_minmax $adv_opts.param_param_wodefault_mandatory_unrestricted_adv_minmax -#end if - #if str($adv_opts.param_param_default_mandatory_unrestricted_adv_minmax): - -param_default_mandatory_unrestricted_adv_minmax $adv_opts.param_param_default_mandatory_unrestricted_adv_minmax -#end if - #if str($adv_opts.param_param_wodefault_optional_unrestricted_adv_minmax): - -param_wodefault_optional_unrestricted_adv_minmax $adv_opts.param_param_wodefault_optional_unrestricted_adv_minmax -#end if - #if str($adv_opts.param_param_default_optional_unrestricted_adv_minmax): - -param_default_optional_unrestricted_adv_minmax $adv_opts.param_param_default_optional_unrestricted_adv_minmax -#end if + #if str($adv_opts.param_param_wodefault_mandatory_unrestricted_adv): + -param_wodefault_mandatory_unrestricted_adv $adv_opts.param_param_wodefault_mandatory_unrestricted_adv + #end if + #if str($adv_opts.param_param_default_mandatory_unrestricted_adv): + -param_default_mandatory_unrestricted_adv $adv_opts.param_param_default_mandatory_unrestricted_adv + #end if + #if str($adv_opts.param_param_wodefault_optional_unrestricted_adv): + -param_wodefault_optional_unrestricted_adv $adv_opts.param_param_wodefault_optional_unrestricted_adv + #end if + #if str($adv_opts.param_param_default_optional_unrestricted_adv): + -param_default_optional_unrestricted_adv $adv_opts.param_param_default_optional_unrestricted_adv + #end if + #if str($adv_opts.param_param_wodefault_mandatory_unrestricted_adv_min): + -param_wodefault_mandatory_unrestricted_adv_min $adv_opts.param_param_wodefault_mandatory_unrestricted_adv_min + #end if + #if str($adv_opts.param_param_default_mandatory_unrestricted_adv_min): + -param_default_mandatory_unrestricted_adv_min $adv_opts.param_param_default_mandatory_unrestricted_adv_min + #end if + #if str($adv_opts.param_param_wodefault_optional_unrestricted_adv_min): + -param_wodefault_optional_unrestricted_adv_min $adv_opts.param_param_wodefault_optional_unrestricted_adv_min + #end if + #if str($adv_opts.param_param_default_optional_unrestricted_adv_min): + -param_default_optional_unrestricted_adv_min $adv_opts.param_param_default_optional_unrestricted_adv_min + #end if + #if str($adv_opts.param_param_wodefault_mandatory_unrestricted_adv_max): + -param_wodefault_mandatory_unrestricted_adv_max $adv_opts.param_param_wodefault_mandatory_unrestricted_adv_max + #end if + #if str($adv_opts.param_param_default_mandatory_unrestricted_adv_max): + -param_default_mandatory_unrestricted_adv_max $adv_opts.param_param_default_mandatory_unrestricted_adv_max + #end if + #if str($adv_opts.param_param_wodefault_optional_unrestricted_adv_max): + -param_wodefault_optional_unrestricted_adv_max $adv_opts.param_param_wodefault_optional_unrestricted_adv_max + #end if + #if str($adv_opts.param_param_default_optional_unrestricted_adv_max): + -param_default_optional_unrestricted_adv_max $adv_opts.param_param_default_optional_unrestricted_adv_max + #end if + #if str($adv_opts.param_param_wodefault_mandatory_unrestricted_adv_minmax): + -param_wodefault_mandatory_unrestricted_adv_minmax $adv_opts.param_param_wodefault_mandatory_unrestricted_adv_minmax + #end if + #if str($adv_opts.param_param_default_mandatory_unrestricted_adv_minmax): + -param_default_mandatory_unrestricted_adv_minmax $adv_opts.param_param_default_mandatory_unrestricted_adv_minmax + #end if + #if str($adv_opts.param_param_wodefault_optional_unrestricted_adv_minmax): + -param_wodefault_optional_unrestricted_adv_minmax $adv_opts.param_param_wodefault_optional_unrestricted_adv_minmax + #end if + #if str($adv_opts.param_param_default_optional_unrestricted_adv_minmax): + -param_default_optional_unrestricted_adv_minmax $adv_opts.param_param_default_optional_unrestricted_adv_minmax + #end if #end if > $param_stdout ]]> diff --git a/tests/test-data/ifile.xml b/tests/test-data/ifile.xml index feddc933..76c471e8 100644 --- a/tests/test-data/ifile.xml +++ b/tests/test-data/ifile.xml @@ -73,66 +73,66 @@ $token #end for #if $adv_opts.adv_opts_selector=='advanced': - #if str($adv_opts.param_mandatory_noformat_adv): - -mandatory_noformat_adv $adv_opts.param_mandatory_noformat_adv -#end if - #if str($adv_opts.param_optional_noformat_adv): - -optional_noformat_adv $adv_opts.param_optional_noformat_adv -#end if - #if str($adv_opts.param_mandatory_oneformat_adv): - -mandatory_oneformat_adv $adv_opts.param_mandatory_oneformat_adv -#end if - #if str($adv_opts.param_optional_oneformat_adv): - -optional_oneformat_adv $adv_opts.param_optional_oneformat_adv -#end if - #if str($adv_opts.param_mandatory_multformat_adv): - -mandatory_multformat_adv $adv_opts.param_mandatory_multformat_adv -#end if - #if str($adv_opts.param_optional_multformat_adv): - -optional_multformat_adv $adv_opts.param_optional_multformat_adv -#end if - #if str($adv_opts.param_default_mandatory_noformat_adv): - -default_mandatory_noformat_adv $adv_opts.param_default_mandatory_noformat_adv -#end if - #if str($adv_opts.param_default_optional_noformat_adv): - -default_optional_noformat_adv $adv_opts.param_default_optional_noformat_adv -#end if - #if str($adv_opts.param_default_mandatory_oneformat_adv): - -default_mandatory_oneformat_adv $adv_opts.param_default_mandatory_oneformat_adv -#end if - #if str($adv_opts.param_default_optional_oneformat_adv): - -default_optional_oneformat_adv $adv_opts.param_default_optional_oneformat_adv -#end if - #if str($adv_opts.param_default_mandatory_multformat_adv): - -default_mandatory_multformat_adv $adv_opts.param_default_mandatory_multformat_adv -#end if - #if str($adv_opts.param_default_optional_multformat_adv): - -default_optional_multformat_adv $adv_opts.param_default_optional_multformat_adv -#end if - -list_mandatory_noformat_adv - #for token in $param_list_mandatory_noformat_adv: - $token - #end for - -list_optional_noformat_adv - #for token in $param_list_optional_noformat_adv: - $token - #end for - -list_mandatory_oneformat_adv - #for token in $param_list_mandatory_oneformat_adv: - $token - #end for - -list_optional_oneformat_adv - #for token in $param_list_optional_oneformat_adv: - $token - #end for - -list_mandatory_multformat_adv - #for token in $param_list_mandatory_multformat_adv: - $token - #end for - -list_optional_multformat_adv - #for token in $param_list_optional_multformat_adv: - $token - #end for + #if str($adv_opts.param_mandatory_noformat_adv): + -mandatory_noformat_adv $adv_opts.param_mandatory_noformat_adv + #end if + #if str($adv_opts.param_optional_noformat_adv): + -optional_noformat_adv $adv_opts.param_optional_noformat_adv + #end if + #if str($adv_opts.param_mandatory_oneformat_adv): + -mandatory_oneformat_adv $adv_opts.param_mandatory_oneformat_adv + #end if + #if str($adv_opts.param_optional_oneformat_adv): + -optional_oneformat_adv $adv_opts.param_optional_oneformat_adv + #end if + #if str($adv_opts.param_mandatory_multformat_adv): + -mandatory_multformat_adv $adv_opts.param_mandatory_multformat_adv + #end if + #if str($adv_opts.param_optional_multformat_adv): + -optional_multformat_adv $adv_opts.param_optional_multformat_adv + #end if + #if str($adv_opts.param_default_mandatory_noformat_adv): + -default_mandatory_noformat_adv $adv_opts.param_default_mandatory_noformat_adv + #end if + #if str($adv_opts.param_default_optional_noformat_adv): + -default_optional_noformat_adv $adv_opts.param_default_optional_noformat_adv + #end if + #if str($adv_opts.param_default_mandatory_oneformat_adv): + -default_mandatory_oneformat_adv $adv_opts.param_default_mandatory_oneformat_adv + #end if + #if str($adv_opts.param_default_optional_oneformat_adv): + -default_optional_oneformat_adv $adv_opts.param_default_optional_oneformat_adv + #end if + #if str($adv_opts.param_default_mandatory_multformat_adv): + -default_mandatory_multformat_adv $adv_opts.param_default_mandatory_multformat_adv + #end if + #if str($adv_opts.param_default_optional_multformat_adv): + -default_optional_multformat_adv $adv_opts.param_default_optional_multformat_adv + #end if + -list_mandatory_noformat_adv + #for token in $param_list_mandatory_noformat_adv: + $token + #end for + -list_optional_noformat_adv + #for token in $param_list_optional_noformat_adv: + $token + #end for + -list_mandatory_oneformat_adv + #for token in $param_list_mandatory_oneformat_adv: + $token + #end for + -list_optional_oneformat_adv + #for token in $param_list_optional_oneformat_adv: + $token + #end for + -list_mandatory_multformat_adv + #for token in $param_list_mandatory_multformat_adv: + $token + #end for + -list_optional_multformat_adv + #for token in $param_list_optional_multformat_adv: + $token + #end for #end if > $param_stdout ]]> diff --git a/tests/test-data/integer.xml b/tests/test-data/integer.xml index 1a8bbdc5..8be1a028 100644 --- a/tests/test-data/integer.xml +++ b/tests/test-data/integer.xml @@ -61,54 +61,54 @@ -default_optional_unrestricted_minmax $param_default_optional_unrestricted_minmax #end if #if $adv_opts.adv_opts_selector=='advanced': - #if str($adv_opts.param_wodefault_mandatory_unrestricted_adv): - -wodefault_mandatory_unrestricted_adv $adv_opts.param_wodefault_mandatory_unrestricted_adv -#end if - #if str($adv_opts.param_default_mandatory_unrestricted_adv): - -default_mandatory_unrestricted_adv $adv_opts.param_default_mandatory_unrestricted_adv -#end if - #if str($adv_opts.param_wodefault_optional_unrestricted_adv): - -wodefault_optional_unrestricted_adv $adv_opts.param_wodefault_optional_unrestricted_adv -#end if - #if str($adv_opts.param_default_optional_unrestricted_adv): - -default_optional_unrestricted_adv $adv_opts.param_default_optional_unrestricted_adv -#end if - #if str($adv_opts.param_wodefault_mandatory_unrestricted_adv_min): - -wodefault_mandatory_unrestricted_adv_min $adv_opts.param_wodefault_mandatory_unrestricted_adv_min -#end if - #if str($adv_opts.param_default_mandatory_unrestricted_adv_min): - -default_mandatory_unrestricted_adv_min $adv_opts.param_default_mandatory_unrestricted_adv_min -#end if - #if str($adv_opts.param_wodefault_optional_unrestricted_adv_min): - -wodefault_optional_unrestricted_adv_min $adv_opts.param_wodefault_optional_unrestricted_adv_min -#end if - #if str($adv_opts.param_default_optional_unrestricted_adv_min): - -default_optional_unrestricted_adv_min $adv_opts.param_default_optional_unrestricted_adv_min -#end if - #if str($adv_opts.param_wodefault_mandatory_unrestricted_adv_max): - -wodefault_mandatory_unrestricted_adv_max $adv_opts.param_wodefault_mandatory_unrestricted_adv_max -#end if - #if str($adv_opts.param_default_mandatory_unrestricted_adv_max): - -default_mandatory_unrestricted_adv_max $adv_opts.param_default_mandatory_unrestricted_adv_max -#end if - #if str($adv_opts.param_wodefault_optional_unrestricted_adv_max): - -wodefault_optional_unrestricted_adv_max $adv_opts.param_wodefault_optional_unrestricted_adv_max -#end if - #if str($adv_opts.param_default_optional_unrestricted_adv_max): - -default_optional_unrestricted_adv_max $adv_opts.param_default_optional_unrestricted_adv_max -#end if - #if str($adv_opts.param_wodefault_mandatory_unrestricted_adv_minmax): - -wodefault_mandatory_unrestricted_adv_minmax $adv_opts.param_wodefault_mandatory_unrestricted_adv_minmax -#end if - #if str($adv_opts.param_default_mandatory_unrestricted_adv_minmax): - -default_mandatory_unrestricted_adv_minmax $adv_opts.param_default_mandatory_unrestricted_adv_minmax -#end if - #if str($adv_opts.param_wodefault_optional_unrestricted_adv_minmax): - -wodefault_optional_unrestricted_adv_minmax $adv_opts.param_wodefault_optional_unrestricted_adv_minmax -#end if - #if str($adv_opts.param_default_optional_unrestricted_adv_minmax): - -default_optional_unrestricted_adv_minmax $adv_opts.param_default_optional_unrestricted_adv_minmax -#end if + #if str($adv_opts.param_wodefault_mandatory_unrestricted_adv): + -wodefault_mandatory_unrestricted_adv $adv_opts.param_wodefault_mandatory_unrestricted_adv + #end if + #if str($adv_opts.param_default_mandatory_unrestricted_adv): + -default_mandatory_unrestricted_adv $adv_opts.param_default_mandatory_unrestricted_adv + #end if + #if str($adv_opts.param_wodefault_optional_unrestricted_adv): + -wodefault_optional_unrestricted_adv $adv_opts.param_wodefault_optional_unrestricted_adv + #end if + #if str($adv_opts.param_default_optional_unrestricted_adv): + -default_optional_unrestricted_adv $adv_opts.param_default_optional_unrestricted_adv + #end if + #if str($adv_opts.param_wodefault_mandatory_unrestricted_adv_min): + -wodefault_mandatory_unrestricted_adv_min $adv_opts.param_wodefault_mandatory_unrestricted_adv_min + #end if + #if str($adv_opts.param_default_mandatory_unrestricted_adv_min): + -default_mandatory_unrestricted_adv_min $adv_opts.param_default_mandatory_unrestricted_adv_min + #end if + #if str($adv_opts.param_wodefault_optional_unrestricted_adv_min): + -wodefault_optional_unrestricted_adv_min $adv_opts.param_wodefault_optional_unrestricted_adv_min + #end if + #if str($adv_opts.param_default_optional_unrestricted_adv_min): + -default_optional_unrestricted_adv_min $adv_opts.param_default_optional_unrestricted_adv_min + #end if + #if str($adv_opts.param_wodefault_mandatory_unrestricted_adv_max): + -wodefault_mandatory_unrestricted_adv_max $adv_opts.param_wodefault_mandatory_unrestricted_adv_max + #end if + #if str($adv_opts.param_default_mandatory_unrestricted_adv_max): + -default_mandatory_unrestricted_adv_max $adv_opts.param_default_mandatory_unrestricted_adv_max + #end if + #if str($adv_opts.param_wodefault_optional_unrestricted_adv_max): + -wodefault_optional_unrestricted_adv_max $adv_opts.param_wodefault_optional_unrestricted_adv_max + #end if + #if str($adv_opts.param_default_optional_unrestricted_adv_max): + -default_optional_unrestricted_adv_max $adv_opts.param_default_optional_unrestricted_adv_max + #end if + #if str($adv_opts.param_wodefault_mandatory_unrestricted_adv_minmax): + -wodefault_mandatory_unrestricted_adv_minmax $adv_opts.param_wodefault_mandatory_unrestricted_adv_minmax + #end if + #if str($adv_opts.param_default_mandatory_unrestricted_adv_minmax): + -default_mandatory_unrestricted_adv_minmax $adv_opts.param_default_mandatory_unrestricted_adv_minmax + #end if + #if str($adv_opts.param_wodefault_optional_unrestricted_adv_minmax): + -wodefault_optional_unrestricted_adv_minmax $adv_opts.param_wodefault_optional_unrestricted_adv_minmax + #end if + #if str($adv_opts.param_default_optional_unrestricted_adv_minmax): + -default_optional_unrestricted_adv_minmax $adv_opts.param_default_optional_unrestricted_adv_minmax + #end if #end if > $param_stdout ]]> diff --git a/tests/test-data/repeat.xml b/tests/test-data/repeat.xml index e7d09790..93c55e18 100644 --- a/tests/test-data/repeat.xml +++ b/tests/test-data/repeat.xml @@ -153,146 +153,146 @@ #end for #end if #if $adv_opts.adv_opts_selector=='advanced': - -#if $adv_opts.rep_param_string_wodefault_optional_adv: --string_wodefault_optional_adv - #for token in $adv_opts.rep_param_string_wodefault_optional_adv: - '$token.param_string_wodefault_optional_adv' - #end for -#end if - -#if $adv_opts.rep_param_string_wodefault_mandatory_adv: --string_wodefault_mandatory_adv - #for token in $adv_opts.rep_param_string_wodefault_mandatory_adv: - '$token.param_string_wodefault_mandatory_adv' - #end for -#end if - -#if $adv_opts.rep_param_string_default_optional_adv: --string_default_optional_adv - #for token in $adv_opts.rep_param_string_default_optional_adv: - '$token.param_string_default_optional_adv' - #end for -#end if - -#if $adv_opts.rep_param_string_default_mandatory_adv: --string_default_mandatory_adv - #for token in $adv_opts.rep_param_string_default_mandatory_adv: - '$token.param_string_default_mandatory_adv' - #end for -#end if - -#if $adv_opts.rep_param_int_wodefault_optional_adv: --int_wodefault_optional_adv - #for token in $adv_opts.rep_param_int_wodefault_optional_adv: - '$token.param_int_wodefault_optional_adv' - #end for -#end if - -#if $adv_opts.rep_param_int_wodefault_mandatory_adv: --int_wodefault_mandatory_adv - #for token in $adv_opts.rep_param_int_wodefault_mandatory_adv: - '$token.param_int_wodefault_mandatory_adv' - #end for -#end if - -#if $adv_opts.rep_param_int_default_optional_adv: --int_default_optional_adv - #for token in $adv_opts.rep_param_int_default_optional_adv: - '$token.param_int_default_optional_adv' - #end for -#end if - -#if $adv_opts.rep_param_int_default_mandatory_adv: --int_default_mandatory_adv - #for token in $adv_opts.rep_param_int_default_mandatory_adv: - '$token.param_int_default_mandatory_adv' - #end for -#end if - -#if $adv_opts.rep_param_int_resticted_wodefault_optional_adv: --int_resticted_wodefault_optional_adv - #for token in $adv_opts.rep_param_int_resticted_wodefault_optional_adv: - '$token.param_int_resticted_wodefault_optional_adv' - #end for -#end if - -#if $adv_opts.rep_param_int_resticted_wodefault_mandatory_adv: --int_resticted_wodefault_mandatory_adv - #for token in $adv_opts.rep_param_int_resticted_wodefault_mandatory_adv: - '$token.param_int_resticted_wodefault_mandatory_adv' - #end for -#end if - -#if $adv_opts.rep_param_int_resticted_default_optional_adv: --int_resticted_default_optional_adv - #for token in $adv_opts.rep_param_int_resticted_default_optional_adv: - '$token.param_int_resticted_default_optional_adv' - #end for -#end if - -#if $adv_opts.rep_param_int_resticted_default_mandatory_adv: --int_resticted_default_mandatory_adv - #for token in $adv_opts.rep_param_int_resticted_default_mandatory_adv: - '$token.param_int_resticted_default_mandatory_adv' - #end for -#end if - -#if $adv_opts.rep_param_double_wodefault_optional_adv: --double_wodefault_optional_adv - #for token in $adv_opts.rep_param_double_wodefault_optional_adv: - '$token.param_double_wodefault_optional_adv' - #end for -#end if - -#if $adv_opts.rep_param_double_wodefault_mandatory_adv: --double_wodefault_mandatory_adv - #for token in $adv_opts.rep_param_double_wodefault_mandatory_adv: - '$token.param_double_wodefault_mandatory_adv' - #end for -#end if - -#if $adv_opts.rep_param_double_default_optional_adv: --double_default_optional_adv - #for token in $adv_opts.rep_param_double_default_optional_adv: - '$token.param_double_default_optional_adv' - #end for -#end if - -#if $adv_opts.rep_param_double_default_mandatory_adv: --double_default_mandatory_adv - #for token in $adv_opts.rep_param_double_default_mandatory_adv: - '$token.param_double_default_mandatory_adv' - #end for -#end if - -#if $adv_opts.rep_param_double_resticted_wodefault_optional_adv: --double_resticted_wodefault_optional_adv - #for token in $adv_opts.rep_param_double_resticted_wodefault_optional_adv: - '$token.param_double_resticted_wodefault_optional_adv' - #end for -#end if - -#if $adv_opts.rep_param_double_resticted_wodefault_mandatory_adv: --double_resticted_wodefault_mandatory_adv - #for token in $adv_opts.rep_param_double_resticted_wodefault_mandatory_adv: - '$token.param_double_resticted_wodefault_mandatory_adv' - #end for -#end if - -#if $adv_opts.rep_param_double_resticted_default_optional_adv: --double_resticted_default_optional_adv - #for token in $adv_opts.rep_param_double_resticted_default_optional_adv: - '$token.param_double_resticted_default_optional_adv' - #end for -#end if - -#if $adv_opts.rep_param_double_resticted_default_mandatory_adv: --double_resticted_default_mandatory_adv - #for token in $adv_opts.rep_param_double_resticted_default_mandatory_adv: - '$token.param_double_resticted_default_mandatory_adv' - #end for -#end if + + #if $adv_opts.rep_param_string_wodefault_optional_adv: + -string_wodefault_optional_adv + #for token in $adv_opts.rep_param_string_wodefault_optional_adv: + '$token.param_string_wodefault_optional_adv' + #end for + #end if + + #if $adv_opts.rep_param_string_wodefault_mandatory_adv: + -string_wodefault_mandatory_adv + #for token in $adv_opts.rep_param_string_wodefault_mandatory_adv: + '$token.param_string_wodefault_mandatory_adv' + #end for + #end if + + #if $adv_opts.rep_param_string_default_optional_adv: + -string_default_optional_adv + #for token in $adv_opts.rep_param_string_default_optional_adv: + '$token.param_string_default_optional_adv' + #end for + #end if + + #if $adv_opts.rep_param_string_default_mandatory_adv: + -string_default_mandatory_adv + #for token in $adv_opts.rep_param_string_default_mandatory_adv: + '$token.param_string_default_mandatory_adv' + #end for + #end if + + #if $adv_opts.rep_param_int_wodefault_optional_adv: + -int_wodefault_optional_adv + #for token in $adv_opts.rep_param_int_wodefault_optional_adv: + '$token.param_int_wodefault_optional_adv' + #end for + #end if + + #if $adv_opts.rep_param_int_wodefault_mandatory_adv: + -int_wodefault_mandatory_adv + #for token in $adv_opts.rep_param_int_wodefault_mandatory_adv: + '$token.param_int_wodefault_mandatory_adv' + #end for + #end if + + #if $adv_opts.rep_param_int_default_optional_adv: + -int_default_optional_adv + #for token in $adv_opts.rep_param_int_default_optional_adv: + '$token.param_int_default_optional_adv' + #end for + #end if + + #if $adv_opts.rep_param_int_default_mandatory_adv: + -int_default_mandatory_adv + #for token in $adv_opts.rep_param_int_default_mandatory_adv: + '$token.param_int_default_mandatory_adv' + #end for + #end if + + #if $adv_opts.rep_param_int_resticted_wodefault_optional_adv: + -int_resticted_wodefault_optional_adv + #for token in $adv_opts.rep_param_int_resticted_wodefault_optional_adv: + '$token.param_int_resticted_wodefault_optional_adv' + #end for + #end if + + #if $adv_opts.rep_param_int_resticted_wodefault_mandatory_adv: + -int_resticted_wodefault_mandatory_adv + #for token in $adv_opts.rep_param_int_resticted_wodefault_mandatory_adv: + '$token.param_int_resticted_wodefault_mandatory_adv' + #end for + #end if + + #if $adv_opts.rep_param_int_resticted_default_optional_adv: + -int_resticted_default_optional_adv + #for token in $adv_opts.rep_param_int_resticted_default_optional_adv: + '$token.param_int_resticted_default_optional_adv' + #end for + #end if + + #if $adv_opts.rep_param_int_resticted_default_mandatory_adv: + -int_resticted_default_mandatory_adv + #for token in $adv_opts.rep_param_int_resticted_default_mandatory_adv: + '$token.param_int_resticted_default_mandatory_adv' + #end for + #end if + + #if $adv_opts.rep_param_double_wodefault_optional_adv: + -double_wodefault_optional_adv + #for token in $adv_opts.rep_param_double_wodefault_optional_adv: + '$token.param_double_wodefault_optional_adv' + #end for + #end if + + #if $adv_opts.rep_param_double_wodefault_mandatory_adv: + -double_wodefault_mandatory_adv + #for token in $adv_opts.rep_param_double_wodefault_mandatory_adv: + '$token.param_double_wodefault_mandatory_adv' + #end for + #end if + + #if $adv_opts.rep_param_double_default_optional_adv: + -double_default_optional_adv + #for token in $adv_opts.rep_param_double_default_optional_adv: + '$token.param_double_default_optional_adv' + #end for + #end if + + #if $adv_opts.rep_param_double_default_mandatory_adv: + -double_default_mandatory_adv + #for token in $adv_opts.rep_param_double_default_mandatory_adv: + '$token.param_double_default_mandatory_adv' + #end for + #end if + + #if $adv_opts.rep_param_double_resticted_wodefault_optional_adv: + -double_resticted_wodefault_optional_adv + #for token in $adv_opts.rep_param_double_resticted_wodefault_optional_adv: + '$token.param_double_resticted_wodefault_optional_adv' + #end for + #end if + + #if $adv_opts.rep_param_double_resticted_wodefault_mandatory_adv: + -double_resticted_wodefault_mandatory_adv + #for token in $adv_opts.rep_param_double_resticted_wodefault_mandatory_adv: + '$token.param_double_resticted_wodefault_mandatory_adv' + #end for + #end if + + #if $adv_opts.rep_param_double_resticted_default_optional_adv: + -double_resticted_default_optional_adv + #for token in $adv_opts.rep_param_double_resticted_default_optional_adv: + '$token.param_double_resticted_default_optional_adv' + #end for + #end if + + #if $adv_opts.rep_param_double_resticted_default_mandatory_adv: + -double_resticted_default_mandatory_adv + #for token in $adv_opts.rep_param_double_resticted_default_mandatory_adv: + '$token.param_double_resticted_default_mandatory_adv' + #end for + #end if #end if > $param_stdout ]]> diff --git a/tests/test-data/select.xml b/tests/test-data/select.xml index 00f83850..c9318af9 100644 --- a/tests/test-data/select.xml +++ b/tests/test-data/select.xml @@ -85,78 +85,78 @@ $tmp #end if #if $adv_opts.adv_opts_selector=='advanced': - #if $adv_opts.param_wodefault_mandatory_adv: - -wodefault_mandatory_adv - '$adv_opts.param_wodefault_mandatory_adv' -#end if - #if $adv_opts.param_default_mandatory_adv: - -default_mandatory_adv - '$adv_opts.param_default_mandatory_adv' -#end if - #if $adv_opts.param_wodefault_optional_adv: - -wodefault_optional_adv - '$adv_opts.param_wodefault_optional_adv' -#end if - #if $adv_opts.param_default_optional_adv: - -default_optional_adv - '$adv_opts.param_default_optional_adv' -#end if - #if $adv_opts.param_wodefault_mandatory_many_adv: - -wodefault_mandatory_many_adv - '$adv_opts.param_wodefault_mandatory_many_adv' -#end if - #if $adv_opts.param_default_mandatory_many_adv: - -default_mandatory_many_adv - '$adv_opts.param_default_mandatory_many_adv' -#end if - #if $adv_opts.param_wodefault_optional_many_adv: - -wodefault_optional_many_adv - '$adv_opts.param_wodefault_optional_many_adv' -#end if - #if $adv_opts.param_default_optional_many_adv: - -default_optional_many_adv - '$adv_opts.param_default_optional_many_adv' -#end if - #if $adv_opts.param_list_wodefault_optional_adv: - #set tmp=' '.join(["'%s'"%str(_) for _ in $adv_opts.param_list_wodefault_optional_adv]) - -list_wodefault_optional_adv - $tmp -#end if - #if $adv_opts.param_list_wodefault_mandatory_adv: - #set tmp=' '.join(["'%s'"%str(_) for _ in $adv_opts.param_list_wodefault_mandatory_adv]) - -list_wodefault_mandatory_adv - $tmp -#end if - #if $adv_opts.param_list_default_optional_adv: - #set tmp=' '.join(["'%s'"%str(_) for _ in $adv_opts.param_list_default_optional_adv]) - -list_default_optional_adv - $tmp -#end if - #if $adv_opts.param_list_default_mandatory_adv: - #set tmp=' '.join(["'%s'"%str(_) for _ in $adv_opts.param_list_default_mandatory_adv]) - -list_default_mandatory_adv - $tmp -#end if - #if $adv_opts.param_list_wodefault_optional_many_adv: - #set tmp=' '.join(["'%s'"%str(_) for _ in $adv_opts.param_list_wodefault_optional_many_adv]) - -list_wodefault_optional_many_adv - $tmp -#end if - #if $adv_opts.param_list_wodefault_mandatory_many_adv: - #set tmp=' '.join(["'%s'"%str(_) for _ in $adv_opts.param_list_wodefault_mandatory_many_adv]) - -list_wodefault_mandatory_many_adv - $tmp -#end if - #if $adv_opts.param_list_default_optional_many_adv: - #set tmp=' '.join(["'%s'"%str(_) for _ in $adv_opts.param_list_default_optional_many_adv]) - -list_default_optional_many_adv - $tmp -#end if - #if $adv_opts.param_list_default_mandatory_many_adv: - #set tmp=' '.join(["'%s'"%str(_) for _ in $adv_opts.param_list_default_mandatory_many_adv]) - -list_default_mandatory_many_adv - $tmp -#end if + #if $adv_opts.param_wodefault_mandatory_adv: + -wodefault_mandatory_adv + '$adv_opts.param_wodefault_mandatory_adv' + #end if + #if $adv_opts.param_default_mandatory_adv: + -default_mandatory_adv + '$adv_opts.param_default_mandatory_adv' + #end if + #if $adv_opts.param_wodefault_optional_adv: + -wodefault_optional_adv + '$adv_opts.param_wodefault_optional_adv' + #end if + #if $adv_opts.param_default_optional_adv: + -default_optional_adv + '$adv_opts.param_default_optional_adv' + #end if + #if $adv_opts.param_wodefault_mandatory_many_adv: + -wodefault_mandatory_many_adv + '$adv_opts.param_wodefault_mandatory_many_adv' + #end if + #if $adv_opts.param_default_mandatory_many_adv: + -default_mandatory_many_adv + '$adv_opts.param_default_mandatory_many_adv' + #end if + #if $adv_opts.param_wodefault_optional_many_adv: + -wodefault_optional_many_adv + '$adv_opts.param_wodefault_optional_many_adv' + #end if + #if $adv_opts.param_default_optional_many_adv: + -default_optional_many_adv + '$adv_opts.param_default_optional_many_adv' + #end if + #if $adv_opts.param_list_wodefault_optional_adv: + #set tmp=' '.join(["'%s'"%str(_) for _ in $adv_opts.param_list_wodefault_optional_adv]) + -list_wodefault_optional_adv + $tmp + #end if + #if $adv_opts.param_list_wodefault_mandatory_adv: + #set tmp=' '.join(["'%s'"%str(_) for _ in $adv_opts.param_list_wodefault_mandatory_adv]) + -list_wodefault_mandatory_adv + $tmp + #end if + #if $adv_opts.param_list_default_optional_adv: + #set tmp=' '.join(["'%s'"%str(_) for _ in $adv_opts.param_list_default_optional_adv]) + -list_default_optional_adv + $tmp + #end if + #if $adv_opts.param_list_default_mandatory_adv: + #set tmp=' '.join(["'%s'"%str(_) for _ in $adv_opts.param_list_default_mandatory_adv]) + -list_default_mandatory_adv + $tmp + #end if + #if $adv_opts.param_list_wodefault_optional_many_adv: + #set tmp=' '.join(["'%s'"%str(_) for _ in $adv_opts.param_list_wodefault_optional_many_adv]) + -list_wodefault_optional_many_adv + $tmp + #end if + #if $adv_opts.param_list_wodefault_mandatory_many_adv: + #set tmp=' '.join(["'%s'"%str(_) for _ in $adv_opts.param_list_wodefault_mandatory_many_adv]) + -list_wodefault_mandatory_many_adv + $tmp + #end if + #if $adv_opts.param_list_default_optional_many_adv: + #set tmp=' '.join(["'%s'"%str(_) for _ in $adv_opts.param_list_default_optional_many_adv]) + -list_default_optional_many_adv + $tmp + #end if + #if $adv_opts.param_list_default_mandatory_many_adv: + #set tmp=' '.join(["'%s'"%str(_) for _ in $adv_opts.param_list_default_mandatory_many_adv]) + -list_default_mandatory_many_adv + $tmp + #end if #end if > $param_stdout ]]> diff --git a/tests/test-data/string.xml b/tests/test-data/string.xml index 5b41dbd3..9e3e7c0d 100644 --- a/tests/test-data/string.xml +++ b/tests/test-data/string.xml @@ -25,18 +25,18 @@ -param_default_optional_unrestricted '$param_param_default_optional_unrestricted' #end if #if $adv_opts.adv_opts_selector=='advanced': - #if str($adv_opts.param_param_wodefault_mandatory_unrestricted_adv): - -param_wodefault_mandatory_unrestricted_adv '$adv_opts.param_param_wodefault_mandatory_unrestricted_adv' -#end if - #if str($adv_opts.param_param_default_mandatory_unrestricted_adv): - -param_default_mandatory_unrestricted_adv '$adv_opts.param_param_default_mandatory_unrestricted_adv' -#end if - #if str($adv_opts.param_param_wodefault_optional_unrestricted_adv): - -param_wodefault_optional_unrestricted_adv '$adv_opts.param_param_wodefault_optional_unrestricted_adv' -#end if - #if str($adv_opts.param_param_default_optional_unrestricted_adv): - -param_default_optional_unrestricted_adv '$adv_opts.param_param_default_optional_unrestricted_adv' -#end if + #if str($adv_opts.param_param_wodefault_mandatory_unrestricted_adv): + -param_wodefault_mandatory_unrestricted_adv '$adv_opts.param_param_wodefault_mandatory_unrestricted_adv' + #end if + #if str($adv_opts.param_param_default_mandatory_unrestricted_adv): + -param_default_mandatory_unrestricted_adv '$adv_opts.param_param_default_mandatory_unrestricted_adv' + #end if + #if str($adv_opts.param_param_wodefault_optional_unrestricted_adv): + -param_wodefault_optional_unrestricted_adv '$adv_opts.param_param_wodefault_optional_unrestricted_adv' + #end if + #if str($adv_opts.param_param_default_optional_unrestricted_adv): + -param_default_optional_unrestricted_adv '$adv_opts.param_param_default_optional_unrestricted_adv' + #end if #end if > $param_stdout ]]> From e33b6379633a5910e8f43637a8b60d6900c1f009 Mon Sep 17 00:00:00 2001 From: M Bernt Date: Sat, 6 Jul 2019 14:50:47 +0200 Subject: [PATCH 017/121] use argument attribute, remove size attribute - add quotes for multiple,optional file inputs - remove the (now deprecated) size attribute of text fields - moved parameter to argument attribute - allow for empty defaults in int/float inputs --- galaxy/converter.py | 41 ++--------- tests/test-data/bool.xml | 8 +-- tests/test-data/float.xml | 64 ++++++++--------- tests/test-data/ifile.xml | 132 ++++++++++++++++++++---------------- tests/test-data/integer.xml | 64 ++++++++--------- tests/test-data/repeat.xml | 80 +++++++++++----------- tests/test-data/select.xml | 64 ++++++++--------- tests/test-data/string.xml | 16 ++--- 8 files changed, 225 insertions(+), 244 deletions(-) diff --git a/galaxy/converter.py b/galaxy/converter.py index 84ea4508..40a247c2 100755 --- a/galaxy/converter.py +++ b/galaxy/converter.py @@ -415,7 +415,9 @@ def create_command(tool, model, **kwargs): if param.type is _InFile: command += command_line_prefix + "\n" command += " #for token in $" + galaxy_parameter_name + ":\n" - command += " $token\n" + command += " #if $token\n" + command += " '$token'\n" + command += " #end if" command += " #end for\n" elif is_selection_parameter(param): command += "#if " + actual_parameter + ":\n" @@ -702,8 +704,6 @@ def create_param_attribute_list(param_node, param, supported_file_formats): if param_type == "text": - # add size attribute... this is the length of a textbox field in Galaxy (it could also be 15x2, for instance) - param_node.attrib["size"] = "30" # add sanitizer nodes, this is needed for special character like "[" # which are used for example by FeatureFinderMultiplex sanitizer_node = SubElement(param_node, "sanitizer") @@ -729,37 +729,6 @@ def create_param_attribute_list(param_node, param, supported_file_formats): # simple boolean with a default if param.default is True: param_node.attrib["checked"] = "true" - else: - if param.type is int or param.type is float: - # galaxy requires "value" to be included for int/float - # since no default was included, we need to figure out one in a clever way... but let the user know - # that we are "thinking" for him/her - logger.warning("Generating default value for parameter [%s]. " - "Galaxy requires the attribute 'value' to be set for integer/floats. " - "Edit the CTD file and provide a suitable default value." % param.name, 1) - # check if there's a min/max and try to use them - default_value = None - if param.restrictions is not None: - if type(param.restrictions) is _NumericRange: - default_value = param.restrictions.n_min - if default_value is None: - default_value = param.restrictions.n_max - if default_value is None: - # no min/max provided... just use 0 and see what happens - default_value = 0 - else: - # should never be here, since we have validated this anyway... - # this code is here just for documentation purposes - # however, better safe than sorry! - # (it could be that the code changes and then we have an ugly scenario) - raise InvalidModelException("Expected either a numeric range for parameter [%(name)s], " - "but instead got [%(type)s]" - % {"name": param.name, "type": type(param.restrictions)}) - else: - # no restrictions and no default value provided... - # make up something - default_value = 0 - param_node.attrib["value"] = str(default_value) label = "%s parameter" % param.name help_text = "" @@ -768,8 +737,8 @@ def create_param_attribute_list(param_node, param, supported_file_formats): label, help_text = generate_label_and_help(param.description) param_node.attrib["label"] = label - param_node.attrib["help"] = "(-%s)" % param.name + " " + help_text - + param_node.attrib["help"] = help_text + param_node.attrib["argument"] = "-"+param.name def generate_label_and_help(desc): help_text = "" diff --git a/tests/test-data/bool.xml b/tests/test-data/bool.xml index 0e331fa1..50ce6974 100644 --- a/tests/test-data/bool.xml +++ b/tests/test-data/bool.xml @@ -21,11 +21,11 @@ $param_false > $param_stdout ]]> - - + + - - + + diff --git a/tests/test-data/float.xml b/tests/test-data/float.xml index 09ec5150..a87ad309 100644 --- a/tests/test-data/float.xml +++ b/tests/test-data/float.xml @@ -113,39 +113,39 @@ > $param_stdout ]]> - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + diff --git a/tests/test-data/ifile.xml b/tests/test-data/ifile.xml index 76c471e8..58dca317 100644 --- a/tests/test-data/ifile.xml +++ b/tests/test-data/ifile.xml @@ -50,28 +50,34 @@ #end if -list_mandatory_noformat #for token in $param_list_mandatory_noformat: - $token - #end for + #if $token + $token + #end if #end for -list_optional_noformat #for token in $param_list_optional_noformat: - $token - #end for + #if $token + $token + #end if #end for -list_mandatory_oneformat #for token in $param_list_mandatory_oneformat: - $token - #end for + #if $token + $token + #end if #end for -list_optional_oneformat #for token in $param_list_optional_oneformat: - $token - #end for + #if $token + $token + #end if #end for -list_mandatory_multformat #for token in $param_list_mandatory_multformat: - $token - #end for + #if $token + $token + #end if #end for -list_optional_multformat #for token in $param_list_optional_multformat: - $token - #end for + #if $token + $token + #end if #end for #if $adv_opts.adv_opts_selector=='advanced': #if str($adv_opts.param_mandatory_noformat_adv): -mandatory_noformat_adv $adv_opts.param_mandatory_noformat_adv @@ -111,45 +117,51 @@ #end if -list_mandatory_noformat_adv #for token in $param_list_mandatory_noformat_adv: - $token - #end for + #if $token + $token + #end if #end for -list_optional_noformat_adv #for token in $param_list_optional_noformat_adv: - $token - #end for + #if $token + $token + #end if #end for -list_mandatory_oneformat_adv #for token in $param_list_mandatory_oneformat_adv: - $token - #end for + #if $token + $token + #end if #end for -list_optional_oneformat_adv #for token in $param_list_optional_oneformat_adv: - $token - #end for + #if $token + $token + #end if #end for -list_mandatory_multformat_adv #for token in $param_list_mandatory_multformat_adv: - $token - #end for + #if $token + $token + #end if #end for -list_optional_multformat_adv #for token in $param_list_optional_multformat_adv: - $token - #end for + #if $token + $token + #end if #end for #end if > $param_stdout ]]> - - - - - - - - - - - - - + + + + + + + + + + + + + @@ -157,7 +169,7 @@ - + @@ -165,7 +177,7 @@ - + @@ -173,7 +185,7 @@ - + @@ -181,7 +193,7 @@ - + @@ -189,7 +201,7 @@ - + @@ -198,19 +210,19 @@ - - - - - - - - - - - - - + + + + + + + + + + + + + @@ -218,7 +230,7 @@ - + @@ -226,7 +238,7 @@ - + @@ -234,7 +246,7 @@ - + @@ -242,7 +254,7 @@ - + @@ -250,7 +262,7 @@ - + diff --git a/tests/test-data/integer.xml b/tests/test-data/integer.xml index 8be1a028..f97ed500 100644 --- a/tests/test-data/integer.xml +++ b/tests/test-data/integer.xml @@ -113,39 +113,39 @@ > $param_stdout ]]> - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + diff --git a/tests/test-data/repeat.xml b/tests/test-data/repeat.xml index 93c55e18..17a1f150 100644 --- a/tests/test-data/repeat.xml +++ b/tests/test-data/repeat.xml @@ -298,7 +298,7 @@ ]]> - + @@ -308,7 +308,7 @@ - + @@ -318,7 +318,7 @@ - + @@ -328,7 +328,7 @@ - + @@ -338,7 +338,7 @@ - + @@ -348,7 +348,7 @@ - + @@ -358,7 +358,7 @@ - + @@ -368,7 +368,7 @@ - + @@ -378,7 +378,7 @@ - + @@ -388,7 +388,7 @@ - + @@ -398,7 +398,7 @@ - + @@ -408,7 +408,7 @@ - + @@ -418,7 +418,7 @@ - + @@ -428,7 +428,7 @@ - + @@ -438,7 +438,7 @@ - + @@ -448,7 +448,7 @@ - + @@ -458,7 +458,7 @@ - + @@ -468,7 +468,7 @@ - + @@ -478,7 +478,7 @@ - + @@ -488,7 +488,7 @@ - + @@ -499,7 +499,7 @@ - + @@ -509,7 +509,7 @@ - + @@ -519,7 +519,7 @@ - + @@ -529,7 +529,7 @@ - + @@ -539,7 +539,7 @@ - + @@ -549,7 +549,7 @@ - + @@ -559,7 +559,7 @@ - + @@ -569,7 +569,7 @@ - + @@ -579,7 +579,7 @@ - + @@ -589,7 +589,7 @@ - + @@ -599,7 +599,7 @@ - + @@ -609,7 +609,7 @@ - + @@ -619,7 +619,7 @@ - + @@ -629,7 +629,7 @@ - + @@ -639,7 +639,7 @@ - + @@ -649,7 +649,7 @@ - + @@ -659,7 +659,7 @@ - + @@ -669,7 +669,7 @@ - + @@ -679,7 +679,7 @@ - + @@ -689,7 +689,7 @@ - + diff --git a/tests/test-data/select.xml b/tests/test-data/select.xml index c9318af9..0faba53d 100644 --- a/tests/test-data/select.xml +++ b/tests/test-data/select.xml @@ -161,7 +161,7 @@ > $param_stdout ]]> - + @@ -169,25 +169,25 @@ value != "select a value" - + - + - + - + @@ -196,34 +196,34 @@ value != "select a value" - + - + - + - + - + @@ -231,26 +231,26 @@ value != "select a value" - + - + - + - + @@ -259,14 +259,14 @@ value != "select a value" - + - + @@ -274,7 +274,7 @@ - + @@ -282,25 +282,25 @@ value != "select a value" - + - + - + - + @@ -309,34 +309,34 @@ value != "select a value" - + - + - + - + - + @@ -344,26 +344,26 @@ value != "select a value" - + - + - + - + @@ -372,14 +372,14 @@ value != "select a value" - + - + diff --git a/tests/test-data/string.xml b/tests/test-data/string.xml index 9e3e7c0d..26541bb6 100644 --- a/tests/test-data/string.xml +++ b/tests/test-data/string.xml @@ -41,7 +41,7 @@ > $param_stdout ]]> - + @@ -49,7 +49,7 @@ - + @@ -57,7 +57,7 @@ - + @@ -65,7 +65,7 @@ - + @@ -74,7 +74,7 @@ - + @@ -82,7 +82,7 @@ - + @@ -90,7 +90,7 @@ - + @@ -98,7 +98,7 @@ - + From 828f6ba703ef1112dd4e8f8e3b5fb9dbaa17dafd Mon Sep 17 00:00:00 2001 From: M Bernt Date: Fri, 19 Jul 2019 10:22:23 +0200 Subject: [PATCH 018/121] int and float wo default need empty value attribute also restored quoting in ifile and repeats --- galaxy/converter.py | 2 ++ tests/test-data/float.xml | 32 ++++++++++++++++---------------- tests/test-data/ifile.xml | 24 ++++++++++++------------ tests/test-data/integer.xml | 32 ++++++++++++++++---------------- tests/test-data/repeat.xml | 32 ++++++++++++++++---------------- 5 files changed, 62 insertions(+), 60 deletions(-) diff --git a/galaxy/converter.py b/galaxy/converter.py index 40a247c2..c78668b2 100755 --- a/galaxy/converter.py +++ b/galaxy/converter.py @@ -729,6 +729,8 @@ def create_param_attribute_list(param_node, param, supported_file_formats): # simple boolean with a default if param.default is True: param_node.attrib["checked"] = "true" + elif param.type is int or param.type is float: + param_node.attrib["value"] = "" label = "%s parameter" % param.name help_text = "" diff --git a/tests/test-data/float.xml b/tests/test-data/float.xml index a87ad309..06ec8623 100644 --- a/tests/test-data/float.xml +++ b/tests/test-data/float.xml @@ -113,38 +113,38 @@ > $param_stdout ]]> - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + diff --git a/tests/test-data/ifile.xml b/tests/test-data/ifile.xml index 58dca317..d2038a0c 100644 --- a/tests/test-data/ifile.xml +++ b/tests/test-data/ifile.xml @@ -51,32 +51,32 @@ -list_mandatory_noformat #for token in $param_list_mandatory_noformat: #if $token - $token + '$token' #end if #end for -list_optional_noformat #for token in $param_list_optional_noformat: #if $token - $token + '$token' #end if #end for -list_mandatory_oneformat #for token in $param_list_mandatory_oneformat: #if $token - $token + '$token' #end if #end for -list_optional_oneformat #for token in $param_list_optional_oneformat: #if $token - $token + '$token' #end if #end for -list_mandatory_multformat #for token in $param_list_mandatory_multformat: #if $token - $token + '$token' #end if #end for -list_optional_multformat #for token in $param_list_optional_multformat: #if $token - $token + '$token' #end if #end for #if $adv_opts.adv_opts_selector=='advanced': #if str($adv_opts.param_mandatory_noformat_adv): @@ -118,32 +118,32 @@ -list_mandatory_noformat_adv #for token in $param_list_mandatory_noformat_adv: #if $token - $token + '$token' #end if #end for -list_optional_noformat_adv #for token in $param_list_optional_noformat_adv: #if $token - $token + '$token' #end if #end for -list_mandatory_oneformat_adv #for token in $param_list_mandatory_oneformat_adv: #if $token - $token + '$token' #end if #end for -list_optional_oneformat_adv #for token in $param_list_optional_oneformat_adv: #if $token - $token + '$token' #end if #end for -list_mandatory_multformat_adv #for token in $param_list_mandatory_multformat_adv: #if $token - $token + '$token' #end if #end for -list_optional_multformat_adv #for token in $param_list_optional_multformat_adv: #if $token - $token + '$token' #end if #end for #end if > $param_stdout diff --git a/tests/test-data/integer.xml b/tests/test-data/integer.xml index f97ed500..eef85e62 100644 --- a/tests/test-data/integer.xml +++ b/tests/test-data/integer.xml @@ -113,38 +113,38 @@ > $param_stdout ]]> - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + diff --git a/tests/test-data/repeat.xml b/tests/test-data/repeat.xml index 17a1f150..b43c7200 100644 --- a/tests/test-data/repeat.xml +++ b/tests/test-data/repeat.xml @@ -338,7 +338,7 @@ - + @@ -348,7 +348,7 @@ - + @@ -378,7 +378,7 @@ - + @@ -388,7 +388,7 @@ - + @@ -418,7 +418,7 @@ - + @@ -428,7 +428,7 @@ - + @@ -458,7 +458,7 @@ - + @@ -468,7 +468,7 @@ - + @@ -539,7 +539,7 @@ - + @@ -549,7 +549,7 @@ - + @@ -579,7 +579,7 @@ - + @@ -589,7 +589,7 @@ - + @@ -619,7 +619,7 @@ - + @@ -629,7 +629,7 @@ - + @@ -659,7 +659,7 @@ - + @@ -669,7 +669,7 @@ - + From 0dfe02efc91bd0b867d2f3a70bfe47e2f9175ef7 Mon Sep 17 00:00:00 2001 From: M Bernt Date: Fri, 19 Jul 2019 10:39:26 +0200 Subject: [PATCH 019/121] simplify code for advanced node generation --- galaxy/converter.py | 38 ++++++++------------------------------ 1 file changed, 8 insertions(+), 30 deletions(-) diff --git a/galaxy/converter.py b/galaxy/converter.py index c78668b2..8ab7cc70 100755 --- a/galaxy/converter.py +++ b/galaxy/converter.py @@ -526,7 +526,7 @@ def create_inputs(tool, model, **kwargs): inputs_node = SubElement(tool, "inputs") # some suites (such as OpenMS) need some advanced options when handling inputs - expand_advanced_node = None + advanced_node = None parameter_hardcoder = kwargs["parameter_hardcoder"] # treat all non output-file/advanced/blacklisted/hardcoded parameters as inputs @@ -539,9 +539,11 @@ def create_inputs(tool, model, **kwargs): if param.type is _OutFile: continue if param.advanced: - continue - - parent_node = inputs_node + if advanced_node is None: + advanced_node = Element("expand", OrderedDict([("macro", ADVANCED_OPTIONS_MACRO_NAME)])) + parent_node = advanced_node + else: + parent_node = inputs_node # for lists we need a repeat tag, execept if # - list of strings with a fixed set of options -> select w multiple=true @@ -556,32 +558,8 @@ def create_inputs(tool, model, **kwargs): param_node = add_child_node(parent_node, "param") create_param_attribute_list(param_node, param, kwargs["supported_file_formats"]) - # tread advanced parameters as inputs - # TODO: merge with previous loop to avoid code duplication? - for param in utils.extract_and_flatten_parameters(model): - # no need to show hardcoded parameters - hardcoded_value = parameter_hardcoder.get_hardcoded_value(param.name, model.name) - if param.name in kwargs["blacklisted_parameters"] or hardcoded_value: - # let's not use an extra level of indentation and use NOP - continue - if param.type is _OutFile: - continue - if not param.advanced: - continue - if expand_advanced_node is None: - expand_advanced_node = add_child_node(inputs_node, "expand", OrderedDict([("macro", ADVANCED_OPTIONS_MACRO_NAME)])) - parent_node = expand_advanced_node - - # for lists we need a repeat tag - if param.is_list and\ - not (param.type is str and param.restrictions is not None) and\ - param.type is not _InFile: - rep_node = add_child_node(parent_node, "repeat") - create_repeat_attribute_list(rep_node, param) - parent_node = rep_node - - param_node = add_child_node(parent_node, "param") - create_param_attribute_list(param_node, param, kwargs["supported_file_formats"]) + if not advanced_node is None: + inputs_node.append(advanced_node) def get_repeat_galaxy_parameter_name(param): From 216bea152e84f24f15aa91ce4a52ca4fc3102f8f Mon Sep 17 00:00:00 2001 From: M Bernt Date: Mon, 22 Jul 2019 16:52:17 +0200 Subject: [PATCH 020/121] Itemlist (wo restrictions) as comma separated list Previously itemlists wo restrictions were rendered as repeats. Problem is that repeats do not allow to enter default values. Also a select is no option since the user can't add values So I suggest to use a text field allowing for comma separated list of strings, integers, or floats. The correctness of the input is verified by by validators and sanitizers. I'm optimistic that listitems do not contain ',' (which would be a problem), since restrictions of restricted itemslists are given as comma separated list. --- galaxy/converter.py | 95 +++- tests/test-data/bool.xml | 2 +- tests/test-data/empty.xml | 2 +- tests/test-data/float.xml | 2 +- tests/test-data/ifile.xml | 26 +- tests/test-data/integer.xml | 2 +- tests/test-data/ofile.xml | 50 +- tests/test-data/repeat.ctd | 16 +- tests/test-data/repeat.xml | 956 +++++++++++++++++------------------- tests/test-data/select.xml | 2 +- tests/test-data/string.xml | 2 +- 11 files changed, 580 insertions(+), 575 deletions(-) diff --git a/galaxy/converter.py b/galaxy/converter.py index 8ab7cc70..62f173b9 100755 --- a/galaxy/converter.py +++ b/galaxy/converter.py @@ -358,7 +358,7 @@ def create_tool(model): initialize the tool @param model the ctd model """ - return Element("tool", OrderedDict([("id", model.name), ("name", model.name), ("version", model.version)])) + return Element("tool", OrderedDict([("id", model.name.replace(" ","_")), ("name", model.name), ("version", model.version)])) def create_description(tool, model): @@ -426,11 +426,9 @@ def create_command(tool, model, **kwargs): command += " $tmp\n" command += "#end if\n" else: - command += "\n#if " + actual_repeat + ":\n" - command += command_line_prefix + "\n" - command += " #for token in " + actual_repeat + ":\n" - command += " '$token." + galaxy_parameter_name + "'\n" - command += " #end for\n" + command += "\n#if str(" + actual_parameter + "):\n" + command += " " + command_line_prefix + "\n" + command += " '" + actual_parameter + "'\n" command += "#end if\n" # logic for other ITEMs else: @@ -545,15 +543,15 @@ def create_inputs(tool, model, **kwargs): else: parent_node = inputs_node - # for lists we need a repeat tag, execept if - # - list of strings with a fixed set of options -> select w multiple=true - # - InFiles -> data w multiple=true - if param.is_list and\ - not (param.type is str and param.restrictions is not None) and\ - param.type is not _InFile: - rep_node = add_child_node(parent_node, "repeat") - create_repeat_attribute_list(rep_node, param) - parent_node = rep_node +# # for lists we need a repeat tag, execept if +# # - list of strings with a fixed set of options -> select w multiple=true +# # - InFiles -> data w multiple=true +# if param.is_list and\ +# not (param.type is str and param.restrictions is not None) and\ +# param.type is not _InFile: +# rep_node = add_child_node(parent_node, "repeat") +# create_repeat_attribute_list(rep_node, param) +# parent_node = rep_node param_node = add_child_node(parent_node, "param") create_param_attribute_list(param_node, param, kwargs["supported_file_formats"]) @@ -592,12 +590,17 @@ def create_param_attribute_list(param_node, param, supported_file_formats): @param param the ctd parameter @param supported_file_formats """ + + # set the name and a first guess for the type (which will be over written + # in some cases .. see below) param_node.attrib["name"] = get_galaxy_parameter_name(param) param_type = TYPE_TO_GALAXY_TYPE[param.type] if param_type is None: raise ModelError("Unrecognized parameter type %(type)s for parameter %(name)s" % {"type": param.type, "name": param.name}) + # ITEMLIST is rendered as text field (even if its integers or floats), an + # exception is files which are treated a bit below if param.is_list: param_type = "text" @@ -661,7 +664,12 @@ def create_param_attribute_list(param_node, param, supported_file_formats): validator_node = add_child_node(param_node, "validator", OrderedDict([("type", "expression"), ("message", "A value needs to be selected")])) validator_node.text = 'value != "select a value"' - elif type(param.restrictions) is _NumericRange: + # numeric ranges (which appear for int and float ITEMS and ITEMLISTS) + # these are reflected by min and max attributes + # since item lists become text parameters + validator these don't need these attributes + elif type(param.restrictions) is _NumericRange and param_type == "text": + pass + elif type(param.restrictions) is _NumericRange and param_type != "text": if param.type is not int and param.type is not float: raise InvalidModelException("Expected either 'int' or 'float' in the numeric range restriction for " "parameter [%(name)s], but instead got [%(type)s]" % @@ -682,13 +690,51 @@ def create_param_attribute_list(param_node, param, supported_file_formats): if param_type == "text": - # add sanitizer nodes, this is needed for special character like "[" + # for repeats (which are rendered as text field in the tool form) that are actually + # integer/floats special validation is necessary (try to convert them and check if + # in the min max range if a range is given) + # in addition (for the convenience of the user) any spaces are removed (due to the + # regex only possible next to the commas and at the begin or end) + if TYPE_TO_GALAXY_TYPE[param.type] in ["integer", "float"]: + message = "a comma separated list of %s values is required" % TYPE_TO_GALAXY_TYPE[param.type] + if TYPE_TO_GALAXY_TYPE[param.type] == "integer": + expression = "^ *[+-]?[0-9]+( *, *[+-]?[0-9]+)* *$" + else: + expression = "^ *[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?( *, *[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?)* *$" + validator_node = SubElement(param_node, "validator", OrderedDict([("type", "regex"), ("message", message )])) + validator_node.text = CDATA(expression) + + if type(param.restrictions) is _NumericRange and not (param.restrictions.n_min is None and param.restrictions.n_max is None): + expression = "len(value.split(',')) == len([_ for _ in value.split(',')" + message = "a comma separated list of %s values " % TYPE_TO_GALAXY_TYPE[param.type] + + if not param.restrictions.n_min is None and not param.restrictions.n_max is None: + expression += " if %s <= %s(_) <= %s" % (param.restrictions.n_min, param.type.__name__, param.restrictions.n_max) + message += "in the range %s:%s " % (param.restrictions.n_min, param.restrictions.n_max) + elif not param.restrictions.n_min is None: + expression += " if %s <= %s(_)" % (param.restrictions.n_min, param.type.__name__) + message += "in the range %s: " % (param.restrictions.n_min) + elif not param.restrictions.n_max is None: + expression += " if %s(_) <= %s" % (param.type.__name__, param.restrictions.n_max) + message += "in the range :%s " % (param.restrictions.n_min) + expression += "])\n" + message += "is required" + validator_node = SubElement(param_node, "validator", OrderedDict([("type", "expression"), ("message", message )])) + validator_node.text = CDATA(expression) + + sanitizer_node = SubElement(param_node, "sanitizer") + valid_node = SubElement(sanitizer_node, "valid", OrderedDict([("initial", "string.printable")])) + SubElement(valid_node, "remove", OrderedDict([("value", ' ')])) + mapping_node = SubElement(sanitizer_node, "mapping", OrderedDict([("initial", "none")])) + SubElement(mapping_node, "add", OrderedDict([("source", ' '),("target", "")])) + + # add sanitizer nodes to other text params, this is needed for special character like "[" # which are used for example by FeatureFinderMultiplex - sanitizer_node = SubElement(param_node, "sanitizer") - - valid_node = SubElement(sanitizer_node, "valid", OrderedDict([("initial", "string.printable")])) - add_child_node(valid_node, "remove", OrderedDict([("value", '\'')])) - add_child_node(valid_node, "remove", OrderedDict([("value", '"')])) + else: + sanitizer_node = SubElement(param_node, "sanitizer") + valid_node = SubElement(sanitizer_node, "valid", OrderedDict([("initial", "string.printable")])) + SubElement(valid_node, "remove", OrderedDict([("value", '\'')])) + SubElement(valid_node, "remove", OrderedDict([("value", '"')])) # check for default value if param.default is not None and param.default is not _Null: @@ -699,7 +745,7 @@ def create_param_attribute_list(param_node, param, supported_file_formats): # we ASSUME that a list of parameters looks like: # $ tool -ignore He Ar Xe # meaning, that, for example, Helium, Argon and Xenon will be ignored - param_node.attrib["value"] = ' '.join(map(str, param.default)) + param_node.attrib["value"] = ','.join(map(str, param.default)) elif param_type != "boolean": param_node.attrib["value"] = str(param.default) @@ -710,11 +756,14 @@ def create_param_attribute_list(param_node, param, supported_file_formats): elif param.type is int or param.type is float: param_node.attrib["value"] = "" + # add label, help, and argument label = "%s parameter" % param.name help_text = "" if param.description is not None: label, help_text = generate_label_and_help(param.description) + if param.is_list and not is_selection_parameter(param): + help_text += " (comma separated list)" param_node.attrib["label"] = label param_node.attrib["help"] = help_text diff --git a/tests/test-data/bool.xml b/tests/test-data/bool.xml index 50ce6974..755af6f6 100644 --- a/tests/test-data/bool.xml +++ b/tests/test-data/bool.xml @@ -1,7 +1,7 @@ - + Bool parameter tests echo BoolTest diff --git a/tests/test-data/empty.xml b/tests/test-data/empty.xml index b66b175c..90a38c5b 100644 --- a/tests/test-data/empty.xml +++ b/tests/test-data/empty.xml @@ -1,7 +1,7 @@ - + String parameter tests echo StringTest diff --git a/tests/test-data/float.xml b/tests/test-data/float.xml index 06ec8623..7db2d3c3 100644 --- a/tests/test-data/float.xml +++ b/tests/test-data/float.xml @@ -1,7 +1,7 @@ - + Floateger parameter tests echo FloatTest diff --git a/tests/test-data/ifile.xml b/tests/test-data/ifile.xml index d2038a0c..11aba389 100644 --- a/tests/test-data/ifile.xml +++ b/tests/test-data/ifile.xml @@ -1,7 +1,7 @@ - + Input File parameter tests echo inFileTest @@ -161,7 +161,7 @@ - + @@ -169,7 +169,7 @@ - + @@ -177,7 +177,7 @@ - + @@ -185,7 +185,7 @@ - + @@ -193,7 +193,7 @@ - + @@ -201,7 +201,7 @@ - + @@ -222,7 +222,7 @@ - + @@ -230,7 +230,7 @@ - + @@ -238,7 +238,7 @@ - + @@ -246,7 +246,7 @@ - + @@ -254,7 +254,7 @@ - + @@ -262,7 +262,7 @@ - + diff --git a/tests/test-data/integer.xml b/tests/test-data/integer.xml index eef85e62..6f79d777 100644 --- a/tests/test-data/integer.xml +++ b/tests/test-data/integer.xml @@ -1,7 +1,7 @@ - + Integer parameter tests echo IntTest diff --git a/tests/test-data/ofile.xml b/tests/test-data/ofile.xml index 99d4d2f4..61b8df54 100644 --- a/tests/test-data/ofile.xml +++ b/tests/test-data/ofile.xml @@ -1,7 +1,7 @@ - + Input File parameter tests echo outFileTest @@ -49,46 +49,34 @@ -default_optional_multformat $param_default_optional_multformat #end if -#if $rep_param_list_mandatory_noformat: --list_mandatory_noformat - #for token in $rep_param_list_mandatory_noformat: - '$token.param_list_mandatory_noformat' - #end for +#if str($param_list_mandatory_noformat): + -list_mandatory_noformat + '$param_list_mandatory_noformat' #end if -#if $rep_param_list_optional_noformat: --list_optional_noformat - #for token in $rep_param_list_optional_noformat: - '$token.param_list_optional_noformat' - #end for +#if str($param_list_optional_noformat): + -list_optional_noformat + '$param_list_optional_noformat' #end if -#if $rep_param_list_mandatory_oneformat: --list_mandatory_oneformat - #for token in $rep_param_list_mandatory_oneformat: - '$token.param_list_mandatory_oneformat' - #end for +#if str($param_list_mandatory_oneformat): + -list_mandatory_oneformat + '$param_list_mandatory_oneformat' #end if -#if $rep_param_list_optional_oneformat: --list_optional_oneformat - #for token in $rep_param_list_optional_oneformat: - '$token.param_list_optional_oneformat' - #end for +#if str($param_list_optional_oneformat): + -list_optional_oneformat + '$param_list_optional_oneformat' #end if -#if $rep_param_list_mandatory_multformat: --list_mandatory_multformat - #for token in $rep_param_list_mandatory_multformat: - '$token.param_list_mandatory_multformat' - #end for +#if str($param_list_mandatory_multformat): + -list_mandatory_multformat + '$param_list_mandatory_multformat' #end if -#if $rep_param_list_optional_multformat: --list_optional_multformat - #for token in $rep_param_list_optional_multformat: - '$token.param_list_optional_multformat' - #end for +#if str($param_list_optional_multformat): + -list_optional_multformat + '$param_list_optional_multformat' #end if #if str($param_mandatory_noformat_adv): -mandatory_noformat_adv $param_mandatory_noformat_adv diff --git a/tests/test-data/repeat.ctd b/tests/test-data/repeat.ctd index 75d90af5..fa7b3021 100644 --- a/tests/test-data/repeat.ctd +++ b/tests/test-data/repeat.ctd @@ -29,8 +29,8 @@ - - + + @@ -52,8 +52,8 @@ - - + + @@ -86,8 +86,8 @@ - - + + @@ -109,8 +109,8 @@ - - + + diff --git a/tests/test-data/repeat.xml b/tests/test-data/repeat.xml index b43c7200..4b32729a 100644 --- a/tests/test-data/repeat.xml +++ b/tests/test-data/repeat.xml @@ -1,7 +1,7 @@ - + Repeat parameter tests echo RepeatTest @@ -13,292 +13,436 @@ $param_stdout ]]> - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -306,9 +450,7 @@ - - - + @@ -316,9 +458,7 @@ - - - + @@ -326,9 +466,7 @@ - - - + @@ -336,368 +474,198 @@ - - - + + - - + + + + - - - + + - - + + + + - - - + + - - + + + + - - - + + - - + + + + - - - + + + - - + + + + - - - + + + - - + + + + - - - + + + - - + + + + - - - + + + - - + + + + - - - + + - - + + + + - - - + + - - + + + + - - - + + - - + + + + - - - + + - - + + + + - - - + + + - - + + + + - - - + + + - - + + + + - - - + + + - - + + + + - - - + + + - - + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/tests/test-data/select.xml b/tests/test-data/select.xml index 0faba53d..97af6b1f 100644 --- a/tests/test-data/select.xml +++ b/tests/test-data/select.xml @@ -1,7 +1,7 @@ - + Select parameter tests echo SelectTest diff --git a/tests/test-data/string.xml b/tests/test-data/string.xml index 26541bb6..ca24b8ed 100644 --- a/tests/test-data/string.xml +++ b/tests/test-data/string.xml @@ -1,7 +1,7 @@ - + String parameter tests echo StringTest From c7d3905cc556c3c6777ab7d08b464e710dbfbf9b Mon Sep 17 00:00:00 2001 From: M Bernt Date: Mon, 22 Jul 2019 17:11:17 +0200 Subject: [PATCH 021/121] remove duplicate format entries in inputs --- galaxy/converter.py | 8 ++++---- tests/test-data/ifile.xml | 24 ++++++++++++------------ 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/galaxy/converter.py b/galaxy/converter.py index 62f173b9..40534cb3 100755 --- a/galaxy/converter.py +++ b/galaxy/converter.py @@ -620,9 +620,9 @@ def create_param_attribute_list(param_node, param, supported_file_formats): if param.restrictions is not None: # join all formats of the file, take mapping from supported_file if available for an entry if type(param.restrictions) is _FileFormat: - param_format = ",".join([get_supported_file_type(i, supported_file_formats) if + param_format = ",".join(set([get_supported_file_type(i, supported_file_formats) if get_supported_file_type(i, supported_file_formats) - else i for i in param.restrictions.formats]) + else i for i in param.restrictions.formats])) else: raise InvalidModelException("Expected 'file type' restrictions for input file [%(name)s], " "but instead got [%(type)s]" @@ -681,9 +681,9 @@ def create_param_attribute_list(param_node, param, supported_file_formats): if param.restrictions.n_max is not None: param_node.attrib["max"] = str(param.restrictions.n_max) elif type(param.restrictions) is _FileFormat: - param_node.attrib["format"] = ','.join([get_supported_file_type(i, supported_file_formats) if + param_node.attrib["format"] = ','.join(set([get_supported_file_type(i, supported_file_formats) if get_supported_file_type(i, supported_file_formats) - else i for i in param.restrictions.formats]) + else i for i in param.restrictions.formats])) else: raise InvalidModelException("Unrecognized restriction type [%(type)s] for parameter [%(name)s]" % {"type": type(param.restrictions), "name": param.name}) diff --git a/tests/test-data/ifile.xml b/tests/test-data/ifile.xml index 11aba389..86534c4c 100644 --- a/tests/test-data/ifile.xml +++ b/tests/test-data/ifile.xml @@ -153,14 +153,14 @@ - - + + - - + + @@ -193,7 +193,7 @@ - + @@ -201,7 +201,7 @@ - + @@ -214,14 +214,14 @@ - - + + - - + + @@ -254,7 +254,7 @@ - + @@ -262,7 +262,7 @@ - + From 0c23fd089cb984d1a67f210a2fa4b725220b9829 Mon Sep 17 00:00:00 2001 From: M Bernt Date: Mon, 22 Jul 2019 17:30:58 +0200 Subject: [PATCH 022/121] remove sanitizer from input files also added itemlists with default to ctd --- galaxy/converter.py | 2 +- tests/test-data/ifile.ctd | 41 ++++++++- tests/test-data/ifile.xml | 180 ++++++++++++++++++-------------------- 3 files changed, 125 insertions(+), 98 deletions(-) diff --git a/galaxy/converter.py b/galaxy/converter.py index 40534cb3..acb1379f 100755 --- a/galaxy/converter.py +++ b/galaxy/converter.py @@ -730,7 +730,7 @@ def create_param_attribute_list(param_node, param, supported_file_formats): # add sanitizer nodes to other text params, this is needed for special character like "[" # which are used for example by FeatureFinderMultiplex - else: + elif not param.type is _InFile: sanitizer_node = SubElement(param_node, "sanitizer") valid_node = SubElement(sanitizer_node, "valid", OrderedDict([("initial", "string.printable")])) SubElement(valid_node, "remove", OrderedDict([("value", '\'')])) diff --git a/tests/test-data/ifile.ctd b/tests/test-data/ifile.ctd index be957cb9..702fa30a 100644 --- a/tests/test-data/ifile.ctd +++ b/tests/test-data/ifile.ctd @@ -20,7 +20,7 @@ - + @@ -28,6 +28,26 @@ + + + + + + + + + + + + + + + + + + + + @@ -50,6 +70,25 @@ + + + + + + + + + + + + + + + + + + + diff --git a/tests/test-data/ifile.xml b/tests/test-data/ifile.xml index 86534c4c..ab941930 100644 --- a/tests/test-data/ifile.xml +++ b/tests/test-data/ifile.xml @@ -78,6 +78,36 @@ #if $token '$token' #end if #end for +-list_default_mandatory_noformat + #for token in $param_list_default_mandatory_noformat: + #if $token + '$token' + #end if #end for +-list_default_optional_noformat + #for token in $param_list_default_optional_noformat: + #if $token + '$token' + #end if #end for +-list_default_mandatory_oneformat + #for token in $param_list_default_mandatory_oneformat: + #if $token + '$token' + #end if #end for +-list_default_optional_oneformat + #for token in $param_list_default_optional_oneformat: + #if $token + '$token' + #end if #end for +-list_default_mandatory_multformat + #for token in $param_list_default_mandatory_multformat: + #if $token + '$token' + #end if #end for +-list_default_optional_multformat + #for token in $param_list_default_optional_multformat: + #if $token + '$token' + #end if #end for #if $adv_opts.adv_opts_selector=='advanced': #if str($adv_opts.param_mandatory_noformat_adv): -mandatory_noformat_adv $adv_opts.param_mandatory_noformat_adv @@ -145,6 +175,36 @@ #if $token '$token' #end if #end for + -list_default_mandatory_noformat_adv + #for token in $param_list_default_mandatory_noformat_adv: + #if $token + '$token' + #end if #end for + -list_default_optional_noformat_adv + #for token in $param_list_default_optional_noformat_adv: + #if $token + '$token' + #end if #end for + -list_default_mandatory_oneformat_adv + #for token in $param_list_default_mandatory_oneformat_adv: + #if $token + '$token' + #end if #end for + -list_default_optional_oneformat_adv + #for token in $param_list_default_optional_oneformat_adv: + #if $token + '$token' + #end if #end for + -list_default_mandatory_multformat_adv + #for token in $param_list_default_mandatory_multformat_adv: + #if $token + '$token' + #end if #end for + -list_default_optional_multformat_adv + #for token in $param_list_default_optional_multformat_adv: + #if $token + '$token' + #end if #end for #end if > $param_stdout ]]> @@ -161,54 +221,18 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + @@ -222,54 +246,18 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + From e40979aaa971e93e54062ed84e7b42bbbaf2af2d Mon Sep 17 00:00:00 2001 From: M Bernt Date: Tue, 23 Jul 2019 09:18:01 +0200 Subject: [PATCH 023/121] infile input needs no help info that a comma separated list is needed --- galaxy/converter.py | 4 +- tests/test-data/ifile.xml | 120 +++++++++++++++++++++++--------------- 2 files changed, 74 insertions(+), 50 deletions(-) diff --git a/galaxy/converter.py b/galaxy/converter.py index acb1379f..74f3ea42 100755 --- a/galaxy/converter.py +++ b/galaxy/converter.py @@ -417,7 +417,7 @@ def create_command(tool, model, **kwargs): command += " #for token in $" + galaxy_parameter_name + ":\n" command += " #if $token\n" command += " '$token'\n" - command += " #end if" + command += " #end if\n" command += " #end for\n" elif is_selection_parameter(param): command += "#if " + actual_parameter + ":\n" @@ -762,7 +762,7 @@ def create_param_attribute_list(param_node, param, supported_file_formats): if param.description is not None: label, help_text = generate_label_and_help(param.description) - if param.is_list and not is_selection_parameter(param): + if param.is_list and not is_selection_parameter(param) and not param.type is _InFile: help_text += " (comma separated list)" param_node.attrib["label"] = label diff --git a/tests/test-data/ifile.xml b/tests/test-data/ifile.xml index ab941930..bd088ec3 100644 --- a/tests/test-data/ifile.xml +++ b/tests/test-data/ifile.xml @@ -52,62 +52,74 @@ #for token in $param_list_mandatory_noformat: #if $token '$token' - #end if #end for + #end if + #end for -list_optional_noformat #for token in $param_list_optional_noformat: #if $token '$token' - #end if #end for + #end if + #end for -list_mandatory_oneformat #for token in $param_list_mandatory_oneformat: #if $token '$token' - #end if #end for + #end if + #end for -list_optional_oneformat #for token in $param_list_optional_oneformat: #if $token '$token' - #end if #end for + #end if + #end for -list_mandatory_multformat #for token in $param_list_mandatory_multformat: #if $token '$token' - #end if #end for + #end if + #end for -list_optional_multformat #for token in $param_list_optional_multformat: #if $token '$token' - #end if #end for + #end if + #end for -list_default_mandatory_noformat #for token in $param_list_default_mandatory_noformat: #if $token '$token' - #end if #end for + #end if + #end for -list_default_optional_noformat #for token in $param_list_default_optional_noformat: #if $token '$token' - #end if #end for + #end if + #end for -list_default_mandatory_oneformat #for token in $param_list_default_mandatory_oneformat: #if $token '$token' - #end if #end for + #end if + #end for -list_default_optional_oneformat #for token in $param_list_default_optional_oneformat: #if $token '$token' - #end if #end for + #end if + #end for -list_default_mandatory_multformat #for token in $param_list_default_mandatory_multformat: #if $token '$token' - #end if #end for + #end if + #end for -list_default_optional_multformat #for token in $param_list_default_optional_multformat: #if $token '$token' - #end if #end for + #end if + #end for #if $adv_opts.adv_opts_selector=='advanced': #if str($adv_opts.param_mandatory_noformat_adv): -mandatory_noformat_adv $adv_opts.param_mandatory_noformat_adv @@ -149,62 +161,74 @@ #for token in $param_list_mandatory_noformat_adv: #if $token '$token' - #end if #end for + #end if + #end for -list_optional_noformat_adv #for token in $param_list_optional_noformat_adv: #if $token '$token' - #end if #end for + #end if + #end for -list_mandatory_oneformat_adv #for token in $param_list_mandatory_oneformat_adv: #if $token '$token' - #end if #end for + #end if + #end for -list_optional_oneformat_adv #for token in $param_list_optional_oneformat_adv: #if $token '$token' - #end if #end for + #end if + #end for -list_mandatory_multformat_adv #for token in $param_list_mandatory_multformat_adv: #if $token '$token' - #end if #end for + #end if + #end for -list_optional_multformat_adv #for token in $param_list_optional_multformat_adv: #if $token '$token' - #end if #end for + #end if + #end for -list_default_mandatory_noformat_adv #for token in $param_list_default_mandatory_noformat_adv: #if $token '$token' - #end if #end for + #end if + #end for -list_default_optional_noformat_adv #for token in $param_list_default_optional_noformat_adv: #if $token '$token' - #end if #end for + #end if + #end for -list_default_mandatory_oneformat_adv #for token in $param_list_default_mandatory_oneformat_adv: #if $token '$token' - #end if #end for + #end if + #end for -list_default_optional_oneformat_adv #for token in $param_list_default_optional_oneformat_adv: #if $token '$token' - #end if #end for + #end if + #end for -list_default_mandatory_multformat_adv #for token in $param_list_default_mandatory_multformat_adv: #if $token '$token' - #end if #end for + #end if + #end for -list_default_optional_multformat_adv #for token in $param_list_default_optional_multformat_adv: #if $token '$token' - #end if #end for + #end if + #end for #end if > $param_stdout ]]> @@ -221,18 +245,18 @@ - - - - - - - - - - - - + + + + + + + + + + + + @@ -246,18 +270,18 @@ - - - - - - - - - - - - + + + + + + + + + + + + From 0f08f32252c547c36c18f690e76cc675da80eb95 Mon Sep 17 00:00:00 2001 From: M Bernt Date: Tue, 23 Jul 2019 15:28:47 +0200 Subject: [PATCH 024/121] bool input for optional outputs - for each optional param_out_x there is now an param_x which controls the output of the output (i.e. there is now a filter using this variable and the parameter is included in the command line depending on the bool) - simplified command line generation - cheetah if statements only for optional parameters - also removed param_ from the parameter names in the ctd files --- common/utils.py | 2 +- galaxy/converter.py | 129 ++++++++++----------- tests/test-data/float.ctd | 64 +++++------ tests/test-data/float.xml | 192 +++++++++++++------------------- tests/test-data/ifile.xml | 216 ++++++------------------------------ tests/test-data/integer.xml | 64 +++-------- tests/test-data/ofile.ctd | 2 +- tests/test-data/ofile.xml | 179 +++++++++++++++--------------- tests/test-data/repeat.xml | 200 +++++++-------------------------- tests/test-data/select.xml | 144 ++++++------------------ tests/test-data/string.ctd | 16 +-- tests/test-data/string.xml | 48 ++++---- 12 files changed, 422 insertions(+), 834 deletions(-) diff --git a/common/utils.py b/common/utils.py index c69ba1b2..020b8664 100644 --- a/common/utils.py +++ b/common/utils.py @@ -325,5 +325,5 @@ def indent(s, indentation=" "): @param indentation the desired indentation @return indented text """ - return "\n".join([indentation+_ for _ in s.splitlines()])+"\n" + return "\n".join([indentation+_ for _ in s.splitlines()]) diff --git a/galaxy/converter.py b/galaxy/converter.py index 74f3ea42..7685d59e 100755 --- a/galaxy/converter.py +++ b/galaxy/converter.py @@ -4,6 +4,7 @@ import string from collections import OrderedDict +import copy from string import strip from lxml import etree from lxml.etree import CDATA, SubElement, Element, ElementTree, ParseError, parse @@ -402,59 +403,55 @@ def create_command(tool, model, **kwargs): else: # parameter is neither blacklisted nor hardcoded... galaxy_parameter_name = get_galaxy_parameter_name(param) - repeat_galaxy_parameter_name = get_repeat_galaxy_parameter_name(param) - if param.advanced and param.type is not _OutFile: + if param.advanced: actual_parameter = "$adv_opts.%s" % galaxy_parameter_name - actual_repeat = "$adv_opts.%s" % repeat_galaxy_parameter_name else: actual_parameter = "$%s" % galaxy_parameter_name - actual_repeat = "$%s" % repeat_galaxy_parameter_name # logic for ITEMLISTs if param.is_list: if param.type is _InFile: - command += command_line_prefix + "\n" - command += " #for token in $" + galaxy_parameter_name + ":\n" - command += " #if $token\n" - command += " '$token'\n" - command += " #end if\n" - command += " #end for\n" + command += command_line_prefix + " " + command += "{# ' '.join([\"'%s'\"%str(_) for _ in " + actual_parameter + " if _ != None])}\n" elif is_selection_parameter(param): - command += "#if " + actual_parameter + ":\n" - command += " #set tmp=' '.join([\"'%s'\"%str(_) for _ in " + actual_parameter + "])\n" - command += " %s\n" % command_line_prefix - command += " $tmp\n" - command += "#end if\n" + command += "%s " % command_line_prefix + " " + command += "{# ' '.join([\"'%s'\"%str(_) for _ in " + actual_parameter + "])}\n" else: - command += "\n#if str(" + actual_parameter + "):\n" - command += " " + command_line_prefix + "\n" - command += " '" + actual_parameter + "'\n" - command += "#end if\n" + command += command_line_prefix + " " + command += "'" + actual_parameter + "'\n" # logic for other ITEMs else: # TODO only useful for text fields, integers or floats # not useful for choices, input fields ... - if is_selection_parameter(param): - command += "#if " + actual_parameter + ":\n" - command += " %s\n" % command_line_prefix - command += " '" + actual_parameter + "'\n" - command += "#end if\n" + command += "%s " % command_line_prefix + command += "'" + actual_parameter + "'\n" elif is_boolean_parameter(param): - command += "%s\n" % actual_parameter + command += "%s" % actual_parameter + "\n" elif TYPE_TO_GALAXY_TYPE[param.type] is 'text': - command += "#if str(" + actual_parameter + "):\n" - command += " %s " % command_line_prefix - command += " '" + actual_parameter + "'\n" - command += "#end if\n" + command += "%s " % command_line_prefix + command += "'" + actual_parameter + "'\n" else: - command += "#if str(" + actual_parameter + "):\n" - command += " %s " % command_line_prefix + command += "%s " % command_line_prefix command += actual_parameter + "\n" - command += "#end if\n" + + # add if statement for mandatory parameters + # - for optional outputs (param_out_x) the presence of the parameter depends on the additional input (param_x) + if not param.required and not is_boolean_parameter(param) and not(param.type is _InFile and param.is_list): + if param.type is _OutFile: + galaxy_parameter_name = get_galaxy_parameter_name(param, True) + if param.advanced: + actual_parameter = "$adv_opts.%s" % galaxy_parameter_name + else: + actual_parameter = "$%s" % galaxy_parameter_name + "param_"+param.name + if is_selection_parameter(param): + command = "#if " + actual_parameter + ":\n" + utils.indent(command) + "\n#end if\n" + else: + command = "#if str(" + actual_parameter + "):\n" + utils.indent(command) + "\n#end if\n" if param.advanced and param.type is not _OutFile: - advanced_command += "%s" % utils.indent(command) + advanced_command += "%s\n" % utils.indent(command) else: final_command += command @@ -493,15 +490,24 @@ def expand_macros(tool, model, **kwargs): expand_node.attrib["macro"] = expand_macro -def get_galaxy_parameter_name(param): +def get_galaxy_parameter_name(param, inparam = False): """ get the name of the parameter used in the galaxy tool - - prepend param_ + - prepend param_ or param_out_ (the latter for outputs, except if inparam is True) - replace : and - by _ + + the idea of inparam is to be used for optional outputs (param_out_x) for + which an additional boolean input (param_x) exists + @param param the parameter + @param inparam get the name of the corresponding input @return the name used for the parameter in the tool form """ - return "param_%s" % utils.extract_param_name(param).replace(":", "_").replace("-", "_") + p = utils.extract_param_name(param).replace(":", "_").replace("-", "_") + if param.type is _OutFile and not inparam: + return "param_out_%s" % p + else: + return "param_%s" % p def get_input_with_same_restrictions(out_param, model, supported_file_formats): @@ -535,8 +541,19 @@ def create_inputs(tool, model, **kwargs): # let's not use an extra level of indentation and use NOP continue if param.type is _OutFile: - continue - if param.advanced: + if not param.required: + _param = copy.deepcopy(param) + _param.type = str + _param.restrictions = _Choices("true,false") + _param.default = "false" + _param.description = "generate output %s (%s)" %(param.name, param.description) + _param.is_list = False + else: + continue + else: + _param = param + + if _param.advanced: if advanced_node is None: advanced_node = Element("expand", OrderedDict([("macro", ADVANCED_OPTIONS_MACRO_NAME)])) parent_node = advanced_node @@ -554,35 +571,12 @@ def create_inputs(tool, model, **kwargs): # parent_node = rep_node param_node = add_child_node(parent_node, "param") - create_param_attribute_list(param_node, param, kwargs["supported_file_formats"]) + create_param_attribute_list(param_node, _param, kwargs["supported_file_formats"]) if not advanced_node is None: inputs_node.append(advanced_node) -def get_repeat_galaxy_parameter_name(param): - """ - get the name of the repeat that contains the parameter - - prepend rep_ - @param param the parameter - @return the name of the repeat - """ - return "rep_" + get_galaxy_parameter_name(param) - - -def create_repeat_attribute_list(rep_node, param): - rep_node.attrib["name"] = get_repeat_galaxy_parameter_name(param) - if param.required: - rep_node.attrib["min"] = "1" - else: - rep_node.attrib["min"] = "0" - # for the ITEMLISTs which have LISTITEM children we only - # need one parameter as it is given as a string - if param.default is not None and param.default is not _Null: - rep_node.attrib["max"] = "1" - rep_node.attrib["title"] = get_galaxy_parameter_name(param) - - def create_param_attribute_list(param_node, param, supported_file_formats): """ get the attributes of input parameters @@ -895,8 +889,9 @@ def create_outputs(parent, model, **kwargs): if param.name in kwargs["blacklisted_parameters"] or hardcoded_value: # let's not use an extra level of indentation and use NOP continue - if param.type is _OutFile: - create_output_node(outputs_node, param, model, kwargs["supported_file_formats"]) + if not param.type is _OutFile: + continue + create_output_node(outputs_node, param, model, kwargs["supported_file_formats"]) # If there are no outputs defined in the ctd the node will have no children # and the stdout will be used as output @@ -944,8 +939,14 @@ def create_output_node(parent, param, model, supported_file_formats): raise InvalidModelException("Unrecognized restriction type [%(type)s] " "for output [%(name)s]" % {"type": type(param.restrictions), "name": param.name}) + data_node.attrib["format"] = data_format + if not param.required: + filter_node = add_child_node(data_node, "filter") + filter_node.text = get_galaxy_parameter_name(param, True) + if param.advanced: + filter_node.text = "advanced_options['adv_opts_selector'] == 'advanced' and advanced_options['" + filter_node.text + "']" return data_node diff --git a/tests/test-data/float.ctd b/tests/test-data/float.ctd index af537d83..58839d43 100644 --- a/tests/test-data/float.ctd +++ b/tests/test-data/float.ctd @@ -6,41 +6,41 @@ - - - - - - - - + + + + + + + + - - - - - - - - + + + + + + + + - - - - - - - - + + + + + + + + - - - - - - - - + + + + + + + + diff --git a/tests/test-data/float.xml b/tests/test-data/float.xml index 7db2d3c3..626e3c66 100644 --- a/tests/test-data/float.xml +++ b/tests/test-data/float.xml @@ -12,140 +12,108 @@ $param_stdout ]]> - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + diff --git a/tests/test-data/ifile.xml b/tests/test-data/ifile.xml index bd088ec3..796819ac 100644 --- a/tests/test-data/ifile.xml +++ b/tests/test-data/ifile.xml @@ -12,223 +12,79 @@ $param_stdout ]]> diff --git a/tests/test-data/integer.xml b/tests/test-data/integer.xml index 6f79d777..6b4ffb68 100644 --- a/tests/test-data/integer.xml +++ b/tests/test-data/integer.xml @@ -12,48 +12,32 @@ - + diff --git a/tests/test-data/ofile.xml b/tests/test-data/ofile.xml index 61b8df54..542f64e5 100644 --- a/tests/test-data/ofile.xml +++ b/tests/test-data/ofile.xml @@ -12,117 +12,120 @@ - + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - + + + + param_wodefault_optional_noformat + + + param_default_optional_noformat + + + + + param_wodefault_optional_oneformat + + + param_default_optional_oneformat + + + + + param_wodefault_optional_multformat + + + param_default_optional_multformat + + + + param_list_optional_noformat + + + + param_list_optional_oneformat + + + + param_list_optional_multformat + + + + advanced_options['adv_opts_selector'] == 'advanced' and advanced_options['param_optional_noformat_adv'] + + + + advanced_options['adv_opts_selector'] == 'advanced' and advanced_options['param_optional_oneformat_adv'] + + + + advanced_options['adv_opts_selector'] == 'advanced' and advanced_options['param_optional_multformat_adv'] + $param_stdout ]]> diff --git a/tests/test-data/select.xml b/tests/test-data/select.xml index 97af6b1f..fd656a5b 100644 --- a/tests/test-data/select.xml +++ b/tests/test-data/select.xml @@ -12,151 +12,71 @@ $param_stdout ]]> diff --git a/tests/test-data/string.ctd b/tests/test-data/string.ctd index 5c0e5fc7..8cd8408a 100644 --- a/tests/test-data/string.ctd +++ b/tests/test-data/string.ctd @@ -6,14 +6,14 @@ - - - - - - - - + + + + + + + + diff --git a/tests/test-data/string.xml b/tests/test-data/string.xml index ca24b8ed..3b65857b 100644 --- a/tests/test-data/string.xml +++ b/tests/test-data/string.xml @@ -12,36 +12,28 @@ $param_stdout ]]> - + @@ -49,7 +41,7 @@ - + @@ -57,7 +49,7 @@ - + @@ -65,7 +57,7 @@ - + @@ -74,7 +66,7 @@ - + @@ -82,7 +74,7 @@ - + @@ -90,7 +82,7 @@ - + @@ -98,7 +90,7 @@ - + From 269e8d5d944374ca1d2efe410d3684c477e7f8a0 Mon Sep 17 00:00:00 2001 From: M Bernt Date: Tue, 23 Jul 2019 16:14:23 +0200 Subject: [PATCH 025/121] further simplify cli generation - simply add quotes to all parameter values - also check select arguments for none --- galaxy/converter.py | 31 ++++-------------- tests/test-data/float.xml | 64 ++++++++++++++++++------------------- tests/test-data/ifile.xml | 48 ++++++++++++++-------------- tests/test-data/integer.xml | 64 ++++++++++++++++++------------------- tests/test-data/ofile.xml | 36 ++++++++++----------- tests/test-data/select.xml | 32 +++++++++---------- 6 files changed, 129 insertions(+), 146 deletions(-) diff --git a/galaxy/converter.py b/galaxy/converter.py index 7685d59e..20a17b8a 100755 --- a/galaxy/converter.py +++ b/galaxy/converter.py @@ -409,31 +409,14 @@ def create_command(tool, model, **kwargs): actual_parameter = "$%s" % galaxy_parameter_name # logic for ITEMLISTs - if param.is_list: - if param.type is _InFile: - command += command_line_prefix + " " - command += "{# ' '.join([\"'%s'\"%str(_) for _ in " + actual_parameter + " if _ != None])}\n" - elif is_selection_parameter(param): - command += "%s " % command_line_prefix + " " - command += "{# ' '.join([\"'%s'\"%str(_) for _ in " + actual_parameter + "])}\n" - else: - command += command_line_prefix + " " - command += "'" + actual_parameter + "'\n" - # logic for other ITEMs + if param.is_list and (param.type is _InFile or is_selection_parameter(param)): + command += command_line_prefix + " " + command += "{# ' '.join([\"'%s'\"%str(_) for _ in " + actual_parameter + " if _ != None])}\n" + elif is_boolean_parameter(param): + command += "%s" % actual_parameter + "\n" else: - # TODO only useful for text fields, integers or floats - # not useful for choices, input fields ... - if is_selection_parameter(param): - command += "%s " % command_line_prefix - command += "'" + actual_parameter + "'\n" - elif is_boolean_parameter(param): - command += "%s" % actual_parameter + "\n" - elif TYPE_TO_GALAXY_TYPE[param.type] is 'text': - command += "%s " % command_line_prefix - command += "'" + actual_parameter + "'\n" - else: - command += "%s " % command_line_prefix - command += actual_parameter + "\n" + command += command_line_prefix + " " + command += "'" + actual_parameter + "'\n" # add if statement for mandatory parameters # - for optional outputs (param_out_x) the presence of the parameter depends on the additional input (param_x) diff --git a/tests/test-data/float.xml b/tests/test-data/float.xml index 626e3c66..223a6c02 100644 --- a/tests/test-data/float.xml +++ b/tests/test-data/float.xml @@ -12,70 +12,70 @@ $param_stdout diff --git a/tests/test-data/ifile.xml b/tests/test-data/ifile.xml index 796819ac..73ced9ea 100644 --- a/tests/test-data/ifile.xml +++ b/tests/test-data/ifile.xml @@ -12,29 +12,29 @@ $param_stdout diff --git a/tests/test-data/ofile.xml b/tests/test-data/ofile.xml index 542f64e5..2e0e6e5c 100644 --- a/tests/test-data/ofile.xml +++ b/tests/test-data/ofile.xml @@ -12,29 +12,29 @@ diff --git a/tests/test-data/select.xml b/tests/test-data/select.xml index fd656a5b..098aa865 100644 --- a/tests/test-data/select.xml +++ b/tests/test-data/select.xml @@ -29,21 +29,21 @@ -default_optional_many '$param_default_optional_many' #end if #if $param_list_wodefault_optional: - -list_wodefault_optional {# ' '.join(["'%s'"%str(_) for _ in $param_list_wodefault_optional])} + -list_wodefault_optional {# ' '.join(["'%s'"%str(_) for _ in $param_list_wodefault_optional if _ != None])} #end if --list_wodefault_mandatory {# ' '.join(["'%s'"%str(_) for _ in $param_list_wodefault_mandatory])} +-list_wodefault_mandatory {# ' '.join(["'%s'"%str(_) for _ in $param_list_wodefault_mandatory if _ != None])} #if $param_list_default_optional: - -list_default_optional {# ' '.join(["'%s'"%str(_) for _ in $param_list_default_optional])} + -list_default_optional {# ' '.join(["'%s'"%str(_) for _ in $param_list_default_optional if _ != None])} #end if --list_default_mandatory {# ' '.join(["'%s'"%str(_) for _ in $param_list_default_mandatory])} +-list_default_mandatory {# ' '.join(["'%s'"%str(_) for _ in $param_list_default_mandatory if _ != None])} #if $param_list_wodefault_optional_many: - -list_wodefault_optional_many {# ' '.join(["'%s'"%str(_) for _ in $param_list_wodefault_optional_many])} + -list_wodefault_optional_many {# ' '.join(["'%s'"%str(_) for _ in $param_list_wodefault_optional_many if _ != None])} #end if --list_wodefault_mandatory_many {# ' '.join(["'%s'"%str(_) for _ in $param_list_wodefault_mandatory_many])} +-list_wodefault_mandatory_many {# ' '.join(["'%s'"%str(_) for _ in $param_list_wodefault_mandatory_many if _ != None])} #if $param_list_default_optional_many: - -list_default_optional_many {# ' '.join(["'%s'"%str(_) for _ in $param_list_default_optional_many])} + -list_default_optional_many {# ' '.join(["'%s'"%str(_) for _ in $param_list_default_optional_many if _ != None])} #end if --list_default_mandatory_many {# ' '.join(["'%s'"%str(_) for _ in $param_list_default_mandatory_many])} +-list_default_mandatory_many {# ' '.join(["'%s'"%str(_) for _ in $param_list_default_mandatory_many if _ != None])} #if $adv_opts.adv_opts_selector=='advanced': -wodefault_mandatory_adv '$adv_opts.param_wodefault_mandatory_adv' -default_mandatory_adv '$adv_opts.param_default_mandatory_adv' @@ -62,21 +62,21 @@ -default_optional_many_adv '$adv_opts.param_default_optional_many_adv' #end if #if $adv_opts.param_list_wodefault_optional_adv: - -list_wodefault_optional_adv {# ' '.join(["'%s'"%str(_) for _ in $adv_opts.param_list_wodefault_optional_adv])} + -list_wodefault_optional_adv {# ' '.join(["'%s'"%str(_) for _ in $adv_opts.param_list_wodefault_optional_adv if _ != None])} #end if - -list_wodefault_mandatory_adv {# ' '.join(["'%s'"%str(_) for _ in $adv_opts.param_list_wodefault_mandatory_adv])} + -list_wodefault_mandatory_adv {# ' '.join(["'%s'"%str(_) for _ in $adv_opts.param_list_wodefault_mandatory_adv if _ != None])} #if $adv_opts.param_list_default_optional_adv: - -list_default_optional_adv {# ' '.join(["'%s'"%str(_) for _ in $adv_opts.param_list_default_optional_adv])} + -list_default_optional_adv {# ' '.join(["'%s'"%str(_) for _ in $adv_opts.param_list_default_optional_adv if _ != None])} #end if - -list_default_mandatory_adv {# ' '.join(["'%s'"%str(_) for _ in $adv_opts.param_list_default_mandatory_adv])} + -list_default_mandatory_adv {# ' '.join(["'%s'"%str(_) for _ in $adv_opts.param_list_default_mandatory_adv if _ != None])} #if $adv_opts.param_list_wodefault_optional_many_adv: - -list_wodefault_optional_many_adv {# ' '.join(["'%s'"%str(_) for _ in $adv_opts.param_list_wodefault_optional_many_adv])} + -list_wodefault_optional_many_adv {# ' '.join(["'%s'"%str(_) for _ in $adv_opts.param_list_wodefault_optional_many_adv if _ != None])} #end if - -list_wodefault_mandatory_many_adv {# ' '.join(["'%s'"%str(_) for _ in $adv_opts.param_list_wodefault_mandatory_many_adv])} + -list_wodefault_mandatory_many_adv {# ' '.join(["'%s'"%str(_) for _ in $adv_opts.param_list_wodefault_mandatory_many_adv if _ != None])} #if $adv_opts.param_list_default_optional_many_adv: - -list_default_optional_many_adv {# ' '.join(["'%s'"%str(_) for _ in $adv_opts.param_list_default_optional_many_adv])} + -list_default_optional_many_adv {# ' '.join(["'%s'"%str(_) for _ in $adv_opts.param_list_default_optional_many_adv if _ != None])} #end if - -list_default_mandatory_many_adv {# ' '.join(["'%s'"%str(_) for _ in $adv_opts.param_list_default_mandatory_many_adv])} + -list_default_mandatory_many_adv {# ' '.join(["'%s'"%str(_) for _ in $adv_opts.param_list_default_mandatory_many_adv if _ != None])} #end if > $param_stdout ]]> From 54032d1332f19d0f00c68b70022385bb13de95d0 Mon Sep 17 00:00:00 2001 From: M Bernt Date: Wed, 24 Jul 2019 13:20:30 +0200 Subject: [PATCH 026/121] link input files to identifier.ext --- galaxy/converter.py | 63 +++++++---- tests/test-data/bool.xml | 3 +- tests/test-data/empty.xml | 3 +- tests/test-data/float.xml | 55 +++++++++- tests/test-data/ifile.xml | 205 ++++++++++++++++++++++++------------ tests/test-data/integer.xml | 51 ++++++++- tests/test-data/ofile.xml | 39 ++++++- tests/test-data/repeat.xml | 63 ++++++++++- tests/test-data/select.xml | 35 +++--- tests/test-data/string.xml | 15 ++- 10 files changed, 421 insertions(+), 111 deletions(-) diff --git a/galaxy/converter.py b/galaxy/converter.py index 20a17b8a..c4427f48 100755 --- a/galaxy/converter.py +++ b/galaxy/converter.py @@ -379,11 +379,15 @@ def create_command(tool, model, **kwargs): @param model the ctd model @param kwargs """ + + # main command + final_preprocessing = "\n" final_command = utils.extract_tool_executable_path(model, kwargs["default_executable_path"]) + '\n' final_command += kwargs["add_to_command_line"] + '\n' advanced_command_start = "#if $adv_opts.adv_opts_selector=='advanced':\n" advanced_command_end = "#end if" advanced_command = "" + parameter_hardcoder = kwargs["parameter_hardcoder"] found_output_parameter = False @@ -391,6 +395,7 @@ def create_command(tool, model, **kwargs): if param.type is _OutFile: found_output_parameter = True command = "" + preprocessing = "" param_name = utils.extract_param_name(param) command_line_prefix = utils.extract_command_line_prefix(param, model) @@ -402,48 +407,61 @@ def create_command(tool, model, **kwargs): command += "%s %s\n" % (command_line_prefix, hardcoded_value) else: # parameter is neither blacklisted nor hardcoded... - galaxy_parameter_name = get_galaxy_parameter_name(param) if param.advanced: - actual_parameter = "$adv_opts.%s" % galaxy_parameter_name + actual_parameter = "adv_opts.%s" % get_galaxy_parameter_name(param) else: - actual_parameter = "$%s" % galaxy_parameter_name + actual_parameter = "%s" % get_galaxy_parameter_name(param) - # logic for ITEMLISTs - if param.is_list and (param.type is _InFile or is_selection_parameter(param)): + # all but bool params need the command line argument (bools have it already in the true/false value) + if not is_boolean_parameter(param): command += command_line_prefix + " " - command += "{# ' '.join([\"'%s'\"%str(_) for _ in " + actual_parameter + " if _ != None])}\n" + + # preprocessing for file inputs: create a link to id.ext in the input directory + if param.type is _InFile: + if param.is_list: + preprocessing += "{# ' && '.join([ 'ls -s \'%s\' \'%s.%s\'' % (_, re.sub('[^\w\-_.]', '_', _.element_identifier)}, _.ext) for _ in " + actual_parameter + " if _ != None ]) } && \n" + command += "{# ' '.join([\"'%s.%s'\"%(re.sub('[^\w\-_.]', '_', _.element_identifier),_.ext) for _ in $" + actual_parameter + " if _ != None])}\n" + else: + preprocessing += "ln -s '$"+ actual_parameter +"' '${re.sub('[^\w\-_.]', '_', "+actual_parameter+".element_identifier)}.${"+actual_parameter+".ext}' &&\n" + command += "'${re.sub('[^\w\-_.]', '_', "+actual_parameter+".element_identifier)}.${"+actual_parameter+".ext}'\n" + + # logic for ITEMLISTs + elif param.is_list and is_selection_parameter(param): + command += "{# ' '.join([\"'%s'\"%str(_) for _ in $" + actual_parameter + "])}\n" elif is_boolean_parameter(param): - command += "%s" % actual_parameter + "\n" + command += "$%s" % actual_parameter + "\n" else: - command += command_line_prefix + " " - command += "'" + actual_parameter + "'\n" + command += "'$" + actual_parameter + "'\n" - # add if statement for mandatory parameters + # add if statement for mandatory parameters and preprocessing # - for optional outputs (param_out_x) the presence of the parameter depends on the additional input (param_x) if not param.required and not is_boolean_parameter(param) and not(param.type is _InFile and param.is_list): if param.type is _OutFile: - galaxy_parameter_name = get_galaxy_parameter_name(param, True) if param.advanced: - actual_parameter = "$adv_opts.%s" % galaxy_parameter_name + actual_parameter = "adv_opts.%s" % get_galaxy_parameter_name(param, True) else: - actual_parameter = "$%s" % galaxy_parameter_name - "param_"+param.name + actual_parameter = "%s" % get_galaxy_parameter_name(param, True) if is_selection_parameter(param): - command = "#if " + actual_parameter + ":\n" + utils.indent(command) + "\n#end if\n" + command = "#if $" + actual_parameter + ":\n" + utils.indent(command) + "\n#end if\n" else: - command = "#if str(" + actual_parameter + "):\n" + utils.indent(command) + "\n#end if\n" + command = "#if str($" + actual_parameter + "):\n" + utils.indent(command) + "\n#end if\n" + preprocessing = "#if str($" + actual_parameter + "):\n" + utils.indent(preprocessing) + "\n#end if\n" if param.advanced and param.type is not _OutFile: advanced_command += "%s\n" % utils.indent(command) else: final_command += command + final_preprocessing += preprocessing + if advanced_command: final_command += "%s%s%s\n" % (advanced_command_start, advanced_command, advanced_command_end) if not found_output_parameter: final_command += "> $param_stdout\n" + final_command = final_preprocessing + final_command + command_node = add_child_node(tool, "command") command_node.text = CDATA(final_command) @@ -593,7 +611,7 @@ def create_param_attribute_list(param_node, param, supported_file_formats): if param.type is _InFile: # assume it's just text unless restrictions are provided - param_format = "txt" + param_format = "data" if param.restrictions is not None: # join all formats of the file, take mapping from supported_file if available for an entry if type(param.restrictions) is _FileFormat: @@ -860,7 +878,9 @@ def create_outputs(parent, model, **kwargs): create outputs section of the Galaxy tool @param tool the Galaxy tool @param model the ctd model - @param kwargs + @param kwargs + - parameter_hardcoder and + - supported_file_formats () """ outputs_node = add_child_node(parent, "outputs") parameter_hardcoder = kwargs["parameter_hardcoder"] @@ -889,6 +909,11 @@ def create_output_node(parent, param, model, supported_file_formats): if data_node.attrib["name"].startswith('param_out_'): data_node.attrib["label"] = "${tool.name} on ${on_string}: %s" % data_node.attrib["name"][10:] + # determine format attribute + # - default is data + # - check if all listed possible formats are supported in Galaxy + # - if there is a single one, then take this + # - otherwise try to determine an input with the same restrictions and use this as format source data_format = "data" if param.restrictions is not None: if type(param.restrictions) is _FileFormat: @@ -922,9 +947,9 @@ def create_output_node(parent, param, model, supported_file_formats): raise InvalidModelException("Unrecognized restriction type [%(type)s] " "for output [%(name)s]" % {"type": type(param.restrictions), "name": param.name}) - data_node.attrib["format"] = data_format + # add filter for optional parameters if not param.required: filter_node = add_child_node(data_node, "filter") filter_node.text = get_galaxy_parameter_name(param, True) diff --git a/tests/test-data/bool.xml b/tests/test-data/bool.xml index 755af6f6..b39a8669 100644 --- a/tests/test-data/bool.xml +++ b/tests/test-data/bool.xml @@ -10,7 +10,8 @@ - - $param_stdout ]]> diff --git a/tests/test-data/float.xml b/tests/test-data/float.xml index 223a6c02..010a253c 100644 --- a/tests/test-data/float.xml +++ b/tests/test-data/float.xml @@ -2,7 +2,7 @@ - Floateger parameter tests + Float parameter tests echo FloatTest macros.xml @@ -10,7 +10,56 @@ - - diff --git a/tests/test-data/ifile.xml b/tests/test-data/ifile.xml index 73ced9ea..62811ce8 100644 --- a/tests/test-data/ifile.xml +++ b/tests/test-data/ifile.xml @@ -10,130 +10,203 @@ - $param_stdout ]]> - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + diff --git a/tests/test-data/integer.xml b/tests/test-data/integer.xml index 47ab0144..77877c30 100644 --- a/tests/test-data/integer.xml +++ b/tests/test-data/integer.xml @@ -10,7 +10,56 @@ - - - - $param_stdout ]]> diff --git a/tests/test-data/string.xml b/tests/test-data/string.xml index 3b65857b..42fca2c7 100644 --- a/tests/test-data/string.xml +++ b/tests/test-data/string.xml @@ -10,7 +10,20 @@ - Date: Fri, 26 Jul 2019 10:50:27 +0200 Subject: [PATCH 027/121] added code to autogenerate a simple test for unit tests idea: take the generated input and output nodes and adapt it to a test case. mainly the optional parameters and outputs are stripped and attributes changes appropriately. adds planemo linting and testing to the unit tests for the future this shall be extended to take a ctd generated by the openms unit tests. the problem with the unit tests is that only files are given on the command line, but all other parameters in a ctd file. using an improved version of this tool the following strategy seems useful: generate the full xml file and extract just the test part. also fixes a few minor errors detected with the test --- .travis.yml | 2 +- galaxy/converter.py | 118 ++++++++++++++--- tests/test-data/bool.xml | 16 ++- tests/test-data/empty.xml | 5 + tests/test-data/float.ctd | 8 +- tests/test-data/float.xml | 136 ++++++++------------ tests/test-data/ifile.ctd | 1 + tests/test-data/ifile.xml | 177 +++++++++++++++----------- tests/test-data/integer.xml | 136 ++++++++------------ tests/test-data/ofile.xml | 91 ++++++------- tests/test-data/repeat.xml | 168 ++++++++++-------------- tests/test-data/select.xml | 128 +++++++++---------- tests/test-data/string.xml | 40 +++--- tests/test-data/test-data/outfile.txt | 0 tests/test-data/test-data/test.ext | 0 tests/test-data/test-data/test2.ext | 0 tests/test_galaxy_cli.py | 18 ++- 17 files changed, 543 insertions(+), 501 deletions(-) create mode 100644 tests/test-data/test-data/outfile.txt create mode 100644 tests/test-data/test-data/test.ext create mode 100644 tests/test-data/test-data/test2.ext diff --git a/.travis.yml b/.travis.yml index be252d41..564f9b2a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -34,7 +34,7 @@ install: - python setup.py install - conda install coverage green codecov - + - conda install -c bioconda planemo script: - python setup.py test diff --git a/galaxy/converter.py b/galaxy/converter.py index c4427f48..2fe388a6 100755 --- a/galaxy/converter.py +++ b/galaxy/converter.py @@ -7,7 +7,7 @@ import copy from string import strip from lxml import etree -from lxml.etree import CDATA, SubElement, Element, ElementTree, ParseError, parse +from lxml.etree import CDATA, SubElement, Element, ElementTree, ParseError, parse, strip_elements from common import utils, logger from common.exceptions import ApplicationException, InvalidModelException @@ -72,6 +72,8 @@ def add_specific_args(parser): "The macros stdio, requirements and advanced_options are " "required. Please see galaxy/macros.xml for an example of a " "valid macros file. All defined macros will be imported.") + parser.add_argument("--test-test", dest="test_test", action='store_true', default=False, required=False, + help="Generate a simple test for the internal unit tests.") def convert_models(args, parsed_ctds): @@ -103,7 +105,8 @@ def convert_models(args, parsed_ctds): skip_tools=skip_tools, macros_file_names=args.macros_files, macros_to_expand=macros_to_expand, - parameter_hardcoder=args.parameter_hardcoder) + parameter_hardcoder=args.parameter_hardcoder, + test_test=args.test_test) # generation of galaxy stubs is ready... now, let's see if we need to generate a tool_conf.xml if args.tool_conf_destination is not None: @@ -277,8 +280,10 @@ def _convert_internal(parsed_ctds, **kwargs): create_description(tool, model) expand_macros(tool, model, **kwargs) create_command(tool, model, **kwargs) - create_inputs(tool, model, **kwargs) - create_outputs(tool, model, **kwargs) + inputs = create_inputs(tool, model, **kwargs) + outputs = create_outputs(tool, model, **kwargs) + if kwargs["test_test"]: + create_tests(tool, copy.deepcopy(inputs), copy.deepcopy(outputs)) create_help(tool, model) # wrap our tool element into a tree to be able to serialize it @@ -390,10 +395,13 @@ def create_command(tool, model, **kwargs): parameter_hardcoder = kwargs["parameter_hardcoder"] + found_input_parameter = False found_output_parameter = False for param in utils.extract_and_flatten_parameters(model): if param.type is _OutFile: found_output_parameter = True + if param.type is _InFile: + found_input_parameter = True command = "" preprocessing = "" param_name = utils.extract_param_name(param) @@ -407,7 +415,7 @@ def create_command(tool, model, **kwargs): command += "%s %s\n" % (command_line_prefix, hardcoded_value) else: # parameter is neither blacklisted nor hardcoded... - if param.advanced: + if param.advanced and not param.type is _OutFile: actual_parameter = "adv_opts.%s" % get_galaxy_parameter_name(param) else: actual_parameter = "%s" % get_galaxy_parameter_name(param) @@ -420,14 +428,14 @@ def create_command(tool, model, **kwargs): if param.type is _InFile: if param.is_list: preprocessing += "{# ' && '.join([ 'ls -s \'%s\' \'%s.%s\'' % (_, re.sub('[^\w\-_.]', '_', _.element_identifier)}, _.ext) for _ in " + actual_parameter + " if _ != None ]) } && \n" - command += "{# ' '.join([\"'%s.%s'\"%(re.sub('[^\w\-_.]', '_', _.element_identifier),_.ext) for _ in $" + actual_parameter + " if _ != None])}\n" + command += "${' '.join([\"'%s.%s'\"%(re.sub('[^\w\-_.]', '_', _.element_identifier),_.ext) for _ in $" + actual_parameter + " if _ != None])}\n" else: preprocessing += "ln -s '$"+ actual_parameter +"' '${re.sub('[^\w\-_.]', '_', "+actual_parameter+".element_identifier)}.${"+actual_parameter+".ext}' &&\n" command += "'${re.sub('[^\w\-_.]', '_', "+actual_parameter+".element_identifier)}.${"+actual_parameter+".ext}'\n" # logic for ITEMLISTs elif param.is_list and is_selection_parameter(param): - command += "{# ' '.join([\"'%s'\"%str(_) for _ in $" + actual_parameter + "])}\n" + command += "${' '.join([\"'%s'\"%str(_) for _ in $" + actual_parameter + "])}\n" elif is_boolean_parameter(param): command += "$%s" % actual_parameter + "\n" else: @@ -436,7 +444,7 @@ def create_command(tool, model, **kwargs): # add if statement for mandatory parameters and preprocessing # - for optional outputs (param_out_x) the presence of the parameter depends on the additional input (param_x) if not param.required and not is_boolean_parameter(param) and not(param.type is _InFile and param.is_list): - if param.type is _OutFile: + if param.type is _OutFile: if param.advanced: actual_parameter = "adv_opts.%s" % get_galaxy_parameter_name(param, True) else: @@ -445,6 +453,7 @@ def create_command(tool, model, **kwargs): command = "#if $" + actual_parameter + ":\n" + utils.indent(command) + "\n#end if\n" else: command = "#if str($" + actual_parameter + "):\n" + utils.indent(command) + "\n#end if\n" + if param.type is _InFile: preprocessing = "#if str($" + actual_parameter + "):\n" + utils.indent(preprocessing) + "\n#end if\n" if param.advanced and param.type is not _OutFile: @@ -462,6 +471,9 @@ def create_command(tool, model, **kwargs): final_command = final_preprocessing + final_command + if found_input_parameter: + final_command = "\n#import re" + final_command + command_node = add_child_node(tool, "command") command_node.text = CDATA(final_command) @@ -527,6 +539,7 @@ def create_inputs(tool, model, **kwargs): @param tool the Galaxy tool @param model the ctd model @param kwargs + @return inputs node """ inputs_node = SubElement(tool, "inputs") @@ -561,22 +574,14 @@ def create_inputs(tool, model, **kwargs): else: parent_node = inputs_node -# # for lists we need a repeat tag, execept if -# # - list of strings with a fixed set of options -> select w multiple=true -# # - InFiles -> data w multiple=true -# if param.is_list and\ -# not (param.type is str and param.restrictions is not None) and\ -# param.type is not _InFile: -# rep_node = add_child_node(parent_node, "repeat") -# create_repeat_attribute_list(rep_node, param) -# parent_node = rep_node - param_node = add_child_node(parent_node, "param") create_param_attribute_list(param_node, _param, kwargs["supported_file_formats"]) if not advanced_node is None: inputs_node.append(advanced_node) + return inputs_node + def create_param_attribute_list(param_node, param, supported_file_formats): """ @@ -633,9 +638,9 @@ def create_param_attribute_list(param_node, param, supported_file_formats): param_node.attrib["type"] = param_type if param_type == "select" and param.default in param.restrictions.choices: - param_node.attrib["optional"] = "False" + param_node.attrib["optional"] = "false" else: - param_node.attrib["optional"] = str(not param.required) + param_node.attrib["optional"] = str(not param.required).lower() # check for parameters with restricted values (which will correspond to a "select" in galaxy) if param.restrictions is not None: @@ -748,7 +753,7 @@ def create_param_attribute_list(param_node, param, supported_file_formats): # simple boolean with a default if param.default is True: param_node.attrib["checked"] = "true" - elif param.type is int or param.type is float: + elif param.type is int or param.type is float or param.type is str: param_node.attrib["value"] = "" # add label, help, and argument @@ -901,6 +906,7 @@ def create_outputs(parent, model, **kwargs): if len(outputs_node) == 0: add_child_node(outputs_node, "data", OrderedDict([("name", "param_stdout"), ("format", "txt"), ("label", "Output from stdout")])) + return outputs_node def create_output_node(parent, param, model, supported_file_formats): @@ -981,6 +987,76 @@ def create_change_format_node(parent, data_formats, input_ref): OrderedDict([("input", input_ref), ("value", data_format), ("format", data_format)])) +def create_tests(parent, inputs, outputs): + """ + create tests section of the Galaxy tool + @param tool the Galaxy tool + @param inputs a copy of the inputs + """ + tests_node = add_child_node(parent, "tests") + test_node = add_child_node(tests_node, "test") + + strip_elements(inputs, "validator", "sanitizer") + for node in inputs.iter(): + if node.tag == "expand": + node.tag = "conditional" + node.attrib["name"] = "adv_opts" + add_child_node(node, "param", OrderedDict([("name", "adv_opts_selector"), ("value", "advanced")])) + if not "type" in node.attrib: + continue + + if (node.attrib["type"] == "select" and "true" in set([_.attrib.get("selected","false") for _ in node])) or\ + (node.attrib["type"] == "select" and node.attrib.get("value", "") != ""): + node.tag = "delete_node" + continue + + # TODO make this optional (ie add aparameter) + if node.attrib["optional"] == "true": + node.tag = "delete_node" + continue + + if node.attrib["type"] == "boolean": + node.attrib["value"] = node.attrib["truevalue"] + elif node.attrib["type"] == "text" and node.attrib["value"] == "": + node.attrib["value"] = "1,2" # use a comma separated list here to cover the repeat (int/float) case + elif node.attrib["type"] == "integer" and node.attrib["value"] == "": + node.attrib["value"] = "1" + elif node.attrib["type"] == "float" and node.attrib["value"] == "": + node.attrib["value"] = "1.0" + elif node.attrib["type"] == "select": + if node.attrib.get("display", None) == "radio" or node.attrib.get("multiple", "false") == "false": + node.attrib["value"] = node[-1].attrib["value"] + elif node.attrib.get("multiple", None) == "true": + node.attrib["value"] = ",".join([ _.attrib["value"] for _ in node ]) + elif node.attrib["type"] == "data": + if node.attrib.get("multiple", "false") == "true": + node.attrib["value"] = "test.ext,test2.ext" + else: + node.attrib["value"] = "test.ext" + for a in set(node.attrib) - set(["name","value","format"]): + del node.attrib[a] + strip_elements(inputs, "delete_node", "option") + for node in inputs: + test_node.append(node) + + for node in outputs.iter(): + if node.tag == "data": + node.tag = "output" + node.attrib["ftype"] = node.attrib["format"] + if len(node) > 0 and node[0].tag == "filter": + node.tag = "delete_node" + node.attrib["value"] = "outfile.txt" + if node.attrib.get("name", None) == "param_stdout": + node.attrib["lines_diff"] = "2" + for a in set(node.attrib) - set(["name","value","ftype", "lines_diff"]): + del node.attrib[a] + strip_elements(outputs, "delete_node") + + for node in outputs: + test_node.append(node) + + + # Shows basic information about the file, such as data ranges and file type. def create_help(tool, model): """ diff --git a/tests/test-data/bool.xml b/tests/test-data/bool.xml index b39a8669..c23672f3 100644 --- a/tests/test-data/bool.xml +++ b/tests/test-data/bool.xml @@ -22,16 +22,24 @@ $param_false > $param_stdout ]]> - - + + - - + + + + + + + + + + + + + + + - - + + - + - + diff --git a/tests/test-data/float.xml b/tests/test-data/float.xml index 010a253c..18a37bff 100644 --- a/tests/test-data/float.xml +++ b/tests/test-data/float.xml @@ -11,54 +11,6 @@ $param_stdout ]]> - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/test-data/ifile.xml b/tests/test-data/ifile.xml index 62811ce8..b7489113 100644 --- a/tests/test-data/ifile.xml +++ b/tests/test-data/ifile.xml @@ -11,6 +11,7 @@ $param_stdout ]]> - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $param_stdout ]]> - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - + + + @@ -164,6 +128,25 @@ echo outFileTest advanced_options['adv_opts_selector'] == 'advanced' and advanced_options['param_optional_multformat_adv'] + + + + + + + + + + + + + + + + + + + $param_stdout ]]> - + @@ -166,7 +106,7 @@ echo RepeatTest - + @@ -174,7 +114,7 @@ echo RepeatTest - + @@ -182,7 +122,7 @@ echo RepeatTest - + @@ -190,7 +130,7 @@ echo RepeatTest - + @@ -201,7 +141,7 @@ echo RepeatTest - + @@ -212,7 +152,7 @@ echo RepeatTest - + @@ -223,7 +163,7 @@ echo RepeatTest - + @@ -234,7 +174,7 @@ echo RepeatTest - + @@ -247,7 +187,7 @@ echo RepeatTest - + @@ -260,7 +200,7 @@ echo RepeatTest - + @@ -273,7 +213,7 @@ echo RepeatTest - + @@ -286,7 +226,7 @@ echo RepeatTest - + @@ -297,7 +237,7 @@ echo RepeatTest - + @@ -308,7 +248,7 @@ echo RepeatTest - + @@ -319,7 +259,7 @@ echo RepeatTest - + @@ -330,7 +270,7 @@ echo RepeatTest - + @@ -343,7 +283,7 @@ echo RepeatTest - + @@ -356,7 +296,7 @@ echo RepeatTest - + @@ -369,7 +309,7 @@ echo RepeatTest - + @@ -383,7 +323,7 @@ echo RepeatTest - + @@ -391,7 +331,7 @@ echo RepeatTest - + @@ -399,7 +339,7 @@ echo RepeatTest - + @@ -407,7 +347,7 @@ echo RepeatTest - + @@ -415,7 +355,7 @@ echo RepeatTest - + @@ -426,7 +366,7 @@ echo RepeatTest - + @@ -437,7 +377,7 @@ echo RepeatTest - + @@ -448,7 +388,7 @@ echo RepeatTest - + @@ -459,7 +399,7 @@ echo RepeatTest - + @@ -472,7 +412,7 @@ echo RepeatTest - + @@ -485,7 +425,7 @@ echo RepeatTest - + @@ -498,7 +438,7 @@ echo RepeatTest - + @@ -511,7 +451,7 @@ echo RepeatTest - + @@ -522,7 +462,7 @@ echo RepeatTest - + @@ -533,7 +473,7 @@ echo RepeatTest - + @@ -544,7 +484,7 @@ echo RepeatTest - + @@ -555,7 +495,7 @@ echo RepeatTest - + @@ -568,7 +508,7 @@ echo RepeatTest - + @@ -581,7 +521,7 @@ echo RepeatTest - + @@ -594,7 +534,7 @@ echo RepeatTest - + @@ -612,6 +552,34 @@ echo RepeatTest + + + + + + + + + + + + + + + + + + + + + + + + + + + + $param_stdout ]]> - - + - value != "select a value" - + - + - + - - + - value != "select a value" - + - + - + - + - - + - value != "select a value" - + - + - + - - + - value != "select a value" - + - + @@ -195,112 +187,104 @@ echo SelectTest - - + - value != "select a value" - + - + - + - - + - value != "select a value" - + - + - + - + - - + - value != "select a value" - + - + - + - - + - value != "select a value" - + - + @@ -312,6 +296,22 @@ echo SelectTest + + + + + + + + + + + + + + + + $param_stdout ]]> - + @@ -54,7 +42,7 @@ echo StringTest - + @@ -62,7 +50,7 @@ echo StringTest - + @@ -70,7 +58,7 @@ echo StringTest - + @@ -79,7 +67,7 @@ echo StringTest - + @@ -87,7 +75,7 @@ echo StringTest - + @@ -95,7 +83,7 @@ echo StringTest - + @@ -103,7 +91,7 @@ echo StringTest - + @@ -116,6 +104,18 @@ echo StringTest + + + + + + + + + + + + Date: Fri, 26 Jul 2019 13:02:55 +0200 Subject: [PATCH 028/121] test w expect_num_outputs and some bug fixes in cli generation - infile test for optional arguments must not test for str(param) but param - some cheetah fixes --- galaxy/converter.py | 19 +-- tests/test-data/bool.xml | 6 +- tests/test-data/empty.xml | 2 +- tests/test-data/float.xml | 2 +- tests/test-data/ifile.xml | 242 ++++++++++++++++++------------------ tests/test-data/integer.xml | 2 +- tests/test-data/ofile.xml | 14 ++- tests/test-data/repeat.xml | 2 +- tests/test-data/select.xml | 2 +- tests/test-data/string.xml | 2 +- 10 files changed, 155 insertions(+), 138 deletions(-) diff --git a/galaxy/converter.py b/galaxy/converter.py index 2fe388a6..79abf352 100755 --- a/galaxy/converter.py +++ b/galaxy/converter.py @@ -427,11 +427,11 @@ def create_command(tool, model, **kwargs): # preprocessing for file inputs: create a link to id.ext in the input directory if param.type is _InFile: if param.is_list: - preprocessing += "{# ' && '.join([ 'ls -s \'%s\' \'%s.%s\'' % (_, re.sub('[^\w\-_.]', '_', _.element_identifier)}, _.ext) for _ in " + actual_parameter + " if _ != None ]) } && \n" - command += "${' '.join([\"'%s.%s'\"%(re.sub('[^\w\-_.]', '_', _.element_identifier),_.ext) for _ in $" + actual_parameter + " if _ != None])}\n" + preprocessing += "${ ' && '.join([ \"ln -s '%s' '%s.%s'\" % (_, re.sub('[^\w\-_.]', '_', _.element_identifier), _.ext) for _ in $" + actual_parameter + " if _ != None ]) } && \n" + command += "${' '.join([\"'%s.%s'\"%(re.sub('[^\w\-_.]', '_', _.element_identifier),_.ext) for _ in $" + actual_parameter + " if _ ])}\n" else: - preprocessing += "ln -s '$"+ actual_parameter +"' '${re.sub('[^\w\-_.]', '_', "+actual_parameter+".element_identifier)}.${"+actual_parameter+".ext}' &&\n" - command += "'${re.sub('[^\w\-_.]', '_', "+actual_parameter+".element_identifier)}.${"+actual_parameter+".ext}'\n" + preprocessing += "ln -s '$"+ actual_parameter +"' '${re.sub(\"[^\w\-_.]\", \"_\", $"+actual_parameter+".element_identifier)}.${"+actual_parameter+".ext}' &&\n" + command += "'${re.sub('[^\w\-_.]', '_', $"+actual_parameter+".element_identifier)}.${"+actual_parameter+".ext}'\n" # logic for ITEMLISTs elif param.is_list and is_selection_parameter(param): @@ -449,12 +449,12 @@ def create_command(tool, model, **kwargs): actual_parameter = "adv_opts.%s" % get_galaxy_parameter_name(param, True) else: actual_parameter = "%s" % get_galaxy_parameter_name(param, True) - if is_selection_parameter(param): + if is_selection_parameter(param) or param.type == _InFile: command = "#if $" + actual_parameter + ":\n" + utils.indent(command) + "\n#end if\n" else: command = "#if str($" + actual_parameter + "):\n" + utils.indent(command) + "\n#end if\n" if param.type is _InFile: - preprocessing = "#if str($" + actual_parameter + "):\n" + utils.indent(preprocessing) + "\n#end if\n" + preprocessing = "#if $" + actual_parameter + ":\n" + utils.indent(preprocessing) + "\n#end if\n" if param.advanced and param.type is not _OutFile: advanced_command += "%s\n" % utils.indent(command) @@ -1011,7 +1011,7 @@ def create_tests(parent, inputs, outputs): continue # TODO make this optional (ie add aparameter) - if node.attrib["optional"] == "true": + if node.attrib["optional"] == "true" and node.attrib["type"]!="boolean": node.tag = "delete_node" continue @@ -1038,11 +1038,12 @@ def create_tests(parent, inputs, outputs): strip_elements(inputs, "delete_node", "option") for node in inputs: test_node.append(node) - + outputs_cnt = 0 for node in outputs.iter(): if node.tag == "data": node.tag = "output" node.attrib["ftype"] = node.attrib["format"] + outputs_cnt += 1 if len(node) > 0 and node[0].tag == "filter": node.tag = "delete_node" node.attrib["value"] = "outfile.txt" @@ -1055,7 +1056,7 @@ def create_tests(parent, inputs, outputs): for node in outputs: test_node.append(node) - + test_node.attrib["expect_num_outputs"] = str(outputs_cnt) # Shows basic information about the file, such as data ranges and file type. def create_help(tool, model): diff --git a/tests/test-data/bool.xml b/tests/test-data/bool.xml index c23672f3..8531d33a 100644 --- a/tests/test-data/bool.xml +++ b/tests/test-data/bool.xml @@ -33,8 +33,12 @@ $param_false - + + + + + diff --git a/tests/test-data/empty.xml b/tests/test-data/empty.xml index 802cdff8..32b64032 100644 --- a/tests/test-data/empty.xml +++ b/tests/test-data/empty.xml @@ -20,7 +20,7 @@ echo StringTest - + diff --git a/tests/test-data/float.xml b/tests/test-data/float.xml index 18a37bff..65a94db4 100644 --- a/tests/test-data/float.xml +++ b/tests/test-data/float.xml @@ -121,7 +121,7 @@ echo FloatTest - + diff --git a/tests/test-data/ifile.xml b/tests/test-data/ifile.xml index b7489113..3bef58f2 100644 --- a/tests/test-data/ifile.xml +++ b/tests/test-data/ifile.xml @@ -12,153 +12,153 @@ $param_stdout ]]> @@ -218,7 +218,7 @@ echo inFileTest - + diff --git a/tests/test-data/integer.xml b/tests/test-data/integer.xml index 44d6b7f7..7eb4a436 100644 --- a/tests/test-data/integer.xml +++ b/tests/test-data/integer.xml @@ -121,7 +121,7 @@ echo IntTest - + diff --git a/tests/test-data/ofile.xml b/tests/test-data/ofile.xml index 667499b0..82e743c6 100644 --- a/tests/test-data/ofile.xml +++ b/tests/test-data/ofile.xml @@ -129,8 +129,20 @@ echo outFileTest - + + + + + + + + + + + + + diff --git a/tests/test-data/repeat.xml b/tests/test-data/repeat.xml index c278470d..2276779c 100644 --- a/tests/test-data/repeat.xml +++ b/tests/test-data/repeat.xml @@ -553,7 +553,7 @@ echo RepeatTest - + diff --git a/tests/test-data/select.xml b/tests/test-data/select.xml index acd9b141..0a9f55ae 100644 --- a/tests/test-data/select.xml +++ b/tests/test-data/select.xml @@ -297,7 +297,7 @@ echo SelectTest - + diff --git a/tests/test-data/string.xml b/tests/test-data/string.xml index 224025c6..922f5bdb 100644 --- a/tests/test-data/string.xml +++ b/tests/test-data/string.xml @@ -105,7 +105,7 @@ echo StringTest - + From e926ad7dc2eb2a472bee6b07759a63bb5cd417f5 Mon Sep 17 00:00:00 2001 From: M Bernt Date: Fri, 26 Jul 2019 13:15:06 +0200 Subject: [PATCH 029/121] add bool as in ctd 1.7 - also make test case use tre/falsevalue depending on checked --- galaxy/converter.py | 41 ++++++++++++++++++++------------- tests/test-data/bool.ctd | 4 ++++ tests/test-data/bool.xml | 16 +++++++++++-- tests/test-data/ifile.xml | 48 +++++++++++++++++++-------------------- tests/test-data/ofile.xml | 24 ++++++++++---------- 5 files changed, 79 insertions(+), 54 deletions(-) diff --git a/galaxy/converter.py b/galaxy/converter.py index 79abf352..b3bc8d63 100755 --- a/galaxy/converter.py +++ b/galaxy/converter.py @@ -427,7 +427,7 @@ def create_command(tool, model, **kwargs): # preprocessing for file inputs: create a link to id.ext in the input directory if param.type is _InFile: if param.is_list: - preprocessing += "${ ' && '.join([ \"ln -s '%s' '%s.%s'\" % (_, re.sub('[^\w\-_.]', '_', _.element_identifier), _.ext) for _ in $" + actual_parameter + " if _ != None ]) } && \n" + preprocessing += "${ ' && '.join([ \"ln -s '%s' '%s.%s'\" % (_, re.sub('[^\w\-_.]', '_', _.element_identifier), _.ext) for _ in $" + actual_parameter + " if _ ]) } && \n" command += "${' '.join([\"'%s.%s'\"%(re.sub('[^\w\-_.]', '_', _.element_identifier),_.ext) for _ in $" + actual_parameter + " if _ ])}\n" else: preprocessing += "ln -s '$"+ actual_parameter +"' '${re.sub(\"[^\w\-_.]\", \"_\", $"+actual_parameter+".element_identifier)}.${"+actual_parameter+".ext}' &&\n" @@ -643,7 +643,7 @@ def create_param_attribute_list(param_node, param, supported_file_formats): param_node.attrib["optional"] = str(not param.required).lower() # check for parameters with restricted values (which will correspond to a "select" in galaxy) - if param.restrictions is not None: + if param.restrictions is not None or param_type == "boolean": # it could be either _Choices or _NumericRange, with special case for boolean types if param_type == "boolean": create_boolean_parameter(param_node, param) @@ -861,21 +861,27 @@ def create_boolean_parameter(param_node, param): # in ctd (1.6.2) bools are strings with restriction true,false # - if the default is false then they are flags # - otherwise the true or false value needs to be added (where the true case is unnecessary) - choices = get_lowercase_list(param.restrictions.choices) - if set(choices) == set(["true","false"]) and param.default == "true": - param_node.attrib["truevalue"] = "" - param_node.attrib["falsevalue"] = "-%s false" % utils.extract_param_name(param) - else: + + if param.type == str: + choices = get_lowercase_list(param.restrictions.choices) + if set(choices) == set(["true","false"]) and param.default == "true": + param_node.attrib["truevalue"] = "" + param_node.attrib["falsevalue"] = "-%s false" % utils.extract_param_name(param) + else: + param_node.attrib["truevalue"] = "-%s" % utils.extract_param_name(param) + param_node.attrib["falsevalue"] = "" + + # set the checked attribute + if param.default is not None: + checked_value = "false" + default = strip(string.lower(param.default)) + if default == "yes" or default == "true": + checked_value = "true" + param_node.attrib["checked"] = checked_value + else: param_node.attrib["truevalue"] = "-%s" % utils.extract_param_name(param) param_node.attrib["falsevalue"] = "" - - # set the checked attribute - if param.default is not None: - checked_value = "false" - default = strip(string.lower(param.default)) - if default == "yes" or default == "true": - checked_value = "true" - param_node.attrib["checked"] = checked_value + param_node.attrib["checked"] = str(param.default).lower() def create_outputs(parent, model, **kwargs): @@ -1016,7 +1022,10 @@ def create_tests(parent, inputs, outputs): continue if node.attrib["type"] == "boolean": - node.attrib["value"] = node.attrib["truevalue"] + if node.attrib["checked"] == "true": + node.attrib["value"] = node.attrib["truevalue"] + else: + node.attrib["value"] = node.attrib["falsevalue"] elif node.attrib["type"] == "text" and node.attrib["value"] == "": node.attrib["value"] = "1,2" # use a comma separated list here to cover the repeat (int/float) case elif node.attrib["type"] == "integer" and node.attrib["value"] == "": diff --git a/tests/test-data/bool.ctd b/tests/test-data/bool.ctd index bec0cc7d..3fab0317 100644 --- a/tests/test-data/bool.ctd +++ b/tests/test-data/bool.ctd @@ -11,6 +11,10 @@ + + + + diff --git a/tests/test-data/bool.xml b/tests/test-data/bool.xml index 8531d33a..c45fbc63 100644 --- a/tests/test-data/bool.xml +++ b/tests/test-data/bool.xml @@ -15,18 +15,26 @@ echo BoolTest $param_true $param_false +$param_flag_true +$param_flag_false #if $adv_opts.adv_opts_selector=='advanced': $adv_opts.param_true_adv $adv_opts.param_false_adv + $adv_opts.param_flag_true_adv + $adv_opts.param_flag_false_adv #end if > $param_stdout ]]> + + + + @@ -35,10 +43,14 @@ $param_false - + + + - + + + diff --git a/tests/test-data/ifile.xml b/tests/test-data/ifile.xml index 3bef58f2..5341bbca 100644 --- a/tests/test-data/ifile.xml +++ b/tests/test-data/ifile.xml @@ -36,18 +36,18 @@ ln -s '$param_default_mandatory_multformat' '${re.sub("[^\w\-_.]", "_", $param_d #if $param_default_optional_multformat: ln -s '$param_default_optional_multformat' '${re.sub("[^\w\-_.]", "_", $param_default_optional_multformat.element_identifier)}.${param_default_optional_multformat.ext}' && #end if -${ ' && '.join([ "ln -s '%s' '%s.%s'" % (_, re.sub('[^\w\-_.]', '_', _.element_identifier), _.ext) for _ in $param_list_mandatory_noformat if _ != None ]) } && -${ ' && '.join([ "ln -s '%s' '%s.%s'" % (_, re.sub('[^\w\-_.]', '_', _.element_identifier), _.ext) for _ in $param_list_optional_noformat if _ != None ]) } && -${ ' && '.join([ "ln -s '%s' '%s.%s'" % (_, re.sub('[^\w\-_.]', '_', _.element_identifier), _.ext) for _ in $param_list_mandatory_oneformat if _ != None ]) } && -${ ' && '.join([ "ln -s '%s' '%s.%s'" % (_, re.sub('[^\w\-_.]', '_', _.element_identifier), _.ext) for _ in $param_list_optional_oneformat if _ != None ]) } && -${ ' && '.join([ "ln -s '%s' '%s.%s'" % (_, re.sub('[^\w\-_.]', '_', _.element_identifier), _.ext) for _ in $param_list_mandatory_multformat if _ != None ]) } && -${ ' && '.join([ "ln -s '%s' '%s.%s'" % (_, re.sub('[^\w\-_.]', '_', _.element_identifier), _.ext) for _ in $param_list_optional_multformat if _ != None ]) } && -${ ' && '.join([ "ln -s '%s' '%s.%s'" % (_, re.sub('[^\w\-_.]', '_', _.element_identifier), _.ext) for _ in $param_list_default_mandatory_noformat if _ != None ]) } && -${ ' && '.join([ "ln -s '%s' '%s.%s'" % (_, re.sub('[^\w\-_.]', '_', _.element_identifier), _.ext) for _ in $param_list_default_optional_noformat if _ != None ]) } && -${ ' && '.join([ "ln -s '%s' '%s.%s'" % (_, re.sub('[^\w\-_.]', '_', _.element_identifier), _.ext) for _ in $param_list_default_mandatory_oneformat if _ != None ]) } && -${ ' && '.join([ "ln -s '%s' '%s.%s'" % (_, re.sub('[^\w\-_.]', '_', _.element_identifier), _.ext) for _ in $param_list_default_optional_oneformat if _ != None ]) } && -${ ' && '.join([ "ln -s '%s' '%s.%s'" % (_, re.sub('[^\w\-_.]', '_', _.element_identifier), _.ext) for _ in $param_list_default_mandatory_multformat if _ != None ]) } && -${ ' && '.join([ "ln -s '%s' '%s.%s'" % (_, re.sub('[^\w\-_.]', '_', _.element_identifier), _.ext) for _ in $param_list_default_optional_multformat if _ != None ]) } && +${ ' && '.join([ "ln -s '%s' '%s.%s'" % (_, re.sub('[^\w\-_.]', '_', _.element_identifier), _.ext) for _ in $param_list_mandatory_noformat if _ ]) } && +${ ' && '.join([ "ln -s '%s' '%s.%s'" % (_, re.sub('[^\w\-_.]', '_', _.element_identifier), _.ext) for _ in $param_list_optional_noformat if _ ]) } && +${ ' && '.join([ "ln -s '%s' '%s.%s'" % (_, re.sub('[^\w\-_.]', '_', _.element_identifier), _.ext) for _ in $param_list_mandatory_oneformat if _ ]) } && +${ ' && '.join([ "ln -s '%s' '%s.%s'" % (_, re.sub('[^\w\-_.]', '_', _.element_identifier), _.ext) for _ in $param_list_optional_oneformat if _ ]) } && +${ ' && '.join([ "ln -s '%s' '%s.%s'" % (_, re.sub('[^\w\-_.]', '_', _.element_identifier), _.ext) for _ in $param_list_mandatory_multformat if _ ]) } && +${ ' && '.join([ "ln -s '%s' '%s.%s'" % (_, re.sub('[^\w\-_.]', '_', _.element_identifier), _.ext) for _ in $param_list_optional_multformat if _ ]) } && +${ ' && '.join([ "ln -s '%s' '%s.%s'" % (_, re.sub('[^\w\-_.]', '_', _.element_identifier), _.ext) for _ in $param_list_default_mandatory_noformat if _ ]) } && +${ ' && '.join([ "ln -s '%s' '%s.%s'" % (_, re.sub('[^\w\-_.]', '_', _.element_identifier), _.ext) for _ in $param_list_default_optional_noformat if _ ]) } && +${ ' && '.join([ "ln -s '%s' '%s.%s'" % (_, re.sub('[^\w\-_.]', '_', _.element_identifier), _.ext) for _ in $param_list_default_mandatory_oneformat if _ ]) } && +${ ' && '.join([ "ln -s '%s' '%s.%s'" % (_, re.sub('[^\w\-_.]', '_', _.element_identifier), _.ext) for _ in $param_list_default_optional_oneformat if _ ]) } && +${ ' && '.join([ "ln -s '%s' '%s.%s'" % (_, re.sub('[^\w\-_.]', '_', _.element_identifier), _.ext) for _ in $param_list_default_mandatory_multformat if _ ]) } && +${ ' && '.join([ "ln -s '%s' '%s.%s'" % (_, re.sub('[^\w\-_.]', '_', _.element_identifier), _.ext) for _ in $param_list_default_optional_multformat if _ ]) } && ln -s '$adv_opts.param_mandatory_noformat_adv' '${re.sub("[^\w\-_.]", "_", $adv_opts.param_mandatory_noformat_adv.element_identifier)}.${adv_opts.param_mandatory_noformat_adv.ext}' && #if $adv_opts.param_optional_noformat_adv: ln -s '$adv_opts.param_optional_noformat_adv' '${re.sub("[^\w\-_.]", "_", $adv_opts.param_optional_noformat_adv.element_identifier)}.${adv_opts.param_optional_noformat_adv.ext}' && @@ -72,18 +72,18 @@ ln -s '$adv_opts.param_default_mandatory_multformat_adv' '${re.sub("[^\w\-_.]", #if $adv_opts.param_default_optional_multformat_adv: ln -s '$adv_opts.param_default_optional_multformat_adv' '${re.sub("[^\w\-_.]", "_", $adv_opts.param_default_optional_multformat_adv.element_identifier)}.${adv_opts.param_default_optional_multformat_adv.ext}' && #end if -${ ' && '.join([ "ln -s '%s' '%s.%s'" % (_, re.sub('[^\w\-_.]', '_', _.element_identifier), _.ext) for _ in $adv_opts.param_list_mandatory_noformat_adv if _ != None ]) } && -${ ' && '.join([ "ln -s '%s' '%s.%s'" % (_, re.sub('[^\w\-_.]', '_', _.element_identifier), _.ext) for _ in $adv_opts.param_list_optional_noformat_adv if _ != None ]) } && -${ ' && '.join([ "ln -s '%s' '%s.%s'" % (_, re.sub('[^\w\-_.]', '_', _.element_identifier), _.ext) for _ in $adv_opts.param_list_mandatory_oneformat_adv if _ != None ]) } && -${ ' && '.join([ "ln -s '%s' '%s.%s'" % (_, re.sub('[^\w\-_.]', '_', _.element_identifier), _.ext) for _ in $adv_opts.param_list_optional_oneformat_adv if _ != None ]) } && -${ ' && '.join([ "ln -s '%s' '%s.%s'" % (_, re.sub('[^\w\-_.]', '_', _.element_identifier), _.ext) for _ in $adv_opts.param_list_mandatory_multformat_adv if _ != None ]) } && -${ ' && '.join([ "ln -s '%s' '%s.%s'" % (_, re.sub('[^\w\-_.]', '_', _.element_identifier), _.ext) for _ in $adv_opts.param_list_optional_multformat_adv if _ != None ]) } && -${ ' && '.join([ "ln -s '%s' '%s.%s'" % (_, re.sub('[^\w\-_.]', '_', _.element_identifier), _.ext) for _ in $adv_opts.param_list_default_mandatory_noformat_adv if _ != None ]) } && -${ ' && '.join([ "ln -s '%s' '%s.%s'" % (_, re.sub('[^\w\-_.]', '_', _.element_identifier), _.ext) for _ in $adv_opts.param_list_default_optional_noformat_adv if _ != None ]) } && -${ ' && '.join([ "ln -s '%s' '%s.%s'" % (_, re.sub('[^\w\-_.]', '_', _.element_identifier), _.ext) for _ in $adv_opts.param_list_default_mandatory_oneformat_adv if _ != None ]) } && -${ ' && '.join([ "ln -s '%s' '%s.%s'" % (_, re.sub('[^\w\-_.]', '_', _.element_identifier), _.ext) for _ in $adv_opts.param_list_default_optional_oneformat_adv if _ != None ]) } && -${ ' && '.join([ "ln -s '%s' '%s.%s'" % (_, re.sub('[^\w\-_.]', '_', _.element_identifier), _.ext) for _ in $adv_opts.param_list_default_mandatory_multformat_adv if _ != None ]) } && -${ ' && '.join([ "ln -s '%s' '%s.%s'" % (_, re.sub('[^\w\-_.]', '_', _.element_identifier), _.ext) for _ in $adv_opts.param_list_default_optional_multformat_adv if _ != None ]) } && +${ ' && '.join([ "ln -s '%s' '%s.%s'" % (_, re.sub('[^\w\-_.]', '_', _.element_identifier), _.ext) for _ in $adv_opts.param_list_mandatory_noformat_adv if _ ]) } && +${ ' && '.join([ "ln -s '%s' '%s.%s'" % (_, re.sub('[^\w\-_.]', '_', _.element_identifier), _.ext) for _ in $adv_opts.param_list_optional_noformat_adv if _ ]) } && +${ ' && '.join([ "ln -s '%s' '%s.%s'" % (_, re.sub('[^\w\-_.]', '_', _.element_identifier), _.ext) for _ in $adv_opts.param_list_mandatory_oneformat_adv if _ ]) } && +${ ' && '.join([ "ln -s '%s' '%s.%s'" % (_, re.sub('[^\w\-_.]', '_', _.element_identifier), _.ext) for _ in $adv_opts.param_list_optional_oneformat_adv if _ ]) } && +${ ' && '.join([ "ln -s '%s' '%s.%s'" % (_, re.sub('[^\w\-_.]', '_', _.element_identifier), _.ext) for _ in $adv_opts.param_list_mandatory_multformat_adv if _ ]) } && +${ ' && '.join([ "ln -s '%s' '%s.%s'" % (_, re.sub('[^\w\-_.]', '_', _.element_identifier), _.ext) for _ in $adv_opts.param_list_optional_multformat_adv if _ ]) } && +${ ' && '.join([ "ln -s '%s' '%s.%s'" % (_, re.sub('[^\w\-_.]', '_', _.element_identifier), _.ext) for _ in $adv_opts.param_list_default_mandatory_noformat_adv if _ ]) } && +${ ' && '.join([ "ln -s '%s' '%s.%s'" % (_, re.sub('[^\w\-_.]', '_', _.element_identifier), _.ext) for _ in $adv_opts.param_list_default_optional_noformat_adv if _ ]) } && +${ ' && '.join([ "ln -s '%s' '%s.%s'" % (_, re.sub('[^\w\-_.]', '_', _.element_identifier), _.ext) for _ in $adv_opts.param_list_default_mandatory_oneformat_adv if _ ]) } && +${ ' && '.join([ "ln -s '%s' '%s.%s'" % (_, re.sub('[^\w\-_.]', '_', _.element_identifier), _.ext) for _ in $adv_opts.param_list_default_optional_oneformat_adv if _ ]) } && +${ ' && '.join([ "ln -s '%s' '%s.%s'" % (_, re.sub('[^\w\-_.]', '_', _.element_identifier), _.ext) for _ in $adv_opts.param_list_default_mandatory_multformat_adv if _ ]) } && +${ ' && '.join([ "ln -s '%s' '%s.%s'" % (_, re.sub('[^\w\-_.]', '_', _.element_identifier), _.ext) for _ in $adv_opts.param_list_default_optional_multformat_adv if _ ]) } && echo inFileTest -mandatory_noformat '${re.sub('[^\w\-_.]', '_', $param_mandatory_noformat.element_identifier)}.${param_mandatory_noformat.ext}' diff --git a/tests/test-data/ofile.xml b/tests/test-data/ofile.xml index 82e743c6..53f7f48d 100644 --- a/tests/test-data/ofile.xml +++ b/tests/test-data/ofile.xml @@ -130,19 +130,19 @@ echo outFileTest - - - - - - - - - + + + + + + + + + - - - + + + From 1b634550f65aa484233b286eaa2b039e807c81d4 Mon Sep 17 00:00:00 2001 From: M Bernt Date: Fri, 26 Jul 2019 15:41:30 +0200 Subject: [PATCH 030/121] allow for list outputs - matching input is now determined based on the ctd formats (not the galaxy formats) since those are more specific - make output a collection TODOs and notes: - structured like probably does not work, except if we force the input to be a collection as well - nice thing would be that the CLI could be easily created without symlinks, see https://planemo.readthedocs.io/en/latest/writing_advanced.html#multiple-output-files section: "2. Computable Element Count" - CLI generation is missing --- galaxy/converter.py | 47 +++++++++++++++++++++++++---------- tests/test-data/filetypes.txt | 1 + tests/test-data/ofile.ctd | 8 ------ tests/test-data/ofile.xml | 35 +------------------------- tests/test_galaxy_cli.py | 3 +++ 5 files changed, 39 insertions(+), 55 deletions(-) diff --git a/galaxy/converter.py b/galaxy/converter.py index b3bc8d63..675e3ff3 100755 --- a/galaxy/converter.py +++ b/galaxy/converter.py @@ -523,14 +523,25 @@ def get_galaxy_parameter_name(param, inparam = False): return "param_%s" % p -def get_input_with_same_restrictions(out_param, model, supported_file_formats): +def get_input_with_same_restrictions(out_param, model): + """ + get the input parameter that has the same restrictions (ctd file_formats) + - input and output must both be lists of both be simple parameters + """ + matching = [] for param in utils.extract_and_flatten_parameters(model): - if param.type is _InFile: - if param.restrictions is not None: - in_param_formats = get_supported_file_types(param.restrictions.formats, supported_file_formats) - out_param_formats = get_supported_file_types(out_param.restrictions.formats, supported_file_formats) - if in_param_formats == out_param_formats: - return param + if not param.type is _InFile or param.restrictions is None: + continue + if param.is_list != out_param.is_list: + continue + in_param_formats = param.restrictions.formats + out_param_formats = out_param.restrictions.formats + if in_param_formats == out_param_formats: + matching.append(param) + if len(matching) == 1: + return matching[0] + else: + return None def create_inputs(tool, model, **kwargs): @@ -916,7 +927,12 @@ def create_outputs(parent, model, **kwargs): def create_output_node(parent, param, model, supported_file_formats): - data_node = add_child_node(parent, "data") + + if not param.is_list: + data_node = add_child_node(parent, "data") + else: + data_node = add_child_node(parent, "collection") + data_node.attrib["name"] = get_galaxy_parameter_name(param) if data_node.attrib["name"].startswith('param_out_'): data_node.attrib["label"] = "${tool.name} on ${on_string}: %s" % data_node.attrib["name"][10:] @@ -950,11 +966,16 @@ def create_output_node(parent, param, model, supported_file_formats): # there is not much we can do, other than catching the exception pass # if there are more than one output file formats try to take the format from the input parameter - if formats: - corresponding_input = get_input_with_same_restrictions(param, model, supported_file_formats) - if corresponding_input is not None: - data_format = "input" - data_node.attrib["metadata_source"] = get_galaxy_parameter_name(corresponding_input) + corresponding_input = get_input_with_same_restrictions(param, model) + if param.is_list and corresponding_input is not None: + data_node.attrib["structured_like"] = get_galaxy_parameter_name(corresponding_input) + # data_node.attrib["inherit_format"] = "true" + data_node.attrib["format_source"] = get_galaxy_parameter_name(corresponding_input) + data_node.attrib["metadata_source"] = get_galaxy_parameter_name(corresponding_input) + elif not param.is_list and formats and corresponding_input is not None: + data_format = "input" + data_node.attrib["format_source"] = get_galaxy_parameter_name(corresponding_input) + data_node.attrib["metadata_source"] = get_galaxy_parameter_name(corresponding_input) else: raise InvalidModelException("Unrecognized restriction type [%(type)s] " "for output [%(name)s]" % {"type": type(param.restrictions), diff --git a/tests/test-data/filetypes.txt b/tests/test-data/filetypes.txt index ae3d61a2..ea290d02 100644 --- a/tests/test-data/filetypes.txt +++ b/tests/test-data/filetypes.txt @@ -1,4 +1,5 @@ # CTD type # Galaxy type # Long Galaxy data type # Mimetype +ewt txt galaxy.datatypes.data:Text ext txt galaxy.datatypes.data:Text eyt txt galaxy.datatypes.data:Text ezt txt galaxy.datatypes.data:Text diff --git a/tests/test-data/ofile.ctd b/tests/test-data/ofile.ctd index 78a6638b..9fb8648d 100644 --- a/tests/test-data/ofile.ctd +++ b/tests/test-data/ofile.ctd @@ -19,14 +19,6 @@ - - - - - - - - diff --git a/tests/test-data/ofile.xml b/tests/test-data/ofile.xml index 53f7f48d..9469202b 100644 --- a/tests/test-data/ofile.xml +++ b/tests/test-data/ofile.xml @@ -37,18 +37,6 @@ echo outFileTest #if str($param_default_optional_multformat): -default_optional_multformat '$param_out_default_optional_multformat' #end if --list_mandatory_noformat '$param_out_list_mandatory_noformat' -#if str($param_list_optional_noformat): - -list_optional_noformat '$param_out_list_optional_noformat' -#end if --list_mandatory_oneformat '$param_out_list_mandatory_oneformat' -#if str($param_list_optional_oneformat): - -list_optional_oneformat '$param_out_list_optional_oneformat' -#end if --list_mandatory_multformat '$param_out_list_mandatory_multformat' -#if str($param_list_optional_multformat): - -list_optional_multformat '$param_out_list_optional_multformat' -#end if -mandatory_noformat_adv '$param_out_mandatory_noformat_adv' #if str($adv_opts.param_optional_noformat_adv): -optional_noformat_adv '$param_out_optional_noformat_adv' @@ -69,9 +57,6 @@ echo outFileTest - - - @@ -103,18 +88,6 @@ echo outFileTest param_default_optional_multformat - - - param_list_optional_noformat - - - - param_list_optional_oneformat - - - - param_list_optional_multformat - advanced_options['adv_opts_selector'] == 'advanced' and advanced_options['param_optional_noformat_adv'] @@ -129,16 +102,13 @@ echo outFileTest - + - - - @@ -151,9 +121,6 @@ echo outFileTest - - - diff --git a/tests/test_galaxy_cli.py b/tests/test_galaxy_cli.py index 802af148..c120f2f7 100644 --- a/tests/test_galaxy_cli.py +++ b/tests/test_galaxy_cli.py @@ -84,6 +84,9 @@ def test_galaxy_cli_ifile_ctd(self): def test_galaxy_cli_ofile_ctd(self): self._compare_cli_output('ofile') + def test_galaxy_cli_ofile_mult_ctd(self): + self._compare_cli_output('ofile-mult') + if __name__ == '__main__': unittest.main() From d40c31bda078504e933626571d75cf744facf99f Mon Sep 17 00:00:00 2001 From: M Bernt Date: Sun, 11 Aug 2019 22:28:26 +0200 Subject: [PATCH 031/121] multiple outputs: - CLI also uses symlinks using identifiers and extension of corresponding input .. symlinks go to a separate dir - output collections now work with `discover_datasets` TODOs: - output collections still have format --- galaxy/converter.py | 34 ++++++++----- tests/test-data/ofile-mult.ctd | 38 +++++++++++++++ tests/test-data/ofile-mult.xml | 88 ++++++++++++++++++++++++++++++++++ 3 files changed, 149 insertions(+), 11 deletions(-) create mode 100644 tests/test-data/ofile-mult.ctd create mode 100644 tests/test-data/ofile-mult.xml diff --git a/galaxy/converter.py b/galaxy/converter.py index 675e3ff3..acf2d3c4 100755 --- a/galaxy/converter.py +++ b/galaxy/converter.py @@ -152,10 +152,10 @@ def parse_macros_files(macros_file_names): else: logger.info("Macro %s found" % name, 1) macros_to_expand.append(name) - except ParseError, e: + except ParseError as e: raise ApplicationException("The macros file " + macros_file_name + " could not be parsed. Cause: " + str(e)) - except IOError, e: + except IOError as e: raise ApplicationException("The macros file " + macros_file_name + " could not be opened. Cause: " + str(e)) @@ -414,11 +414,18 @@ def create_command(tool, model, **kwargs): if hardcoded_value: command += "%s %s\n" % (command_line_prefix, hardcoded_value) else: - # parameter is neither blacklisted nor hardcoded... + # in the else branch the parameter is neither blacklisted nor hardcoded... + + # determine the name of the cheetah variable that is used to get the argument + # for the parameter + corresponding_input = None + if param.type is _OutFile and param.is_list: + corresponding_input = get_input_with_same_restrictions(param, model) + + + actual_parameter = get_galaxy_parameter_name(param) if param.advanced and not param.type is _OutFile: - actual_parameter = "adv_opts.%s" % get_galaxy_parameter_name(param) - else: - actual_parameter = "%s" % get_galaxy_parameter_name(param) + actual_parameter = "adv_opts." + actual_parameter # all but bool params need the command line argument (bools have it already in the true/false value) if not is_boolean_parameter(param): @@ -432,7 +439,13 @@ def create_command(tool, model, **kwargs): else: preprocessing += "ln -s '$"+ actual_parameter +"' '${re.sub(\"[^\w\-_.]\", \"_\", $"+actual_parameter+".element_identifier)}.${"+actual_parameter+".ext}' &&\n" command += "'${re.sub('[^\w\-_.]', '_', $"+actual_parameter+".element_identifier)}.${"+actual_parameter+".ext}'\n" - + elif param.type is _OutFile and not corresponding_input is None: + preprocessing += "mkdir " + actual_parameter + " && \n" + actual_input_parameter = get_galaxy_parameter_name( corresponding_input ) + if param.is_list: + command += "${' '.join([\"'"+ actual_parameter +"/%s.%s'\"%(re.sub('[^\w\-_.]', '_', _.element_identifier),_.ext) for _ in $" + actual_input_parameter + " if _ ])}\n" + else: + command += actual_parameter+"/'${re.sub('[^\w\-_.]', '_', $"+actual_input_parameter+".element_identifier)}.${"+actual_input_parameter+".ext}'\n" # logic for ITEMLISTs elif param.is_list and is_selection_parameter(param): command += "${' '.join([\"'%s'\"%str(_) for _ in $" + actual_parameter + "])}\n" @@ -968,10 +981,9 @@ def create_output_node(parent, param, model, supported_file_formats): # if there are more than one output file formats try to take the format from the input parameter corresponding_input = get_input_with_same_restrictions(param, model) if param.is_list and corresponding_input is not None: - data_node.attrib["structured_like"] = get_galaxy_parameter_name(corresponding_input) - # data_node.attrib["inherit_format"] = "true" - data_node.attrib["format_source"] = get_galaxy_parameter_name(corresponding_input) - data_node.attrib["metadata_source"] = get_galaxy_parameter_name(corresponding_input) + # TODO + add_child_node(data_node, "discover_datasets", + OrderedDict([("pattern", "__name_and_ext__"), ("directory", get_galaxy_parameter_name(param))])) elif not param.is_list and formats and corresponding_input is not None: data_format = "input" data_node.attrib["format_source"] = get_galaxy_parameter_name(corresponding_input) diff --git a/tests/test-data/ofile-mult.ctd b/tests/test-data/ofile-mult.ctd new file mode 100644 index 00000000..07c7e7d3 --- /dev/null +++ b/tests/test-data/ofile-mult.ctd @@ -0,0 +1,38 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/test-data/ofile-mult.xml b/tests/test-data/ofile-mult.xml new file mode 100644 index 00000000..4dd31b9a --- /dev/null +++ b/tests/test-data/ofile-mult.xml @@ -0,0 +1,88 @@ + + + + + Input File parameter tests + + echo outFileTest + macros.xml + + + + + + + + + + + + + + + + + + + + param_optional_mandatoryinput + + + + + + + param_optional_optionalinput + + + + + + + + + + + + + + param_optional_mandatoryinput + + + + + + + param_optional_optionalinput + + + + + From 3d6ccf3cbe4bac380c971f897020b9c06d6f80c6 Mon Sep 17 00:00:00 2001 From: M Bernt Date: Mon, 11 Nov 2019 11:49:59 +0100 Subject: [PATCH 032/121] make default data type txt since data seems unavailable for upload --- galaxy/converter.py | 36 +++++++++++++++++++++++++++--------- 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/galaxy/converter.py b/galaxy/converter.py index acf2d3c4..c32cd9f4 100755 --- a/galaxy/converter.py +++ b/galaxy/converter.py @@ -15,8 +15,8 @@ from CTDopts.CTDopts import _InFile, _OutFile, ParameterGroup, _Choices, _NumericRange, _FileFormat, ModelError, _Null -TYPE_TO_GALAXY_TYPE = {int: 'integer', float: 'float', str: 'text', bool: 'boolean', _InFile: 'data', - _OutFile: 'data', _Choices: 'select'} +TYPE_TO_GALAXY_TYPE = {int: 'integer', float: 'float', str: 'text', bool: 'boolean', _InFile: 'txt', + _OutFile: 'txt', _Choices: 'select'} STDIO_MACRO_NAME = "stdio" REQUIREMENTS_MACRO_NAME = "requirements" ADVANCED_OPTIONS_MACRO_NAME = "advanced_options" @@ -286,6 +286,8 @@ def _convert_internal(parsed_ctds, **kwargs): create_tests(tool, copy.deepcopy(inputs), copy.deepcopy(outputs)) create_help(tool, model) + move_citations(tool) + # wrap our tool element into a tree to be able to serialize it tree = ElementTree(tool) logger.info("Writing to %s" % utils.get_filename(output_file), 1) @@ -515,6 +517,14 @@ def expand_macros(tool, model, **kwargs): expand_node = add_child_node(tool, "expand") expand_node.attrib["macro"] = expand_macro +def move_citations(tool): + """ + move expansions of the citations/references macro to the end of the tool file + """ + for expand in tool.iter('expand'): + if expand.attrib["macro"] in ["references", "citations"]: + tool.append(expand) + def get_galaxy_parameter_name(param, inparam = False): """ @@ -640,7 +650,7 @@ def create_param_attribute_list(param_node, param, supported_file_formats): if param.type is _InFile: # assume it's just text unless restrictions are provided - param_format = "data" + param_format = TYPE_TO_GALAXY_TYPE[_InFile] if param.restrictions is not None: # join all formats of the file, take mapping from supported_file if available for an entry if type(param.restrictions) is _FileFormat: @@ -945,17 +955,18 @@ def create_output_node(parent, param, model, supported_file_formats): data_node = add_child_node(parent, "data") else: data_node = add_child_node(parent, "collection") + data_node.attrib["type"] = "list" data_node.attrib["name"] = get_galaxy_parameter_name(param) if data_node.attrib["name"].startswith('param_out_'): data_node.attrib["label"] = "${tool.name} on ${on_string}: %s" % data_node.attrib["name"][10:] # determine format attribute - # - default is data + # - default is txt # - check if all listed possible formats are supported in Galaxy # - if there is a single one, then take this # - otherwise try to determine an input with the same restrictions and use this as format source - data_format = "data" + data_format = "txt" if param.restrictions is not None: if type(param.restrictions) is _FileFormat: # set the first data output node to the first file format @@ -992,7 +1003,9 @@ def create_output_node(parent, param, model, supported_file_formats): raise InvalidModelException("Unrecognized restriction type [%(type)s] " "for output [%(name)s]" % {"type": type(param.restrictions), "name": param.name}) - data_node.attrib["format"] = data_format + # data output has fomat .. collection output has no format + if not param.is_list: + data_node.attrib["format"] = data_format # add filter for optional parameters if not param.required: @@ -1071,11 +1084,13 @@ def create_tests(parent, inputs, outputs): elif node.attrib.get("multiple", None) == "true": node.attrib["value"] = ",".join([ _.attrib["value"] for _ in node ]) elif node.attrib["type"] == "data": + node.attrib["ftype"] = node.attrib["format"] if node.attrib.get("multiple", "false") == "true": node.attrib["value"] = "test.ext,test2.ext" else: node.attrib["value"] = "test.ext" - for a in set(node.attrib) - set(["name","value","format"]): + for node in inputs.iter(): + for a in set(node.attrib) - set(["name","value","ftype"]): del node.attrib[a] strip_elements(inputs, "delete_node", "option") for node in inputs: @@ -1086,14 +1101,17 @@ def create_tests(parent, inputs, outputs): node.tag = "output" node.attrib["ftype"] = node.attrib["format"] outputs_cnt += 1 + node.attrib["value"] = "outfile.txt" + if node.tag == "collection": + del node.attrib["value"] + node.tag = "output_collection" if len(node) > 0 and node[0].tag == "filter": node.tag = "delete_node" - node.attrib["value"] = "outfile.txt" if node.attrib.get("name", None) == "param_stdout": node.attrib["lines_diff"] = "2" for a in set(node.attrib) - set(["name","value","ftype", "lines_diff"]): del node.attrib[a] - strip_elements(outputs, "delete_node") + strip_elements(outputs, "delete_node", "discover_datasets", "filter") for node in outputs: test_node.append(node) From 134d66ac88e577f1fe9758d5f1e22aed08a28b5f Mon Sep 17 00:00:00 2001 From: M Bernt Date: Wed, 13 Nov 2019 10:06:05 +0100 Subject: [PATCH 033/121] cont. --- galaxy/converter.py | 45 ++++--- tests/test-data/bool.xml | 32 ++--- tests/test-data/empty.xml | 2 +- tests/test-data/filetypes.txt | 4 +- tests/test-data/float.xml | 56 ++++---- tests/test-data/ifile.xml | 240 ++++++++++++++++----------------- tests/test-data/integer.xml | 56 ++++---- tests/test-data/macros.xml | 4 +- tests/test-data/ofile-mult.ctd | 8 +- tests/test-data/ofile-mult.xml | 52 +++---- tests/test-data/ofile.ctd | 6 +- tests/test-data/ofile.xml | 96 ++++++------- tests/test-data/repeat.xml | 68 +++++----- tests/test-data/select.xml | 56 ++++---- tests/test-data/string.xml | 20 +-- 15 files changed, 371 insertions(+), 374 deletions(-) diff --git a/galaxy/converter.py b/galaxy/converter.py index c32cd9f4..2179b953 100755 --- a/galaxy/converter.py +++ b/galaxy/converter.py @@ -19,9 +19,9 @@ _OutFile: 'txt', _Choices: 'select'} STDIO_MACRO_NAME = "stdio" REQUIREMENTS_MACRO_NAME = "requirements" -ADVANCED_OPTIONS_MACRO_NAME = "advanced_options" +ADVANCED_OPTIONS_NAME = "adv_opts_" -REQUIRED_MACROS = [REQUIREMENTS_MACRO_NAME, STDIO_MACRO_NAME, ADVANCED_OPTIONS_MACRO_NAME] +REQUIRED_MACROS = [REQUIREMENTS_MACRO_NAME, STDIO_MACRO_NAME, ADVANCED_OPTIONS_NAME+"macro"] class ExitCode: @@ -172,7 +172,7 @@ def parse_macros_files(macros_file_names): % ", ".join(missing_needed_macros)) # we do not need to "expand" the advanced_options macro - macros_to_expand.remove(ADVANCED_OPTIONS_MACRO_NAME) + macros_to_expand.remove(ADVANCED_OPTIONS_NAME+"macro") return macros_to_expand @@ -391,7 +391,7 @@ def create_command(tool, model, **kwargs): final_preprocessing = "\n" final_command = utils.extract_tool_executable_path(model, kwargs["default_executable_path"]) + '\n' final_command += kwargs["add_to_command_line"] + '\n' - advanced_command_start = "#if $adv_opts.adv_opts_selector=='advanced':\n" + advanced_command_start = "#if ${aon}cond.{aon}selector=='advanced':\n".format(aon=ADVANCED_OPTIONS_NAME) advanced_command_end = "#end if" advanced_command = "" @@ -427,7 +427,7 @@ def create_command(tool, model, **kwargs): actual_parameter = get_galaxy_parameter_name(param) if param.advanced and not param.type is _OutFile: - actual_parameter = "adv_opts." + actual_parameter + actual_parameter = ADVANCED_OPTIONS_NAME+"cond." + actual_parameter # all but bool params need the command line argument (bools have it already in the true/false value) if not is_boolean_parameter(param): @@ -461,7 +461,7 @@ def create_command(tool, model, **kwargs): if not param.required and not is_boolean_parameter(param) and not(param.type is _InFile and param.is_list): if param.type is _OutFile: if param.advanced: - actual_parameter = "adv_opts.%s" % get_galaxy_parameter_name(param, True) + actual_parameter = ADVANCED_OPTIONS_NAME+"cond.%s" % get_galaxy_parameter_name(param, True) else: actual_parameter = "%s" % get_galaxy_parameter_name(param, True) if is_selection_parameter(param) or param.type == _InFile: @@ -471,7 +471,7 @@ def create_command(tool, model, **kwargs): if param.type is _InFile: preprocessing = "#if $" + actual_parameter + ":\n" + utils.indent(preprocessing) + "\n#end if\n" - if param.advanced and param.type is not _OutFile: + if param.advanced: advanced_command += "%s\n" % utils.indent(command) else: final_command += command @@ -603,7 +603,7 @@ def create_inputs(tool, model, **kwargs): if _param.advanced: if advanced_node is None: - advanced_node = Element("expand", OrderedDict([("macro", ADVANCED_OPTIONS_MACRO_NAME)])) + advanced_node = Element("expand", OrderedDict([("macro", ADVANCED_OPTIONS_NAME+"macro")])) parent_node = advanced_node else: parent_node = inputs_node @@ -1012,7 +1012,7 @@ def create_output_node(parent, param, model, supported_file_formats): filter_node = add_child_node(data_node, "filter") filter_node.text = get_galaxy_parameter_name(param, True) if param.advanced: - filter_node.text = "advanced_options['adv_opts_selector'] == 'advanced' and advanced_options['" + filter_node.text + "']" + filter_node.text = "{aon}cond['{aon}selector'] == 'advanced' and {aon}cond['".format(aon = ADVANCED_OPTIONS_NAME) + filter_node.text + "']" return data_node @@ -1052,8 +1052,8 @@ def create_tests(parent, inputs, outputs): for node in inputs.iter(): if node.tag == "expand": node.tag = "conditional" - node.attrib["name"] = "adv_opts" - add_child_node(node, "param", OrderedDict([("name", "adv_opts_selector"), ("value", "advanced")])) + node.attrib["name"] = ADVANCED_OPTIONS_NAME+"cond" + add_child_node(node, "param", OrderedDict([("name", ADVANCED_OPTIONS_NAME+"selector"), ("value", "advanced")])) if not "type" in node.attrib: continue @@ -1069,9 +1069,9 @@ def create_tests(parent, inputs, outputs): if node.attrib["type"] == "boolean": if node.attrib["checked"] == "true": - node.attrib["value"] = node.attrib["truevalue"] + node.attrib["value"] = "true" # node.attrib["truevalue"] else: - node.attrib["value"] = node.attrib["falsevalue"] + node.attrib["value"] = "false" # node.attrib["falsevalue"] elif node.attrib["type"] == "text" and node.attrib["value"] == "": node.attrib["value"] = "1,2" # use a comma separated list here to cover the repeat (int/float) case elif node.attrib["type"] == "integer" and node.attrib["value"] == "": @@ -1084,7 +1084,7 @@ def create_tests(parent, inputs, outputs): elif node.attrib.get("multiple", None) == "true": node.attrib["value"] = ",".join([ _.attrib["value"] for _ in node ]) elif node.attrib["type"] == "data": - node.attrib["ftype"] = node.attrib["format"] + node.attrib["ftype"] = node.attrib["format"].split(',')[0] if node.attrib.get("multiple", "false") == "true": node.attrib["value"] = "test.ext,test2.ext" else: @@ -1097,16 +1097,23 @@ def create_tests(parent, inputs, outputs): test_node.append(node) outputs_cnt = 0 for node in outputs.iter(): + if node.tag == "data" or node.tag == "collection": + # assuming that all filters avaluate to false + has_filter = False + for c in node: + if c.tag == "filter": + has_filter = True + break + if not has_filter: + outputs_cnt += 1 + else: + node.tag = "delete_node" if node.tag == "data": node.tag = "output" node.attrib["ftype"] = node.attrib["format"] - outputs_cnt += 1 - node.attrib["value"] = "outfile.txt" + node.attrib["value"] = "outfile.txt" if node.tag == "collection": - del node.attrib["value"] node.tag = "output_collection" - if len(node) > 0 and node[0].tag == "filter": - node.tag = "delete_node" if node.attrib.get("name", None) == "param_stdout": node.attrib["lines_diff"] = "2" for a in set(node.attrib) - set(["name","value","ftype", "lines_diff"]): diff --git a/tests/test-data/bool.xml b/tests/test-data/bool.xml index c45fbc63..c74fbe92 100644 --- a/tests/test-data/bool.xml +++ b/tests/test-data/bool.xml @@ -9,7 +9,6 @@ - $param_stdout ]]> @@ -30,7 +29,7 @@ $param_flag_false - + @@ -42,15 +41,15 @@ $param_flag_false - - - - - - - - - + + + + + + + + + @@ -60,4 +59,5 @@ $param_flag_false For more information, visit http://docs.are/unavailable.html]]> + diff --git a/tests/test-data/empty.xml b/tests/test-data/empty.xml index 32b64032..07302234 100644 --- a/tests/test-data/empty.xml +++ b/tests/test-data/empty.xml @@ -9,7 +9,6 @@ - + diff --git a/tests/test-data/filetypes.txt b/tests/test-data/filetypes.txt index ea290d02..4137ff50 100644 --- a/tests/test-data/filetypes.txt +++ b/tests/test-data/filetypes.txt @@ -1,5 +1,5 @@ # CTD type # Galaxy type # Long Galaxy data type # Mimetype ewt txt galaxy.datatypes.data:Text ext txt galaxy.datatypes.data:Text -eyt txt galaxy.datatypes.data:Text -ezt txt galaxy.datatypes.data:Text +eyt tabular galaxy.datatypes.tabular:Tabular +ezt tabular galaxy.datatypes.tabular:Tabular diff --git a/tests/test-data/float.xml b/tests/test-data/float.xml index 65a94db4..0cc8a417 100644 --- a/tests/test-data/float.xml +++ b/tests/test-data/float.xml @@ -9,7 +9,6 @@ - $param_stdout @@ -98,7 +97,7 @@ echo FloatTest - + @@ -130,7 +129,7 @@ echo FloatTest - + @@ -148,4 +147,5 @@ echo FloatTest For more information, visit http://docs.are/unavailable.html]]> + diff --git a/tests/test-data/ifile.xml b/tests/test-data/ifile.xml index 5341bbca..822fe653 100644 --- a/tests/test-data/ifile.xml +++ b/tests/test-data/ifile.xml @@ -9,7 +9,6 @@ - $param_stdout ]]> - - + + - - - - + + + + - - - - + + + + - - - - + + + + - - - - - + + + + + - - - - + + + + - - - - + + + + - - - - + + + + - - + + @@ -219,31 +218,31 @@ echo inFileTest - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + @@ -253,4 +252,5 @@ echo inFileTest For more information, visit http://docs.are/unavailable.html]]> + diff --git a/tests/test-data/integer.xml b/tests/test-data/integer.xml index 7eb4a436..afe21a66 100644 --- a/tests/test-data/integer.xml +++ b/tests/test-data/integer.xml @@ -9,7 +9,6 @@ - $param_stdout @@ -98,7 +97,7 @@ echo IntTest - + @@ -130,7 +129,7 @@ echo IntTest - + @@ -148,4 +147,5 @@ echo IntTest For more information, visit http://docs.are/unavailable.html]]> + diff --git a/tests/test-data/macros.xml b/tests/test-data/macros.xml index acf44cb0..4d7a246a 100644 --- a/tests/test-data/macros.xml +++ b/tests/test-data/macros.xml @@ -16,8 +16,8 @@ doi:10.1186/1471-2105-9-163 - - + + diff --git a/tests/test-data/ofile-mult.ctd b/tests/test-data/ofile-mult.ctd index 07c7e7d3..70c8d4a1 100644 --- a/tests/test-data/ofile-mult.ctd +++ b/tests/test-data/ofile-mult.ctd @@ -1,9 +1,9 @@ - - - + + + - + diff --git a/tests/test-data/ofile-mult.xml b/tests/test-data/ofile-mult.xml index 4dd31b9a..bc1161b2 100644 --- a/tests/test-data/ofile-mult.xml +++ b/tests/test-data/ofile-mult.xml @@ -1,15 +1,14 @@ - - - Input File parameter tests + + + multiple Output File parameter tests - echo outFileTest + echo MultoutFileTest macros.xml - - - + + - + - + param_optional_mandatoryinput - + - + param_optional_optionalinput - - - - - - - - - - - param_optional_mandatoryinput - - - - - - - param_optional_optionalinput - + + + + + + + - + diff --git a/tests/test-data/ofile.ctd b/tests/test-data/ofile.ctd index 9fb8648d..d25f1bf1 100644 --- a/tests/test-data/ofile.ctd +++ b/tests/test-data/ofile.ctd @@ -1,9 +1,9 @@ - - + + - + diff --git a/tests/test-data/ofile.xml b/tests/test-data/ofile.xml index 9469202b..068c74a3 100644 --- a/tests/test-data/ofile.xml +++ b/tests/test-data/ofile.xml @@ -2,14 +2,13 @@ - Input File parameter tests + Outut File parameter tests echo outFileTest macros.xml - @@ -57,19 +58,19 @@ echo outFileTest - + - - - + + + param_wodefault_optional_noformat - + param_default_optional_noformat @@ -80,54 +81,55 @@ echo outFileTest param_default_optional_oneformat - - - + + + param_wodefault_optional_multformat - + param_default_optional_multformat - - - advanced_options['adv_opts_selector'] == 'advanced' and advanced_options['param_optional_noformat_adv'] + + + adv_opts_cond['adv_opts_selector'] == 'advanced' and adv_opts_cond['param_optional_noformat_adv'] - advanced_options['adv_opts_selector'] == 'advanced' and advanced_options['param_optional_oneformat_adv'] + adv_opts_cond['adv_opts_selector'] == 'advanced' and adv_opts_cond['param_optional_oneformat_adv'] - - - advanced_options['adv_opts_selector'] == 'advanced' and advanced_options['param_optional_multformat_adv'] + + + adv_opts_cond['adv_opts_selector'] == 'advanced' and adv_opts_cond['param_optional_multformat_adv'] - - - - - - - - - - - + + + + + + + + + + + - - + + - - - + + + - + - + diff --git a/tests/test-data/repeat.xml b/tests/test-data/repeat.xml index 2276779c..d5862da7 100644 --- a/tests/test-data/repeat.xml +++ b/tests/test-data/repeat.xml @@ -9,7 +9,6 @@ - $param_stdout ]]> @@ -322,7 +321,7 @@ echo RepeatTest - + @@ -564,7 +563,7 @@ echo RepeatTest - + @@ -584,4 +583,5 @@ echo RepeatTest For more information, visit http://docs.are/unavailable.html]]> + diff --git a/tests/test-data/select.xml b/tests/test-data/select.xml index 0a9f55ae..d866ed77 100644 --- a/tests/test-data/select.xml +++ b/tests/test-data/select.xml @@ -9,7 +9,6 @@ - $param_stdout ]]> @@ -186,7 +185,7 @@ echo SelectTest - + @@ -302,7 +301,7 @@ echo SelectTest - + @@ -316,4 +315,5 @@ echo SelectTest For more information, visit http://docs.are/unavailable.html]]> + diff --git a/tests/test-data/string.xml b/tests/test-data/string.xml index 922f5bdb..eb50dfd1 100644 --- a/tests/test-data/string.xml +++ b/tests/test-data/string.xml @@ -9,7 +9,6 @@ - $param_stdout @@ -66,7 +65,7 @@ echo StringTest - + @@ -108,7 +107,7 @@ echo StringTest - + @@ -120,4 +119,5 @@ echo StringTest For more information, visit http://docs.are/unavailable.html]]> + From 8b68ea2918746c0b4aceec3aa1e956e10a968947 Mon Sep 17 00:00:00 2001 From: M Bernt Date: Wed, 13 Nov 2019 10:25:27 +0100 Subject: [PATCH 034/121] fixed category in test ctd files --- tests/test-data/bool.ctd | 2 +- tests/test-data/empty.ctd | 2 +- tests/test-data/float.ctd | 2 +- tests/test-data/ifile.ctd | 2 +- tests/test-data/integer.ctd | 2 +- tests/test-data/ofile-mult.ctd | 2 +- tests/test-data/ofile.ctd | 2 +- tests/test-data/repeat.ctd | 2 +- tests/test-data/select.ctd | 2 +- tests/test-data/string.ctd | 2 +- 10 files changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/test-data/bool.ctd b/tests/test-data/bool.ctd index 3fab0317..468a2f44 100644 --- a/tests/test-data/bool.ctd +++ b/tests/test-data/bool.ctd @@ -1,5 +1,5 @@ - + diff --git a/tests/test-data/empty.ctd b/tests/test-data/empty.ctd index 1ffc19de..3dda4edd 100644 --- a/tests/test-data/empty.ctd +++ b/tests/test-data/empty.ctd @@ -1,5 +1,5 @@ - + diff --git a/tests/test-data/float.ctd b/tests/test-data/float.ctd index f3f06a48..1317dc19 100644 --- a/tests/test-data/float.ctd +++ b/tests/test-data/float.ctd @@ -1,5 +1,5 @@ - + diff --git a/tests/test-data/ifile.ctd b/tests/test-data/ifile.ctd index fcadd466..629bd694 100644 --- a/tests/test-data/ifile.ctd +++ b/tests/test-data/ifile.ctd @@ -1,5 +1,5 @@ - + diff --git a/tests/test-data/integer.ctd b/tests/test-data/integer.ctd index a358e3c6..486686e6 100644 --- a/tests/test-data/integer.ctd +++ b/tests/test-data/integer.ctd @@ -1,5 +1,5 @@ - + diff --git a/tests/test-data/ofile-mult.ctd b/tests/test-data/ofile-mult.ctd index 70c8d4a1..c585ba77 100644 --- a/tests/test-data/ofile-mult.ctd +++ b/tests/test-data/ofile-mult.ctd @@ -1,5 +1,5 @@ - + diff --git a/tests/test-data/ofile.ctd b/tests/test-data/ofile.ctd index d25f1bf1..bf164f94 100644 --- a/tests/test-data/ofile.ctd +++ b/tests/test-data/ofile.ctd @@ -1,5 +1,5 @@ - + diff --git a/tests/test-data/repeat.ctd b/tests/test-data/repeat.ctd index fa7b3021..71a8b4a4 100644 --- a/tests/test-data/repeat.ctd +++ b/tests/test-data/repeat.ctd @@ -1,5 +1,5 @@ - + diff --git a/tests/test-data/select.ctd b/tests/test-data/select.ctd index 2abc52de..f928a19b 100644 --- a/tests/test-data/select.ctd +++ b/tests/test-data/select.ctd @@ -1,5 +1,5 @@ - + diff --git a/tests/test-data/string.ctd b/tests/test-data/string.ctd index 8cd8408a..8829408f 100644 --- a/tests/test-data/string.ctd +++ b/tests/test-data/string.ctd @@ -1,5 +1,5 @@ - + From d772ea258c9fff1b49b3bffdd2dd8fbc240f0d33 Mon Sep 17 00:00:00 2001 From: M Bernt Date: Wed, 13 Nov 2019 13:09:25 +0100 Subject: [PATCH 035/121] add argument attribute --- galaxy/converter.py | 12 +++-- tests/test-data/bool.xml | 18 +++---- tests/test-data/empty.xml | 2 +- tests/test-data/float.xml | 66 +++++++++++------------ tests/test-data/ifile.xml | 98 +++++++++++++++++----------------- tests/test-data/integer.xml | 66 +++++++++++------------ tests/test-data/ofile-mult.xml | 14 ++--- tests/test-data/ofile.xml | 20 +++---- tests/test-data/repeat.xml | 82 ++++++++++++++-------------- tests/test-data/select.xml | 66 +++++++++++------------ tests/test-data/string.xml | 18 +++---- 11 files changed, 233 insertions(+), 229 deletions(-) diff --git a/galaxy/converter.py b/galaxy/converter.py index 2179b953..987422c3 100755 --- a/galaxy/converter.py +++ b/galaxy/converter.py @@ -285,7 +285,7 @@ def _convert_internal(parsed_ctds, **kwargs): if kwargs["test_test"]: create_tests(tool, copy.deepcopy(inputs), copy.deepcopy(outputs)) create_help(tool, model) - + # citations are required to be at the end move_citations(tool) # wrap our tool element into a tree to be able to serialize it @@ -586,8 +586,10 @@ def create_inputs(tool, model, **kwargs): # no need to show hardcoded parameters hardcoded_value = parameter_hardcoder.get_hardcoded_value(param.name, model.name) if param.name in kwargs["blacklisted_parameters"] or hardcoded_value: - # let's not use an extra level of indentation and use NOP continue + # optional outfiles: create an additional bool input which is used to filter for the output + # mandatory outpiles: no input node needed + # inputs: create the input param if param.type is _OutFile: if not param.required: _param = copy.deepcopy(param) @@ -608,9 +610,11 @@ def create_inputs(tool, model, **kwargs): else: parent_node = inputs_node + # create the actual param node and fill the attributes param_node = add_child_node(parent_node, "param") create_param_attribute_list(param_node, _param, kwargs["supported_file_formats"]) + # if there is an advanced section then append it at the end of the inputs if not advanced_node is None: inputs_node.append(advanced_node) @@ -625,9 +629,10 @@ def create_param_attribute_list(param_node, param, supported_file_formats): @param supported_file_formats """ - # set the name and a first guess for the type (which will be over written + # set the name, argument and a first guess for the type (which will be over written # in some cases .. see below) param_node.attrib["name"] = get_galaxy_parameter_name(param) + param_node.attrib["argument"] = str(param) param_type = TYPE_TO_GALAXY_TYPE[param.type] if param_type is None: raise ModelError("Unrecognized parameter type %(type)s for parameter %(name)s" @@ -801,7 +806,6 @@ def create_param_attribute_list(param_node, param, supported_file_formats): param_node.attrib["label"] = label param_node.attrib["help"] = help_text - param_node.attrib["argument"] = "-"+param.name def generate_label_and_help(desc): help_text = "" diff --git a/tests/test-data/bool.xml b/tests/test-data/bool.xml index c74fbe92..c7ead423 100644 --- a/tests/test-data/bool.xml +++ b/tests/test-data/bool.xml @@ -1,6 +1,6 @@ - + Bool parameter tests @@ -25,15 +25,15 @@ $param_flag_false > $param_stdout ]]> - - - - + + + + - - - - + + + + diff --git a/tests/test-data/empty.xml b/tests/test-data/empty.xml index 07302234..afdcb5d6 100644 --- a/tests/test-data/empty.xml +++ b/tests/test-data/empty.xml @@ -1,6 +1,6 @@ - + String parameter tests diff --git a/tests/test-data/float.xml b/tests/test-data/float.xml index 0cc8a417..81e19d97 100644 --- a/tests/test-data/float.xml +++ b/tests/test-data/float.xml @@ -1,6 +1,6 @@ - + Float parameter tests @@ -81,39 +81,39 @@ echo FloatTest > $param_stdout ]]> - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + diff --git a/tests/test-data/ifile.xml b/tests/test-data/ifile.xml index 822fe653..0141c92f 100644 --- a/tests/test-data/ifile.xml +++ b/tests/test-data/ifile.xml @@ -1,6 +1,6 @@ - + Input File parameter tests @@ -162,55 +162,55 @@ echo inFileTest > $param_stdout ]]> - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/test-data/integer.xml b/tests/test-data/integer.xml index afe21a66..577af9a6 100644 --- a/tests/test-data/integer.xml +++ b/tests/test-data/integer.xml @@ -1,6 +1,6 @@ - + Integer parameter tests @@ -81,39 +81,39 @@ echo IntTest > $param_stdout ]]> - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + diff --git a/tests/test-data/ofile-mult.xml b/tests/test-data/ofile-mult.xml index bc1161b2..dc17d142 100644 --- a/tests/test-data/ofile-mult.xml +++ b/tests/test-data/ofile-mult.xml @@ -1,6 +1,6 @@ - + multiple Output File parameter tests @@ -35,12 +35,12 @@ echo MultoutFileTest -inp_optional_optionalinput ${' '.join(["'%s.%s'"%(re.sub('[^\w\-_.]', '_', _.element_identifier),_.ext) for _ in $param_inp_optional_optionalinput if _ ])} ]]> - - - - - - + + + + + + diff --git a/tests/test-data/ofile.xml b/tests/test-data/ofile.xml index 068c74a3..0e4a353e 100644 --- a/tests/test-data/ofile.xml +++ b/tests/test-data/ofile.xml @@ -1,6 +1,6 @@ - + Outut File parameter tests @@ -52,16 +52,16 @@ echo outFileTest #end if ]]> - - - - - - + + + + + + - - - + + + diff --git a/tests/test-data/repeat.xml b/tests/test-data/repeat.xml index d5862da7..8be0ed94 100644 --- a/tests/test-data/repeat.xml +++ b/tests/test-data/repeat.xml @@ -1,6 +1,6 @@ - + Repeat parameter tests @@ -97,7 +97,7 @@ echo RepeatTest > $param_stdout ]]> - + @@ -105,7 +105,7 @@ echo RepeatTest - + @@ -113,7 +113,7 @@ echo RepeatTest - + @@ -121,7 +121,7 @@ echo RepeatTest - + @@ -129,7 +129,7 @@ echo RepeatTest - + @@ -140,7 +140,7 @@ echo RepeatTest - + @@ -151,7 +151,7 @@ echo RepeatTest - + @@ -162,7 +162,7 @@ echo RepeatTest - + @@ -173,7 +173,7 @@ echo RepeatTest - + @@ -186,7 +186,7 @@ echo RepeatTest - + @@ -199,7 +199,7 @@ echo RepeatTest - + @@ -212,7 +212,7 @@ echo RepeatTest - + @@ -225,7 +225,7 @@ echo RepeatTest - + @@ -236,7 +236,7 @@ echo RepeatTest - + @@ -247,7 +247,7 @@ echo RepeatTest - + @@ -258,7 +258,7 @@ echo RepeatTest - + @@ -269,7 +269,7 @@ echo RepeatTest - + @@ -282,7 +282,7 @@ echo RepeatTest - + @@ -295,7 +295,7 @@ echo RepeatTest - + @@ -308,7 +308,7 @@ echo RepeatTest - + @@ -322,7 +322,7 @@ echo RepeatTest - + @@ -330,7 +330,7 @@ echo RepeatTest - + @@ -338,7 +338,7 @@ echo RepeatTest - + @@ -346,7 +346,7 @@ echo RepeatTest - + @@ -354,7 +354,7 @@ echo RepeatTest - + @@ -365,7 +365,7 @@ echo RepeatTest - + @@ -376,7 +376,7 @@ echo RepeatTest - + @@ -387,7 +387,7 @@ echo RepeatTest - + @@ -398,7 +398,7 @@ echo RepeatTest - + @@ -411,7 +411,7 @@ echo RepeatTest - + @@ -424,7 +424,7 @@ echo RepeatTest - + @@ -437,7 +437,7 @@ echo RepeatTest - + @@ -450,7 +450,7 @@ echo RepeatTest - + @@ -461,7 +461,7 @@ echo RepeatTest - + @@ -472,7 +472,7 @@ echo RepeatTest - + @@ -483,7 +483,7 @@ echo RepeatTest - + @@ -494,7 +494,7 @@ echo RepeatTest - + @@ -507,7 +507,7 @@ echo RepeatTest - + @@ -520,7 +520,7 @@ echo RepeatTest - + @@ -533,7 +533,7 @@ echo RepeatTest - + diff --git a/tests/test-data/select.xml b/tests/test-data/select.xml index d866ed77..04f124c0 100644 --- a/tests/test-data/select.xml +++ b/tests/test-data/select.xml @@ -1,6 +1,6 @@ - + Select parameter tests @@ -81,104 +81,104 @@ echo SelectTest > $param_stdout ]]> - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -186,104 +186,104 @@ echo SelectTest - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + diff --git a/tests/test-data/string.xml b/tests/test-data/string.xml index eb50dfd1..b3830f4e 100644 --- a/tests/test-data/string.xml +++ b/tests/test-data/string.xml @@ -1,6 +1,6 @@ - + String parameter tests @@ -33,7 +33,7 @@ echo StringTest > $param_stdout ]]> - + @@ -41,7 +41,7 @@ echo StringTest - + @@ -49,7 +49,7 @@ echo StringTest - + @@ -57,7 +57,7 @@ echo StringTest - + @@ -66,7 +66,7 @@ echo StringTest - + @@ -74,7 +74,7 @@ echo StringTest - + @@ -82,7 +82,7 @@ echo StringTest - + @@ -90,7 +90,7 @@ echo StringTest - + From ab94c12368a62df16f5745f22e7332d1190057bf Mon Sep 17 00:00:00 2001 From: M Bernt Date: Wed, 13 Nov 2019 19:40:31 +0100 Subject: [PATCH 036/121] - add option to integrate automatically generated tests - use executable token - fix argument attribute --- galaxy/converter.py | 225 +++++++++++++++++---------------- tests/test-data/bool.xml | 4 +- tests/test-data/empty.xml | 4 +- tests/test-data/float.xml | 4 +- tests/test-data/ifile.xml | 4 +- tests/test-data/integer.xml | 4 +- tests/test-data/ofile-mult.xml | 4 +- tests/test-data/ofile.xml | 4 +- tests/test-data/repeat.xml | 4 +- tests/test-data/select.xml | 4 +- tests/test-data/string.xml | 4 +- 11 files changed, 138 insertions(+), 127 deletions(-) diff --git a/galaxy/converter.py b/galaxy/converter.py index 987422c3..df5dd453 100755 --- a/galaxy/converter.py +++ b/galaxy/converter.py @@ -72,6 +72,11 @@ def add_specific_args(parser): "The macros stdio, requirements and advanced_options are " "required. Please see galaxy/macros.xml for an example of a " "valid macros file. All defined macros will be imported.") + parser.add_argument("--test-macros", dest="test_macros_files", default=[], nargs="*", + action="append", required=None, help="Import tests from the files given file(s) as macros. " + "The macro names must end with the id of the tools") + parser.add_argument("--test-macros-prefix", dest="test_macros_prefix", default=[], nargs="*", + action="append", required=None, help="The prefix of the macro name in the corresponding trest macros file") parser.add_argument("--test-test", dest="test_test", action='store_true', default=False, required=False, help="Generate a simple test for the internal unit tests.") @@ -87,7 +92,7 @@ def convert_models(args, parsed_ctds): validate_and_prepare_args(args) # extract the names of the macros and check that we have found the ones we need - macros_to_expand = parse_macros_files(args.macros_files) + macros_to_expand = parse_macros_files(args.macros_files, required_macros=REQUIRED_MACROS, dont_expand=[ADVANCED_OPTIONS_NAME+"macro", "references"]) # parse the given supported file-formats file supported_file_formats = parse_file_formats(args.formats_file) @@ -95,7 +100,6 @@ def convert_models(args, parsed_ctds): # parse the skip/required tools files skip_tools = parse_tools_list_file(args.skip_tools_file) required_tools = parse_tools_list_file(args.required_tools_file) - _convert_internal(parsed_ctds, supported_file_formats=supported_file_formats, default_executable_path=args.default_executable_path, @@ -106,7 +110,9 @@ def convert_models(args, parsed_ctds): macros_file_names=args.macros_files, macros_to_expand=macros_to_expand, parameter_hardcoder=args.parameter_hardcoder, - test_test=args.test_test) + test_test=args.test_test, + test_macros_file_names=args.test_macros_files, + test_macros_prefix=args.test_macros_prefix) # generation of galaxy stubs is ready... now, let's see if we need to generate a tool_conf.xml if args.tool_conf_destination is not None: @@ -134,11 +140,10 @@ def parse_tools_list_file(tools_list_file): return tools_list -def parse_macros_files(macros_file_names): +def parse_macros_files(macros_file_names, required_macros = [], dont_expand=[]): """ """ - macros_to_expand = list() - + macros_to_expand = set() for macros_file_name in macros_file_names: try: macros_file = open(macros_file_name) @@ -149,9 +154,9 @@ def parse_macros_files(macros_file_names): if name in macros_to_expand: logger.warning("Macro %s has already been found. Duplicate found in file %s." % (name, macros_file_name), 0) - else: - logger.info("Macro %s found" % name, 1) - macros_to_expand.append(name) + continue + logger.info("Macro %s found" % name, 1) + macros_to_expand.add(name) except ParseError as e: raise ApplicationException("The macros file " + macros_file_name + " could not be parsed. Cause: " + str(e)) @@ -161,7 +166,7 @@ def parse_macros_files(macros_file_names): # we depend on "stdio", "requirements" and "advanced_options" to exist on all the given macros files missing_needed_macros = [] - for required_macro in REQUIRED_MACROS: + for required_macro in required_macros: if required_macro not in macros_to_expand: missing_needed_macros.append(required_macro) @@ -170,9 +175,11 @@ def parse_macros_files(macros_file_names): "The following required macro(s) were not found in any of the given macros files: %s, " "see galaxy/macros.xml for an example of a valid macros file." % ", ".join(missing_needed_macros)) + + # remove macros that should not be expanded + for m in dont_expand: + macros_to_expand.discard(m) - # we do not need to "expand" the advanced_options macro - macros_to_expand.remove(ADVANCED_OPTIONS_NAME+"macro") return macros_to_expand @@ -226,6 +233,8 @@ def validate_and_prepare_args(args): # flatten macros_files to make sure that we have a list containing file names and not a list of lists utils.flatten_list_of_lists(args, "macros_files") + utils.flatten_list_of_lists(args, "test_macros_files") + utils.flatten_list_of_lists(args, "test_macros_prefix") # check that the arguments point to a valid, existing path input_variables_to_check = ["skip_tools_file", "required_tools_file", "macros_files", "formats_file"] @@ -278,15 +287,19 @@ def _convert_internal(parsed_ctds, **kwargs): tool = create_tool(model) write_header(tool, model) create_description(tool, model) - expand_macros(tool, model, **kwargs) + import_macros(tool, model, **kwargs) + expand_macros(tool, kwargs["macros_to_expand"]) create_command(tool, model, **kwargs) inputs = create_inputs(tool, model, **kwargs) outputs = create_outputs(tool, model, **kwargs) if kwargs["test_test"]: - create_tests(tool, copy.deepcopy(inputs), copy.deepcopy(outputs)) + create_tests(tool, inputs=copy.deepcopy(inputs), outputs=copy.deepcopy(outputs)) + if kwargs["test_macros_prefix"]: + create_tests(tool, test_macros_prefix=kwargs['test_macros_prefix'], name=model.name) + create_help(tool, model) # citations are required to be at the end - move_citations(tool) + expand_macros(tool, ["references"]) # wrap our tool element into a tree to be able to serialize it tree = ElementTree(tool) @@ -389,7 +402,7 @@ def create_command(tool, model, **kwargs): # main command final_preprocessing = "\n" - final_command = utils.extract_tool_executable_path(model, kwargs["default_executable_path"]) + '\n' + final_command = "@EXECUTABLE@" + '\n' final_command += kwargs["add_to_command_line"] + '\n' advanced_command_start = "#if ${aon}cond.{aon}selector=='advanced':\n".format(aon=ADVANCED_OPTIONS_NAME) advanced_command_end = "#end if" @@ -494,8 +507,7 @@ def create_command(tool, model, **kwargs): # creates the xml elements needed to import the needed macros files -# and to "expand" the macros -def expand_macros(tool, model, **kwargs): +def import_macros(tool, model, **kwargs): """ @param tool the Galaxy tool @param model the ctd model @@ -507,24 +519,19 @@ def expand_macros(tool, model, **kwargs): token_node.text = utils.extract_tool_executable_path(model, kwargs["default_executable_path"]) # add nodes - for macro_file_name in kwargs["macros_file_names"]: + for macro_file_name in kwargs["macros_file_names"]+kwargs["test_macros_file_names"]: macro_file = open(macro_file_name) import_node = add_child_node(macros_node, "import") # do not add the path of the file, rather, just its basename import_node.text = os.path.basename(macro_file.name) + +# and to "expand" the macros in a node +def expand_macros(node, macros_to_expand): # add nodes - for expand_macro in kwargs["macros_to_expand"]: - expand_node = add_child_node(tool, "expand") + for expand_macro in macros_to_expand: + expand_node = add_child_node(node, "expand") expand_node.attrib["macro"] = expand_macro -def move_citations(tool): - """ - move expansions of the citations/references macro to the end of the tool file - """ - for expand in tool.iter('expand'): - if expand.attrib["macro"] in ["references", "citations"]: - tool.append(expand) - def get_galaxy_parameter_name(param, inparam = False): """ @@ -632,7 +639,7 @@ def create_param_attribute_list(param_node, param, supported_file_formats): # set the name, argument and a first guess for the type (which will be over written # in some cases .. see below) param_node.attrib["name"] = get_galaxy_parameter_name(param) - param_node.attrib["argument"] = str(param) + param_node.attrib["argument"] = "-"+param.name param_type = TYPE_TO_GALAXY_TYPE[param.type] if param_type is None: raise ModelError("Unrecognized parameter type %(type)s for parameter %(name)s" @@ -1043,91 +1050,95 @@ def create_change_format_node(parent, data_formats, input_ref): OrderedDict([("input", input_ref), ("value", data_format), ("format", data_format)])) -def create_tests(parent, inputs, outputs): +def create_tests(parent, inputs=None, outputs=None, test_macros_prefix=None, name=None): """ create tests section of the Galaxy tool @param tool the Galaxy tool @param inputs a copy of the inputs """ tests_node = add_child_node(parent, "tests") - test_node = add_child_node(tests_node, "test") - - strip_elements(inputs, "validator", "sanitizer") - for node in inputs.iter(): - if node.tag == "expand": - node.tag = "conditional" - node.attrib["name"] = ADVANCED_OPTIONS_NAME+"cond" - add_child_node(node, "param", OrderedDict([("name", ADVANCED_OPTIONS_NAME+"selector"), ("value", "advanced")])) - if not "type" in node.attrib: - continue - - if (node.attrib["type"] == "select" and "true" in set([_.attrib.get("selected","false") for _ in node])) or\ - (node.attrib["type"] == "select" and node.attrib.get("value", "") != ""): - node.tag = "delete_node" - continue - - # TODO make this optional (ie add aparameter) - if node.attrib["optional"] == "true" and node.attrib["type"]!="boolean": - node.tag = "delete_node" - continue - if node.attrib["type"] == "boolean": - if node.attrib["checked"] == "true": - node.attrib["value"] = "true" # node.attrib["truevalue"] - else: - node.attrib["value"] = "false" # node.attrib["falsevalue"] - elif node.attrib["type"] == "text" and node.attrib["value"] == "": - node.attrib["value"] = "1,2" # use a comma separated list here to cover the repeat (int/float) case - elif node.attrib["type"] == "integer" and node.attrib["value"] == "": - node.attrib["value"] = "1" - elif node.attrib["type"] == "float" and node.attrib["value"] == "": - node.attrib["value"] = "1.0" - elif node.attrib["type"] == "select": - if node.attrib.get("display", None) == "radio" or node.attrib.get("multiple", "false") == "false": - node.attrib["value"] = node[-1].attrib["value"] - elif node.attrib.get("multiple", None) == "true": - node.attrib["value"] = ",".join([ _.attrib["value"] for _ in node ]) - elif node.attrib["type"] == "data": - node.attrib["ftype"] = node.attrib["format"].split(',')[0] - if node.attrib.get("multiple", "false") == "true": - node.attrib["value"] = "test.ext,test2.ext" - else: - node.attrib["value"] = "test.ext" - for node in inputs.iter(): - for a in set(node.attrib) - set(["name","value","ftype"]): - del node.attrib[a] - strip_elements(inputs, "delete_node", "option") - for node in inputs: - test_node.append(node) - outputs_cnt = 0 - for node in outputs.iter(): - if node.tag == "data" or node.tag == "collection": - # assuming that all filters avaluate to false - has_filter = False - for c in node: - if c.tag == "filter": - has_filter = True - break - if not has_filter: - outputs_cnt += 1 - else: + if not (inputs is None or outputs is None): + test_node = add_child_node(tests_node, "test") + strip_elements(inputs, "validator", "sanitizer") + for node in inputs.iter(): + if node.tag == "expand": + node.tag = "conditional" + node.attrib["name"] = ADVANCED_OPTIONS_NAME+"cond" + add_child_node(node, "param", OrderedDict([("name", ADVANCED_OPTIONS_NAME+"selector"), ("value", "advanced")])) + if not "type" in node.attrib: + continue + + if (node.attrib["type"] == "select" and "true" in set([_.attrib.get("selected","false") for _ in node])) or\ + (node.attrib["type"] == "select" and node.attrib.get("value", "") != ""): + node.tag = "delete_node" + continue + + # TODO make this optional (ie add aparameter) + if node.attrib["optional"] == "true" and node.attrib["type"]!="boolean": node.tag = "delete_node" - if node.tag == "data": - node.tag = "output" - node.attrib["ftype"] = node.attrib["format"] - node.attrib["value"] = "outfile.txt" - if node.tag == "collection": - node.tag = "output_collection" - if node.attrib.get("name", None) == "param_stdout": - node.attrib["lines_diff"] = "2" - for a in set(node.attrib) - set(["name","value","ftype", "lines_diff"]): - del node.attrib[a] - strip_elements(outputs, "delete_node", "discover_datasets", "filter") - - for node in outputs: - test_node.append(node) - - test_node.attrib["expect_num_outputs"] = str(outputs_cnt) + continue + + if node.attrib["type"] == "boolean": + if node.attrib["checked"] == "true": + node.attrib["value"] = "true" # node.attrib["truevalue"] + else: + node.attrib["value"] = "false" # node.attrib["falsevalue"] + elif node.attrib["type"] == "text" and node.attrib["value"] == "": + node.attrib["value"] = "1,2" # use a comma separated list here to cover the repeat (int/float) case + elif node.attrib["type"] == "integer" and node.attrib["value"] == "": + node.attrib["value"] = "1" + elif node.attrib["type"] == "float" and node.attrib["value"] == "": + node.attrib["value"] = "1.0" + elif node.attrib["type"] == "select": + if node.attrib.get("display", None) == "radio" or node.attrib.get("multiple", "false") == "false": + node.attrib["value"] = node[-1].attrib["value"] + elif node.attrib.get("multiple", None) == "true": + node.attrib["value"] = ",".join([ _.attrib["value"] for _ in node ]) + elif node.attrib["type"] == "data": + node.attrib["ftype"] = node.attrib["format"].split(',')[0] + if node.attrib.get("multiple", "false") == "true": + node.attrib["value"] = "test.ext,test2.ext" + else: + node.attrib["value"] = "test.ext" + for node in inputs.iter(): + for a in set(node.attrib) - set(["name","value","ftype"]): + del node.attrib[a] + strip_elements(inputs, "delete_node", "option") + for node in inputs: + test_node.append(node) + + outputs_cnt = 0 + for node in outputs.iter(): + if node.tag == "data" or node.tag == "collection": + # assuming that all filters avaluate to false + has_filter = False + for c in node: + if c.tag == "filter": + has_filter = True + break + if not has_filter: + outputs_cnt += 1 + else: + node.tag = "delete_node" + if node.tag == "data": + node.tag = "output" + node.attrib["ftype"] = node.attrib["format"] + node.attrib["value"] = "outfile.txt" + if node.tag == "collection": + node.tag = "output_collection" + if node.attrib.get("name", None) == "param_stdout": + node.attrib["lines_diff"] = "2" + for a in set(node.attrib) - set(["name","value","ftype", "lines_diff"]): + del node.attrib[a] + strip_elements(outputs, "delete_node", "discover_datasets", "filter") + + for node in outputs: + test_node.append(node) + test_node.attrib["expect_num_outputs"] = str(outputs_cnt) + elif not (test_macros_prefix is None or name is None): + expand_macros(tests_node, [ p + name for p in test_macros_prefix]) + # Shows basic information about the file, such as data ranges and file type. def create_help(tool, model): diff --git a/tests/test-data/bool.xml b/tests/test-data/bool.xml index c7ead423..3542b582 100644 --- a/tests/test-data/bool.xml +++ b/tests/test-data/bool.xml @@ -7,10 +7,10 @@ echo BoolTest macros.xml - + echo StringTest macros.xml - + $param_stdout ]]> diff --git a/tests/test-data/float.xml b/tests/test-data/float.xml index 81e19d97..1ce8ea57 100644 --- a/tests/test-data/float.xml +++ b/tests/test-data/float.xml @@ -7,10 +7,10 @@ echo FloatTest macros.xml - + echo inFileTest macros.xml - + echo IntTest macros.xml - + echo MultoutFileTest macros.xml - + echo outFileTest macros.xml - + echo RepeatTest macros.xml - + echo SelectTest macros.xml - + echo StringTest macros.xml - + Date: Sat, 16 Nov 2019 15:08:31 +0100 Subject: [PATCH 037/121] - improve parsing of hardcoded values file for flags the value column can now contain a single # which is removed when parsing - allow to overwrite tool version from command line - automatically set galaxy tool version - updated macro file - ITEMLISTs (wo restrictions, i.e. rendered as text) need to be split by ',' on the commandline --- common/utils.py | 39 +++++++-------- galaxy/converter.py | 81 ++++++++++++++++++++++++-------- galaxy/macros.xml | 21 ++++++--- tests/test-data/bool.xml | 6 +-- tests/test-data/empty.xml | 6 +-- tests/test-data/float.xml | 6 +-- tests/test-data/ifile.xml | 78 +++++++++++++++--------------- tests/test-data/integer.xml | 6 +-- tests/test-data/ofile-mult.xml | 6 +-- tests/test-data/ofile.xml | 6 +-- tests/test-data/repeat.xml | 86 +++++++++++++++++----------------- tests/test-data/select.ctd | 2 +- tests/test-data/select.xml | 38 +++++++-------- tests/test-data/string.xml | 6 +-- 14 files changed, 219 insertions(+), 168 deletions(-) diff --git a/common/utils.py b/common/utils.py index 020b8664..73ba341f 100644 --- a/common/utils.py +++ b/common/utils.py @@ -169,26 +169,27 @@ def parse_hardcoded_parameters(hardcoded_parameters_file): for line in f: line_number += 1 if line is None or not line.strip() or line.strip().startswith("#"): - pass + continue + # the third column must not be obtained as a whole, and not split + parsed_hardcoded_parameter = [ _ for _ in line.strip().split("\t") if _ != ""] + # valid lines contain two or three columns + if not (2 <= len(parsed_hardcoded_parameter) <= 3): + warning("Invalid line at line number %d of the given hardcoded parameters file. Line will be" + "ignored:\n%s" % (line_number, line), 0) + continue + + parameter_name = parsed_hardcoded_parameter[0] + hardcoded_value = parsed_hardcoded_parameter[1] + if hardcoded_value == "#": + hardcoded_value = "" + tool_names = None + if len(parsed_hardcoded_parameter) == 3: + tool_names = parsed_hardcoded_parameter[2].split(',') + if tool_names: + for tool_name in tool_names: + parameter_hardcoder.register_parameter(parameter_name, hardcoded_value, tool_name.strip()) else: - # the third column must not be obtained as a whole, and not split - parsed_hardcoded_parameter = [ _ for _ in line.strip().split("\t") if _ != ""] - # valid lines contain two or three columns - if not (2 <= len(parsed_hardcoded_parameter) <= 3): - warning("Invalid line at line number %d of the given hardcoded parameters file. Line will be" - "ignored:\n%s" % (line_number, line), 0) - continue - - parameter_name = parsed_hardcoded_parameter[0] - hardcoded_value = parsed_hardcoded_parameter[1] - tool_names = None - if len(parsed_hardcoded_parameter) == 3: - tool_names = parsed_hardcoded_parameter[2].split(',') - if tool_names: - for tool_name in tool_names: - parameter_hardcoder.register_parameter(parameter_name, hardcoded_value, tool_name.strip()) - else: - parameter_hardcoder.register_parameter(parameter_name, hardcoded_value) + parameter_hardcoder.register_parameter(parameter_name, hardcoded_value) return parameter_hardcoder diff --git a/galaxy/converter.py b/galaxy/converter.py index df5dd453..2738d1b1 100755 --- a/galaxy/converter.py +++ b/galaxy/converter.py @@ -1,6 +1,7 @@ #!/usr/bin/env python # encoding: utf-8 import os +import os.path import string from collections import OrderedDict @@ -79,6 +80,8 @@ def add_specific_args(parser): action="append", required=None, help="The prefix of the macro name in the corresponding trest macros file") parser.add_argument("--test-test", dest="test_test", action='store_true', default=False, required=False, help="Generate a simple test for the internal unit tests.") + parser.add_argument("--tool-version", dest="tool_version", required=False, default = None, + help="Tool version to use (if not given its extracted from the CTD)") def convert_models(args, parsed_ctds): @@ -89,10 +92,13 @@ def convert_models(args, parsed_ctds): @param parsed_ctds the ctds """ # validate and prepare the passed arguments - validate_and_prepare_args(args) - + validate_and_prepare_args(args, parsed_ctds[0].ctd_model) + # extract the names of the macros and check that we have found the ones we need - macros_to_expand = parse_macros_files(args.macros_files, required_macros=REQUIRED_MACROS, dont_expand=[ADVANCED_OPTIONS_NAME+"macro", "references"]) + macros_to_expand = parse_macros_files(args.macros_files, + tool_version=args.tool_version, + required_macros=REQUIRED_MACROS, + dont_expand=[ADVANCED_OPTIONS_NAME+"macro", "references"]) # parse the given supported file-formats file supported_file_formats = parse_file_formats(args.formats_file) @@ -112,7 +118,8 @@ def convert_models(args, parsed_ctds): parameter_hardcoder=args.parameter_hardcoder, test_test=args.test_test, test_macros_file_names=args.test_macros_files, - test_macros_prefix=args.test_macros_prefix) + test_macros_prefix=args.test_macros_prefix, + tool_version = args.tool_version) # generation of galaxy stubs is ready... now, let's see if we need to generate a tool_conf.xml if args.tool_conf_destination is not None: @@ -140,10 +147,10 @@ def parse_tools_list_file(tools_list_file): return tools_list -def parse_macros_files(macros_file_names, required_macros = [], dont_expand=[]): +def parse_macros_files(macros_file_names, tool_version, required_macros = [], dont_expand=[]): """ """ - macros_to_expand = set() + macros_to_expand = [] for macros_file_name in macros_file_names: try: macros_file = open(macros_file_name) @@ -156,13 +163,31 @@ def parse_macros_files(macros_file_names, required_macros = [], dont_expand=[]): (name, macros_file_name), 0) continue logger.info("Macro %s found" % name, 1) - macros_to_expand.add(name) + macros_to_expand.append(name) + bump_galaxy_version = True + for xml_element in root.findall("token"): + if xml_element.attrib["name"] == "@TOOL_VERSION@": + if tool_version > xml_element.text: + bump_galaxy_version = False + xml_element.text = tool_version + if xml_element.attrib["name"] == "@GALAXY_VERSION@": + if bump_galaxy_version: + xml_element.text = str( int(xml_element.text) + 1 ) + else: + xml_element.text = "0" + + except ParseError as e: raise ApplicationException("The macros file " + macros_file_name + " could not be parsed. Cause: " + str(e)) except IOError as e: raise ApplicationException("The macros file " + macros_file_name + " could not be opened. Cause: " + str(e)) + else: + macros_file.close() + with open(os.path.basename(macros_file_name), "w") as macros_file: + tree = ElementTree(root) + tree.write(macros_file, encoding="UTF-8", xml_declaration=True, pretty_print=True) # we depend on "stdio", "requirements" and "advanced_options" to exist on all the given macros files missing_needed_macros = [] @@ -178,7 +203,11 @@ def parse_macros_files(macros_file_names, required_macros = [], dont_expand=[]): # remove macros that should not be expanded for m in dont_expand: - macros_to_expand.discard(m) + try: + idx = macros_to_expand.index(m) + del macros_to_expand[idx] + except ValueError: + pass return macros_to_expand @@ -219,7 +248,7 @@ def parse_file_formats(formats_file): return supported_formats -def validate_and_prepare_args(args): +def validate_and_prepare_args(args, model): """ check command line arguments @param args command line arguments @@ -251,8 +280,10 @@ def validate_and_prepare_args(args): if not args.macros_files: # list is empty, provide the default value logger.warning("Using default macros from galaxy/macros.xml", 0) - args.macros_files = ["galaxy/macros.xml"] + args.macros_files = [os.path.dirname(os.path.abspath( __file__ ))+"/macros.xml"] + if args.tool_version == None: + args.tool_version = model.version def get_preferred_file_extension(): """ @@ -304,7 +335,8 @@ def _convert_internal(parsed_ctds, **kwargs): # wrap our tool element into a tree to be able to serialize it tree = ElementTree(tool) logger.info("Writing to %s" % utils.get_filename(output_file), 1) - tree.write(open(output_file, 'w'), encoding="UTF-8", xml_declaration=True, pretty_print=True) + with open(output_file, 'w') as fh: + tree.write(fh, encoding="UTF-8", xml_declaration=True, pretty_print=True) def write_header(tool, model): @@ -379,7 +411,8 @@ def create_tool(model): initialize the tool @param model the ctd model """ - return Element("tool", OrderedDict([("id", model.name.replace(" ","_")), ("name", model.name), ("version", model.version)])) + + return Element("tool", OrderedDict([("id", model.name.replace(" ","_")), ("name", model.name), ("version", "@TOOL_VERSION@+galaxy@GALAXY_VERSION@")])) def create_description(tool, model): @@ -402,6 +435,7 @@ def create_command(tool, model, **kwargs): # main command final_preprocessing = "\n" + advanced_preprocessing = "" final_command = "@EXECUTABLE@" + '\n' final_command += kwargs["add_to_command_line"] + '\n' advanced_command_start = "#if ${aon}cond.{aon}selector=='advanced':\n".format(aon=ADVANCED_OPTIONS_NAME) @@ -426,7 +460,7 @@ def create_command(tool, model, **kwargs): continue hardcoded_value = parameter_hardcoder.get_hardcoded_value(param_name, model.name) - if hardcoded_value: + if not hardcoded_value is None: command += "%s %s\n" % (command_line_prefix, hardcoded_value) else: # in the else branch the parameter is neither blacklisted nor hardcoded... @@ -462,8 +496,10 @@ def create_command(tool, model, **kwargs): else: command += actual_parameter+"/'${re.sub('[^\w\-_.]', '_', $"+actual_input_parameter+".element_identifier)}.${"+actual_input_parameter+".ext}'\n" # logic for ITEMLISTs - elif param.is_list and is_selection_parameter(param): + elif is_selection_parameter(param): command += "${' '.join([\"'%s'\"%str(_) for _ in $" + actual_parameter + "])}\n" + elif param.is_list : + command += "${' '.join([\"'%s'\"%str(_) for _ in $" + actual_parameter + ".split(',')])}\n" elif is_boolean_parameter(param): command += "$%s" % actual_parameter + "\n" else: @@ -484,15 +520,18 @@ def create_command(tool, model, **kwargs): if param.type is _InFile: preprocessing = "#if $" + actual_parameter + ":\n" + utils.indent(preprocessing) + "\n#end if\n" - if param.advanced: + if param.advanced and hardcoded_value is None: advanced_command += "%s\n" % utils.indent(command) + if preprocessing != "": + advanced_preprocessing += "%s\n" % utils.indent(preprocessing) else: final_command += command - - final_preprocessing += preprocessing + final_preprocessing += preprocessing if advanced_command: final_command += "%s%s%s\n" % (advanced_command_start, advanced_command, advanced_command_end) + if advanced_preprocessing: + final_preprocessing += "%s%s%s\n" % (advanced_command_start, advanced_preprocessing, advanced_command_end) if not found_output_parameter: final_command += "> $param_stdout\n" @@ -503,6 +542,7 @@ def create_command(tool, model, **kwargs): final_command = "\n#import re" + final_command command_node = add_child_node(tool, "command") + command_node.attrib["detect_errors"] = "aggressive" command_node.text = CDATA(final_command) @@ -592,7 +632,7 @@ def create_inputs(tool, model, **kwargs): for param in utils.extract_and_flatten_parameters(model): # no need to show hardcoded parameters hardcoded_value = parameter_hardcoder.get_hardcoded_value(param.name, model.name) - if param.name in kwargs["blacklisted_parameters"] or hardcoded_value: + if param.name in kwargs["blacklisted_parameters"] or not hardcoded_value is None: continue # optional outfiles: create an additional bool input which is used to filter for the output # mandatory outpiles: no input node needed @@ -1014,8 +1054,9 @@ def create_output_node(parent, param, model, supported_file_formats): raise InvalidModelException("Unrecognized restriction type [%(type)s] " "for output [%(name)s]" % {"type": type(param.restrictions), "name": param.name}) - # data output has fomat .. collection output has no format - if not param.is_list: + # data output has fomat (except if fromat_source has been added already) + # note .. collection output has no format + if not param.is_list and not "format_source" in data_node.attrib: data_node.attrib["format"] = data_format # add filter for optional parameters diff --git a/galaxy/macros.xml b/galaxy/macros.xml index a582994f..afcf1a23 100644 --- a/galaxy/macros.xml +++ b/galaxy/macros.xml @@ -3,21 +3,28 @@ You can edit this file to add your own macros, if you so desire, or you can add additional macro files using the m/macros parameter --> + 2.3 + 0 - @EXECUTABLE@ + openms + - - - - + + + - - + + + doi:10.1186/1471-2105-9-163 + + + + diff --git a/tests/test-data/bool.xml b/tests/test-data/bool.xml index 3542b582..fd2fce55 100644 --- a/tests/test-data/bool.xml +++ b/tests/test-data/bool.xml @@ -1,15 +1,15 @@ - + Bool parameter tests echo BoolTest macros.xml - - + - + String parameter tests echo StringTest macros.xml - - + $param_stdout diff --git a/tests/test-data/float.xml b/tests/test-data/float.xml index 1ce8ea57..fd666265 100644 --- a/tests/test-data/float.xml +++ b/tests/test-data/float.xml @@ -1,15 +1,15 @@ - + Float parameter tests echo FloatTest macros.xml - - + - + Input File parameter tests echo inFileTest macros.xml - - + - + Integer parameter tests echo IntTest macros.xml - - + - + multiple Output File parameter tests echo MultoutFileTest macros.xml - - + - + Outut File parameter tests echo outFileTest macros.xml - - + - + Repeat parameter tests echo RepeatTest macros.xml - - + $param_stdout ]]> diff --git a/tests/test-data/select.ctd b/tests/test-data/select.ctd index f928a19b..352424ff 100644 --- a/tests/test-data/select.ctd +++ b/tests/test-data/select.ctd @@ -6,7 +6,7 @@ - + diff --git a/tests/test-data/select.xml b/tests/test-data/select.xml index e480781f..7c10615e 100644 --- a/tests/test-data/select.xml +++ b/tests/test-data/select.xml @@ -1,32 +1,32 @@ - + Select parameter tests echo SelectTest macros.xml - - + - + String parameter tests echo StringTest macros.xml - - + Date: Tue, 26 Nov 2019 14:51:44 +0100 Subject: [PATCH 038/121] cont - also sanitize lists - add function to apply galaxy specific changes to parameters - allow for tests files to be used multiple times (via symlinks) --- galaxy/converter.py | 104 ++++++----- tests/test-data/ifile.xml | 252 +++++++++++++++++--------- tests/test-data/ofile-mult.xml | 20 +- tests/test-data/repeat.xml | 8 - tests/test-data/select.xml | 192 ++++++++++++++++++-- tests/test-data/string.xml | 8 - tests/test-data/test-data/0test.ext | 1 + tests/test-data/test-data/0test2.ext | 1 + tests/test-data/test-data/10test.ext | 1 + tests/test-data/test-data/10test2.ext | 1 + tests/test-data/test-data/11test.ext | 1 + tests/test-data/test-data/11test2.ext | 1 + tests/test-data/test-data/1test.ext | 1 + tests/test-data/test-data/1test2.ext | 1 + tests/test-data/test-data/2test.ext | 1 + tests/test-data/test-data/2test2.ext | 1 + tests/test-data/test-data/3test.ext | 1 + tests/test-data/test-data/3test2.ext | 1 + tests/test-data/test-data/4test.ext | 1 + tests/test-data/test-data/4test2.ext | 1 + tests/test-data/test-data/5test.ext | 1 + tests/test-data/test-data/5test2.ext | 1 + tests/test-data/test-data/6test.ext | 1 + tests/test-data/test-data/6test2.ext | 1 + tests/test-data/test-data/7test.ext | 1 + tests/test-data/test-data/7test2.ext | 1 + tests/test-data/test-data/8test.ext | 1 + tests/test-data/test-data/8test2.ext | 1 + tests/test-data/test-data/9test.ext | 1 + tests/test-data/test-data/9test2.ext | 1 + 30 files changed, 443 insertions(+), 165 deletions(-) create mode 120000 tests/test-data/test-data/0test.ext create mode 120000 tests/test-data/test-data/0test2.ext create mode 120000 tests/test-data/test-data/10test.ext create mode 120000 tests/test-data/test-data/10test2.ext create mode 120000 tests/test-data/test-data/11test.ext create mode 120000 tests/test-data/test-data/11test2.ext create mode 120000 tests/test-data/test-data/1test.ext create mode 120000 tests/test-data/test-data/1test2.ext create mode 120000 tests/test-data/test-data/2test.ext create mode 120000 tests/test-data/test-data/2test2.ext create mode 120000 tests/test-data/test-data/3test.ext create mode 120000 tests/test-data/test-data/3test2.ext create mode 120000 tests/test-data/test-data/4test.ext create mode 120000 tests/test-data/test-data/4test2.ext create mode 120000 tests/test-data/test-data/5test.ext create mode 120000 tests/test-data/test-data/5test2.ext create mode 120000 tests/test-data/test-data/6test.ext create mode 120000 tests/test-data/test-data/6test2.ext create mode 120000 tests/test-data/test-data/7test.ext create mode 120000 tests/test-data/test-data/7test2.ext create mode 120000 tests/test-data/test-data/8test.ext create mode 120000 tests/test-data/test-data/8test2.ext create mode 120000 tests/test-data/test-data/9test.ext create mode 120000 tests/test-data/test-data/9test2.ext diff --git a/galaxy/converter.py b/galaxy/converter.py index 2738d1b1..746ade54 100755 --- a/galaxy/converter.py +++ b/galaxy/converter.py @@ -83,6 +83,20 @@ def add_specific_args(parser): parser.add_argument("--tool-version", dest="tool_version", required=False, default = None, help="Tool version to use (if not given its extracted from the CTD)") +def modify_param_for_galaxy(param): + """ + some parameters need galaxy specific modifications + """ + if param.type is _InFile: + #if file default is given (happens for external applications and + # files for which the default is taken from share/OpenMS) set the + # parm to not required and remove the default (external applications + # need to be taken care by hardcoded values and the other cases + # are chosen automatically if not specified on the command line) + if param.required and not (param.default is None or param.default is _Null): + param.required = False + param.default = _Null + return param def convert_models(args, parsed_ctds): """ @@ -304,8 +318,6 @@ def _convert_internal(parsed_ctds, **kwargs): """ for parsed_ctd in parsed_ctds: model = parsed_ctd.ctd_model - origin_file = parsed_ctd.input_file - output_file = parsed_ctd.suggested_output_file if kwargs["skip_tools"] is not None and model.name in kwargs["skip_tools"]: logger.info("Skipping tool %s" % model.name, 0) @@ -313,30 +325,32 @@ def _convert_internal(parsed_ctds, **kwargs): elif kwargs["required_tools"] is not None and model.name not in kwargs["required_tools"]: logger.info("Tool %s is not required, skipping it" % model.name, 0) continue - else: - logger.info("Converting %s (source %s)" % (model.name, utils.get_filename(origin_file)), 0) - tool = create_tool(model) - write_header(tool, model) - create_description(tool, model) - import_macros(tool, model, **kwargs) - expand_macros(tool, kwargs["macros_to_expand"]) - create_command(tool, model, **kwargs) - inputs = create_inputs(tool, model, **kwargs) - outputs = create_outputs(tool, model, **kwargs) - if kwargs["test_test"]: - create_tests(tool, inputs=copy.deepcopy(inputs), outputs=copy.deepcopy(outputs)) - if kwargs["test_macros_prefix"]: - create_tests(tool, test_macros_prefix=kwargs['test_macros_prefix'], name=model.name) - - create_help(tool, model) - # citations are required to be at the end - expand_macros(tool, ["references"]) - - # wrap our tool element into a tree to be able to serialize it - tree = ElementTree(tool) - logger.info("Writing to %s" % utils.get_filename(output_file), 1) - with open(output_file, 'w') as fh: - tree.write(fh, encoding="UTF-8", xml_declaration=True, pretty_print=True) + + origin_file = parsed_ctd.input_file + output_file = parsed_ctd.suggested_output_file + logger.info("Converting %s (source %s)" % (model.name, utils.get_filename(origin_file)), 0) + tool = create_tool(model) + write_header(tool, model) + create_description(tool, model) + import_macros(tool, model, **kwargs) + expand_macros(tool, kwargs["macros_to_expand"]) + create_command(tool, model, **kwargs) + inputs = create_inputs(tool, model, **kwargs) + outputs = create_outputs(tool, model, **kwargs) + if kwargs["test_test"]: + create_tests(tool, inputs=copy.deepcopy(inputs), outputs=copy.deepcopy(outputs)) + if kwargs["test_macros_prefix"]: + create_tests(tool, test_macros_prefix=kwargs['test_macros_prefix'], name=model.name) + + create_help(tool, model) + # citations are required to be at the end + expand_macros(tool, ["references"]) + + # wrap our tool element into a tree to be able to serialize it + tree = ElementTree(tool) + logger.info("Writing to %s" % utils.get_filename(output_file), 1) + with open(output_file, 'w') as fh: + tree.write(fh, encoding="UTF-8", xml_declaration=True, pretty_print=True) def write_header(tool, model): @@ -447,6 +461,7 @@ def create_command(tool, model, **kwargs): found_input_parameter = False found_output_parameter = False for param in utils.extract_and_flatten_parameters(model): + param = modify_param_for_galaxy(param) if param.type is _OutFile: found_output_parameter = True if param.type is _InFile: @@ -470,7 +485,6 @@ def create_command(tool, model, **kwargs): corresponding_input = None if param.type is _OutFile and param.is_list: corresponding_input = get_input_with_same_restrictions(param, model) - actual_parameter = get_galaxy_parameter_name(param) if param.advanced and not param.type is _OutFile: @@ -496,7 +510,7 @@ def create_command(tool, model, **kwargs): else: command += actual_parameter+"/'${re.sub('[^\w\-_.]', '_', $"+actual_input_parameter+".element_identifier)}.${"+actual_input_parameter+".ext}'\n" # logic for ITEMLISTs - elif is_selection_parameter(param): + elif is_selection_parameter(param) and param.is_list: command += "${' '.join([\"'%s'\"%str(_) for _ in $" + actual_parameter + "])}\n" elif param.is_list : command += "${' '.join([\"'%s'\"%str(_) for _ in $" + actual_parameter + ".split(',')])}\n" @@ -507,7 +521,7 @@ def create_command(tool, model, **kwargs): # add if statement for mandatory parameters and preprocessing # - for optional outputs (param_out_x) the presence of the parameter depends on the additional input (param_x) - if not param.required and not is_boolean_parameter(param) and not(param.type is _InFile and param.is_list): + if not param.required and not is_boolean_parameter(param): # and not(param.type is _InFile and param.is_list): if param.type is _OutFile: if param.advanced: actual_parameter = ADVANCED_OPTIONS_NAME+"cond.%s" % get_galaxy_parameter_name(param, True) @@ -627,13 +641,14 @@ def create_inputs(tool, model, **kwargs): # some suites (such as OpenMS) need some advanced options when handling inputs advanced_node = None parameter_hardcoder = kwargs["parameter_hardcoder"] - # treat all non output-file/advanced/blacklisted/hardcoded parameters as inputs for param in utils.extract_and_flatten_parameters(model): + param = modify_param_for_galaxy(param) # no need to show hardcoded parameters hardcoded_value = parameter_hardcoder.get_hardcoded_value(param.name, model.name) if param.name in kwargs["blacklisted_parameters"] or not hardcoded_value is None: continue + # optional outfiles: create an additional bool input which is used to filter for the output # mandatory outpiles: no input node needed # inputs: create the input param @@ -679,7 +694,7 @@ def create_param_attribute_list(param_node, param, supported_file_formats): # set the name, argument and a first guess for the type (which will be over written # in some cases .. see below) param_node.attrib["name"] = get_galaxy_parameter_name(param) - param_node.attrib["argument"] = "-"+param.name + param_node.attrib["argument"] = "-%s"%utils.extract_param_name(param) param_type = TYPE_TO_GALAXY_TYPE[param.type] if param_type is None: raise ModelError("Unrecognized parameter type %(type)s for parameter %(name)s" @@ -814,17 +829,18 @@ def create_param_attribute_list(param_node, param, supported_file_formats): mapping_node = SubElement(sanitizer_node, "mapping", OrderedDict([("initial", "none")])) SubElement(mapping_node, "add", OrderedDict([("source", ' '),("target", "")])) - # add sanitizer nodes to other text params, this is needed for special character like "[" - # which are used for example by FeatureFinderMultiplex - elif not param.type is _InFile: - sanitizer_node = SubElement(param_node, "sanitizer") - valid_node = SubElement(sanitizer_node, "valid", OrderedDict([("initial", "string.printable")])) - SubElement(valid_node, "remove", OrderedDict([("value", '\'')])) - SubElement(valid_node, "remove", OrderedDict([("value", '"')])) + # add sanitizer nodes to + # - text (only those that are not actually integer selects which are treated above) and + # - select params, + # this is needed for special character like "[" which are used for example by FeatureFinderMultiplex + if ((param_type == "text" and not TYPE_TO_GALAXY_TYPE[param.type] in ["integer", "float"]) or is_selection_parameter(param)) and not param.type is _InFile: + sanitizer_node = SubElement(param_node, "sanitizer") + valid_node = SubElement(sanitizer_node, "valid", OrderedDict([("initial", "string.printable")])) + SubElement(valid_node, "remove", OrderedDict([("value", '\'')])) # check for default value - if param.default is not None and param.default is not _Null: - # defaults of selects are set via the selected attribute of the options + if not (param.default is None or param.default is _Null): + # defaults of selects are set via the selected attribute of the options (happens above) if param_type == "select": pass elif type(param.default) is list: @@ -982,7 +998,7 @@ def create_outputs(parent, model, **kwargs): parameter_hardcoder = kwargs["parameter_hardcoder"] for param in utils.extract_and_flatten_parameters(model): - + param = modify_param_for_galaxy(param) # no need to show hardcoded parameters hardcoded_value = parameter_hardcoder.get_hardcoded_value(param.name, model.name) if param.name in kwargs["blacklisted_parameters"] or hardcoded_value: @@ -1100,6 +1116,7 @@ def create_tests(parent, inputs=None, outputs=None, test_macros_prefix=None, nam tests_node = add_child_node(parent, "tests") if not (inputs is None or outputs is None): + fidx = 0 test_node = add_child_node(tests_node, "test") strip_elements(inputs, "validator", "sanitizer") for node in inputs.iter(): @@ -1139,9 +1156,10 @@ def create_tests(parent, inputs=None, outputs=None, test_macros_prefix=None, nam elif node.attrib["type"] == "data": node.attrib["ftype"] = node.attrib["format"].split(',')[0] if node.attrib.get("multiple", "false") == "true": - node.attrib["value"] = "test.ext,test2.ext" + node.attrib["value"] = "{fidx}test.ext,{fidx}test2.ext".format(fidx=fidx) else: - node.attrib["value"] = "test.ext" + node.attrib["value"] = "{fidx}test.ext".format(fidx=fidx) + fidx += 1 for node in inputs.iter(): for a in set(node.attrib) - set(["name","value","ftype"]): del node.attrib[a] diff --git a/tests/test-data/ifile.xml b/tests/test-data/ifile.xml index bafb79db..670c67d7 100644 --- a/tests/test-data/ifile.xml +++ b/tests/test-data/ifile.xml @@ -23,30 +23,54 @@ ln -s '$param_mandatory_multformat' '${re.sub("[^\w\-_.]", "_", $param_mandatory #if $param_optional_multformat: ln -s '$param_optional_multformat' '${re.sub("[^\w\-_.]", "_", $param_optional_multformat.element_identifier)}.${param_optional_multformat.ext}' && #end if -ln -s '$param_default_mandatory_noformat' '${re.sub("[^\w\-_.]", "_", $param_default_mandatory_noformat.element_identifier)}.${param_default_mandatory_noformat.ext}' && +#if $param_default_mandatory_noformat: + ln -s '$param_default_mandatory_noformat' '${re.sub("[^\w\-_.]", "_", $param_default_mandatory_noformat.element_identifier)}.${param_default_mandatory_noformat.ext}' && +#end if #if $param_default_optional_noformat: ln -s '$param_default_optional_noformat' '${re.sub("[^\w\-_.]", "_", $param_default_optional_noformat.element_identifier)}.${param_default_optional_noformat.ext}' && #end if -ln -s '$param_default_mandatory_oneformat' '${re.sub("[^\w\-_.]", "_", $param_default_mandatory_oneformat.element_identifier)}.${param_default_mandatory_oneformat.ext}' && +#if $param_default_mandatory_oneformat: + ln -s '$param_default_mandatory_oneformat' '${re.sub("[^\w\-_.]", "_", $param_default_mandatory_oneformat.element_identifier)}.${param_default_mandatory_oneformat.ext}' && +#end if #if $param_default_optional_oneformat: ln -s '$param_default_optional_oneformat' '${re.sub("[^\w\-_.]", "_", $param_default_optional_oneformat.element_identifier)}.${param_default_optional_oneformat.ext}' && #end if -ln -s '$param_default_mandatory_multformat' '${re.sub("[^\w\-_.]", "_", $param_default_mandatory_multformat.element_identifier)}.${param_default_mandatory_multformat.ext}' && +#if $param_default_mandatory_multformat: + ln -s '$param_default_mandatory_multformat' '${re.sub("[^\w\-_.]", "_", $param_default_mandatory_multformat.element_identifier)}.${param_default_mandatory_multformat.ext}' && +#end if #if $param_default_optional_multformat: ln -s '$param_default_optional_multformat' '${re.sub("[^\w\-_.]", "_", $param_default_optional_multformat.element_identifier)}.${param_default_optional_multformat.ext}' && #end if ${ ' && '.join([ "ln -s '%s' '%s.%s'" % (_, re.sub('[^\w\-_.]', '_', _.element_identifier), _.ext) for _ in $param_list_mandatory_noformat if _ ]) } && -${ ' && '.join([ "ln -s '%s' '%s.%s'" % (_, re.sub('[^\w\-_.]', '_', _.element_identifier), _.ext) for _ in $param_list_optional_noformat if _ ]) } && +#if $param_list_optional_noformat: + ${ ' && '.join([ "ln -s '%s' '%s.%s'" % (_, re.sub('[^\w\-_.]', '_', _.element_identifier), _.ext) for _ in $param_list_optional_noformat if _ ]) } && +#end if ${ ' && '.join([ "ln -s '%s' '%s.%s'" % (_, re.sub('[^\w\-_.]', '_', _.element_identifier), _.ext) for _ in $param_list_mandatory_oneformat if _ ]) } && -${ ' && '.join([ "ln -s '%s' '%s.%s'" % (_, re.sub('[^\w\-_.]', '_', _.element_identifier), _.ext) for _ in $param_list_optional_oneformat if _ ]) } && +#if $param_list_optional_oneformat: + ${ ' && '.join([ "ln -s '%s' '%s.%s'" % (_, re.sub('[^\w\-_.]', '_', _.element_identifier), _.ext) for _ in $param_list_optional_oneformat if _ ]) } && +#end if ${ ' && '.join([ "ln -s '%s' '%s.%s'" % (_, re.sub('[^\w\-_.]', '_', _.element_identifier), _.ext) for _ in $param_list_mandatory_multformat if _ ]) } && -${ ' && '.join([ "ln -s '%s' '%s.%s'" % (_, re.sub('[^\w\-_.]', '_', _.element_identifier), _.ext) for _ in $param_list_optional_multformat if _ ]) } && -${ ' && '.join([ "ln -s '%s' '%s.%s'" % (_, re.sub('[^\w\-_.]', '_', _.element_identifier), _.ext) for _ in $param_list_default_mandatory_noformat if _ ]) } && -${ ' && '.join([ "ln -s '%s' '%s.%s'" % (_, re.sub('[^\w\-_.]', '_', _.element_identifier), _.ext) for _ in $param_list_default_optional_noformat if _ ]) } && -${ ' && '.join([ "ln -s '%s' '%s.%s'" % (_, re.sub('[^\w\-_.]', '_', _.element_identifier), _.ext) for _ in $param_list_default_mandatory_oneformat if _ ]) } && -${ ' && '.join([ "ln -s '%s' '%s.%s'" % (_, re.sub('[^\w\-_.]', '_', _.element_identifier), _.ext) for _ in $param_list_default_optional_oneformat if _ ]) } && -${ ' && '.join([ "ln -s '%s' '%s.%s'" % (_, re.sub('[^\w\-_.]', '_', _.element_identifier), _.ext) for _ in $param_list_default_mandatory_multformat if _ ]) } && -${ ' && '.join([ "ln -s '%s' '%s.%s'" % (_, re.sub('[^\w\-_.]', '_', _.element_identifier), _.ext) for _ in $param_list_default_optional_multformat if _ ]) } && +#if $param_list_optional_multformat: + ${ ' && '.join([ "ln -s '%s' '%s.%s'" % (_, re.sub('[^\w\-_.]', '_', _.element_identifier), _.ext) for _ in $param_list_optional_multformat if _ ]) } && +#end if +#if $param_list_default_mandatory_noformat: + ${ ' && '.join([ "ln -s '%s' '%s.%s'" % (_, re.sub('[^\w\-_.]', '_', _.element_identifier), _.ext) for _ in $param_list_default_mandatory_noformat if _ ]) } && +#end if +#if $param_list_default_optional_noformat: + ${ ' && '.join([ "ln -s '%s' '%s.%s'" % (_, re.sub('[^\w\-_.]', '_', _.element_identifier), _.ext) for _ in $param_list_default_optional_noformat if _ ]) } && +#end if +#if $param_list_default_mandatory_oneformat: + ${ ' && '.join([ "ln -s '%s' '%s.%s'" % (_, re.sub('[^\w\-_.]', '_', _.element_identifier), _.ext) for _ in $param_list_default_mandatory_oneformat if _ ]) } && +#end if +#if $param_list_default_optional_oneformat: + ${ ' && '.join([ "ln -s '%s' '%s.%s'" % (_, re.sub('[^\w\-_.]', '_', _.element_identifier), _.ext) for _ in $param_list_default_optional_oneformat if _ ]) } && +#end if +#if $param_list_default_mandatory_multformat: + ${ ' && '.join([ "ln -s '%s' '%s.%s'" % (_, re.sub('[^\w\-_.]', '_', _.element_identifier), _.ext) for _ in $param_list_default_mandatory_multformat if _ ]) } && +#end if +#if $param_list_default_optional_multformat: + ${ ' && '.join([ "ln -s '%s' '%s.%s'" % (_, re.sub('[^\w\-_.]', '_', _.element_identifier), _.ext) for _ in $param_list_default_optional_multformat if _ ]) } && +#end if #if $adv_opts_cond.adv_opts_selector=='advanced': ln -s '$adv_opts_cond.param_mandatory_noformat_adv' '${re.sub("[^\w\-_.]", "_", $adv_opts_cond.param_mandatory_noformat_adv.element_identifier)}.${adv_opts_cond.param_mandatory_noformat_adv.ext}' && #if $adv_opts_cond.param_optional_noformat_adv: @@ -60,30 +84,54 @@ ${ ' && '.join([ "ln -s '%s' '%s.%s'" % (_, re.sub('[^\w\-_.]', '_', _.element_i #if $adv_opts_cond.param_optional_multformat_adv: ln -s '$adv_opts_cond.param_optional_multformat_adv' '${re.sub("[^\w\-_.]", "_", $adv_opts_cond.param_optional_multformat_adv.element_identifier)}.${adv_opts_cond.param_optional_multformat_adv.ext}' && #end if - ln -s '$adv_opts_cond.param_default_mandatory_noformat_adv' '${re.sub("[^\w\-_.]", "_", $adv_opts_cond.param_default_mandatory_noformat_adv.element_identifier)}.${adv_opts_cond.param_default_mandatory_noformat_adv.ext}' && + #if $adv_opts_cond.param_default_mandatory_noformat_adv: + ln -s '$adv_opts_cond.param_default_mandatory_noformat_adv' '${re.sub("[^\w\-_.]", "_", $adv_opts_cond.param_default_mandatory_noformat_adv.element_identifier)}.${adv_opts_cond.param_default_mandatory_noformat_adv.ext}' && + #end if #if $adv_opts_cond.param_default_optional_noformat_adv: ln -s '$adv_opts_cond.param_default_optional_noformat_adv' '${re.sub("[^\w\-_.]", "_", $adv_opts_cond.param_default_optional_noformat_adv.element_identifier)}.${adv_opts_cond.param_default_optional_noformat_adv.ext}' && #end if - ln -s '$adv_opts_cond.param_default_mandatory_oneformat_adv' '${re.sub("[^\w\-_.]", "_", $adv_opts_cond.param_default_mandatory_oneformat_adv.element_identifier)}.${adv_opts_cond.param_default_mandatory_oneformat_adv.ext}' && + #if $adv_opts_cond.param_default_mandatory_oneformat_adv: + ln -s '$adv_opts_cond.param_default_mandatory_oneformat_adv' '${re.sub("[^\w\-_.]", "_", $adv_opts_cond.param_default_mandatory_oneformat_adv.element_identifier)}.${adv_opts_cond.param_default_mandatory_oneformat_adv.ext}' && + #end if #if $adv_opts_cond.param_default_optional_oneformat_adv: ln -s '$adv_opts_cond.param_default_optional_oneformat_adv' '${re.sub("[^\w\-_.]", "_", $adv_opts_cond.param_default_optional_oneformat_adv.element_identifier)}.${adv_opts_cond.param_default_optional_oneformat_adv.ext}' && #end if - ln -s '$adv_opts_cond.param_default_mandatory_multformat_adv' '${re.sub("[^\w\-_.]", "_", $adv_opts_cond.param_default_mandatory_multformat_adv.element_identifier)}.${adv_opts_cond.param_default_mandatory_multformat_adv.ext}' && + #if $adv_opts_cond.param_default_mandatory_multformat_adv: + ln -s '$adv_opts_cond.param_default_mandatory_multformat_adv' '${re.sub("[^\w\-_.]", "_", $adv_opts_cond.param_default_mandatory_multformat_adv.element_identifier)}.${adv_opts_cond.param_default_mandatory_multformat_adv.ext}' && + #end if #if $adv_opts_cond.param_default_optional_multformat_adv: ln -s '$adv_opts_cond.param_default_optional_multformat_adv' '${re.sub("[^\w\-_.]", "_", $adv_opts_cond.param_default_optional_multformat_adv.element_identifier)}.${adv_opts_cond.param_default_optional_multformat_adv.ext}' && #end if ${ ' && '.join([ "ln -s '%s' '%s.%s'" % (_, re.sub('[^\w\-_.]', '_', _.element_identifier), _.ext) for _ in $adv_opts_cond.param_list_mandatory_noformat_adv if _ ]) } && - ${ ' && '.join([ "ln -s '%s' '%s.%s'" % (_, re.sub('[^\w\-_.]', '_', _.element_identifier), _.ext) for _ in $adv_opts_cond.param_list_optional_noformat_adv if _ ]) } && + #if $adv_opts_cond.param_list_optional_noformat_adv: + ${ ' && '.join([ "ln -s '%s' '%s.%s'" % (_, re.sub('[^\w\-_.]', '_', _.element_identifier), _.ext) for _ in $adv_opts_cond.param_list_optional_noformat_adv if _ ]) } && + #end if ${ ' && '.join([ "ln -s '%s' '%s.%s'" % (_, re.sub('[^\w\-_.]', '_', _.element_identifier), _.ext) for _ in $adv_opts_cond.param_list_mandatory_oneformat_adv if _ ]) } && - ${ ' && '.join([ "ln -s '%s' '%s.%s'" % (_, re.sub('[^\w\-_.]', '_', _.element_identifier), _.ext) for _ in $adv_opts_cond.param_list_optional_oneformat_adv if _ ]) } && + #if $adv_opts_cond.param_list_optional_oneformat_adv: + ${ ' && '.join([ "ln -s '%s' '%s.%s'" % (_, re.sub('[^\w\-_.]', '_', _.element_identifier), _.ext) for _ in $adv_opts_cond.param_list_optional_oneformat_adv if _ ]) } && + #end if ${ ' && '.join([ "ln -s '%s' '%s.%s'" % (_, re.sub('[^\w\-_.]', '_', _.element_identifier), _.ext) for _ in $adv_opts_cond.param_list_mandatory_multformat_adv if _ ]) } && - ${ ' && '.join([ "ln -s '%s' '%s.%s'" % (_, re.sub('[^\w\-_.]', '_', _.element_identifier), _.ext) for _ in $adv_opts_cond.param_list_optional_multformat_adv if _ ]) } && - ${ ' && '.join([ "ln -s '%s' '%s.%s'" % (_, re.sub('[^\w\-_.]', '_', _.element_identifier), _.ext) for _ in $adv_opts_cond.param_list_default_mandatory_noformat_adv if _ ]) } && - ${ ' && '.join([ "ln -s '%s' '%s.%s'" % (_, re.sub('[^\w\-_.]', '_', _.element_identifier), _.ext) for _ in $adv_opts_cond.param_list_default_optional_noformat_adv if _ ]) } && - ${ ' && '.join([ "ln -s '%s' '%s.%s'" % (_, re.sub('[^\w\-_.]', '_', _.element_identifier), _.ext) for _ in $adv_opts_cond.param_list_default_mandatory_oneformat_adv if _ ]) } && - ${ ' && '.join([ "ln -s '%s' '%s.%s'" % (_, re.sub('[^\w\-_.]', '_', _.element_identifier), _.ext) for _ in $adv_opts_cond.param_list_default_optional_oneformat_adv if _ ]) } && - ${ ' && '.join([ "ln -s '%s' '%s.%s'" % (_, re.sub('[^\w\-_.]', '_', _.element_identifier), _.ext) for _ in $adv_opts_cond.param_list_default_mandatory_multformat_adv if _ ]) } && - ${ ' && '.join([ "ln -s '%s' '%s.%s'" % (_, re.sub('[^\w\-_.]', '_', _.element_identifier), _.ext) for _ in $adv_opts_cond.param_list_default_optional_multformat_adv if _ ]) } && + #if $adv_opts_cond.param_list_optional_multformat_adv: + ${ ' && '.join([ "ln -s '%s' '%s.%s'" % (_, re.sub('[^\w\-_.]', '_', _.element_identifier), _.ext) for _ in $adv_opts_cond.param_list_optional_multformat_adv if _ ]) } && + #end if + #if $adv_opts_cond.param_list_default_mandatory_noformat_adv: + ${ ' && '.join([ "ln -s '%s' '%s.%s'" % (_, re.sub('[^\w\-_.]', '_', _.element_identifier), _.ext) for _ in $adv_opts_cond.param_list_default_mandatory_noformat_adv if _ ]) } && + #end if + #if $adv_opts_cond.param_list_default_optional_noformat_adv: + ${ ' && '.join([ "ln -s '%s' '%s.%s'" % (_, re.sub('[^\w\-_.]', '_', _.element_identifier), _.ext) for _ in $adv_opts_cond.param_list_default_optional_noformat_adv if _ ]) } && + #end if + #if $adv_opts_cond.param_list_default_mandatory_oneformat_adv: + ${ ' && '.join([ "ln -s '%s' '%s.%s'" % (_, re.sub('[^\w\-_.]', '_', _.element_identifier), _.ext) for _ in $adv_opts_cond.param_list_default_mandatory_oneformat_adv if _ ]) } && + #end if + #if $adv_opts_cond.param_list_default_optional_oneformat_adv: + ${ ' && '.join([ "ln -s '%s' '%s.%s'" % (_, re.sub('[^\w\-_.]', '_', _.element_identifier), _.ext) for _ in $adv_opts_cond.param_list_default_optional_oneformat_adv if _ ]) } && + #end if + #if $adv_opts_cond.param_list_default_mandatory_multformat_adv: + ${ ' && '.join([ "ln -s '%s' '%s.%s'" % (_, re.sub('[^\w\-_.]', '_', _.element_identifier), _.ext) for _ in $adv_opts_cond.param_list_default_mandatory_multformat_adv if _ ]) } && + #end if + #if $adv_opts_cond.param_list_default_optional_multformat_adv: + ${ ' && '.join([ "ln -s '%s' '%s.%s'" % (_, re.sub('[^\w\-_.]', '_', _.element_identifier), _.ext) for _ in $adv_opts_cond.param_list_default_optional_multformat_adv if _ ]) } && + #end if #end if @EXECUTABLE@ @@ -99,30 +147,54 @@ ${ ' && '.join([ "ln -s '%s' '%s.%s'" % (_, re.sub('[^\w\-_.]', '_', _.element_i #if $param_optional_multformat: -optional_multformat '${re.sub('[^\w\-_.]', '_', $param_optional_multformat.element_identifier)}.${param_optional_multformat.ext}' #end if --default_mandatory_noformat '${re.sub('[^\w\-_.]', '_', $param_default_mandatory_noformat.element_identifier)}.${param_default_mandatory_noformat.ext}' +#if $param_default_mandatory_noformat: + -default_mandatory_noformat '${re.sub('[^\w\-_.]', '_', $param_default_mandatory_noformat.element_identifier)}.${param_default_mandatory_noformat.ext}' +#end if #if $param_default_optional_noformat: -default_optional_noformat '${re.sub('[^\w\-_.]', '_', $param_default_optional_noformat.element_identifier)}.${param_default_optional_noformat.ext}' #end if --default_mandatory_oneformat '${re.sub('[^\w\-_.]', '_', $param_default_mandatory_oneformat.element_identifier)}.${param_default_mandatory_oneformat.ext}' +#if $param_default_mandatory_oneformat: + -default_mandatory_oneformat '${re.sub('[^\w\-_.]', '_', $param_default_mandatory_oneformat.element_identifier)}.${param_default_mandatory_oneformat.ext}' +#end if #if $param_default_optional_oneformat: -default_optional_oneformat '${re.sub('[^\w\-_.]', '_', $param_default_optional_oneformat.element_identifier)}.${param_default_optional_oneformat.ext}' #end if --default_mandatory_multformat '${re.sub('[^\w\-_.]', '_', $param_default_mandatory_multformat.element_identifier)}.${param_default_mandatory_multformat.ext}' +#if $param_default_mandatory_multformat: + -default_mandatory_multformat '${re.sub('[^\w\-_.]', '_', $param_default_mandatory_multformat.element_identifier)}.${param_default_mandatory_multformat.ext}' +#end if #if $param_default_optional_multformat: -default_optional_multformat '${re.sub('[^\w\-_.]', '_', $param_default_optional_multformat.element_identifier)}.${param_default_optional_multformat.ext}' #end if -list_mandatory_noformat ${' '.join(["'%s.%s'"%(re.sub('[^\w\-_.]', '_', _.element_identifier),_.ext) for _ in $param_list_mandatory_noformat if _ ])} --list_optional_noformat ${' '.join(["'%s.%s'"%(re.sub('[^\w\-_.]', '_', _.element_identifier),_.ext) for _ in $param_list_optional_noformat if _ ])} +#if $param_list_optional_noformat: + -list_optional_noformat ${' '.join(["'%s.%s'"%(re.sub('[^\w\-_.]', '_', _.element_identifier),_.ext) for _ in $param_list_optional_noformat if _ ])} +#end if -list_mandatory_oneformat ${' '.join(["'%s.%s'"%(re.sub('[^\w\-_.]', '_', _.element_identifier),_.ext) for _ in $param_list_mandatory_oneformat if _ ])} --list_optional_oneformat ${' '.join(["'%s.%s'"%(re.sub('[^\w\-_.]', '_', _.element_identifier),_.ext) for _ in $param_list_optional_oneformat if _ ])} +#if $param_list_optional_oneformat: + -list_optional_oneformat ${' '.join(["'%s.%s'"%(re.sub('[^\w\-_.]', '_', _.element_identifier),_.ext) for _ in $param_list_optional_oneformat if _ ])} +#end if -list_mandatory_multformat ${' '.join(["'%s.%s'"%(re.sub('[^\w\-_.]', '_', _.element_identifier),_.ext) for _ in $param_list_mandatory_multformat if _ ])} --list_optional_multformat ${' '.join(["'%s.%s'"%(re.sub('[^\w\-_.]', '_', _.element_identifier),_.ext) for _ in $param_list_optional_multformat if _ ])} --list_default_mandatory_noformat ${' '.join(["'%s.%s'"%(re.sub('[^\w\-_.]', '_', _.element_identifier),_.ext) for _ in $param_list_default_mandatory_noformat if _ ])} --list_default_optional_noformat ${' '.join(["'%s.%s'"%(re.sub('[^\w\-_.]', '_', _.element_identifier),_.ext) for _ in $param_list_default_optional_noformat if _ ])} --list_default_mandatory_oneformat ${' '.join(["'%s.%s'"%(re.sub('[^\w\-_.]', '_', _.element_identifier),_.ext) for _ in $param_list_default_mandatory_oneformat if _ ])} --list_default_optional_oneformat ${' '.join(["'%s.%s'"%(re.sub('[^\w\-_.]', '_', _.element_identifier),_.ext) for _ in $param_list_default_optional_oneformat if _ ])} --list_default_mandatory_multformat ${' '.join(["'%s.%s'"%(re.sub('[^\w\-_.]', '_', _.element_identifier),_.ext) for _ in $param_list_default_mandatory_multformat if _ ])} --list_default_optional_multformat ${' '.join(["'%s.%s'"%(re.sub('[^\w\-_.]', '_', _.element_identifier),_.ext) for _ in $param_list_default_optional_multformat if _ ])} +#if $param_list_optional_multformat: + -list_optional_multformat ${' '.join(["'%s.%s'"%(re.sub('[^\w\-_.]', '_', _.element_identifier),_.ext) for _ in $param_list_optional_multformat if _ ])} +#end if +#if $param_list_default_mandatory_noformat: + -list_default_mandatory_noformat ${' '.join(["'%s.%s'"%(re.sub('[^\w\-_.]', '_', _.element_identifier),_.ext) for _ in $param_list_default_mandatory_noformat if _ ])} +#end if +#if $param_list_default_optional_noformat: + -list_default_optional_noformat ${' '.join(["'%s.%s'"%(re.sub('[^\w\-_.]', '_', _.element_identifier),_.ext) for _ in $param_list_default_optional_noformat if _ ])} +#end if +#if $param_list_default_mandatory_oneformat: + -list_default_mandatory_oneformat ${' '.join(["'%s.%s'"%(re.sub('[^\w\-_.]', '_', _.element_identifier),_.ext) for _ in $param_list_default_mandatory_oneformat if _ ])} +#end if +#if $param_list_default_optional_oneformat: + -list_default_optional_oneformat ${' '.join(["'%s.%s'"%(re.sub('[^\w\-_.]', '_', _.element_identifier),_.ext) for _ in $param_list_default_optional_oneformat if _ ])} +#end if +#if $param_list_default_mandatory_multformat: + -list_default_mandatory_multformat ${' '.join(["'%s.%s'"%(re.sub('[^\w\-_.]', '_', _.element_identifier),_.ext) for _ in $param_list_default_mandatory_multformat if _ ])} +#end if +#if $param_list_default_optional_multformat: + -list_default_optional_multformat ${' '.join(["'%s.%s'"%(re.sub('[^\w\-_.]', '_', _.element_identifier),_.ext) for _ in $param_list_default_optional_multformat if _ ])} +#end if #if $adv_opts_cond.adv_opts_selector=='advanced': -mandatory_noformat_adv '${re.sub('[^\w\-_.]', '_', $adv_opts_cond.param_mandatory_noformat_adv.element_identifier)}.${adv_opts_cond.param_mandatory_noformat_adv.ext}' #if $adv_opts_cond.param_optional_noformat_adv: @@ -136,30 +208,54 @@ ${ ' && '.join([ "ln -s '%s' '%s.%s'" % (_, re.sub('[^\w\-_.]', '_', _.element_i #if $adv_opts_cond.param_optional_multformat_adv: -optional_multformat_adv '${re.sub('[^\w\-_.]', '_', $adv_opts_cond.param_optional_multformat_adv.element_identifier)}.${adv_opts_cond.param_optional_multformat_adv.ext}' #end if - -default_mandatory_noformat_adv '${re.sub('[^\w\-_.]', '_', $adv_opts_cond.param_default_mandatory_noformat_adv.element_identifier)}.${adv_opts_cond.param_default_mandatory_noformat_adv.ext}' + #if $adv_opts_cond.param_default_mandatory_noformat_adv: + -default_mandatory_noformat_adv '${re.sub('[^\w\-_.]', '_', $adv_opts_cond.param_default_mandatory_noformat_adv.element_identifier)}.${adv_opts_cond.param_default_mandatory_noformat_adv.ext}' + #end if #if $adv_opts_cond.param_default_optional_noformat_adv: -default_optional_noformat_adv '${re.sub('[^\w\-_.]', '_', $adv_opts_cond.param_default_optional_noformat_adv.element_identifier)}.${adv_opts_cond.param_default_optional_noformat_adv.ext}' #end if - -default_mandatory_oneformat_adv '${re.sub('[^\w\-_.]', '_', $adv_opts_cond.param_default_mandatory_oneformat_adv.element_identifier)}.${adv_opts_cond.param_default_mandatory_oneformat_adv.ext}' + #if $adv_opts_cond.param_default_mandatory_oneformat_adv: + -default_mandatory_oneformat_adv '${re.sub('[^\w\-_.]', '_', $adv_opts_cond.param_default_mandatory_oneformat_adv.element_identifier)}.${adv_opts_cond.param_default_mandatory_oneformat_adv.ext}' + #end if #if $adv_opts_cond.param_default_optional_oneformat_adv: -default_optional_oneformat_adv '${re.sub('[^\w\-_.]', '_', $adv_opts_cond.param_default_optional_oneformat_adv.element_identifier)}.${adv_opts_cond.param_default_optional_oneformat_adv.ext}' #end if - -default_mandatory_multformat_adv '${re.sub('[^\w\-_.]', '_', $adv_opts_cond.param_default_mandatory_multformat_adv.element_identifier)}.${adv_opts_cond.param_default_mandatory_multformat_adv.ext}' + #if $adv_opts_cond.param_default_mandatory_multformat_adv: + -default_mandatory_multformat_adv '${re.sub('[^\w\-_.]', '_', $adv_opts_cond.param_default_mandatory_multformat_adv.element_identifier)}.${adv_opts_cond.param_default_mandatory_multformat_adv.ext}' + #end if #if $adv_opts_cond.param_default_optional_multformat_adv: -default_optional_multformat_adv '${re.sub('[^\w\-_.]', '_', $adv_opts_cond.param_default_optional_multformat_adv.element_identifier)}.${adv_opts_cond.param_default_optional_multformat_adv.ext}' #end if -list_mandatory_noformat_adv ${' '.join(["'%s.%s'"%(re.sub('[^\w\-_.]', '_', _.element_identifier),_.ext) for _ in $adv_opts_cond.param_list_mandatory_noformat_adv if _ ])} - -list_optional_noformat_adv ${' '.join(["'%s.%s'"%(re.sub('[^\w\-_.]', '_', _.element_identifier),_.ext) for _ in $adv_opts_cond.param_list_optional_noformat_adv if _ ])} + #if $adv_opts_cond.param_list_optional_noformat_adv: + -list_optional_noformat_adv ${' '.join(["'%s.%s'"%(re.sub('[^\w\-_.]', '_', _.element_identifier),_.ext) for _ in $adv_opts_cond.param_list_optional_noformat_adv if _ ])} + #end if -list_mandatory_oneformat_adv ${' '.join(["'%s.%s'"%(re.sub('[^\w\-_.]', '_', _.element_identifier),_.ext) for _ in $adv_opts_cond.param_list_mandatory_oneformat_adv if _ ])} - -list_optional_oneformat_adv ${' '.join(["'%s.%s'"%(re.sub('[^\w\-_.]', '_', _.element_identifier),_.ext) for _ in $adv_opts_cond.param_list_optional_oneformat_adv if _ ])} + #if $adv_opts_cond.param_list_optional_oneformat_adv: + -list_optional_oneformat_adv ${' '.join(["'%s.%s'"%(re.sub('[^\w\-_.]', '_', _.element_identifier),_.ext) for _ in $adv_opts_cond.param_list_optional_oneformat_adv if _ ])} + #end if -list_mandatory_multformat_adv ${' '.join(["'%s.%s'"%(re.sub('[^\w\-_.]', '_', _.element_identifier),_.ext) for _ in $adv_opts_cond.param_list_mandatory_multformat_adv if _ ])} - -list_optional_multformat_adv ${' '.join(["'%s.%s'"%(re.sub('[^\w\-_.]', '_', _.element_identifier),_.ext) for _ in $adv_opts_cond.param_list_optional_multformat_adv if _ ])} - -list_default_mandatory_noformat_adv ${' '.join(["'%s.%s'"%(re.sub('[^\w\-_.]', '_', _.element_identifier),_.ext) for _ in $adv_opts_cond.param_list_default_mandatory_noformat_adv if _ ])} - -list_default_optional_noformat_adv ${' '.join(["'%s.%s'"%(re.sub('[^\w\-_.]', '_', _.element_identifier),_.ext) for _ in $adv_opts_cond.param_list_default_optional_noformat_adv if _ ])} - -list_default_mandatory_oneformat_adv ${' '.join(["'%s.%s'"%(re.sub('[^\w\-_.]', '_', _.element_identifier),_.ext) for _ in $adv_opts_cond.param_list_default_mandatory_oneformat_adv if _ ])} - -list_default_optional_oneformat_adv ${' '.join(["'%s.%s'"%(re.sub('[^\w\-_.]', '_', _.element_identifier),_.ext) for _ in $adv_opts_cond.param_list_default_optional_oneformat_adv if _ ])} - -list_default_mandatory_multformat_adv ${' '.join(["'%s.%s'"%(re.sub('[^\w\-_.]', '_', _.element_identifier),_.ext) for _ in $adv_opts_cond.param_list_default_mandatory_multformat_adv if _ ])} - -list_default_optional_multformat_adv ${' '.join(["'%s.%s'"%(re.sub('[^\w\-_.]', '_', _.element_identifier),_.ext) for _ in $adv_opts_cond.param_list_default_optional_multformat_adv if _ ])} + #if $adv_opts_cond.param_list_optional_multformat_adv: + -list_optional_multformat_adv ${' '.join(["'%s.%s'"%(re.sub('[^\w\-_.]', '_', _.element_identifier),_.ext) for _ in $adv_opts_cond.param_list_optional_multformat_adv if _ ])} + #end if + #if $adv_opts_cond.param_list_default_mandatory_noformat_adv: + -list_default_mandatory_noformat_adv ${' '.join(["'%s.%s'"%(re.sub('[^\w\-_.]', '_', _.element_identifier),_.ext) for _ in $adv_opts_cond.param_list_default_mandatory_noformat_adv if _ ])} + #end if + #if $adv_opts_cond.param_list_default_optional_noformat_adv: + -list_default_optional_noformat_adv ${' '.join(["'%s.%s'"%(re.sub('[^\w\-_.]', '_', _.element_identifier),_.ext) for _ in $adv_opts_cond.param_list_default_optional_noformat_adv if _ ])} + #end if + #if $adv_opts_cond.param_list_default_mandatory_oneformat_adv: + -list_default_mandatory_oneformat_adv ${' '.join(["'%s.%s'"%(re.sub('[^\w\-_.]', '_', _.element_identifier),_.ext) for _ in $adv_opts_cond.param_list_default_mandatory_oneformat_adv if _ ])} + #end if + #if $adv_opts_cond.param_list_default_optional_oneformat_adv: + -list_default_optional_oneformat_adv ${' '.join(["'%s.%s'"%(re.sub('[^\w\-_.]', '_', _.element_identifier),_.ext) for _ in $adv_opts_cond.param_list_default_optional_oneformat_adv if _ ])} + #end if + #if $adv_opts_cond.param_list_default_mandatory_multformat_adv: + -list_default_mandatory_multformat_adv ${' '.join(["'%s.%s'"%(re.sub('[^\w\-_.]', '_', _.element_identifier),_.ext) for _ in $adv_opts_cond.param_list_default_mandatory_multformat_adv if _ ])} + #end if + #if $adv_opts_cond.param_list_default_optional_multformat_adv: + -list_default_optional_multformat_adv ${' '.join(["'%s.%s'"%(re.sub('[^\w\-_.]', '_', _.element_identifier),_.ext) for _ in $adv_opts_cond.param_list_default_optional_multformat_adv if _ ])} + #end if #end if > $param_stdout ]]> @@ -170,11 +266,11 @@ ${ ' && '.join([ "ln -s '%s' '%s.%s'" % (_, re.sub('[^\w\-_.]', '_', _.element_i - + - + - + @@ -182,11 +278,11 @@ ${ ' && '.join([ "ln -s '%s' '%s.%s'" % (_, re.sub('[^\w\-_.]', '_', _.element_i - + - + - + @@ -195,11 +291,11 @@ ${ ' && '.join([ "ln -s '%s' '%s.%s'" % (_, re.sub('[^\w\-_.]', '_', _.element_i - + - + - + @@ -207,11 +303,11 @@ ${ ' && '.join([ "ln -s '%s' '%s.%s'" % (_, re.sub('[^\w\-_.]', '_', _.element_i - + - + - + @@ -220,31 +316,19 @@ ${ ' && '.join([ "ln -s '%s' '%s.%s'" % (_, re.sub('[^\w\-_.]', '_', _.element_i - - - - - - - - - - - - + + + + + + - - - - - - - - - - - - + + + + + + diff --git a/tests/test-data/ofile-mult.xml b/tests/test-data/ofile-mult.xml index 2886c808..74f3b727 100644 --- a/tests/test-data/ofile-mult.xml +++ b/tests/test-data/ofile-mult.xml @@ -17,8 +17,12 @@ mkdir param_out_mandatory_optionalinput && mkdir param_out_optional_optionalinput && ${ ' && '.join([ "ln -s '%s' '%s.%s'" % (_, re.sub('[^\w\-_.]', '_', _.element_identifier), _.ext) for _ in $param_inp_mandatory_mandatoryinput if _ ]) } && ${ ' && '.join([ "ln -s '%s' '%s.%s'" % (_, re.sub('[^\w\-_.]', '_', _.element_identifier), _.ext) for _ in $param_inp_optional_mandatoryinput if _ ]) } && -${ ' && '.join([ "ln -s '%s' '%s.%s'" % (_, re.sub('[^\w\-_.]', '_', _.element_identifier), _.ext) for _ in $param_inp_mandatory_optionalinput if _ ]) } && -${ ' && '.join([ "ln -s '%s' '%s.%s'" % (_, re.sub('[^\w\-_.]', '_', _.element_identifier), _.ext) for _ in $param_inp_optional_optionalinput if _ ]) } && +#if $param_inp_mandatory_optionalinput: + ${ ' && '.join([ "ln -s '%s' '%s.%s'" % (_, re.sub('[^\w\-_.]', '_', _.element_identifier), _.ext) for _ in $param_inp_mandatory_optionalinput if _ ]) } && +#end if +#if $param_inp_optional_optionalinput: + ${ ' && '.join([ "ln -s '%s' '%s.%s'" % (_, re.sub('[^\w\-_.]', '_', _.element_identifier), _.ext) for _ in $param_inp_optional_optionalinput if _ ]) } && +#end if @EXECUTABLE@ -mandatory_mandatoryinput ${' '.join(["'param_out_mandatory_mandatoryinput/%s.%s'"%(re.sub('[^\w\-_.]', '_', _.element_identifier),_.ext) for _ in $param_inp_mandatory_mandatoryinput if _ ])} @@ -31,8 +35,12 @@ ${ ' && '.join([ "ln -s '%s' '%s.%s'" % (_, re.sub('[^\w\-_.]', '_', _.element_i #end if -inp_mandatory_mandatoryinput ${' '.join(["'%s.%s'"%(re.sub('[^\w\-_.]', '_', _.element_identifier),_.ext) for _ in $param_inp_mandatory_mandatoryinput if _ ])} -inp_optional_mandatoryinput ${' '.join(["'%s.%s'"%(re.sub('[^\w\-_.]', '_', _.element_identifier),_.ext) for _ in $param_inp_optional_mandatoryinput if _ ])} --inp_mandatory_optionalinput ${' '.join(["'%s.%s'"%(re.sub('[^\w\-_.]', '_', _.element_identifier),_.ext) for _ in $param_inp_mandatory_optionalinput if _ ])} --inp_optional_optionalinput ${' '.join(["'%s.%s'"%(re.sub('[^\w\-_.]', '_', _.element_identifier),_.ext) for _ in $param_inp_optional_optionalinput if _ ])} +#if $param_inp_mandatory_optionalinput: + -inp_mandatory_optionalinput ${' '.join(["'%s.%s'"%(re.sub('[^\w\-_.]', '_', _.element_identifier),_.ext) for _ in $param_inp_mandatory_optionalinput if _ ])} +#end if +#if $param_inp_optional_optionalinput: + -inp_optional_optionalinput ${' '.join(["'%s.%s'"%(re.sub('[^\w\-_.]', '_', _.element_identifier),_.ext) for _ in $param_inp_optional_optionalinput if _ ])} +#end if ]]> @@ -62,8 +70,8 @@ ${ ' && '.join([ "ln -s '%s' '%s.%s'" % (_, re.sub('[^\w\-_.]', '_', _.element_i - - + + diff --git a/tests/test-data/repeat.xml b/tests/test-data/repeat.xml index 2fc6e323..66986a90 100644 --- a/tests/test-data/repeat.xml +++ b/tests/test-data/repeat.xml @@ -101,7 +101,6 @@ - @@ -109,7 +108,6 @@ - @@ -117,7 +115,6 @@ - @@ -125,7 +122,6 @@ - @@ -326,7 +322,6 @@ - @@ -334,7 +329,6 @@ - @@ -342,7 +336,6 @@ - @@ -350,7 +343,6 @@ - diff --git a/tests/test-data/select.xml b/tests/test-data/select.xml index 7c10615e..18faf91c 100644 --- a/tests/test-data/select.xml +++ b/tests/test-data/select.xml @@ -12,21 +12,21 @@ B + + + + + + + + + + + + + + + + + + + + @@ -111,6 +131,11 @@ + + + + + @@ -118,6 +143,11 @@ + + + + + @@ -125,6 +155,11 @@ + + + + + @@ -132,30 +167,55 @@ + + + + + + + + + + + + + + + + + + + + + + + + + @@ -163,6 +223,11 @@ + + + + + @@ -170,6 +235,11 @@ + + + + + @@ -177,6 +247,11 @@ + + + + + @@ -184,6 +259,11 @@ + + + + + @@ -191,24 +271,44 @@ + + + + + + + + + + + + + + + + + + + + @@ -216,6 +316,11 @@ + + + + + @@ -223,6 +328,11 @@ + + + + + @@ -230,6 +340,11 @@ + + + + + @@ -237,30 +352,55 @@ + + + + + + + + + + + + + + + + + + + + + + + + + @@ -268,6 +408,11 @@ + + + + + @@ -275,6 +420,11 @@ + + + + + @@ -282,6 +432,11 @@ + + + + + @@ -289,6 +444,11 @@ + + + + + diff --git a/tests/test-data/string.xml b/tests/test-data/string.xml index 076520dc..8edbceba 100644 --- a/tests/test-data/string.xml +++ b/tests/test-data/string.xml @@ -37,7 +37,6 @@ - @@ -45,7 +44,6 @@ - @@ -53,7 +51,6 @@ - @@ -61,7 +58,6 @@ - @@ -70,7 +66,6 @@ - @@ -78,7 +73,6 @@ - @@ -86,7 +80,6 @@ - @@ -94,7 +87,6 @@ - diff --git a/tests/test-data/test-data/0test.ext b/tests/test-data/test-data/0test.ext new file mode 120000 index 00000000..769e1336 --- /dev/null +++ b/tests/test-data/test-data/0test.ext @@ -0,0 +1 @@ +test.ext \ No newline at end of file diff --git a/tests/test-data/test-data/0test2.ext b/tests/test-data/test-data/0test2.ext new file mode 120000 index 00000000..f6d8b812 --- /dev/null +++ b/tests/test-data/test-data/0test2.ext @@ -0,0 +1 @@ +test2.ext \ No newline at end of file diff --git a/tests/test-data/test-data/10test.ext b/tests/test-data/test-data/10test.ext new file mode 120000 index 00000000..769e1336 --- /dev/null +++ b/tests/test-data/test-data/10test.ext @@ -0,0 +1 @@ +test.ext \ No newline at end of file diff --git a/tests/test-data/test-data/10test2.ext b/tests/test-data/test-data/10test2.ext new file mode 120000 index 00000000..f6d8b812 --- /dev/null +++ b/tests/test-data/test-data/10test2.ext @@ -0,0 +1 @@ +test2.ext \ No newline at end of file diff --git a/tests/test-data/test-data/11test.ext b/tests/test-data/test-data/11test.ext new file mode 120000 index 00000000..769e1336 --- /dev/null +++ b/tests/test-data/test-data/11test.ext @@ -0,0 +1 @@ +test.ext \ No newline at end of file diff --git a/tests/test-data/test-data/11test2.ext b/tests/test-data/test-data/11test2.ext new file mode 120000 index 00000000..f6d8b812 --- /dev/null +++ b/tests/test-data/test-data/11test2.ext @@ -0,0 +1 @@ +test2.ext \ No newline at end of file diff --git a/tests/test-data/test-data/1test.ext b/tests/test-data/test-data/1test.ext new file mode 120000 index 00000000..769e1336 --- /dev/null +++ b/tests/test-data/test-data/1test.ext @@ -0,0 +1 @@ +test.ext \ No newline at end of file diff --git a/tests/test-data/test-data/1test2.ext b/tests/test-data/test-data/1test2.ext new file mode 120000 index 00000000..f6d8b812 --- /dev/null +++ b/tests/test-data/test-data/1test2.ext @@ -0,0 +1 @@ +test2.ext \ No newline at end of file diff --git a/tests/test-data/test-data/2test.ext b/tests/test-data/test-data/2test.ext new file mode 120000 index 00000000..769e1336 --- /dev/null +++ b/tests/test-data/test-data/2test.ext @@ -0,0 +1 @@ +test.ext \ No newline at end of file diff --git a/tests/test-data/test-data/2test2.ext b/tests/test-data/test-data/2test2.ext new file mode 120000 index 00000000..f6d8b812 --- /dev/null +++ b/tests/test-data/test-data/2test2.ext @@ -0,0 +1 @@ +test2.ext \ No newline at end of file diff --git a/tests/test-data/test-data/3test.ext b/tests/test-data/test-data/3test.ext new file mode 120000 index 00000000..769e1336 --- /dev/null +++ b/tests/test-data/test-data/3test.ext @@ -0,0 +1 @@ +test.ext \ No newline at end of file diff --git a/tests/test-data/test-data/3test2.ext b/tests/test-data/test-data/3test2.ext new file mode 120000 index 00000000..f6d8b812 --- /dev/null +++ b/tests/test-data/test-data/3test2.ext @@ -0,0 +1 @@ +test2.ext \ No newline at end of file diff --git a/tests/test-data/test-data/4test.ext b/tests/test-data/test-data/4test.ext new file mode 120000 index 00000000..769e1336 --- /dev/null +++ b/tests/test-data/test-data/4test.ext @@ -0,0 +1 @@ +test.ext \ No newline at end of file diff --git a/tests/test-data/test-data/4test2.ext b/tests/test-data/test-data/4test2.ext new file mode 120000 index 00000000..f6d8b812 --- /dev/null +++ b/tests/test-data/test-data/4test2.ext @@ -0,0 +1 @@ +test2.ext \ No newline at end of file diff --git a/tests/test-data/test-data/5test.ext b/tests/test-data/test-data/5test.ext new file mode 120000 index 00000000..769e1336 --- /dev/null +++ b/tests/test-data/test-data/5test.ext @@ -0,0 +1 @@ +test.ext \ No newline at end of file diff --git a/tests/test-data/test-data/5test2.ext b/tests/test-data/test-data/5test2.ext new file mode 120000 index 00000000..f6d8b812 --- /dev/null +++ b/tests/test-data/test-data/5test2.ext @@ -0,0 +1 @@ +test2.ext \ No newline at end of file diff --git a/tests/test-data/test-data/6test.ext b/tests/test-data/test-data/6test.ext new file mode 120000 index 00000000..769e1336 --- /dev/null +++ b/tests/test-data/test-data/6test.ext @@ -0,0 +1 @@ +test.ext \ No newline at end of file diff --git a/tests/test-data/test-data/6test2.ext b/tests/test-data/test-data/6test2.ext new file mode 120000 index 00000000..f6d8b812 --- /dev/null +++ b/tests/test-data/test-data/6test2.ext @@ -0,0 +1 @@ +test2.ext \ No newline at end of file diff --git a/tests/test-data/test-data/7test.ext b/tests/test-data/test-data/7test.ext new file mode 120000 index 00000000..769e1336 --- /dev/null +++ b/tests/test-data/test-data/7test.ext @@ -0,0 +1 @@ +test.ext \ No newline at end of file diff --git a/tests/test-data/test-data/7test2.ext b/tests/test-data/test-data/7test2.ext new file mode 120000 index 00000000..f6d8b812 --- /dev/null +++ b/tests/test-data/test-data/7test2.ext @@ -0,0 +1 @@ +test2.ext \ No newline at end of file diff --git a/tests/test-data/test-data/8test.ext b/tests/test-data/test-data/8test.ext new file mode 120000 index 00000000..769e1336 --- /dev/null +++ b/tests/test-data/test-data/8test.ext @@ -0,0 +1 @@ +test.ext \ No newline at end of file diff --git a/tests/test-data/test-data/8test2.ext b/tests/test-data/test-data/8test2.ext new file mode 120000 index 00000000..f6d8b812 --- /dev/null +++ b/tests/test-data/test-data/8test2.ext @@ -0,0 +1 @@ +test2.ext \ No newline at end of file diff --git a/tests/test-data/test-data/9test.ext b/tests/test-data/test-data/9test.ext new file mode 120000 index 00000000..769e1336 --- /dev/null +++ b/tests/test-data/test-data/9test.ext @@ -0,0 +1 @@ +test.ext \ No newline at end of file diff --git a/tests/test-data/test-data/9test2.ext b/tests/test-data/test-data/9test2.ext new file mode 120000 index 00000000..f6d8b812 --- /dev/null +++ b/tests/test-data/test-data/9test2.ext @@ -0,0 +1 @@ +test2.ext \ No newline at end of file From 5aaa7d0cb522eb9736475ff3e340a726d348ad54 Mon Sep 17 00:00:00 2001 From: M Bernt Date: Tue, 26 Nov 2019 16:26:31 +0100 Subject: [PATCH 039/121] switch to exit_code (instead of aggressive) --- galaxy/converter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/galaxy/converter.py b/galaxy/converter.py index 746ade54..60fec83e 100755 --- a/galaxy/converter.py +++ b/galaxy/converter.py @@ -556,7 +556,7 @@ def create_command(tool, model, **kwargs): final_command = "\n#import re" + final_command command_node = add_child_node(tool, "command") - command_node.attrib["detect_errors"] = "aggressive" + command_node.attrib["detect_errors"] = "exit_code" command_node.text = CDATA(final_command) From 732d0a45ced0ddf5eb2c65ca689de3d827394081 Mon Sep 17 00:00:00 2001 From: M Bernt Date: Tue, 26 Nov 2019 17:18:52 +0100 Subject: [PATCH 040/121] switched to yml hased hardcoder --- common/utils.py | 71 ++++++++++++++++++++++++++++++------------------- 1 file changed, 44 insertions(+), 27 deletions(-) diff --git a/common/utils.py b/common/utils.py index 73ba341f..b84272cc 100644 --- a/common/utils.py +++ b/common/utils.py @@ -163,33 +163,50 @@ def add_common_parameters(parser, version, last_updated): def parse_hardcoded_parameters(hardcoded_parameters_file): parameter_hardcoder = ParameterHardcoder() - if hardcoded_parameters_file is not None: - line_number = 0 - with open(hardcoded_parameters_file) as f: - for line in f: - line_number += 1 - if line is None or not line.strip() or line.strip().startswith("#"): - continue - # the third column must not be obtained as a whole, and not split - parsed_hardcoded_parameter = [ _ for _ in line.strip().split("\t") if _ != ""] - # valid lines contain two or three columns - if not (2 <= len(parsed_hardcoded_parameter) <= 3): - warning("Invalid line at line number %d of the given hardcoded parameters file. Line will be" - "ignored:\n%s" % (line_number, line), 0) - continue - - parameter_name = parsed_hardcoded_parameter[0] - hardcoded_value = parsed_hardcoded_parameter[1] - if hardcoded_value == "#": - hardcoded_value = "" - tool_names = None - if len(parsed_hardcoded_parameter) == 3: - tool_names = parsed_hardcoded_parameter[2].split(',') - if tool_names: - for tool_name in tool_names: - parameter_hardcoder.register_parameter(parameter_name, hardcoded_value, tool_name.strip()) - else: - parameter_hardcoder.register_parameter(parameter_name, hardcoded_value) + if hardcoded_parameters_file is None: + return parameter_hardcoder + from yaml import load, dump + try: + from yaml import CLoader as Loader, CDumper as Dumper + except ImportError: + from yaml import Loader, Dumper + + with open(hardcoded_parameters_file) as f: + data = load(f, Loader=Loader) + + for parameter_name in data: + hardcoded_value = data[parameter_name].get("value", None) + tool_names = data[parameter_name].get("tools", [None]) + for tool_name in tool_names: + if not tool_name is None: + tool_name = tool_name.strip() + parameter_hardcoder.register_parameter(parameter_name, hardcoded_value, tool_name) +# line_number = 0 +# with open(hardcoded_parameters_file) as f: +# for line in f: +# line_number += 1 +# if line is None or not line.strip() or line.strip().startswith("#"): +# continue +# # the third column must not be obtained as a whole, and not split +# parsed_hardcoded_parameter = [ _ for _ in line.strip().split("\t") if _ != ""] +# # valid lines contain two or three columns +# if not (2 <= len(parsed_hardcoded_parameter) <= 3): +# warning("Invalid line at line number %d of the given hardcoded parameters file. Line will be" +# "ignored:\n%s" % (line_number, line), 0) +# continue +# +# parameter_name = parsed_hardcoded_parameter[0] +# hardcoded_value = parsed_hardcoded_parameter[1] +# if hardcoded_value == "#": +# hardcoded_value = "" +# tool_names = None +# if len(parsed_hardcoded_parameter) == 3: +# tool_names = parsed_hardcoded_parameter[2].split(',') +# if tool_names: +# for tool_name in tool_names: +# parameter_hardcoder.register_parameter(parameter_name, hardcoded_value, tool_name.strip()) +# else: +# parameter_hardcoder.register_parameter(parameter_name, hardcoded_value) return parameter_hardcoder From 1123e37d7cdb7064f3d39616548e3b612b00177b Mon Sep 17 00:00:00 2001 From: M Bernt Date: Wed, 27 Nov 2019 10:27:05 +0100 Subject: [PATCH 041/121] switch to json hardcoder --- common/utils.py | 63 +++++++++++++------------------------------------ 1 file changed, 16 insertions(+), 47 deletions(-) diff --git a/common/utils.py b/common/utils.py index b84272cc..4729f867 100644 --- a/common/utils.py +++ b/common/utils.py @@ -39,12 +39,10 @@ def __init__(self): # the most specific value will be returned in case of overlap def get_hardcoded_value(self, parameter_name, tool_name): # look for the value that would apply for all tools - generic_value = self.parameter_map.get(parameter_name, None) - specific_value = self.parameter_map.get(self.build_key(parameter_name, tool_name), None) - if specific_value is not None: - return specific_value - - return generic_value + try: + return self.parameter_map[self.build_key(parameter_name, tool_name)] + except KeyError: + return self.parameter_map.get(parameter_name, None) def register_parameter(self, parameter_name, parameter_value, tool_name=None): self.parameter_map[self.build_key(parameter_name, tool_name)] = parameter_value @@ -165,49 +163,20 @@ def parse_hardcoded_parameters(hardcoded_parameters_file): parameter_hardcoder = ParameterHardcoder() if hardcoded_parameters_file is None: return parameter_hardcoder - from yaml import load, dump - try: - from yaml import CLoader as Loader, CDumper as Dumper - except ImportError: - from yaml import Loader, Dumper - - with open(hardcoded_parameters_file) as f: - data = load(f, Loader=Loader) + + import json + with open(hardcoded_parameters_file) as fp: + data = json.load(fp) for parameter_name in data: - hardcoded_value = data[parameter_name].get("value", None) - tool_names = data[parameter_name].get("tools", [None]) - for tool_name in tool_names: - if not tool_name is None: - tool_name = tool_name.strip() - parameter_hardcoder.register_parameter(parameter_name, hardcoded_value, tool_name) -# line_number = 0 -# with open(hardcoded_parameters_file) as f: -# for line in f: -# line_number += 1 -# if line is None or not line.strip() or line.strip().startswith("#"): -# continue -# # the third column must not be obtained as a whole, and not split -# parsed_hardcoded_parameter = [ _ for _ in line.strip().split("\t") if _ != ""] -# # valid lines contain two or three columns -# if not (2 <= len(parsed_hardcoded_parameter) <= 3): -# warning("Invalid line at line number %d of the given hardcoded parameters file. Line will be" -# "ignored:\n%s" % (line_number, line), 0) -# continue -# -# parameter_name = parsed_hardcoded_parameter[0] -# hardcoded_value = parsed_hardcoded_parameter[1] -# if hardcoded_value == "#": -# hardcoded_value = "" -# tool_names = None -# if len(parsed_hardcoded_parameter) == 3: -# tool_names = parsed_hardcoded_parameter[2].split(',') -# if tool_names: -# for tool_name in tool_names: -# parameter_hardcoder.register_parameter(parameter_name, hardcoded_value, tool_name.strip()) -# else: -# parameter_hardcoder.register_parameter(parameter_name, hardcoded_value) - + for el in data[parameter_name]: + hardcoded_value = el.get("value", None) + tool_names = el.get("tools", [None]) + for tool_name in tool_names: + if not tool_name is None: + tool_name = tool_name.strip() + print("register %s %s %s", (parameter_name,hardcoded_value, tool_name)) + parameter_hardcoder.register_parameter(parameter_name, hardcoded_value, tool_name) return parameter_hardcoder From a193c9138de70b956e9646ad374c407e3a060c64 Mon Sep 17 00:00:00 2001 From: M Bernt Date: Wed, 27 Nov 2019 10:27:29 +0100 Subject: [PATCH 042/121] add special case for booleans if restrictions are false,true the value needs to be given in both cases --- galaxy/converter.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/galaxy/converter.py b/galaxy/converter.py index 60fec83e..ce420da8 100755 --- a/galaxy/converter.py +++ b/galaxy/converter.py @@ -959,13 +959,17 @@ def create_boolean_parameter(param_node, param): TODO: true and false values can be way more than 'true' and 'false' but for that we need CTD support """ + # in ctd (1.6.2) bools are strings with restriction true,false # - if the default is false then they are flags # - otherwise the true or false value needs to be added (where the true case is unnecessary) - + # A special case are restrictions false,true which are not treated as flags if param.type == str: choices = get_lowercase_list(param.restrictions.choices) - if set(choices) == set(["true","false"]) and param.default == "true": + if choices == ["false", "true"]: + param_node.attrib["truevalue"] = "-%s true" % utils.extract_param_name(param) + param_node.attrib["falsevalue"] = "-%s false" % utils.extract_param_name(param) + elif set(choices) == set(["true","false"]) and param.default == "true": param_node.attrib["truevalue"] = "" param_node.attrib["falsevalue"] = "-%s false" % utils.extract_param_name(param) else: From af2dcef2a04b0d94b18481a4b5f14203bc86907a Mon Sep 17 00:00:00 2001 From: M Bernt Date: Mon, 9 Dec 2019 14:00:50 +0100 Subject: [PATCH 043/121] big update hardcoder: - add possibility to blacklist parameters - add possibility to rewrite ctd/xml attributes - remove old blacklist parameter xml generation: - added quote function to savely string parameters that are rendered from list of strings - most validators and sanitizers are now via macros - rewrote output handling (corresponding input and type param) and added tests - error detection aggressive -> exit_codde --- common/utils.py | 72 ++- convert.py | 2 +- galaxy/converter.py | 324 +++++++--- galaxy/macros.xml | 43 ++ tests/test-data/bool.xml | 2 +- tests/test-data/empty.xml | 2 +- tests/test-data/filetypes.txt | 4 +- tests/test-data/float.xml | 2 +- tests/test-data/ifile.xml | 2 +- tests/test-data/integer.xml | 2 +- tests/test-data/macros.xml | 49 +- tests/test-data/ofile-corresponding-input.ctd | 38 ++ tests/test-data/ofile-corresponding-input.xml | 126 ++++ tests/test-data/ofile-mult.xml | 69 ++- tests/test-data/ofile-typeparam.ctd | 48 ++ tests/test-data/ofile-typeparam.xml | 182 ++++++ tests/test-data/ofile.ctd | 27 +- tests/test-data/ofile.xml | 117 ++-- tests/test-data/repeat.xml | 586 +++++------------- tests/test-data/select.xml | 234 ++----- tests/test-data/string.xml | 62 +- tests/test_galaxy_cli.py | 6 + 22 files changed, 1124 insertions(+), 875 deletions(-) create mode 100644 tests/test-data/ofile-corresponding-input.ctd create mode 100644 tests/test-data/ofile-corresponding-input.xml create mode 100644 tests/test-data/ofile-typeparam.ctd create mode 100644 tests/test-data/ofile-typeparam.xml diff --git a/common/utils.py b/common/utils.py index 4729f867..3f0a7428 100644 --- a/common/utils.py +++ b/common/utils.py @@ -7,8 +7,9 @@ from string import strip from logger import info, error, warning +from common import logger from common.exceptions import ApplicationException -from CTDopts.CTDopts import CTDModel, ParameterGroup +from CTDopts.CTDopts import _InFile, _OutFile, CTDModel, ParameterGroup MESSAGE_INDENTATION_INCREMENT = 2 @@ -34,8 +35,44 @@ def __init__(self): # XtandemAdapter#adapter -> xtandem.exe # adapter -> adapter.exe self.separator = "!" + + # hard coded values self.parameter_map = {} + # ctd/xml attributes to overwrite + self.attribute_map = {} + + # blacklisted parameters + self.blacklist = set() + + def register_blacklist(self, parameter_name, tool_name): + k = self.build_key(parameter_name, tool_name) + self.blacklist.add(k) + + # the most specific value will be returned in case of overlap + def get_blacklist(self, parameter_name, tool_name): + # look for the value that would apply for all tools + if self.build_key(parameter_name, tool_name) in self.blacklist: + return True + elif parameter_name in self.blacklist: + return True + else: + return False + + def register_attribute(self, parameter_name, attribute, value, tool_name): + k = self.build_key(parameter_name, tool_name) + if not self.attribute_map.has_key(k): + self.attribute_map[k] = {} + self.attribute_map[k][attribute] = value + + # the most specific value will be returned in case of overlap + def get_hardcoded_attributes(self, parameter_name, tool_name): + # look for the value that would apply for all tools + try: + return self.attribute_map[self.build_key(parameter_name, tool_name)] + except KeyError: + return self.attribute_map.get(parameter_name, None) + # the most specific value will be returned in case of overlap def get_hardcoded_value(self, parameter_name, tool_name): # look for the value that would apply for all tools @@ -99,7 +136,7 @@ def parse_input_ctds(xsd_location, input_ctds, output_destination, output_file_e try: info("Loading validation schema from %s" % xsd_location, 0) schema = etree.XMLSchema(etree.parse(xsd_location)) - except Exception, e: + except Exception as e: error("Could not load validation schema %s. Reason: %s" % (xsd_location, str(e)), 0) else: warning("Validation against a schema has not been enabled.", 0) @@ -126,7 +163,7 @@ def validate_against_schema(ctd_file, schema): try: parser = etree.XMLParser(schema=schema) etree.parse(ctd_file, parser=parser) - except etree.XMLSyntaxError, e: + except etree.XMLSyntaxError as e: raise ApplicationException("Invalid CTD file %s. Reason: %s" % (ctd_file, str(e))) @@ -141,10 +178,6 @@ def add_common_parameters(parser, version, last_updated): parser.add_argument("-x", "--default-executable-path", dest="default_executable_path", help="Use this executable path when is not present in the CTD", default=None, required=False) - parser.add_argument("-b", "--blacklist-parameters", dest="blacklisted_parameters", default=[], nargs="+", - action="append", - help="List of parameters that will be ignored and won't appear on the galaxy stub", - required=False) parser.add_argument("-p", "--hardcoded-parameters", dest="hardcoded_parameters", default=None, required=False, help="File containing hardcoded values for the given parameters. Run with '-h' or '--help' " "to see a brief example on the format of this file.") @@ -169,14 +202,35 @@ def parse_hardcoded_parameters(hardcoded_parameters_file): data = json.load(fp) for parameter_name in data: + if parameter_name == "#": + continue for el in data[parameter_name]: hardcoded_value = el.get("value", None) tool_names = el.get("tools", [None]) for tool_name in tool_names: if not tool_name is None: tool_name = tool_name.strip() - print("register %s %s %s", (parameter_name,hardcoded_value, tool_name)) - parameter_hardcoder.register_parameter(parameter_name, hardcoded_value, tool_name) + if hardcoded_value is not None: + if hardcoded_value == '@': + parameter_hardcoder.register_blacklist(parameter_name, tool_name) + else: + parameter_hardcoder.register_parameter(parameter_name, hardcoded_value, tool_name) + else: + for a in el: + if a in ["tools","value"]: + continue + if el[a] == "output-file": + el[a] = _OutFile + if el[a] == "input-file": + el[a] = _InFile + + parameter_hardcoder.register_attribute(parameter_name, a, el[a], tool_name) + # hard coded values + logger.info("parameter_map %s" %parameter_hardcoder.parameter_map) + logger.info("attribute_map %s" %parameter_hardcoder.attribute_map) + logger.info("blacklist %s" %parameter_hardcoder.blacklist) + + return parameter_hardcoder diff --git a/convert.py b/convert.py index 4442a44b..4e28ef8b 100644 --- a/convert.py +++ b/convert.py @@ -243,7 +243,7 @@ def main(argv=None): def validate_and_prepare_common_arguments(args): # flatten lists of lists to a list containing elements - lists_to_flatten = ["input_files", "blacklisted_parameters"] + lists_to_flatten = ["input_files"] for list_to_flatten in lists_to_flatten: utils.flatten_list_of_lists(args, list_to_flatten) diff --git a/galaxy/converter.py b/galaxy/converter.py index ce420da8..2f08a798 100755 --- a/galaxy/converter.py +++ b/galaxy/converter.py @@ -16,6 +16,7 @@ from CTDopts.CTDopts import _InFile, _OutFile, ParameterGroup, _Choices, _NumericRange, _FileFormat, ModelError, _Null +# mapping to CTD types to Galaxy types TYPE_TO_GALAXY_TYPE = {int: 'integer', float: 'float', str: 'text', bool: 'boolean', _InFile: 'txt', _OutFile: 'txt', _Choices: 'select'} STDIO_MACRO_NAME = "stdio" @@ -112,7 +113,9 @@ def convert_models(args, parsed_ctds): macros_to_expand = parse_macros_files(args.macros_files, tool_version=args.tool_version, required_macros=REQUIRED_MACROS, - dont_expand=[ADVANCED_OPTIONS_NAME+"macro", "references"]) + dont_expand=[ADVANCED_OPTIONS_NAME+"macro", "references", + "list_string_val", "list_string_san", + "list_float_valsan", "list_integer_valsan"]) # parse the given supported file-formats file supported_file_formats = parse_file_formats(args.formats_file) @@ -124,7 +127,6 @@ def convert_models(args, parsed_ctds): supported_file_formats=supported_file_formats, default_executable_path=args.default_executable_path, add_to_command_line=args.add_to_command_line, - blacklisted_parameters=args.blacklisted_parameters, required_tools=required_tools, skip_tools=skip_tools, macros_file_names=args.macros_files, @@ -186,7 +188,7 @@ def parse_macros_files(macros_file_names, tool_version, required_macros = [], do xml_element.text = tool_version if xml_element.attrib["name"] == "@GALAXY_VERSION@": if bump_galaxy_version: - xml_element.text = str( int(xml_element.text) + 1 ) + xml_element.text = str(int(xml_element.text) + 1) else: xml_element.text = "0" @@ -316,6 +318,8 @@ def _convert_internal(parsed_ctds, **kwargs): expand_macros, create_command, create_inputs, create_outputs @return a tuple containing the model, output destination, origin file """ + + parameter_hardcoder = kwargs["parameter_hardcoder"] for parsed_ctd in parsed_ctds: model = parsed_ctd.ctd_model @@ -328,6 +332,16 @@ def _convert_internal(parsed_ctds, **kwargs): origin_file = parsed_ctd.input_file output_file = parsed_ctd.suggested_output_file + + # overwrite attributes of the parsed ctd parameters as specified in hardcoded parameterd json + for param in utils.extract_and_flatten_parameters(model): + hardcoded_attributes = parameter_hardcoder.get_hardcoded_attributes(param.name, model.name) + if hardcoded_attributes != None: + for a in hardcoded_attributes: + if not hasattr(param, a): + continue + setattr(param, a, hardcoded_attributes[a]) + logger.info("Converting %s (source %s)" % (model.name, utils.get_filename(origin_file)), 0) tool = create_tool(model) write_header(tool, model) @@ -344,7 +358,7 @@ def _convert_internal(parsed_ctds, **kwargs): create_help(tool, model) # citations are required to be at the end - expand_macros(tool, ["references"]) + expand_macro(tool, "references") # wrap our tool element into a tree to be able to serialize it tree = ElementTree(tool) @@ -375,7 +389,7 @@ def generate_tool_conf(parsed_ctds, tool_conf_destination, galaxy_tool_path, def if category not in categories_to_tools: categories_to_tools[category] = [] categories_to_tools[category].append(utils.get_filename(parsed_ctd.suggested_output_file)) - + # at this point, we should have a map for all categories->tools toolbox_node = Element("toolbox") @@ -448,15 +462,16 @@ def create_command(tool, model, **kwargs): """ # main command - final_preprocessing = "\n" + final_preprocessing = "\n@QUOTE_FOO@\n" advanced_preprocessing = "" - final_command = "@EXECUTABLE@" + '\n' + final_command = "@EXECUTABLE@\n" final_command += kwargs["add_to_command_line"] + '\n' advanced_command_start = "#if ${aon}cond.{aon}selector=='advanced':\n".format(aon=ADVANCED_OPTIONS_NAME) advanced_command_end = "#end if" advanced_command = "" parameter_hardcoder = kwargs["parameter_hardcoder"] + supported_file_formats = kwargs["supported_file_formats"] found_input_parameter = False found_output_parameter = False @@ -471,21 +486,15 @@ def create_command(tool, model, **kwargs): param_name = utils.extract_param_name(param) command_line_prefix = utils.extract_command_line_prefix(param, model) - if param.name in kwargs["blacklisted_parameters"]: + if parameter_hardcoder.get_blacklist(param_name, model.name): continue hardcoded_value = parameter_hardcoder.get_hardcoded_value(param_name, model.name) - if not hardcoded_value is None: + if hardcoded_value is not None: command += "%s %s\n" % (command_line_prefix, hardcoded_value) else: # in the else branch the parameter is neither blacklisted nor hardcoded... - # determine the name of the cheetah variable that is used to get the argument - # for the parameter - corresponding_input = None - if param.type is _OutFile and param.is_list: - corresponding_input = get_input_with_same_restrictions(param, model) - actual_parameter = get_galaxy_parameter_name(param) if param.advanced and not param.type is _OutFile: actual_parameter = ADVANCED_OPTIONS_NAME+"cond." + actual_parameter @@ -494,26 +503,56 @@ def create_command(tool, model, **kwargs): if not is_boolean_parameter(param): command += command_line_prefix + " " - # preprocessing for file inputs: create a link to id.ext in the input directory + # preprocessing for file inputs: + # - create a dir with name param.name + # - create a link to id.ext in this directory + # rationale: in the autogenerated tests the same file was used as input to multiple parameters + # this leads to conflicts while linking... might also be better in general if param.type is _InFile: + preprocessing += "mkdir %s &&\n" %actual_parameter if param.is_list: - preprocessing += "${ ' && '.join([ \"ln -s '%s' '%s.%s'\" % (_, re.sub('[^\w\-_.]', '_', _.element_identifier), _.ext) for _ in $" + actual_parameter + " if _ ]) } && \n" - command += "${' '.join([\"'%s.%s'\"%(re.sub('[^\w\-_.]', '_', _.element_identifier),_.ext) for _ in $" + actual_parameter + " if _ ])}\n" + preprocessing += "${ ' && '.join([ \"ln -s '%s' '"+actual_parameter+"/%s.%s'\" % (_, re.sub('[^\w\-_.]', '_', _.element_identifier), _.ext) for _ in $" + actual_parameter + " if _ ]) } && \n" + command += "${' '.join([\"'"+actual_parameter+"/%s.%s'\"%(re.sub('[^\w\-_.]', '_', _.element_identifier),_.ext) for _ in $" + actual_parameter + " if _ ])}\n" else: - preprocessing += "ln -s '$"+ actual_parameter +"' '${re.sub(\"[^\w\-_.]\", \"_\", $"+actual_parameter+".element_identifier)}.${"+actual_parameter+".ext}' &&\n" - command += "'${re.sub('[^\w\-_.]', '_', $"+actual_parameter+".element_identifier)}.${"+actual_parameter+".ext}'\n" - elif param.type is _OutFile and not corresponding_input is None: - preprocessing += "mkdir " + actual_parameter + " && \n" - actual_input_parameter = get_galaxy_parameter_name( corresponding_input ) + preprocessing += "ln -s '$"+ actual_parameter +"' '"+actual_parameter+"/${re.sub(\"[^\w\-_.]\", \"_\", $"+actual_parameter+".element_identifier)}.${"+actual_parameter+".ext}' &&\n" + command += "'"+actual_parameter+"/${re.sub('[^\w\-_.]', '_', $"+actual_parameter+".element_identifier)}.${"+actual_parameter+".ext}'\n" + elif param.type is _OutFile: + # check if there is a parameter that sets the format + # if so we add an extension to the generated files which will be used to + # determine the format in the output tag + # in all other cases (corresponding input / there is only one allowed format) + # the format will be set in the output tag + type_param = get_out_type_param(param, model) + if type_param is not None: + type_param_name = get_galaxy_parameter_name(type_param) + if type_param.advanced: + type_param_name = ADVANCED_OPTIONS_NAME+"cond." + type_param_name + if param.is_list: - command += "${' '.join([\"'"+ actual_parameter +"/%s.%s'\"%(re.sub('[^\w\-_.]', '_', _.element_identifier),_.ext) for _ in $" + actual_input_parameter + " if _ ])}\n" + preprocessing += "mkdir " + actual_parameter + " && \n" + corresponding_input, fmt_from_corresponding = get_corresponding_input(param, model) + if corresponding_input is None: + raise Exception() + actual_input_parameter = get_galaxy_parameter_name(corresponding_input) + if corresponding_input.advanced: + actual_input_parameter = ADVANCED_OPTIONS_NAME+"cond." + actual_input_parameter + if type_param is not None: + command += "${' '.join([\"'"+ actual_parameter +"/%s.%s'\"%(re.sub('[^\w\-_.]', '_', _.element_identifier), "+type_param_name+") for _ in $" + actual_input_parameter + " if _ ])}\n" + else: + command += "${' '.join([\"'"+ actual_parameter +"/%s'\"%(re.sub('[^\w\-_.]', '_', _.element_identifier)) for _ in $" + actual_input_parameter + " if _ ])}\n" else: - command += actual_parameter+"/'${re.sub('[^\w\-_.]', '_', $"+actual_input_parameter+".element_identifier)}.${"+actual_input_parameter+".ext}'\n" - # logic for ITEMLISTs + if type_param is not None: + preprocessing += "mkdir " + actual_parameter + " && \n" + command += actual_parameter+"/output.${"+type_param_name+"}'\n" + else: + command += "'$" +actual_parameter+ "'\n" + + # select with multiple = true elif is_selection_parameter(param) and param.is_list: - command += "${' '.join([\"'%s'\"%str(_) for _ in $" + actual_parameter + "])}\n" - elif param.is_list : - command += "${' '.join([\"'%s'\"%str(_) for _ in $" + actual_parameter + ".split(',')])}\n" + command += "${' '.join([\"'%s'\"%str(_) for _ in str($" + actual_parameter + ").split(',')])}\n" + elif param.is_list: + command += "$quote($%s" % actual_parameter + ")\n" + #command += "${' '.join([\"'%s'\"%str(_) for _ in $" + actual_parameter + "])}\n" elif is_boolean_parameter(param): command += "$%s" % actual_parameter + "\n" else: @@ -579,6 +618,11 @@ def import_macros(tool, model, **kwargs): # do not add the path of the file, rather, just its basename import_node.text = os.path.basename(macro_file.name) +def expand_macro(node, macro): + expand_node = add_child_node(node, "expand") + expand_node.attrib["macro"] = macro + return expand_node + # and to "expand" the macros in a node def expand_macros(node, macros_to_expand): # add nodes @@ -607,21 +651,70 @@ def get_galaxy_parameter_name(param, inparam = False): return "param_%s" % p -def get_input_with_same_restrictions(out_param, model): +def get_out_type_param(out_param, model): + """ + check if there is a parameter that has the same name with appended _type + and return it if present, otherwise return None + """ + + for param in utils.extract_and_flatten_parameters(model): + if param.name == out_param.name + "_type": + return param + return None + + +def is_out_type_param(param, model): + """ + check if the parameter is output_type parameter + - the name ends with _type and there is an output parameter without this suffix + and return True iff this is the case + """ + if not param.name.endswith("_type"): + return False + for out_param in utils.extract_and_flatten_parameters(model): + if out_param.type is not _OutFile: + continue + if param.name == out_param.name + "_type": + return True + return False + + +def get_corresponding_input(out_param, model): + """ + get the input parameter corresponding to the given output + + 1st try to get the input with the type (single file/list) and same format restrictions + if this fails get the input that has the same type + in both cases there must be only one such input + + return the found input parameter and True iff the 1st case applied + """ + c = get_input_with_same_restrictions(out_param, model, True) + if c == None: + return (get_input_with_same_restrictions(out_param, model, False), False) + else: + return (c, True) + + +def get_input_with_same_restrictions(out_param, model, check_formats): """ get the input parameter that has the same restrictions (ctd file_formats) - input and output must both be lists of both be simple parameters """ + matching = [] for param in utils.extract_and_flatten_parameters(model): - if not param.type is _InFile or param.restrictions is None: - continue - if param.is_list != out_param.is_list: + if not param.type is _InFile: continue - in_param_formats = param.restrictions.formats - out_param_formats = out_param.restrictions.formats - if in_param_formats == out_param_formats: - matching.append(param) + if param.is_list == out_param.is_list: + if check_formats: + if param.restrictions is None and out_param.restrictions is None: + matching.append(param) + elif param.restrictions is not None and out_param.restrictions is not None and param.restrictions.formats == out_param.restrictions.formats: + matching.append(param) + else: + matching.append(param) + logger.error("gisr %s %s"%(out_param.name, [_.name for _ in matching])) if len(matching) == 1: return matching[0] else: @@ -641,12 +734,13 @@ def create_inputs(tool, model, **kwargs): # some suites (such as OpenMS) need some advanced options when handling inputs advanced_node = None parameter_hardcoder = kwargs["parameter_hardcoder"] + # treat all non output-file/advanced/blacklisted/hardcoded parameters as inputs for param in utils.extract_and_flatten_parameters(model): param = modify_param_for_galaxy(param) # no need to show hardcoded parameters hardcoded_value = parameter_hardcoder.get_hardcoded_value(param.name, model.name) - if param.name in kwargs["blacklisted_parameters"] or not hardcoded_value is None: + if parameter_hardcoder.get_blacklist(param.name, model.name) or not hardcoded_value is None: continue # optional outfiles: create an additional bool input which is used to filter for the output @@ -672,9 +766,14 @@ def create_inputs(tool, model, **kwargs): else: parent_node = inputs_node - # create the actual param node and fill the attributes + # create the actuayield/l param node and fill the attributes param_node = add_child_node(parent_node, "param") - create_param_attribute_list(param_node, _param, kwargs["supported_file_formats"]) + create_param_attribute_list(param_node, _param, model, kwargs["supported_file_formats"]) + +# hardcoded_attributes = parameter_hardcoder.get_hardcoded_attributes(param.name, model.name) +# if hardcoded_attributes != None: +# for a in hardcoded_attributes: +# param_node.attrib[a] = str(hardcoded_attributes[a]) # if there is an advanced section then append it at the end of the inputs if not advanced_node is None: @@ -683,7 +782,7 @@ def create_inputs(tool, model, **kwargs): return inputs_node -def create_param_attribute_list(param_node, param, supported_file_formats): +def create_param_attribute_list(param_node, param, model, supported_file_formats): """ get the attributes of input parameters @param param_node the galaxy tool param node @@ -749,6 +848,11 @@ def create_param_attribute_list(param_node, param, supported_file_formats): if param_type == "boolean": create_boolean_parameter(param_node, param) elif type(param.restrictions) is _Choices: + + # if the parameter is used to select the output file type the options need to be replaced with the Galaxy data types + if is_out_type_param(param, model): + param.restrictions.choices = get_supported_file_types(param.restrictions.choices, supported_file_formats) + # add a nothing selected option to mandatory options w/o default if param_node.attrib["optional"] == "False" and (param.default is None or param.default is _Null): param.restrictions.choices.insert(0, "select a value") @@ -794,49 +898,35 @@ def create_param_attribute_list(param_node, param, supported_file_formats): # for repeats (which are rendered as text field in the tool form) that are actually # integer/floats special validation is necessary (try to convert them and check if # in the min max range if a range is given) - # in addition (for the convenience of the user) any spaces are removed (due to the - # regex only possible next to the commas and at the begin or end) if TYPE_TO_GALAXY_TYPE[param.type] in ["integer", "float"]: - message = "a comma separated list of %s values is required" % TYPE_TO_GALAXY_TYPE[param.type] - if TYPE_TO_GALAXY_TYPE[param.type] == "integer": - expression = "^ *[+-]?[0-9]+( *, *[+-]?[0-9]+)* *$" - else: - expression = "^ *[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?( *, *[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?)* *$" - validator_node = SubElement(param_node, "validator", OrderedDict([("type", "regex"), ("message", message )])) - validator_node.text = CDATA(expression) + valsan = expand_macro(param_node, "list_%s_valsan" %TYPE_TO_GALAXY_TYPE[param.type]) if type(param.restrictions) is _NumericRange and not (param.restrictions.n_min is None and param.restrictions.n_max is None): - expression = "len(value.split(',')) == len([_ for _ in value.split(',')" - message = "a comma separated list of %s values " % TYPE_TO_GALAXY_TYPE[param.type] + expression = "len(value.split(' ')) == len([_ for _ in value.split(' ') if " + message = "a space separated list of %s values " % TYPE_TO_GALAXY_TYPE[param.type] if not param.restrictions.n_min is None and not param.restrictions.n_max is None: - expression += " if %s <= %s(_) <= %s" % (param.restrictions.n_min, param.type.__name__, param.restrictions.n_max) + expression += " %s <= %s(_) <= %s" % (param.restrictions.n_min, param.type.__name__, param.restrictions.n_max) message += "in the range %s:%s " % (param.restrictions.n_min, param.restrictions.n_max) elif not param.restrictions.n_min is None: - expression += " if %s <= %s(_)" % (param.restrictions.n_min, param.type.__name__) + expression += " %s <= %s(_)" % (param.restrictions.n_min, param.type.__name__) message += "in the range %s: " % (param.restrictions.n_min) elif not param.restrictions.n_max is None: - expression += " if %s(_) <= %s" % (param.type.__name__, param.restrictions.n_max) + expression += " %s(_) <= %s" % (param.type.__name__, param.restrictions.n_max) message += "in the range :%s " % (param.restrictions.n_min) expression += "])\n" message += "is required" - validator_node = SubElement(param_node, "validator", OrderedDict([("type", "expression"), ("message", message )])) + validator_node = SubElement(valsan, "validator", OrderedDict([("type", "expression"), ("message", message )])) validator_node.text = CDATA(expression) - sanitizer_node = SubElement(param_node, "sanitizer") - valid_node = SubElement(sanitizer_node, "valid", OrderedDict([("initial", "string.printable")])) - SubElement(valid_node, "remove", OrderedDict([("value", ' ')])) - mapping_node = SubElement(sanitizer_node, "mapping", OrderedDict([("initial", "none")])) - SubElement(mapping_node, "add", OrderedDict([("source", ' '),("target", "")])) - # add sanitizer nodes to # - text (only those that are not actually integer selects which are treated above) and # - select params, # this is needed for special character like "[" which are used for example by FeatureFinderMultiplex if ((param_type == "text" and not TYPE_TO_GALAXY_TYPE[param.type] in ["integer", "float"]) or is_selection_parameter(param)) and not param.type is _InFile: - sanitizer_node = SubElement(param_node, "sanitizer") - valid_node = SubElement(sanitizer_node, "valid", OrderedDict([("initial", "string.printable")])) - SubElement(valid_node, "remove", OrderedDict([("value", '\'')])) + if not is_selection_parameter(param): + valsan = expand_macro(param_node, "list_string_val") + valsan = expand_macro(param_node, "list_string_san") # check for default value if not (param.default is None or param.default is _Null): @@ -847,7 +937,7 @@ def create_param_attribute_list(param_node, param, supported_file_formats): # we ASSUME that a list of parameters looks like: # $ tool -ignore He Ar Xe # meaning, that, for example, Helium, Argon and Xenon will be ignored - param_node.attrib["value"] = ','.join(map(str, param.default)) + param_node.attrib["value"] = ' '.join(map(str, param.default)) elif param_type != "boolean": param_node.attrib["value"] = str(param.default) @@ -1005,7 +1095,7 @@ def create_outputs(parent, model, **kwargs): param = modify_param_for_galaxy(param) # no need to show hardcoded parameters hardcoded_value = parameter_hardcoder.get_hardcoded_value(param.name, model.name) - if param.name in kwargs["blacklisted_parameters"] or hardcoded_value: + if parameter_hardcoder.get_blacklist(param.name, model.name) or hardcoded_value: # let's not use an extra level of indentation and use NOP continue if not param.type is _OutFile: @@ -1022,22 +1112,27 @@ def create_outputs(parent, model, **kwargs): def create_output_node(parent, param, model, supported_file_formats): + # add a data node / collection + discover_datasets + # in the former case we just set the discover_node equal to the data node + # then we can just use this to set the common format attribute if not param.is_list: data_node = add_child_node(parent, "data") + discover_node = data_node else: data_node = add_child_node(parent, "collection") data_node.attrib["type"] = "list" + discover_node = add_child_node(data_node, "discover_datasets", + OrderedDict([("directory", get_galaxy_parameter_name(param))])) data_node.attrib["name"] = get_galaxy_parameter_name(param) if data_node.attrib["name"].startswith('param_out_'): data_node.attrib["label"] = "${tool.name} on ${on_string}: %s" % data_node.attrib["name"][10:] # determine format attribute - # - default is txt # - check if all listed possible formats are supported in Galaxy # - if there is a single one, then take this # - otherwise try to determine an input with the same restrictions and use this as format source - data_format = "txt" + formats = set() if param.restrictions is not None: if type(param.restrictions) is _FileFormat: # set the first data output node to the first file format @@ -1052,32 +1147,54 @@ def create_output_node(parent, param, model, supported_file_formats): if output: logger.warning("Parameter " + param.name + " has the following unsupported format(s):" + ','.join(output), 1) - data_format = ','.join(output) formats = get_supported_file_types(param.restrictions.formats, supported_file_formats) - try: - data_format = formats.pop() - except KeyError: - # there is not much we can do, other than catching the exception - pass - # if there are more than one output file formats try to take the format from the input parameter - corresponding_input = get_input_with_same_restrictions(param, model) - if param.is_list and corresponding_input is not None: - # TODO - add_child_node(data_node, "discover_datasets", - OrderedDict([("pattern", "__name_and_ext__"), ("directory", get_galaxy_parameter_name(param))])) - elif not param.is_list and formats and corresponding_input is not None: - data_format = "input" - data_node.attrib["format_source"] = get_galaxy_parameter_name(corresponding_input) - data_node.attrib["metadata_source"] = get_galaxy_parameter_name(corresponding_input) + if len(formats) == 0: + raise InvalidModelException("No supported formats [%(type)s] " + "for output [%(name)s]" % {"type": type(param.restrictions), + "name": param.name}) else: raise InvalidModelException("Unrecognized restriction type [%(type)s] " "for output [%(name)s]" % {"type": type(param.restrictions), "name": param.name}) - # data output has fomat (except if fromat_source has been added already) - # note .. collection output has no format - if not param.is_list and not "format_source" in data_node.attrib: - data_node.attrib["format"] = data_format + + type_param = get_out_type_param(param, model) + corresponding_input, fmt_from_corresponding = get_corresponding_input(param, model) + + # if there is only a single possible output format we set this + logger.error("%s %s %s %s %s" %(param.name, formats, type_param, fmt_from_corresponding, corresponding_input)) + if len(formats) == 1: + discover_node.attrib["format"] = formats.pop() + if param.is_list: + discover_node.attrib["pattern"] = "__name__" + # if there is another parameter where the user selects the format + # then this format was added as file extension on the CLI, now we can discover this + elif type_param is not None: + if not param.is_list: + discover_node = add_child_node(data_node, "discover_datasets", + OrderedDict([("directory", get_galaxy_parameter_name(param))])) + discover_node.attrib["pattern"] = "__name_and_ext__" + elif corresponding_input is not None: + if fmt_from_corresponding: + data_node.attrib["format_source"] = get_galaxy_parameter_name(corresponding_input) + if param.is_list: + discover_node.attrib["pattern"] = "__name__" + data_node.attrib["structured_like"] = get_galaxy_parameter_name(corresponding_input) + #data_node.attrib["inherit_format"] = "true" + else: + data_node.attrib["metadata_source"] = get_galaxy_parameter_name(corresponding_input) + else: + if not param.is_list: + data_node.attrib["auto_format"] = "true" + else: + raise InvalidModelException("No way to know the format for" + "for output [%(name)s]" % {"name": param.name}) + + +# # data output has fomat (except if fromat_source has been added already) +# # note .. collection output has no format +# if not param.is_list and not "format_source" in data_node.attrib: +# data_node.attrib["format"] = data_format # add filter for optional parameters if not param.required: @@ -1091,14 +1208,20 @@ def create_output_node(parent, param, model, supported_file_formats): # Get the supported file format for one given format def get_supported_file_type(format_name, supported_file_formats): if format_name in supported_file_formats.keys(): - return supported_file_formats.get(format_name, DataType(format_name, format_name)).galaxy_extension + return supported_file_formats[format_name].galaxy_extension else: return None def get_supported_file_types(formats, supported_file_formats): - return set([supported_file_formats.get(format_name, DataType(format_name, format_name)).galaxy_extension - for format_name in formats if format_name in supported_file_formats.keys()]) + r = set() + for f in formats: + if f in supported_file_formats: + r.add( supported_file_formats[f].galaxy_extension ) + return r +# print f, f in supported_file_formats, supported_file_formats[f].galaxy_extension +# return set([supported_file_formats[_].galaxy_extension +# for _ in formats if _ in supported_file_formats]) def create_change_format_node(parent, data_formats, input_ref): @@ -1122,9 +1245,9 @@ def create_tests(parent, inputs=None, outputs=None, test_macros_prefix=None, nam if not (inputs is None or outputs is None): fidx = 0 test_node = add_child_node(tests_node, "test") - strip_elements(inputs, "validator", "sanitizer") + strip_elements(inputs, "validator", "sanitizer" ) for node in inputs.iter(): - if node.tag == "expand": + if node.tag == "expand" and node.attrib["macro"]== ADVANCED_OPTIONS_NAME+"macro": node.tag = "conditional" node.attrib["name"] = ADVANCED_OPTIONS_NAME+"cond" add_child_node(node, "param", OrderedDict([("name", ADVANCED_OPTIONS_NAME+"selector"), ("value", "advanced")])) @@ -1147,16 +1270,16 @@ def create_tests(parent, inputs=None, outputs=None, test_macros_prefix=None, nam else: node.attrib["value"] = "false" # node.attrib["falsevalue"] elif node.attrib["type"] == "text" and node.attrib["value"] == "": - node.attrib["value"] = "1,2" # use a comma separated list here to cover the repeat (int/float) case + node.attrib["value"] = "1 2" # use a space separated list here to cover the repeat (int/float) case elif node.attrib["type"] == "integer" and node.attrib["value"] == "": node.attrib["value"] = "1" elif node.attrib["type"] == "float" and node.attrib["value"] == "": node.attrib["value"] = "1.0" elif node.attrib["type"] == "select": if node.attrib.get("display", None) == "radio" or node.attrib.get("multiple", "false") == "false": - node.attrib["value"] = node[-1].attrib["value"] + node.attrib["value"] = node[0].attrib["value"] elif node.attrib.get("multiple", None) == "true": - node.attrib["value"] = ",".join([ _.attrib["value"] for _ in node ]) + node.attrib["value"] = ",".join([ _.attrib["value"] for _ in node if _.attrib.has_key("value") ]) elif node.attrib["type"] == "data": node.attrib["ftype"] = node.attrib["format"].split(',')[0] if node.attrib.get("multiple", "false") == "true": @@ -1167,7 +1290,7 @@ def create_tests(parent, inputs=None, outputs=None, test_macros_prefix=None, nam for node in inputs.iter(): for a in set(node.attrib) - set(["name","value","ftype"]): del node.attrib[a] - strip_elements(inputs, "delete_node", "option") + strip_elements(inputs, "delete_node", "option", "expand") for node in inputs: test_node.append(node) @@ -1186,7 +1309,10 @@ def create_tests(parent, inputs=None, outputs=None, test_macros_prefix=None, nam node.tag = "delete_node" if node.tag == "data": node.tag = "output" - node.attrib["ftype"] = node.attrib["format"] + try: + node.attrib["ftype"] = node.attrib["format"] + except KeyError: + pass node.attrib["value"] = "outfile.txt" if node.tag == "collection": node.tag = "output_collection" diff --git a/galaxy/macros.xml b/galaxy/macros.xml index afcf1a23..b0de2352 100644 --- a/galaxy/macros.xml +++ b/galaxy/macros.xml @@ -35,4 +35,47 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/test-data/bool.xml b/tests/test-data/bool.xml index fd2fce55..5afe7e61 100644 --- a/tests/test-data/bool.xml +++ b/tests/test-data/bool.xml @@ -9,7 +9,7 @@ - - $param_stdout diff --git a/tests/test-data/filetypes.txt b/tests/test-data/filetypes.txt index 4137ff50..befe60cc 100644 --- a/tests/test-data/filetypes.txt +++ b/tests/test-data/filetypes.txt @@ -1,5 +1,5 @@ # CTD type # Galaxy type # Long Galaxy data type # Mimetype -ewt txt galaxy.datatypes.data:Text -ext txt galaxy.datatypes.data:Text +ewt tsv galaxy.datatypes.data:Text +ext tsv galaxy.datatypes.data:Text eyt tabular galaxy.datatypes.tabular:Tabular ezt tabular galaxy.datatypes.tabular:Tabular diff --git a/tests/test-data/float.xml b/tests/test-data/float.xml index fd666265..334207b4 100644 --- a/tests/test-data/float.xml +++ b/tests/test-data/float.xml @@ -9,7 +9,7 @@ - - - + 2.3 + 0 - bash + openms + + @@ -28,4 +32,47 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/test-data/ofile-corresponding-input.ctd b/tests/test-data/ofile-corresponding-input.ctd new file mode 100644 index 00000000..5e35e051 --- /dev/null +++ b/tests/test-data/ofile-corresponding-input.ctd @@ -0,0 +1,38 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/test-data/ofile-corresponding-input.xml b/tests/test-data/ofile-corresponding-input.xml new file mode 100644 index 00000000..fee7b498 --- /dev/null +++ b/tests/test-data/ofile-corresponding-input.xml @@ -0,0 +1,126 @@ + + + + + Outut File parameter tests-corresponding-input + + echo outFileTest-corresponding-input + macros.xml + + + + + + + + + + + + + + + + + + + + + + + + + param_optional_noformat + + + + param_optional_oneformat + + + + param_optional_multformat + + + + adv_opts_cond['adv_opts_selector'] == 'advanced' and adv_opts_cond['param_optional_noformat_adv'] + + + + adv_opts_cond['adv_opts_selector'] == 'advanced' and adv_opts_cond['param_optional_oneformat_adv'] + + + + adv_opts_cond['adv_opts_selector'] == 'advanced' and adv_opts_cond['param_optional_multformat_adv'] + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/test-data/ofile-mult.xml b/tests/test-data/ofile-mult.xml index 74f3b727..deaad2ad 100644 --- a/tests/test-data/ofile-mult.xml +++ b/tests/test-data/ofile-mult.xml @@ -9,60 +9,76 @@ - - - - - + + + + + + + + + + - + - + param_optional_mandatoryinput - - + + - - + + param_optional_optionalinput @@ -70,8 +86,11 @@ ${ ' && '.join([ "ln -s '%s' '%s.%s'" % (_, re.sub('[^\w\-_.]', '_', _.element_i - - + + + + + diff --git a/tests/test-data/ofile-typeparam.ctd b/tests/test-data/ofile-typeparam.ctd new file mode 100644 index 00000000..1dfb3ba8 --- /dev/null +++ b/tests/test-data/ofile-typeparam.ctd @@ -0,0 +1,48 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/test-data/ofile-typeparam.xml b/tests/test-data/ofile-typeparam.xml new file mode 100644 index 00000000..f7a9ff3c --- /dev/null +++ b/tests/test-data/ofile-typeparam.xml @@ -0,0 +1,182 @@ + + + + + Outut File parameter tests-typeparam + + echo outFileTest-typeparam + macros.xml + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + param_optional_noformat + + + + + + + param_optional_multformat + + + + + + + adv_opts_cond['adv_opts_selector'] == 'advanced' and adv_opts_cond['param_optional_noformat_adv'] + + + + + + + adv_opts_cond['adv_opts_selector'] == 'advanced' and adv_opts_cond['param_optional_multformat_adv'] + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/test-data/ofile.ctd b/tests/test-data/ofile.ctd index bf164f94..2c031f5c 100644 --- a/tests/test-data/ofile.ctd +++ b/tests/test-data/ofile.ctd @@ -5,19 +5,20 @@ - - - - - - - - - - - - - + + + + + + + + + diff --git a/tests/test-data/ofile.xml b/tests/test-data/ofile.xml index 1072e795..7b7f3704 100644 --- a/tests/test-data/ofile.xml +++ b/tests/test-data/ofile.xml @@ -9,34 +9,26 @@ - - - - - - - + + + + + + + - - - - param_wodefault_optional_noformat - - - param_default_optional_noformat - - - - - param_wodefault_optional_oneformat - - - param_default_optional_oneformat + + + param_optional_noformat - - - - param_wodefault_optional_multformat + + + param_optional_oneformat - - param_default_optional_multformat + + + param_optional_multformat - - + + adv_opts_cond['adv_opts_selector'] == 'advanced' and adv_opts_cond['param_optional_noformat_adv'] - - + + adv_opts_cond['adv_opts_selector'] == 'advanced' and adv_opts_cond['param_optional_oneformat_adv'] - - + + adv_opts_cond['adv_opts_selector'] == 'advanced' and adv_opts_cond['param_optional_multformat_adv'] - - - - - - - + + + + - - - - - - - - - + + + + + + - $param_stdout ]]> - - - - - + + - - - - - + + - - - - - - + + + - - - - - - + + + - - - - - - - - - + - - - - - - - - - + - - - - - - - - - - + + - - - - - - - - - - + + - - + - - - - - - - - + - - + - - - - - - - - + - - - + + - - - - - - - - + - - - + + - - - - - - - - + - - - - - - - - - + - - - - - - - - - + - - - - - - - - - - + + - - - - - - - - - - + + - - + - - - - - - - - + - - + - - - - - - - - + - - - + + - - - - - - - - + - - - + + - - - - - - - - + - - - - - + + - - - - - + + - - - - - - + + + - - - - - - + + + - - - - - - - - - + - - - - - - - - - + - - - - - - - - - - + + - - - - - - - - - - + + - - + - - - - - - - - + - - + - - - - - - - - + - - - + + - - - - - - - - + - - - + + - - - - - - - - + - - - - - - - - - + - - - - - - - - - + - - - - - - - - - - + + - - - - - - - - - - + + - - + - - - - - - - - + - - + - - - - - - - - + - - - + + - - - - - - - - + - - - + + - - - - - - - - + @@ -545,27 +281,27 @@ - - - - - - - - - - + + + + + + + + + + - - - - - - - - - - + + + + + + + + + + diff --git a/tests/test-data/select.xml b/tests/test-data/select.xml index 18faf91c..ea1aee1d 100644 --- a/tests/test-data/select.xml +++ b/tests/test-data/select.xml @@ -9,7 +9,7 @@ - $param_stdout ]]> @@ -86,44 +86,28 @@ - - - - - + - - - - - + - - - - - + - - - - - + @@ -131,11 +115,7 @@ - - - - - + @@ -143,11 +123,7 @@ - - - - - + @@ -155,11 +131,7 @@ - - - - - + @@ -167,55 +139,35 @@ - - - - - + - - - - - + - - - - - + - - - - - + - - - - - + @@ -223,11 +175,7 @@ - - - - - + @@ -235,11 +183,7 @@ - - - - - + @@ -247,11 +191,7 @@ - - - - - + @@ -259,11 +199,7 @@ - - - - - + @@ -271,44 +207,28 @@ - - - - - + - - - - - + - - - - - + - - - - - + @@ -316,11 +236,7 @@ - - - - - + @@ -328,11 +244,7 @@ - - - - - + @@ -340,11 +252,7 @@ - - - - - + @@ -352,55 +260,35 @@ - - - - - + - - - - - + - - - - - + - - - - - + - - - - - + @@ -408,11 +296,7 @@ - - - - - + @@ -420,11 +304,7 @@ - - - - - + @@ -432,11 +312,7 @@ - - - - - + @@ -444,11 +320,7 @@ - - - - - + @@ -457,13 +329,13 @@ - - + + - - + + diff --git a/tests/test-data/string.xml b/tests/test-data/string.xml index 8edbceba..e237ee22 100644 --- a/tests/test-data/string.xml +++ b/tests/test-data/string.xml @@ -9,7 +9,7 @@ - - - - - - + + - - - - - + + - - - - - + + - - - - - + + - - - - - + + - - - - - + + - - - - - + + - - - - - + + @@ -97,10 +73,10 @@ - + - + diff --git a/tests/test_galaxy_cli.py b/tests/test_galaxy_cli.py index c120f2f7..9decf211 100644 --- a/tests/test_galaxy_cli.py +++ b/tests/test_galaxy_cli.py @@ -84,6 +84,12 @@ def test_galaxy_cli_ifile_ctd(self): def test_galaxy_cli_ofile_ctd(self): self._compare_cli_output('ofile') + def test_galaxy_cli_ofile_corresponding_input_ctd(self): + self._compare_cli_output('ofile-corresponding-input') + + def test_galaxy_cli_ofile_typeparam(self): + self._compare_cli_output('ofile-typeparam') + def test_galaxy_cli_ofile_mult_ctd(self): self._compare_cli_output('ofile-mult') From e92ea6d16f1a3301a4912e058e0a7d44a2dbec9e Mon Sep 17 00:00:00 2001 From: M Bernt Date: Mon, 9 Dec 2019 14:10:18 +0100 Subject: [PATCH 044/121] remove . from element identifiers largely to avoid problems in testing: in tests the identifiers of the inputs contain the file extensions. When using these identifiers for the names of outputs OpenMS sometimes complains about wrong extensions, e.g. FileMerger with test input XYZ.mzML - `FileMerger -rt_concat:trafo_out XYZ.mzML` does not work because OpenMS insists that mzML is the wrong file type, but - `FileMerger -rt_concat:trafo_out XYZ_mzML works` --- galaxy/converter.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/galaxy/converter.py b/galaxy/converter.py index 2f08a798..cfc2f98a 100755 --- a/galaxy/converter.py +++ b/galaxy/converter.py @@ -511,11 +511,11 @@ def create_command(tool, model, **kwargs): if param.type is _InFile: preprocessing += "mkdir %s &&\n" %actual_parameter if param.is_list: - preprocessing += "${ ' && '.join([ \"ln -s '%s' '"+actual_parameter+"/%s.%s'\" % (_, re.sub('[^\w\-_.]', '_', _.element_identifier), _.ext) for _ in $" + actual_parameter + " if _ ]) } && \n" - command += "${' '.join([\"'"+actual_parameter+"/%s.%s'\"%(re.sub('[^\w\-_.]', '_', _.element_identifier),_.ext) for _ in $" + actual_parameter + " if _ ])}\n" + preprocessing += "${ ' && '.join([ \"ln -s '%s' '"+actual_parameter+"/%s.%s'\" % (_, re.sub('[^\w\-_]', '_', _.element_identifier), _.ext) for _ in $" + actual_parameter + " if _ ]) } && \n" + command += "${' '.join([\"'"+actual_parameter+"/%s.%s'\"%(re.sub('[^\w\-_]', '_', _.element_identifier),_.ext) for _ in $" + actual_parameter + " if _ ])}\n" else: - preprocessing += "ln -s '$"+ actual_parameter +"' '"+actual_parameter+"/${re.sub(\"[^\w\-_.]\", \"_\", $"+actual_parameter+".element_identifier)}.${"+actual_parameter+".ext}' &&\n" - command += "'"+actual_parameter+"/${re.sub('[^\w\-_.]', '_', $"+actual_parameter+".element_identifier)}.${"+actual_parameter+".ext}'\n" + preprocessing += "ln -s '$"+ actual_parameter +"' '"+actual_parameter+"/${re.sub(\"[^\w\-_]\", \"_\", $"+actual_parameter+".element_identifier)}.${"+actual_parameter+".ext}' &&\n" + command += "'"+actual_parameter+"/${re.sub('[^\w\-_]', '_', $"+actual_parameter+".element_identifier)}.${"+actual_parameter+".ext}'\n" elif param.type is _OutFile: # check if there is a parameter that sets the format # if so we add an extension to the generated files which will be used to @@ -537,9 +537,9 @@ def create_command(tool, model, **kwargs): if corresponding_input.advanced: actual_input_parameter = ADVANCED_OPTIONS_NAME+"cond." + actual_input_parameter if type_param is not None: - command += "${' '.join([\"'"+ actual_parameter +"/%s.%s'\"%(re.sub('[^\w\-_.]', '_', _.element_identifier), "+type_param_name+") for _ in $" + actual_input_parameter + " if _ ])}\n" + command += "${' '.join([\"'"+ actual_parameter +"/%s.%s'\"%(re.sub('[^\w\-_]', '_', _.element_identifier), "+type_param_name+") for _ in $" + actual_input_parameter + " if _ ])}\n" else: - command += "${' '.join([\"'"+ actual_parameter +"/%s'\"%(re.sub('[^\w\-_.]', '_', _.element_identifier)) for _ in $" + actual_input_parameter + " if _ ])}\n" + command += "${' '.join([\"'"+ actual_parameter +"/%s'\"%(re.sub('[^\w\-_]', '_', _.element_identifier)) for _ in $" + actual_input_parameter + " if _ ])}\n" else: if type_param is not None: preprocessing += "mkdir " + actual_parameter + " && \n" From aab34955c8221e41a30bc2f5cae4d270f9c7cd3d Mon Sep 17 00:00:00 2001 From: M Bernt Date: Tue, 10 Dec 2019 17:13:36 +0100 Subject: [PATCH 045/121] fixed output handling - added token with data type conversion functions galaxy <-> openms - improved macro parsing (try block now only covers the reading) --- galaxy/converter.py | 220 +++++++++++++++++++++++++++----------------- 1 file changed, 134 insertions(+), 86 deletions(-) diff --git a/galaxy/converter.py b/galaxy/converter.py index cfc2f98a..5732a887 100755 --- a/galaxy/converter.py +++ b/galaxy/converter.py @@ -109,16 +109,18 @@ def convert_models(args, parsed_ctds): # validate and prepare the passed arguments validate_and_prepare_args(args, parsed_ctds[0].ctd_model) + # parse the given supported file-formats file + supported_file_formats = parse_file_formats(args.formats_file) + # extract the names of the macros and check that we have found the ones we need macros_to_expand = parse_macros_files(args.macros_files, tool_version=args.tool_version, + supported_file_types=supported_file_formats, required_macros=REQUIRED_MACROS, dont_expand=[ADVANCED_OPTIONS_NAME+"macro", "references", "list_string_val", "list_string_san", "list_float_valsan", "list_integer_valsan"]) - # parse the given supported file-formats file - supported_file_formats = parse_file_formats(args.formats_file) # parse the skip/required tools files skip_tools = parse_tools_list_file(args.skip_tools_file) @@ -163,7 +165,7 @@ def parse_tools_list_file(tools_list_file): return tools_list -def parse_macros_files(macros_file_names, tool_version, required_macros = [], dont_expand=[]): +def parse_macros_files(macros_file_names, tool_version, supported_file_types, required_macros = [], dont_expand=[]): """ """ macros_to_expand = [] @@ -180,19 +182,6 @@ def parse_macros_files(macros_file_names, tool_version, required_macros = [], do continue logger.info("Macro %s found" % name, 1) macros_to_expand.append(name) - bump_galaxy_version = True - for xml_element in root.findall("token"): - if xml_element.attrib["name"] == "@TOOL_VERSION@": - if tool_version > xml_element.text: - bump_galaxy_version = False - xml_element.text = tool_version - if xml_element.attrib["name"] == "@GALAXY_VERSION@": - if bump_galaxy_version: - xml_element.text = str(int(xml_element.text) + 1) - else: - xml_element.text = "0" - - except ParseError as e: raise ApplicationException("The macros file " + macros_file_name + " could not be parsed. Cause: " + str(e)) @@ -201,6 +190,41 @@ def parse_macros_files(macros_file_names, tool_version, required_macros = [], do str(e)) else: macros_file.close() + + tool_ver_tk = root.find("token[@name='@TOOL_VERSION@']") + galaxy_ver_tk = root.find("token[@name='@GALAXY_VERSION@']") + if tool_ver_tk is None: + tool_ver_tk = add_child_node(root, "token", OrderedDict([("name", "@TOOL_VERSION@")])) + tool_ver_tk.text = tool_version + if galaxy_ver_tk is None: + galaxy_ver_tk = add_child_node(root, "token", OrderedDict([("name", "@GALAXY_VERSION@")])) + galaxy_ver_tk.text = "0" + if tool_version == tool_ver_tk.text: + galaxy_ver_tk.text = str(int(galaxy_ver_tk.text) + 1) + else: + tool_ver_tk.text = tool_version + galaxy_ver_tk.text = "0" + + ext_foo = root.find("token[@name='@EXT_FOO@']") + if ext_foo is None: + ext_foo = add_child_node(root, "token", OrderedDict([("name", "@EXT_FOO@")])) + + o2g = {} + g2o = {} + for s in supported_file_types: + o2g[s] = supported_file_types[s].galaxy_extension + if supported_file_types[s].galaxy_extension not in g2o: + g2o[supported_file_types[s].galaxy_extension] = s + ext_foo.text = CDATA("""#def oms2gxyext(o) + #set m=%s + #return m[o] +#end def +#def gxy2omsext(g) + #set m=%s + #return m[o] +#end def +"""%(str(o2g), str(g2o))) + with open(os.path.basename(macros_file_name), "w") as macros_file: tree = ElementTree(root) tree.write(macros_file, encoding="UTF-8", xml_declaration=True, pretty_print=True) @@ -462,13 +486,15 @@ def create_command(tool, model, **kwargs): """ # main command - final_preprocessing = "\n@QUOTE_FOO@\n" + final_preprocessing = "\n@QUOTE_FOO@\n@EXT_FOO@\n" advanced_preprocessing = "" final_command = "@EXECUTABLE@\n" final_command += kwargs["add_to_command_line"] + '\n' advanced_command_start = "#if ${aon}cond.{aon}selector=='advanced':\n".format(aon=ADVANCED_OPTIONS_NAME) advanced_command_end = "#end if" advanced_command = "" + final_postprocessing = "" + advanced_postprocessing = "" parameter_hardcoder = kwargs["parameter_hardcoder"] supported_file_formats = kwargs["supported_file_formats"] @@ -483,6 +509,7 @@ def create_command(tool, model, **kwargs): found_input_parameter = True command = "" preprocessing = "" + postprocessing = "" param_name = utils.extract_param_name(param) command_line_prefix = utils.extract_command_line_prefix(param, model) @@ -502,7 +529,7 @@ def create_command(tool, model, **kwargs): # all but bool params need the command line argument (bools have it already in the true/false value) if not is_boolean_parameter(param): command += command_line_prefix + " " - + # preprocessing for file inputs: # - create a dir with name param.name # - create a link to id.ext in this directory @@ -522,28 +549,43 @@ def create_command(tool, model, **kwargs): # determine the format in the output tag # in all other cases (corresponding input / there is only one allowed format) # the format will be set in the output tag + formats = get_formats(param, supported_file_formats, TYPE_TO_GALAXY_TYPE[param.type]) type_param = get_out_type_param(param, model) - if type_param is not None: - type_param_name = get_galaxy_parameter_name(type_param) - if type_param.advanced: - type_param_name = ADVANCED_OPTIONS_NAME+"cond." + type_param_name - if param.is_list: - preprocessing += "mkdir " + actual_parameter + " && \n" corresponding_input, fmt_from_corresponding = get_corresponding_input(param, model) if corresponding_input is None: raise Exception() actual_input_parameter = get_galaxy_parameter_name(corresponding_input) if corresponding_input.advanced: actual_input_parameter = ADVANCED_OPTIONS_NAME+"cond." + actual_input_parameter - if type_param is not None: + else: + actual_input_parameter = None + + if type_param is not None: + type_param_name = get_galaxy_parameter_name(type_param) + if type_param.advanced: + type_param_name = ADVANCED_OPTIONS_NAME+"cond." + type_param_name + if len(formats)==1: + if param.is_list: + preprocessing += "mkdir " + actual_parameter + " && \n" + command += "${' '.join([\"'"+ actual_parameter +"/%s'\"%(re.sub('[^\w\-_]', '_', _.element_identifier)) for _ in $" + actual_input_parameter + " if _ ])}\n" + else: + command += "'$" +actual_parameter+ "'\n" + elif type_param is not None: + if param.is_list: command += "${' '.join([\"'"+ actual_parameter +"/%s.%s'\"%(re.sub('[^\w\-_]', '_', _.element_identifier), "+type_param_name+") for _ in $" + actual_input_parameter + " if _ ])}\n" + postprocessing += "${' '.join([\"mv -n '"+ actual_parameter +"/%(id)s.%(omsext)s' '" +actual_parameter+ "/%(id)s.%(gext)s'\"%{\"id\": re.sub('[^\w\-_]', '_', _.element_identifier), \"omsext\":"+type_param_name+", \"gext\": oms2gxyext("+type_param_name+")} for _ in $" + actual_input_parameter + " if _ ])}\n" else: - command += "${' '.join([\"'"+ actual_parameter +"/%s'\"%(re.sub('[^\w\-_]', '_', _.element_identifier)) for _ in $" + actual_input_parameter + " if _ ])}\n" + # 1st create file with openms extension (often required by openms) + # then move it to the actual place specified by the parameter + # the format is then set by the tag using + preprocessing += "mkdir " + actual_parameter + " && \n" + command += "'" + actual_parameter+"/output.${"+type_param_name+"}'\n" + postprocessing += "&& mv '" + actual_parameter+"/output.${"+type_param_name+"}' '$" + actual_parameter + "'" else: - if type_param is not None: + if param.is_list: preprocessing += "mkdir " + actual_parameter + " && \n" - command += actual_parameter+"/output.${"+type_param_name+"}'\n" + command += "${' '.join([\"'"+ actual_parameter +"/%s.%s'\"%(re.sub('[^\w\-_]', '_', _.element_identifier), _.ext) for _ in $" + actual_input_parameter + " if _ ])}\n" else: command += "'$" +actual_parameter+ "'\n" @@ -577,19 +619,24 @@ def create_command(tool, model, **kwargs): advanced_command += "%s\n" % utils.indent(command) if preprocessing != "": advanced_preprocessing += "%s\n" % utils.indent(preprocessing) + if postprocessing != "": + advanced_postprocessing += "%s\n" % utils.indent(postprocessing) else: final_command += command final_preprocessing += preprocessing + final_postprocessing += postprocessing if advanced_command: final_command += "%s%s%s\n" % (advanced_command_start, advanced_command, advanced_command_end) if advanced_preprocessing: final_preprocessing += "%s%s%s\n" % (advanced_command_start, advanced_preprocessing, advanced_command_end) + if advanced_postprocessing: + final_postprocessing += "%s%s%s\n" % (advanced_command_start, advanced_postprocessing, advanced_command_end) if not found_output_parameter: final_command += "> $param_stdout\n" - final_command = final_preprocessing + final_command + final_command = final_preprocessing + final_command + final_postprocessing if found_input_parameter: final_command = "\n#import re" + final_command @@ -714,7 +761,6 @@ def get_input_with_same_restrictions(out_param, model, check_formats): matching.append(param) else: matching.append(param) - logger.error("gisr %s %s"%(out_param.name, [_.name for _ in matching])) if len(matching) == 1: return matching[0] else: @@ -782,6 +828,45 @@ def create_inputs(tool, model, **kwargs): return inputs_node +def get_formats(param, supported_file_formats, default = None): + """ + determine format attribute + - check if all listed possible formats are supported in Galaxy + - if there is a single one, then take this + - otherwise try to determine an input with the same restrictions and use this as format source + """ + formats = set() + if param.restrictions is not None: + if type(param.restrictions) is _FileFormat: + # set the first data output node to the first file format + + # check if there are formats that have not been registered yet... + output = list() + for format_name in param.restrictions.formats: + if format_name not in supported_file_formats.keys(): + output.append(str(format_name)) + + # warn only if there's about to complain + if output: + logger.warning("Parameter " + param.name + " has the following unsupported format(s):" + + ','.join(output), 1) + + formats = get_supported_file_types(param.restrictions.formats, supported_file_formats) + else: + raise InvalidModelException("Unrecognized restriction type [%(type)s] " + "for [%(name)s]" % {"type": type(param.restrictions), + "name": param.name}) + if len(formats) == 0: + if default != None: + formats.add(default) + else: + raise InvalidModelException("No supported formats [%(type)s] " + "for [%(name)s]" % {"type": type(param.restrictions), + "name": param.name}) + + return formats + + def create_param_attribute_list(param_node, param, model, supported_file_formats): """ get the attributes of input parameters @@ -816,20 +901,8 @@ def create_param_attribute_list(param_node, param, model, supported_file_formats if param.type is _InFile: # assume it's just text unless restrictions are provided - param_format = TYPE_TO_GALAXY_TYPE[_InFile] - if param.restrictions is not None: - # join all formats of the file, take mapping from supported_file if available for an entry - if type(param.restrictions) is _FileFormat: - param_format = ",".join(set([get_supported_file_type(i, supported_file_formats) if - get_supported_file_type(i, supported_file_formats) - else i for i in param.restrictions.formats])) - else: - raise InvalidModelException("Expected 'file type' restrictions for input file [%(name)s], " - "but instead got [%(type)s]" - % {"name": param.name, "type": type(param.restrictions)}) - param_node.attrib["type"] = "data" - param_node.attrib["format"] = param_format + param_node.attrib["format"] = ",".join(get_formats(param, supported_file_formats, TYPE_TO_GALAXY_TYPE[_InFile])) # in the case of multiple input set multiple flag if param.is_list: param_node.attrib["multiple"] = "true" @@ -850,8 +923,8 @@ def create_param_attribute_list(param_node, param, model, supported_file_formats elif type(param.restrictions) is _Choices: # if the parameter is used to select the output file type the options need to be replaced with the Galaxy data types - if is_out_type_param(param, model): - param.restrictions.choices = get_supported_file_types(param.restrictions.choices, supported_file_formats) +# if is_out_type_param(param, model): +# param.restrictions.choices = get_supported_file_types(param.restrictions.choices, supported_file_formats) # add a nothing selected option to mandatory options w/o default if param_node.attrib["optional"] == "False" and (param.default is None or param.default is _Null): @@ -886,9 +959,8 @@ def create_param_attribute_list(param_node, param, model, supported_file_formats if param.restrictions.n_max is not None: param_node.attrib["max"] = str(param.restrictions.n_max) elif type(param.restrictions) is _FileFormat: - param_node.attrib["format"] = ','.join(set([get_supported_file_type(i, supported_file_formats) if - get_supported_file_type(i, supported_file_formats) - else i for i in param.restrictions.formats])) + # has already been handled + pass else: raise InvalidModelException("Unrecognized restriction type [%(type)s] for parameter [%(name)s]" % {"type": type(param.restrictions), "name": param.name}) @@ -1128,41 +1200,12 @@ def create_output_node(parent, param, model, supported_file_formats): if data_node.attrib["name"].startswith('param_out_'): data_node.attrib["label"] = "${tool.name} on ${on_string}: %s" % data_node.attrib["name"][10:] - # determine format attribute - # - check if all listed possible formats are supported in Galaxy - # - if there is a single one, then take this - # - otherwise try to determine an input with the same restrictions and use this as format source - formats = set() - if param.restrictions is not None: - if type(param.restrictions) is _FileFormat: - # set the first data output node to the first file format - - # check if there are formats that have not been registered yet... - output = list() - for format_name in param.restrictions.formats: - if not format_name in supported_file_formats.keys(): - output.append(str(format_name)) - - # warn only if there's about to complain - if output: - logger.warning("Parameter " + param.name + " has the following unsupported format(s):" - + ','.join(output), 1) - - formats = get_supported_file_types(param.restrictions.formats, supported_file_formats) - if len(formats) == 0: - raise InvalidModelException("No supported formats [%(type)s] " - "for output [%(name)s]" % {"type": type(param.restrictions), - "name": param.name}) - else: - raise InvalidModelException("Unrecognized restriction type [%(type)s] " - "for output [%(name)s]" % {"type": type(param.restrictions), - "name": param.name}) - + formats = get_formats(param, supported_file_formats, TYPE_TO_GALAXY_TYPE[_OutFile]) type_param = get_out_type_param(param, model) corresponding_input, fmt_from_corresponding = get_corresponding_input(param, model) # if there is only a single possible output format we set this - logger.error("%s %s %s %s %s" %(param.name, formats, type_param, fmt_from_corresponding, corresponding_input)) +# logger.error("%s %s %s %s %s" %(param.name, formats, type_param, fmt_from_corresponding, corresponding_input)) if len(formats) == 1: discover_node.attrib["format"] = formats.pop() if param.is_list: @@ -1171,17 +1214,22 @@ def create_output_node(parent, param, model, supported_file_formats): # then this format was added as file extension on the CLI, now we can discover this elif type_param is not None: if not param.is_list: - discover_node = add_child_node(data_node, "discover_datasets", - OrderedDict([("directory", get_galaxy_parameter_name(param))])) - discover_node.attrib["pattern"] = "__name_and_ext__" + change_node = add_child_node(data_node, "change_format") + for r in type_param.restrictions.choices: + f = get_supported_file_type(r, supported_file_formats) + #TODO this should not happen for fully specified fileformats file + if f is None: + f = r + add_child_node(change_node, "when", OrderedDict([("input", get_galaxy_parameter_name(type_param)), ("value", r), ("format", f)])) + else: + discover_node.attrib["pattern"] = "__name_and_ext__" elif corresponding_input is not None: - if fmt_from_corresponding: - data_node.attrib["format_source"] = get_galaxy_parameter_name(corresponding_input) if param.is_list: - discover_node.attrib["pattern"] = "__name__" - data_node.attrib["structured_like"] = get_galaxy_parameter_name(corresponding_input) + discover_node.attrib["pattern"] = "__name_and_ext__" +# data_node.attrib["structured_like"] = get_galaxy_parameter_name(corresponding_input) #data_node.attrib["inherit_format"] = "true" else: + data_node.attrib["format_source"] = get_galaxy_parameter_name(corresponding_input) data_node.attrib["metadata_source"] = get_galaxy_parameter_name(corresponding_input) else: if not param.is_list: From 3c2902c948190a7f68821945a517666b0fd86333 Mon Sep 17 00:00:00 2001 From: M Bernt Date: Fri, 13 Dec 2019 16:09:11 +0100 Subject: [PATCH 046/121] updates: no if for optional strings - do not surround optional string parameters by if - updated test xmls for previous commit - added hardcoded params to test command --- galaxy/converter.py | 18 +- galaxy/macros.xml | 33 ++ tests/test-data/bool.xml | 2 + tests/test-data/empty.xml | 2 + tests/test-data/float.xml | 2 + tests/test-data/ifile.xml | 322 ++++++++++-------- tests/test-data/integer.xml | 2 + tests/test-data/ofile-corresponding-input.xml | 32 +- tests/test-data/ofile-mult.xml | 47 +-- tests/test-data/ofile-typeparam.xml | 158 +++++---- tests/test-data/ofile.xml | 20 +- tests/test-data/repeat.xml | 122 ++++--- tests/test-data/select.xml | 2 + tests/test-data/string.xml | 18 +- tests/test_galaxy_cli.py | 3 +- 15 files changed, 436 insertions(+), 347 deletions(-) diff --git a/galaxy/converter.py b/galaxy/converter.py index 5732a887..fdb08983 100755 --- a/galaxy/converter.py +++ b/galaxy/converter.py @@ -600,9 +600,13 @@ def create_command(tool, model, **kwargs): else: command += "'$" + actual_parameter + "'\n" - # add if statement for mandatory parameters and preprocessing - # - for optional outputs (param_out_x) the presence of the parameter depends on the additional input (param_x) - if not param.required and not is_boolean_parameter(param): # and not(param.type is _InFile and param.is_list): + # add if statement for optional parameters and preprocessing + # - for optional outputs (param_out_x) the presence of the parameter + # depends on the additional input (param_x) -> need no if + # - real string parameters (i.e. ctd type string wo restrictions) also + # need no if (otherwise the empty string could not be provided) + if not ( param.required or is_boolean_parameter(param) or (param.type == str and param.restrictions is None)): + # and not(param.type is _InFile and param.is_list): if param.type is _OutFile: if param.advanced: actual_parameter = ADVANCED_OPTIONS_NAME+"cond.%s" % get_galaxy_parameter_name(param, True) @@ -612,6 +616,7 @@ def create_command(tool, model, **kwargs): command = "#if $" + actual_parameter + ":\n" + utils.indent(command) + "\n#end if\n" else: command = "#if str($" + actual_parameter + "):\n" + utils.indent(command) + "\n#end if\n" + # TODO this should be if preprocessing != ""? if param.type is _InFile: preprocessing = "#if $" + actual_parameter + ":\n" + utils.indent(preprocessing) + "\n#end if\n" @@ -665,11 +670,13 @@ def import_macros(tool, model, **kwargs): # do not add the path of the file, rather, just its basename import_node.text = os.path.basename(macro_file.name) + def expand_macro(node, macro): expand_node = add_child_node(node, "expand") expand_node.attrib["macro"] = macro return expand_node + # and to "expand" the macros in a node def expand_macros(node, macros_to_expand): # add nodes @@ -884,7 +891,7 @@ def create_param_attribute_list(param_node, param, model, supported_file_formats raise ModelError("Unrecognized parameter type %(type)s for parameter %(name)s" % {"type": param.type, "name": param.name}) - # ITEMLIST is rendered as text field (even if its integers or floats), an + # ITEMLIST is rendered as text field (even if its integers or floats), an # exception is files which are treated a bit below if param.is_list: param_type = "text" @@ -922,7 +929,8 @@ def create_param_attribute_list(param_node, param, model, supported_file_formats create_boolean_parameter(param_node, param) elif type(param.restrictions) is _Choices: - # if the parameter is used to select the output file type the options need to be replaced with the Galaxy data types + # TODO if the parameter is used to select the output file type the + # options need to be replaced with the Galaxy data types # if is_out_type_param(param, model): # param.restrictions.choices = get_supported_file_types(param.restrictions.choices, supported_file_formats) diff --git a/galaxy/macros.xml b/galaxy/macros.xml index b0de2352..f0fa24e7 100644 --- a/galaxy/macros.xml +++ b/galaxy/macros.xml @@ -78,4 +78,37 @@ + + + +#def quote(s): + #set $s = [ _ for _ in $s.split(" ") if _ != "" ] + #set $q = False + #for $i, $p in enumerate($s): + #if $p == "": + #continue + #end if + #if $p.startswith('"'): + #set $q = True + #end if + #if not $q: + #set $s[i] = '"%s"'%p + #end if + #if $p.endswith('"'): + #set $q = False + #end if + #end for + #return " ".join($s) +#end def + + + diff --git a/tests/test-data/bool.xml b/tests/test-data/bool.xml index 5afe7e61..bd53df20 100644 --- a/tests/test-data/bool.xml +++ b/tests/test-data/bool.xml @@ -10,6 +10,8 @@ $param_stdout diff --git a/tests/test-data/float.xml b/tests/test-data/float.xml index 334207b4..0c14955d 100644 --- a/tests/test-data/float.xml +++ b/tests/test-data/float.xml @@ -10,6 +10,8 @@ $param_stdout @@ -262,53 +312,53 @@ ${ ' && '.join([ "ln -s '%s' '%s.%s'" % (_, re.sub('[^\w\-_.]', '_', _.element_i - - - - + + + + - - - - + + + + - - - - + + + + - - - - + + + + - - - - + + + + - - - - + + + + - - - - + + + + - - - - + + + + @@ -317,18 +367,18 @@ ${ ' && '.join([ "ln -s '%s' '%s.%s'" % (_, re.sub('[^\w\-_.]', '_', _.element_i - - + + - - + + - - + + - - + + diff --git a/tests/test-data/integer.xml b/tests/test-data/integer.xml index 8cd10526..49d85980 100644 --- a/tests/test-data/integer.xml +++ b/tests/test-data/integer.xml @@ -10,6 +10,8 @@ - - - - - - + + param_optional_noformat @@ -83,8 +77,8 @@ ln -s '$param_input_multformat' 'param_input_multformat/${re.sub("[^\w\-_.]", "_ param_optional_multformat - - + + adv_opts_cond['adv_opts_selector'] == 'advanced' and adv_opts_cond['param_optional_noformat_adv'] @@ -110,10 +104,10 @@ ln -s '$param_input_multformat' 'param_input_multformat/${re.sub("[^\w\-_.]", "_ - + - + diff --git a/tests/test-data/ofile-mult.xml b/tests/test-data/ofile-mult.xml index deaad2ad..a8fdb38c 100644 --- a/tests/test-data/ofile-mult.xml +++ b/tests/test-data/ofile-mult.xml @@ -12,44 +12,40 @@ @@ -59,12 +55,6 @@ ${ ' && '.join([ "ln -s '%s' 'param_inp_optional_mandatoryinput/%s.%s'" % (_, re - - - - - - @@ -74,11 +64,11 @@ ${ ' && '.join([ "ln -s '%s' 'param_inp_optional_mandatoryinput/%s.%s'" % (_, re param_optional_mandatoryinput - - + + - - + + param_optional_optionalinput @@ -88,9 +78,6 @@ ${ ' && '.join([ "ln -s '%s' 'param_inp_optional_mandatoryinput/%s.%s'" % (_, re - - - diff --git a/tests/test-data/ofile-typeparam.xml b/tests/test-data/ofile-typeparam.xml index f7a9ff3c..84b1fa3e 100644 --- a/tests/test-data/ofile-typeparam.xml +++ b/tests/test-data/ofile-typeparam.xml @@ -12,19 +12,16 @@ - - + + + - - + + + - - + + + - - + + + @@ -90,88 +92,110 @@ ln -s '$param_decoy_input_multformat' 'param_decoy_input_multformat/${re.sub("[^ - - - - - - + + + - - + + + - - + + + - - + + + - - - - - + + param_optional_noformat - + + + + + - + + + + + param_optional_multformat - - - - - + + adv_opts_cond['adv_opts_selector'] == 'advanced' and adv_opts_cond['param_optional_noformat_adv'] - + + + + + - + + + + + adv_opts_cond['adv_opts_selector'] == 'advanced' and adv_opts_cond['param_optional_multformat_adv'] - - + + - - + + - - + + - - + + - - - - + + + + + + + + + + + + + + + + - - - - - - + + param_optional_noformat @@ -70,8 +64,8 @@ param_optional_multformat - - + + adv_opts_cond['adv_opts_selector'] == 'advanced' and adv_opts_cond['param_optional_noformat_adv'] @@ -94,10 +88,10 @@ - + - + diff --git a/tests/test-data/repeat.xml b/tests/test-data/repeat.xml index 45b9992d..ff82b867 100644 --- a/tests/test-data/repeat.xml +++ b/tests/test-data/repeat.xml @@ -10,89 +10,83 @@ $param_stdout ]]> @@ -127,25 +121,25 @@ - - - - @@ -163,25 +157,25 @@ - - - - @@ -216,25 +210,25 @@ - - - - @@ -252,25 +246,25 @@ - - - - diff --git a/tests/test-data/select.xml b/tests/test-data/select.xml index ea1aee1d..9bc1f7c9 100644 --- a/tests/test-data/select.xml +++ b/tests/test-data/select.xml @@ -10,6 +10,8 @@ $param_stdout ]]> diff --git a/tests/test_galaxy_cli.py b/tests/test_galaxy_cli.py index 9decf211..b1eab620 100644 --- a/tests/test_galaxy_cli.py +++ b/tests/test_galaxy_cli.py @@ -17,12 +17,13 @@ def _compare_cli_output(self, fileprefix): in_pth = to_test_data('{}.ctd'.format(fileprefix)) macro_pth = to_test_data('macros.xml') ftypes_pth = to_test_data('filetypes.txt') + hcparam_pth = to_test_data('hardcoded_params.json') tmp = tempfile.mkdtemp() out_file = os.path.join(tmp, '{}.xml'.format(fileprefix)) #out_file = to_test_data('{}.xml'.format(fileprefix)) - cmd = ['CTDConverter', 'galaxy', '-i', in_pth, '-o', out_file, '-f', ftypes_pth, '-m', macro_pth, '-b', 'version', '--test-test'] + cmd = ['CTDConverter', 'galaxy', '-i', in_pth, '-o', out_file, '-f', ftypes_pth, '-m', macro_pth, '--test-test', '-p', hcparam_pth] print(cmd) popen = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) From 6981bed4c464dc8a3889d465a25fa2f7de17bdbd Mon Sep 17 00:00:00 2001 From: M Bernt Date: Thu, 19 Dec 2019 15:33:54 +0100 Subject: [PATCH 047/121] improvements test macros, type conversion, command line - automatically add empty test macros if missing - galaxy2openms datatype foo add txt if missing (because this is the fallback datatype) - simplified code for command line generation + pre/post-processing code for optional arguments get also an if --- common/utils.py | 2 +- galaxy/converter.py | 215 ++++++--- tests/test-data/bool.xml | 15 +- tests/test-data/empty.xml | 7 +- tests/test-data/float.xml | 135 +++--- tests/test-data/ifile.xml | 441 ++++++++++-------- tests/test-data/integer.xml | 135 +++--- tests/test-data/macros.xml | 44 +- tests/test-data/mock.py | 29 ++ tests/test-data/ofile-corresponding-input.ctd | 2 +- tests/test-data/ofile-corresponding-input.xml | 82 ++-- tests/test-data/ofile-mult.ctd | 2 +- tests/test-data/ofile-mult.xml | 87 ++-- tests/test-data/ofile-typeparam.ctd | 2 +- tests/test-data/ofile-typeparam.xml | 112 +++-- tests/test-data/ofile.ctd | 2 +- tests/test-data/ofile.xml | 59 ++- tests/test-data/repeat.xml | 159 ++++--- tests/test-data/select.xml | 135 +++--- tests/test-data/string.xml | 31 +- 20 files changed, 1024 insertions(+), 672 deletions(-) create mode 100755 tests/test-data/mock.py diff --git a/common/utils.py b/common/utils.py index 3f0a7428..6d4dda70 100644 --- a/common/utils.py +++ b/common/utils.py @@ -366,5 +366,5 @@ def indent(s, indentation=" "): @param indentation the desired indentation @return indented text """ - return "\n".join([indentation+_ for _ in s.splitlines()]) + return [indentation+_ for _ in s] diff --git a/galaxy/converter.py b/galaxy/converter.py index fdb08983..b5daf5b9 100755 --- a/galaxy/converter.py +++ b/galaxy/converter.py @@ -121,6 +121,7 @@ def convert_models(args, parsed_ctds): "list_string_val", "list_string_san", "list_float_valsan", "list_integer_valsan"]) + check_test_macros(args.test_macros_files, args.test_macros_prefix, parsed_ctds) # parse the skip/required tools files skip_tools = parse_tools_list_file(args.skip_tools_file) @@ -139,6 +140,9 @@ def convert_models(args, parsed_ctds): test_macros_prefix=args.test_macros_prefix, tool_version = args.tool_version) + + + # generation of galaxy stubs is ready... now, let's see if we need to generate a tool_conf.xml if args.tool_conf_destination is not None: generate_tool_conf(parsed_ctds, args.tool_conf_destination, @@ -185,6 +189,8 @@ def parse_macros_files(macros_file_names, tool_version, supported_file_types, re except ParseError as e: raise ApplicationException("The macros file " + macros_file_name + " could not be parsed. Cause: " + str(e)) + + except IOError as e: raise ApplicationException("The macros file " + macros_file_name + " could not be opened. Cause: " + str(e)) @@ -215,17 +221,21 @@ def parse_macros_files(macros_file_names, tool_version, supported_file_types, re o2g[s] = supported_file_types[s].galaxy_extension if supported_file_types[s].galaxy_extension not in g2o: g2o[supported_file_types[s].galaxy_extension] = s + # make sure that the backup data type is in the map + if 'txt' not in g2o: + g2o['txt'] = 'txt' + ext_foo.text = CDATA("""#def oms2gxyext(o) #set m=%s #return m[o] #end def #def gxy2omsext(g) #set m=%s - #return m[o] + #return m[g] #end def """%(str(o2g), str(g2o))) - with open(os.path.basename(macros_file_name), "w") as macros_file: + with open(macros_file_name, "w") as macros_file: tree = ElementTree(root) tree.write(macros_file, encoding="UTF-8", xml_declaration=True, pretty_print=True) @@ -252,10 +262,47 @@ def parse_macros_files(macros_file_names, tool_version, supported_file_types, re return macros_to_expand +def check_test_macros(test_macros_files, test_macros_prefix, parsed_ctds): + + tool_ids = set() + for parsed_ctd in parsed_ctds: + model = parsed_ctd.ctd_model + tool_ids.add( model.name.replace(" ","_") ) + + for mf, mp in zip(test_macros_files, test_macros_prefix): + macro_ids = set() + try: + with open(mf) as macros_file: + root = parse(macros_file).getroot() + for xml_element in root.findall("xml"): + name = xml_element.attrib["name"] + if not name.startswith(mp): + logger.warning("Testmacro with invalid prefix %s." %(mp), 0) + continue + name = name[len(mp):] + macro_ids.add(name) + + except ParseError as e: + raise ApplicationException("The macros file " + mf + " could not be parsed. Cause: " + + str(e)) + except IOError as e: + raise ApplicationException("The macros file " + mf + " could not be opened. Cause: " + + str(e)) + for t in tool_ids - macro_ids: + logger.error("missing %s"%t) + add_child_node(root, "xml", OrderedDict([("name", mp+t)])) + + if len( macro_ids - tool_ids): + logger.warning("Unnecessary macros in %s: %s"%(mf, macro_ids - tool_ids)) + with open(mf, "w") as macros_file: + tree = ElementTree(root) + tree.write(macros_file, encoding="UTF-8", xml_declaration=True, pretty_print=True) + + def parse_file_formats(formats_file): """ """ - supported_formats = {} + supported_formats = OrderedDict() if formats_file is not None: line_number = 0 with open(formats_file) as f: @@ -372,6 +419,7 @@ def _convert_internal(parsed_ctds, **kwargs): create_description(tool, model) import_macros(tool, model, **kwargs) expand_macros(tool, kwargs["macros_to_expand"]) +# command, inputs, outputs = create_cio(tool, model, **kwargs) create_command(tool, model, **kwargs) inputs = create_inputs(tool, model, **kwargs) outputs = create_outputs(tool, model, **kwargs) @@ -390,6 +438,25 @@ def _convert_internal(parsed_ctds, **kwargs): with open(output_file, 'w') as fh: tree.write(fh, encoding="UTF-8", xml_declaration=True, pretty_print=True) +# def create_cio(tool, model, **kwargs): +# cnode = add_child_node(tool, "command", OrderedDict([("detect_errors", "exit_code")])) +# inode = add_child_node(tool, "inputs") +# onode = add_child_node(parent, "outputs") +# +# parameter_hardcoder = kwargs["parameter_hardcoder"] +# for param in utils.extract_and_flatten_parameters(model): +# param = modify_param_for_galaxy(param) +# # no need to show hardcoded parameters +# hardcoded_value = parameter_hardcoder.get_hardcoded_value(param.name, model.name) +# +# if parameter_hardcoder.get_blacklist(param.name, model.name) or hardcoded_value: +# continue +# if not param.type is _OutFile: +# continue +# create_output_node(outputs_node, param, model, kwargs["supported_file_formats"]) +# +# return cnode, inode, onode + def write_header(tool, model): """ @@ -486,15 +553,15 @@ def create_command(tool, model, **kwargs): """ # main command - final_preprocessing = "\n@QUOTE_FOO@\n@EXT_FOO@\n" - advanced_preprocessing = "" - final_command = "@EXECUTABLE@\n" - final_command += kwargs["add_to_command_line"] + '\n' - advanced_command_start = "#if ${aon}cond.{aon}selector=='advanced':\n".format(aon=ADVANCED_OPTIONS_NAME) + final_cmd = OrderedDict([('preprocessing', []), ('command', []), ('postprocessing', [])]) + advanced_cmd = {'preprocessing': [], 'command': [], 'postprocessing': []} + + final_cmd['preprocessing'].extend(["@QUOTE_FOO@", "@EXT_FOO@"]) + final_cmd['command'].append("@EXECUTABLE@") + final_cmd['command'].extend(kwargs["add_to_command_line"]) + + advanced_command_start = "#if ${aon}cond.{aon}selector=='advanced':".format(aon=ADVANCED_OPTIONS_NAME) advanced_command_end = "#end if" - advanced_command = "" - final_postprocessing = "" - advanced_postprocessing = "" parameter_hardcoder = kwargs["parameter_hardcoder"] supported_file_formats = kwargs["supported_file_formats"] @@ -507,18 +574,18 @@ def create_command(tool, model, **kwargs): found_output_parameter = True if param.type is _InFile: found_input_parameter = True - command = "" - preprocessing = "" - postprocessing = "" + + param_cmd = {'preprocessing': [], 'command': [], 'postprocessing': []} param_name = utils.extract_param_name(param) command_line_prefix = utils.extract_command_line_prefix(param, model) - if parameter_hardcoder.get_blacklist(param_name, model.name): + # TODO use utils.extract_param_name(param).replace(":", "_")? Then hardcoding ctd variables (with :) and tool variables (with _) can be distinguished + if parameter_hardcoder.get_blacklist(utils.extract_param_name(param), model.name): continue hardcoded_value = parameter_hardcoder.get_hardcoded_value(param_name, model.name) if hardcoded_value is not None: - command += "%s %s\n" % (command_line_prefix, hardcoded_value) + param_cmd['command'].append("%s %s" % (command_line_prefix, hardcoded_value)) else: # in the else branch the parameter is neither blacklisted nor hardcoded... @@ -528,7 +595,7 @@ def create_command(tool, model, **kwargs): # all but bool params need the command line argument (bools have it already in the true/false value) if not is_boolean_parameter(param): - command += command_line_prefix + " " + param_cmd['command'].append(command_line_prefix) # preprocessing for file inputs: # - create a dir with name param.name @@ -536,13 +603,13 @@ def create_command(tool, model, **kwargs): # rationale: in the autogenerated tests the same file was used as input to multiple parameters # this leads to conflicts while linking... might also be better in general if param.type is _InFile: - preprocessing += "mkdir %s &&\n" %actual_parameter + param_cmd['preprocessing'].append("mkdir %s &&" %actual_parameter) if param.is_list: - preprocessing += "${ ' && '.join([ \"ln -s '%s' '"+actual_parameter+"/%s.%s'\" % (_, re.sub('[^\w\-_]', '_', _.element_identifier), _.ext) for _ in $" + actual_parameter + " if _ ]) } && \n" - command += "${' '.join([\"'"+actual_parameter+"/%s.%s'\"%(re.sub('[^\w\-_]', '_', _.element_identifier),_.ext) for _ in $" + actual_parameter + " if _ ])}\n" + param_cmd['preprocessing'].append("${ ' && '.join([ \"ln -s '%s' '"+actual_parameter+"/%s.%s'\" % (_, re.sub('[^\w\-_]', '_', _.element_identifier),$gxy2omsext(_.ext)) for _ in $" + actual_parameter + " if _ ]) } &&") + param_cmd['command'].append("${' '.join([\"'"+actual_parameter+"/%s.%s'\"%(re.sub('[^\w\-_]', '_', _.element_identifier),$gxy2omsext(_.ext)) for _ in $" + actual_parameter + " if _ ])}") else: - preprocessing += "ln -s '$"+ actual_parameter +"' '"+actual_parameter+"/${re.sub(\"[^\w\-_]\", \"_\", $"+actual_parameter+".element_identifier)}.${"+actual_parameter+".ext}' &&\n" - command += "'"+actual_parameter+"/${re.sub('[^\w\-_]', '_', $"+actual_parameter+".element_identifier)}.${"+actual_parameter+".ext}'\n" + param_cmd['preprocessing'].append("ln -s '$"+ actual_parameter +"' '"+actual_parameter+"/${re.sub(\"[^\w\-_]\", \"_\", $"+actual_parameter+".element_identifier)}.$gxy2omsext($"+actual_parameter+".ext)' &&") + param_cmd['command'].append("'"+actual_parameter+"/${re.sub('[^\w\-_]', '_', $"+actual_parameter+".element_identifier)}.${gxy2omsext($"+actual_parameter+".ext)}'") elif param.type is _OutFile: # check if there is a parameter that sets the format # if so we add an extension to the generated files which will be used to @@ -550,7 +617,7 @@ def create_command(tool, model, **kwargs): # in all other cases (corresponding input / there is only one allowed format) # the format will be set in the output tag formats = get_formats(param, supported_file_formats, TYPE_TO_GALAXY_TYPE[param.type]) - type_param = get_out_type_param(param, model) + type_param = get_out_type_param(param, model, parameter_hardcoder) if param.is_list: corresponding_input, fmt_from_corresponding = get_corresponding_input(param, model) if corresponding_input is None: @@ -567,38 +634,38 @@ def create_command(tool, model, **kwargs): type_param_name = ADVANCED_OPTIONS_NAME+"cond." + type_param_name if len(formats)==1: if param.is_list: - preprocessing += "mkdir " + actual_parameter + " && \n" - command += "${' '.join([\"'"+ actual_parameter +"/%s'\"%(re.sub('[^\w\-_]', '_', _.element_identifier)) for _ in $" + actual_input_parameter + " if _ ])}\n" + param_cmd['preprocessing'].append("mkdir " + actual_parameter + " &&") + param_cmd['command'].append("${' '.join([\"'"+ actual_parameter +"/%s'\"%(re.sub('[^\w\-_]', '_', _.element_identifier)) for _ in $" + actual_input_parameter + " if _ ])}") else: - command += "'$" +actual_parameter+ "'\n" + param_cmd['command'].append("'$" +actual_parameter+ "'") elif type_param is not None: if param.is_list: - command += "${' '.join([\"'"+ actual_parameter +"/%s.%s'\"%(re.sub('[^\w\-_]', '_', _.element_identifier), "+type_param_name+") for _ in $" + actual_input_parameter + " if _ ])}\n" - postprocessing += "${' '.join([\"mv -n '"+ actual_parameter +"/%(id)s.%(omsext)s' '" +actual_parameter+ "/%(id)s.%(gext)s'\"%{\"id\": re.sub('[^\w\-_]', '_', _.element_identifier), \"omsext\":"+type_param_name+", \"gext\": oms2gxyext("+type_param_name+")} for _ in $" + actual_input_parameter + " if _ ])}\n" + param_cmd['command'].append("${' '.join([\"'"+ actual_parameter +"/%s.%s'\"%(re.sub('[^\w\-_]', '_', _.element_identifier), "+type_param_name+") for _ in $" + actual_input_parameter + " if _ ])}") + param_cmd['postprocessing'].append("${' '.join([\"mv -n '"+ actual_parameter +"/%(id)s.%(omsext)s' '" +actual_parameter+ "/%(id)s.%(gext)s'\"%{\"id\": re.sub('[^\w\-_]', '_', _.element_identifier), \"omsext\":"+type_param_name+", \"gext\": oms2gxyext("+type_param_name+")} for _ in $" + actual_input_parameter + " if _ ])}") else: # 1st create file with openms extension (often required by openms) # then move it to the actual place specified by the parameter # the format is then set by the tag using - preprocessing += "mkdir " + actual_parameter + " && \n" - command += "'" + actual_parameter+"/output.${"+type_param_name+"}'\n" - postprocessing += "&& mv '" + actual_parameter+"/output.${"+type_param_name+"}' '$" + actual_parameter + "'" + param_cmd['preprocessing'].append("mkdir " + actual_parameter + " &&") + param_cmd['command'].append("'" + actual_parameter+"/output.${"+type_param_name+"}'") + param_cmd['postprocessing'].append("&& mv '" + actual_parameter+"/output.${"+type_param_name+"}' '$" + actual_parameter + "'") else: if param.is_list: - preprocessing += "mkdir " + actual_parameter + " && \n" - command += "${' '.join([\"'"+ actual_parameter +"/%s.%s'\"%(re.sub('[^\w\-_]', '_', _.element_identifier), _.ext) for _ in $" + actual_input_parameter + " if _ ])}\n" + param_cmd['preprocessing'].append("mkdir " + actual_parameter + " && ") + param_cmd['command'].append("${' '.join([\"'"+ actual_parameter +"/%s.%s'\"%(re.sub('[^\w\-_]', '_', _.element_identifier), _.ext) for _ in $" + actual_input_parameter + " if _ ])}") else: - command += "'$" +actual_parameter+ "'\n" + param_cmd['command'].append("'$" +actual_parameter+ "'") # select with multiple = true elif is_selection_parameter(param) and param.is_list: - command += "${' '.join([\"'%s'\"%str(_) for _ in str($" + actual_parameter + ").split(',')])}\n" + param_cmd['command'].append("${' '.join([\"'%s'\"%str(_) for _ in str($" + actual_parameter + ").split(',')])}") elif param.is_list: - command += "$quote($%s" % actual_parameter + ")\n" + param_cmd['command'].append("$quote($%s" % actual_parameter + ")") #command += "${' '.join([\"'%s'\"%str(_) for _ in $" + actual_parameter + "])}\n" elif is_boolean_parameter(param): - command += "$%s" % actual_parameter + "\n" + param_cmd['command'].append("$%s" % actual_parameter + "") else: - command += "'$" + actual_parameter + "'\n" + param_cmd['command'].append("'$" + actual_parameter + "'") # add if statement for optional parameters and preprocessing # - for optional outputs (param_out_x) the presence of the parameter @@ -612,43 +679,40 @@ def create_command(tool, model, **kwargs): actual_parameter = ADVANCED_OPTIONS_NAME+"cond.%s" % get_galaxy_parameter_name(param, True) else: actual_parameter = "%s" % get_galaxy_parameter_name(param, True) - if is_selection_parameter(param) or param.type == _InFile: - command = "#if $" + actual_parameter + ":\n" + utils.indent(command) + "\n#end if\n" - else: - command = "#if str($" + actual_parameter + "):\n" + utils.indent(command) + "\n#end if\n" - # TODO this should be if preprocessing != ""? - if param.type is _InFile: - preprocessing = "#if $" + actual_parameter + ":\n" + utils.indent(preprocessing) + "\n#end if\n" - - if param.advanced and hardcoded_value is None: - advanced_command += "%s\n" % utils.indent(command) - if preprocessing != "": - advanced_preprocessing += "%s\n" % utils.indent(preprocessing) - if postprocessing != "": - advanced_postprocessing += "%s\n" % utils.indent(postprocessing) - else: - final_command += command - final_preprocessing += preprocessing - final_postprocessing += postprocessing - if advanced_command: - final_command += "%s%s%s\n" % (advanced_command_start, advanced_command, advanced_command_end) - if advanced_preprocessing: - final_preprocessing += "%s%s%s\n" % (advanced_command_start, advanced_preprocessing, advanced_command_end) - if advanced_postprocessing: - final_postprocessing += "%s%s%s\n" % (advanced_command_start, advanced_postprocessing, advanced_command_end) + for stage in param_cmd: + if len(param_cmd[stage]) == 0: + continue + if is_selection_parameter(param) or param.type == _InFile: + param_cmd[stage] = ["#if $" + actual_parameter + ":"] + utils.indent(param_cmd[stage]) + ["#end if"] + else: + param_cmd[stage] = ["#if str($" + actual_parameter + "):"] + utils.indent(param_cmd[stage]) + ["#end if"] + + for stage in param_cmd: + if len(param_cmd[stage]) == 0: + continue + if param.advanced and hardcoded_value is None: + advanced_cmd[stage].extend(utils.indent(param_cmd[stage])) + else: + final_cmd[stage].extend(utils.indent(param_cmd[stage])) + + for stage in advanced_cmd: + if len(advanced_cmd[stage])==0: + continue + advanced_cmd[stage] = [advanced_command_start] + advanced_cmd[stage] + [advanced_command_end] + final_cmd[stage].extend(advanced_cmd[stage]) if not found_output_parameter: - final_command += "> $param_stdout\n" + final_cmd['command'].append("> $param_stdout") - final_command = final_preprocessing + final_command + final_postprocessing if found_input_parameter: - final_command = "\n#import re" + final_command + final_cmd['command'] = ["#import re"] + final_cmd['command'] command_node = add_child_node(tool, "command") command_node.attrib["detect_errors"] = "exit_code" - command_node.text = CDATA(final_command) + + command_node.text = CDATA("\n".join(sum(final_cmd.values(), []))) # creates the xml elements needed to import the needed macros files @@ -705,13 +769,16 @@ def get_galaxy_parameter_name(param, inparam = False): return "param_%s" % p -def get_out_type_param(out_param, model): +def get_out_type_param(out_param, model, parameter_hardcoder): """ check if there is a parameter that has the same name with appended _type and return it if present, otherwise return None """ + if parameter_hardcoder.get_blacklist(out_param.name + "_type", model.name): + return None for param in utils.extract_and_flatten_parameters(model): + if param.name == out_param.name + "_type": return param return None @@ -792,8 +859,8 @@ def create_inputs(tool, model, **kwargs): for param in utils.extract_and_flatten_parameters(model): param = modify_param_for_galaxy(param) # no need to show hardcoded parameters - hardcoded_value = parameter_hardcoder.get_hardcoded_value(param.name, model.name) - if parameter_hardcoder.get_blacklist(param.name, model.name) or not hardcoded_value is None: + hardcoded_value = parameter_hardcoder.get_hardcoded_value(utils.extract_param_name(param), model.name) + if parameter_hardcoder.get_blacklist(utils.extract_param_name(param), model.name) or not hardcoded_value is None: continue # optional outfiles: create an additional bool input which is used to filter for the output @@ -1174,13 +1241,13 @@ def create_outputs(parent, model, **kwargs): for param in utils.extract_and_flatten_parameters(model): param = modify_param_for_galaxy(param) # no need to show hardcoded parameters - hardcoded_value = parameter_hardcoder.get_hardcoded_value(param.name, model.name) - if parameter_hardcoder.get_blacklist(param.name, model.name) or hardcoded_value: + hardcoded_value = parameter_hardcoder.get_hardcoded_value(utils.extract_param_name(param), model.name) + if parameter_hardcoder.get_blacklist(utils.extract_param_name(param), model.name) or hardcoded_value: # let's not use an extra level of indentation and use NOP continue if not param.type is _OutFile: continue - create_output_node(outputs_node, param, model, kwargs["supported_file_formats"]) + create_output_node(outputs_node, param, model, kwargs["supported_file_formats"], parameter_hardcoder) # If there are no outputs defined in the ctd the node will have no children # and the stdout will be used as output @@ -1190,7 +1257,7 @@ def create_outputs(parent, model, **kwargs): return outputs_node -def create_output_node(parent, param, model, supported_file_formats): +def create_output_node(parent, param, model, supported_file_formats, parameter_hardcoder): # add a data node / collection + discover_datasets # in the former case we just set the discover_node equal to the data node @@ -1209,7 +1276,7 @@ def create_output_node(parent, param, model, supported_file_formats): data_node.attrib["label"] = "${tool.name} on ${on_string}: %s" % data_node.attrib["name"][10:] formats = get_formats(param, supported_file_formats, TYPE_TO_GALAXY_TYPE[_OutFile]) - type_param = get_out_type_param(param, model) + type_param = get_out_type_param(param, model, parameter_hardcoder) corresponding_input, fmt_from_corresponding = get_corresponding_input(param, model) # if there is only a single possible output format we set this diff --git a/tests/test-data/bool.xml b/tests/test-data/bool.xml index bd53df20..859e90c1 100644 --- a/tests/test-data/bool.xml +++ b/tests/test-data/bool.xml @@ -9,23 +9,20 @@ - $param_stdout -]]> +> $param_stdout]]> diff --git a/tests/test-data/empty.xml b/tests/test-data/empty.xml index 72bf323d..2cc8f0f3 100644 --- a/tests/test-data/empty.xml +++ b/tests/test-data/empty.xml @@ -9,13 +9,10 @@ - $param_stdout -]]> +> $param_stdout]]> diff --git a/tests/test-data/float.xml b/tests/test-data/float.xml index 0c14955d..853cccc6 100644 --- a/tests/test-data/float.xml +++ b/tests/test-data/float.xml @@ -9,79 +9,108 @@ - $param_stdout -]]> +> $param_stdout]]> diff --git a/tests/test-data/ifile.xml b/tests/test-data/ifile.xml index 0d3d5e42..1d2ad891 100644 --- a/tests/test-data/ifile.xml +++ b/tests/test-data/ifile.xml @@ -9,306 +9,351 @@ - $param_stdout -]]> +> $param_stdout]]> diff --git a/tests/test-data/integer.xml b/tests/test-data/integer.xml index 49d85980..0d2bfbb1 100644 --- a/tests/test-data/integer.xml +++ b/tests/test-data/integer.xml @@ -9,79 +9,108 @@ - $param_stdout -]]> +> $param_stdout]]> diff --git a/tests/test-data/macros.xml b/tests/test-data/macros.xml index dd30fab7..bccb9956 100644 --- a/tests/test-data/macros.xml +++ b/tests/test-data/macros.xml @@ -1,7 +1,7 @@ - 2.3 - 0 + 2.3.0 + 8 openms @@ -35,7 +35,7 @@ - + ^ *((?:\"[^\"]*\" +)|(?:[^ \"]+ +))*((?:\"[^\"]*\")|(?:[^ \"]+)) *$ @@ -51,7 +51,7 @@ - + ^ *[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?( *[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?)* *$ @@ -65,7 +65,7 @@ - + ^ *[+-]?[0-9]+( *[+-]?[0-9]+)* *$ @@ -75,4 +75,36 @@ - + + + +#def quote(s): + #set $s = [ _ for _ in $s.split(" ") if _ != "" ] + #set $q = False + #for $i, $p in enumerate($s): + #if $p == "": + #continue + #end if + #if $p.startswith('"'): + #set $q = True + #end if + #if not $q: + #set $s[i] = '"%s"'%p + #end if + #if $p.endswith('"'): + #set $q = False + #end if + #end for + #return " ".join($s) +#end def + + + diff --git a/tests/test-data/mock.py b/tests/test-data/mock.py new file mode 100755 index 00000000..b7d48120 --- /dev/null +++ b/tests/test-data/mock.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python + +# little mock app to handle ouput file parameters +# i.e. to create them + +import os +import re +import sys + +wd=os.path.dirname(__file__) + +oparam=set() +ctd = sys.argv[1] +with open(os.path.join(wd, ctd)) as cf: + for line in cf: + m = re.search(r'type="output-file"', line) + if m is None: + continue + m = re.search(r'name="([^"]+)"', line) + oparam.add(m.group(1)) + +i = 2 +while i < len(sys.argv): + if sys.argv[i].startswith("-"): + param = sys.argv[i][1:] + if param in oparam: + of = open(sys.argv[i+1], "w") + of.close() + i += 1 diff --git a/tests/test-data/ofile-corresponding-input.ctd b/tests/test-data/ofile-corresponding-input.ctd index 5e35e051..c4bd3c65 100644 --- a/tests/test-data/ofile-corresponding-input.ctd +++ b/tests/test-data/ofile-corresponding-input.ctd @@ -1,5 +1,5 @@ - + diff --git a/tests/test-data/ofile-corresponding-input.xml b/tests/test-data/ofile-corresponding-input.xml index a6b2f402..f7d5c372 100644 --- a/tests/test-data/ofile-corresponding-input.xml +++ b/tests/test-data/ofile-corresponding-input.xml @@ -1,56 +1,68 @@ - + Outut File parameter tests-corresponding-input - echo outFileTest-corresponding-input + '$__tool_directory__/mock.py' ofile-corresponding-input.ctd macros.xml - +#end if]]> diff --git a/tests/test-data/ofile-mult.ctd b/tests/test-data/ofile-mult.ctd index c585ba77..5ea7ec6d 100644 --- a/tests/test-data/ofile-mult.ctd +++ b/tests/test-data/ofile-mult.ctd @@ -1,5 +1,5 @@ - + diff --git a/tests/test-data/ofile-mult.xml b/tests/test-data/ofile-mult.xml index a8fdb38c..ffead8fe 100644 --- a/tests/test-data/ofile-mult.xml +++ b/tests/test-data/ofile-mult.xml @@ -1,53 +1,62 @@ - + multiple Output File parameter tests - echo MultoutFileTest + '$__tool_directory__/mock.py' ofile-mult.ctd macros.xml - + -mandatory_mandatoryinput + ${' '.join(["'param_out_mandatory_mandatoryinput/%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier)) for _ in $param_inp_mandatory_mandatoryinput if _ ])} + #if str($param_optional_mandatoryinput): + -optional_mandatoryinput + ${' '.join(["'param_out_optional_mandatoryinput/%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier)) for _ in $param_inp_optional_mandatoryinput if _ ])} + #end if + -mandatory_optionalinput + ${' '.join(["'param_out_mandatory_optionalinput/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier), _.ext) for _ in $param_inp_mandatory_optionalinput if _ ])} + #if str($param_optional_optionalinput): + -optional_optionalinput + ${' '.join(["'param_out_optional_optionalinput/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier), _.ext) for _ in $param_inp_optional_optionalinput if _ ])} + #end if + -inp_mandatory_mandatoryinput + ${' '.join(["'param_inp_mandatory_mandatoryinput/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier),$gxy2omsext(_.ext)) for _ in $param_inp_mandatory_mandatoryinput if _ ])} + -inp_optional_mandatoryinput + ${' '.join(["'param_inp_optional_mandatoryinput/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier),$gxy2omsext(_.ext)) for _ in $param_inp_optional_mandatoryinput if _ ])} + #if $param_inp_mandatory_optionalinput: + -inp_mandatory_optionalinput + ${' '.join(["'param_inp_mandatory_optionalinput/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier),$gxy2omsext(_.ext)) for _ in $param_inp_mandatory_optionalinput if _ ])} + #end if + #if $param_inp_optional_optionalinput: + -inp_optional_optionalinput + ${' '.join(["'param_inp_optional_optionalinput/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier),$gxy2omsext(_.ext)) for _ in $param_inp_optional_optionalinput if _ ])} + #end if]]> diff --git a/tests/test-data/ofile-typeparam.ctd b/tests/test-data/ofile-typeparam.ctd index 1dfb3ba8..164e92e7 100644 --- a/tests/test-data/ofile-typeparam.ctd +++ b/tests/test-data/ofile-typeparam.ctd @@ -1,5 +1,5 @@ - + diff --git a/tests/test-data/ofile-typeparam.xml b/tests/test-data/ofile-typeparam.xml index 84b1fa3e..314e50fd 100644 --- a/tests/test-data/ofile-typeparam.xml +++ b/tests/test-data/ofile-typeparam.xml @@ -1,66 +1,92 @@ - + Outut File parameter tests-typeparam - echo outFileTest-typeparam + '$__tool_directory__/mock.py' ofile-typeparam.ctd macros.xml - + #if str($adv_opts_cond.param_optional_multformat_adv): + && mv 'param_out_optional_multformat_adv/output.${adv_opts_cond.param_optional_multformat_adv_type}' '$param_out_optional_multformat_adv' + #end if +#end if]]> diff --git a/tests/test-data/ofile.ctd b/tests/test-data/ofile.ctd index 2c031f5c..4b042bcf 100644 --- a/tests/test-data/ofile.ctd +++ b/tests/test-data/ofile.ctd @@ -1,5 +1,5 @@ - + diff --git a/tests/test-data/ofile.xml b/tests/test-data/ofile.xml index b6fac06f..6e1c0194 100644 --- a/tests/test-data/ofile.xml +++ b/tests/test-data/ofile.xml @@ -1,46 +1,55 @@ - + Outut File parameter tests - echo outFileTest + '$__tool_directory__/mock.py' ofile.ctd macros.xml - +#end if]]> diff --git a/tests/test-data/repeat.xml b/tests/test-data/repeat.xml index ff82b867..61d92310 100644 --- a/tests/test-data/repeat.xml +++ b/tests/test-data/repeat.xml @@ -9,87 +9,124 @@ - $param_stdout -]]> +> $param_stdout]]> diff --git a/tests/test-data/select.xml b/tests/test-data/select.xml index 9bc1f7c9..dd1b74ca 100644 --- a/tests/test-data/select.xml +++ b/tests/test-data/select.xml @@ -9,79 +9,108 @@ - $param_stdout -]]> +> $param_stdout]]> diff --git a/tests/test-data/string.xml b/tests/test-data/string.xml index 9bcaeba2..4383c7d1 100644 --- a/tests/test-data/string.xml +++ b/tests/test-data/string.xml @@ -9,23 +9,28 @@ - $param_stdout -]]> +> $param_stdout]]> From 977995939ea05e91aceec9087d30d542b7e4d477 Mon Sep 17 00:00:00 2001 From: M Bernt Date: Thu, 19 Dec 2019 16:07:16 +0100 Subject: [PATCH 048/121] single quotes for itemlist strings --- galaxy/macros.xml | 22 ++++++++++++---------- tests/test-data/macros.xml | 36 +++++++++++++++++++++--------------- 2 files changed, 33 insertions(+), 25 deletions(-) diff --git a/galaxy/macros.xml b/galaxy/macros.xml index f0fa24e7..b3e74a9a 100644 --- a/galaxy/macros.xml +++ b/galaxy/macros.xml @@ -38,7 +38,7 @@ - + @@ -88,27 +88,29 @@ #if $p == "": #continue #end if - #if $p.startswith('"'): + #if $p.startswith("'"): #set $q = True #end if + #if p.startswith('-'): + p = "\\" + p + #end if #if not $q: - #set $s[i] = '"%s"'%p + #set $s[i] = "'%s'" % p #end if - #if $p.endswith('"'): + #if $p.endswith("'"): #set $q = False #end if #end for #return " ".join($s) #end def - - + + - +]]> diff --git a/tests/test-data/macros.xml b/tests/test-data/macros.xml index bccb9956..6d9f3d70 100644 --- a/tests/test-data/macros.xml +++ b/tests/test-data/macros.xml @@ -1,7 +1,10 @@ + 2.3.0 - 8 + 0 openms @@ -35,7 +38,7 @@ - ^ *((?:\"[^\"]*\" +)|(?:[^ \"]+ +))*((?:\"[^\"]*\")|(?:[^ \"]+)) *$ + @@ -51,7 +54,7 @@ - ^ *[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?( *[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?)* *$ + @@ -65,7 +68,7 @@ - ^ *[+-]?[0-9]+( *[+-]?[0-9]+)* *$ + @@ -83,19 +86,22 @@ #set $q = False #for $i, $p in enumerate($s): #if $p == "": - #continue - #end if - #if $p.startswith('"'): + #continue + #end if + #if $p.startswith("'"): #set $q = True - #end if + #end if + #if p.startswith('-'): + p = "\\" + p + #end if #if not $q: - #set $s[i] = '"%s"'%p - #end if - #if $p.endswith('"'): - #set $q = False - #end if - #end for - #return " ".join($s) + #set $s[i] = "'%s'" % p + #end if + #if $p.endswith("'"): + #set $q = False + #end if + #end for + #return " ".join($s) #end def From 84bef83ade979dec945489c820a2b9250f805799 Mon Sep 17 00:00:00 2001 From: M Bernt Date: Fri, 20 Dec 2019 15:11:15 +0100 Subject: [PATCH 049/121] fixes for listitems - default values with spaces need quoting - quote list item values starting with a dash - link macros in test data to the template --- galaxy/converter.py | 12 +- galaxy/macros.xml | 233 +++++++++++++++++++------------------ tests/test-data/macros.xml | 117 +------------------ tests/test-data/repeat.ctd | 72 ++++++------ tests/test-data/repeat.xml | 124 ++++++++++---------- 5 files changed, 226 insertions(+), 332 deletions(-) mode change 100644 => 120000 tests/test-data/macros.xml diff --git a/galaxy/converter.py b/galaxy/converter.py index b5daf5b9..e1d0bc0c 100755 --- a/galaxy/converter.py +++ b/galaxy/converter.py @@ -1065,7 +1065,15 @@ def create_param_attribute_list(param_node, param, model, supported_file_formats message += "is required" validator_node = SubElement(valsan, "validator", OrderedDict([("type", "expression"), ("message", message )])) validator_node.text = CDATA(expression) - + else: + # add quotes to the default values (only if they include spaces .. then the UI looks nicer) + if not (param.default is None or param.default is _Null) and param.type is not _InFile: + if type(param.default) is list: + for i, d in enumerate(param.default): + if " " in d: + param.default[i] = "'%s'" %d + elif " " in param.default: + param.default = "'%s'" %param.default # add sanitizer nodes to # - text (only those that are not actually integer selects which are treated above) and # - select params, @@ -1102,7 +1110,7 @@ def create_param_attribute_list(param_node, param, model, supported_file_formats if param.description is not None: label, help_text = generate_label_and_help(param.description) if param.is_list and not is_selection_parameter(param) and not param.type is _InFile: - help_text += " (comma separated list)" + help_text += " (space separated list, in order to allow for spaces in list items surround them by single quotes)" param_node.attrib["label"] = label param_node.attrib["help"] = help_text diff --git a/galaxy/macros.xml b/galaxy/macros.xml index b3e74a9a..15fa17b2 100644 --- a/galaxy/macros.xml +++ b/galaxy/macros.xml @@ -1,116 +1,117 @@ - - - - 2.3 - 0 - - - openms - - - - - - - - - - - - - doi:10.1186/1471-2105-9-163 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -#def quote(s): - #set $s = [ _ for _ in $s.split(" ") if _ != "" ] - #set $q = False - #for $i, $p in enumerate($s): - #if $p == "": - #continue - #end if - #if $p.startswith("'"): - #set $q = True - #end if - #if p.startswith('-'): - p = "\\" + p - #end if - #if not $q: - #set $s[i] = "'%s'" % p - #end if - #if $p.endswith("'"): - #set $q = False - #end if - #end for - #return " ".join($s) -#end def - - - + + + + 2.3.0 + 3 + + + openms + + + + + + + + + + + + + doi:10.1186/1471-2105-9-163 + + + + + + + + + + + + + + + + + + ^ *((?:\'[^\']*\' +)|(?:[^ \']+ +))*((?:\'[^\']*\')|(?:[^ \']+)) *$ + + + + + + + + + + + + + + + + + ^ *[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?( *[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?)* *$ + + + + + + + + + + + + + + ^ *[+-]?[0-9]+( *[+-]?[0-9]+)* *$ + + + + + + + + + + + + +#def quote(s): + #set $s = [ _ for _ in $s.split(" ") if _ != "" ] + #set $q = False + #for $i, $p in enumerate($s): + #if $p == "": + #continue + #end if + #if $p.startswith("'"): + #set $q = True + #end if + #if p.startswith('-'): + p = "\\" + p + #end if + #if not $q: + #set $s[i] = "'%s'" % p + #end if + #if $p.endswith("'"): + #set $q = False + #end if + #end for + #return " ".join($s) +#end def + + + diff --git a/tests/test-data/macros.xml b/tests/test-data/macros.xml deleted file mode 100644 index 6d9f3d70..00000000 --- a/tests/test-data/macros.xml +++ /dev/null @@ -1,116 +0,0 @@ - - - - 2.3.0 - 0 - - - openms - - - - - - - - - - - - - doi:10.1186/1471-2105-9-163 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -#def quote(s): - #set $s = [ _ for _ in $s.split(" ") if _ != "" ] - #set $q = False - #for $i, $p in enumerate($s): - #if $p == "": - #continue - #end if - #if $p.startswith("'"): - #set $q = True - #end if - #if p.startswith('-'): - p = "\\" + p - #end if - #if not $q: - #set $s[i] = "'%s'" % p - #end if - #if $p.endswith("'"): - #set $q = False - #end if - #end for - #return " ".join($s) -#end def - - - diff --git a/tests/test-data/macros.xml b/tests/test-data/macros.xml new file mode 120000 index 00000000..63e52a86 --- /dev/null +++ b/tests/test-data/macros.xml @@ -0,0 +1 @@ +../../galaxy/macros.xml \ No newline at end of file diff --git a/tests/test-data/repeat.ctd b/tests/test-data/repeat.ctd index 71a8b4a4..b022bcfb 100644 --- a/tests/test-data/repeat.ctd +++ b/tests/test-data/repeat.ctd @@ -9,33 +9,33 @@ - - + + - - + + - + - + - - + + - + - - + + @@ -43,22 +43,22 @@ - + - + - - + + - + - - + + @@ -66,33 +66,33 @@ - - + + - - + + - + - + - - + + - + - - + + @@ -100,22 +100,22 @@ - + - + - - + + - + - - + + diff --git a/tests/test-data/repeat.xml b/tests/test-data/repeat.xml index 61d92310..8804202c 100644 --- a/tests/test-data/repeat.xml +++ b/tests/test-data/repeat.xml @@ -128,180 +128,180 @@ #end if > $param_stdout]]> - + - + - + - + - + - + - + - + - + - - + - - + - + - - + - + - + - + - + - - + - - + - + - - + - + - + - + - + - + - + - + - + - - + - - + - + - - + - + - + - + - + - - + - - + - + - @@ -313,26 +313,26 @@ - + - + - + - + - + - + - + - + - + - + From bbef513d33d6a488f4ca477ee240cdca2c71d95e Mon Sep 17 00:00:00 2001 From: M Bernt Date: Sat, 21 Dec 2019 17:17:37 +0100 Subject: [PATCH 050/121] fixed ofile command line handling mockpy also checks input files --- galaxy/converter.py | 70 ++-- tests/test-data/bool.xml | 9 +- tests/test-data/empty.xml | 1 + tests/test-data/float.xml | 97 ++--- tests/test-data/ifile.xml | 386 +++++++++--------- tests/test-data/integer.xml | 97 ++--- tests/test-data/mock.py | 27 +- tests/test-data/ofile-corresponding-input.ctd | 3 - tests/test-data/ofile-corresponding-input.xml | 106 +++-- tests/test-data/ofile-mult-typeparam.ctd | 35 ++ tests/test-data/ofile-mult-typeparam.xml | 77 ++++ tests/test-data/ofile-mult.xml | 92 +++-- tests/test-data/ofile-typeparam.ctd | 8 +- tests/test-data/ofile-typeparam.xml | 126 +++--- tests/test-data/ofile.xml | 75 +++- tests/test-data/repeat.xml | 113 ++--- tests/test-data/select.xml | 97 ++--- tests/test-data/string.xml | 17 +- tests/test_galaxy_cli.py | 2 + 19 files changed, 825 insertions(+), 613 deletions(-) create mode 100644 tests/test-data/ofile-mult-typeparam.ctd create mode 100644 tests/test-data/ofile-mult-typeparam.xml diff --git a/galaxy/converter.py b/galaxy/converter.py index e1d0bc0c..d235d94c 100755 --- a/galaxy/converter.py +++ b/galaxy/converter.py @@ -556,7 +556,7 @@ def create_command(tool, model, **kwargs): final_cmd = OrderedDict([('preprocessing', []), ('command', []), ('postprocessing', [])]) advanced_cmd = {'preprocessing': [], 'command': [], 'postprocessing': []} - final_cmd['preprocessing'].extend(["@QUOTE_FOO@", "@EXT_FOO@"]) + final_cmd['preprocessing'].extend(["@QUOTE_FOO@", "@EXT_FOO@", "#import re"]) final_cmd['command'].append("@EXECUTABLE@") final_cmd['command'].extend(kwargs["add_to_command_line"]) @@ -566,14 +566,11 @@ def create_command(tool, model, **kwargs): parameter_hardcoder = kwargs["parameter_hardcoder"] supported_file_formats = kwargs["supported_file_formats"] - found_input_parameter = False found_output_parameter = False for param in utils.extract_and_flatten_parameters(model): param = modify_param_for_galaxy(param) if param.type is _OutFile: found_output_parameter = True - if param.type is _InFile: - found_input_parameter = True param_cmd = {'preprocessing': [], 'command': [], 'postprocessing': []} param_name = utils.extract_param_name(param) @@ -605,11 +602,11 @@ def create_command(tool, model, **kwargs): if param.type is _InFile: param_cmd['preprocessing'].append("mkdir %s &&" %actual_parameter) if param.is_list: - param_cmd['preprocessing'].append("${ ' && '.join([ \"ln -s '%s' '"+actual_parameter+"/%s.%s'\" % (_, re.sub('[^\w\-_]', '_', _.element_identifier),$gxy2omsext(_.ext)) for _ in $" + actual_parameter + " if _ ]) } &&") + param_cmd['preprocessing'].append("${ ' '.join([ \"ln -s '%s' '"+actual_parameter+"/%s.%s' &&\" % (_, re.sub('[^\w\-_]', '_', _.element_identifier),$gxy2omsext(_.ext)) for _ in $" + actual_parameter + " if _ ]) }") param_cmd['command'].append("${' '.join([\"'"+actual_parameter+"/%s.%s'\"%(re.sub('[^\w\-_]', '_', _.element_identifier),$gxy2omsext(_.ext)) for _ in $" + actual_parameter + " if _ ])}") else: param_cmd['preprocessing'].append("ln -s '$"+ actual_parameter +"' '"+actual_parameter+"/${re.sub(\"[^\w\-_]\", \"_\", $"+actual_parameter+".element_identifier)}.$gxy2omsext($"+actual_parameter+".ext)' &&") - param_cmd['command'].append("'"+actual_parameter+"/${re.sub('[^\w\-_]', '_', $"+actual_parameter+".element_identifier)}.${gxy2omsext($"+actual_parameter+".ext)}'") + param_cmd['command'].append("'"+actual_parameter+"/${re.sub('[^\w\-_]', '_', $"+actual_parameter+".element_identifier)}.$gxy2omsext($"+actual_parameter+".ext)'") elif param.type is _OutFile: # check if there is a parameter that sets the format # if so we add an extension to the generated files which will be used to @@ -618,10 +615,8 @@ def create_command(tool, model, **kwargs): # the format will be set in the output tag formats = get_formats(param, supported_file_formats, TYPE_TO_GALAXY_TYPE[param.type]) type_param = get_out_type_param(param, model, parameter_hardcoder) - if param.is_list: - corresponding_input, fmt_from_corresponding = get_corresponding_input(param, model) - if corresponding_input is None: - raise Exception() + corresponding_input, fmt_from_corresponding = get_corresponding_input(param, model) + if corresponding_input is not None : actual_input_parameter = get_galaxy_parameter_name(corresponding_input) if corresponding_input.advanced: actual_input_parameter = ADVANCED_OPTIONS_NAME+"cond." + actual_input_parameter @@ -632,28 +627,51 @@ def create_command(tool, model, **kwargs): type_param_name = get_galaxy_parameter_name(type_param) if type_param.advanced: type_param_name = ADVANCED_OPTIONS_NAME+"cond." + type_param_name + + param_cmd['preprocessing'].append("mkdir " + actual_parameter + " &&") + + # if there is only one format (the outoput node sets format using the format attribute of the data/discover node) + # - single file: create a link with the oms extension and write through that to the actual file + # - lists: write to files with the oms extension and remove the extension afterwards (discovery with __name__) if len(formats)==1: + fmt = formats.pop() if param.is_list: - param_cmd['preprocessing'].append("mkdir " + actual_parameter + " &&") - param_cmd['command'].append("${' '.join([\"'"+ actual_parameter +"/%s'\"%(re.sub('[^\w\-_]', '_', _.element_identifier)) for _ in $" + actual_input_parameter + " if _ ])}") + logger.info("1 fmt + list " + actual_parameter) + param_cmd['command'].append("${' '.join([\"'"+ actual_parameter +"/%s.%s'\"%(re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(\""+ fmt +"\")) for _ in $" + actual_input_parameter + " if _ ])}") + param_cmd['postprocessing'].append("${' '.join([\"&& mv -n '"+ actual_parameter +"/%(id)s.%(gext)s' '" +actual_parameter+ "/%(id)s'\"%{\"id\": re.sub('[^\w\-_]', '_', _.element_identifier), \"gext\": $gxy2omsext(\""+fmt+"\")} for _ in $" + actual_input_parameter + " if _ ])}") else: - param_cmd['command'].append("'$" +actual_parameter+ "'") + logger.info("1 fmt + dataset " + actual_parameter) + param_cmd['preprocessing'].append("ln -s '$"+actual_parameter+"' '"+actual_parameter+"/output.${gxy2omsext(\"" + fmt + "\")}' &&") + param_cmd['command'].append("'"+actual_parameter+"/output.${gxy2omsext(\"" + fmt + "\")}'") + + # if there is a type parameter then we use the type selected by the user + # - single: create a link with the oms extension and write through that to the actual file output is treated via change_format + # - list: let the command create output files with the oms extensions, postprocessing renames them to the galaxy extensions, output is then discover + __name_and_ext__ elif type_param is not None: if param.is_list: + logger.info("type + list " + actual_parameter) param_cmd['command'].append("${' '.join([\"'"+ actual_parameter +"/%s.%s'\"%(re.sub('[^\w\-_]', '_', _.element_identifier), "+type_param_name+") for _ in $" + actual_input_parameter + " if _ ])}") - param_cmd['postprocessing'].append("${' '.join([\"mv -n '"+ actual_parameter +"/%(id)s.%(omsext)s' '" +actual_parameter+ "/%(id)s.%(gext)s'\"%{\"id\": re.sub('[^\w\-_]', '_', _.element_identifier), \"omsext\":"+type_param_name+", \"gext\": oms2gxyext("+type_param_name+")} for _ in $" + actual_input_parameter + " if _ ])}") + param_cmd['postprocessing'].append("${' '.join([\"&& mv -n '"+ actual_parameter +"/%(id)s.%(omsext)s' '" +actual_parameter+ "/%(id)s.%(gext)s'\"%{\"id\": re.sub('[^\w\-_]', '_', _.element_identifier), \"omsext\":"+type_param_name+", \"gext\": oms2gxyext(\""+type_param_name+"\")} for _ in $" + actual_input_parameter + " if _ ])}") else: + logger.info("type + dataset " + actual_parameter) # 1st create file with openms extension (often required by openms) # then move it to the actual place specified by the parameter # the format is then set by the tag using - param_cmd['preprocessing'].append("mkdir " + actual_parameter + " &&") + param_cmd['preprocessing'].append("ln -s '$"+actual_parameter+"' '"+actual_parameter+"/output.${"+type_param_name+"}' &&") param_cmd['command'].append("'" + actual_parameter+"/output.${"+type_param_name+"}'") - param_cmd['postprocessing'].append("&& mv '" + actual_parameter+"/output.${"+type_param_name+"}' '$" + actual_parameter + "'") - else: + elif actual_input_parameter is not None: if param.is_list: - param_cmd['preprocessing'].append("mkdir " + actual_parameter + " && ") + logger.info("actual + list " + actual_parameter) param_cmd['command'].append("${' '.join([\"'"+ actual_parameter +"/%s.%s'\"%(re.sub('[^\w\-_]', '_', _.element_identifier), _.ext) for _ in $" + actual_input_parameter + " if _ ])}") else: + logger.info("actual + dataset " + actual_parameter) + param_cmd['preprocessing'].append("ln -s '$"+actual_parameter+"' '"+actual_parameter+"/output.${"+actual_input_parameter+".ext}' &&") + param_cmd['command'].append("'" + actual_parameter+"/output.${"+actual_input_parameter+".ext}'") + else: + if param.is_list: + raise Exception( "output parameter itemlist %s without corresponding input" ) + else: + logger.info("else + dataset " + actual_parameter) param_cmd['command'].append("'$" +actual_parameter+ "'") # select with multiple = true @@ -692,23 +710,19 @@ def create_command(tool, model, **kwargs): if len(param_cmd[stage]) == 0: continue if param.advanced and hardcoded_value is None: - advanced_cmd[stage].extend(utils.indent(param_cmd[stage])) + advanced_cmd[stage].extend(param_cmd[stage]) else: - final_cmd[stage].extend(utils.indent(param_cmd[stage])) + final_cmd[stage].extend(param_cmd[stage]) for stage in advanced_cmd: if len(advanced_cmd[stage])==0: continue - advanced_cmd[stage] = [advanced_command_start] + advanced_cmd[stage] + [advanced_command_end] + advanced_cmd[stage] = [advanced_command_start] + utils.indent(advanced_cmd[stage]) + [advanced_command_end] final_cmd[stage].extend(advanced_cmd[stage]) if not found_output_parameter: final_cmd['command'].append("> $param_stdout") - - if found_input_parameter: - final_cmd['command'] = ["#import re"] + final_cmd['command'] - command_node = add_child_node(tool, "command") command_node.attrib["detect_errors"] = "exit_code" @@ -1336,7 +1350,7 @@ def create_output_node(parent, param, model, supported_file_formats, parameter_h return data_node -# Get the supported file format for one given format +# Get the supported file format for one given format (w galaxy extensions) def get_supported_file_type(format_name, supported_file_formats): if format_name in supported_file_formats.keys(): return supported_file_formats[format_name].galaxy_extension @@ -1376,7 +1390,7 @@ def create_tests(parent, inputs=None, outputs=None, test_macros_prefix=None, nam if not (inputs is None or outputs is None): fidx = 0 test_node = add_child_node(tests_node, "test") - strip_elements(inputs, "validator", "sanitizer" ) + strip_elements(inputs, "validator", "sanitizer") for node in inputs.iter(): if node.tag == "expand" and node.attrib["macro"]== ADVANCED_OPTIONS_NAME+"macro": node.tag = "conditional" @@ -1451,7 +1465,7 @@ def create_tests(parent, inputs=None, outputs=None, test_macros_prefix=None, nam node.attrib["lines_diff"] = "2" for a in set(node.attrib) - set(["name","value","ftype", "lines_diff"]): del node.attrib[a] - strip_elements(outputs, "delete_node", "discover_datasets", "filter") + strip_elements(outputs, "delete_node", "discover_datasets", "filter", "change_format") for node in outputs: test_node.append(node) diff --git a/tests/test-data/bool.xml b/tests/test-data/bool.xml index 859e90c1..60d52f34 100644 --- a/tests/test-data/bool.xml +++ b/tests/test-data/bool.xml @@ -11,11 +11,12 @@ $param_stdout]]> diff --git a/tests/test-data/float.xml b/tests/test-data/float.xml index 853cccc6..1b209d5c 100644 --- a/tests/test-data/float.xml +++ b/tests/test-data/float.xml @@ -11,55 +11,56 @@ - - @@ -27,7 +25,6 @@ - diff --git a/tests/test-data/ofile-corresponding-input.xml b/tests/test-data/ofile-corresponding-input.xml index f7d5c372..15c2fd24 100644 --- a/tests/test-data/ofile-corresponding-input.xml +++ b/tests/test-data/ofile-corresponding-input.xml @@ -11,56 +11,94 @@ diff --git a/tests/test-data/ofile-mult-typeparam.ctd b/tests/test-data/ofile-mult-typeparam.ctd new file mode 100644 index 00000000..983b2447 --- /dev/null +++ b/tests/test-data/ofile-mult-typeparam.ctd @@ -0,0 +1,35 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/test-data/ofile-mult-typeparam.xml b/tests/test-data/ofile-mult-typeparam.xml new file mode 100644 index 00000000..55d67ca5 --- /dev/null +++ b/tests/test-data/ofile-mult-typeparam.xml @@ -0,0 +1,77 @@ + + + + + multiple Output File parameter tests + + '$__tool_directory__/mock.py' ofile-mult-typeparam.ctd + macros.xml + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/test-data/ofile-mult.xml b/tests/test-data/ofile-mult.xml index ffead8fe..ad8c0e96 100644 --- a/tests/test-data/ofile-mult.xml +++ b/tests/test-data/ofile-mult.xml @@ -11,52 +11,56 @@ +-mandatory_mandatoryinput +${' '.join(["'param_out_mandatory_mandatoryinput/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext("tsv")) for _ in $param_inp_mandatory_mandatoryinput if _ ])} +#if str($param_optional_mandatoryinput): + -optional_mandatoryinput + ${' '.join(["'param_out_optional_mandatoryinput/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext("tsv")) for _ in $param_inp_optional_mandatoryinput if _ ])} +#end if +-mandatory_optionalinput +${' '.join(["'param_out_mandatory_optionalinput/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier), _.ext) for _ in $param_inp_mandatory_optionalinput if _ ])} +#if str($param_optional_optionalinput): + -optional_optionalinput + ${' '.join(["'param_out_optional_optionalinput/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier), _.ext) for _ in $param_inp_optional_optionalinput if _ ])} +#end if +-inp_mandatory_mandatoryinput +${' '.join(["'param_inp_mandatory_mandatoryinput/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier),$gxy2omsext(_.ext)) for _ in $param_inp_mandatory_mandatoryinput if _ ])} +-inp_optional_mandatoryinput +${' '.join(["'param_inp_optional_mandatoryinput/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier),$gxy2omsext(_.ext)) for _ in $param_inp_optional_mandatoryinput if _ ])} +#if $param_inp_mandatory_optionalinput: + -inp_mandatory_optionalinput + ${' '.join(["'param_inp_mandatory_optionalinput/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier),$gxy2omsext(_.ext)) for _ in $param_inp_mandatory_optionalinput if _ ])} +#end if +#if $param_inp_optional_optionalinput: + -inp_optional_optionalinput + ${' '.join(["'param_inp_optional_optionalinput/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier),$gxy2omsext(_.ext)) for _ in $param_inp_optional_optionalinput if _ ])} +#end if +${' '.join(["&& mv -n 'param_out_mandatory_mandatoryinput/%(id)s.%(gext)s' 'param_out_mandatory_mandatoryinput/%(id)s'"%{"id": re.sub('[^\w\-_]', '_', _.element_identifier), "gext": $gxy2omsext("tsv")} for _ in $param_inp_mandatory_mandatoryinput if _ ])} +#if str($param_optional_mandatoryinput): + ${' '.join(["&& mv -n 'param_out_optional_mandatoryinput/%(id)s.%(gext)s' 'param_out_optional_mandatoryinput/%(id)s'"%{"id": re.sub('[^\w\-_]', '_', _.element_identifier), "gext": $gxy2omsext("tsv")} for _ in $param_inp_optional_mandatoryinput if _ ])} +#end if]]> diff --git a/tests/test-data/ofile-typeparam.ctd b/tests/test-data/ofile-typeparam.ctd index 164e92e7..86afac64 100644 --- a/tests/test-data/ofile-typeparam.ctd +++ b/tests/test-data/ofile-typeparam.ctd @@ -17,25 +17,25 @@ For optional outputs an additional bool param should be generated that triggers the output (must be incorporated in the CLI and the filter) --> - + - + - + - + diff --git a/tests/test-data/ofile-typeparam.xml b/tests/test-data/ofile-typeparam.xml index 314e50fd..b12f3194 100644 --- a/tests/test-data/ofile-typeparam.xml +++ b/tests/test-data/ofile-typeparam.xml @@ -11,81 +11,87 @@ @@ -207,21 +213,9 @@ - - - - - - - + - - - - - - - + Date: Sun, 29 Dec 2019 18:10:16 +0100 Subject: [PATCH 051/121] outfile handling improvements - since writing throuh links did not work (links were just overwritten) swith to mv - add format select param in cases where there is no way to determine the output format automatically - make the supported file types an ordered dict --- galaxy/converter.py | 115 ++++++++++++++++++++++++++------------------ 1 file changed, 69 insertions(+), 46 deletions(-) diff --git a/galaxy/converter.py b/galaxy/converter.py index d235d94c..db43615a 100755 --- a/galaxy/converter.py +++ b/galaxy/converter.py @@ -627,51 +627,61 @@ def create_command(tool, model, **kwargs): type_param_name = get_galaxy_parameter_name(type_param) if type_param.advanced: type_param_name = ADVANCED_OPTIONS_NAME+"cond." + type_param_name + elif len(formats) > 1 and (corresponding_input == None or not fmt_from_corresponding) and not param.is_list: + type_param_name = get_galaxy_parameter_name(param, True) + "_type" + if param.advanced: + type_param_name = ADVANCED_OPTIONS_NAME+"cond." + type_param_name + else: + type_param_name = None param_cmd['preprocessing'].append("mkdir " + actual_parameter + " &&") # if there is only one format (the outoput node sets format using the format attribute of the data/discover node) - # - single file: create a link with the oms extension and write through that to the actual file + # - single file: write to temp file with oms extension and move this to the actual result file # - lists: write to files with the oms extension and remove the extension afterwards (discovery with __name__) if len(formats)==1: fmt = formats.pop() if param.is_list: - logger.info("1 fmt + list " + actual_parameter) + logger.info("1 fmt + list %s -> %s" % (actual_parameter, actual_input_parameter)) param_cmd['command'].append("${' '.join([\"'"+ actual_parameter +"/%s.%s'\"%(re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(\""+ fmt +"\")) for _ in $" + actual_input_parameter + " if _ ])}") param_cmd['postprocessing'].append("${' '.join([\"&& mv -n '"+ actual_parameter +"/%(id)s.%(gext)s' '" +actual_parameter+ "/%(id)s'\"%{\"id\": re.sub('[^\w\-_]', '_', _.element_identifier), \"gext\": $gxy2omsext(\""+fmt+"\")} for _ in $" + actual_input_parameter + " if _ ])}") else: - logger.info("1 fmt + dataset " + actual_parameter) - param_cmd['preprocessing'].append("ln -s '$"+actual_parameter+"' '"+actual_parameter+"/output.${gxy2omsext(\"" + fmt + "\")}' &&") + logger.info("1 fmt + dataset %s" % actual_parameter) param_cmd['command'].append("'"+actual_parameter+"/output.${gxy2omsext(\"" + fmt + "\")}'") + param_cmd['postprocessing'].append("&& mv '"+actual_parameter+"/output.${gxy2omsext(\"" + fmt + "\")}' '$"+actual_parameter+"'") # if there is a type parameter then we use the type selected by the user - # - single: create a link with the oms extension and write through that to the actual file output is treated via change_format + # - single: write to temp file with the oms extension and mv it to the actual file output which is treated via change_format # - list: let the command create output files with the oms extensions, postprocessing renames them to the galaxy extensions, output is then discover + __name_and_ext__ - elif type_param is not None: + elif type_param_name is not None: if param.is_list: - logger.info("type + list " + actual_parameter) + logger.info("type + list %s" %actual_parameter) param_cmd['command'].append("${' '.join([\"'"+ actual_parameter +"/%s.%s'\"%(re.sub('[^\w\-_]', '_', _.element_identifier), "+type_param_name+") for _ in $" + actual_input_parameter + " if _ ])}") param_cmd['postprocessing'].append("${' '.join([\"&& mv -n '"+ actual_parameter +"/%(id)s.%(omsext)s' '" +actual_parameter+ "/%(id)s.%(gext)s'\"%{\"id\": re.sub('[^\w\-_]', '_', _.element_identifier), \"omsext\":"+type_param_name+", \"gext\": oms2gxyext(\""+type_param_name+"\")} for _ in $" + actual_input_parameter + " if _ ])}") else: - logger.info("type + dataset " + actual_parameter) + logger.info("type + dataset %s" %actual_parameter) # 1st create file with openms extension (often required by openms) # then move it to the actual place specified by the parameter # the format is then set by the tag using - param_cmd['preprocessing'].append("ln -s '$"+actual_parameter+"' '"+actual_parameter+"/output.${"+type_param_name+"}' &&") param_cmd['command'].append("'" + actual_parameter+"/output.${"+type_param_name+"}'") + param_cmd['postprocessing'].append("&& mv '"+actual_parameter+"/output.${"+type_param_name+"}' '$"+actual_parameter+"'") elif actual_input_parameter is not None: if param.is_list: - logger.info("actual + list " + actual_parameter) + logger.info("actual + list %s" %actual_parameter) param_cmd['command'].append("${' '.join([\"'"+ actual_parameter +"/%s.%s'\"%(re.sub('[^\w\-_]', '_', _.element_identifier), _.ext) for _ in $" + actual_input_parameter + " if _ ])}") else: - logger.info("actual + dataset " + actual_parameter) - param_cmd['preprocessing'].append("ln -s '$"+actual_parameter+"' '"+actual_parameter+"/output.${"+actual_input_parameter+".ext}' &&") - param_cmd['command'].append("'" + actual_parameter+"/output.${"+actual_input_parameter+".ext}'") + logger.info("actual + dataset %s %s %s" % (actual_parameter, actual_input_parameter, corresponding_input.is_list)) + if corresponding_input.is_list: + param_cmd['command'].append("'" + actual_parameter+"/output.${"+actual_input_parameter+"[0].ext}'") + param_cmd['postprocessing'].append("&& mv '"+actual_parameter+"/output.${"+actual_input_parameter+"[0].ext}' '$"+actual_parameter+"'") + else: + param_cmd['command'].append("'" + actual_parameter+"/output.${"+actual_input_parameter+".ext}'") + param_cmd['postprocessing'].append("&& mv '"+actual_parameter+"/output.${"+actual_input_parameter+".ext}' '$"+actual_parameter+"'") else: if param.is_list: raise Exception( "output parameter itemlist %s without corresponding input" ) else: - logger.info("else + dataset " + actual_parameter) + logger.info("else + dataset %s" % actual_parameter) param_cmd['command'].append("'$" +actual_parameter+ "'") # select with multiple = true @@ -701,7 +711,7 @@ def create_command(tool, model, **kwargs): for stage in param_cmd: if len(param_cmd[stage]) == 0: continue - if is_selection_parameter(param) or param.type == _InFile: + if is_selection_parameter(param) or param.type is _OutFile or param.type is _InFile: param_cmd[stage] = ["#if $" + actual_parameter + ":"] + utils.indent(param_cmd[stage]) + ["#end if"] else: param_cmd[stage] = ["#if str($" + actual_parameter + "):"] + utils.indent(param_cmd[stage]) + ["#end if"] @@ -838,17 +848,23 @@ def get_input_with_same_restrictions(out_param, model, check_formats): """ matching = [] - for param in utils.extract_and_flatten_parameters(model): - if not param.type is _InFile: - continue - if param.is_list == out_param.is_list: - if check_formats: - if param.restrictions is None and out_param.restrictions is None: - matching.append(param) - elif param.restrictions is not None and out_param.restrictions is not None and param.restrictions.formats == out_param.restrictions.formats: + + for allow_different_type in [False, True]: + for param in utils.extract_and_flatten_parameters(model): + if not param.type is _InFile: + continue +# logger.error("%s %s %s %s %s %s" %(out_param.name, param.name, param.is_list, out_param.is_list, param.restrictions, out_param.restrictions)) + if allow_different_type or param.is_list == out_param.is_list: + if check_formats: + if param.restrictions is None and out_param.restrictions is None: + matching.append(param) + elif param.restrictions is not None and out_param.restrictions is not None and param.restrictions.formats == out_param.restrictions.formats: + matching.append(param) + else: matching.append(param) - else: - matching.append(param) +# logger.error("match %s "%([ _.name for _ in matching])) + if len(matching) > 0: + break if len(matching) == 1: return matching[0] else: @@ -876,33 +892,38 @@ def create_inputs(tool, model, **kwargs): hardcoded_value = parameter_hardcoder.get_hardcoded_value(utils.extract_param_name(param), model.name) if parameter_hardcoder.get_blacklist(utils.extract_param_name(param), model.name) or not hardcoded_value is None: continue - - # optional outfiles: create an additional bool input which is used to filter for the output - # mandatory outpiles: no input node needed - # inputs: create the input param - if param.type is _OutFile: - if not param.required: - _param = copy.deepcopy(param) - _param.type = str - _param.restrictions = _Choices("true,false") - _param.default = "false" - _param.description = "generate output %s (%s)" %(param.name, param.description) - _param.is_list = False - else: - continue - else: - _param = param - - if _param.advanced: + + if param.advanced: if advanced_node is None: advanced_node = Element("expand", OrderedDict([("macro", ADVANCED_OPTIONS_NAME+"macro")])) parent_node = advanced_node else: parent_node = inputs_node - # create the actuayield/l param node and fill the attributes + # sometimes special inputs are needed for outfiles: + add_formats = False + if param.type is _OutFile: + # if there are multiple possible output formats, but no parameter to choose the type or a + # corresponding input then add a selection parameter + formats = get_formats(param, kwargs["supported_file_formats"], TYPE_TO_GALAXY_TYPE[_OutFile]) + type_param = get_out_type_param(param, model, parameter_hardcoder) + corresponding_input, fmt_from_corresponding = get_corresponding_input(param, model) + if len(formats) > 1 and type_param is None and (corresponding_input is None or not fmt_from_corresponding) and not param.is_list: + fmt_select = add_child_node(parent_node, "param", OrderedDict([("name", get_galaxy_parameter_name(param, True)+"_type"), ("type", "select"), ("optional", "false"), ("label", "File type of output %s (%s)" % (param.name, param.description))])) + for f in formats: + option_node = add_child_node(fmt_select, "option", OrderedDict([("value", f)]), f) + + # create an additional bool input which is used to filter for the output + # mandatory outpiles: no input node needed + # inputs: create the input param + if not param.required: + add_child_node(parent_node, "param", OrderedDict([("type", "boolean"), ("name", get_galaxy_parameter_name(param, True)), ("label", "Generate output %s (%s)" %(param.name, param.description))])) + + continue + + # create the actual param node and fill the attributes param_node = add_child_node(parent_node, "param") - create_param_attribute_list(param_node, _param, model, kwargs["supported_file_formats"]) + create_param_attribute_list(param_node, param, model, kwargs["supported_file_formats"]) # hardcoded_attributes = parameter_hardcoder.get_hardcoded_attributes(param.name, model.name) # if hardcoded_attributes != None: @@ -1486,7 +1507,7 @@ def create_help(tool, model): help_node.text = CDATA(utils.extract_tool_help_text(model)) -def add_child_node(parent_node, child_node_name, attributes=OrderedDict([])): +def add_child_node(parent_node, child_node_name, attributes=OrderedDict([]), text=None): """ helper function to add a child node using the given name to the given parent node @param parent_node the parent @@ -1495,4 +1516,6 @@ def add_child_node(parent_node, child_node_name, attributes=OrderedDict([])): @return the created child node """ child_node = SubElement(parent_node, child_node_name, attributes) + if text is not None: + child_node.text = text return child_node From 4c3f53ab1ae58a19295be8e20bde0723a48469ae Mon Sep 17 00:00:00 2001 From: M Bernt Date: Sun, 29 Dec 2019 18:34:41 +0100 Subject: [PATCH 052/121] add missing attributes to output boolean --- galaxy/converter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/galaxy/converter.py b/galaxy/converter.py index db43615a..547bf8ff 100755 --- a/galaxy/converter.py +++ b/galaxy/converter.py @@ -917,7 +917,7 @@ def create_inputs(tool, model, **kwargs): # mandatory outpiles: no input node needed # inputs: create the input param if not param.required: - add_child_node(parent_node, "param", OrderedDict([("type", "boolean"), ("name", get_galaxy_parameter_name(param, True)), ("label", "Generate output %s (%s)" %(param.name, param.description))])) + add_child_node(parent_node, "param", OrderedDict([("type", "boolean"), ("name", get_galaxy_parameter_name(param, True)), ("optional", "false"), ("checked", "false"), ("label", "Generate output %s (%s)" %(param.name, param.description))])) continue From b76860444a7b0c30f61eab8cd686df522f791126 Mon Sep 17 00:00:00 2001 From: M Bernt Date: Sun, 29 Dec 2019 19:39:22 +0100 Subject: [PATCH 053/121] update test data --- tests/test-data/ofile-corresponding-input.xml | 74 ++++++++------ tests/test-data/ofile-mult.xml | 14 +-- tests/test-data/ofile-typeparam.xml | 50 ++++++---- tests/test-data/ofile.xml | 98 +++++++++++++------ 4 files changed, 149 insertions(+), 87 deletions(-) diff --git a/tests/test-data/ofile-corresponding-input.xml b/tests/test-data/ofile-corresponding-input.xml index 15c2fd24..18711a3a 100644 --- a/tests/test-data/ofile-corresponding-input.xml +++ b/tests/test-data/ofile-corresponding-input.xml @@ -13,22 +13,16 @@ @EXT_FOO@ #import re mkdir param_out_mandatory_noformat && -ln -s '$param_out_mandatory_noformat' 'param_out_mandatory_noformat/output.${gxy2omsext("txt")}' && -#if str($param_optional_noformat): +#if $param_optional_noformat: mkdir param_out_optional_noformat && - ln -s '$param_out_optional_noformat' 'param_out_optional_noformat/output.${gxy2omsext("txt")}' && #end if mkdir param_out_mandatory_oneformat && -ln -s '$param_out_mandatory_oneformat' 'param_out_mandatory_oneformat/output.${gxy2omsext("tsv")}' && -#if str($param_optional_oneformat): +#if $param_optional_oneformat: mkdir param_out_optional_oneformat && - ln -s '$param_out_optional_oneformat' 'param_out_optional_oneformat/output.${gxy2omsext("tsv")}' && #end if mkdir param_out_mandatory_multformat && -ln -s '$param_out_mandatory_multformat' 'param_out_mandatory_multformat/output.${param_input_multformat.ext}' && -#if str($param_optional_multformat): +#if $param_optional_multformat: mkdir param_out_optional_multformat && - ln -s '$param_out_optional_multformat' 'param_out_optional_multformat/output.${param_input_multformat.ext}' && #end if mkdir param_input_noformat && ln -s '$param_input_noformat' 'param_input_noformat/${re.sub("[^\w\-_]", "_", $param_input_noformat.element_identifier)}.$gxy2omsext($param_input_noformat.ext)' && @@ -38,40 +32,34 @@ mkdir param_input_multformat && ln -s '$param_input_multformat' 'param_input_multformat/${re.sub("[^\w\-_]", "_", $param_input_multformat.element_identifier)}.$gxy2omsext($param_input_multformat.ext)' && #if $adv_opts_cond.adv_opts_selector=='advanced': mkdir param_out_mandatory_noformat_adv && - ln -s '$param_out_mandatory_noformat_adv' 'param_out_mandatory_noformat_adv/output.${gxy2omsext("txt")}' && - #if str($adv_opts_cond.param_optional_noformat_adv): + #if $adv_opts_cond.param_optional_noformat_adv: mkdir param_out_optional_noformat_adv && - ln -s '$param_out_optional_noformat_adv' 'param_out_optional_noformat_adv/output.${gxy2omsext("txt")}' && #end if mkdir param_out_mandatory_oneformat_adv && - ln -s '$param_out_mandatory_oneformat_adv' 'param_out_mandatory_oneformat_adv/output.${gxy2omsext("tsv")}' && - #if str($adv_opts_cond.param_optional_oneformat_adv): + #if $adv_opts_cond.param_optional_oneformat_adv: mkdir param_out_optional_oneformat_adv && - ln -s '$param_out_optional_oneformat_adv' 'param_out_optional_oneformat_adv/output.${gxy2omsext("tsv")}' && #end if mkdir param_out_mandatory_multformat_adv && - ln -s '$param_out_mandatory_multformat_adv' 'param_out_mandatory_multformat_adv/output.${param_input_multformat.ext}' && - #if str($adv_opts_cond.param_optional_multformat_adv): + #if $adv_opts_cond.param_optional_multformat_adv: mkdir param_out_optional_multformat_adv && - ln -s '$param_out_optional_multformat_adv' 'param_out_optional_multformat_adv/output.${param_input_multformat.ext}' && #end if #end if @EXECUTABLE@ -mandatory_noformat 'param_out_mandatory_noformat/output.${gxy2omsext("txt")}' -#if str($param_optional_noformat): +#if $param_optional_noformat: -optional_noformat 'param_out_optional_noformat/output.${gxy2omsext("txt")}' #end if -mandatory_oneformat 'param_out_mandatory_oneformat/output.${gxy2omsext("tsv")}' -#if str($param_optional_oneformat): +#if $param_optional_oneformat: -optional_oneformat 'param_out_optional_oneformat/output.${gxy2omsext("tsv")}' #end if -mandatory_multformat 'param_out_mandatory_multformat/output.${param_input_multformat.ext}' -#if str($param_optional_multformat): +#if $param_optional_multformat: -optional_multformat 'param_out_optional_multformat/output.${param_input_multformat.ext}' #end if @@ -84,34 +72,60 @@ ln -s '$param_input_multformat' 'param_input_multformat/${re.sub("[^\w\-_]", "_" #if $adv_opts_cond.adv_opts_selector=='advanced': -mandatory_noformat_adv 'param_out_mandatory_noformat_adv/output.${gxy2omsext("txt")}' - #if str($adv_opts_cond.param_optional_noformat_adv): + #if $adv_opts_cond.param_optional_noformat_adv: -optional_noformat_adv 'param_out_optional_noformat_adv/output.${gxy2omsext("txt")}' #end if -mandatory_oneformat_adv 'param_out_mandatory_oneformat_adv/output.${gxy2omsext("tsv")}' - #if str($adv_opts_cond.param_optional_oneformat_adv): + #if $adv_opts_cond.param_optional_oneformat_adv: -optional_oneformat_adv 'param_out_optional_oneformat_adv/output.${gxy2omsext("tsv")}' #end if -mandatory_multformat_adv 'param_out_mandatory_multformat_adv/output.${param_input_multformat.ext}' - #if str($adv_opts_cond.param_optional_multformat_adv): + #if $adv_opts_cond.param_optional_multformat_adv: -optional_multformat_adv 'param_out_optional_multformat_adv/output.${param_input_multformat.ext}' #end if +#end if +&& mv 'param_out_mandatory_noformat/output.${gxy2omsext("txt")}' '$param_out_mandatory_noformat' +#if $param_optional_noformat: + && mv 'param_out_optional_noformat/output.${gxy2omsext("txt")}' '$param_out_optional_noformat' +#end if +&& mv 'param_out_mandatory_oneformat/output.${gxy2omsext("tsv")}' '$param_out_mandatory_oneformat' +#if $param_optional_oneformat: + && mv 'param_out_optional_oneformat/output.${gxy2omsext("tsv")}' '$param_out_optional_oneformat' +#end if +&& mv 'param_out_mandatory_multformat/output.${param_input_multformat.ext}' '$param_out_mandatory_multformat' +#if $param_optional_multformat: + && mv 'param_out_optional_multformat/output.${param_input_multformat.ext}' '$param_out_optional_multformat' +#end if +#if $adv_opts_cond.adv_opts_selector=='advanced': + && mv 'param_out_mandatory_noformat_adv/output.${gxy2omsext("txt")}' '$param_out_mandatory_noformat_adv' + #if $adv_opts_cond.param_optional_noformat_adv: + && mv 'param_out_optional_noformat_adv/output.${gxy2omsext("txt")}' '$param_out_optional_noformat_adv' + #end if + && mv 'param_out_mandatory_oneformat_adv/output.${gxy2omsext("tsv")}' '$param_out_mandatory_oneformat_adv' + #if $adv_opts_cond.param_optional_oneformat_adv: + && mv 'param_out_optional_oneformat_adv/output.${gxy2omsext("tsv")}' '$param_out_optional_oneformat_adv' + #end if + && mv 'param_out_mandatory_multformat_adv/output.${param_input_multformat.ext}' '$param_out_mandatory_multformat_adv' + #if $adv_opts_cond.param_optional_multformat_adv: + && mv 'param_out_optional_multformat_adv/output.${param_input_multformat.ext}' '$param_out_optional_multformat_adv' + #end if #end if]]> - - - + + + - - - + + + diff --git a/tests/test-data/ofile-mult.xml b/tests/test-data/ofile-mult.xml index ad8c0e96..b7d3782d 100644 --- a/tests/test-data/ofile-mult.xml +++ b/tests/test-data/ofile-mult.xml @@ -13,11 +13,11 @@ @EXT_FOO@ #import re mkdir param_out_mandatory_mandatoryinput && -#if str($param_optional_mandatoryinput): +#if $param_optional_mandatoryinput: mkdir param_out_optional_mandatoryinput && #end if mkdir param_out_mandatory_optionalinput && -#if str($param_optional_optionalinput): +#if $param_optional_optionalinput: mkdir param_out_optional_optionalinput && #end if mkdir param_inp_mandatory_mandatoryinput && @@ -35,13 +35,13 @@ ${ ' '.join([ "ln -s '%s' 'param_inp_optional_mandatoryinput/%s.%s' &&" % (_, re @EXECUTABLE@ -mandatory_mandatoryinput ${' '.join(["'param_out_mandatory_mandatoryinput/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext("tsv")) for _ in $param_inp_mandatory_mandatoryinput if _ ])} -#if str($param_optional_mandatoryinput): +#if $param_optional_mandatoryinput: -optional_mandatoryinput ${' '.join(["'param_out_optional_mandatoryinput/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext("tsv")) for _ in $param_inp_optional_mandatoryinput if _ ])} #end if -mandatory_optionalinput ${' '.join(["'param_out_mandatory_optionalinput/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier), _.ext) for _ in $param_inp_mandatory_optionalinput if _ ])} -#if str($param_optional_optionalinput): +#if $param_optional_optionalinput: -optional_optionalinput ${' '.join(["'param_out_optional_optionalinput/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier), _.ext) for _ in $param_inp_optional_optionalinput if _ ])} #end if @@ -58,12 +58,12 @@ ${' '.join(["'param_inp_optional_mandatoryinput/%s.%s'"%(re.sub('[^\w\-_]', '_', ${' '.join(["'param_inp_optional_optionalinput/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier),$gxy2omsext(_.ext)) for _ in $param_inp_optional_optionalinput if _ ])} #end if ${' '.join(["&& mv -n 'param_out_mandatory_mandatoryinput/%(id)s.%(gext)s' 'param_out_mandatory_mandatoryinput/%(id)s'"%{"id": re.sub('[^\w\-_]', '_', _.element_identifier), "gext": $gxy2omsext("tsv")} for _ in $param_inp_mandatory_mandatoryinput if _ ])} -#if str($param_optional_mandatoryinput): +#if $param_optional_mandatoryinput: ${' '.join(["&& mv -n 'param_out_optional_mandatoryinput/%(id)s.%(gext)s' 'param_out_optional_mandatoryinput/%(id)s'"%{"id": re.sub('[^\w\-_]', '_', _.element_identifier), "gext": $gxy2omsext("tsv")} for _ in $param_inp_optional_mandatoryinput if _ ])} #end if]]> - - + + diff --git a/tests/test-data/ofile-typeparam.xml b/tests/test-data/ofile-typeparam.xml index b12f3194..ecbcdc1b 100644 --- a/tests/test-data/ofile-typeparam.xml +++ b/tests/test-data/ofile-typeparam.xml @@ -13,16 +13,12 @@ @EXT_FOO@ #import re mkdir param_out_mandatory_noformat && -ln -s '$param_out_mandatory_noformat' 'param_out_mandatory_noformat/output.${gxy2omsext("txt")}' && -#if str($param_optional_noformat): +#if $param_optional_noformat: mkdir param_out_optional_noformat && - ln -s '$param_out_optional_noformat' 'param_out_optional_noformat/output.${gxy2omsext("txt")}' && #end if mkdir param_out_mandatory_multformat && -ln -s '$param_out_mandatory_multformat' 'param_out_mandatory_multformat/output.${param_mandatory_multformat_type}' && -#if str($param_optional_multformat): +#if $param_optional_multformat: mkdir param_out_optional_multformat && - ln -s '$param_out_optional_multformat' 'param_out_optional_multformat/output.${param_optional_multformat_type}' && #end if mkdir param_decoy_input_noformat && ln -s '$param_decoy_input_noformat' 'param_decoy_input_noformat/${re.sub("[^\w\-_]", "_", $param_decoy_input_noformat.element_identifier)}.$gxy2omsext($param_decoy_input_noformat.ext)' && @@ -32,16 +28,12 @@ mkdir param_decoy_input_multformat && ln -s '$param_decoy_input_multformat' 'param_decoy_input_multformat/${re.sub("[^\w\-_]", "_", $param_decoy_input_multformat.element_identifier)}.$gxy2omsext($param_decoy_input_multformat.ext)' && #if $adv_opts_cond.adv_opts_selector=='advanced': mkdir param_out_mandatory_noformat_adv && - ln -s '$param_out_mandatory_noformat_adv' 'param_out_mandatory_noformat_adv/output.${gxy2omsext("txt")}' && - #if str($adv_opts_cond.param_optional_noformat_adv): + #if $adv_opts_cond.param_optional_noformat_adv: mkdir param_out_optional_noformat_adv && - ln -s '$param_out_optional_noformat_adv' 'param_out_optional_noformat_adv/output.${gxy2omsext("txt")}' && #end if mkdir param_out_mandatory_multformat_adv && - ln -s '$param_out_mandatory_multformat_adv' 'param_out_mandatory_multformat_adv/output.${adv_opts_cond.param_mandatory_multformat_adv_type}' && - #if str($adv_opts_cond.param_optional_multformat_adv): + #if $adv_opts_cond.param_optional_multformat_adv: mkdir param_out_optional_multformat_adv && - ln -s '$param_out_optional_multformat_adv' 'param_out_optional_multformat_adv/output.${adv_opts_cond.param_optional_multformat_adv_type}' && #end if #end if @EXECUTABLE@ @@ -51,7 +43,7 @@ ln -s '$param_decoy_input_multformat' 'param_decoy_input_multformat/${re.sub("[^ 'param_out_mandatory_noformat/output.${gxy2omsext("txt")}' -optional_noformat_type '$param_optional_noformat_type' -#if str($param_optional_noformat): +#if $param_optional_noformat: -optional_noformat 'param_out_optional_noformat/output.${gxy2omsext("txt")}' #end if @@ -61,7 +53,7 @@ ln -s '$param_decoy_input_multformat' 'param_decoy_input_multformat/${re.sub("[^ 'param_out_mandatory_multformat/output.${param_mandatory_multformat_type}' -optional_multformat_type '$param_optional_multformat_type' -#if str($param_optional_multformat): +#if $param_optional_multformat: -optional_multformat 'param_out_optional_multformat/output.${param_optional_multformat_type}' #end if @@ -78,7 +70,7 @@ ln -s '$param_decoy_input_multformat' 'param_decoy_input_multformat/${re.sub("[^ 'param_out_mandatory_noformat_adv/output.${gxy2omsext("txt")}' -optional_noformat_adv_type '$adv_opts_cond.param_optional_noformat_adv_type' - #if str($adv_opts_cond.param_optional_noformat_adv): + #if $adv_opts_cond.param_optional_noformat_adv: -optional_noformat_adv 'param_out_optional_noformat_adv/output.${gxy2omsext("txt")}' #end if @@ -88,10 +80,28 @@ ln -s '$param_decoy_input_multformat' 'param_decoy_input_multformat/${re.sub("[^ 'param_out_mandatory_multformat_adv/output.${adv_opts_cond.param_mandatory_multformat_adv_type}' -optional_multformat_adv_type '$adv_opts_cond.param_optional_multformat_adv_type' - #if str($adv_opts_cond.param_optional_multformat_adv): + #if $adv_opts_cond.param_optional_multformat_adv: -optional_multformat_adv 'param_out_optional_multformat_adv/output.${adv_opts_cond.param_optional_multformat_adv_type}' #end if +#end if +&& mv 'param_out_mandatory_noformat/output.${gxy2omsext("txt")}' '$param_out_mandatory_noformat' +#if $param_optional_noformat: + && mv 'param_out_optional_noformat/output.${gxy2omsext("txt")}' '$param_out_optional_noformat' +#end if +&& mv 'param_out_mandatory_multformat/output.${param_mandatory_multformat_type}' '$param_out_mandatory_multformat' +#if $param_optional_multformat: + && mv 'param_out_optional_multformat/output.${param_optional_multformat_type}' '$param_out_optional_multformat' +#end if +#if $adv_opts_cond.adv_opts_selector=='advanced': + && mv 'param_out_mandatory_noformat_adv/output.${gxy2omsext("txt")}' '$param_out_mandatory_noformat_adv' + #if $adv_opts_cond.param_optional_noformat_adv: + && mv 'param_out_optional_noformat_adv/output.${gxy2omsext("txt")}' '$param_out_optional_noformat_adv' + #end if + && mv 'param_out_mandatory_multformat_adv/output.${adv_opts_cond.param_mandatory_multformat_adv_type}' '$param_out_mandatory_multformat_adv' + #if $adv_opts_cond.param_optional_multformat_adv: + && mv 'param_out_optional_multformat_adv/output.${adv_opts_cond.param_optional_multformat_adv_type}' '$param_out_optional_multformat_adv' + #end if #end if]]> @@ -106,7 +116,7 @@ ln -s '$param_decoy_input_multformat' 'param_decoy_input_multformat/${re.sub("[^ - + @@ -119,7 +129,7 @@ ln -s '$param_decoy_input_multformat' 'param_decoy_input_multformat/${re.sub("[^ - + @@ -136,7 +146,7 @@ ln -s '$param_decoy_input_multformat' 'param_decoy_input_multformat/${re.sub("[^ - + @@ -149,7 +159,7 @@ ln -s '$param_decoy_input_multformat' 'param_decoy_input_multformat/${re.sub("[^ - + diff --git a/tests/test-data/ofile.xml b/tests/test-data/ofile.xml index a039107c..962773d0 100644 --- a/tests/test-data/ofile.xml +++ b/tests/test-data/ofile.xml @@ -13,86 +13,120 @@ @EXT_FOO@ #import re mkdir param_out_mandatory_noformat && -ln -s '$param_out_mandatory_noformat' 'param_out_mandatory_noformat/output.${gxy2omsext("txt")}' && -#if str($param_optional_noformat): +#if $param_optional_noformat: mkdir param_out_optional_noformat && - ln -s '$param_out_optional_noformat' 'param_out_optional_noformat/output.${gxy2omsext("txt")}' && #end if mkdir param_out_mandatory_oneformat && -ln -s '$param_out_mandatory_oneformat' 'param_out_mandatory_oneformat/output.${gxy2omsext("tsv")}' && -#if str($param_optional_oneformat): +#if $param_optional_oneformat: mkdir param_out_optional_oneformat && - ln -s '$param_out_optional_oneformat' 'param_out_optional_oneformat/output.${gxy2omsext("tsv")}' && #end if mkdir param_out_mandatory_multformat && -#if str($param_optional_multformat): +#if $param_optional_multformat: mkdir param_out_optional_multformat && #end if #if $adv_opts_cond.adv_opts_selector=='advanced': mkdir param_out_mandatory_noformat_adv && - ln -s '$param_out_mandatory_noformat_adv' 'param_out_mandatory_noformat_adv/output.${gxy2omsext("txt")}' && - #if str($adv_opts_cond.param_optional_noformat_adv): + #if $adv_opts_cond.param_optional_noformat_adv: mkdir param_out_optional_noformat_adv && - ln -s '$param_out_optional_noformat_adv' 'param_out_optional_noformat_adv/output.${gxy2omsext("txt")}' && #end if mkdir param_out_mandatory_oneformat_adv && - ln -s '$param_out_mandatory_oneformat_adv' 'param_out_mandatory_oneformat_adv/output.${gxy2omsext("tsv")}' && - #if str($adv_opts_cond.param_optional_oneformat_adv): + #if $adv_opts_cond.param_optional_oneformat_adv: mkdir param_out_optional_oneformat_adv && - ln -s '$param_out_optional_oneformat_adv' 'param_out_optional_oneformat_adv/output.${gxy2omsext("tsv")}' && #end if mkdir param_out_mandatory_multformat_adv && - #if str($adv_opts_cond.param_optional_multformat_adv): + #if $adv_opts_cond.param_optional_multformat_adv: mkdir param_out_optional_multformat_adv && #end if #end if @EXECUTABLE@ -mandatory_noformat 'param_out_mandatory_noformat/output.${gxy2omsext("txt")}' -#if str($param_optional_noformat): +#if $param_optional_noformat: -optional_noformat 'param_out_optional_noformat/output.${gxy2omsext("txt")}' #end if -mandatory_oneformat 'param_out_mandatory_oneformat/output.${gxy2omsext("tsv")}' -#if str($param_optional_oneformat): +#if $param_optional_oneformat: -optional_oneformat 'param_out_optional_oneformat/output.${gxy2omsext("tsv")}' #end if -mandatory_multformat -'$param_out_mandatory_multformat' -#if str($param_optional_multformat): +'param_out_mandatory_multformat/output.${param_mandatory_multformat_type}' +#if $param_optional_multformat: -optional_multformat - '$param_out_optional_multformat' + 'param_out_optional_multformat/output.${param_optional_multformat_type}' #end if #if $adv_opts_cond.adv_opts_selector=='advanced': -mandatory_noformat_adv 'param_out_mandatory_noformat_adv/output.${gxy2omsext("txt")}' - #if str($adv_opts_cond.param_optional_noformat_adv): + #if $adv_opts_cond.param_optional_noformat_adv: -optional_noformat_adv 'param_out_optional_noformat_adv/output.${gxy2omsext("txt")}' #end if -mandatory_oneformat_adv 'param_out_mandatory_oneformat_adv/output.${gxy2omsext("tsv")}' - #if str($adv_opts_cond.param_optional_oneformat_adv): + #if $adv_opts_cond.param_optional_oneformat_adv: -optional_oneformat_adv 'param_out_optional_oneformat_adv/output.${gxy2omsext("tsv")}' #end if -mandatory_multformat_adv - '$param_out_mandatory_multformat_adv' - #if str($adv_opts_cond.param_optional_multformat_adv): + 'param_out_mandatory_multformat_adv/output.${adv_opts_cond.param_mandatory_multformat_adv_type}' + #if $adv_opts_cond.param_optional_multformat_adv: -optional_multformat_adv - '$param_out_optional_multformat_adv' + 'param_out_optional_multformat_adv/output.${adv_opts_cond.param_optional_multformat_adv_type}' + #end if +#end if +&& mv 'param_out_mandatory_noformat/output.${gxy2omsext("txt")}' '$param_out_mandatory_noformat' +#if $param_optional_noformat: + && mv 'param_out_optional_noformat/output.${gxy2omsext("txt")}' '$param_out_optional_noformat' +#end if +&& mv 'param_out_mandatory_oneformat/output.${gxy2omsext("tsv")}' '$param_out_mandatory_oneformat' +#if $param_optional_oneformat: + && mv 'param_out_optional_oneformat/output.${gxy2omsext("tsv")}' '$param_out_optional_oneformat' +#end if +&& mv 'param_out_mandatory_multformat/output.${param_mandatory_multformat_type}' '$param_out_mandatory_multformat' +#if $param_optional_multformat: + && mv 'param_out_optional_multformat/output.${param_optional_multformat_type}' '$param_out_optional_multformat' +#end if +#if $adv_opts_cond.adv_opts_selector=='advanced': + && mv 'param_out_mandatory_noformat_adv/output.${gxy2omsext("txt")}' '$param_out_mandatory_noformat_adv' + #if $adv_opts_cond.param_optional_noformat_adv: + && mv 'param_out_optional_noformat_adv/output.${gxy2omsext("txt")}' '$param_out_optional_noformat_adv' + #end if + && mv 'param_out_mandatory_oneformat_adv/output.${gxy2omsext("tsv")}' '$param_out_mandatory_oneformat_adv' + #if $adv_opts_cond.param_optional_oneformat_adv: + && mv 'param_out_optional_oneformat_adv/output.${gxy2omsext("tsv")}' '$param_out_optional_oneformat_adv' + #end if + && mv 'param_out_mandatory_multformat_adv/output.${adv_opts_cond.param_mandatory_multformat_adv_type}' '$param_out_mandatory_multformat_adv' + #if $adv_opts_cond.param_optional_multformat_adv: + && mv 'param_out_optional_multformat_adv/output.${adv_opts_cond.param_optional_multformat_adv_type}' '$param_out_optional_multformat_adv' #end if #end if]]> - - - + + + + + + + + + + + - - - + + + + + + + + + + + @@ -125,10 +159,14 @@ mkdir param_out_mandatory_multformat && + + + + From 2e676eec750061242c54b1b20cc11940cf397e20 Mon Sep 17 00:00:00 2001 From: M Bernt Date: Sun, 29 Dec 2019 20:15:11 +0100 Subject: [PATCH 054/121] add script to create Galaxy tool xml files for tests --- create_galaxy_tests.sh | 1 + tests/test-data/hardcoded_params.json | 5 +++++ 2 files changed, 6 insertions(+) create mode 100755 create_galaxy_tests.sh create mode 100644 tests/test-data/hardcoded_params.json diff --git a/create_galaxy_tests.sh b/create_galaxy_tests.sh new file mode 100755 index 00000000..3766eefc --- /dev/null +++ b/create_galaxy_tests.sh @@ -0,0 +1 @@ +python2 convert.py galaxy -i tests/test-data/bool.ctd tests/test-data/empty.ctd tests/test-data/float.ctd tests/test-data/ifile.ctd tests/test-data/integer.ctd tests/test-data/ofile-corresponding-input.ctd tests/test-data/ofile.ctd tests/test-data/ofile-mult.ctd tests/test-data/ofile-mult-typeparam.ctd tests/test-data/ofile-typeparam.ctd tests/test-data/repeat.ctd tests/test-data/select.ctd tests/test-data/string.ctd -o tests/test-data/ -m tests/test-data/macros.xml -f tests/test-data/filetypes.txt --test-test -p tests/test-data/hardcoded_params.json diff --git a/tests/test-data/hardcoded_params.json b/tests/test-data/hardcoded_params.json new file mode 100644 index 00000000..a5efbd8f --- /dev/null +++ b/tests/test-data/hardcoded_params.json @@ -0,0 +1,5 @@ +{ + "#": "blacklist parameters", + + "version": [{"value": "@"}] +} From da3a9ae87bfc432ed738f7daaa41c6fe7cb39ec3 Mon Sep 17 00:00:00 2001 From: M Bernt Date: Mon, 30 Dec 2019 12:08:03 +0100 Subject: [PATCH 055/121] revert to double quotes .. for now --- galaxy/converter.py | 8 +-- galaxy/macros.xml | 14 +++--- tests/test-data/float.xml | 64 ++++++++++++------------ tests/test-data/integer.xml | 64 ++++++++++++------------ tests/test-data/ofile-mult-typeparam.xml | 4 +- tests/test-data/ofile-typeparam.xml | 16 +++--- tests/test-data/repeat.xml | 12 ++--- tests/test-data/select.xml | 64 ++++++++++++------------ tests/test-data/string.xml | 16 +++--- 9 files changed, 132 insertions(+), 130 deletions(-) diff --git a/galaxy/converter.py b/galaxy/converter.py index 547bf8ff..7c3429cc 100755 --- a/galaxy/converter.py +++ b/galaxy/converter.py @@ -686,14 +686,14 @@ def create_command(tool, model, **kwargs): # select with multiple = true elif is_selection_parameter(param) and param.is_list: - param_cmd['command'].append("${' '.join([\"'%s'\"%str(_) for _ in str($" + actual_parameter + ").split(',')])}") + param_cmd['command'].append("${' '.join(['\"%s\"'%str(_) for _ in str($" + actual_parameter + ").split(',')])}") elif param.is_list: param_cmd['command'].append("$quote($%s" % actual_parameter + ")") #command += "${' '.join([\"'%s'\"%str(_) for _ in $" + actual_parameter + "])}\n" elif is_boolean_parameter(param): param_cmd['command'].append("$%s" % actual_parameter + "") else: - param_cmd['command'].append("'$" + actual_parameter + "'") + param_cmd['command'].append('"$' + actual_parameter + '"') # add if statement for optional parameters and preprocessing # - for optional outputs (param_out_x) the presence of the parameter @@ -1106,9 +1106,9 @@ def create_param_attribute_list(param_node, param, model, supported_file_formats if type(param.default) is list: for i, d in enumerate(param.default): if " " in d: - param.default[i] = "'%s'" %d + param.default[i] = '"%s"' %d elif " " in param.default: - param.default = "'%s'" %param.default + param.default = '"%s"' %param.default # add sanitizer nodes to # - text (only those that are not actually integer selects which are treated above) and # - select params, diff --git a/galaxy/macros.xml b/galaxy/macros.xml index 15fa17b2..b25a80ca 100644 --- a/galaxy/macros.xml +++ b/galaxy/macros.xml @@ -4,7 +4,7 @@ add additional macro files using the m/macros parameter --> 2.3.0 - 3 + 12 openms @@ -38,7 +38,7 @@ - ^ *((?:\'[^\']*\' +)|(?:[^ \']+ +))*((?:\'[^\']*\')|(?:[^ \']+)) *$ + ^ *((?:\"[^\"]*\" +)|(?:[^ \"]+ +))*((?:\"[^\"]*\")|(?:[^ \"]+)) *$ @@ -89,16 +89,18 @@ #if $p == "": #continue #end if - #if $p.startswith("'"): + #if $p.startswith('"'): #set $q = True #end if #if p.startswith('-'): - p = "\\" + p + #set p = "\\" + p + #elif p.startswith('"-'): + #set p = "\\" + p[1:] #end if #if not $q: - #set $s[i] = "'%s'" % p + #set $s[i] = '"%s"' % p #end if - #if $p.endswith("'"): + #if $p.endswith('"'): #set $q = False #end if #end for diff --git a/tests/test-data/float.xml b/tests/test-data/float.xml index 1b209d5c..0ddf3828 100644 --- a/tests/test-data/float.xml +++ b/tests/test-data/float.xml @@ -14,101 +14,101 @@ #import re @EXECUTABLE@ -wodefault_mandatory_unrestricted -'$param_wodefault_mandatory_unrestricted' +"$param_wodefault_mandatory_unrestricted" -default_mandatory_unrestricted -'$param_default_mandatory_unrestricted' +"$param_default_mandatory_unrestricted" #if str($param_wodefault_optional_unrestricted): -wodefault_optional_unrestricted - '$param_wodefault_optional_unrestricted' + "$param_wodefault_optional_unrestricted" #end if #if str($param_default_optional_unrestricted): -default_optional_unrestricted - '$param_default_optional_unrestricted' + "$param_default_optional_unrestricted" #end if -wodefault_mandatory_unrestricted_min -'$param_wodefault_mandatory_unrestricted_min' +"$param_wodefault_mandatory_unrestricted_min" -default_mandatory_unrestricted_min -'$param_default_mandatory_unrestricted_min' +"$param_default_mandatory_unrestricted_min" #if str($param_wodefault_optional_unrestricted_min): -wodefault_optional_unrestricted_min - '$param_wodefault_optional_unrestricted_min' + "$param_wodefault_optional_unrestricted_min" #end if #if str($param_default_optional_unrestricted_min): -default_optional_unrestricted_min - '$param_default_optional_unrestricted_min' + "$param_default_optional_unrestricted_min" #end if -wodefault_mandatory_unrestricted_max -'$param_wodefault_mandatory_unrestricted_max' +"$param_wodefault_mandatory_unrestricted_max" -default_mandatory_unrestricted_max -'$param_default_mandatory_unrestricted_max' +"$param_default_mandatory_unrestricted_max" #if str($param_wodefault_optional_unrestricted_max): -wodefault_optional_unrestricted_max - '$param_wodefault_optional_unrestricted_max' + "$param_wodefault_optional_unrestricted_max" #end if #if str($param_default_optional_unrestricted_max): -default_optional_unrestricted_max - '$param_default_optional_unrestricted_max' + "$param_default_optional_unrestricted_max" #end if -wodefault_mandatory_unrestricted_minmax -'$param_wodefault_mandatory_unrestricted_minmax' +"$param_wodefault_mandatory_unrestricted_minmax" -default_mandatory_unrestricted_minmax -'$param_default_mandatory_unrestricted_minmax' +"$param_default_mandatory_unrestricted_minmax" #if str($param_wodefault_optional_unrestricted_minmax): -wodefault_optional_unrestricted_minmax - '$param_wodefault_optional_unrestricted_minmax' + "$param_wodefault_optional_unrestricted_minmax" #end if #if str($param_default_optional_unrestricted_minmax): -default_optional_unrestricted_minmax - '$param_default_optional_unrestricted_minmax' + "$param_default_optional_unrestricted_minmax" #end if #if $adv_opts_cond.adv_opts_selector=='advanced': -wodefault_mandatory_unrestricted_adv - '$adv_opts_cond.param_wodefault_mandatory_unrestricted_adv' + "$adv_opts_cond.param_wodefault_mandatory_unrestricted_adv" -default_mandatory_unrestricted_adv - '$adv_opts_cond.param_default_mandatory_unrestricted_adv' + "$adv_opts_cond.param_default_mandatory_unrestricted_adv" #if str($adv_opts_cond.param_wodefault_optional_unrestricted_adv): -wodefault_optional_unrestricted_adv - '$adv_opts_cond.param_wodefault_optional_unrestricted_adv' + "$adv_opts_cond.param_wodefault_optional_unrestricted_adv" #end if #if str($adv_opts_cond.param_default_optional_unrestricted_adv): -default_optional_unrestricted_adv - '$adv_opts_cond.param_default_optional_unrestricted_adv' + "$adv_opts_cond.param_default_optional_unrestricted_adv" #end if -wodefault_mandatory_unrestricted_adv_min - '$adv_opts_cond.param_wodefault_mandatory_unrestricted_adv_min' + "$adv_opts_cond.param_wodefault_mandatory_unrestricted_adv_min" -default_mandatory_unrestricted_adv_min - '$adv_opts_cond.param_default_mandatory_unrestricted_adv_min' + "$adv_opts_cond.param_default_mandatory_unrestricted_adv_min" #if str($adv_opts_cond.param_wodefault_optional_unrestricted_adv_min): -wodefault_optional_unrestricted_adv_min - '$adv_opts_cond.param_wodefault_optional_unrestricted_adv_min' + "$adv_opts_cond.param_wodefault_optional_unrestricted_adv_min" #end if #if str($adv_opts_cond.param_default_optional_unrestricted_adv_min): -default_optional_unrestricted_adv_min - '$adv_opts_cond.param_default_optional_unrestricted_adv_min' + "$adv_opts_cond.param_default_optional_unrestricted_adv_min" #end if -wodefault_mandatory_unrestricted_adv_max - '$adv_opts_cond.param_wodefault_mandatory_unrestricted_adv_max' + "$adv_opts_cond.param_wodefault_mandatory_unrestricted_adv_max" -default_mandatory_unrestricted_adv_max - '$adv_opts_cond.param_default_mandatory_unrestricted_adv_max' + "$adv_opts_cond.param_default_mandatory_unrestricted_adv_max" #if str($adv_opts_cond.param_wodefault_optional_unrestricted_adv_max): -wodefault_optional_unrestricted_adv_max - '$adv_opts_cond.param_wodefault_optional_unrestricted_adv_max' + "$adv_opts_cond.param_wodefault_optional_unrestricted_adv_max" #end if #if str($adv_opts_cond.param_default_optional_unrestricted_adv_max): -default_optional_unrestricted_adv_max - '$adv_opts_cond.param_default_optional_unrestricted_adv_max' + "$adv_opts_cond.param_default_optional_unrestricted_adv_max" #end if -wodefault_mandatory_unrestricted_adv_minmax - '$adv_opts_cond.param_wodefault_mandatory_unrestricted_adv_minmax' + "$adv_opts_cond.param_wodefault_mandatory_unrestricted_adv_minmax" -default_mandatory_unrestricted_adv_minmax - '$adv_opts_cond.param_default_mandatory_unrestricted_adv_minmax' + "$adv_opts_cond.param_default_mandatory_unrestricted_adv_minmax" #if str($adv_opts_cond.param_wodefault_optional_unrestricted_adv_minmax): -wodefault_optional_unrestricted_adv_minmax - '$adv_opts_cond.param_wodefault_optional_unrestricted_adv_minmax' + "$adv_opts_cond.param_wodefault_optional_unrestricted_adv_minmax" #end if #if str($adv_opts_cond.param_default_optional_unrestricted_adv_minmax): -default_optional_unrestricted_adv_minmax - '$adv_opts_cond.param_default_optional_unrestricted_adv_minmax' + "$adv_opts_cond.param_default_optional_unrestricted_adv_minmax" #end if #end if > $param_stdout]]> diff --git a/tests/test-data/integer.xml b/tests/test-data/integer.xml index 373262d0..ae525d79 100644 --- a/tests/test-data/integer.xml +++ b/tests/test-data/integer.xml @@ -14,101 +14,101 @@ #import re @EXECUTABLE@ -wodefault_mandatory_unrestricted -'$param_wodefault_mandatory_unrestricted' +"$param_wodefault_mandatory_unrestricted" -default_mandatory_unrestricted -'$param_default_mandatory_unrestricted' +"$param_default_mandatory_unrestricted" #if str($param_wodefault_optional_unrestricted): -wodefault_optional_unrestricted - '$param_wodefault_optional_unrestricted' + "$param_wodefault_optional_unrestricted" #end if #if str($param_default_optional_unrestricted): -default_optional_unrestricted - '$param_default_optional_unrestricted' + "$param_default_optional_unrestricted" #end if -wodefault_mandatory_unrestricted_min -'$param_wodefault_mandatory_unrestricted_min' +"$param_wodefault_mandatory_unrestricted_min" -default_mandatory_unrestricted_min -'$param_default_mandatory_unrestricted_min' +"$param_default_mandatory_unrestricted_min" #if str($param_wodefault_optional_unrestricted_min): -wodefault_optional_unrestricted_min - '$param_wodefault_optional_unrestricted_min' + "$param_wodefault_optional_unrestricted_min" #end if #if str($param_default_optional_unrestricted_min): -default_optional_unrestricted_min - '$param_default_optional_unrestricted_min' + "$param_default_optional_unrestricted_min" #end if -wodefault_mandatory_unrestricted_max -'$param_wodefault_mandatory_unrestricted_max' +"$param_wodefault_mandatory_unrestricted_max" -default_mandatory_unrestricted_max -'$param_default_mandatory_unrestricted_max' +"$param_default_mandatory_unrestricted_max" #if str($param_wodefault_optional_unrestricted_max): -wodefault_optional_unrestricted_max - '$param_wodefault_optional_unrestricted_max' + "$param_wodefault_optional_unrestricted_max" #end if #if str($param_default_optional_unrestricted_max): -default_optional_unrestricted_max - '$param_default_optional_unrestricted_max' + "$param_default_optional_unrestricted_max" #end if -wodefault_mandatory_unrestricted_minmax -'$param_wodefault_mandatory_unrestricted_minmax' +"$param_wodefault_mandatory_unrestricted_minmax" -default_mandatory_unrestricted_minmax -'$param_default_mandatory_unrestricted_minmax' +"$param_default_mandatory_unrestricted_minmax" #if str($param_wodefault_optional_unrestricted_minmax): -wodefault_optional_unrestricted_minmax - '$param_wodefault_optional_unrestricted_minmax' + "$param_wodefault_optional_unrestricted_minmax" #end if #if str($param_default_optional_unrestricted_minmax): -default_optional_unrestricted_minmax - '$param_default_optional_unrestricted_minmax' + "$param_default_optional_unrestricted_minmax" #end if #if $adv_opts_cond.adv_opts_selector=='advanced': -wodefault_mandatory_unrestricted_adv - '$adv_opts_cond.param_wodefault_mandatory_unrestricted_adv' + "$adv_opts_cond.param_wodefault_mandatory_unrestricted_adv" -default_mandatory_unrestricted_adv - '$adv_opts_cond.param_default_mandatory_unrestricted_adv' + "$adv_opts_cond.param_default_mandatory_unrestricted_adv" #if str($adv_opts_cond.param_wodefault_optional_unrestricted_adv): -wodefault_optional_unrestricted_adv - '$adv_opts_cond.param_wodefault_optional_unrestricted_adv' + "$adv_opts_cond.param_wodefault_optional_unrestricted_adv" #end if #if str($adv_opts_cond.param_default_optional_unrestricted_adv): -default_optional_unrestricted_adv - '$adv_opts_cond.param_default_optional_unrestricted_adv' + "$adv_opts_cond.param_default_optional_unrestricted_adv" #end if -wodefault_mandatory_unrestricted_adv_min - '$adv_opts_cond.param_wodefault_mandatory_unrestricted_adv_min' + "$adv_opts_cond.param_wodefault_mandatory_unrestricted_adv_min" -default_mandatory_unrestricted_adv_min - '$adv_opts_cond.param_default_mandatory_unrestricted_adv_min' + "$adv_opts_cond.param_default_mandatory_unrestricted_adv_min" #if str($adv_opts_cond.param_wodefault_optional_unrestricted_adv_min): -wodefault_optional_unrestricted_adv_min - '$adv_opts_cond.param_wodefault_optional_unrestricted_adv_min' + "$adv_opts_cond.param_wodefault_optional_unrestricted_adv_min" #end if #if str($adv_opts_cond.param_default_optional_unrestricted_adv_min): -default_optional_unrestricted_adv_min - '$adv_opts_cond.param_default_optional_unrestricted_adv_min' + "$adv_opts_cond.param_default_optional_unrestricted_adv_min" #end if -wodefault_mandatory_unrestricted_adv_max - '$adv_opts_cond.param_wodefault_mandatory_unrestricted_adv_max' + "$adv_opts_cond.param_wodefault_mandatory_unrestricted_adv_max" -default_mandatory_unrestricted_adv_max - '$adv_opts_cond.param_default_mandatory_unrestricted_adv_max' + "$adv_opts_cond.param_default_mandatory_unrestricted_adv_max" #if str($adv_opts_cond.param_wodefault_optional_unrestricted_adv_max): -wodefault_optional_unrestricted_adv_max - '$adv_opts_cond.param_wodefault_optional_unrestricted_adv_max' + "$adv_opts_cond.param_wodefault_optional_unrestricted_adv_max" #end if #if str($adv_opts_cond.param_default_optional_unrestricted_adv_max): -default_optional_unrestricted_adv_max - '$adv_opts_cond.param_default_optional_unrestricted_adv_max' + "$adv_opts_cond.param_default_optional_unrestricted_adv_max" #end if -wodefault_mandatory_unrestricted_adv_minmax - '$adv_opts_cond.param_wodefault_mandatory_unrestricted_adv_minmax' + "$adv_opts_cond.param_wodefault_mandatory_unrestricted_adv_minmax" -default_mandatory_unrestricted_adv_minmax - '$adv_opts_cond.param_default_mandatory_unrestricted_adv_minmax' + "$adv_opts_cond.param_default_mandatory_unrestricted_adv_minmax" #if str($adv_opts_cond.param_wodefault_optional_unrestricted_adv_minmax): -wodefault_optional_unrestricted_adv_minmax - '$adv_opts_cond.param_wodefault_optional_unrestricted_adv_minmax' + "$adv_opts_cond.param_wodefault_optional_unrestricted_adv_minmax" #end if #if str($adv_opts_cond.param_default_optional_unrestricted_adv_minmax): -default_optional_unrestricted_adv_minmax - '$adv_opts_cond.param_default_optional_unrestricted_adv_minmax' + "$adv_opts_cond.param_default_optional_unrestricted_adv_minmax" #end if #end if > $param_stdout]]> diff --git a/tests/test-data/ofile-mult-typeparam.xml b/tests/test-data/ofile-mult-typeparam.xml index 55d67ca5..ecd5ae06 100644 --- a/tests/test-data/ofile-mult-typeparam.xml +++ b/tests/test-data/ofile-mult-typeparam.xml @@ -22,11 +22,11 @@ ${ ' '.join([ "ln -s '%s' 'param_inp_optional_mandatoryinput/%s.%s' &&" % (_, re #end if @EXECUTABLE@ -mandatory_mandatoryinput_type -'$param_mandatory_mandatoryinput_type' +"$param_mandatory_mandatoryinput_type" -mandatory_mandatoryinput ${' '.join(["'param_out_mandatory_mandatoryinput/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext("tsv")) for _ in $param_inp_optional_mandatoryinput if _ ])} -mandatory_optionalinput_type -'$param_mandatory_optionalinput_type' +"$param_mandatory_optionalinput_type" -mandatory_optionalinput ${' '.join(["'param_out_mandatory_optionalinput/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier), param_mandatory_optionalinput_type) for _ in $param_inp_mandatory_optionalinput if _ ])} -inp_optional_mandatoryinput diff --git a/tests/test-data/ofile-typeparam.xml b/tests/test-data/ofile-typeparam.xml index ecbcdc1b..e79192a9 100644 --- a/tests/test-data/ofile-typeparam.xml +++ b/tests/test-data/ofile-typeparam.xml @@ -38,21 +38,21 @@ ln -s '$param_decoy_input_multformat' 'param_decoy_input_multformat/${re.sub("[^ #end if @EXECUTABLE@ -mandatory_noformat_type -'$param_mandatory_noformat_type' +"$param_mandatory_noformat_type" -mandatory_noformat 'param_out_mandatory_noformat/output.${gxy2omsext("txt")}' -optional_noformat_type -'$param_optional_noformat_type' +"$param_optional_noformat_type" #if $param_optional_noformat: -optional_noformat 'param_out_optional_noformat/output.${gxy2omsext("txt")}' #end if -mandatory_multformat_type -'$param_mandatory_multformat_type' +"$param_mandatory_multformat_type" -mandatory_multformat 'param_out_mandatory_multformat/output.${param_mandatory_multformat_type}' -optional_multformat_type -'$param_optional_multformat_type' +"$param_optional_multformat_type" #if $param_optional_multformat: -optional_multformat 'param_out_optional_multformat/output.${param_optional_multformat_type}' @@ -65,21 +65,21 @@ ln -s '$param_decoy_input_multformat' 'param_decoy_input_multformat/${re.sub("[^ 'param_decoy_input_multformat/${re.sub('[^\w\-_]', '_', $param_decoy_input_multformat.element_identifier)}.$gxy2omsext($param_decoy_input_multformat.ext)' #if $adv_opts_cond.adv_opts_selector=='advanced': -mandatory_noformat_adv_type - '$adv_opts_cond.param_mandatory_noformat_adv_type' + "$adv_opts_cond.param_mandatory_noformat_adv_type" -mandatory_noformat_adv 'param_out_mandatory_noformat_adv/output.${gxy2omsext("txt")}' -optional_noformat_adv_type - '$adv_opts_cond.param_optional_noformat_adv_type' + "$adv_opts_cond.param_optional_noformat_adv_type" #if $adv_opts_cond.param_optional_noformat_adv: -optional_noformat_adv 'param_out_optional_noformat_adv/output.${gxy2omsext("txt")}' #end if -mandatory_multformat_adv_type - '$adv_opts_cond.param_mandatory_multformat_adv_type' + "$adv_opts_cond.param_mandatory_multformat_adv_type" -mandatory_multformat_adv 'param_out_mandatory_multformat_adv/output.${adv_opts_cond.param_mandatory_multformat_adv_type}' -optional_multformat_adv_type - '$adv_opts_cond.param_optional_multformat_adv_type' + "$adv_opts_cond.param_optional_multformat_adv_type" #if $adv_opts_cond.param_optional_multformat_adv: -optional_multformat_adv 'param_out_optional_multformat_adv/output.${adv_opts_cond.param_optional_multformat_adv_type}' diff --git a/tests/test-data/repeat.xml b/tests/test-data/repeat.xml index b94782af..2a56475a 100644 --- a/tests/test-data/repeat.xml +++ b/tests/test-data/repeat.xml @@ -137,11 +137,11 @@ $quote($param_double_resticted_default_mandatory) - + - + @@ -226,11 +226,11 @@ $quote($param_double_resticted_default_mandatory) - + - + @@ -314,7 +314,7 @@ $quote($param_double_resticted_default_mandatory) - + @@ -325,7 +325,7 @@ $quote($param_double_resticted_default_mandatory) - + diff --git a/tests/test-data/select.xml b/tests/test-data/select.xml index e82297a8..622f434e 100644 --- a/tests/test-data/select.xml +++ b/tests/test-data/select.xml @@ -14,102 +14,102 @@ #import re @EXECUTABLE@ -wodefault_mandatory -'$param_wodefault_mandatory' +"$param_wodefault_mandatory" -default_mandatory -'$param_default_mandatory' +"$param_default_mandatory" #if $param_wodefault_optional: -wodefault_optional - '$param_wodefault_optional' + "$param_wodefault_optional" #end if #if $param_default_optional: -default_optional - '$param_default_optional' + "$param_default_optional" #end if -wodefault_mandatory_many -'$param_wodefault_mandatory_many' +"$param_wodefault_mandatory_many" -default_mandatory_many -'$param_default_mandatory_many' +"$param_default_mandatory_many" #if $param_wodefault_optional_many: -wodefault_optional_many - '$param_wodefault_optional_many' + "$param_wodefault_optional_many" #end if #if $param_default_optional_many: -default_optional_many - '$param_default_optional_many' + "$param_default_optional_many" #end if #if $param_list_wodefault_optional: -list_wodefault_optional - ${' '.join(["'%s'"%str(_) for _ in str($param_list_wodefault_optional).split(',')])} + ${' '.join(['"%s"'%str(_) for _ in str($param_list_wodefault_optional).split(',')])} #end if -list_wodefault_mandatory -${' '.join(["'%s'"%str(_) for _ in str($param_list_wodefault_mandatory).split(',')])} +${' '.join(['"%s"'%str(_) for _ in str($param_list_wodefault_mandatory).split(',')])} #if $param_list_default_optional: -list_default_optional - ${' '.join(["'%s'"%str(_) for _ in str($param_list_default_optional).split(',')])} + ${' '.join(['"%s"'%str(_) for _ in str($param_list_default_optional).split(',')])} #end if -list_default_mandatory -${' '.join(["'%s'"%str(_) for _ in str($param_list_default_mandatory).split(',')])} +${' '.join(['"%s"'%str(_) for _ in str($param_list_default_mandatory).split(',')])} #if $param_list_wodefault_optional_many: -list_wodefault_optional_many - ${' '.join(["'%s'"%str(_) for _ in str($param_list_wodefault_optional_many).split(',')])} + ${' '.join(['"%s"'%str(_) for _ in str($param_list_wodefault_optional_many).split(',')])} #end if -list_wodefault_mandatory_many -${' '.join(["'%s'"%str(_) for _ in str($param_list_wodefault_mandatory_many).split(',')])} +${' '.join(['"%s"'%str(_) for _ in str($param_list_wodefault_mandatory_many).split(',')])} #if $param_list_default_optional_many: -list_default_optional_many - ${' '.join(["'%s'"%str(_) for _ in str($param_list_default_optional_many).split(',')])} + ${' '.join(['"%s"'%str(_) for _ in str($param_list_default_optional_many).split(',')])} #end if -list_default_mandatory_many -${' '.join(["'%s'"%str(_) for _ in str($param_list_default_mandatory_many).split(',')])} +${' '.join(['"%s"'%str(_) for _ in str($param_list_default_mandatory_many).split(',')])} #if $adv_opts_cond.adv_opts_selector=='advanced': -wodefault_mandatory_adv - '$adv_opts_cond.param_wodefault_mandatory_adv' + "$adv_opts_cond.param_wodefault_mandatory_adv" -default_mandatory_adv - '$adv_opts_cond.param_default_mandatory_adv' + "$adv_opts_cond.param_default_mandatory_adv" #if $adv_opts_cond.param_wodefault_optional_adv: -wodefault_optional_adv - '$adv_opts_cond.param_wodefault_optional_adv' + "$adv_opts_cond.param_wodefault_optional_adv" #end if #if $adv_opts_cond.param_default_optional_adv: -default_optional_adv - '$adv_opts_cond.param_default_optional_adv' + "$adv_opts_cond.param_default_optional_adv" #end if -wodefault_mandatory_many_adv - '$adv_opts_cond.param_wodefault_mandatory_many_adv' + "$adv_opts_cond.param_wodefault_mandatory_many_adv" -default_mandatory_many_adv - '$adv_opts_cond.param_default_mandatory_many_adv' + "$adv_opts_cond.param_default_mandatory_many_adv" #if $adv_opts_cond.param_wodefault_optional_many_adv: -wodefault_optional_many_adv - '$adv_opts_cond.param_wodefault_optional_many_adv' + "$adv_opts_cond.param_wodefault_optional_many_adv" #end if #if $adv_opts_cond.param_default_optional_many_adv: -default_optional_many_adv - '$adv_opts_cond.param_default_optional_many_adv' + "$adv_opts_cond.param_default_optional_many_adv" #end if #if $adv_opts_cond.param_list_wodefault_optional_adv: -list_wodefault_optional_adv - ${' '.join(["'%s'"%str(_) for _ in str($adv_opts_cond.param_list_wodefault_optional_adv).split(',')])} + ${' '.join(['"%s"'%str(_) for _ in str($adv_opts_cond.param_list_wodefault_optional_adv).split(',')])} #end if -list_wodefault_mandatory_adv - ${' '.join(["'%s'"%str(_) for _ in str($adv_opts_cond.param_list_wodefault_mandatory_adv).split(',')])} + ${' '.join(['"%s"'%str(_) for _ in str($adv_opts_cond.param_list_wodefault_mandatory_adv).split(',')])} #if $adv_opts_cond.param_list_default_optional_adv: -list_default_optional_adv - ${' '.join(["'%s'"%str(_) for _ in str($adv_opts_cond.param_list_default_optional_adv).split(',')])} + ${' '.join(['"%s"'%str(_) for _ in str($adv_opts_cond.param_list_default_optional_adv).split(',')])} #end if -list_default_mandatory_adv - ${' '.join(["'%s'"%str(_) for _ in str($adv_opts_cond.param_list_default_mandatory_adv).split(',')])} + ${' '.join(['"%s"'%str(_) for _ in str($adv_opts_cond.param_list_default_mandatory_adv).split(',')])} #if $adv_opts_cond.param_list_wodefault_optional_many_adv: -list_wodefault_optional_many_adv - ${' '.join(["'%s'"%str(_) for _ in str($adv_opts_cond.param_list_wodefault_optional_many_adv).split(',')])} + ${' '.join(['"%s"'%str(_) for _ in str($adv_opts_cond.param_list_wodefault_optional_many_adv).split(',')])} #end if -list_wodefault_mandatory_many_adv - ${' '.join(["'%s'"%str(_) for _ in str($adv_opts_cond.param_list_wodefault_mandatory_many_adv).split(',')])} + ${' '.join(['"%s"'%str(_) for _ in str($adv_opts_cond.param_list_wodefault_mandatory_many_adv).split(',')])} #if $adv_opts_cond.param_list_default_optional_many_adv: -list_default_optional_many_adv - ${' '.join(["'%s'"%str(_) for _ in str($adv_opts_cond.param_list_default_optional_many_adv).split(',')])} + ${' '.join(['"%s"'%str(_) for _ in str($adv_opts_cond.param_list_default_optional_many_adv).split(',')])} #end if -list_default_mandatory_many_adv - ${' '.join(["'%s'"%str(_) for _ in str($adv_opts_cond.param_list_default_mandatory_many_adv).split(',')])} + ${' '.join(['"%s"'%str(_) for _ in str($adv_opts_cond.param_list_default_mandatory_many_adv).split(',')])} #end if > $param_stdout]]> diff --git a/tests/test-data/string.xml b/tests/test-data/string.xml index d933520b..bec44bc9 100644 --- a/tests/test-data/string.xml +++ b/tests/test-data/string.xml @@ -14,22 +14,22 @@ #import re @EXECUTABLE@ -wodefault_mandatory_unrestricted -'$param_wodefault_mandatory_unrestricted' +"$param_wodefault_mandatory_unrestricted" -default_mandatory_unrestricted -'$param_default_mandatory_unrestricted' +"$param_default_mandatory_unrestricted" -wodefault_optional_unrestricted -'$param_wodefault_optional_unrestricted' +"$param_wodefault_optional_unrestricted" -default_optional_unrestricted -'$param_default_optional_unrestricted' +"$param_default_optional_unrestricted" #if $adv_opts_cond.adv_opts_selector=='advanced': -wodefault_mandatory_unrestricted_adv - '$adv_opts_cond.param_wodefault_mandatory_unrestricted_adv' + "$adv_opts_cond.param_wodefault_mandatory_unrestricted_adv" -default_mandatory_unrestricted_adv - '$adv_opts_cond.param_default_mandatory_unrestricted_adv' + "$adv_opts_cond.param_default_mandatory_unrestricted_adv" -wodefault_optional_unrestricted_adv - '$adv_opts_cond.param_wodefault_optional_unrestricted_adv' + "$adv_opts_cond.param_wodefault_optional_unrestricted_adv" -default_optional_unrestricted_adv - '$adv_opts_cond.param_default_optional_unrestricted_adv' + "$adv_opts_cond.param_default_optional_unrestricted_adv" #end if > $param_stdout]]> From 0b492330a8b20ea9b3195886b0ce672f7af72fa3 Mon Sep 17 00:00:00 2001 From: M Bernt Date: Wed, 8 Jan 2020 17:27:28 +0100 Subject: [PATCH 056/121] hardcoding CTD restrictions, format selector - implement overwriting of CTD restrictions via hardcoded params - format selectors use Galaxt data format - dont autobump galaxy version --- common/utils.py | 5 - galaxy/converter.py | 102 +++++++++--------- tests/test-data/ifile.xml | 48 ++++----- tests/test-data/ofile-corresponding-input.xml | 6 +- tests/test-data/ofile-mult-typeparam.xml | 10 +- tests/test-data/ofile-typeparam.xml | 54 +++++----- tests/test-data/ofile.xml | 24 ++--- 7 files changed, 124 insertions(+), 125 deletions(-) diff --git a/common/utils.py b/common/utils.py index 6d4dda70..7de2d02d 100644 --- a/common/utils.py +++ b/common/utils.py @@ -225,11 +225,6 @@ def parse_hardcoded_parameters(hardcoded_parameters_file): el[a] = _InFile parameter_hardcoder.register_attribute(parameter_name, a, el[a], tool_name) - # hard coded values - logger.info("parameter_map %s" %parameter_hardcoder.parameter_map) - logger.info("attribute_map %s" %parameter_hardcoder.attribute_map) - logger.info("blacklist %s" %parameter_hardcoder.blacklist) - return parameter_hardcoder diff --git a/galaxy/converter.py b/galaxy/converter.py index 7c3429cc..97d4559b 100755 --- a/galaxy/converter.py +++ b/galaxy/converter.py @@ -13,7 +13,7 @@ from common import utils, logger from common.exceptions import ApplicationException, InvalidModelException -from CTDopts.CTDopts import _InFile, _OutFile, ParameterGroup, _Choices, _NumericRange, _FileFormat, ModelError, _Null +from CTDopts.CTDopts import _InFile, _OutFile, ParameterGroup, _Choices, _NumericRange, _FileFormat, ModelError, _Null, CTDTYPE_TO_TYPE # mapping to CTD types to Galaxy types @@ -206,7 +206,7 @@ def parse_macros_files(macros_file_names, tool_version, supported_file_types, re galaxy_ver_tk = add_child_node(root, "token", OrderedDict([("name", "@GALAXY_VERSION@")])) galaxy_ver_tk.text = "0" if tool_version == tool_ver_tk.text: - galaxy_ver_tk.text = str(int(galaxy_ver_tk.text) + 1) + galaxy_ver_tk.text = str(int(galaxy_ver_tk.text)) else: tool_ver_tk.text = tool_version galaxy_ver_tk.text = "0" @@ -215,12 +215,8 @@ def parse_macros_files(macros_file_names, tool_version, supported_file_types, re if ext_foo is None: ext_foo = add_child_node(root, "token", OrderedDict([("name", "@EXT_FOO@")])) - o2g = {} - g2o = {} - for s in supported_file_types: - o2g[s] = supported_file_types[s].galaxy_extension - if supported_file_types[s].galaxy_extension not in g2o: - g2o[supported_file_types[s].galaxy_extension] = s + g2o, o2g = get_fileformat_maps(supported_file_types) + # make sure that the backup data type is in the map if 'txt' not in g2o: g2o['txt'] = 'txt' @@ -335,6 +331,20 @@ def parse_file_formats(formats_file): return supported_formats +def get_fileformat_maps(supported_formats): + """ + convenience functions to compute dictionaries mapping + Galaxy data types <-> CTD formats + """ + o2g = {} + g2o = {} + for s in supported_formats: + o2g[s] = supported_formats[s].galaxy_extension + if supported_formats[s].galaxy_extension not in g2o: + g2o[supported_formats[s].galaxy_extension] = s + return g2o, o2g + + def validate_and_prepare_args(args, model): """ check command line arguments @@ -411,7 +421,16 @@ def _convert_internal(parsed_ctds, **kwargs): for a in hardcoded_attributes: if not hasattr(param, a): continue - setattr(param, a, hardcoded_attributes[a]) + logger.error("%s %s"%(param.name,type(getattr(param, a))) ) + if type(getattr(param, a)) is _FileFormat: + setattr(param, a, _FileFormat(str(hardcoded_attributes[a]))) + elif type(getattr(param, a)) is _Choices: + logger.error("SET CTD %s %s %s" %(param.name, a, hardcoded_attributes[a])) + setattr(param, a, _Choices(str(hardcoded_attributes[a]))) + elif type(getattr(param, a)) is _NumericRange: + raise Exception("Overwriting of Numeric Range not implemented") + else: + setattr(param, a, hardcoded_attributes[a]) logger.info("Converting %s (source %s)" % (model.name, utils.get_filename(origin_file)), 0) tool = create_tool(model) @@ -438,25 +457,6 @@ def _convert_internal(parsed_ctds, **kwargs): with open(output_file, 'w') as fh: tree.write(fh, encoding="UTF-8", xml_declaration=True, pretty_print=True) -# def create_cio(tool, model, **kwargs): -# cnode = add_child_node(tool, "command", OrderedDict([("detect_errors", "exit_code")])) -# inode = add_child_node(tool, "inputs") -# onode = add_child_node(parent, "outputs") -# -# parameter_hardcoder = kwargs["parameter_hardcoder"] -# for param in utils.extract_and_flatten_parameters(model): -# param = modify_param_for_galaxy(param) -# # no need to show hardcoded parameters -# hardcoded_value = parameter_hardcoder.get_hardcoded_value(param.name, model.name) -# -# if parameter_hardcoder.get_blacklist(param.name, model.name) or hardcoded_value: -# continue -# if not param.type is _OutFile: -# continue -# create_output_node(outputs_node, param, model, kwargs["supported_file_formats"]) -# -# return cnode, inode, onode - def write_header(tool, model): """ @@ -606,7 +606,7 @@ def create_command(tool, model, **kwargs): param_cmd['command'].append("${' '.join([\"'"+actual_parameter+"/%s.%s'\"%(re.sub('[^\w\-_]', '_', _.element_identifier),$gxy2omsext(_.ext)) for _ in $" + actual_parameter + " if _ ])}") else: param_cmd['preprocessing'].append("ln -s '$"+ actual_parameter +"' '"+actual_parameter+"/${re.sub(\"[^\w\-_]\", \"_\", $"+actual_parameter+".element_identifier)}.$gxy2omsext($"+actual_parameter+".ext)' &&") - param_cmd['command'].append("'"+actual_parameter+"/${re.sub('[^\w\-_]', '_', $"+actual_parameter+".element_identifier)}.$gxy2omsext($"+actual_parameter+".ext)'") + param_cmd['command'].append("'"+actual_parameter+"/${re.sub(\"[^\w\-_]\", \"_\", $"+actual_parameter+".element_identifier)}.$gxy2omsext($"+actual_parameter+".ext)'") elif param.type is _OutFile: # check if there is a parameter that sets the format # if so we add an extension to the generated files which will be used to @@ -642,11 +642,11 @@ def create_command(tool, model, **kwargs): if len(formats)==1: fmt = formats.pop() if param.is_list: - logger.info("1 fmt + list %s -> %s" % (actual_parameter, actual_input_parameter)) + logger.info("1 fmt + list %s -> %s" % (actual_parameter, actual_input_parameter), 1) param_cmd['command'].append("${' '.join([\"'"+ actual_parameter +"/%s.%s'\"%(re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(\""+ fmt +"\")) for _ in $" + actual_input_parameter + " if _ ])}") param_cmd['postprocessing'].append("${' '.join([\"&& mv -n '"+ actual_parameter +"/%(id)s.%(gext)s' '" +actual_parameter+ "/%(id)s'\"%{\"id\": re.sub('[^\w\-_]', '_', _.element_identifier), \"gext\": $gxy2omsext(\""+fmt+"\")} for _ in $" + actual_input_parameter + " if _ ])}") else: - logger.info("1 fmt + dataset %s" % actual_parameter) + logger.info("1 fmt + dataset %s" % actual_parameter, 1) param_cmd['command'].append("'"+actual_parameter+"/output.${gxy2omsext(\"" + fmt + "\")}'") param_cmd['postprocessing'].append("&& mv '"+actual_parameter+"/output.${gxy2omsext(\"" + fmt + "\")}' '$"+actual_parameter+"'") @@ -655,11 +655,11 @@ def create_command(tool, model, **kwargs): # - list: let the command create output files with the oms extensions, postprocessing renames them to the galaxy extensions, output is then discover + __name_and_ext__ elif type_param_name is not None: if param.is_list: - logger.info("type + list %s" %actual_parameter) + logger.info("type + list %s" %actual_parameter, 1) param_cmd['command'].append("${' '.join([\"'"+ actual_parameter +"/%s.%s'\"%(re.sub('[^\w\-_]', '_', _.element_identifier), "+type_param_name+") for _ in $" + actual_input_parameter + " if _ ])}") param_cmd['postprocessing'].append("${' '.join([\"&& mv -n '"+ actual_parameter +"/%(id)s.%(omsext)s' '" +actual_parameter+ "/%(id)s.%(gext)s'\"%{\"id\": re.sub('[^\w\-_]', '_', _.element_identifier), \"omsext\":"+type_param_name+", \"gext\": oms2gxyext(\""+type_param_name+"\")} for _ in $" + actual_input_parameter + " if _ ])}") else: - logger.info("type + dataset %s" %actual_parameter) + logger.info("type + dataset %s" %actual_parameter, 1) # 1st create file with openms extension (often required by openms) # then move it to the actual place specified by the parameter # the format is then set by the tag using @@ -667,10 +667,10 @@ def create_command(tool, model, **kwargs): param_cmd['postprocessing'].append("&& mv '"+actual_parameter+"/output.${"+type_param_name+"}' '$"+actual_parameter+"'") elif actual_input_parameter is not None: if param.is_list: - logger.info("actual + list %s" %actual_parameter) + logger.info("actual + list %s" %actual_parameter, 1) param_cmd['command'].append("${' '.join([\"'"+ actual_parameter +"/%s.%s'\"%(re.sub('[^\w\-_]', '_', _.element_identifier), _.ext) for _ in $" + actual_input_parameter + " if _ ])}") else: - logger.info("actual + dataset %s %s %s" % (actual_parameter, actual_input_parameter, corresponding_input.is_list)) + logger.info("actual + dataset %s %s %s" % (actual_parameter, actual_input_parameter, corresponding_input.is_list), 1) if corresponding_input.is_list: param_cmd['command'].append("'" + actual_parameter+"/output.${"+actual_input_parameter+"[0].ext}'") param_cmd['postprocessing'].append("&& mv '"+actual_parameter+"/output.${"+actual_input_parameter+"[0].ext}' '$"+actual_parameter+"'") @@ -681,7 +681,7 @@ def create_command(tool, model, **kwargs): if param.is_list: raise Exception( "output parameter itemlist %s without corresponding input" ) else: - logger.info("else + dataset %s" % actual_parameter) + logger.info("else + dataset %s" % actual_parameter, 1) param_cmd['command'].append("'$" +actual_parameter+ "'") # select with multiple = true @@ -910,8 +910,9 @@ def create_inputs(tool, model, **kwargs): corresponding_input, fmt_from_corresponding = get_corresponding_input(param, model) if len(formats) > 1 and type_param is None and (corresponding_input is None or not fmt_from_corresponding) and not param.is_list: fmt_select = add_child_node(parent_node, "param", OrderedDict([("name", get_galaxy_parameter_name(param, True)+"_type"), ("type", "select"), ("optional", "false"), ("label", "File type of output %s (%s)" % (param.name, param.description))])) + g2o, o2g = get_fileformat_maps(kwargs["supported_file_formats"]) for f in formats: - option_node = add_child_node(fmt_select, "option", OrderedDict([("value", f)]), f) + option_node = add_child_node(fmt_select, "option", OrderedDict([("value", g2o[f])]), f) # create an additional bool input which is used to filter for the output # mandatory outpiles: no input node needed @@ -925,10 +926,10 @@ def create_inputs(tool, model, **kwargs): param_node = add_child_node(parent_node, "param") create_param_attribute_list(param_node, param, model, kwargs["supported_file_formats"]) -# hardcoded_attributes = parameter_hardcoder.get_hardcoded_attributes(param.name, model.name) -# if hardcoded_attributes != None: -# for a in hardcoded_attributes: -# param_node.attrib[a] = str(hardcoded_attributes[a]) + hardcoded_attributes = parameter_hardcoder.get_hardcoded_attributes("param_"+param.name, model.name) + if hardcoded_attributes != None: + for a in hardcoded_attributes: + param_node.attrib[a] = str(hardcoded_attributes[a]) # if there is an advanced section then append it at the end of the inputs if not advanced_node is None: @@ -947,24 +948,22 @@ def get_formats(param, supported_file_formats, default = None): formats = set() if param.restrictions is not None: if type(param.restrictions) is _FileFormat: - # set the first data output node to the first file format - # check if there are formats that have not been registered yet... output = list() for format_name in param.restrictions.formats: if format_name not in supported_file_formats.keys(): output.append(str(format_name)) - # warn only if there's about to complain if output: logger.warning("Parameter " + param.name + " has the following unsupported format(s):" + ','.join(output), 1) - + formats = get_supported_file_types(param.restrictions.formats, supported_file_formats) else: raise InvalidModelException("Unrecognized restriction type [%(type)s] " "for [%(name)s]" % {"type": type(param.restrictions), "name": param.name}) + if len(formats) == 0: if default != None: formats.add(default) @@ -984,6 +983,8 @@ def create_param_attribute_list(param_node, param, model, supported_file_formats @param supported_file_formats """ + g2o, o2g = get_fileformat_maps(supported_file_formats) + # set the name, argument and a first guess for the type (which will be over written # in some cases .. see below) param_node.attrib["name"] = get_galaxy_parameter_name(param) @@ -1042,7 +1043,10 @@ def create_param_attribute_list(param_node, param, model, supported_file_formats # create as many diff --git a/tests/test-data/empty.ctd b/tests/test-data/empty.ctd index 3dda4edd..0876084b 100644 --- a/tests/test-data/empty.ctd +++ b/tests/test-data/empty.ctd @@ -1,5 +1,5 @@ - + diff --git a/tests/test-data/empty.xml b/tests/test-data/empty.xml index 5f4dc5bf..f700355c 100644 --- a/tests/test-data/empty.xml +++ b/tests/test-data/empty.xml @@ -26,6 +26,6 @@ +For more information, visit http://www.openms.de]]> diff --git a/tests/test-data/float.ctd b/tests/test-data/float.ctd index 1317dc19..045e123c 100644 --- a/tests/test-data/float.ctd +++ b/tests/test-data/float.ctd @@ -1,5 +1,5 @@ - + diff --git a/tests/test-data/float.xml b/tests/test-data/float.xml index 0ddf3828..7cbfa092 100644 --- a/tests/test-data/float.xml +++ b/tests/test-data/float.xml @@ -178,6 +178,6 @@ +For more information, visit http://www.openms.de]]> diff --git a/tests/test-data/ifile.ctd b/tests/test-data/ifile.ctd index 629bd694..ba5f2466 100644 --- a/tests/test-data/ifile.ctd +++ b/tests/test-data/ifile.ctd @@ -1,5 +1,5 @@ - + diff --git a/tests/test-data/ifile.xml b/tests/test-data/ifile.xml index d5caa68a..b9fdb3a5 100644 --- a/tests/test-data/ifile.xml +++ b/tests/test-data/ifile.xml @@ -432,6 +432,6 @@ ${' '.join(["'param_list_mandatory_multformat/%s.%s'"%(re.sub('[^\w\-_]', '_', _ +For more information, visit http://www.openms.de]]> diff --git a/tests/test-data/integer.ctd b/tests/test-data/integer.ctd index 486686e6..908978c0 100644 --- a/tests/test-data/integer.ctd +++ b/tests/test-data/integer.ctd @@ -1,5 +1,5 @@ - + diff --git a/tests/test-data/integer.xml b/tests/test-data/integer.xml index ae525d79..6f54f8e3 100644 --- a/tests/test-data/integer.xml +++ b/tests/test-data/integer.xml @@ -178,6 +178,6 @@ +For more information, visit http://www.openms.de]]> diff --git a/tests/test-data/ofile-corresponding-input.ctd b/tests/test-data/ofile-corresponding-input.ctd index 46fec2fb..f73ad663 100644 --- a/tests/test-data/ofile-corresponding-input.ctd +++ b/tests/test-data/ofile-corresponding-input.ctd @@ -1,5 +1,5 @@ - + diff --git a/tests/test-data/ofile-corresponding-input.xml b/tests/test-data/ofile-corresponding-input.xml index 167239d3..1ada40a5 100644 --- a/tests/test-data/ofile-corresponding-input.xml +++ b/tests/test-data/ofile-corresponding-input.xml @@ -179,6 +179,6 @@ ln -s '$param_input_multformat' 'param_input_multformat/${re.sub("[^\w\-_]", "_" +For more information, visit http://www.openms.de]]> diff --git a/tests/test-data/ofile-mult-typeparam.ctd b/tests/test-data/ofile-mult-typeparam.ctd index 983b2447..6ea3e351 100644 --- a/tests/test-data/ofile-mult-typeparam.ctd +++ b/tests/test-data/ofile-mult-typeparam.ctd @@ -1,5 +1,5 @@ - + diff --git a/tests/test-data/ofile-mult-typeparam.xml b/tests/test-data/ofile-mult-typeparam.xml index e6b06807..f7c007dc 100644 --- a/tests/test-data/ofile-mult-typeparam.xml +++ b/tests/test-data/ofile-mult-typeparam.xml @@ -72,6 +72,6 @@ ${' '.join(["&& mv -n 'param_out_mandatory_optionalinput/%(id)s.%(omsext)s' 'par +For more information, visit http://www.openms.de]]> diff --git a/tests/test-data/ofile-mult.ctd b/tests/test-data/ofile-mult.ctd index 5ea7ec6d..78dee0fb 100644 --- a/tests/test-data/ofile-mult.ctd +++ b/tests/test-data/ofile-mult.ctd @@ -1,5 +1,5 @@ - + diff --git a/tests/test-data/ofile-mult.xml b/tests/test-data/ofile-mult.xml index b7d3782d..51dd2013 100644 --- a/tests/test-data/ofile-mult.xml +++ b/tests/test-data/ofile-mult.xml @@ -98,6 +98,6 @@ ${' '.join(["&& mv -n 'param_out_mandatory_mandatoryinput/%(id)s.%(gext)s' 'para +For more information, visit http://www.openms.de]]> diff --git a/tests/test-data/ofile-typeparam.ctd b/tests/test-data/ofile-typeparam.ctd index 86afac64..f8b1f533 100644 --- a/tests/test-data/ofile-typeparam.ctd +++ b/tests/test-data/ofile-typeparam.ctd @@ -1,5 +1,5 @@ - + diff --git a/tests/test-data/ofile-typeparam.xml b/tests/test-data/ofile-typeparam.xml index cba8d1e2..1348a90f 100644 --- a/tests/test-data/ofile-typeparam.xml +++ b/tests/test-data/ofile-typeparam.xml @@ -231,6 +231,6 @@ ln -s '$param_decoy_input_multformat' 'param_decoy_input_multformat/${re.sub("[^ +For more information, visit http://www.openms.de]]> diff --git a/tests/test-data/ofile.ctd b/tests/test-data/ofile.ctd index 4b042bcf..a206e8e5 100644 --- a/tests/test-data/ofile.ctd +++ b/tests/test-data/ofile.ctd @@ -1,5 +1,5 @@ - + diff --git a/tests/test-data/ofile.xml b/tests/test-data/ofile.xml index 806793c4..8331bf51 100644 --- a/tests/test-data/ofile.xml +++ b/tests/test-data/ofile.xml @@ -181,6 +181,6 @@ mkdir param_out_mandatory_multformat && +For more information, visit http://www.openms.de]]> diff --git a/tests/test-data/repeat.ctd b/tests/test-data/repeat.ctd index b022bcfb..0806beca 100644 --- a/tests/test-data/repeat.ctd +++ b/tests/test-data/repeat.ctd @@ -1,5 +1,5 @@ - + diff --git a/tests/test-data/repeat.xml b/tests/test-data/repeat.xml index 2a56475a..956587af 100644 --- a/tests/test-data/repeat.xml +++ b/tests/test-data/repeat.xml @@ -342,6 +342,6 @@ $quote($param_double_resticted_default_mandatory) +For more information, visit http://www.openms.de]]> diff --git a/tests/test-data/select.ctd b/tests/test-data/select.ctd index 352424ff..1d634b19 100644 --- a/tests/test-data/select.ctd +++ b/tests/test-data/select.ctd @@ -1,5 +1,5 @@ - + diff --git a/tests/test-data/select.xml b/tests/test-data/select.xml index b80b1a6d..b134fcba 100644 --- a/tests/test-data/select.xml +++ b/tests/test-data/select.xml @@ -378,6 +378,6 @@ ${' '.join(['"%s"'%str(_) for _ in str($param_list_default_mandatory_many).split +For more information, visit http://www.openms.de]]> diff --git a/tests/test-data/string.ctd b/tests/test-data/string.ctd index 8829408f..cc2fe622 100644 --- a/tests/test-data/string.ctd +++ b/tests/test-data/string.ctd @@ -1,5 +1,5 @@ - + diff --git a/tests/test-data/string.xml b/tests/test-data/string.xml index bec44bc9..882e8638 100644 --- a/tests/test-data/string.xml +++ b/tests/test-data/string.xml @@ -86,6 +86,6 @@ +For more information, visit http://www.openms.de]]> From b04e191bc66f47b98ec9d3db9cabf0830b8df948 Mon Sep 17 00:00:00 2001 From: M Bernt Date: Tue, 14 Jan 2020 13:45:41 +0100 Subject: [PATCH 060/121] add test for param, label and help generation --- tests/test-data/label-help.ctd | 22 ++++++++ tests/test-data/label-help.xml | 95 ++++++++++++++++++++++++++++++++++ 2 files changed, 117 insertions(+) create mode 100644 tests/test-data/label-help.ctd create mode 100644 tests/test-data/label-help.xml diff --git a/tests/test-data/label-help.ctd b/tests/test-data/label-help.ctd new file mode 100644 index 00000000..e07d555d --- /dev/null +++ b/tests/test-data/label-help.ctd @@ -0,0 +1,22 @@ + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/test-data/label-help.xml b/tests/test-data/label-help.xml new file mode 100644 index 00000000..11781077 --- /dev/null +++ b/tests/test-data/label-help.xml @@ -0,0 +1,95 @@ + + + + + Lebel Help parameter tests + + echo LabelHelpTest + macros.xml + + + + $param_stdout]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From 4a4149c9c6e96ab31f856f2d8fdcceea2804ec07 Mon Sep 17 00:00:00 2001 From: M Bernt Date: Tue, 14 Jan 2020 14:30:06 +0100 Subject: [PATCH 061/121] simplify and improve label/help generation - add file types for infiles - add i.e. splitting for description --- galaxy/converter.py | 63 +++++++----- tests/test-data/ifile.xml | 96 +++++++++---------- tests/test-data/label-help.ctd | 17 ++-- tests/test-data/label-help.xml | 23 ++--- tests/test-data/ofile-corresponding-input.xml | 6 +- tests/test-data/ofile-mult-typeparam.xml | 4 +- tests/test-data/ofile-mult.xml | 8 +- tests/test-data/ofile-typeparam.xml | 6 +- 8 files changed, 114 insertions(+), 109 deletions(-) diff --git a/galaxy/converter.py b/galaxy/converter.py index dda4d7cf..24e46691 100755 --- a/galaxy/converter.py +++ b/galaxy/converter.py @@ -2,6 +2,7 @@ # encoding: utf-8 import os import os.path +import re import string from collections import OrderedDict @@ -1163,6 +1164,11 @@ def create_param_attribute_list(param_node, param, model, supported_file_formats label, help_text = generate_label_and_help(param.description) if param.is_list and not is_selection_parameter(param) and not param.type is _InFile: help_text += " (space separated list, in order to allow for spaces in list items surround them by single quotes)" + if param.type is _InFile: + if param.is_list: + help_text += " select %s data sets(s)" % (",".join(get_formats(param, supported_file_formats, TYPE_TO_GALAXY_TYPE[_InFile]))) + else: + help_text += " select a %s data set" % (",".join(get_formats(param, supported_file_formats, TYPE_TO_GALAXY_TYPE[_InFile]))) param_node.attrib["label"] = label param_node.attrib["help"] = help_text @@ -1185,32 +1191,39 @@ def generate_label_and_help(desc): first_word.capitalize() desc = first_word + " " + rest label = desc.decode("utf8") - - # Try to split the label if it is too long + + # split delimiters ".,?!;(" if len(desc) > 50: - # find an example and put everything before in the label and the e.g. in the help - if desc.find("e.g.") > 1 : - label, help_text = desc.split("e.g.",1) - help_text = "e.g." + help_text - else: - # find the end of the first sentence - # look for ". " because some labels contain .file or something similar - delimiter = "" - if desc.find(". ") > 1 and desc.find("? ") > 1: - if desc.find(". ") < desc.find("? "): - delimiter = ". " - else: - delimiter = "? " - elif desc.find(". ") > 1: - delimiter = ". " - elif desc.find("? ") > 1: - delimiter = "? " - if delimiter != "": - label, help_text = desc.split(delimiter, 1) - - # add the question mark back - if delimiter == "? ": - label += "? " + m = re.search(r"([.?!] |e\.g\.|\(e\.g\.|i\.e\.|\(i\.e\.)", desc) + if m is not None: + label = desc[:m.start()].rstrip(".?!, ") + help_text = desc[m.start():].lstrip(".?!, ") + +# # Try to split the label if it is too long +# if len(desc) > 50: +# # find an example and put everything before in the label and the e.g. in the help +# if desc.find("e.g.") > 1 : +# label, help_text = desc.split("e.g.",1) +# help_text = "e.g." + help_text +# else: +# # find the end of the first sentence +# # look for ". " because some labels contain .file or something similar +# delimiter = "" +# if desc.find(". ") > 1 and desc.find("? ") > 1: +# if desc.find(". ") < desc.find("? "): +# delimiter = ". " +# else: +# delimiter = "? " +# elif desc.find(". ") > 1: +# delimiter = ". " +# elif desc.find("? ") > 1: +# delimiter = "? " +# if delimiter != "": +# label, help_text = desc.split(delimiter, 1) +# +# # add the question mark back +# if delimiter == "? ": +# label += "? " # remove all linebreaks label = label.rstrip().rstrip('
').rstrip() diff --git a/tests/test-data/ifile.xml b/tests/test-data/ifile.xml index b9fdb3a5..a889143d 100644 --- a/tests/test-data/ifile.xml +++ b/tests/test-data/ifile.xml @@ -355,55 +355,55 @@ ${' '.join(["'param_list_mandatory_multformat/%s.%s'"%(re.sub('[^\w\-_]', '_', _ #end if > $param_stdout]]> - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/test-data/label-help.ctd b/tests/test-data/label-help.ctd index e07d555d..7cda0295 100644 --- a/tests/test-data/label-help.ctd +++ b/tests/test-data/label-help.ctd @@ -6,16 +6,15 @@ - - - - + + + - - - - - + + + + + diff --git a/tests/test-data/label-help.xml b/tests/test-data/label-help.xml index 11781077..4d2bd348 100644 --- a/tests/test-data/label-help.xml +++ b/tests/test-data/label-help.xml @@ -15,8 +15,6 @@ @EXECUTABLE@ -do "$param_do" --co -"$param_co" -qm "$param_qm" -ex @@ -33,39 +31,35 @@ "$param_ptie" > $param_stdout]]> - + - + - + - + - + - + - + - - - - - + @@ -76,7 +70,6 @@ - diff --git a/tests/test-data/ofile-corresponding-input.xml b/tests/test-data/ofile-corresponding-input.xml index 1ada40a5..b60229d8 100644 --- a/tests/test-data/ofile-corresponding-input.xml +++ b/tests/test-data/ofile-corresponding-input.xml @@ -119,9 +119,9 @@ ln -s '$param_input_multformat' 'param_input_multformat/${re.sub("[^\w\-_]", "_" - - - + + + diff --git a/tests/test-data/ofile-mult-typeparam.xml b/tests/test-data/ofile-mult-typeparam.xml index f7c007dc..5f2efeeb 100644 --- a/tests/test-data/ofile-mult-typeparam.xml +++ b/tests/test-data/ofile-mult-typeparam.xml @@ -49,8 +49,8 @@ ${' '.join(["&& mv -n 'param_out_mandatory_optionalinput/%(id)s.%(omsext)s' 'par - - + + diff --git a/tests/test-data/ofile-mult.xml b/tests/test-data/ofile-mult.xml index 51dd2013..a39ad165 100644 --- a/tests/test-data/ofile-mult.xml +++ b/tests/test-data/ofile-mult.xml @@ -64,10 +64,10 @@ ${' '.join(["&& mv -n 'param_out_mandatory_mandatoryinput/%(id)s.%(gext)s' 'para - - - - + + + + diff --git a/tests/test-data/ofile-typeparam.xml b/tests/test-data/ofile-typeparam.xml index 1348a90f..9e4f4c7d 100644 --- a/tests/test-data/ofile-typeparam.xml +++ b/tests/test-data/ofile-typeparam.xml @@ -130,9 +130,9 @@ ln -s '$param_decoy_input_multformat' 'param_decoy_input_multformat/${re.sub("[^ - - - + + + From 8539619b29b43c1e31d78e1e5ae6b46eefe08cdf Mon Sep 17 00:00:00 2001 From: M Bernt Date: Thu, 16 Jan 2020 18:38:53 +0100 Subject: [PATCH 062/121] add Galaxy section analoguous to CTD nodes --- common/utils.py | 82 ++++++++++++++++++++++++++++++++------------- galaxy/converter.py | 58 +++++++++++++++++++++----------- 2 files changed, 97 insertions(+), 43 deletions(-) diff --git a/common/utils.py b/common/utils.py index c33d1ac0..761cb60e 100644 --- a/common/utils.py +++ b/common/utils.py @@ -284,23 +284,56 @@ def extract_tool_executable_path(model, default_executable_path): return command -def extract_and_flatten_parameters(ctd_model): - parameters = [] - if len(ctd_model.parameters.parameters) > 0: - # use this to put parameters that are to be processed - # we know that CTDModel has one parent ParameterGroup - pending = [ctd_model.parameters] - while len(pending) > 0: - # take one element from 'pending' - parameter = pending.pop() - if type(parameter) is not ParameterGroup: - parameters.append(parameter) - else: - # append the first-level children of this ParameterGroup - pending.extend(parameter.parameters.values()) - # returned the reversed list of parameters (as it is now, - # we have the last parameter in the CTD as first in the list) - return reversed(parameters) +def _extract_and_flatten_parameters(parameter_group, nodes=False): + """ + get the parameters of a OptionGroup as generator + """ + for parameter in parameter_group.parameters.itervalues(): + if type(parameter) is not ParameterGroup: + yield parameter + else: + if nodes: + yield parameter + for p in _extract_and_flatten_parameters(parameter, nodes): + yield p + + +def extract_and_flatten_parameters(ctd_model, nodes=False): + """ + get the parameters of a CTD as generator + """ + + names = [_.name for _ in ctd_model.parameters.parameters.values()] + if names == ["version", "1"]: + return _extract_and_flatten_parameters(ctd_model.parameters.parameters["1"], nodes) + else: + return _extract_and_flatten_parameters(ctd_model.parameters, nodes) + +# for parameter in ctd_model.parameters.parameters: +# if type(parameter) is not ParameterGroup: +# yield parameter +# else: +# for p in extract_and_flatten_parameters(parameter): +# yield p + +# +# +# parameters = [] +# if len(ctd_model.parameters.parameters) > 0: +# # use this to put parameters that are to be processed +# # we know that CTDModel has one parent ParameterGroup +# pending = [ctd_model.parameters] +# while len(pending) > 0: +# # take one element from 'pending' +# parameter = pending.pop() +# if type(parameter) is not ParameterGroup: +# parameters.append(parameter) +# else: +# # append the first-level children of this ParameterGroup +# pending.extend(parameter.parameters.values()) +# # returned the reversed list of parameters (as it is now, +# # we have the last parameter in the CTD as first in the list) +# return reversed(parameters) # some parameters are mapped to command line options, this method helps resolve those mappings, if any @@ -334,19 +367,22 @@ def _extract_param_cli_name(param, ctd_model): return resolve_param_mapping(param, ctd_model) -def extract_param_name(param): - # we generate parameters with colons for subgroups, but not for the two topmost parents (OpenMS legacy) +def extract_param_path(param): if type(param.parent) == ParameterGroup: if not hasattr(param.parent.parent, "parent"): - return param.name + return [param.name] elif not hasattr(param.parent.parent.parent, "parent"): - return param.name + return [param.name] else: - return extract_param_name(param.parent) + ":" + param.name + return extract_param_path(param.parent) + [ param.name ] else: - return param.name + return [param.name] +def extract_param_name(param): + # we generate parameters with colons for subgroups, but not for the two topmost parents (OpenMS legacy) + return ":".join(extract_param_path(param)) + def extract_command_line_prefix(param, ctd_model): param_name = extract_param_name(param) param_cli_name = _extract_param_cli_name(param, ctd_model) diff --git a/galaxy/converter.py b/galaxy/converter.py index 24e46691..9399fd4e 100755 --- a/galaxy/converter.py +++ b/galaxy/converter.py @@ -14,7 +14,7 @@ from common import utils, logger from common.exceptions import ApplicationException, InvalidModelException -from CTDopts.CTDopts import _InFile, _OutFile, ParameterGroup, _Choices, _NumericRange, _FileFormat, ModelError, _Null, CTDTYPE_TO_TYPE +from CTDopts.CTDopts import _InFile, _OutFile, ParameterGroup, _Choices, _NumericRange, _FileFormat, ModelError, _Null, CTDTYPE_TO_TYPE, ParameterGroup # mapping to CTD types to Galaxy types @@ -587,10 +587,7 @@ def create_command(tool, model, **kwargs): else: # in the else branch the parameter is neither blacklisted nor hardcoded... - actual_parameter = get_galaxy_parameter_name(param) - if param.advanced and not param.type is _OutFile: - actual_parameter = ADVANCED_OPTIONS_NAME+"cond." + actual_parameter - + actual_parameter = get_galaxy_parameter_path(param) # all but bool params need the command line argument (bools have it already in the true/false value) if not is_boolean_parameter(param): param_cmd['command'].append(command_line_prefix) @@ -618,9 +615,7 @@ def create_command(tool, model, **kwargs): type_param = get_out_type_param(param, model, parameter_hardcoder) corresponding_input, fmt_from_corresponding = get_corresponding_input(param, model) if corresponding_input is not None : - actual_input_parameter = get_galaxy_parameter_name(corresponding_input) - if corresponding_input.advanced: - actual_input_parameter = ADVANCED_OPTIONS_NAME+"cond." + actual_input_parameter + actual_input_parameter = get_galaxy_parameter_path(corresponding_input) else: actual_input_parameter = None @@ -629,9 +624,7 @@ def create_command(tool, model, **kwargs): if type_param.advanced: type_param_name = ADVANCED_OPTIONS_NAME+"cond." + type_param_name elif len(formats) > 1 and (corresponding_input == None or not fmt_from_corresponding) and not param.is_list: - type_param_name = get_galaxy_parameter_name(param, True) + "_type" - if param.advanced: - type_param_name = ADVANCED_OPTIONS_NAME+"cond." + type_param_name + type_param_name = get_galaxy_parameter_path(param, True) + "_type" else: type_param_name = None @@ -704,10 +697,7 @@ def create_command(tool, model, **kwargs): if not ( param.required or is_boolean_parameter(param) or (param.type == str and param.restrictions is None)): # and not(param.type is _InFile and param.is_list): if param.type is _OutFile: - if param.advanced: - actual_parameter = ADVANCED_OPTIONS_NAME+"cond.%s" % get_galaxy_parameter_name(param, True) - else: - actual_parameter = "%s" % get_galaxy_parameter_name(param, True) + actual_parameter = get_galaxy_parameter_path(param, True) for stage in param_cmd: if len(param_cmd[stage]) == 0: @@ -777,6 +767,20 @@ def expand_macros(node, macros_to_expand): expand_node.attrib["macro"] = expand_macro +def get_galaxy_parameter_path(param, inparam = False): + """ + get the complete cheetah path for a parameter + """ + parameter = get_galaxy_parameter_name(param, inparam) + path = utils.extract_param_path(param) + if len(path) > 1: + return ".".join( [ "section_"+_ for _ in path[:-1]] ) +"."+ parameter + elif param.advanced and (not param.type is _OutFile or inparam): + return ADVANCED_OPTIONS_NAME+"cond." + parameter + else: + return parameter + + def get_galaxy_parameter_name(param, inparam = False): """ get the name of the parameter used in the galaxy tool @@ -884,20 +888,29 @@ def create_inputs(tool, model, **kwargs): @return inputs node """ inputs_node = SubElement(tool, "inputs") + section_nodes = dict() + section_params = dict() # some suites (such as OpenMS) need some advanced options when handling inputs advanced_node = None parameter_hardcoder = kwargs["parameter_hardcoder"] # treat all non output-file/advanced/blacklisted/hardcoded parameters as inputs - for param in utils.extract_and_flatten_parameters(model): + for param in utils.extract_and_flatten_parameters(model, True): + if type(param) is ParameterGroup: + section_params[utils.extract_param_name(param)] = param + section_nodes[utils.extract_param_name(param)] = Element("section", OrderedDict([("name", "section_"+param.name), ("description", param.description), ("expanded", "false")])) + continue + param = modify_param_for_galaxy(param) # no need to show hardcoded parameters hardcoded_value = parameter_hardcoder.get_hardcoded_value(utils.extract_param_name(param), model.name) if parameter_hardcoder.get_blacklist(utils.extract_param_name(param), model.name) or not hardcoded_value is None: continue - if param.advanced: + if section_nodes.has_key(utils.extract_param_name(param.parent)): + parent_node = section_nodes[utils.extract_param_name(param.parent)] + elif param.advanced: if advanced_node is None: advanced_node = Element("expand", OrderedDict([("macro", ADVANCED_OPTIONS_NAME+"macro")])) parent_node = advanced_node @@ -935,6 +948,11 @@ def create_inputs(tool, model, **kwargs): for a in hardcoded_attributes: param_node.attrib[a] = str(hardcoded_attributes[a]) + for sn in section_nodes: + if utils.extract_param_name(section_params[sn].parent) in section_nodes: + section_nodes[ utils.extract_param_name(section_params[sn].parent) ].append(section_nodes[sn]) + else: + inputs_node.append(section_nodes[sn]) # if there is an advanced section then append it at the end of the inputs if not advanced_node is None: inputs_node.append(advanced_node) @@ -1342,7 +1360,7 @@ def create_output_node(parent, param, model, supported_file_formats, parameter_h data_node = add_child_node(parent, "collection") data_node.attrib["type"] = "list" discover_node = add_child_node(data_node, "discover_datasets", - OrderedDict([("directory", get_galaxy_parameter_name(param))])) + OrderedDict([("directory", get_galaxy_parameter_path(param))])) data_node.attrib["name"] = get_galaxy_parameter_name(param) if data_node.attrib["name"].startswith('param_out_'): @@ -1377,8 +1395,8 @@ def create_output_node(parent, param, model, supported_file_formats, parameter_h # data_node.attrib["structured_like"] = get_galaxy_parameter_name(corresponding_input) #data_node.attrib["inherit_format"] = "true" else: - data_node.attrib["format_source"] = get_galaxy_parameter_name(corresponding_input) - data_node.attrib["metadata_source"] = get_galaxy_parameter_name(corresponding_input) + data_node.attrib["format_source"] = get_galaxy_parameter_path(corresponding_input) + data_node.attrib["metadata_source"] = get_galaxy_parameter_path(corresponding_input) else: if not param.is_list: data_node.attrib["auto_format"] = "true" From 0882c5b662f94332175f9c2c3d68e95815372ab7 Mon Sep 17 00:00:00 2001 From: M Bernt Date: Fri, 17 Jan 2020 09:32:04 +0100 Subject: [PATCH 063/121] sections: description -> title --- galaxy/converter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/galaxy/converter.py b/galaxy/converter.py index 9399fd4e..de36e04e 100755 --- a/galaxy/converter.py +++ b/galaxy/converter.py @@ -899,7 +899,7 @@ def create_inputs(tool, model, **kwargs): for param in utils.extract_and_flatten_parameters(model, True): if type(param) is ParameterGroup: section_params[utils.extract_param_name(param)] = param - section_nodes[utils.extract_param_name(param)] = Element("section", OrderedDict([("name", "section_"+param.name), ("description", param.description), ("expanded", "false")])) + section_nodes[utils.extract_param_name(param)] = Element("section", OrderedDict([("name", "section_"+param.name), ("title", param.description), ("expanded", "false")])) continue param = modify_param_for_galaxy(param) From 30f70622607b5dca17286ab3cd036245488efa68 Mon Sep 17 00:00:00 2001 From: M Bernt Date: Fri, 17 Jan 2020 09:42:13 +0100 Subject: [PATCH 064/121] sections: title and help --- galaxy/converter.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/galaxy/converter.py b/galaxy/converter.py index de36e04e..6ef3913c 100755 --- a/galaxy/converter.py +++ b/galaxy/converter.py @@ -898,8 +898,9 @@ def create_inputs(tool, model, **kwargs): # treat all non output-file/advanced/blacklisted/hardcoded parameters as inputs for param in utils.extract_and_flatten_parameters(model, True): if type(param) is ParameterGroup: + title, help_text = generate_label_and_help(param.description) section_params[utils.extract_param_name(param)] = param - section_nodes[utils.extract_param_name(param)] = Element("section", OrderedDict([("name", "section_"+param.name), ("title", param.description), ("expanded", "false")])) + section_nodes[utils.extract_param_name(param)] = Element("section", OrderedDict([("name", "section_"+param.name), ("title", title), ("help", help_text), ("expanded", "false")])) continue param = modify_param_for_galaxy(param) From 83c09cbb19c8ff469b3d389c0c5176b710b99488 Mon Sep 17 00:00:00 2001 From: M Bernt Date: Sat, 18 Jan 2020 15:11:46 +0100 Subject: [PATCH 065/121] prototype for CTD based input - let the Galaxy tool write a inputs configfile (json) - let the binary write ctd with defaults - use helper python script to fill actual values in CTD - call the tool with the CTD --- galaxy/converter.py | 163 +++++++++++++++++++++++++------------------- 1 file changed, 93 insertions(+), 70 deletions(-) diff --git a/galaxy/converter.py b/galaxy/converter.py index 6ef3913c..1aa9c2ca 100755 --- a/galaxy/converter.py +++ b/galaxy/converter.py @@ -8,6 +8,7 @@ from collections import OrderedDict import copy from string import strip +from StringIO import StringIO from lxml import etree from lxml.etree import CDATA, SubElement, Element, ElementTree, ParseError, parse, strip_elements @@ -441,6 +442,7 @@ def _convert_internal(parsed_ctds, **kwargs): expand_macros(tool, kwargs["macros_to_expand"]) # command, inputs, outputs = create_cio(tool, model, **kwargs) create_command(tool, model, **kwargs) + create_configfile(tool, model, **kwargs) inputs = create_inputs(tool, model, **kwargs) outputs = create_outputs(tool, model, **kwargs) if kwargs["test_test"]: @@ -546,6 +548,21 @@ def create_description(tool, model): description.text = model.opt_attribs["description"] +def get_by_path(dataDict, mapList): + for k in mapList: dataDict = dataDict[k] + return dataDict + +def set_by_path(dic, keys, value): + for key in keys[:-1]: + dic = dic.setdefault(key, {}) + dic[keys[-1]] = value + +def create_configfile(tool, model, **kwargs): + + configfiles_node = add_child_node(tool, "configfiles") + inputs_node = add_child_node(configfiles_node, "inputs", OrderedDict([("name", "args_json"), ("data_style", "paths")])) + + def create_command(tool, model, **kwargs): """ @param tool the Galaxy tool @@ -557,10 +574,15 @@ def create_command(tool, model, **kwargs): final_cmd = OrderedDict([('preprocessing', []), ('command', []), ('postprocessing', [])]) advanced_cmd = {'preprocessing': [], 'command': [], 'postprocessing': []} - final_cmd['preprocessing'].extend(["@QUOTE_FOO@", "@EXT_FOO@", "#import re"]) - final_cmd['command'].append("@EXECUTABLE@") + final_cmd['preprocessing'].extend(["@QUOTE_FOO@", "@EXT_FOO@", "#import re", "", "## Preprocessing"]) + final_cmd['command'].extend(["", "## Main program call"]) + final_cmd['command'].append("""export PYTHONPATH='$__tool_directory__/CTDopts' && +@EXECUTABLE@ -write_ctd ./ && +python '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' && +@EXECUTABLE@ -ini @EXECUTABLE@.ctd""") final_cmd['command'].extend(kwargs["add_to_command_line"]) - + final_cmd['postprocessing'].extend(["", "## Postprocessing"]) + advanced_command_start = "#if ${aon}cond.{aon}selector=='advanced':".format(aon=ADVANCED_OPTIONS_NAME) advanced_command_end = "#end if" @@ -583,13 +605,14 @@ def create_command(tool, model, **kwargs): hardcoded_value = parameter_hardcoder.get_hardcoded_value(param_name, model.name) if hardcoded_value is not None: - param_cmd['command'].append("%s %s" % (command_line_prefix, hardcoded_value)) + pass # TODO hardcoded values should go to + # param_cmd['command'].append("%s %s" % (command_line_prefix, hardcoded_value)) else: # in the else branch the parameter is neither blacklisted nor hardcoded... actual_parameter = get_galaxy_parameter_path(param) # all but bool params need the command line argument (bools have it already in the true/false value) - if not is_boolean_parameter(param): + if param.type is _OutFile or param.type is _InFile: param_cmd['command'].append(command_line_prefix) # preprocessing for file inputs: @@ -598,13 +621,13 @@ def create_command(tool, model, **kwargs): # rationale: in the autogenerated tests the same file was used as input to multiple parameters # this leads to conflicts while linking... might also be better in general if param.type is _InFile: - param_cmd['preprocessing'].append("mkdir %s &&" %actual_parameter) + param_cmd['preprocessing'].append("mkdir %s &&" %param.name) if param.is_list: - param_cmd['preprocessing'].append("${ ' '.join([ \"ln -s '%s' '"+actual_parameter+"/%s.%s' &&\" % (_, re.sub('[^\w\-_]', '_', _.element_identifier),$gxy2omsext(_.ext)) for _ in $" + actual_parameter + " if _ ]) }") - param_cmd['command'].append("${' '.join([\"'"+actual_parameter+"/%s.%s'\"%(re.sub('[^\w\-_]', '_', _.element_identifier),$gxy2omsext(_.ext)) for _ in $" + actual_parameter + " if _ ])}") + param_cmd['preprocessing'].append("${ ' '.join([ \"ln -s '%s' '"+param.name+"/%s.%s' &&\" % (_, re.sub('[^\w\-_]', '_', _.element_identifier),$gxy2omsext(_.ext)) for _ in $" + param.name + " if _ ]) }") + param_cmd['command'].append("${' '.join([\"'"+param.name+"/%s.%s'\"%(re.sub('[^\w\-_]', '_', _.element_identifier),$gxy2omsext(_.ext)) for _ in $" + param.name + " if _ ])}") else: - param_cmd['preprocessing'].append("ln -s '$"+ actual_parameter +"' '"+actual_parameter+"/${re.sub(\"[^\w\-_]\", \"_\", $"+actual_parameter+".element_identifier)}.$gxy2omsext($"+actual_parameter+".ext)' &&") - param_cmd['command'].append("'"+actual_parameter+"/${re.sub(\"[^\w\-_]\", \"_\", $"+actual_parameter+".element_identifier)}.$gxy2omsext($"+actual_parameter+".ext)'") + param_cmd['preprocessing'].append("ln -s '$"+ param.name +"' '"+param.name+"/${re.sub(\"[^\w\-_]\", \"_\", $"+param.name+".element_identifier)}.$gxy2omsext($"+param.name+".ext)' &&") + param_cmd['command'].append("'"+param.name+"/${re.sub(\"[^\w\-_]\", \"_\", $"+param.name+".element_identifier)}.$gxy2omsext($"+param.name+".ext)'") elif param.type is _OutFile: # check if there is a parameter that sets the format # if so we add an extension to the generated files which will be used to @@ -628,7 +651,7 @@ def create_command(tool, model, **kwargs): else: type_param_name = None - param_cmd['preprocessing'].append("mkdir " + actual_parameter + " &&") + param_cmd['preprocessing'].append("mkdir " + param.name + " &&") # if there is only one format (the outoput node sets format using the format attribute of the data/discover node) # - single file: write to temp file with oms extension and move this to the actual result file @@ -636,58 +659,58 @@ def create_command(tool, model, **kwargs): if len(formats)==1: fmt = formats.pop() if param.is_list: - logger.info("1 fmt + list %s -> %s" % (actual_parameter, actual_input_parameter), 1) - param_cmd['command'].append("${' '.join([\"'"+ actual_parameter +"/%s.%s'\"%(re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(\""+ fmt +"\")) for _ in $" + actual_input_parameter + " if _ ])}") - param_cmd['postprocessing'].append("${' '.join([\"&& mv -n '"+ actual_parameter +"/%(id)s.%(gext)s' '" +actual_parameter+ "/%(id)s'\"%{\"id\": re.sub('[^\w\-_]', '_', _.element_identifier), \"gext\": $gxy2omsext(\""+fmt+"\")} for _ in $" + actual_input_parameter + " if _ ])}") + logger.info("1 fmt + list %s -> %s" % (param.name, actual_input_parameter), 1) + param_cmd['command'].append("${' '.join([\"'"+ param.name +"/%s.%s'\"%(re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(\""+ fmt +"\")) for _ in $" + actual_input_parameter + " if _ ])}") + param_cmd['postprocessing'].append("${' '.join([\"&& mv -n '"+ param.name +"/%(id)s.%(gext)s' '" +param.name+ "/%(id)s'\"%{\"id\": re.sub('[^\w\-_]', '_', _.element_identifier), \"gext\": $gxy2omsext(\""+fmt+"\")} for _ in $" + actual_input_parameter + " if _ ])}") else: - logger.info("1 fmt + dataset %s" % actual_parameter, 1) - param_cmd['command'].append("'"+actual_parameter+"/output.${gxy2omsext(\"" + fmt + "\")}'") - param_cmd['postprocessing'].append("&& mv '"+actual_parameter+"/output.${gxy2omsext(\"" + fmt + "\")}' '$"+actual_parameter+"'") + logger.info("1 fmt + dataset %s" % param.name, 1) + param_cmd['command'].append("'"+param.name+"/output.${gxy2omsext(\"" + fmt + "\")}'") + param_cmd['postprocessing'].append("&& mv '"+param.name+"/output.${gxy2omsext(\"" + fmt + "\")}' '$out_"+param.name+"'") # if there is a type parameter then we use the type selected by the user # - single: write to temp file with the oms extension and mv it to the actual file output which is treated via change_format # - list: let the command create output files with the oms extensions, postprocessing renames them to the galaxy extensions, output is then discover + __name_and_ext__ elif type_param_name is not None: if param.is_list: - logger.info("type + list %s" %actual_parameter, 1) - param_cmd['command'].append("${' '.join([\"'"+ actual_parameter +"/%s.%s'\"%(re.sub('[^\w\-_]', '_', _.element_identifier), "+type_param_name+") for _ in $" + actual_input_parameter + " if _ ])}") - param_cmd['postprocessing'].append("${' '.join([\"&& mv -n '"+ actual_parameter +"/%(id)s.%(omsext)s' '" +actual_parameter+ "/%(id)s.%(gext)s'\"%{\"id\": re.sub('[^\w\-_]', '_', _.element_identifier), \"omsext\":"+type_param_name+", \"gext\": oms2gxyext(\""+type_param_name+"\")} for _ in $" + actual_input_parameter + " if _ ])}") + logger.info("type + list %s" %param.name, 1) + param_cmd['command'].append("${' '.join([\"'"+ param.name +"/%s.%s'\"%(re.sub('[^\w\-_]', '_', _.element_identifier), "+type_param_name+") for _ in $" + actual_input_parameter + " if _ ])}") + param_cmd['postprocessing'].append("${' '.join([\"&& mv -n '"+ param.name +"/%(id)s.%(omsext)s' '" +param.name+ "/%(id)s.%(gext)s'\"%{\"id\": re.sub('[^\w\-_]', '_', _.element_identifier), \"omsext\":"+type_param_name+", \"gext\": oms2gxyext(\""+type_param_name+"\")} for _ in $" + actual_input_parameter + " if _ ])}") else: - logger.info("type + dataset %s" %actual_parameter, 1) + logger.info("type + dataset %s" %param.name, 1) # 1st create file with openms extension (often required by openms) # then move it to the actual place specified by the parameter # the format is then set by the tag using - param_cmd['command'].append("'" + actual_parameter+"/output.${"+type_param_name+"}'") - param_cmd['postprocessing'].append("&& mv '"+actual_parameter+"/output.${"+type_param_name+"}' '$"+actual_parameter+"'") + param_cmd['command'].append("'" + param.name+"/output.${"+type_param_name+"}'") + param_cmd['postprocessing'].append("&& mv '"+param.name+"/output.${"+type_param_name+"}' '$out_"+param.name+"'") elif actual_input_parameter is not None: if param.is_list: - logger.info("actual + list %s" %actual_parameter, 1) - param_cmd['command'].append("${' '.join([\"'"+ actual_parameter +"/%s.%s'\"%(re.sub('[^\w\-_]', '_', _.element_identifier), _.ext) for _ in $" + actual_input_parameter + " if _ ])}") + logger.info("actual + list %s" %param.name, 1) + param_cmd['command'].append("${' '.join([\"'"+ param.name +"/%s.%s'\"%(re.sub('[^\w\-_]', '_', _.element_identifier), _.ext) for _ in $" + actual_input_parameter + " if _ ])}") else: - logger.info("actual + dataset %s %s %s" % (actual_parameter, actual_input_parameter, corresponding_input.is_list), 1) + logger.info("actual + dataset %s %s %s" % (param.name, actual_input_parameter, corresponding_input.is_list), 1) if corresponding_input.is_list: - param_cmd['command'].append("'" + actual_parameter+"/output.${"+actual_input_parameter+"[0].ext}'") - param_cmd['postprocessing'].append("&& mv '"+actual_parameter+"/output.${"+actual_input_parameter+"[0].ext}' '$"+actual_parameter+"'") + param_cmd['command'].append("'" + param.name+"/output.${"+actual_input_parameter+"[0].ext}'") + param_cmd['postprocessing'].append("&& mv '"+param.name+"/output.${"+actual_input_parameter+"[0].ext}' '$out_"+param.name+"'") else: - param_cmd['command'].append("'" + actual_parameter+"/output.${"+actual_input_parameter+".ext}'") - param_cmd['postprocessing'].append("&& mv '"+actual_parameter+"/output.${"+actual_input_parameter+".ext}' '$"+actual_parameter+"'") + param_cmd['command'].append("'" + param.name+"/output.${"+actual_input_parameter+".ext}'") + param_cmd['postprocessing'].append("&& mv '"+param.name+"/output.${"+actual_input_parameter+".ext}' '$out_"+param.name+"'") else: if param.is_list: raise Exception( "output parameter itemlist %s without corresponding input" ) else: - logger.info("else + dataset %s" % actual_parameter, 1) - param_cmd['command'].append("'$" +actual_parameter+ "'") - - # select with multiple = true - elif is_selection_parameter(param) and param.is_list: - param_cmd['command'].append("${' '.join(['\"%s\"'%str(_) for _ in str($" + actual_parameter + ").split(',')])}") - elif param.is_list: - param_cmd['command'].append("$quote($%s" % actual_parameter + ")") - #command += "${' '.join([\"'%s'\"%str(_) for _ in $" + actual_parameter + "])}\n" - elif is_boolean_parameter(param): - param_cmd['command'].append("$%s" % actual_parameter + "") - else: - param_cmd['command'].append('"$' + actual_parameter + '"') + logger.info("else + dataset %s" % param.name, 1) + param_cmd['command'].append("'$out_" +param.name+ "'") + +# # select with multiple = true +# elif is_selection_parameter(param) and param.is_list: +# param_cmd['command'].append("${' '.join(['\"%s\"'%str(_) for _ in str($" + actual_parameter + ").split(',')])}") +# elif param.is_list: +# param_cmd['command'].append("$quote($%s" % actual_parameter + ")") +# #command += "${' '.join([\"'%s'\"%str(_) for _ in $" + actual_parameter + "])}\n" +# elif is_boolean_parameter(param): +# param_cmd['command'].append("$%s" % actual_parameter + "") +# else: +# param_cmd['command'].append('"$' + actual_parameter + '"') # add if statement for optional parameters and preprocessing # - for optional outputs (param_out_x) the presence of the parameter @@ -704,11 +727,11 @@ def create_command(tool, model, **kwargs): continue # special case for optional itemlists: for those if no option is selected only the parameter must be specified if is_selection_parameter(param) and param.is_list and param.required == False: - param_cmd[stage] = [param_cmd[stage][0]] + ["#if $" + actual_parameter + ":"] + utils.indent(param_cmd[stage][1:]) + ["#end if"] + param_cmd[stage] = [param_cmd[stage][0]] + ["#if $" + param.name + ":"] + utils.indent(param_cmd[stage][1:]) + ["#end if"] elif is_selection_parameter(param) or param.type is _OutFile or param.type is _InFile: - param_cmd[stage] = ["#if $" + actual_parameter + ":"] + utils.indent(param_cmd[stage]) + ["#end if"] + param_cmd[stage] = ["#if $" + param.name + ":"] + utils.indent(param_cmd[stage]) + ["#end if"] else: - param_cmd[stage] = ["#if str($" + actual_parameter + "):"] + utils.indent(param_cmd[stage]) + ["#end if"] + param_cmd[stage] = ["#if str($" + param.name + "):"] + utils.indent(param_cmd[stage]) + ["#end if"] for stage in param_cmd: if len(param_cmd[stage]) == 0: @@ -730,8 +753,8 @@ def create_command(tool, model, **kwargs): command_node = add_child_node(tool, "command") command_node.attrib["detect_errors"] = "exit_code" - command_node.text = CDATA("\n".join(sum(final_cmd.values(), []))) + command_node.text = CDATA("\n".join(sum(final_cmd.values(), []))) # creates the xml elements needed to import the needed macros files def import_macros(tool, model, **kwargs): @@ -773,8 +796,8 @@ def get_galaxy_parameter_path(param, inparam = False): """ parameter = get_galaxy_parameter_name(param, inparam) path = utils.extract_param_path(param) - if len(path) > 1: - return ".".join( [ "section_"+_ for _ in path[:-1]] ) +"."+ parameter + if len(path) > 1 and (not param.type is _OutFile or inparam): + return ".".join( [ "section_"+_ for _ in path[:-1]] ).replace("-", "_") +"."+ parameter elif param.advanced and (not param.type is _OutFile or inparam): return ADVANCED_OPTIONS_NAME+"cond." + parameter else: @@ -794,6 +817,9 @@ def get_galaxy_parameter_name(param, inparam = False): @param inparam get the name of the corresponding input @return the name used for the parameter in the tool form """ + if type(param) is ParameterGroup: + return param.name.replace("-", "_") + p = utils.extract_param_name(param).replace(":", "_").replace("-", "_") if param.type is _OutFile and not inparam: return "param_out_%s" % p @@ -900,7 +926,7 @@ def create_inputs(tool, model, **kwargs): if type(param) is ParameterGroup: title, help_text = generate_label_and_help(param.description) section_params[utils.extract_param_name(param)] = param - section_nodes[utils.extract_param_name(param)] = Element("section", OrderedDict([("name", "section_"+param.name), ("title", title), ("help", help_text), ("expanded", "false")])) + section_nodes[utils.extract_param_name(param)] = Element("section", OrderedDict([("name", param.name), ("title", title), ("help", help_text), ("expanded", "false")])) continue param = modify_param_for_galaxy(param) @@ -927,7 +953,7 @@ def create_inputs(tool, model, **kwargs): type_param = get_out_type_param(param, model, parameter_hardcoder) corresponding_input, fmt_from_corresponding = get_corresponding_input(param, model) if len(formats) > 1 and type_param is None and (corresponding_input is None or not fmt_from_corresponding) and not param.is_list: - fmt_select = add_child_node(parent_node, "param", OrderedDict([("name", get_galaxy_parameter_name(param, True)+"_type"), ("type", "select"), ("optional", "false"), ("label", "File type of output %s (%s)" % (param.name, param.description))])) + fmt_select = add_child_node(parent_node, "param", OrderedDict([("name", param.name + "_type"), ("type", "select"), ("optional", "false"), ("label", "File type of output %s (%s)" % (param.name, param.description))])) g2o, o2g = get_fileformat_maps(kwargs["supported_file_formats"]) for f in formats: option_node = add_child_node(fmt_select, "option", OrderedDict([("value", g2o[f])]), f) @@ -936,15 +962,16 @@ def create_inputs(tool, model, **kwargs): # mandatory outpiles: no input node needed # inputs: create the input param if not param.required: - add_child_node(parent_node, "param", OrderedDict([("type", "boolean"), ("name", get_galaxy_parameter_name(param, True)), ("optional", "false"), ("checked", "false"), ("label", "Generate output %s (%s)" %(param.name, param.description))])) - + add_child_node(parent_node, "param", OrderedDict([("type", "boolean"), ("name", param.name), ("optional", "false"), ("checked", "false"), ("label", "Generate output %s (%s)" %(param.name, param.description))])) + else: + add_child_node(parent_node, "param", OrderedDict([("type", "hidden"), ("name", param.name)])) continue # create the actual param node and fill the attributes param_node = add_child_node(parent_node, "param") create_param_attribute_list(param_node, param, model, kwargs["supported_file_formats"]) - hardcoded_attributes = parameter_hardcoder.get_hardcoded_attributes("param_"+param.name, model.name) + hardcoded_attributes = parameter_hardcoder.get_hardcoded_attributes(param.name, model.name) if hardcoded_attributes != None: for a in hardcoded_attributes: param_node.attrib[a] = str(hardcoded_attributes[a]) @@ -1016,7 +1043,7 @@ def create_param_attribute_list(param_node, param, model, supported_file_formats # set the name, argument and a first guess for the type (which will be over written # in some cases .. see below) - param_node.attrib["name"] = get_galaxy_parameter_name(param) + param_node.attrib["name"] = param.name param_node.attrib["argument"] = "-%s"%utils.extract_param_name(param) param_type = TYPE_TO_GALAXY_TYPE[param.type] if param_type is None: @@ -1141,8 +1168,8 @@ def create_param_attribute_list(param_node, param, model, supported_file_formats for i, d in enumerate(param.default): if " " in d: param.default[i] = '"%s"' %d - elif " " in param.default: - param.default = '"%s"' %param.default +# elif " " in param.default: +# param.default = '"%s"' %param.default # add sanitizer nodes to # - text (only those that are not actually integer selects which are treated above) and # - select params, @@ -1295,15 +1322,12 @@ def create_boolean_parameter(param_node, param): # A special case are restrictions false,true which are not treated as flags if param.type == str: choices = get_lowercase_list(param.restrictions.choices) - if choices == ["false", "true"]: - param_node.attrib["truevalue"] = "-%s true" % utils.extract_param_name(param) - param_node.attrib["falsevalue"] = "-%s false" % utils.extract_param_name(param) - elif set(choices) == set(["true","false"]) and param.default == "true": - param_node.attrib["truevalue"] = "" - param_node.attrib["falsevalue"] = "-%s false" % utils.extract_param_name(param) + if set(choices) == set(["true","false"]): + param_node.attrib["truevalue"] = "true" + param_node.attrib["falsevalue"] = "false" else: - param_node.attrib["truevalue"] = "-%s" % utils.extract_param_name(param) - param_node.attrib["falsevalue"] = "" + param_node.attrib["truevalue"] = choices[0] + param_node.attrib["falsevalue"] = choices[1] # set the checked attribute if param.default is not None: @@ -1313,8 +1337,8 @@ def create_boolean_parameter(param_node, param): checked_value = "true" param_node.attrib["checked"] = checked_value else: - param_node.attrib["truevalue"] = "-%s" % utils.extract_param_name(param) - param_node.attrib["falsevalue"] = "" + param_node.attrib["truevalue"] = "true" + param_node.attrib["falsevalue"] = "false" param_node.attrib["checked"] = str(param.default).lower() @@ -1363,9 +1387,8 @@ def create_output_node(parent, param, model, supported_file_formats, parameter_h discover_node = add_child_node(data_node, "discover_datasets", OrderedDict([("directory", get_galaxy_parameter_path(param))])) - data_node.attrib["name"] = get_galaxy_parameter_name(param) - if data_node.attrib["name"].startswith('param_out_'): - data_node.attrib["label"] = "${tool.name} on ${on_string}: %s" % data_node.attrib["name"][10:] + data_node.attrib["name"] = "out_"+param.name + data_node.attrib["label"] = "${tool.name} on ${on_string}: %s" % param.name formats = get_formats(param, supported_file_formats, TYPE_TO_GALAXY_TYPE[_OutFile]) type_param = get_out_type_param(param, model, parameter_hardcoder) From 260768aa4d34dfc4397768f0d62b5e4edc5437f2 Mon Sep 17 00:00:00 2001 From: M Bernt Date: Thu, 23 Jan 2020 18:53:15 +0100 Subject: [PATCH 066/121] also parse params + test only generation - add ability to parse params files (in addition to ctd files) - add functionality to extract a test case (--test-only) from a ctd/params file - bug fixes in ctd based tool generation --- README.md | 14 +- common/utils.py | 46 +- convert.py | 6 +- galaxy/converter.py | 158 ++-- galaxy/macros.xml | 7 +- tests/test-data/bool | 1 + tests/test-data/bool.ctd | 2 +- tests/test-data/bool.xml | 63 +- tests/test-data/empty | 1 + tests/test-data/empty.ctd | 8 +- tests/test-data/empty.xml | 25 +- tests/test-data/fill_ctd.py | 16 + tests/test-data/float | 1 + tests/test-data/float.ctd | 34 +- tests/test-data/float.xml | 191 +---- tests/test-data/ifile | 1 + tests/test-data/ifile.ctd | 2 +- tests/test-data/ifile.xml | 705 ++++++++++-------- tests/test-data/integer | 1 + tests/test-data/integer.ctd | 34 +- tests/test-data/integer.xml | 191 +---- tests/test-data/label-help | 1 + tests/test-data/label-help.ctd | 2 +- tests/test-data/label-help.xml | 69 +- tests/test-data/mock.py | 69 +- tests/test-data/ofile | 1 + tests/test-data/ofile-corresponding-input | 1 + tests/test-data/ofile-corresponding-input.ctd | 2 +- tests/test-data/ofile-corresponding-input.xml | 252 ++++--- tests/test-data/ofile-mult | 1 + tests/test-data/ofile-mult-typeparam | 1 + tests/test-data/ofile-mult-typeparam.ctd | 2 +- tests/test-data/ofile-mult-typeparam.xml | 82 +- tests/test-data/ofile-mult.ctd | 2 +- tests/test-data/ofile-mult.xml | 135 ++-- tests/test-data/ofile-typeparam | 1 + tests/test-data/ofile-typeparam.ctd | 2 +- tests/test-data/ofile-typeparam.xml | 256 +++---- tests/test-data/ofile.ctd | 2 +- tests/test-data/ofile.xml | 215 +++--- tests/test-data/repeat | 1 + tests/test-data/repeat.ctd | 2 +- tests/test-data/repeat.xml | 255 ++----- tests/test-data/select | 1 + tests/test-data/select.ctd | 2 +- tests/test-data/select.xml | 199 ++--- tests/test-data/string | 1 + tests/test-data/string.ctd | 2 +- tests/test-data/string.xml | 63 +- 49 files changed, 1486 insertions(+), 1643 deletions(-) create mode 120000 tests/test-data/bool create mode 120000 tests/test-data/empty create mode 100644 tests/test-data/fill_ctd.py create mode 120000 tests/test-data/float create mode 120000 tests/test-data/ifile create mode 120000 tests/test-data/integer create mode 120000 tests/test-data/label-help create mode 120000 tests/test-data/ofile create mode 120000 tests/test-data/ofile-corresponding-input create mode 120000 tests/test-data/ofile-mult create mode 120000 tests/test-data/ofile-mult-typeparam create mode 120000 tests/test-data/ofile-typeparam create mode 120000 tests/test-data/repeat create mode 120000 tests/test-data/select create mode 120000 tests/test-data/string diff --git a/README.md b/README.md index d1852944..97688d4c 100644 --- a/README.md +++ b/README.md @@ -178,7 +178,19 @@ CTDs can contain an `` element that will be used when executing The following invocation of the converter will use `/opt/suite/bin` as a prefix when providing the executable path in the output files for any input CTD that lacks the `` section: $ python convert.py [FORMAT] -x /opt/suite/bin ... - + +### Tests + +Tests for Galaxy tools are generated with: + +``` +for i in tests/test-data/*ctd +do +b=$(basename $i .ctd) +python2 convert.py galaxy -i tests/test-data/$b.ctd -o tests/test-data/$b.xml -m tests/test-data/macros.xml -f tests/test-data/filetypes.txt --test-test -p tests/test-data/hardcoded_params.json --tool-version 5.0.011 -x $(pwd)/tests/test-data/ +done +``` + [CTDopts]: https://github.com/genericworkflownodes/CTDopts [CTDSchema]: https://github.com/WorkflowConversion/CTDSchema diff --git a/common/utils.py b/common/utils.py index 761cb60e..16bbef70 100644 --- a/common/utils.py +++ b/common/utils.py @@ -9,7 +9,7 @@ from common import logger from common.exceptions import ApplicationException -from CTDopts.CTDopts import _InFile, _OutFile, CTDModel, ParameterGroup +from CTDopts.CTDopts import _InFile, _OutFile, CTDModel, ParameterGroup, Parameters, Parameter, ModelTypeError MESSAGE_INDENTATION_INCREMENT = 2 @@ -150,7 +150,20 @@ def parse_input_ctds(xsd_location, input_ctds, output_destination, output_file_e if is_converting_multiple_ctds: output_file = os.path.join(output_file, get_filename_without_suffix(input_ctd) + "." + output_file_extension) info("Parsing %s" % input_ctd) - parsed_ctds.append(ParsedCTD(CTDModel(from_file=input_ctd), input_ctd, output_file)) + + model = None + try: + model = CTDModel(from_file=input_ctd) + except ModelTypeError: + pass + try: + model = Parameters(from_file=input_ctd) + except ModelTypeError: + pass + assert model != None, "Could not parse %s, seems to be no CTD/PARAMS"%(input_ctd) + + + parsed_ctds.append(ParsedCTD(model, input_ctd, output_file)) return parsed_ctds @@ -288,13 +301,13 @@ def _extract_and_flatten_parameters(parameter_group, nodes=False): """ get the parameters of a OptionGroup as generator """ - for parameter in parameter_group.parameters.itervalues(): - if type(parameter) is not ParameterGroup: + for parameter in parameter_group.itervalues(): + if type(parameter) is Parameter: yield parameter else: if nodes: yield parameter - for p in _extract_and_flatten_parameters(parameter, nodes): + for p in _extract_and_flatten_parameters(parameter.parameters, nodes): yield p @@ -302,13 +315,17 @@ def extract_and_flatten_parameters(ctd_model, nodes=False): """ get the parameters of a CTD as generator """ - - names = [_.name for _ in ctd_model.parameters.parameters.values()] - if names == ["version", "1"]: - return _extract_and_flatten_parameters(ctd_model.parameters.parameters["1"], nodes) + if type(ctd_model) is CTDModel: + return _extract_and_flatten_parameters(ctd_model.parameters.parameters, nodes) else: return _extract_and_flatten_parameters(ctd_model.parameters, nodes) +# names = [_.name for _ in ctd_model.parameters.values()] +# if names == ["version", "1"]: +# return _extract_and_flatten_parameters(ctd_model.parameters.parameters["1"], nodes) +# else: +# return _extract_and_flatten_parameters(ctd_model, nodes) + # for parameter in ctd_model.parameters.parameters: # if type(parameter) is not ParameterGroup: # yield parameter @@ -340,7 +357,11 @@ def extract_and_flatten_parameters(ctd_model, nodes=False): def resolve_param_mapping(param, ctd_model): # go through all mappings and find if the given param appears as a reference name in a mapping element param_mapping = None - for cli_element in ctd_model.cli: + ctd_model_cli = [] + if hasattr(ctd_model, "cli"): + ctd_model_cli = ctd_model.cli + + for cli_element in ctd_model_cli: for mapping_element in cli_element.mappings: if mapping_element.reference_name == param.name: if param_mapping is not None: @@ -360,7 +381,7 @@ def _extract_param_cli_name(param, ctd_model): elif not hasattr(param.parent.parent.parent, 'parent'): return resolve_param_mapping(param, ctd_model) else: - if ctd_model.cli: + if hasattr(ctd_model, "cli") and ctd_model.cli: warning("Using nested parameter sections (NODE elements) is not compatible with ", 1) return extract_param_name(param.parent) + ":" + resolve_param_mapping(param, ctd_model) else: @@ -368,7 +389,7 @@ def _extract_param_cli_name(param, ctd_model): def extract_param_path(param): - if type(param.parent) == ParameterGroup: + if type(param.parent) == ParameterGroup or type(param.parent) == Parameters: if not hasattr(param.parent.parent, "parent"): return [param.name] elif not hasattr(param.parent.parent.parent, "parent"): @@ -383,6 +404,7 @@ def extract_param_name(param): # we generate parameters with colons for subgroups, but not for the two topmost parents (OpenMS legacy) return ":".join(extract_param_path(param)) + def extract_command_line_prefix(param, ctd_model): param_name = extract_param_name(param) param_cli_name = _extract_param_cli_name(param, ctd_model) diff --git a/convert.py b/convert.py index 4e28ef8b..c5992403 100644 --- a/convert.py +++ b/convert.py @@ -222,19 +222,19 @@ def main(argv=None): print("Interrupted...") return 0 - except ApplicationException, e: + except ApplicationException as e: traceback.print_exc() utils.error("CTDConverter could not complete the requested operation.", 0) utils.error("Reason: " + e.msg, 0) return 1 - except ModelError, e: + except ModelError as e: traceback.print_exc() utils.error("There seems to be a problem with one of your input CTDs.", 0) utils.error("Reason: " + e.msg, 0) return 1 - except Exception, e: + except Exception as e: traceback.print_exc() utils.error("CTDConverter could not complete the requested operation.", 0) utils.error("Reason: " + e.msg, 0) diff --git a/galaxy/converter.py b/galaxy/converter.py index 1aa9c2ca..84dbe7c6 100755 --- a/galaxy/converter.py +++ b/galaxy/converter.py @@ -83,6 +83,8 @@ def add_specific_args(parser): action="append", required=None, help="The prefix of the macro name in the corresponding trest macros file") parser.add_argument("--test-test", dest="test_test", action='store_true', default=False, required=False, help="Generate a simple test for the internal unit tests.") + parser.add_argument("--test-only", dest="test_only", action='store_true', default=False, required=False, + help="Generate only the test section.") parser.add_argument("--tool-version", dest="tool_version", required=False, default = None, help="Tool version to use (if not given its extracted from the CTD)") @@ -138,6 +140,7 @@ def convert_models(args, parsed_ctds): macros_to_expand=macros_to_expand, parameter_hardcoder=args.parameter_hardcoder, test_test=args.test_test, + test_only=args.test_only, test_macros_file_names=args.test_macros_files, test_macros_prefix=args.test_macros_prefix, tool_version = args.tool_version) @@ -146,13 +149,15 @@ def convert_models(args, parsed_ctds): # generation of galaxy stubs is ready... now, let's see if we need to generate a tool_conf.xml - if args.tool_conf_destination is not None: - generate_tool_conf(parsed_ctds, args.tool_conf_destination, - args.galaxy_tool_path, args.default_category) + # TODO remove tool conf .. deprecated +# if args.tool_conf_destination is not None: +# generate_tool_conf(parsed_ctds, args.tool_conf_destination, +# args.galaxy_tool_path, args.default_category) + # TODO remove datatypes_conf generation # generate datatypes_conf.xml - if args.data_types_destination is not None: - generate_data_type_conf(supported_file_formats, args.data_types_destination) +# if args.data_types_destination is not None: +# generate_data_type_conf(supported_file_formats, args.data_types_destination) def parse_tools_list_file(tools_list_file): @@ -402,6 +407,16 @@ def _convert_internal(parsed_ctds, **kwargs): @return a tuple containing the model, output destination, origin file """ + if "test_only" in kwargs and kwargs["test_only"]: + for parsed_ctd in parsed_ctds: + test = create_test_only(parsed_ctd.ctd_model) + tree = ElementTree(test) + output_file = parsed_ctd.suggested_output_file + logger.info("Writing to %s" % utils.get_filename(output_file), 1) + with open(output_file, 'w') as fh: + tree.write(fh, encoding="UTF-8", xml_declaration=False, pretty_print=True) + return + parameter_hardcoder = kwargs["parameter_hardcoder"] for parsed_ctd in parsed_ctds: model = parsed_ctd.ctd_model @@ -423,11 +438,11 @@ def _convert_internal(parsed_ctds, **kwargs): for a in hardcoded_attributes: if not hasattr(param, a): continue - logger.error("%s %s"%(param.name,type(getattr(param, a))) ) + # logger.error("%s %s"%(param.name,type(getattr(param, a))) ) if type(getattr(param, a)) is _FileFormat: setattr(param, a, _FileFormat(str(hardcoded_attributes[a]))) elif type(getattr(param, a)) is _Choices: - logger.error("SET CTD %s %s %s" %(param.name, a, hardcoded_attributes[a])) + # logger.error("SET CTD %s %s %s" %(param.name, a, hardcoded_attributes[a])) setattr(param, a, _Choices(str(hardcoded_attributes[a]))) elif type(getattr(param, a)) is _NumericRange: raise Exception("Overwriting of Numeric Range not implemented") @@ -575,10 +590,18 @@ def create_command(tool, model, **kwargs): advanced_cmd = {'preprocessing': [], 'command': [], 'postprocessing': []} final_cmd['preprocessing'].extend(["@QUOTE_FOO@", "@EXT_FOO@", "#import re", "", "## Preprocessing"]) + + # - call the executable with -write_ctd to write the ctd file (with defaults) + # - use fill_ctd.py to overwrite the defaults in the ctd file with the + # Galaxy parameters in the JSON file (from inputs config file) + # (note the base name call is only necessary for the CTDconverter tests) + # - feed the ctd file to the executable (with -ini) + # note: input and output file parameters are still given on the command line + # - output file parameters are not included in the JSON file + # - input and output files are accessed through links / files that have the correct extension final_cmd['command'].extend(["", "## Main program call"]) - final_cmd['command'].append("""export PYTHONPATH='$__tool_directory__/CTDopts' && -@EXECUTABLE@ -write_ctd ./ && -python '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' && + final_cmd['command'].append("""@EXECUTABLE@ -write_ctd ./ && +python '$__tool_directory__/fill_ctd.py' "\$(basename "@EXECUTABLE@.ctd")" '$args_json' && @EXECUTABLE@ -ini @EXECUTABLE@.ctd""") final_cmd['command'].extend(kwargs["add_to_command_line"]) final_cmd['postprocessing'].extend(["", "## Postprocessing"]) @@ -643,11 +666,9 @@ def create_command(tool, model, **kwargs): actual_input_parameter = None if type_param is not None: - type_param_name = get_galaxy_parameter_name(type_param) - if type_param.advanced: - type_param_name = ADVANCED_OPTIONS_NAME+"cond." + type_param_name + type_param_name = get_galaxy_parameter_path(type_param) elif len(formats) > 1 and (corresponding_input == None or not fmt_from_corresponding) and not param.is_list: - type_param_name = get_galaxy_parameter_path(param, True) + "_type" + type_param_name = get_galaxy_parameter_path(param, "type") else: type_param_name = None @@ -665,7 +686,7 @@ def create_command(tool, model, **kwargs): else: logger.info("1 fmt + dataset %s" % param.name, 1) param_cmd['command'].append("'"+param.name+"/output.${gxy2omsext(\"" + fmt + "\")}'") - param_cmd['postprocessing'].append("&& mv '"+param.name+"/output.${gxy2omsext(\"" + fmt + "\")}' '$out_"+param.name+"'") + param_cmd['postprocessing'].append("&& mv '"+param.name+"/output.${gxy2omsext(\"" + fmt + "\")}' '$"+param.name+"'") # if there is a type parameter then we use the type selected by the user # - single: write to temp file with the oms extension and mv it to the actual file output which is treated via change_format @@ -681,7 +702,7 @@ def create_command(tool, model, **kwargs): # then move it to the actual place specified by the parameter # the format is then set by the tag using param_cmd['command'].append("'" + param.name+"/output.${"+type_param_name+"}'") - param_cmd['postprocessing'].append("&& mv '"+param.name+"/output.${"+type_param_name+"}' '$out_"+param.name+"'") + param_cmd['postprocessing'].append("&& mv '"+param.name+"/output.${"+type_param_name+"}' '$"+actual_parameter+"'") elif actual_input_parameter is not None: if param.is_list: logger.info("actual + list %s" %param.name, 1) @@ -690,10 +711,10 @@ def create_command(tool, model, **kwargs): logger.info("actual + dataset %s %s %s" % (param.name, actual_input_parameter, corresponding_input.is_list), 1) if corresponding_input.is_list: param_cmd['command'].append("'" + param.name+"/output.${"+actual_input_parameter+"[0].ext}'") - param_cmd['postprocessing'].append("&& mv '"+param.name+"/output.${"+actual_input_parameter+"[0].ext}' '$out_"+param.name+"'") + param_cmd['postprocessing'].append("&& mv '"+param.name+"/output.${"+actual_input_parameter+"[0].ext}' '$"+actual_parameter+"'") else: param_cmd['command'].append("'" + param.name+"/output.${"+actual_input_parameter+".ext}'") - param_cmd['postprocessing'].append("&& mv '"+param.name+"/output.${"+actual_input_parameter+".ext}' '$out_"+param.name+"'") + param_cmd['postprocessing'].append("&& mv '"+param.name+"/output.${"+actual_input_parameter+".ext}' '$"+actual_parameter+"'") else: if param.is_list: raise Exception( "output parameter itemlist %s without corresponding input" ) @@ -719,19 +740,17 @@ def create_command(tool, model, **kwargs): # need no if (otherwise the empty string could not be provided) if not ( param.required or is_boolean_parameter(param) or (param.type == str and param.restrictions is None)): # and not(param.type is _InFile and param.is_list): - if param.type is _OutFile: - actual_parameter = get_galaxy_parameter_path(param, True) - + actual_parameter = get_galaxy_parameter_path(param, "FLAG") for stage in param_cmd: if len(param_cmd[stage]) == 0: continue # special case for optional itemlists: for those if no option is selected only the parameter must be specified if is_selection_parameter(param) and param.is_list and param.required == False: - param_cmd[stage] = [param_cmd[stage][0]] + ["#if $" + param.name + ":"] + utils.indent(param_cmd[stage][1:]) + ["#end if"] - elif is_selection_parameter(param) or param.type is _OutFile or param.type is _InFile: - param_cmd[stage] = ["#if $" + param.name + ":"] + utils.indent(param_cmd[stage]) + ["#end if"] + param_cmd[stage] = [param_cmd[stage][0]] + ["#if $" + actual_parameter + ":"] + utils.indent(param_cmd[stage][1:]) + ["#end if"] + elif is_selection_parameter(param) or param.type is _OutFile or param.type is _InFile: + param_cmd[stage] = ["#if $" + actual_parameter + ":"] + utils.indent(param_cmd[stage]) + ["#end if"] else: - param_cmd[stage] = ["#if str($" + param.name + "):"] + utils.indent(param_cmd[stage]) + ["#end if"] + param_cmd[stage] = ["#if str($" + actual_parameter + "):"] + utils.indent(param_cmd[stage]) + ["#end if"] for stage in param_cmd: if len(param_cmd[stage]) == 0: @@ -794,37 +813,34 @@ def get_galaxy_parameter_path(param, inparam = False): """ get the complete cheetah path for a parameter """ - parameter = get_galaxy_parameter_name(param, inparam) + p = get_galaxy_parameter_name(param, inparam) path = utils.extract_param_path(param) if len(path) > 1 and (not param.type is _OutFile or inparam): - return ".".join( [ "section_"+_ for _ in path[:-1]] ).replace("-", "_") +"."+ parameter + return (".".join( [ "section_"+_ for _ in path[:-1]] )+"."+p).replace("-", "_") elif param.advanced and (not param.type is _OutFile or inparam): - return ADVANCED_OPTIONS_NAME+"cond." + parameter + return ADVANCED_OPTIONS_NAME+"cond." + p else: - return parameter + return p def get_galaxy_parameter_name(param, inparam = False): """ get the name of the parameter used in the galaxy tool - - prepend param_ or param_out_ (the latter for outputs, except if inparam is True) - replace : and - by _ + - add _FLAG for output parameters if inparam is True - the idea of inparam is to be used for optional outputs (param_out_x) for - which an additional boolean input (param_x) exists + the idea of inparam is to be used for optional outputs (out_x) for + which an additional boolean input (out_x_FLAG) exists @param param the parameter @param inparam get the name of the corresponding input @return the name used for the parameter in the tool form """ - if type(param) is ParameterGroup: - return param.name.replace("-", "_") - - p = utils.extract_param_name(param).replace(":", "_").replace("-", "_") - if param.type is _OutFile and not inparam: - return "param_out_%s" % p + p = param.name.replace("-", "_") + if param.type is _OutFile and inparam: + return "%s_%s" % (p, inparam) else: - return "param_%s" % p + return "%s" % p def get_out_type_param(out_param, model, parameter_hardcoder): @@ -836,7 +852,6 @@ def get_out_type_param(out_param, model, parameter_hardcoder): return None for param in utils.extract_and_flatten_parameters(model): - if param.name == out_param.name + "_type": return param return None @@ -962,9 +977,9 @@ def create_inputs(tool, model, **kwargs): # mandatory outpiles: no input node needed # inputs: create the input param if not param.required: - add_child_node(parent_node, "param", OrderedDict([("type", "boolean"), ("name", param.name), ("optional", "false"), ("checked", "false"), ("label", "Generate output %s (%s)" %(param.name, param.description))])) - else: - add_child_node(parent_node, "param", OrderedDict([("type", "hidden"), ("name", param.name)])) + add_child_node(parent_node, "param", OrderedDict([("type", "boolean"), ("name", param.name + "_FLAG"), ("optional", "false"), ("checked", "false"), ("label", "Generate output %s (%s)" %(param.name, param.description))])) +# else: +# add_child_node(parent_node, "param", OrderedDict([("type", "hidden"), ("name", param.name)])) continue # create the actual param node and fill the attributes @@ -1387,7 +1402,7 @@ def create_output_node(parent, param, model, supported_file_formats, parameter_h discover_node = add_child_node(data_node, "discover_datasets", OrderedDict([("directory", get_galaxy_parameter_path(param))])) - data_node.attrib["name"] = "out_"+param.name + data_node.attrib["name"] = param.name data_node.attrib["label"] = "${tool.name} on ${on_string}: %s" % param.name formats = get_formats(param, supported_file_formats, TYPE_TO_GALAXY_TYPE[_OutFile]) @@ -1436,9 +1451,11 @@ def create_output_node(parent, param, model, supported_file_formats, parameter_h # add filter for optional parameters if not param.required: - filter_node = add_child_node(data_node, "filter") - filter_node.text = get_galaxy_parameter_name(param, True) - if param.advanced: + filter_node = add_child_node(data_node, "filter") + filter_path = utils.extract_param_path(param) + filter_path[-1] += "_FLAG" + filter_node.text = filter_path[0] + "".join([ '["'+ _ +'"]' for _ in filter_path[1:] ]) + if len(filter_path) == 1 and param.advanced: filter_node.text = "{aon}cond['{aon}selector'] == 'advanced' and {aon}cond['".format(aon = ADVANCED_OPTIONS_NAME) + filter_node.text + "']" return data_node @@ -1567,6 +1584,53 @@ def create_tests(parent, inputs=None, outputs=None, test_macros_prefix=None, nam expand_macros(tests_node, [ p + name for p in test_macros_prefix]) +def create_test_only(model, **kwargs): + + parameter_hardcoder = kwargs["parameter_hardcoder"] + + section_nodes = dict() + section_params = dict() + + test = Element("test") + advanced = None + + for param in utils.extract_and_flatten_parameters(model, True): + try: + parent = section_nodes[utils.extract_param_name(param.parent)] + except KeyError: + parent = test + + if section_nodes.has_key(utils.extract_param_name(param.parent)): + parent = section_nodes[utils.extract_param_name(param.parent)] + elif type(param) is not ParameterGroup and param.advanced: + if advanced is None: + advanced = add_child_node(test, "conditional", OrderedDict([("name", "adv_opts_cond")])) + adv_sel = add_child_node(advanced, "param", OrderedDict([("name", "adv_opts_selector"), ("value", "advanced")])) + parent = advanced + else: + parent = test + + if type(param) is ParameterGroup: + section_params[utils.extract_param_name(param)] = param + section_nodes[utils.extract_param_name(param)] = add_child_node(parent, "section", OrderedDict([("name", param.name)])) + continue + if param.type is _OutFile and not param.required: + nd = add_child_node(parent, "param", OrderedDict([("name", param.name+"_FLAG"),("value", "false")])) + + formats = get_formats(param, kwargs["supported_file_formats"], TYPE_TO_GALAXY_TYPE[_OutFile]) + type_param = get_out_type_param(param, model, parameter_hardcoder) + corresponding_input, fmt_from_corresponding = get_corresponding_input(param, model) + if len(formats) > 1 and type_param is None and (corresponding_input is None or not fmt_from_corresponding) and not param.is_list: + fmt_select = add_child_node(parent, "param", OrderedDict([("name", param.name + "_type"), ("value", "")])) + + if param.is_list and type(param.default) is not _Null: + value = '"' + '" "'.join(map(str, param.default)) + '"' + else: + value = str(param.default) + + nd = add_child_node(parent, "param" if param.type is not _OutFile else "output", OrderedDict([("name", param.name),("value", value)])) + return test + # Shows basic information about the file, such as data ranges and file type. def create_help(tool, model): """ diff --git a/galaxy/macros.xml b/galaxy/macros.xml index b25a80ca..ecb88310 100644 --- a/galaxy/macros.xml +++ b/galaxy/macros.xml @@ -3,11 +3,12 @@ You can edit this file to add your own macros, if you so desire, or you can add additional macro files using the m/macros parameter --> - 2.3.0 - 12 + 5.0.011 + 0 - openms + bash + ctdopts diff --git a/tests/test-data/bool b/tests/test-data/bool new file mode 120000 index 00000000..16807da6 --- /dev/null +++ b/tests/test-data/bool @@ -0,0 +1 @@ +mock.py \ No newline at end of file diff --git a/tests/test-data/bool.ctd b/tests/test-data/bool.ctd index d05a1375..289c0612 100644 --- a/tests/test-data/bool.ctd +++ b/tests/test-data/bool.ctd @@ -1,5 +1,5 @@ - + diff --git a/tests/test-data/bool.xml b/tests/test-data/bool.xml index ff604035..8030b7ef 100644 --- a/tests/test-data/bool.xml +++ b/tests/test-data/bool.xml @@ -1,10 +1,10 @@ - + Bool parameter tests - echo BoolTest + /home/berntm/projects/tools-galaxyp/tools/openms/CTDConverter/tests/test-data/bool macros.xml @@ -12,28 +12,29 @@ $param_stdout]]> + +## Preprocessing + +## Main program call +@EXECUTABLE@ -write_ctd ./ && +python '$__tool_directory__/fill_ctd.py' "\$(basename "@EXECUTABLE@.ctd")" '$args_json' && +@EXECUTABLE@ -ini @EXECUTABLE@.ctd +> $param_stdout + +## Postprocessing]]> + + + - - - - + + + + - - - - + + + + @@ -41,15 +42,15 @@ $param_flag_false - - - - + + + + - - - - + + + + @@ -58,6 +59,6 @@ $param_flag_false +For more information, visit http://www.openms.de/unavailable.html]]> diff --git a/tests/test-data/empty b/tests/test-data/empty new file mode 120000 index 00000000..16807da6 --- /dev/null +++ b/tests/test-data/empty @@ -0,0 +1 @@ +mock.py \ No newline at end of file diff --git a/tests/test-data/empty.ctd b/tests/test-data/empty.ctd index 0876084b..6f01c63f 100644 --- a/tests/test-data/empty.ctd +++ b/tests/test-data/empty.ctd @@ -1,9 +1,9 @@ - - - + + + - + diff --git a/tests/test-data/empty.xml b/tests/test-data/empty.xml index f700355c..a1a5a682 100644 --- a/tests/test-data/empty.xml +++ b/tests/test-data/empty.xml @@ -1,10 +1,10 @@ - - String parameter tests + + empty ctd test - echo StringTest + /home/berntm/projects/tools-galaxyp/tools/openms/CTDConverter/tests/test-data/empty macros.xml @@ -12,8 +12,19 @@ $param_stdout]]> + +## Preprocessing + +## Main program call +@EXECUTABLE@ -write_ctd ./ && +python '$__tool_directory__/fill_ctd.py' "\$(basename "@EXECUTABLE@.ctd")" '$args_json' && +@EXECUTABLE@ -ini @EXECUTABLE@.ctd +> $param_stdout + +## Postprocessing]]> + + + @@ -23,9 +34,9 @@ - +For more information, visit http://www.openms.de/unavailable.html]]> diff --git a/tests/test-data/fill_ctd.py b/tests/test-data/fill_ctd.py new file mode 100644 index 00000000..c342d627 --- /dev/null +++ b/tests/test-data/fill_ctd.py @@ -0,0 +1,16 @@ +import json +import sys + +from CTDopts.CTDopts import CTDModel + +input_ctd = sys.argv[1] +with open(sys.argv[2]) as fh: + args = json.load(fh) + +if "adv_opts_cond" in args: + args.update(args["adv_opts_cond"]) + del args["adv_opts_cond"] + +model = CTDModel(from_file=input_ctd) +# model.write_ctd(input_ctd, arg_dict = {'1':args}) +model.write_ctd(input_ctd) diff --git a/tests/test-data/float b/tests/test-data/float new file mode 120000 index 00000000..16807da6 --- /dev/null +++ b/tests/test-data/float @@ -0,0 +1 @@ +mock.py \ No newline at end of file diff --git a/tests/test-data/float.ctd b/tests/test-data/float.ctd index 045e123c..5fc283fb 100644 --- a/tests/test-data/float.ctd +++ b/tests/test-data/float.ctd @@ -1,45 +1,45 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + diff --git a/tests/test-data/float.xml b/tests/test-data/float.xml index 7cbfa092..40fbb3c5 100644 --- a/tests/test-data/float.xml +++ b/tests/test-data/float.xml @@ -1,10 +1,10 @@ - + Float parameter tests - echo FloatTest + /home/berntm/projects/tools-galaxyp/tools/openms/CTDConverter/tests/test-data/float macros.xml @@ -12,140 +12,37 @@ $param_stdout]]> + +## Preprocessing + +## Main program call +@EXECUTABLE@ -write_ctd ./ && +python '$__tool_directory__/fill_ctd.py' "\$(basename "@EXECUTABLE@.ctd")" '$args_json' && +@EXECUTABLE@ -ini @EXECUTABLE@.ctd +> $param_stdout + +## Postprocessing]]> + + + - - - - - - - - - - - - - - - - + + + + + + + + - - - - - - - - - - - - - - - - + + + + + + + + @@ -153,23 +50,15 @@ - - - - - - - - + + + + - - - - - - - - + + + + @@ -178,6 +67,6 @@ +For more information, visit http://www.openms.de/unavailable.html]]> diff --git a/tests/test-data/ifile b/tests/test-data/ifile new file mode 120000 index 00000000..16807da6 --- /dev/null +++ b/tests/test-data/ifile @@ -0,0 +1 @@ +mock.py \ No newline at end of file diff --git a/tests/test-data/ifile.ctd b/tests/test-data/ifile.ctd index ba5f2466..c221a000 100644 --- a/tests/test-data/ifile.ctd +++ b/tests/test-data/ifile.ctd @@ -1,5 +1,5 @@ - + diff --git a/tests/test-data/ifile.xml b/tests/test-data/ifile.xml index a889143d..e4d78c21 100644 --- a/tests/test-data/ifile.xml +++ b/tests/test-data/ifile.xml @@ -1,10 +1,10 @@ - + Input File parameter tests - echo inFileTest + /home/berntm/projects/tools-galaxyp/tools/openms/CTDConverter/tests/test-data/ifile macros.xml @@ -12,398 +12,457 @@ $param_stdout]]> +> $param_stdout + +## Postprocessing]]> + + + - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + @@ -411,19 +470,7 @@ ${' '.join(["'param_list_mandatory_multformat/%s.%s'"%(re.sub('[^\w\-_]', '_', _ - - - - - - - - - - - - @@ -432,6 +479,6 @@ ${' '.join(["'param_list_mandatory_multformat/%s.%s'"%(re.sub('[^\w\-_]', '_', _ +For more information, visit http://www.openms.de/unavailable.html]]> diff --git a/tests/test-data/integer b/tests/test-data/integer new file mode 120000 index 00000000..16807da6 --- /dev/null +++ b/tests/test-data/integer @@ -0,0 +1 @@ +mock.py \ No newline at end of file diff --git a/tests/test-data/integer.ctd b/tests/test-data/integer.ctd index 908978c0..d4eedcea 100644 --- a/tests/test-data/integer.ctd +++ b/tests/test-data/integer.ctd @@ -1,45 +1,45 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + diff --git a/tests/test-data/integer.xml b/tests/test-data/integer.xml index 6f54f8e3..577c3ed7 100644 --- a/tests/test-data/integer.xml +++ b/tests/test-data/integer.xml @@ -1,10 +1,10 @@ - + Integer parameter tests - echo IntTest + /home/berntm/projects/tools-galaxyp/tools/openms/CTDConverter/tests/test-data/integer macros.xml @@ -12,140 +12,37 @@ $param_stdout]]> + +## Preprocessing + +## Main program call +@EXECUTABLE@ -write_ctd ./ && +python '$__tool_directory__/fill_ctd.py' "\$(basename "@EXECUTABLE@.ctd")" '$args_json' && +@EXECUTABLE@ -ini @EXECUTABLE@.ctd +> $param_stdout + +## Postprocessing]]> + + + - - - - - - - - - - - - - - - - + + + + + + + + - - - - - - - - - - - - - - - - + + + + + + + + @@ -153,23 +50,15 @@ - - - - - - - - + + + + - - - - - - - - + + + + @@ -178,6 +67,6 @@ +For more information, visit http://www.openms.de/unavailable.html]]> diff --git a/tests/test-data/label-help b/tests/test-data/label-help new file mode 120000 index 00000000..16807da6 --- /dev/null +++ b/tests/test-data/label-help @@ -0,0 +1 @@ +mock.py \ No newline at end of file diff --git a/tests/test-data/label-help.ctd b/tests/test-data/label-help.ctd index 7cda0295..d2d851aa 100644 --- a/tests/test-data/label-help.ctd +++ b/tests/test-data/label-help.ctd @@ -1,5 +1,5 @@ - + diff --git a/tests/test-data/label-help.xml b/tests/test-data/label-help.xml index 4d2bd348..88c5facc 100644 --- a/tests/test-data/label-help.xml +++ b/tests/test-data/label-help.xml @@ -1,10 +1,10 @@ - + Lebel Help parameter tests - echo LabelHelpTest + /home/berntm/projects/tools-galaxyp/tools/openms/CTDConverter/tests/test-data/label-help macros.xml @@ -12,54 +12,49 @@ $param_stdout]]> + +## Preprocessing + +## Main program call +@EXECUTABLE@ -write_ctd ./ && +python '$__tool_directory__/fill_ctd.py' "\$(basename "@EXECUTABLE@.ctd")" '$args_json' && +@EXECUTABLE@ -ini @EXECUTABLE@.ctd +> $param_stdout + +## Postprocessing]]> + + + - + - + - + - + - + - + - + - + @@ -69,20 +64,20 @@ - - - - - - - - + + + + + + + + +For more information, visit http://www.openms.de/unavailable.html]]> diff --git a/tests/test-data/mock.py b/tests/test-data/mock.py index 17dcb8bf..81c1683d 100755 --- a/tests/test-data/mock.py +++ b/tests/test-data/mock.py @@ -5,34 +5,51 @@ import os import re +import shutil import sys -wd=os.path.dirname(__file__) -fparam = {"input": set(), "output": set()} -ctd = sys.argv[1] -with open(os.path.join(wd, ctd)) as cf: - for line in cf: - m = re.search(r'type="(input|output)-file"', line) - if m is not None: - n = re.search(r'name="([^"]+)"', line) - fparam[m.group(1)].add(n.group(1)) - +# from argparse import ArgumentParser +# parser = ArgumentParser(prog="mock.py", description="MOCK", add_help=True) +# parser.add_argument("-w", "-write_ctd", dest="write_ctd", metavar='PATH', default=None, required=False, +# help="Write CTD to given path") +# parser.add_argument("-i", "-ini", dest="ini", metavar='CTDFILE', default=None, required=False, +# help="Process CTDFILE") +# parser.add_argument('moreargs', metavar='ARGS', type=str, nargs='*', help='more arguments') +# args = parser.parse_args() +print(sys.argv) + +wd=os.path.dirname(__file__) +bn=os.path.splitext( os.path.basename(__file__) )[0] -i = 2 -while i < len(sys.argv): - if sys.argv[i].startswith("-"): - param = sys.argv[i][1:] - if param in fparam["input"] or param in fparam["output"]: - if param in fparam["input"]: - mode = "r" - else: - mode = "w" +if sys.argv[1] == "-write_ctd": + shutil.copyfile(os.path.join(wd, bn+".ctd"), os.path.join(sys.argv[2], bn+".ctd")) +elif sys.argv[1] == "-ini": + fparam = {"input": set(), "output": set()} + with open(sys.argv[2]) as cf: + for line in cf: + m = re.search(r'type="(input|output)-file"', line) + if m is not None: + n = re.search(r'name="([^"]+)"', line) + fparam[m.group(1)].add(n.group(1)) - while i+1 < len(sys.argv): - if sys.argv[i+1].startswith("-"): - break - of = open(sys.argv[i+1], mode) - of.close() - i += 1 - i += 1 + i = 3 + while i < len(sys.argv): + if sys.argv[i].startswith("-"): + param = sys.argv[i][1:] + if param in fparam["input"] or param in fparam["output"]: + if param in fparam["input"]: + mode = "r" + else: + mode = "w" + + while i+1 < len(sys.argv): + if sys.argv[i+1].startswith("-"): + break + of = open(sys.argv[i+1], mode) + of.close() + i += 1 + i += 1 +else: + sys.stderr.write("Either -write_ctd or -ini must be given") + sys.exit(1) diff --git a/tests/test-data/ofile b/tests/test-data/ofile new file mode 120000 index 00000000..16807da6 --- /dev/null +++ b/tests/test-data/ofile @@ -0,0 +1 @@ +mock.py \ No newline at end of file diff --git a/tests/test-data/ofile-corresponding-input b/tests/test-data/ofile-corresponding-input new file mode 120000 index 00000000..16807da6 --- /dev/null +++ b/tests/test-data/ofile-corresponding-input @@ -0,0 +1 @@ +mock.py \ No newline at end of file diff --git a/tests/test-data/ofile-corresponding-input.ctd b/tests/test-data/ofile-corresponding-input.ctd index f73ad663..d5043afb 100644 --- a/tests/test-data/ofile-corresponding-input.ctd +++ b/tests/test-data/ofile-corresponding-input.ctd @@ -1,5 +1,5 @@ - + diff --git a/tests/test-data/ofile-corresponding-input.xml b/tests/test-data/ofile-corresponding-input.xml index b60229d8..845ef3e2 100644 --- a/tests/test-data/ofile-corresponding-input.xml +++ b/tests/test-data/ofile-corresponding-input.xml @@ -1,10 +1,10 @@ - + Outut File parameter tests-corresponding-input - '$__tool_directory__/mock.py' ofile-corresponding-input.ctd + /home/berntm/projects/tools-galaxyp/tools/openms/CTDConverter/tests/test-data/ofile-corresponding-input macros.xml @@ -12,173 +12,193 @@ + + + - - - - - - + + + + + + - - - + + + - - - param_optional_noformat + + + optional_noformat_FLAG - - - param_optional_oneformat + + + optional_oneformat_FLAG - - - param_optional_multformat + + + optional_multformat_FLAG - - - adv_opts_cond['adv_opts_selector'] == 'advanced' and adv_opts_cond['param_optional_noformat_adv'] + + + adv_opts_cond['adv_opts_selector'] == 'advanced' and adv_opts_cond['optional_noformat_adv_FLAG'] - - - adv_opts_cond['adv_opts_selector'] == 'advanced' and adv_opts_cond['param_optional_oneformat_adv'] + + + adv_opts_cond['adv_opts_selector'] == 'advanced' and adv_opts_cond['optional_oneformat_adv_FLAG'] - - - adv_opts_cond['adv_opts_selector'] == 'advanced' and adv_opts_cond['param_optional_multformat_adv'] + + + adv_opts_cond['adv_opts_selector'] == 'advanced' and adv_opts_cond['optional_multformat_adv_FLAG'] - - - - - - + + + - - - + + + - - - - - - + + + + + + +For more information, visit http://www.openms.de/unavailable.html]]> diff --git a/tests/test-data/ofile-mult b/tests/test-data/ofile-mult new file mode 120000 index 00000000..16807da6 --- /dev/null +++ b/tests/test-data/ofile-mult @@ -0,0 +1 @@ +mock.py \ No newline at end of file diff --git a/tests/test-data/ofile-mult-typeparam b/tests/test-data/ofile-mult-typeparam new file mode 120000 index 00000000..16807da6 --- /dev/null +++ b/tests/test-data/ofile-mult-typeparam @@ -0,0 +1 @@ +mock.py \ No newline at end of file diff --git a/tests/test-data/ofile-mult-typeparam.ctd b/tests/test-data/ofile-mult-typeparam.ctd index 6ea3e351..24c42abe 100644 --- a/tests/test-data/ofile-mult-typeparam.ctd +++ b/tests/test-data/ofile-mult-typeparam.ctd @@ -1,5 +1,5 @@ - + diff --git a/tests/test-data/ofile-mult-typeparam.xml b/tests/test-data/ofile-mult-typeparam.xml index 5f2efeeb..ea6169a7 100644 --- a/tests/test-data/ofile-mult-typeparam.xml +++ b/tests/test-data/ofile-mult-typeparam.xml @@ -1,10 +1,10 @@ - + multiple Output File parameter tests - '$__tool_directory__/mock.py' ofile-mult-typeparam.ctd + /home/berntm/projects/tools-galaxyp/tools/openms/CTDConverter/tests/test-data/ofile-mult-typeparam macros.xml @@ -12,66 +12,76 @@ + +## Postprocessing +${' '.join(["&& mv -n 'mandatory_mandatoryinput/%(id)s.%(gext)s' 'mandatory_mandatoryinput/%(id)s'"%{"id": re.sub('[^\w\-_]', '_', _.element_identifier), "gext": $gxy2omsext("tsv")} for _ in $inp_optional_mandatoryinput if _ ])} +${' '.join(["&& mv -n 'mandatory_optionalinput/%(id)s.%(omsext)s' 'mandatory_optionalinput/%(id)s.%(gext)s'"%{"id": re.sub('[^\w\-_]', '_', _.element_identifier), "omsext":mandatory_optionalinput_type, "gext": oms2gxyext("mandatory_optionalinput_type")} for _ in $inp_mandatory_optionalinput if _ ])}]]> + + + - + - + - - + + - - + + - - + + - - - - - + + + + +For more information, visit http://www.openms.de/unavailable.html]]> diff --git a/tests/test-data/ofile-mult.ctd b/tests/test-data/ofile-mult.ctd index 78dee0fb..9526004c 100644 --- a/tests/test-data/ofile-mult.ctd +++ b/tests/test-data/ofile-mult.ctd @@ -1,5 +1,5 @@ - + diff --git a/tests/test-data/ofile-mult.xml b/tests/test-data/ofile-mult.xml index a39ad165..cf64fa66 100644 --- a/tests/test-data/ofile-mult.xml +++ b/tests/test-data/ofile-mult.xml @@ -1,10 +1,10 @@ - + multiple Output File parameter tests - '$__tool_directory__/mock.py' ofile-mult.ctd + /home/berntm/projects/tools-galaxyp/tools/openms/CTDConverter/tests/test-data/ofile-mult macros.xml @@ -12,92 +12,109 @@ + + + - - - - - - + + + + + + - - + + - - - param_optional_mandatoryinput + + + optional_mandatoryinput_FLAG - - + + - - - param_optional_optionalinput + + + optional_optionalinput_FLAG - - - - - - + + + + +For more information, visit http://www.openms.de/unavailable.html]]> diff --git a/tests/test-data/ofile-typeparam b/tests/test-data/ofile-typeparam new file mode 120000 index 00000000..16807da6 --- /dev/null +++ b/tests/test-data/ofile-typeparam @@ -0,0 +1 @@ +mock.py \ No newline at end of file diff --git a/tests/test-data/ofile-typeparam.ctd b/tests/test-data/ofile-typeparam.ctd index f8b1f533..685a990f 100644 --- a/tests/test-data/ofile-typeparam.ctd +++ b/tests/test-data/ofile-typeparam.ctd @@ -1,5 +1,5 @@ - + diff --git a/tests/test-data/ofile-typeparam.xml b/tests/test-data/ofile-typeparam.xml index 9e4f4c7d..60e76820 100644 --- a/tests/test-data/ofile-typeparam.xml +++ b/tests/test-data/ofile-typeparam.xml @@ -1,10 +1,10 @@ - + Outut File parameter tests-typeparam - '$__tool_directory__/mock.py' ofile-typeparam.ctd + /home/berntm/projects/tools-galaxyp/tools/openms/CTDConverter/tests/test-data/ofile-typeparam macros.xml @@ -12,225 +12,229 @@ + + + - + - + - - + + - + - - - - + + + + - + - + - - + + - + - + - - - param_optional_noformat + + + optional_noformat_FLAG - + - - - + + + - + - - - + + + - param_optional_multformat + optional_multformat_FLAG - - - adv_opts_cond['adv_opts_selector'] == 'advanced' and adv_opts_cond['param_optional_noformat_adv'] + + + adv_opts_cond['adv_opts_selector'] == 'advanced' and adv_opts_cond['optional_noformat_adv_FLAG'] - + - - - + + + - + - - - + + + - adv_opts_cond['adv_opts_selector'] == 'advanced' and adv_opts_cond['param_optional_multformat_adv'] + adv_opts_cond['adv_opts_selector'] == 'advanced' and adv_opts_cond['optional_multformat_adv_FLAG'] - - - - - - - - - + + + + + + - - - - - - + + + + + + - - - - + + + + +For more information, visit http://www.openms.de/unavailable.html]]> diff --git a/tests/test-data/ofile.ctd b/tests/test-data/ofile.ctd index a206e8e5..fcb5045b 100644 --- a/tests/test-data/ofile.ctd +++ b/tests/test-data/ofile.ctd @@ -1,5 +1,5 @@ - + diff --git a/tests/test-data/ofile.xml b/tests/test-data/ofile.xml index 8331bf51..836c78b4 100644 --- a/tests/test-data/ofile.xml +++ b/tests/test-data/ofile.xml @@ -1,10 +1,10 @@ - + Outut File parameter tests - '$__tool_directory__/mock.py' ofile.ctd + /home/berntm/projects/tools-galaxyp/tools/openms/CTDConverter/tests/test-data/ofile macros.xml @@ -12,175 +12,186 @@ + + + - - - + + + - + - + - - - + + + - + - + - - - param_optional_noformat + + + optional_noformat_FLAG - - - param_optional_oneformat + + + optional_oneformat_FLAG - - - param_optional_multformat + + + optional_multformat_FLAG - - - adv_opts_cond['adv_opts_selector'] == 'advanced' and adv_opts_cond['param_optional_noformat_adv'] + + + adv_opts_cond['adv_opts_selector'] == 'advanced' and adv_opts_cond['optional_noformat_adv_FLAG'] - - - adv_opts_cond['adv_opts_selector'] == 'advanced' and adv_opts_cond['param_optional_oneformat_adv'] + + + adv_opts_cond['adv_opts_selector'] == 'advanced' and adv_opts_cond['optional_oneformat_adv_FLAG'] - - - adv_opts_cond['adv_opts_selector'] == 'advanced' and adv_opts_cond['param_optional_multformat_adv'] + + + adv_opts_cond['adv_opts_selector'] == 'advanced' and adv_opts_cond['optional_multformat_adv_FLAG'] - - - - - + + + + + - - - - - + + + + + - - - - - - + + + + + + +For more information, visit http://www.openms.de/unavailable.html]]> diff --git a/tests/test-data/repeat b/tests/test-data/repeat new file mode 120000 index 00000000..16807da6 --- /dev/null +++ b/tests/test-data/repeat @@ -0,0 +1 @@ +mock.py \ No newline at end of file diff --git a/tests/test-data/repeat.ctd b/tests/test-data/repeat.ctd index 0806beca..147d6fd2 100644 --- a/tests/test-data/repeat.ctd +++ b/tests/test-data/repeat.ctd @@ -1,5 +1,5 @@ - + diff --git a/tests/test-data/repeat.xml b/tests/test-data/repeat.xml index 956587af..71a166cb 100644 --- a/tests/test-data/repeat.xml +++ b/tests/test-data/repeat.xml @@ -1,10 +1,10 @@ - + Repeat parameter tests - echo RepeatTest + /home/berntm/projects/tools-galaxyp/tools/openms/CTDConverter/tests/test-data/repeat macros.xml @@ -12,295 +12,192 @@ $param_stdout]]> + +## Preprocessing + +## Main program call +@EXECUTABLE@ -write_ctd ./ && +python '$__tool_directory__/fill_ctd.py' "\$(basename "@EXECUTABLE@.ctd")" '$args_json' && +@EXECUTABLE@ -ini @EXECUTABLE@.ctd +> $param_stdout + +## Postprocessing]]> + + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -313,27 +210,27 @@ $quote($param_double_resticted_default_mandatory) - - - - - - - - - - + + + + + + + + + + - - - - - - - - - - + + + + + + + + + + @@ -342,6 +239,6 @@ $quote($param_double_resticted_default_mandatory) +For more information, visit http://www.openms.de/unavailable.html]]> diff --git a/tests/test-data/select b/tests/test-data/select new file mode 120000 index 00000000..16807da6 --- /dev/null +++ b/tests/test-data/select @@ -0,0 +1 @@ +mock.py \ No newline at end of file diff --git a/tests/test-data/select.ctd b/tests/test-data/select.ctd index 1d634b19..c54db80b 100644 --- a/tests/test-data/select.ctd +++ b/tests/test-data/select.ctd @@ -1,5 +1,5 @@ - + diff --git a/tests/test-data/select.xml b/tests/test-data/select.xml index b134fcba..e703dfa7 100644 --- a/tests/test-data/select.xml +++ b/tests/test-data/select.xml @@ -1,10 +1,10 @@ - + Select parameter tests - echo SelectTest + /home/berntm/projects/tools-galaxyp/tools/openms/CTDConverter/tests/test-data/select macros.xml @@ -12,136 +12,49 @@ $param_stdout]]> + +## Preprocessing + +## Main program call +@EXECUTABLE@ -write_ctd ./ && +python '$__tool_directory__/fill_ctd.py' "\$(basename "@EXECUTABLE@.ctd")" '$args_json' && +@EXECUTABLE@ -ini @EXECUTABLE@.ctd +> $param_stdout + +## Postprocessing]]> + + + - + - + - + - + - + @@ -149,7 +62,7 @@ ${' '.join(['"%s"'%str(_) for _ in str($param_list_default_mandatory_many).split - + @@ -157,7 +70,7 @@ ${' '.join(['"%s"'%str(_) for _ in str($param_list_default_mandatory_many).split - + @@ -165,7 +78,7 @@ ${' '.join(['"%s"'%str(_) for _ in str($param_list_default_mandatory_many).split - + @@ -173,35 +86,35 @@ ${' '.join(['"%s"'%str(_) for _ in str($param_list_default_mandatory_many).split - + - + - + - + - + @@ -209,7 +122,7 @@ ${' '.join(['"%s"'%str(_) for _ in str($param_list_default_mandatory_many).split - + @@ -217,7 +130,7 @@ ${' '.join(['"%s"'%str(_) for _ in str($param_list_default_mandatory_many).split - + @@ -225,7 +138,7 @@ ${' '.join(['"%s"'%str(_) for _ in str($param_list_default_mandatory_many).split - + @@ -234,35 +147,35 @@ ${' '.join(['"%s"'%str(_) for _ in str($param_list_default_mandatory_many).split - + - + - + - + - + @@ -270,7 +183,7 @@ ${' '.join(['"%s"'%str(_) for _ in str($param_list_default_mandatory_many).split - + @@ -278,7 +191,7 @@ ${' '.join(['"%s"'%str(_) for _ in str($param_list_default_mandatory_many).split - + @@ -286,7 +199,7 @@ ${' '.join(['"%s"'%str(_) for _ in str($param_list_default_mandatory_many).split - + @@ -294,35 +207,35 @@ ${' '.join(['"%s"'%str(_) for _ in str($param_list_default_mandatory_many).split - + - + - + - + - + @@ -330,7 +243,7 @@ ${' '.join(['"%s"'%str(_) for _ in str($param_list_default_mandatory_many).split - + @@ -338,7 +251,7 @@ ${' '.join(['"%s"'%str(_) for _ in str($param_list_default_mandatory_many).split - + @@ -346,7 +259,7 @@ ${' '.join(['"%s"'%str(_) for _ in str($param_list_default_mandatory_many).split - + @@ -361,15 +274,15 @@ ${' '.join(['"%s"'%str(_) for _ in str($param_list_default_mandatory_many).split - - - - + + + + - - - - + + + + @@ -378,6 +291,6 @@ ${' '.join(['"%s"'%str(_) for _ in str($param_list_default_mandatory_many).split +For more information, visit http://www.openms.de/unavailable.html]]> diff --git a/tests/test-data/string b/tests/test-data/string new file mode 120000 index 00000000..16807da6 --- /dev/null +++ b/tests/test-data/string @@ -0,0 +1 @@ +mock.py \ No newline at end of file diff --git a/tests/test-data/string.ctd b/tests/test-data/string.ctd index cc2fe622..fc3d289c 100644 --- a/tests/test-data/string.ctd +++ b/tests/test-data/string.ctd @@ -1,5 +1,5 @@ - + diff --git a/tests/test-data/string.xml b/tests/test-data/string.xml index 882e8638..5295e8b0 100644 --- a/tests/test-data/string.xml +++ b/tests/test-data/string.xml @@ -1,10 +1,10 @@ - + String parameter tests - echo StringTest + /home/berntm/projects/tools-galaxyp/tools/openms/CTDConverter/tests/test-data/string macros.xml @@ -12,57 +12,50 @@ $param_stdout]]> + +## Preprocessing + +## Main program call +@EXECUTABLE@ -write_ctd ./ && +python '$__tool_directory__/fill_ctd.py' "\$(basename "@EXECUTABLE@.ctd")" '$args_json' && +@EXECUTABLE@ -ini @EXECUTABLE@.ctd +> $param_stdout + +## Postprocessing]]> + + + - + - + - + - + - + - + - + - + @@ -73,11 +66,11 @@ - - + + - - + + @@ -86,6 +79,6 @@ +For more information, visit http://www.openms.de/unavailable.html]]> From 11d01033d53dcd8a47b572c5dd5b02943a193cbf Mon Sep 17 00:00:00 2001 From: M Bernt Date: Fri, 24 Jan 2020 11:31:06 +0100 Subject: [PATCH 067/121] update test --- .travis.yml | 8 +++---- README.md | 3 ++- tests/test-data/bool.xml | 2 +- tests/test-data/empty.xml | 2 +- tests/test-data/float.xml | 2 +- tests/test-data/ifile.xml | 2 +- tests/test-data/integer.xml | 2 +- tests/test-data/label-help.xml | 2 +- tests/test-data/ofile-corresponding-input.xml | 2 +- tests/test-data/ofile-mult-typeparam.xml | 2 +- tests/test-data/ofile-mult.xml | 2 +- tests/test-data/ofile-typeparam.xml | 2 +- tests/test-data/ofile.xml | 2 +- tests/test-data/repeat.xml | 2 +- tests/test-data/select.xml | 2 +- tests/test-data/string.xml | 2 +- tests/test_galaxy_cli.py | 21 ++++++++++--------- 17 files changed, 31 insertions(+), 29 deletions(-) diff --git a/.travis.yml b/.travis.yml index 564f9b2a..8150610f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,9 +22,8 @@ install: - conda config --add channels bioconda - conda config --add channels anaconda - conda config --add channels workflowconversion - -# - git clone https://github.com/WorkflowConversion/CTDopts $HOME/CTDopts - - git clone -b topic/empty_numeric_defaults https://github.com/bernt-matthias/CTDopts $HOME/CTDopts + # TODO could be replaced by conda package once it is merged and released + - git clone -b topic/no-1-2x https://github.com/bernt-matthias/CTDopts $HOME/CTDopts - cd $HOME/CTDopts - python setup.py install - conda install -c conda-forge lxml @@ -34,7 +33,8 @@ install: - python setup.py install - conda install coverage green codecov - - conda install -c bioconda planemo + #- conda install -c bioconda planemo + - pip install planemo script: - python setup.py test diff --git a/README.md b/README.md index 97688d4c..a998835d 100644 --- a/README.md +++ b/README.md @@ -184,10 +184,11 @@ The following invocation of the converter will use `/opt/suite/bin` as a prefix Tests for Galaxy tools are generated with: ``` +PATH=$(pwd)/tests/test-data/:$PATH for i in tests/test-data/*ctd do b=$(basename $i .ctd) -python2 convert.py galaxy -i tests/test-data/$b.ctd -o tests/test-data/$b.xml -m tests/test-data/macros.xml -f tests/test-data/filetypes.txt --test-test -p tests/test-data/hardcoded_params.json --tool-version 5.0.011 -x $(pwd)/tests/test-data/ +python2 convert.py galaxy -i tests/test-data/$b.ctd -o tests/test-data/$b.xml -m tests/test-data/macros.xml -f tests/test-data/filetypes.txt --test-test -p tests/test-data/hardcoded_params.json --tool-version 5.0.011 done ``` diff --git a/tests/test-data/bool.xml b/tests/test-data/bool.xml index 8030b7ef..48272519 100644 --- a/tests/test-data/bool.xml +++ b/tests/test-data/bool.xml @@ -4,7 +4,7 @@ Bool parameter tests - /home/berntm/projects/tools-galaxyp/tools/openms/CTDConverter/tests/test-data/bool + bool macros.xml diff --git a/tests/test-data/empty.xml b/tests/test-data/empty.xml index a1a5a682..a02c575a 100644 --- a/tests/test-data/empty.xml +++ b/tests/test-data/empty.xml @@ -4,7 +4,7 @@ empty ctd test - /home/berntm/projects/tools-galaxyp/tools/openms/CTDConverter/tests/test-data/empty + empty macros.xml diff --git a/tests/test-data/float.xml b/tests/test-data/float.xml index 40fbb3c5..3fcd22ac 100644 --- a/tests/test-data/float.xml +++ b/tests/test-data/float.xml @@ -4,7 +4,7 @@ Float parameter tests - /home/berntm/projects/tools-galaxyp/tools/openms/CTDConverter/tests/test-data/float + float macros.xml diff --git a/tests/test-data/ifile.xml b/tests/test-data/ifile.xml index e4d78c21..16e19493 100644 --- a/tests/test-data/ifile.xml +++ b/tests/test-data/ifile.xml @@ -4,7 +4,7 @@ Input File parameter tests - /home/berntm/projects/tools-galaxyp/tools/openms/CTDConverter/tests/test-data/ifile + ifile macros.xml diff --git a/tests/test-data/integer.xml b/tests/test-data/integer.xml index 577c3ed7..14068270 100644 --- a/tests/test-data/integer.xml +++ b/tests/test-data/integer.xml @@ -4,7 +4,7 @@ Integer parameter tests - /home/berntm/projects/tools-galaxyp/tools/openms/CTDConverter/tests/test-data/integer + integer macros.xml diff --git a/tests/test-data/label-help.xml b/tests/test-data/label-help.xml index 88c5facc..678a94c5 100644 --- a/tests/test-data/label-help.xml +++ b/tests/test-data/label-help.xml @@ -4,7 +4,7 @@ Lebel Help parameter tests - /home/berntm/projects/tools-galaxyp/tools/openms/CTDConverter/tests/test-data/label-help + label-help macros.xml diff --git a/tests/test-data/ofile-corresponding-input.xml b/tests/test-data/ofile-corresponding-input.xml index 845ef3e2..52b1f1c2 100644 --- a/tests/test-data/ofile-corresponding-input.xml +++ b/tests/test-data/ofile-corresponding-input.xml @@ -4,7 +4,7 @@ Outut File parameter tests-corresponding-input - /home/berntm/projects/tools-galaxyp/tools/openms/CTDConverter/tests/test-data/ofile-corresponding-input + ofile-corresponding-input macros.xml diff --git a/tests/test-data/ofile-mult-typeparam.xml b/tests/test-data/ofile-mult-typeparam.xml index ea6169a7..9d4ab7dd 100644 --- a/tests/test-data/ofile-mult-typeparam.xml +++ b/tests/test-data/ofile-mult-typeparam.xml @@ -4,7 +4,7 @@ multiple Output File parameter tests - /home/berntm/projects/tools-galaxyp/tools/openms/CTDConverter/tests/test-data/ofile-mult-typeparam + ofile-mult-typeparam macros.xml diff --git a/tests/test-data/ofile-mult.xml b/tests/test-data/ofile-mult.xml index cf64fa66..d9a85c3b 100644 --- a/tests/test-data/ofile-mult.xml +++ b/tests/test-data/ofile-mult.xml @@ -4,7 +4,7 @@ multiple Output File parameter tests - /home/berntm/projects/tools-galaxyp/tools/openms/CTDConverter/tests/test-data/ofile-mult + ofile-mult macros.xml diff --git a/tests/test-data/ofile-typeparam.xml b/tests/test-data/ofile-typeparam.xml index 60e76820..e7f3a80e 100644 --- a/tests/test-data/ofile-typeparam.xml +++ b/tests/test-data/ofile-typeparam.xml @@ -4,7 +4,7 @@ Outut File parameter tests-typeparam - /home/berntm/projects/tools-galaxyp/tools/openms/CTDConverter/tests/test-data/ofile-typeparam + ofile-typeparam macros.xml diff --git a/tests/test-data/ofile.xml b/tests/test-data/ofile.xml index 836c78b4..31273bae 100644 --- a/tests/test-data/ofile.xml +++ b/tests/test-data/ofile.xml @@ -4,7 +4,7 @@ Outut File parameter tests - /home/berntm/projects/tools-galaxyp/tools/openms/CTDConverter/tests/test-data/ofile + ofile macros.xml diff --git a/tests/test-data/repeat.xml b/tests/test-data/repeat.xml index 71a166cb..a629573e 100644 --- a/tests/test-data/repeat.xml +++ b/tests/test-data/repeat.xml @@ -4,7 +4,7 @@ Repeat parameter tests - /home/berntm/projects/tools-galaxyp/tools/openms/CTDConverter/tests/test-data/repeat + repeat macros.xml diff --git a/tests/test-data/select.xml b/tests/test-data/select.xml index e703dfa7..b028cccb 100644 --- a/tests/test-data/select.xml +++ b/tests/test-data/select.xml @@ -4,7 +4,7 @@ Select parameter tests - /home/berntm/projects/tools-galaxyp/tools/openms/CTDConverter/tests/test-data/select + select macros.xml diff --git a/tests/test-data/string.xml b/tests/test-data/string.xml index 5295e8b0..0b2e987e 100644 --- a/tests/test-data/string.xml +++ b/tests/test-data/string.xml @@ -4,7 +4,7 @@ String parameter tests - /home/berntm/projects/tools-galaxyp/tools/openms/CTDConverter/tests/test-data/string + string macros.xml diff --git a/tests/test_galaxy_cli.py b/tests/test_galaxy_cli.py index 202a0a3e..63ce0807 100644 --- a/tests/test_galaxy_cli.py +++ b/tests/test_galaxy_cli.py @@ -22,19 +22,20 @@ def _compare_cli_output(self, fileprefix): out_file = os.path.join(tmp, '{}.xml'.format(fileprefix)) #out_file = to_test_data('{}.xml'.format(fileprefix)) + my_env = os.environ.copy() + my_env["PATH"] = fileprefix+ ":" + my_env["PATH"] + cmd = ['CTDConverter', 'galaxy', '-i', in_pth, '-o', out_file, '-f', ftypes_pth, '-m', macro_pth, '--test-test', '-p', hcparam_pth, "--tool-version", "5.0.011"] + print("cmd %s" % cmd) - cmd = ['CTDConverter', 'galaxy', '-i', in_pth, '-o', out_file, '-f', ftypes_pth, '-m', macro_pth, '--test-test', '-p', hcparam_pth] - print(cmd) - - popen = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + popen = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=my_env) output, err = popen.communicate() - print(output) - print(err) + print("out %s" % output) + print("err %s" % err) old_file_pth = to_test_data('{}.xml'.format(fileprefix)) - print(out_file) - print(old_file_pth) + print("out_file %s" % out_file) + print("old_file_path %s" % old_file_pth) new_l = file2list(out_file) old_l = file2list(old_file_pth) @@ -43,7 +44,7 @@ def _compare_cli_output(self, fileprefix): self.assertEqual(new_l[i].rstrip(), old_l[i].rstrip()) cmd = ['planemo', 'l', out_file] - print(cmd) + print("cmd %s" % str(cmd)) popen = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) output, err = popen.communicate() print(output) @@ -51,7 +52,7 @@ def _compare_cli_output(self, fileprefix): self.assertEqual(err, 0) cmd = ['planemo', 't', out_file] - print(cmd) + print(str(cmd)) popen = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) output, err = popen.communicate() print(output) From 5ee83cced9e047974c983f8d3e02d8f530b38888 Mon Sep 17 00:00:00 2001 From: M Bernt Date: Fri, 24 Jan 2020 17:05:22 +0100 Subject: [PATCH 068/121] python3 and pep8 --- common/exceptions.py | 2 +- common/utils.py | 45 +- convert.py | 20 +- galaxy/converter.py | 449 +++++++++--------- galaxy/macros.xml | 2 +- setup.py | 3 + tests/test-data/ifile.xml | 96 ++-- tests/test-data/mock.py | 16 +- tests/test-data/ofile-corresponding-input.xml | 2 +- tests/test-data/ofile-mult-typeparam.xml | 18 +- tests/test-data/ofile-mult.xml | 32 +- tests/test_galaxy_cli.py | 10 +- 12 files changed, 349 insertions(+), 346 deletions(-) diff --git a/common/exceptions.py b/common/exceptions.py index ef398d30..315b4a33 100644 --- a/common/exceptions.py +++ b/common/exceptions.py @@ -42,4 +42,4 @@ def __str__(self): return self.msg def __unicode__(self): - return self.msg \ No newline at end of file + return self.msg diff --git a/common/utils.py b/common/utils.py index 16bbef70..9a9be413 100644 --- a/common/utils.py +++ b/common/utils.py @@ -1,11 +1,10 @@ #!/usr/bin/env python # encoding: utf-8 +import json import ntpath import os from lxml import etree -from string import strip -from logger import info, error, warning from common import logger from common.exceptions import ApplicationException @@ -44,11 +43,11 @@ def __init__(self): # blacklisted parameters self.blacklist = set() - + def register_blacklist(self, parameter_name, tool_name): k = self.build_key(parameter_name, tool_name) self.blacklist.add(k) - + # the most specific value will be returned in case of overlap def get_blacklist(self, parameter_name, tool_name): # look for the value that would apply for all tools @@ -61,7 +60,7 @@ def get_blacklist(self, parameter_name, tool_name): def register_attribute(self, parameter_name, attribute, value, tool_name): k = self.build_key(parameter_name, tool_name) - if not self.attribute_map.has_key(k): + if k not in self.attribute_map: self.attribute_map[k] = {} self.attribute_map[k][attribute] = value @@ -108,9 +107,9 @@ def validate_argument_is_valid_path(args, argument_name): if member_value is not None: if isinstance(member_value, list): for file_name in member_value: - paths_to_check.append(strip(str(file_name))) + paths_to_check.append(str(file_name).strip()) else: - paths_to_check.append(strip(str(member_value))) + paths_to_check.append(str(member_value).strip()) for path_to_check in paths_to_check: validate_path_exists(path_to_check) @@ -134,12 +133,12 @@ def parse_input_ctds(xsd_location, input_ctds, output_destination, output_file_e schema = None if xsd_location is not None: try: - info("Loading validation schema from %s" % xsd_location, 0) + logger.info("Loading validation schema from %s" % xsd_location, 0) schema = etree.XMLSchema(etree.parse(xsd_location)) except Exception as e: - error("Could not load validation schema %s. Reason: %s" % (xsd_location, str(e)), 0) + logger.error("Could not load validation schema %s. Reason: %s" % (xsd_location, str(e)), 0) else: - warning("Validation against a schema has not been enabled.", 0) + logger.warning("Validation against a schema has not been enabled.", 0) for input_ctd in input_ctds: if schema is not None: @@ -149,7 +148,7 @@ def parse_input_ctds(xsd_location, input_ctds, output_destination, output_file_e # if multiple inputs are being converted, we need to generate a different output_file for each input if is_converting_multiple_ctds: output_file = os.path.join(output_file, get_filename_without_suffix(input_ctd) + "." + output_file_extension) - info("Parsing %s" % input_ctd) + logger.info("Parsing %s" % input_ctd) model = None try: @@ -160,8 +159,7 @@ def parse_input_ctds(xsd_location, input_ctds, output_destination, output_file_e model = Parameters(from_file=input_ctd) except ModelTypeError: pass - assert model != None, "Could not parse %s, seems to be no CTD/PARAMS"%(input_ctd) - + assert model is not None, "Could not parse %s, seems to be no CTD/PARAMS" % (input_ctd) parsed_ctds.append(ParsedCTD(model, input_ctd, output_file)) @@ -209,8 +207,6 @@ def parse_hardcoded_parameters(hardcoded_parameters_file): parameter_hardcoder = ParameterHardcoder() if hardcoded_parameters_file is None: return parameter_hardcoder - - import json with open(hardcoded_parameters_file) as fp: data = json.load(fp) @@ -221,7 +217,7 @@ def parse_hardcoded_parameters(hardcoded_parameters_file): hardcoded_value = el.get("value", None) tool_names = el.get("tools", [None]) for tool_name in tool_names: - if not tool_name is None: + if tool_name is not None: tool_name = tool_name.strip() if hardcoded_value is not None: if hardcoded_value == '@': @@ -230,7 +226,7 @@ def parse_hardcoded_parameters(hardcoded_parameters_file): parameter_hardcoder.register_parameter(parameter_name, hardcoded_value, tool_name) else: for a in el: - if a in ["tools","value"]: + if a in ["tools", "value"]: continue if el[a] == "output-file": el[a] = _OutFile @@ -301,7 +297,7 @@ def _extract_and_flatten_parameters(parameter_group, nodes=False): """ get the parameters of a OptionGroup as generator """ - for parameter in parameter_group.itervalues(): + for parameter in parameter_group.values(): if type(parameter) is Parameter: yield parameter else: @@ -333,8 +329,6 @@ def extract_and_flatten_parameters(ctd_model, nodes=False): # for p in extract_and_flatten_parameters(parameter): # yield p -# -# # parameters = [] # if len(ctd_model.parameters.parameters) > 0: # # use this to put parameters that are to be processed @@ -365,8 +359,8 @@ def resolve_param_mapping(param, ctd_model): for mapping_element in cli_element.mappings: if mapping_element.reference_name == param.name: if param_mapping is not None: - warning("The parameter %s has more than one mapping in the section. " - "The first found mapping, %s, will be used." % (param.name, param_mapping), 1) + logger.warning("The parameter %s has more than one mapping in the section. " + "The first found mapping, %s, will be used." % (param.name, param_mapping), 1) else: param_mapping = cli_element.option_identifier @@ -382,7 +376,7 @@ def _extract_param_cli_name(param, ctd_model): return resolve_param_mapping(param, ctd_model) else: if hasattr(ctd_model, "cli") and ctd_model.cli: - warning("Using nested parameter sections (NODE elements) is not compatible with ", 1) + logger.warning("Using nested parameter sections (NODE elements) is not compatible with ", 1) return extract_param_name(param.parent) + ":" + resolve_param_mapping(param, ctd_model) else: return resolve_param_mapping(param, ctd_model) @@ -395,7 +389,7 @@ def extract_param_path(param): elif not hasattr(param.parent.parent.parent, "parent"): return [param.name] else: - return extract_param_path(param.parent) + [ param.name ] + return extract_param_path(param.parent) + [param.name] else: return [param.name] @@ -421,5 +415,4 @@ def indent(s, indentation=" "): @param indentation the desired indentation @return indented text """ - return [indentation+_ for _ in s] - + return [indentation + _ for _ in s] diff --git a/convert.py b/convert.py index c5992403..ec23474a 100644 --- a/convert.py +++ b/convert.py @@ -16,7 +16,7 @@ program_build_date = str(__updated__) program_version_message = '%%(prog)s %s (%s)' % (program_version, program_build_date) program_short_description = "CTDConverter - A project from the WorkflowConversion family " \ - "(https://github.com/WorkflowConversion/CTDConverter)" + "(https://github.com/WorkflowConversion/CTDConverter)" program_usage = ''' USAGE: @@ -173,7 +173,7 @@ def main(argv=None): # at this point we cannot parse the arguments, because each converter takes different arguments, meaning each # converter will register its own parameters after we've registered the basic ones... we have to do it old school if len(argv) < 2: - utils.error("Not enough arguments provided") + utils.logger.error("Not enough arguments provided") print("\nUsage: $ python convert.py [TARGET] [ARGUMENTS]\n\n" + "Where:\n" + " target: one of 'cwl' or 'galaxy'\n\n" + @@ -192,10 +192,10 @@ def main(argv=None): print(program_license) return 0 else: - utils.error("Unrecognized target engine. Supported targets are 'cwl' and 'galaxy'.") + utils.logger.error("Unrecognized target engine. Supported targets are 'cwl' and 'galaxy'.") return 1 - utils.info("Using %s converter" % target) + utils.logger.info("Using %s converter" % target) try: # Setup argument parser @@ -224,20 +224,20 @@ def main(argv=None): except ApplicationException as e: traceback.print_exc() - utils.error("CTDConverter could not complete the requested operation.", 0) - utils.error("Reason: " + e.msg, 0) + utils.logger.error("CTDConverter could not complete the requested operation.", 0) + utils.logger.error("Reason: " + e.msg, 0) return 1 except ModelError as e: traceback.print_exc() - utils.error("There seems to be a problem with one of your input CTDs.", 0) - utils.error("Reason: " + e.msg, 0) + utils.logger.error("There seems to be a problem with one of your input CTDs.", 0) + utils.logger.error("Reason: " + e.msg, 0) return 1 except Exception as e: traceback.print_exc() - utils.error("CTDConverter could not complete the requested operation.", 0) - utils.error("Reason: " + e.msg, 0) + utils.logger.error("CTDConverter could not complete the requested operation.", 0) + utils.logger.error("Reason: " + e.msg, 0) return 2 diff --git a/galaxy/converter.py b/galaxy/converter.py index 84dbe7c6..24694bec 100755 --- a/galaxy/converter.py +++ b/galaxy/converter.py @@ -7,8 +7,7 @@ from collections import OrderedDict import copy -from string import strip -from StringIO import StringIO +from io import StringIO from lxml import etree from lxml.etree import CDATA, SubElement, Element, ElementTree, ParseError, parse, strip_elements @@ -25,14 +24,14 @@ REQUIREMENTS_MACRO_NAME = "requirements" ADVANCED_OPTIONS_NAME = "adv_opts_" -REQUIRED_MACROS = [REQUIREMENTS_MACRO_NAME, STDIO_MACRO_NAME, ADVANCED_OPTIONS_NAME+"macro"] +REQUIRED_MACROS = [REQUIREMENTS_MACRO_NAME, STDIO_MACRO_NAME, ADVANCED_OPTIONS_NAME + "macro"] class ExitCode: def __init__(self, code_range="", level="", description=None): self.range = code_range self.level = level - self.description = description + self.description = description class DataType: @@ -52,48 +51,50 @@ def add_specific_args(parser): help="File containing the supported file formats. Run with '-h' or '--help' to see a " "brief example on the layout of this file.", default=None, required=False) parser.add_argument("-a", "--add-to-command-line", dest="add_to_command_line", - help="Adds content to the command line", default="", required=False) + help="Adds content to the command line", default="", required=False) parser.add_argument("-d", "--datatypes-destination", dest="data_types_destination", - help="Specify the location of a datatypes_conf.xml to modify and add the registered " - "data types. If the provided destination does not exist, a new file will be created.", - default=None, required=False) + help="Specify the location of a datatypes_conf.xml to modify and add the registered " + "data types. If the provided destination does not exist, a new file will be created.", + default=None, required=False) parser.add_argument("-c", "--default-category", dest="default_category", default="DEFAULT", required=False, - help="Default category to use for tools lacking a category when generating tool_conf.xml") + help="Default category to use for tools lacking a category when generating tool_conf.xml") parser.add_argument("-t", "--tool-conf-destination", dest="tool_conf_destination", default=None, required=False, - help="Specify the location of an existing tool_conf.xml that will be modified to include " - "the converted tools. If the provided destination does not exist, a new file will" - "be created.") + help="Specify the location of an existing tool_conf.xml that will be modified to include " + "the converted tools. If the provided destination does not exist, a new file will" + "be created.") parser.add_argument("-g", "--galaxy-tool-path", dest="galaxy_tool_path", default=None, required=False, - help="The path that will be prepended to the file names when generating tool_conf.xml") + help="The path that will be prepended to the file names when generating tool_conf.xml") parser.add_argument("-r", "--required-tools", dest="required_tools_file", default=None, required=False, - help="Each line of the file will be interpreted as a tool name that needs translation. " - "Run with '-h' or '--help' to see a brief example on the format of this file.") + help="Each line of the file will be interpreted as a tool name that needs translation. " + "Run with '-h' or '--help' to see a brief example on the format of this file.") parser.add_argument("-s", "--skip-tools", dest="skip_tools_file", default=None, required=False, - help="File containing a list of tools for which a Galaxy stub will not be generated. " - "Run with '-h' or '--help' to see a brief example on the format of this file.") + help="File containing a list of tools for which a Galaxy stub will not be generated. " + "Run with '-h' or '--help' to see a brief example on the format of this file.") parser.add_argument("-m", "--macros", dest="macros_files", default=[], nargs="*", - action="append", required=None, help="Import the additional given file(s) as macros. " - "The macros stdio, requirements and advanced_options are " - "required. Please see galaxy/macros.xml for an example of a " - "valid macros file. All defined macros will be imported.") + action="append", required=None, help="Import the additional given file(s) as macros. " + "The macros stdio, requirements and advanced_options are " + "required. Please see galaxy/macros.xml for an example of a " + "valid macros file. All defined macros will be imported.") parser.add_argument("--test-macros", dest="test_macros_files", default=[], nargs="*", - action="append", required=None, help="Import tests from the files given file(s) as macros. " - "The macro names must end with the id of the tools") + action="append", required=None, + help="Import tests from the files given file(s) as macros. " + "The macro names must end with the id of the tools") parser.add_argument("--test-macros-prefix", dest="test_macros_prefix", default=[], nargs="*", - action="append", required=None, help="The prefix of the macro name in the corresponding trest macros file") + action="append", required=None, help="The prefix of the macro name in the corresponding trest macros file") parser.add_argument("--test-test", dest="test_test", action='store_true', default=False, required=False, - help="Generate a simple test for the internal unit tests.") + help="Generate a simple test for the internal unit tests.") parser.add_argument("--test-only", dest="test_only", action='store_true', default=False, required=False, - help="Generate only the test section.") - parser.add_argument("--tool-version", dest="tool_version", required=False, default = None, - help="Tool version to use (if not given its extracted from the CTD)") + help="Generate only the test section.") + parser.add_argument("--tool-version", dest="tool_version", required=False, default=None, + help="Tool version to use (if not given its extracted from the CTD)") + def modify_param_for_galaxy(param): """ some parameters need galaxy specific modifications """ if param.type is _InFile: - #if file default is given (happens for external applications and + # if file default is given (happens for external applications and # files for which the default is taken from share/OpenMS) set the # parm to not required and remove the default (external applications # need to be taken care by hardcoded values and the other cases @@ -103,25 +104,26 @@ def modify_param_for_galaxy(param): param.default = _Null return param + def convert_models(args, parsed_ctds): """ main conversion function @param args command line arguments - @param parsed_ctds the ctds + @param parsed_ctds the ctds """ # validate and prepare the passed arguments validate_and_prepare_args(args, parsed_ctds[0].ctd_model) - + # parse the given supported file-formats file supported_file_formats = parse_file_formats(args.formats_file) - + # extract the names of the macros and check that we have found the ones we need macros_to_expand = parse_macros_files(args.macros_files, tool_version=args.tool_version, supported_file_types=supported_file_formats, required_macros=REQUIRED_MACROS, - dont_expand=[ADVANCED_OPTIONS_NAME+"macro", "references", + dont_expand=[ADVANCED_OPTIONS_NAME + "macro", "references", "list_string_val", "list_string_san", "list_float_valsan", "list_integer_valsan"]) @@ -143,11 +145,7 @@ def convert_models(args, parsed_ctds): test_only=args.test_only, test_macros_file_names=args.test_macros_files, test_macros_prefix=args.test_macros_prefix, - tool_version = args.tool_version) - - - - + tool_version=args.tool_version) # generation of galaxy stubs is ready... now, let's see if we need to generate a tool_conf.xml # TODO remove tool conf .. deprecated # if args.tool_conf_destination is not None: @@ -176,7 +174,7 @@ def parse_tools_list_file(tools_list_file): return tools_list -def parse_macros_files(macros_file_names, tool_version, supported_file_types, required_macros = [], dont_expand=[]): +def parse_macros_files(macros_file_names, tool_version, supported_file_types, required_macros=[], dont_expand=[]): """ """ macros_to_expand = [] @@ -189,15 +187,14 @@ def parse_macros_files(macros_file_names, tool_version, supported_file_types, re name = xml_element.attrib["name"] if name in macros_to_expand: logger.warning("Macro %s has already been found. Duplicate found in file %s." % - (name, macros_file_name), 0) + (name, macros_file_name), 0) continue logger.info("Macro %s found" % name, 1) macros_to_expand.append(name) except ParseError as e: raise ApplicationException("The macros file " + macros_file_name + " could not be parsed. Cause: " + str(e)) - - + except IOError as e: raise ApplicationException("The macros file " + macros_file_name + " could not be opened. Cause: " + str(e)) @@ -227,7 +224,7 @@ def parse_macros_files(macros_file_names, tool_version, supported_file_types, re # make sure that the backup data type is in the map if 'txt' not in g2o: g2o['txt'] = 'txt' - + ext_foo.text = CDATA("""#def oms2gxyext(o) #set m=%s #return m[o] @@ -236,11 +233,13 @@ def parse_macros_files(macros_file_names, tool_version, supported_file_types, re #set m=%s #return m[g] #end def -"""%(str(o2g), str(g2o))) +""" % (str(o2g), str(g2o))) - with open(macros_file_name, "w") as macros_file: - tree = ElementTree(root) - tree.write(macros_file, encoding="UTF-8", xml_declaration=True, pretty_print=True) + tree = ElementTree(root) + tree.write(macros_file_name, encoding="UTF-8", xml_declaration=True, pretty_print=True) +# with open(macros_file_name, "w") as macros_file: +# tree = ElementTree(root) +# tree.write(macros_file, encoding="UTF-8", xml_declaration=True, pretty_print=True) # we depend on "stdio", "requirements" and "advanced_options" to exist on all the given macros files missing_needed_macros = [] @@ -253,7 +252,7 @@ def parse_macros_files(macros_file_names, tool_version, supported_file_types, re "The following required macro(s) were not found in any of the given macros files: %s, " "see galaxy/macros.xml for an example of a valid macros file." % ", ".join(missing_needed_macros)) - + # remove macros that should not be expanded for m in dont_expand: try: @@ -266,11 +265,11 @@ def parse_macros_files(macros_file_names, tool_version, supported_file_types, re def check_test_macros(test_macros_files, test_macros_prefix, parsed_ctds): - + tool_ids = set() for parsed_ctd in parsed_ctds: model = parsed_ctd.ctd_model - tool_ids.add( model.name.replace(" ","_") ) + tool_ids.add(model.name.replace(" ", "_")) for mf, mp in zip(test_macros_files, test_macros_prefix): macro_ids = set() @@ -280,7 +279,7 @@ def check_test_macros(test_macros_files, test_macros_prefix, parsed_ctds): for xml_element in root.findall("xml"): name = xml_element.attrib["name"] if not name.startswith(mp): - logger.warning("Testmacro with invalid prefix %s." %(mp), 0) + logger.warning("Testmacro with invalid prefix %s." % (mp), 0) continue name = name[len(mp):] macro_ids.add(name) @@ -292,11 +291,11 @@ def check_test_macros(test_macros_files, test_macros_prefix, parsed_ctds): raise ApplicationException("The macros file " + mf + " could not be opened. Cause: " + str(e)) for t in tool_ids - macro_ids: - logger.error("missing %s"%t) - add_child_node(root, "xml", OrderedDict([("name", mp+t)])) + logger.error("missing %s" % t) + add_child_node(root, "xml", OrderedDict([("name", mp + t)])) - if len( macro_ids - tool_ids): - logger.warning("Unnecessary macros in %s: %s"%(mf, macro_ids - tool_ids)) + if len(macro_ids - tool_ids): + logger.warning("Unnecessary macros in %s: %s" % (mf, macro_ids - tool_ids)) with open(mf, "w") as macros_file: tree = ElementTree(root) tree.write(macros_file, encoding="UTF-8", xml_declaration=True, pretty_print=True) @@ -340,7 +339,7 @@ def parse_file_formats(formats_file): def get_fileformat_maps(supported_formats): """ - convenience functions to compute dictionaries mapping + convenience functions to compute dictionaries mapping Galaxy data types <-> CTD formats """ o2g = {} @@ -384,11 +383,12 @@ def validate_and_prepare_args(args, model): if not args.macros_files: # list is empty, provide the default value logger.warning("Using default macros from galaxy/macros.xml", 0) - args.macros_files = [os.path.dirname(os.path.abspath( __file__ ))+"/macros.xml"] + args.macros_files = [os.path.dirname(os.path.abspath(__file__)) + "/macros.xml"] - if args.tool_version == None: + if args.tool_version is None: args.tool_version = model.version + def get_preferred_file_extension(): """ get the file extension for the output files @@ -434,11 +434,11 @@ def _convert_internal(parsed_ctds, **kwargs): # overwrite attributes of the parsed ctd parameters as specified in hardcoded parameterd json for param in utils.extract_and_flatten_parameters(model): hardcoded_attributes = parameter_hardcoder.get_hardcoded_attributes(param.name, model.name) - if hardcoded_attributes != None: + if hardcoded_attributes is not None: for a in hardcoded_attributes: if not hasattr(param, a): continue - # logger.error("%s %s"%(param.name,type(getattr(param, a))) ) + # logger.error("%s %s"%(param.name, type(getattr(param, a))) ) if type(getattr(param, a)) is _FileFormat: setattr(param, a, _FileFormat(str(hardcoded_attributes[a]))) elif type(getattr(param, a)) is _Choices: @@ -448,7 +448,7 @@ def _convert_internal(parsed_ctds, **kwargs): raise Exception("Overwriting of Numeric Range not implemented") else: setattr(param, a, hardcoded_attributes[a]) - + logger.info("Converting %s (source %s)" % (model.name, utils.get_filename(origin_file)), 0) tool = create_tool(model) write_header(tool, model) @@ -472,8 +472,7 @@ def _convert_internal(parsed_ctds, **kwargs): # wrap our tool element into a tree to be able to serialize it tree = ElementTree(tool) logger.info("Writing to %s" % utils.get_filename(output_file), 1) - with open(output_file, 'w') as fh: - tree.write(fh, encoding="UTF-8", xml_declaration=True, pretty_print=True) + tree.write(output_file, encoding="UTF-8", xml_declaration=True, pretty_print=True) def write_header(tool, model): @@ -501,23 +500,23 @@ def generate_tool_conf(parsed_ctds, tool_conf_destination, galaxy_tool_path, def # at this point, we should have a map for all categories->tools toolbox_node = Element("toolbox") - + if galaxy_tool_path is not None and not galaxy_tool_path.strip().endswith("/"): galaxy_tool_path = galaxy_tool_path.strip() + "/" if galaxy_tool_path is None: galaxy_tool_path = "" - + for category, file_names in categories_to_tools.iteritems(): section_node = add_child_node(toolbox_node, "section") section_node.attrib["id"] = "section-id-" + "".join(category.split()) section_node.attrib["name"] = category - + for filename in file_names: tool_node = add_child_node(section_node, "tool") tool_node.attrib["file"] = galaxy_tool_path + filename toolconf_tree = ElementTree(toolbox_node) - toolconf_tree.write(open(tool_conf_destination,'w'), encoding="UTF-8", xml_declaration=True, pretty_print=True) + toolconf_tree.write(open(tool_conf_destination, 'w'), encoding="UTF-8", xml_declaration=True, pretty_print=True) logger.info("Generated Galaxy tool_conf.xml in %s" % tool_conf_destination, 0) @@ -539,7 +538,7 @@ def generate_data_type_conf(supported_file_formats, data_types_destination): data_type_node.attrib["mimetype"] = data_type.mimetype data_types_tree = ElementTree(data_types_node) - data_types_tree.write(open(data_types_destination,'w'), encoding="UTF-8", xml_declaration=True, pretty_print=True) + data_types_tree.write(open(data_types_destination, 'w'), encoding="UTF-8", xml_declaration=True, pretty_print=True) logger.info("Generated Galaxy datatypes_conf.xml in %s" % data_types_destination, 0) @@ -549,7 +548,7 @@ def create_tool(model): @param model the ctd model """ - return Element("tool", OrderedDict([("id", model.name.replace(" ","_")), ("name", model.name), ("version", "@TOOL_VERSION@+galaxy@GALAXY_VERSION@")])) + return Element("tool", OrderedDict([("id", model.name.replace(" ", "_")), ("name", model.name), ("version", "@TOOL_VERSION@+galaxy@GALAXY_VERSION@")])) def create_description(tool, model): @@ -559,21 +558,24 @@ def create_description(tool, model): @param model the ctd model """ if "description" in model.opt_attribs.keys() and model.opt_attribs["description"] is not None: - description = SubElement(tool,"description") + description = SubElement(tool, "description") description.text = model.opt_attribs["description"] -def get_by_path(dataDict, mapList): - for k in mapList: dataDict = dataDict[k] +def get_by_path(dataDict, mapList): + for k in mapList: + dataDict = dataDict[k] return dataDict + def set_by_path(dic, keys, value): for key in keys[:-1]: dic = dic.setdefault(key, {}) dic[keys[-1]] = value + def create_configfile(tool, model, **kwargs): - + configfiles_node = add_child_node(tool, "configfiles") inputs_node = add_child_node(configfiles_node, "inputs", OrderedDict([("name", "args_json"), ("data_style", "paths")])) @@ -588,7 +590,7 @@ def create_command(tool, model, **kwargs): # main command final_cmd = OrderedDict([('preprocessing', []), ('command', []), ('postprocessing', [])]) advanced_cmd = {'preprocessing': [], 'command': [], 'postprocessing': []} - + final_cmd['preprocessing'].extend(["@QUOTE_FOO@", "@EXT_FOO@", "#import re", "", "## Preprocessing"]) # - call the executable with -write_ctd to write the ctd file (with defaults) @@ -597,15 +599,15 @@ def create_command(tool, model, **kwargs): # (note the base name call is only necessary for the CTDconverter tests) # - feed the ctd file to the executable (with -ini) # note: input and output file parameters are still given on the command line - # - output file parameters are not included in the JSON file + # - output file parameters are not included in the JSON file # - input and output files are accessed through links / files that have the correct extension final_cmd['command'].extend(["", "## Main program call"]) final_cmd['command'].append("""@EXECUTABLE@ -write_ctd ./ && python '$__tool_directory__/fill_ctd.py' "\$(basename "@EXECUTABLE@.ctd")" '$args_json' && -@EXECUTABLE@ -ini @EXECUTABLE@.ctd""") +@EXECUTABLE@ -ini @EXECUTABLE@.ctd""") final_cmd['command'].extend(kwargs["add_to_command_line"]) final_cmd['postprocessing'].extend(["", "## Postprocessing"]) - + advanced_command_start = "#if ${aon}cond.{aon}selector=='advanced':".format(aon=ADVANCED_OPTIONS_NAME) advanced_command_end = "#end if" @@ -617,7 +619,7 @@ def create_command(tool, model, **kwargs): param = modify_param_for_galaxy(param) if param.type is _OutFile: found_output_parameter = True - + param_cmd = {'preprocessing': [], 'command': [], 'postprocessing': []} param_name = utils.extract_param_name(param) command_line_prefix = utils.extract_command_line_prefix(param, model) @@ -628,7 +630,7 @@ def create_command(tool, model, **kwargs): hardcoded_value = parameter_hardcoder.get_hardcoded_value(param_name, model.name) if hardcoded_value is not None: - pass # TODO hardcoded values should go to + pass # TODO hardcoded values should go to # param_cmd['command'].append("%s %s" % (command_line_prefix, hardcoded_value)) else: # in the else branch the parameter is neither blacklisted nor hardcoded... @@ -637,20 +639,20 @@ def create_command(tool, model, **kwargs): # all but bool params need the command line argument (bools have it already in the true/false value) if param.type is _OutFile or param.type is _InFile: param_cmd['command'].append(command_line_prefix) - - # preprocessing for file inputs: + + # preprocessing for file inputs: # - create a dir with name param.name # - create a link to id.ext in this directory # rationale: in the autogenerated tests the same file was used as input to multiple parameters - # this leads to conflicts while linking... might also be better in general + # this leads to conflicts while linking... might also be better in general if param.type is _InFile: - param_cmd['preprocessing'].append("mkdir %s &&" %param.name) + param_cmd['preprocessing'].append("mkdir %s &&" % param.name) if param.is_list: - param_cmd['preprocessing'].append("${ ' '.join([ \"ln -s '%s' '"+param.name+"/%s.%s' &&\" % (_, re.sub('[^\w\-_]', '_', _.element_identifier),$gxy2omsext(_.ext)) for _ in $" + param.name + " if _ ]) }") - param_cmd['command'].append("${' '.join([\"'"+param.name+"/%s.%s'\"%(re.sub('[^\w\-_]', '_', _.element_identifier),$gxy2omsext(_.ext)) for _ in $" + param.name + " if _ ])}") + param_cmd['preprocessing'].append("${ ' '.join([\"ln -s '%s' '" + param.name + "/%s.%s' &&\" % (_, re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $" + param.name + " if _]) }") + param_cmd['command'].append("${' '.join([\"'" + param.name + "/%s.%s'\"%(re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $" + param.name + " if _])}") else: - param_cmd['preprocessing'].append("ln -s '$"+ param.name +"' '"+param.name+"/${re.sub(\"[^\w\-_]\", \"_\", $"+param.name+".element_identifier)}.$gxy2omsext($"+param.name+".ext)' &&") - param_cmd['command'].append("'"+param.name+"/${re.sub(\"[^\w\-_]\", \"_\", $"+param.name+".element_identifier)}.$gxy2omsext($"+param.name+".ext)'") + param_cmd['preprocessing'].append("ln -s '$" + param.name + "' '" + param.name + "/${re.sub(\"[^\w\-_]\", \"_\", $" + param.name + ".element_identifier)}.$gxy2omsext($" + param.name + ".ext)' &&") + param_cmd['command'].append("'" + param.name + "/${re.sub(\"[^\w\-_]\", \"_\", $" + param.name + ".element_identifier)}.$gxy2omsext($" + param.name + ".ext)'") elif param.type is _OutFile: # check if there is a parameter that sets the format # if so we add an extension to the generated files which will be used to @@ -660,15 +662,15 @@ def create_command(tool, model, **kwargs): formats = get_formats(param, supported_file_formats, TYPE_TO_GALAXY_TYPE[param.type]) type_param = get_out_type_param(param, model, parameter_hardcoder) corresponding_input, fmt_from_corresponding = get_corresponding_input(param, model) - if corresponding_input is not None : + if corresponding_input is not None: actual_input_parameter = get_galaxy_parameter_path(corresponding_input) else: actual_input_parameter = None if type_param is not None: type_param_name = get_galaxy_parameter_path(type_param) - elif len(formats) > 1 and (corresponding_input == None or not fmt_from_corresponding) and not param.is_list: - type_param_name = get_galaxy_parameter_path(param, "type") + elif len(formats) > 1 and (corresponding_input is None or not fmt_from_corresponding) and not param.is_list: + type_param_name = get_galaxy_parameter_path(param, "type") else: type_param_name = None @@ -677,50 +679,50 @@ def create_command(tool, model, **kwargs): # if there is only one format (the outoput node sets format using the format attribute of the data/discover node) # - single file: write to temp file with oms extension and move this to the actual result file # - lists: write to files with the oms extension and remove the extension afterwards (discovery with __name__) - if len(formats)==1: + if len(formats) == 1: fmt = formats.pop() if param.is_list: logger.info("1 fmt + list %s -> %s" % (param.name, actual_input_parameter), 1) - param_cmd['command'].append("${' '.join([\"'"+ param.name +"/%s.%s'\"%(re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(\""+ fmt +"\")) for _ in $" + actual_input_parameter + " if _ ])}") - param_cmd['postprocessing'].append("${' '.join([\"&& mv -n '"+ param.name +"/%(id)s.%(gext)s' '" +param.name+ "/%(id)s'\"%{\"id\": re.sub('[^\w\-_]', '_', _.element_identifier), \"gext\": $gxy2omsext(\""+fmt+"\")} for _ in $" + actual_input_parameter + " if _ ])}") + param_cmd['command'].append("${' '.join([\"'" + param.name + "/%s.%s'\"%(re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(\"" + fmt + "\")) for _ in $" + actual_input_parameter + " if _])}") + param_cmd['postprocessing'].append("${' '.join([\"&& mv -n '" + param.name + "/%(id)s.%(gext)s' '" + param.name + "/%(id)s'\"%{\"id\": re.sub('[^\w\-_]', '_', _.element_identifier), \"gext\": $gxy2omsext(\"" + fmt + "\")} for _ in $" + actual_input_parameter + " if _])}") else: logger.info("1 fmt + dataset %s" % param.name, 1) - param_cmd['command'].append("'"+param.name+"/output.${gxy2omsext(\"" + fmt + "\")}'") - param_cmd['postprocessing'].append("&& mv '"+param.name+"/output.${gxy2omsext(\"" + fmt + "\")}' '$"+param.name+"'") + param_cmd['command'].append("'" + param.name + "/output.${gxy2omsext(\"" + fmt + "\")}'") + param_cmd['postprocessing'].append("&& mv '" + param.name + "/output.${gxy2omsext(\"" + fmt + "\")}' '$" + param.name + "'") # if there is a type parameter then we use the type selected by the user # - single: write to temp file with the oms extension and mv it to the actual file output which is treated via change_format # - list: let the command create output files with the oms extensions, postprocessing renames them to the galaxy extensions, output is then discover + __name_and_ext__ elif type_param_name is not None: if param.is_list: - logger.info("type + list %s" %param.name, 1) - param_cmd['command'].append("${' '.join([\"'"+ param.name +"/%s.%s'\"%(re.sub('[^\w\-_]', '_', _.element_identifier), "+type_param_name+") for _ in $" + actual_input_parameter + " if _ ])}") - param_cmd['postprocessing'].append("${' '.join([\"&& mv -n '"+ param.name +"/%(id)s.%(omsext)s' '" +param.name+ "/%(id)s.%(gext)s'\"%{\"id\": re.sub('[^\w\-_]', '_', _.element_identifier), \"omsext\":"+type_param_name+", \"gext\": oms2gxyext(\""+type_param_name+"\")} for _ in $" + actual_input_parameter + " if _ ])}") + logger.info("type + list %s" % param.name, 1) + param_cmd['command'].append("${' '.join([\"'" + param.name + "/%s.%s'\"%(re.sub('[^\w\-_]', '_', _.element_identifier), " + type_param_name + ") for _ in $" + actual_input_parameter + " if _])}") + param_cmd['postprocessing'].append("${' '.join([\"&& mv -n '" + param.name + "/%(id)s.%(omsext)s' '" + param.name + "/%(id)s.%(gext)s'\"%{\"id\": re.sub('[^\w\-_]', '_', _.element_identifier), \"omsext\":" + type_param_name + ", \"gext\": oms2gxyext(\"" + type_param_name + "\")} for _ in $" + actual_input_parameter + " if _])}") else: - logger.info("type + dataset %s" %param.name, 1) + logger.info("type + dataset %s" % param.name, 1) # 1st create file with openms extension (often required by openms) # then move it to the actual place specified by the parameter # the format is then set by the tag using - param_cmd['command'].append("'" + param.name+"/output.${"+type_param_name+"}'") - param_cmd['postprocessing'].append("&& mv '"+param.name+"/output.${"+type_param_name+"}' '$"+actual_parameter+"'") + param_cmd['command'].append("'" + param.name + "/output.${" + type_param_name + "}'") + param_cmd['postprocessing'].append("&& mv '" + param.name + "/output.${" + type_param_name + "}' '$" + actual_parameter + "'") elif actual_input_parameter is not None: if param.is_list: - logger.info("actual + list %s" %param.name, 1) - param_cmd['command'].append("${' '.join([\"'"+ param.name +"/%s.%s'\"%(re.sub('[^\w\-_]', '_', _.element_identifier), _.ext) for _ in $" + actual_input_parameter + " if _ ])}") + logger.info("actual + list %s" % param.name, 1) + param_cmd['command'].append("${' '.join([\"'" + param.name + "/%s.%s'\"%(re.sub('[^\w\-_]', '_', _.element_identifier), _.ext) for _ in $" + actual_input_parameter + " if _])}") else: logger.info("actual + dataset %s %s %s" % (param.name, actual_input_parameter, corresponding_input.is_list), 1) if corresponding_input.is_list: - param_cmd['command'].append("'" + param.name+"/output.${"+actual_input_parameter+"[0].ext}'") - param_cmd['postprocessing'].append("&& mv '"+param.name+"/output.${"+actual_input_parameter+"[0].ext}' '$"+actual_parameter+"'") + param_cmd['command'].append("'" + param.name + "/output.${" + actual_input_parameter + "[0].ext}'") + param_cmd['postprocessing'].append("&& mv '" + param.name + "/output.${" + actual_input_parameter + "[0].ext}' '$" + actual_parameter + "'") else: - param_cmd['command'].append("'" + param.name+"/output.${"+actual_input_parameter+".ext}'") - param_cmd['postprocessing'].append("&& mv '"+param.name+"/output.${"+actual_input_parameter+".ext}' '$"+actual_parameter+"'") + param_cmd['command'].append("'" + param.name + "/output.${" + actual_input_parameter + ".ext}'") + param_cmd['postprocessing'].append("&& mv '" + param.name + "/output.${" + actual_input_parameter + ".ext}' '$" + actual_parameter + "'") else: if param.is_list: - raise Exception( "output parameter itemlist %s without corresponding input" ) + raise Exception("output parameter itemlist %s without corresponding input") else: logger.info("else + dataset %s" % param.name, 1) - param_cmd['command'].append("'$out_" +param.name+ "'") + param_cmd['command'].append("'$out_" + param.name + "'") # # select with multiple = true # elif is_selection_parameter(param) and param.is_list: @@ -732,20 +734,20 @@ def create_command(tool, model, **kwargs): # param_cmd['command'].append("$%s" % actual_parameter + "") # else: # param_cmd['command'].append('"$' + actual_parameter + '"') - + # add if statement for optional parameters and preprocessing # - for optional outputs (param_out_x) the presence of the parameter # depends on the additional input (param_x) -> need no if # - real string parameters (i.e. ctd type string wo restrictions) also # need no if (otherwise the empty string could not be provided) - if not ( param.required or is_boolean_parameter(param) or (param.type == str and param.restrictions is None)): - # and not(param.type is _InFile and param.is_list): + if not (param.required or is_boolean_parameter(param) or (param.type == str and param.restrictions is None)): + # and not(param.type is _InFile and param.is_list): actual_parameter = get_galaxy_parameter_path(param, "FLAG") for stage in param_cmd: if len(param_cmd[stage]) == 0: continue # special case for optional itemlists: for those if no option is selected only the parameter must be specified - if is_selection_parameter(param) and param.is_list and param.required == False: + if is_selection_parameter(param) and param.is_list and param.required is False: param_cmd[stage] = [param_cmd[stage][0]] + ["#if $" + actual_parameter + ":"] + utils.indent(param_cmd[stage][1:]) + ["#end if"] elif is_selection_parameter(param) or param.type is _OutFile or param.type is _InFile: param_cmd[stage] = ["#if $" + actual_parameter + ":"] + utils.indent(param_cmd[stage]) + ["#end if"] @@ -761,7 +763,7 @@ def create_command(tool, model, **kwargs): final_cmd[stage].extend(param_cmd[stage]) for stage in advanced_cmd: - if len(advanced_cmd[stage])==0: + if len(advanced_cmd[stage]) == 0: continue advanced_cmd[stage] = [advanced_command_start] + utils.indent(advanced_cmd[stage]) + [advanced_command_end] final_cmd[stage].extend(advanced_cmd[stage]) @@ -772,12 +774,12 @@ def create_command(tool, model, **kwargs): command_node = add_child_node(tool, "command") command_node.attrib["detect_errors"] = "exit_code" - command_node.text = CDATA("\n".join(sum(final_cmd.values(), []))) -# creates the xml elements needed to import the needed macros files + def import_macros(tool, model, **kwargs): """ + creates the xml elements needed to import the needed macros files @param tool the Galaxy tool @param model the ctd model @param kwargs @@ -788,7 +790,7 @@ def import_macros(tool, model, **kwargs): token_node.text = utils.extract_tool_executable_path(model, kwargs["default_executable_path"]) # add nodes - for macro_file_name in kwargs["macros_file_names"]+kwargs["test_macros_file_names"]: + for macro_file_name in kwargs["macros_file_names"] + kwargs["test_macros_file_names"]: macro_file = open(macro_file_name) import_node = add_child_node(macros_node, "import") # do not add the path of the file, rather, just its basename @@ -809,25 +811,25 @@ def expand_macros(node, macros_to_expand): expand_node.attrib["macro"] = expand_macro -def get_galaxy_parameter_path(param, inparam = False): +def get_galaxy_parameter_path(param, inparam=False): """ get the complete cheetah path for a parameter """ p = get_galaxy_parameter_name(param, inparam) path = utils.extract_param_path(param) - if len(path) > 1 and (not param.type is _OutFile or inparam): - return (".".join( [ "section_"+_ for _ in path[:-1]] )+"."+p).replace("-", "_") - elif param.advanced and (not param.type is _OutFile or inparam): - return ADVANCED_OPTIONS_NAME+"cond." + p + if len(path) > 1 and (param.type is not _OutFile or inparam): + return (".".join(["section_" + _ for _ in path[:-1]]) + "." + p).replace("-", "_") + elif param.advanced and (param.type is not _OutFile or inparam): + return ADVANCED_OPTIONS_NAME + "cond." + p else: return p -def get_galaxy_parameter_name(param, inparam = False): +def get_galaxy_parameter_name(param, inparam=False): """ get the name of the parameter used in the galaxy tool - replace : and - by _ - - add _FLAG for output parameters if inparam is True + - add value inpram for output parameters if not False the idea of inparam is to be used for optional outputs (out_x) for which an additional boolean input (out_x_FLAG) exists @@ -859,7 +861,7 @@ def get_out_type_param(out_param, model, parameter_hardcoder): def is_out_type_param(param, model): """ - check if the parameter is output_type parameter + check if the parameter is output_type parameter - the name ends with _type and there is an output parameter without this suffix and return True iff this is the case """ @@ -884,7 +886,7 @@ def get_corresponding_input(out_param, model): return the found input parameter and True iff the 1st case applied """ c = get_input_with_same_restrictions(out_param, model, True) - if c == None: + if c is None: return (get_input_with_same_restrictions(out_param, model, False), False) else: return (c, True) @@ -897,10 +899,10 @@ def get_input_with_same_restrictions(out_param, model, check_formats): """ matching = [] - + for allow_different_type in [False, True]: for param in utils.extract_and_flatten_parameters(model): - if not param.type is _InFile: + if param.type is not _InFile: continue # logger.error("%s %s %s %s %s %s" %(out_param.name, param.name, param.is_list, out_param.is_list, param.restrictions, out_param.restrictions)) if allow_different_type or param.is_list == out_param.is_list: @@ -911,7 +913,7 @@ def get_input_with_same_restrictions(out_param, model, check_formats): matching.append(param) else: matching.append(param) -# logger.error("match %s "%([ _.name for _ in matching])) +# logger.error("match %s "%([_.name for _ in matching])) if len(matching) > 0: break if len(matching) == 1: @@ -947,22 +949,22 @@ def create_inputs(tool, model, **kwargs): param = modify_param_for_galaxy(param) # no need to show hardcoded parameters hardcoded_value = parameter_hardcoder.get_hardcoded_value(utils.extract_param_name(param), model.name) - if parameter_hardcoder.get_blacklist(utils.extract_param_name(param), model.name) or not hardcoded_value is None: + if parameter_hardcoder.get_blacklist(utils.extract_param_name(param), model.name) or hardcoded_value is not None: continue - - if section_nodes.has_key(utils.extract_param_name(param.parent)): + + if utils.extract_param_name(param.parent) in section_nodes: parent_node = section_nodes[utils.extract_param_name(param.parent)] elif param.advanced: if advanced_node is None: - advanced_node = Element("expand", OrderedDict([("macro", ADVANCED_OPTIONS_NAME+"macro")])) + advanced_node = Element("expand", OrderedDict([("macro", ADVANCED_OPTIONS_NAME + "macro")])) parent_node = advanced_node else: parent_node = inputs_node - # sometimes special inputs are needed for outfiles: + # sometimes special inputs are needed for outfiles: add_formats = False if param.type is _OutFile: - # if there are multiple possible output formats, but no parameter to choose the type or a + # if there are multiple possible output formats, but no parameter to choose the type or a # corresponding input then add a selection parameter formats = get_formats(param, kwargs["supported_file_formats"], TYPE_TO_GALAXY_TYPE[_OutFile]) type_param = get_out_type_param(param, model, parameter_hardcoder) @@ -977,31 +979,32 @@ def create_inputs(tool, model, **kwargs): # mandatory outpiles: no input node needed # inputs: create the input param if not param.required: - add_child_node(parent_node, "param", OrderedDict([("type", "boolean"), ("name", param.name + "_FLAG"), ("optional", "false"), ("checked", "false"), ("label", "Generate output %s (%s)" %(param.name, param.description))])) + add_child_node(parent_node, "param", OrderedDict([("type", "boolean"), ("name", param.name + "_FLAG"), ("optional", "false"), ("checked", "false"), ("label", "Generate output %s (%s)" % (param.name, param.description))])) # else: # add_child_node(parent_node, "param", OrderedDict([("type", "hidden"), ("name", param.name)])) continue - + # create the actual param node and fill the attributes param_node = add_child_node(parent_node, "param") create_param_attribute_list(param_node, param, model, kwargs["supported_file_formats"]) hardcoded_attributes = parameter_hardcoder.get_hardcoded_attributes(param.name, model.name) - if hardcoded_attributes != None: + if hardcoded_attributes is not None: for a in hardcoded_attributes: param_node.attrib[a] = str(hardcoded_attributes[a]) for sn in section_nodes: if utils.extract_param_name(section_params[sn].parent) in section_nodes: - section_nodes[ utils.extract_param_name(section_params[sn].parent) ].append(section_nodes[sn]) + section_nodes[utils.extract_param_name(section_params[sn].parent)].append(section_nodes[sn]) else: inputs_node.append(section_nodes[sn]) # if there is an advanced section then append it at the end of the inputs - if not advanced_node is None: + if advanced_node is not None: inputs_node.append(advanced_node) return inputs_node + def is_default(value, param): """ check if the value is the default of the param or if the value is in the defaults of param @@ -1009,7 +1012,7 @@ def is_default(value, param): return param.default == value or (type(param.default) is list and value in param.default) -def get_formats(param, supported_file_formats, default = None): +def get_formats(param, supported_file_formats, default=None): """ determine format attribute - check if all listed possible formats are supported in Galaxy @@ -1026,22 +1029,22 @@ def get_formats(param, supported_file_formats, default = None): output.append(str(format_name)) # warn only if there's about to complain if output: - logger.warning("Parameter " + param.name + " has the following unsupported format(s):" - + ','.join(output), 1) - + logger.warning("Parameter " + param.name + " has the following unsupported format(s):" + + ','.join(output), 1) + formats = get_supported_file_types(param.restrictions.formats, supported_file_formats) else: raise InvalidModelException("Unrecognized restriction type [%(type)s] " "for [%(name)s]" % {"type": type(param.restrictions), - "name": param.name}) + "name": param.name}) if len(formats) == 0: - if default != None: + if default is not None: formats.add(default) else: raise InvalidModelException("No supported formats [%(type)s] " "for [%(name)s]" % {"type": type(param.restrictions), - "name": param.name}) + "name": param.name}) return formats @@ -1049,17 +1052,17 @@ def get_formats(param, supported_file_formats, default = None): def create_param_attribute_list(param_node, param, model, supported_file_formats): """ get the attributes of input parameters - @param param_node the galaxy tool param node - @param param the ctd parameter + @param param_node the galaxy tool param node + @param param the ctd parameter @param supported_file_formats """ g2o, o2g = get_fileformat_maps(supported_file_formats) - # set the name, argument and a first guess for the type (which will be over written + # set the name, argument and a first guess for the type (which will be over written # in some cases .. see below) param_node.attrib["name"] = param.name - param_node.attrib["argument"] = "-%s"%utils.extract_param_name(param) + param_node.attrib["argument"] = "-%s" % utils.extract_param_name(param) param_type = TYPE_TO_GALAXY_TYPE[param.type] if param_type is None: raise ModelError("Unrecognized parameter type %(type)s for parameter %(name)s" @@ -1076,10 +1079,10 @@ def create_param_attribute_list(param_node, param, model, supported_file_formats param_node.attrib["display"] = "radio" if param.is_list: param_node.attrib["multiple"] = "true" - + if is_boolean_parameter(param): param_type = "boolean" - + if param.type is _InFile: # assume it's just text unless restrictions are provided param_node.attrib["type"] = "data" @@ -1104,12 +1107,12 @@ def create_param_attribute_list(param_node, param, model, supported_file_formats # TODO if the parameter is used to select the output file type the # options need to be replaced with the Galaxy data types -# if is_out_type_param(param, model): -# param.restrictions.choices = get_supported_file_types(param.restrictions.choices, supported_file_formats) + # if is_out_type_param(param, model): + # param.restrictions.choices = get_supported_file_types(param.restrictions.choices, supported_file_formats) -# # add a nothing selected option to mandatory options w/o default -# if param_node.attrib["optional"] == "False" and (param.default is None or param.default is _Null): -# param.restrictions.choices.insert(0, "select a value") + # # add a nothing selected option to mandatory options w/o default + # if param_node.attrib["optional"] == "False" and (param.default is None or param.default is _Null): + # param.restrictions.choices.insert(0, "select a value") # create as many diff --git a/tests/test-data/ofile-mult.xml b/tests/test-data/ofile-mult.xml index d9a85c3b..1b6ed7c8 100644 --- a/tests/test-data/ofile-mult.xml +++ b/tests/test-data/ofile-mult.xml @@ -24,19 +24,19 @@ mkdir mandatory_optionalinput && #end if #if $inp_mandatory_mandatoryinput: mkdir inp_mandatory_mandatoryinput && - ${ ' '.join([ "ln -s '%s' 'inp_mandatory_mandatoryinput/%s.%s' &&" % (_, re.sub('[^\w\-_]', '_', _.element_identifier),$gxy2omsext(_.ext)) for _ in $inp_mandatory_mandatoryinput if _ ]) } + ${ ' '.join(["ln -s '%s' 'inp_mandatory_mandatoryinput/%s.%s' &&" % (_, re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $inp_mandatory_mandatoryinput if _]) } #end if #if $inp_optional_mandatoryinput: mkdir inp_optional_mandatoryinput && - ${ ' '.join([ "ln -s '%s' 'inp_optional_mandatoryinput/%s.%s' &&" % (_, re.sub('[^\w\-_]', '_', _.element_identifier),$gxy2omsext(_.ext)) for _ in $inp_optional_mandatoryinput if _ ]) } + ${ ' '.join(["ln -s '%s' 'inp_optional_mandatoryinput/%s.%s' &&" % (_, re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $inp_optional_mandatoryinput if _]) } #end if #if $inp_mandatory_optionalinput: mkdir inp_mandatory_optionalinput && - ${ ' '.join([ "ln -s '%s' 'inp_mandatory_optionalinput/%s.%s' &&" % (_, re.sub('[^\w\-_]', '_', _.element_identifier),$gxy2omsext(_.ext)) for _ in $inp_mandatory_optionalinput if _ ]) } + ${ ' '.join(["ln -s '%s' 'inp_mandatory_optionalinput/%s.%s' &&" % (_, re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $inp_mandatory_optionalinput if _]) } #end if #if $inp_optional_optionalinput: mkdir inp_optional_optionalinput && - ${ ' '.join([ "ln -s '%s' 'inp_optional_optionalinput/%s.%s' &&" % (_, re.sub('[^\w\-_]', '_', _.element_identifier),$gxy2omsext(_.ext)) for _ in $inp_optional_optionalinput if _ ]) } + ${ ' '.join(["ln -s '%s' 'inp_optional_optionalinput/%s.%s' &&" % (_, re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $inp_optional_optionalinput if _]) } #end if ## Main program call @@ -44,38 +44,38 @@ mkdir mandatory_optionalinput && python '$__tool_directory__/fill_ctd.py' "\$(basename "@EXECUTABLE@.ctd")" '$args_json' && @EXECUTABLE@ -ini @EXECUTABLE@.ctd -mandatory_mandatoryinput -${' '.join(["'mandatory_mandatoryinput/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext("tsv")) for _ in $inp_mandatory_mandatoryinput if _ ])} +${' '.join(["'mandatory_mandatoryinput/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext("tsv")) for _ in $inp_mandatory_mandatoryinput if _])} #if $optional_mandatoryinput_FLAG: -optional_mandatoryinput - ${' '.join(["'optional_mandatoryinput/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext("tsv")) for _ in $inp_optional_mandatoryinput if _ ])} + ${' '.join(["'optional_mandatoryinput/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext("tsv")) for _ in $inp_optional_mandatoryinput if _])} #end if -mandatory_optionalinput -${' '.join(["'mandatory_optionalinput/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier), _.ext) for _ in $inp_mandatory_optionalinput if _ ])} +${' '.join(["'mandatory_optionalinput/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier), _.ext) for _ in $inp_mandatory_optionalinput if _])} #if $optional_optionalinput_FLAG: -optional_optionalinput - ${' '.join(["'optional_optionalinput/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier), _.ext) for _ in $inp_optional_optionalinput if _ ])} + ${' '.join(["'optional_optionalinput/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier), _.ext) for _ in $inp_optional_optionalinput if _])} #end if #if $inp_mandatory_mandatoryinput: -inp_mandatory_mandatoryinput - ${' '.join(["'inp_mandatory_mandatoryinput/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier),$gxy2omsext(_.ext)) for _ in $inp_mandatory_mandatoryinput if _ ])} + ${' '.join(["'inp_mandatory_mandatoryinput/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $inp_mandatory_mandatoryinput if _])} #end if #if $inp_optional_mandatoryinput: -inp_optional_mandatoryinput - ${' '.join(["'inp_optional_mandatoryinput/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier),$gxy2omsext(_.ext)) for _ in $inp_optional_mandatoryinput if _ ])} + ${' '.join(["'inp_optional_mandatoryinput/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $inp_optional_mandatoryinput if _])} #end if #if $inp_mandatory_optionalinput: -inp_mandatory_optionalinput - ${' '.join(["'inp_mandatory_optionalinput/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier),$gxy2omsext(_.ext)) for _ in $inp_mandatory_optionalinput if _ ])} + ${' '.join(["'inp_mandatory_optionalinput/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $inp_mandatory_optionalinput if _])} #end if #if $inp_optional_optionalinput: -inp_optional_optionalinput - ${' '.join(["'inp_optional_optionalinput/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier),$gxy2omsext(_.ext)) for _ in $inp_optional_optionalinput if _ ])} + ${' '.join(["'inp_optional_optionalinput/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $inp_optional_optionalinput if _])} #end if ## Postprocessing -${' '.join(["&& mv -n 'mandatory_mandatoryinput/%(id)s.%(gext)s' 'mandatory_mandatoryinput/%(id)s'"%{"id": re.sub('[^\w\-_]', '_', _.element_identifier), "gext": $gxy2omsext("tsv")} for _ in $inp_mandatory_mandatoryinput if _ ])} +${' '.join(["&& mv -n 'mandatory_mandatoryinput/%(id)s.%(gext)s' 'mandatory_mandatoryinput/%(id)s'"%{"id": re.sub('[^\w\-_]', '_', _.element_identifier), "gext": $gxy2omsext("tsv")} for _ in $inp_mandatory_mandatoryinput if _])} #if $optional_mandatoryinput_FLAG: - ${' '.join(["&& mv -n 'optional_mandatoryinput/%(id)s.%(gext)s' 'optional_mandatoryinput/%(id)s'"%{"id": re.sub('[^\w\-_]', '_', _.element_identifier), "gext": $gxy2omsext("tsv")} for _ in $inp_optional_mandatoryinput if _ ])} + ${' '.join(["&& mv -n 'optional_mandatoryinput/%(id)s.%(gext)s' 'optional_mandatoryinput/%(id)s'"%{"id": re.sub('[^\w\-_]', '_', _.element_identifier), "gext": $gxy2omsext("tsv")} for _ in $inp_optional_mandatoryinput if _])} #end if]]> @@ -85,8 +85,8 @@ ${' '.join(["&& mv -n 'mandatory_mandatoryinput/%(id)s.%(gext)s' 'mandatory_mand - - + + diff --git a/tests/test_galaxy_cli.py b/tests/test_galaxy_cli.py index 63ce0807..bc34cea8 100644 --- a/tests/test_galaxy_cli.py +++ b/tests/test_galaxy_cli.py @@ -3,14 +3,17 @@ import subprocess import tempfile + def to_test_data(*args): return os.path.abspath(os.path.join(os.path.dirname(os.path.realpath(__file__)), "test-data", *args)) + def file2list(pth): with open(pth) as f: lines = f.readlines() return lines + class GalaxyCliTestCase(unittest.TestCase): def _compare_cli_output(self, fileprefix): @@ -21,9 +24,9 @@ def _compare_cli_output(self, fileprefix): tmp = tempfile.mkdtemp() out_file = os.path.join(tmp, '{}.xml'.format(fileprefix)) - #out_file = to_test_data('{}.xml'.format(fileprefix)) + # out_file = to_test_data('{}.xml'.format(fileprefix)) my_env = os.environ.copy() - my_env["PATH"] = fileprefix+ ":" + my_env["PATH"] + my_env["PATH"] = fileprefix + ":" + my_env["PATH"] cmd = ['CTDConverter', 'galaxy', '-i', in_pth, '-o', out_file, '-f', ftypes_pth, '-m', macro_pth, '--test-test', '-p', hcparam_pth, "--tool-version", "5.0.011"] print("cmd %s" % cmd) @@ -42,7 +45,7 @@ def _compare_cli_output(self, fileprefix): for i in range(0, len(new_l)): self.assertEqual(new_l[i].rstrip(), old_l[i].rstrip()) - + cmd = ['planemo', 'l', out_file] print("cmd %s" % str(cmd)) popen = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) @@ -98,5 +101,6 @@ def test_galaxy_cli_ofile_mult_ctd(self): def test_galaxy_cli_ofile_mult_typeparam_ctd(self): self._compare_cli_output('ofile-mult-typeparam') + if __name__ == '__main__': unittest.main() From 3e704ef2a169280502b3562c7dfa74417dd67a2b Mon Sep 17 00:00:00 2001 From: M Bernt Date: Fri, 24 Jan 2020 17:06:44 +0100 Subject: [PATCH 069/121] f --- .travis.yml | 18 +++++++++++------- tests/test_galaxy_cli.py | 39 +++++++++++++++++++++++---------------- 2 files changed, 34 insertions(+), 23 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8150610f..6d26429e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,6 @@ language: python python: - - "2.7" + - "3.6" install: - sudo apt-get update @@ -12,16 +12,20 @@ install: - bash miniconda.sh -b -p $HOME/miniconda - export PATH="$HOME/miniconda/bin:$PATH" - hash -r - - conda config --set always_yes yes --set changeps1 no + - conda config --set always_yes yes --set changeps1 no --set auto_activate_base True - conda update -q conda + - conda init + - source activate - conda info -a - - - conda create -q -n test-environment python=$TRAVIS_PYTHON_VERSION - - source activate test-environment + +# - conda create -q -n test-environment python=$TRAVIS_PYTHON_VERSION +# - source activate test-environment - conda config --add channels conda-forge - conda config --add channels bioconda - conda config --add channels anaconda - conda config --add channels workflowconversion + + - conda install python=$TRAVIS_PYTHON_VERSION # TODO could be replaced by conda package once it is merged and released - git clone -b topic/no-1-2x https://github.com/bernt-matthias/CTDopts $HOME/CTDopts - cd $HOME/CTDopts @@ -33,10 +37,10 @@ install: - python setup.py install - conda install coverage green codecov - #- conda install -c bioconda planemo - pip install planemo script: - - python setup.py test + - conda info && python setup.py test + - planemo l tools/ after_script: - python -m codecov diff --git a/tests/test_galaxy_cli.py b/tests/test_galaxy_cli.py index bc34cea8..c8dec729 100644 --- a/tests/test_galaxy_cli.py +++ b/tests/test_galaxy_cli.py @@ -1,5 +1,6 @@ import unittest import os +from shutil import copyfile import subprocess import tempfile @@ -45,22 +46,28 @@ def _compare_cli_output(self, fileprefix): for i in range(0, len(new_l)): self.assertEqual(new_l[i].rstrip(), old_l[i].rstrip()) - - cmd = ['planemo', 'l', out_file] - print("cmd %s" % str(cmd)) - popen = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - output, err = popen.communicate() - print(output) - print(err) - self.assertEqual(err, 0) - - cmd = ['planemo', 't', out_file] - print(str(cmd)) - popen = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - output, err = popen.communicate() - print(output) - print(err) - self.assertEqual(err, 0) + if not os.path.exists("tools"): + os.symlink(to_test_data(""), "tools") +# if not os.path.isdir("tools"): +# os.makedirs("tools") +# if not os.path.isfile("tools/macros.xml"): +# copyfile(macro_pth, os.path.join("tools", 'macros.xml')) +# copyfile(out_file, os.path.join("tools", '{}.xml'.format(fileprefix))) +# cmd = ['planemo', 'l', out_file] +# print("cmd %s" % str(cmd)) +# popen = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) +# output, err = popen.communicate() +# print(output) +# print(err) +# self.assertEqual(err, None) +# +# cmd = ['planemo', 't', out_file] +# print(str(cmd)) +# popen = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) +# output, err = popen.communicate() +# print(output) +# print(err) +# self.assertEqual(err, None) def test_galaxy_cli_bool_ctd(self): self._compare_cli_output('bool') From 036061e8bffc469c8bf6fc73b4582885535398a4 Mon Sep 17 00:00:00 2001 From: M Bernt Date: Fri, 24 Jan 2020 18:43:17 +0100 Subject: [PATCH 070/121] make formats sorted --- galaxy/converter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/galaxy/converter.py b/galaxy/converter.py index 24694bec..9019e5bc 100755 --- a/galaxy/converter.py +++ b/galaxy/converter.py @@ -1045,7 +1045,7 @@ def get_formats(param, supported_file_formats, default=None): raise InvalidModelException("No supported formats [%(type)s] " "for [%(name)s]" % {"type": type(param.restrictions), "name": param.name}) - + formats = list(formats).sorted() return formats From 519ce2fac9860304cba79628a3dc70a857d9d764 Mon Sep 17 00:00:00 2001 From: M Bernt Date: Fri, 24 Jan 2020 18:54:07 +0100 Subject: [PATCH 071/121] fix --- galaxy/converter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/galaxy/converter.py b/galaxy/converter.py index 9019e5bc..f633ad69 100755 --- a/galaxy/converter.py +++ b/galaxy/converter.py @@ -1045,7 +1045,7 @@ def get_formats(param, supported_file_formats, default=None): raise InvalidModelException("No supported formats [%(type)s] " "for [%(name)s]" % {"type": type(param.restrictions), "name": param.name}) - formats = list(formats).sorted() + formats = sorted(formats) return formats From 73987166791deb1f4483685f371cc88bbff83d4b Mon Sep 17 00:00:00 2001 From: M Bernt Date: Mon, 27 Jan 2020 15:11:05 +0100 Subject: [PATCH 072/121] update test data for filetype sorting --- tests/test-data/ifile.xml | 32 ++++++++++++++--------------- tests/test-data/ofile-typeparam.xml | 2 +- tests/test-data/ofile.xml | 16 +++++++-------- 3 files changed, 25 insertions(+), 25 deletions(-) diff --git a/tests/test-data/ifile.xml b/tests/test-data/ifile.xml index b9be3690..a8caafa6 100644 --- a/tests/test-data/ifile.xml +++ b/tests/test-data/ifile.xml @@ -418,51 +418,51 @@ python '$__tool_directory__/fill_ctd.py' "\$(basename "@EXECUTABLE@.ctd")" '$arg - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + diff --git a/tests/test-data/ofile-typeparam.xml b/tests/test-data/ofile-typeparam.xml index e7f3a80e..be489de4 100644 --- a/tests/test-data/ofile-typeparam.xml +++ b/tests/test-data/ofile-typeparam.xml @@ -139,7 +139,7 @@ python '$__tool_directory__/fill_ctd.py' "\$(basename "@EXECUTABLE@.ctd")" '$arg - + diff --git a/tests/test-data/ofile.xml b/tests/test-data/ofile.xml index 31273bae..8d515520 100644 --- a/tests/test-data/ofile.xml +++ b/tests/test-data/ofile.xml @@ -118,24 +118,24 @@ python '$__tool_directory__/fill_ctd.py' "\$(basename "@EXECUTABLE@.ctd")" '$arg - + - + - + - + @@ -170,14 +170,14 @@ python '$__tool_directory__/fill_ctd.py' "\$(basename "@EXECUTABLE@.ctd")" '$arg - - + + - - + + From 218efdebc61e0808497aee3adf88275f3973eb1a Mon Sep 17 00:00:00 2001 From: M Bernt Date: Mon, 27 Jan 2020 15:23:29 +0100 Subject: [PATCH 073/121] no lint but test --- .travis.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 6d26429e..f0dc6438 100644 --- a/.travis.yml +++ b/.travis.yml @@ -40,7 +40,11 @@ install: - pip install planemo script: - conda info && python setup.py test - - planemo l tools/ +# TODO fail expected due to +# https://github.com/galaxyproject/galaxy/pull/9081 which will be in 20.01 and +# empty.xml (no inputs) could be removed and +# - planemo l tools/ + - planemo t tools/ after_script: - python -m codecov From 94ac13ba526284d814c687094eb6481758e63811 Mon Sep 17 00:00:00 2001 From: M Bernt Date: Mon, 27 Jan 2020 19:29:48 +0100 Subject: [PATCH 074/121] testing improvements and fixes - rename select -> select_test (select is reserved keyword) - remove uneccessary basename call - set PATH for `planemo t` --- .travis.yml | 7 +++++-- galaxy/converter.py | 3 +-- tests/test-data/bool.xml | 2 +- tests/test-data/empty.xml | 2 +- tests/test-data/float.xml | 2 +- tests/test-data/ifile.xml | 2 +- tests/test-data/integer.xml | 2 +- tests/test-data/label-help.xml | 2 +- tests/test-data/ofile-corresponding-input.xml | 2 +- tests/test-data/ofile-mult-typeparam.xml | 2 +- tests/test-data/ofile-mult.xml | 2 +- tests/test-data/ofile-typeparam.xml | 2 +- tests/test-data/ofile.xml | 2 +- tests/test-data/repeat.xml | 2 +- tests/test-data/{select => select_test} | 0 tests/test-data/{select.ctd => select_test.ctd} | 2 +- tests/test-data/{select.xml => select_test.xml} | 6 +++--- tests/test-data/string.xml | 2 +- tests/test_galaxy_cli.py | 7 ------- 19 files changed, 23 insertions(+), 28 deletions(-) rename tests/test-data/{select => select_test} (100%) rename tests/test-data/{select.ctd => select_test.ctd} (98%) rename tests/test-data/{select.xml => select_test.xml} (98%) diff --git a/.travis.yml b/.travis.yml index f0dc6438..bec2eea0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -43,8 +43,11 @@ script: # TODO fail expected due to # https://github.com/galaxyproject/galaxy/pull/9081 which will be in 20.01 and # empty.xml (no inputs) could be removed and -# - planemo l tools/ - - planemo t tools/ +# - planemo l tests/test-data/ + + # planemo test content of tests/test-data (this is OK, because the previous + # tests ensure equality of the xmls that are generated and those in the package) + - export PATH=$(pwd)/tests/test-data:$PATH && planemo t tests/test-data/ after_script: - python -m codecov diff --git a/galaxy/converter.py b/galaxy/converter.py index f633ad69..5946f7a9 100755 --- a/galaxy/converter.py +++ b/galaxy/converter.py @@ -596,14 +596,13 @@ def create_command(tool, model, **kwargs): # - call the executable with -write_ctd to write the ctd file (with defaults) # - use fill_ctd.py to overwrite the defaults in the ctd file with the # Galaxy parameters in the JSON file (from inputs config file) - # (note the base name call is only necessary for the CTDconverter tests) # - feed the ctd file to the executable (with -ini) # note: input and output file parameters are still given on the command line # - output file parameters are not included in the JSON file # - input and output files are accessed through links / files that have the correct extension final_cmd['command'].extend(["", "## Main program call"]) final_cmd['command'].append("""@EXECUTABLE@ -write_ctd ./ && -python '$__tool_directory__/fill_ctd.py' "\$(basename "@EXECUTABLE@.ctd")" '$args_json' && +python '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' && @EXECUTABLE@ -ini @EXECUTABLE@.ctd""") final_cmd['command'].extend(kwargs["add_to_command_line"]) final_cmd['postprocessing'].extend(["", "## Postprocessing"]) diff --git a/tests/test-data/bool.xml b/tests/test-data/bool.xml index 48272519..d5cb1fa8 100644 --- a/tests/test-data/bool.xml +++ b/tests/test-data/bool.xml @@ -17,7 +17,7 @@ ## Main program call @EXECUTABLE@ -write_ctd ./ && -python '$__tool_directory__/fill_ctd.py' "\$(basename "@EXECUTABLE@.ctd")" '$args_json' && +python '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' && @EXECUTABLE@ -ini @EXECUTABLE@.ctd > $param_stdout diff --git a/tests/test-data/empty.xml b/tests/test-data/empty.xml index a02c575a..47ed931c 100644 --- a/tests/test-data/empty.xml +++ b/tests/test-data/empty.xml @@ -17,7 +17,7 @@ ## Main program call @EXECUTABLE@ -write_ctd ./ && -python '$__tool_directory__/fill_ctd.py' "\$(basename "@EXECUTABLE@.ctd")" '$args_json' && +python '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' && @EXECUTABLE@ -ini @EXECUTABLE@.ctd > $param_stdout diff --git a/tests/test-data/float.xml b/tests/test-data/float.xml index 3fcd22ac..682fb632 100644 --- a/tests/test-data/float.xml +++ b/tests/test-data/float.xml @@ -17,7 +17,7 @@ ## Main program call @EXECUTABLE@ -write_ctd ./ && -python '$__tool_directory__/fill_ctd.py' "\$(basename "@EXECUTABLE@.ctd")" '$args_json' && +python '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' && @EXECUTABLE@ -ini @EXECUTABLE@.ctd > $param_stdout diff --git a/tests/test-data/ifile.xml b/tests/test-data/ifile.xml index a8caafa6..e639bd96 100644 --- a/tests/test-data/ifile.xml +++ b/tests/test-data/ifile.xml @@ -211,7 +211,7 @@ ## Main program call @EXECUTABLE@ -write_ctd ./ && -python '$__tool_directory__/fill_ctd.py' "\$(basename "@EXECUTABLE@.ctd")" '$args_json' && +python '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' && @EXECUTABLE@ -ini @EXECUTABLE@.ctd #if $mandatory_noformat: -mandatory_noformat diff --git a/tests/test-data/integer.xml b/tests/test-data/integer.xml index 14068270..8d542be9 100644 --- a/tests/test-data/integer.xml +++ b/tests/test-data/integer.xml @@ -17,7 +17,7 @@ ## Main program call @EXECUTABLE@ -write_ctd ./ && -python '$__tool_directory__/fill_ctd.py' "\$(basename "@EXECUTABLE@.ctd")" '$args_json' && +python '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' && @EXECUTABLE@ -ini @EXECUTABLE@.ctd > $param_stdout diff --git a/tests/test-data/label-help.xml b/tests/test-data/label-help.xml index 678a94c5..8096552f 100644 --- a/tests/test-data/label-help.xml +++ b/tests/test-data/label-help.xml @@ -17,7 +17,7 @@ ## Main program call @EXECUTABLE@ -write_ctd ./ && -python '$__tool_directory__/fill_ctd.py' "\$(basename "@EXECUTABLE@.ctd")" '$args_json' && +python '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' && @EXECUTABLE@ -ini @EXECUTABLE@.ctd > $param_stdout diff --git a/tests/test-data/ofile-corresponding-input.xml b/tests/test-data/ofile-corresponding-input.xml index 9cc19a2c..8ccc6a2a 100644 --- a/tests/test-data/ofile-corresponding-input.xml +++ b/tests/test-data/ofile-corresponding-input.xml @@ -55,7 +55,7 @@ mkdir mandatory_multformat && ## Main program call @EXECUTABLE@ -write_ctd ./ && -python '$__tool_directory__/fill_ctd.py' "\$(basename "@EXECUTABLE@.ctd")" '$args_json' && +python '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' && @EXECUTABLE@ -ini @EXECUTABLE@.ctd -mandatory_noformat 'mandatory_noformat/output.${gxy2omsext("txt")}' diff --git a/tests/test-data/ofile-mult-typeparam.xml b/tests/test-data/ofile-mult-typeparam.xml index 6b306ae7..01dbcc4a 100644 --- a/tests/test-data/ofile-mult-typeparam.xml +++ b/tests/test-data/ofile-mult-typeparam.xml @@ -27,7 +27,7 @@ mkdir mandatory_optionalinput && ## Main program call @EXECUTABLE@ -write_ctd ./ && -python '$__tool_directory__/fill_ctd.py' "\$(basename "@EXECUTABLE@.ctd")" '$args_json' && +python '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' && @EXECUTABLE@ -ini @EXECUTABLE@.ctd -mandatory_mandatoryinput ${' '.join(["'mandatory_mandatoryinput/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext("tsv")) for _ in $inp_optional_mandatoryinput if _])} diff --git a/tests/test-data/ofile-mult.xml b/tests/test-data/ofile-mult.xml index 1b6ed7c8..bb9aa40b 100644 --- a/tests/test-data/ofile-mult.xml +++ b/tests/test-data/ofile-mult.xml @@ -41,7 +41,7 @@ mkdir mandatory_optionalinput && ## Main program call @EXECUTABLE@ -write_ctd ./ && -python '$__tool_directory__/fill_ctd.py' "\$(basename "@EXECUTABLE@.ctd")" '$args_json' && +python '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' && @EXECUTABLE@ -ini @EXECUTABLE@.ctd -mandatory_mandatoryinput ${' '.join(["'mandatory_mandatoryinput/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext("tsv")) for _ in $inp_mandatory_mandatoryinput if _])} diff --git a/tests/test-data/ofile-typeparam.xml b/tests/test-data/ofile-typeparam.xml index be489de4..7b11acd9 100644 --- a/tests/test-data/ofile-typeparam.xml +++ b/tests/test-data/ofile-typeparam.xml @@ -47,7 +47,7 @@ mkdir mandatory_multformat && ## Main program call @EXECUTABLE@ -write_ctd ./ && -python '$__tool_directory__/fill_ctd.py' "\$(basename "@EXECUTABLE@.ctd")" '$args_json' && +python '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' && @EXECUTABLE@ -ini @EXECUTABLE@.ctd -mandatory_noformat 'mandatory_noformat/output.${gxy2omsext("txt")}' diff --git a/tests/test-data/ofile.xml b/tests/test-data/ofile.xml index 8d515520..686d29cb 100644 --- a/tests/test-data/ofile.xml +++ b/tests/test-data/ofile.xml @@ -43,7 +43,7 @@ mkdir mandatory_multformat && ## Main program call @EXECUTABLE@ -write_ctd ./ && -python '$__tool_directory__/fill_ctd.py' "\$(basename "@EXECUTABLE@.ctd")" '$args_json' && +python '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' && @EXECUTABLE@ -ini @EXECUTABLE@.ctd -mandatory_noformat 'mandatory_noformat/output.${gxy2omsext("txt")}' diff --git a/tests/test-data/repeat.xml b/tests/test-data/repeat.xml index a629573e..66ff3d52 100644 --- a/tests/test-data/repeat.xml +++ b/tests/test-data/repeat.xml @@ -17,7 +17,7 @@ ## Main program call @EXECUTABLE@ -write_ctd ./ && -python '$__tool_directory__/fill_ctd.py' "\$(basename "@EXECUTABLE@.ctd")" '$args_json' && +python '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' && @EXECUTABLE@ -ini @EXECUTABLE@.ctd > $param_stdout diff --git a/tests/test-data/select b/tests/test-data/select_test similarity index 100% rename from tests/test-data/select rename to tests/test-data/select_test diff --git a/tests/test-data/select.ctd b/tests/test-data/select_test.ctd similarity index 98% rename from tests/test-data/select.ctd rename to tests/test-data/select_test.ctd index c54db80b..5b704b63 100644 --- a/tests/test-data/select.ctd +++ b/tests/test-data/select_test.ctd @@ -1,5 +1,5 @@ - + diff --git a/tests/test-data/select.xml b/tests/test-data/select_test.xml similarity index 98% rename from tests/test-data/select.xml rename to tests/test-data/select_test.xml index b028cccb..80094ac9 100644 --- a/tests/test-data/select.xml +++ b/tests/test-data/select_test.xml @@ -1,10 +1,10 @@ - + Select parameter tests - select + select_test macros.xml @@ -17,7 +17,7 @@ ## Main program call @EXECUTABLE@ -write_ctd ./ && -python '$__tool_directory__/fill_ctd.py' "\$(basename "@EXECUTABLE@.ctd")" '$args_json' && +python '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' && @EXECUTABLE@ -ini @EXECUTABLE@.ctd > $param_stdout diff --git a/tests/test-data/string.xml b/tests/test-data/string.xml index 0b2e987e..0053c51a 100644 --- a/tests/test-data/string.xml +++ b/tests/test-data/string.xml @@ -17,7 +17,7 @@ ## Main program call @EXECUTABLE@ -write_ctd ./ && -python '$__tool_directory__/fill_ctd.py' "\$(basename "@EXECUTABLE@.ctd")" '$args_json' && +python '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' && @EXECUTABLE@ -ini @EXECUTABLE@.ctd > $param_stdout diff --git a/tests/test_galaxy_cli.py b/tests/test_galaxy_cli.py index c8dec729..cd965f33 100644 --- a/tests/test_galaxy_cli.py +++ b/tests/test_galaxy_cli.py @@ -46,13 +46,6 @@ def _compare_cli_output(self, fileprefix): for i in range(0, len(new_l)): self.assertEqual(new_l[i].rstrip(), old_l[i].rstrip()) - if not os.path.exists("tools"): - os.symlink(to_test_data(""), "tools") -# if not os.path.isdir("tools"): -# os.makedirs("tools") -# if not os.path.isfile("tools/macros.xml"): -# copyfile(macro_pth, os.path.join("tools", 'macros.xml')) -# copyfile(out_file, os.path.join("tools", '{}.xml'.format(fileprefix))) # cmd = ['planemo', 'l', out_file] # print("cmd %s" % str(cmd)) # popen = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) From 8211b523b7441a46ebf91c5d648480546bf131f3 Mon Sep 17 00:00:00 2001 From: M Bernt Date: Mon, 27 Jan 2020 19:35:35 +0100 Subject: [PATCH 075/121] also rename select in the tests --- tests/test_galaxy_cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_galaxy_cli.py b/tests/test_galaxy_cli.py index cd965f33..ba57c5c1 100644 --- a/tests/test_galaxy_cli.py +++ b/tests/test_galaxy_cli.py @@ -78,7 +78,7 @@ def test_galaxy_cli_repeat_ctd(self): self._compare_cli_output('repeat') def test_galaxy_cli_select_ctd(self): - self._compare_cli_output('select') + self._compare_cli_output('select_test') def test_galaxy_cli_string_ctd(self): self._compare_cli_output('string') From 45ed5de27b3b83104935b9b44b5df53216034871 Mon Sep 17 00:00:00 2001 From: M Bernt Date: Mon, 27 Jan 2020 20:09:37 +0100 Subject: [PATCH 076/121] add debug msgwq --- tests/test-data/bool.xml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test-data/bool.xml b/tests/test-data/bool.xml index d5cb1fa8..a7f80e78 100644 --- a/tests/test-data/bool.xml +++ b/tests/test-data/bool.xml @@ -13,6 +13,8 @@ @EXT_FOO@ #import re +pip list && + ## Preprocessing ## Main program call From 3aff090285784841c1590e243ae598bffc14874d Mon Sep 17 00:00:00 2001 From: M Bernt Date: Tue, 28 Jan 2020 15:23:38 +0100 Subject: [PATCH 077/121] try to use planemo with deactivated conda env --- .travis.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index bec2eea0..60287d90 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,7 +12,7 @@ install: - bash miniconda.sh -b -p $HOME/miniconda - export PATH="$HOME/miniconda/bin:$PATH" - hash -r - - conda config --set always_yes yes --set changeps1 no --set auto_activate_base True + - conda config --set always_yes yes --set changeps1 no --set auto_activate_base False - conda update -q conda - conda init - source activate @@ -35,9 +35,10 @@ install: - conda install libxml2 - cd $TRAVIS_BUILD_DIR - python setup.py install - - conda install coverage green codecov - - pip install planemo + + - virtualenv .venv + - . .venv/bin/activate && pip install planemo && deactivate script: - conda info && python setup.py test # TODO fail expected due to @@ -47,7 +48,7 @@ script: # planemo test content of tests/test-data (this is OK, because the previous # tests ensure equality of the xmls that are generated and those in the package) - - export PATH=$(pwd)/tests/test-data:$PATH && planemo t tests/test-data/ + - conda deactivate &&. .venv/bin/activate && export PATH=$(pwd)/tests/test-data:$PATH && planemo t tests/test-data/ after_script: - python -m codecov From 5a1085b236f646cbb37c091aa72cb49546d158cf Mon Sep 17 00:00:00 2001 From: M Bernt Date: Tue, 28 Jan 2020 15:51:58 +0100 Subject: [PATCH 078/121] try to switch to pip --- .travis.yml | 72 ++++++++++++++++++++++++++++------------------------- 1 file changed, 38 insertions(+), 34 deletions(-) diff --git a/.travis.yml b/.travis.yml index 60287d90..f372513b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,43 +4,47 @@ python: install: - sudo apt-get update - - if [[ "$TRAVIS_PYTHON_VERSION" == "2.7" ]]; then - wget https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh -O miniconda.sh; - else - wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh; - fi - - bash miniconda.sh -b -p $HOME/miniconda - - export PATH="$HOME/miniconda/bin:$PATH" - - hash -r - - conda config --set always_yes yes --set changeps1 no --set auto_activate_base False - - conda update -q conda - - conda init - - source activate - - conda info -a - -# - conda create -q -n test-environment python=$TRAVIS_PYTHON_VERSION -# - source activate test-environment - - conda config --add channels conda-forge - - conda config --add channels bioconda - - conda config --add channels anaconda - - conda config --add channels workflowconversion +# - if [[ "$TRAVIS_PYTHON_VERSION" == "2.7" ]]; then +# wget https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh -O miniconda.sh; +# else +# wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh; +# fi +# - bash miniconda.sh -b -p $HOME/miniconda +# - export PATH="$HOME/miniconda/bin:$PATH" +# - hash -r +# - conda config --set always_yes yes --set changeps1 no --set auto_activate_base False +# - conda update -q conda +# - conda init +# - source activate +# - conda info -a +# +# # - conda create -q -n test-environment python=$TRAVIS_PYTHON_VERSION +# # - source activate test-environment +# - conda config --add channels conda-forge +# - conda config --add channels bioconda +# - conda config --add channels anaconda +# - conda config --add channels workflowconversion +# +# - conda install python=$TRAVIS_PYTHON_VERSION +# # TODO could be replaced by conda package once it is merged and released +# - git clone -b topic/no-1-2x https://github.com/bernt-matthias/CTDopts $HOME/CTDopts +# - cd $HOME/CTDopts +# - python setup.py install +# - conda install -c conda-forge lxml +# - conda install -c conda-forge ruamel.yaml +# - conda install libxml2 +# - cd $TRAVIS_BUILD_DIR +# - python setup.py install +# - conda install coverage green codecov - - conda install python=$TRAVIS_PYTHON_VERSION - # TODO could be replaced by conda package once it is merged and released - - git clone -b topic/no-1-2x https://github.com/bernt-matthias/CTDopts $HOME/CTDopts - - cd $HOME/CTDopts - - python setup.py install - - conda install -c conda-forge lxml - - conda install -c conda-forge ruamel.yaml - - conda install libxml2 +# - virtualenv .venv +# - . .venv/bin/activate + - pip install git+https://github.com/bernt-matthias/CTDopts@topic/no-1-2x + - pip install lxml ruamel.yaml planemo - cd $TRAVIS_BUILD_DIR - python setup.py install - - conda install coverage green codecov - - - virtualenv .venv - - . .venv/bin/activate && pip install planemo && deactivate script: - - conda info && python setup.py test + - python setup.py test # TODO fail expected due to # https://github.com/galaxyproject/galaxy/pull/9081 which will be in 20.01 and # empty.xml (no inputs) could be removed and @@ -48,7 +52,7 @@ script: # planemo test content of tests/test-data (this is OK, because the previous # tests ensure equality of the xmls that are generated and those in the package) - - conda deactivate &&. .venv/bin/activate && export PATH=$(pwd)/tests/test-data:$PATH && planemo t tests/test-data/ + - export PATH=$(pwd)/tests/test-data:$PATH && planemo t tests/test-data/ after_script: - python -m codecov From 8646134b0bc75d2af7a6bf477edab8a38347d1d2 Mon Sep 17 00:00:00 2001 From: M Bernt Date: Tue, 28 Jan 2020 17:27:37 +0100 Subject: [PATCH 079/121] fix test data --- tests/test-data/bool.xml | 2 -- tests/test-data/select_test.xml | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/test-data/bool.xml b/tests/test-data/bool.xml index a7f80e78..d5cb1fa8 100644 --- a/tests/test-data/bool.xml +++ b/tests/test-data/bool.xml @@ -13,8 +13,6 @@ @EXT_FOO@ #import re -pip list && - ## Preprocessing ## Main program call diff --git a/tests/test-data/select_test.xml b/tests/test-data/select_test.xml index 80094ac9..aba27bf5 100644 --- a/tests/test-data/select_test.xml +++ b/tests/test-data/select_test.xml @@ -1,7 +1,7 @@ - + Select parameter tests select_test From 0cb3d32c032516db091948972796407a2cb24c39 Mon Sep 17 00:00:00 2001 From: M Bernt Date: Tue, 4 Feb 2020 10:29:17 +0100 Subject: [PATCH 080/121] fill_ctd: galaxyfy json reading --- tests/test-data/fill_ctd.py | 80 +++++++++++++++++++++++++++++++++++-- 1 file changed, 76 insertions(+), 4 deletions(-) diff --git a/tests/test-data/fill_ctd.py b/tests/test-data/fill_ctd.py index c342d627..bbd45cbb 100644 --- a/tests/test-data/fill_ctd.py +++ b/tests/test-data/fill_ctd.py @@ -1,16 +1,88 @@ +from functools import reduce # forward compatibility for Python 3 import json +import operator import sys -from CTDopts.CTDopts import CTDModel +from CTDopts.CTDopts import CTDModel, _Null, _InFile + + +def getFromDict(dataDict, mapList): + return reduce(operator.getitem, mapList, dataDict) + + +def setInDict(dataDict, mapList, value): + getFromDict(dataDict, mapList[:-1])[mapList[-1]] = value + + +def _json_object_hook(d): + """ + wee helper to transform the json written by galaxy + while loading + - sections are dicts -> recurse + - True/False (bool objects) -> "true"/"false" (lowercase string) + - None -> "" (empty string) + """ + for k in d: + if type(d[k]) is bool: + d[k] = str(d[k]).lower() + elif type(d[k]) is list and len(d[k]) == 1 and d[k][0] is None: + d[k] = [] + elif d[k] is None: + d[k] = "" + return d + + +def qstring2list(qs): + """ + transform a space separated string that is quoted by " into a list + """ + lst = list() + qs = qs.split(" ") + quoted = False + for p in qs: + if p == "": + continue + if not quoted: + if p.startswith('"'): + p = p[1:] + + if p.startswith('"') and p.endswith('"'): + p = p[1:-1] + elif p.startswith('"'): + lst.append("") + quoted = True + p = p[1:]+" " + elif p.endswith('"'): + quoted = False + p = p[:-1] + if quoted: + lst[-1] += p + else: + lst.append(p) + return lst input_ctd = sys.argv[1] with open(sys.argv[2]) as fh: - args = json.load(fh) + args = json.load(fh, object_hook=_json_object_hook) if "adv_opts_cond" in args: args.update(args["adv_opts_cond"]) del args["adv_opts_cond"] model = CTDModel(from_file=input_ctd) -# model.write_ctd(input_ctd, arg_dict = {'1':args}) -model.write_ctd(input_ctd) + +# transform values from json that +# - correspond to unrestricted ITEMLIST which are represented as strings +# ("=quoted and space separated) in Galaxy to lists +# - optional data input parameters that have defaults and for which no +# value is given are overwritten with the default +for p in model.get_parameters(): + if p.is_list and p.restrictions is None: + v = getFromDict(args, p.get_lineage(name_only=True)) + setInDict(args, p.get_lineage(name_only=True), qstring2list(v)) + if p.type is _InFile and p.default not in [None, _Null]: + v = getFromDict(args, p.get_lineage(name_only=True)) + if v in [[], ""]: + setInDict(args, p.get_lineage(name_only=True), p.default) + +model.write_ctd(input_ctd, arg_dict=args) From 369888b7780314f6e2ce58390b8c5996dc1cb275 Mon Sep 17 00:00:00 2001 From: M Bernt Date: Tue, 4 Feb 2020 10:32:42 +0100 Subject: [PATCH 081/121] remove codecov from travis test --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index f372513b..b67634f7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -54,5 +54,5 @@ script: # tests ensure equality of the xmls that are generated and those in the package) - export PATH=$(pwd)/tests/test-data:$PATH && planemo t tests/test-data/ -after_script: -- python -m codecov +# after_script: +# - python -m codecov From 47cb609803f989b783cd2244d34d02c2f1997989 Mon Sep 17 00:00:00 2001 From: M Bernt Date: Tue, 4 Feb 2020 10:33:59 +0100 Subject: [PATCH 082/121] remove unnecessary prints from test script --- tests/test_galaxy_cli.py | 25 +++++-------------------- 1 file changed, 5 insertions(+), 20 deletions(-) diff --git a/tests/test_galaxy_cli.py b/tests/test_galaxy_cli.py index ba57c5c1..afd84211 100644 --- a/tests/test_galaxy_cli.py +++ b/tests/test_galaxy_cli.py @@ -29,38 +29,23 @@ def _compare_cli_output(self, fileprefix): my_env = os.environ.copy() my_env["PATH"] = fileprefix + ":" + my_env["PATH"] cmd = ['CTDConverter', 'galaxy', '-i', in_pth, '-o', out_file, '-f', ftypes_pth, '-m', macro_pth, '--test-test', '-p', hcparam_pth, "--tool-version", "5.0.011"] - print("cmd %s" % cmd) +# print("cmd %s" % cmd) popen = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=my_env) output, err = popen.communicate() - print("out %s" % output) - print("err %s" % err) +# print("out %s" % output) +# print("err %s" % err) old_file_pth = to_test_data('{}.xml'.format(fileprefix)) - print("out_file %s" % out_file) - print("old_file_path %s" % old_file_pth) +# print("out_file %s" % out_file) +# print("old_file_path %s" % old_file_pth) new_l = file2list(out_file) old_l = file2list(old_file_pth) for i in range(0, len(new_l)): self.assertEqual(new_l[i].rstrip(), old_l[i].rstrip()) -# cmd = ['planemo', 'l', out_file] -# print("cmd %s" % str(cmd)) -# popen = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) -# output, err = popen.communicate() -# print(output) -# print(err) -# self.assertEqual(err, None) -# -# cmd = ['planemo', 't', out_file] -# print(str(cmd)) -# popen = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) -# output, err = popen.communicate() -# print(output) -# print(err) -# self.assertEqual(err, None) def test_galaxy_cli_bool_ctd(self): self._compare_cli_output('bool') From 68415c11cfdcaef070cbf000009ee2a841026384 Mon Sep 17 00:00:00 2001 From: M Bernt Date: Tue, 4 Feb 2020 10:34:31 +0100 Subject: [PATCH 083/121] test generation fixes and more temporarily use local CTDopts test generation - started to add treatment of hard coded values - lower case boolean attributes - fix lists to be separated and quoted properly - fix attribute for outputs needs to be file, not value furthermore - fix a bunch of type checks for _Null - more type comparisons using is instead of == - py3 fixes (Etree needs file name not handle) --- galaxy/converter.py | 48 ++++++++++++++++++++++++++++----------------- 1 file changed, 30 insertions(+), 18 deletions(-) diff --git a/galaxy/converter.py b/galaxy/converter.py index 5946f7a9..9abc3e15 100755 --- a/galaxy/converter.py +++ b/galaxy/converter.py @@ -99,7 +99,7 @@ def modify_param_for_galaxy(param): # parm to not required and remove the default (external applications # need to be taken care by hardcoded values and the other cases # are chosen automatically if not specified on the command line) - if param.required and not (param.default is None or param.default is _Null): + if param.required and not (param.default is None or type(param.default) is _Null): param.required = False param.default = _Null return param @@ -296,9 +296,8 @@ def check_test_macros(test_macros_files, test_macros_prefix, parsed_ctds): if len(macro_ids - tool_ids): logger.warning("Unnecessary macros in %s: %s" % (mf, macro_ids - tool_ids)) - with open(mf, "w") as macros_file: - tree = ElementTree(root) - tree.write(macros_file, encoding="UTF-8", xml_declaration=True, pretty_print=True) + tree = ElementTree(root) + tree.write(mf, encoding="UTF-8", xml_declaration=True, pretty_print=True) def parse_file_formats(formats_file): @@ -409,12 +408,11 @@ def _convert_internal(parsed_ctds, **kwargs): if "test_only" in kwargs and kwargs["test_only"]: for parsed_ctd in parsed_ctds: - test = create_test_only(parsed_ctd.ctd_model) + test = create_test_only(parsed_ctd.ctd_model, **kwargs) tree = ElementTree(test) output_file = parsed_ctd.suggested_output_file logger.info("Writing to %s" % utils.get_filename(output_file), 1) - with open(output_file, 'w') as fh: - tree.write(fh, encoding="UTF-8", xml_declaration=False, pretty_print=True) + tree.write(output_file, encoding="UTF-8", xml_declaration=False, pretty_print=True) return parameter_hardcoder = kwargs["parameter_hardcoder"] @@ -600,9 +598,12 @@ def create_command(tool, model, **kwargs): # note: input and output file parameters are still given on the command line # - output file parameters are not included in the JSON file # - input and output files are accessed through links / files that have the correct extension + # TODO remove the PYTHONPATH export CTDopts should come from conda https://github.com/WorkflowConversion/CTDopts/pulls; also python3 -> python? final_cmd['command'].extend(["", "## Main program call"]) - final_cmd['command'].append("""@EXECUTABLE@ -write_ctd ./ && -python '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' && + final_cmd['command'].append(""" +export PYTHONPATH='$__tool_directory__/CTDopts' && +@EXECUTABLE@ -write_ctd ./ && +python3 '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' && @EXECUTABLE@ -ini @EXECUTABLE@.ctd""") final_cmd['command'].extend(kwargs["add_to_command_line"]) final_cmd['postprocessing'].extend(["", "## Postprocessing"]) @@ -739,7 +740,7 @@ def create_command(tool, model, **kwargs): # depends on the additional input (param_x) -> need no if # - real string parameters (i.e. ctd type string wo restrictions) also # need no if (otherwise the empty string could not be provided) - if not (param.required or is_boolean_parameter(param) or (param.type == str and param.restrictions is None)): + if not (param.required or is_boolean_parameter(param) or (param.type is str and param.restrictions is None)): # and not(param.type is _InFile and param.is_list): actual_parameter = get_galaxy_parameter_path(param, "FLAG") for stage in param_cmd: @@ -1066,7 +1067,6 @@ def create_param_attribute_list(param_node, param, model, supported_file_formats if param_type is None: raise ModelError("Unrecognized parameter type %(type)s for parameter %(name)s" % {"type": param.type, "name": param.name}) - # ITEMLIST is rendered as text field (even if its integers or floats), an # exception is files which are treated a bit below if param.is_list: @@ -1126,7 +1126,7 @@ def create_param_attribute_list(param_node, param, model, supported_file_formats else: option_node.attrib["selected"] = "false" # add validator to check that "nothing selected" is not seletcedto mandatory options w/o default - if param_node.attrib["optional"] == "False" and (param.default is None or param.default is _Null): + if param_node.attrib["optional"] == "False" and (param.default is None or type(param.default) is _Null): validator_node = add_child_node(param_node, "validator", OrderedDict([("type", "expression"), ("message", "A value needs to be selected")])) validator_node.text = 'value != "select a value"' @@ -1179,7 +1179,7 @@ def create_param_attribute_list(param_node, param, model, supported_file_formats validator_node.text = CDATA(expression) else: # add quotes to the default values (only if they include spaces .. then the UI looks nicer) - if not (param.default is None or param.default is _Null) and param.type is not _InFile: + if not (param.default is None or type(param.default) is _Null) and param.type is not _InFile: if type(param.default) is list: for i, d in enumerate(param.default): if " " in d: @@ -1196,7 +1196,7 @@ def create_param_attribute_list(param_node, param, model, supported_file_formats valsan = expand_macro(param_node, "list_string_san") # check for default value - if not (param.default is None or param.default is _Null): + if not (param.default is None or type(param.default) is _Null): # defaults of selects are set via the selected attribute of the options (happens above) if param_type == "select": pass @@ -1535,7 +1535,6 @@ def create_tests(parent, inputs=None, outputs=None, test_macros_prefix=None, nam if node.attrib.get("display", None) == "radio" or node.attrib.get("multiple", "false") == "false": node.attrib["value"] = node[0].attrib["value"] elif node.attrib.get("multiple", None) == "true": - print(node) node.attrib["value"] = ",".join([_.attrib["value"] for _ in node if "value" in _.attrib]) elif node.attrib["type"] == "data": node.attrib["ftype"] = node.attrib["format"].split(',')[0] @@ -1597,6 +1596,10 @@ def create_test_only(model, **kwargs): advanced = None for param in utils.extract_and_flatten_parameters(model, True): + # no need to show hardcoded parameters + hardcoded_value = parameter_hardcoder.get_hardcoded_value(utils.extract_param_name(param), model.name) + if parameter_hardcoder.get_blacklist(utils.extract_param_name(param), model.name) or hardcoded_value is not None: + continue try: parent = section_nodes[utils.extract_param_name(param.parent)] except KeyError: @@ -1617,7 +1620,8 @@ def create_test_only(model, **kwargs): section_nodes[utils.extract_param_name(param)] = add_child_node(parent, "section", OrderedDict([("name", param.name)])) continue if param.type is _OutFile and not param.required: - nd = add_child_node(parent, "param", OrderedDict([("name", param.name + "_FLAG"), ("value", "false")])) + flag = str(param.default not in ["", [], _Null]).lower() + nd = add_child_node(parent, "param", OrderedDict([("name", param.name + "_FLAG"), ("value", flag)])) formats = get_formats(param, kwargs["supported_file_formats"], TYPE_TO_GALAXY_TYPE[_OutFile]) type_param = get_out_type_param(param, model, parameter_hardcoder) @@ -1626,11 +1630,19 @@ def create_test_only(model, **kwargs): fmt_select = add_child_node(parent, "param", OrderedDict([("name", param.name + "_type"), ("value", "")])) if param.is_list and type(param.default) is not _Null: - value = '"' + '" "'.join(map(str, param.default)) + '"' + if param.type is _InFile: + value = ','.join(map(str, param.default)) + if param.type is str: + value = '"' + '" "'.join(map(str, param.default)) + '"' + else: + value = ' '.join(map(str, param.default)) else: value = str(param.default) - nd = add_child_node(parent, "param" if param.type is not _OutFile else "output", OrderedDict([("name", param.name), ("value", value)])) + if param.type is not _OutFile: + nd = add_child_node(parent, "param", OrderedDict([("name", param.name), ("value", value)])) + else: + nd = add_child_node(parent, "output", OrderedDict([("name", param.name), ("file", value)])) return test From a9062f35278832d2e1f5e47958925a58a55483fd Mon Sep 17 00:00:00 2001 From: M Bernt Date: Tue, 4 Feb 2020 10:40:27 +0100 Subject: [PATCH 084/121] simplify code for cli mapping generation --- common/utils.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/common/utils.py b/common/utils.py index 9a9be413..82f6209c 100644 --- a/common/utils.py +++ b/common/utils.py @@ -370,14 +370,9 @@ def resolve_param_mapping(param, ctd_model): def _extract_param_cli_name(param, ctd_model): # we generate parameters with colons for subgroups, but not for the two topmost parents (OpenMS legacy) if type(param.parent) == ParameterGroup: - if not hasattr(param.parent.parent, 'parent'): - return resolve_param_mapping(param, ctd_model) - elif not hasattr(param.parent.parent.parent, 'parent'): - return resolve_param_mapping(param, ctd_model) - else: - if hasattr(ctd_model, "cli") and ctd_model.cli: - logger.warning("Using nested parameter sections (NODE elements) is not compatible with ", 1) - return extract_param_name(param.parent) + ":" + resolve_param_mapping(param, ctd_model) + if hasattr(ctd_model, "cli") and ctd_model.cli: + logger.warning("Using nested parameter sections (NODE elements) is not compatible with ", 1) + return ":".join(param.get_lineage(name_only=True)[:-1]) + ":" + resolve_param_mapping(param, ctd_model) else: return resolve_param_mapping(param, ctd_model) @@ -396,7 +391,7 @@ def extract_param_path(param): def extract_param_name(param): # we generate parameters with colons for subgroups, but not for the two topmost parents (OpenMS legacy) - return ":".join(extract_param_path(param)) + return ":".join(param.get_lineage(name_only=True)) def extract_command_line_prefix(param, ctd_model): From e5009366c82233729e9b02f13742afc71ac64eba Mon Sep 17 00:00:00 2001 From: M Bernt Date: Mon, 10 Feb 2020 14:55:52 +0100 Subject: [PATCH 085/121] add nodes to tests - add NODE to all tests - fixes for tests with sections - make mock use CTDopts --- README.md | 2 +- common/utils.py | 19 +- galaxy/converter.py | 61 +- galaxy/macros.xml | 6 +- tests/test-data/bool.ctd | 12 + tests/test-data/bool.xml | 24 +- tests/test-data/empty.xml | 4 +- tests/test-data/fill_ctd.py | 22 +- tests/test-data/float.ctd | 40 + tests/test-data/float.xml | 32 +- tests/test-data/ifile.ctd | 87 ++ tests/test-data/ifile.xml | 740 +++++++++++++----- tests/test-data/integer.ctd | 43 +- tests/test-data/integer.xml | 32 +- tests/test-data/label-help.xml | 12 +- tests/test-data/mock.py | 19 +- tests/test-data/ofile-corresponding-input.ctd | 21 +- tests/test-data/ofile-corresponding-input.xml | 169 +++- tests/test-data/ofile-mult-typeparam.ctd | 14 + tests/test-data/ofile-mult-typeparam.xml | 89 ++- tests/test-data/ofile-mult.xml | 46 +- tests/test-data/ofile-typeparam.ctd | 34 +- tests/test-data/ofile-typeparam.xml | 275 +++++-- tests/test-data/ofile.ctd | 22 +- tests/test-data/ofile.xml | 156 +++- tests/test-data/repeat.xml | 244 ------ tests/test-data/{repeat => repeat_test} | 0 .../test-data/{repeat.ctd => repeat_test.ctd} | 117 ++- tests/test-data/repeat_test.xml | 446 +++++++++++ tests/test-data/select_test.ctd | 66 ++ tests/test-data/select_test.xml | 496 +++++++++--- tests/test-data/string.ctd | 11 + tests/test-data/string.xml | 44 +- tests/test-data/test-data/12test.ext | 1 + tests/test-data/test-data/12test2.ext | 1 + tests/test-data/test-data/13test.ext | 1 + tests/test-data/test-data/13test2.ext | 1 + tests/test-data/test-data/14test.ext | 1 + tests/test-data/test-data/14test2.ext | 1 + tests/test-data/test-data/15test.ext | 1 + tests/test-data/test-data/15test2.ext | 1 + tests/test-data/test-data/16test.ext | 1 + tests/test-data/test-data/16test2.ext | 1 + tests/test-data/test-data/17test.ext | 1 + tests/test-data/test-data/17test2.ext | 1 + tests/test-data/test-data/18test.ext | 1 + tests/test-data/test-data/18test2.ext | 1 + tests/test-data/test-data/19test.ext | 1 + tests/test-data/test-data/19test2.ext | 1 + tests/test-data/test-data/20test.ext | 1 + tests/test-data/test-data/20test2.ext | 1 + tests/test-data/test-data/21test.ext | 1 + tests/test-data/test-data/21test2.ext | 1 + tests/test-data/test-data/22test.ext | 1 + tests/test-data/test-data/22test2.ext | 1 + tests/test-data/test-data/23test.ext | 1 + tests/test-data/test-data/23test2.ext | 1 + 57 files changed, 2665 insertions(+), 764 deletions(-) delete mode 100644 tests/test-data/repeat.xml rename tests/test-data/{repeat => repeat_test} (100%) rename tests/test-data/{repeat.ctd => repeat_test.ctd} (50%) create mode 100644 tests/test-data/repeat_test.xml create mode 120000 tests/test-data/test-data/12test.ext create mode 120000 tests/test-data/test-data/12test2.ext create mode 120000 tests/test-data/test-data/13test.ext create mode 120000 tests/test-data/test-data/13test2.ext create mode 120000 tests/test-data/test-data/14test.ext create mode 120000 tests/test-data/test-data/14test2.ext create mode 120000 tests/test-data/test-data/15test.ext create mode 120000 tests/test-data/test-data/15test2.ext create mode 120000 tests/test-data/test-data/16test.ext create mode 120000 tests/test-data/test-data/16test2.ext create mode 120000 tests/test-data/test-data/17test.ext create mode 120000 tests/test-data/test-data/17test2.ext create mode 120000 tests/test-data/test-data/18test.ext create mode 120000 tests/test-data/test-data/18test2.ext create mode 120000 tests/test-data/test-data/19test.ext create mode 120000 tests/test-data/test-data/19test2.ext create mode 120000 tests/test-data/test-data/20test.ext create mode 120000 tests/test-data/test-data/20test2.ext create mode 120000 tests/test-data/test-data/21test.ext create mode 120000 tests/test-data/test-data/21test2.ext create mode 120000 tests/test-data/test-data/22test.ext create mode 120000 tests/test-data/test-data/22test2.ext create mode 120000 tests/test-data/test-data/23test.ext create mode 120000 tests/test-data/test-data/23test2.ext diff --git a/README.md b/README.md index a998835d..442d1002 100644 --- a/README.md +++ b/README.md @@ -188,7 +188,7 @@ PATH=$(pwd)/tests/test-data/:$PATH for i in tests/test-data/*ctd do b=$(basename $i .ctd) -python2 convert.py galaxy -i tests/test-data/$b.ctd -o tests/test-data/$b.xml -m tests/test-data/macros.xml -f tests/test-data/filetypes.txt --test-test -p tests/test-data/hardcoded_params.json --tool-version 5.0.011 +python convert.py galaxy -i tests/test-data/$b.ctd -o tests/test-data/$b.xml -m tests/test-data/macros.xml -f tests/test-data/filetypes.txt --test-test -p tests/test-data/hardcoded_params.json --tool-version 5.0.011 done ``` diff --git a/common/utils.py b/common/utils.py index 82f6209c..676f0081 100644 --- a/common/utils.py +++ b/common/utils.py @@ -378,15 +378,16 @@ def _extract_param_cli_name(param, ctd_model): def extract_param_path(param): - if type(param.parent) == ParameterGroup or type(param.parent) == Parameters: - if not hasattr(param.parent.parent, "parent"): - return [param.name] - elif not hasattr(param.parent.parent.parent, "parent"): - return [param.name] - else: - return extract_param_path(param.parent) + [param.name] - else: - return [param.name] + return param.get_lineage(name_only=True) +# if type(param.parent) == ParameterGroup or type(param.parent) == Parameters: +# if not hasattr(param.parent.parent, "parent"): +# return [param.name] +# elif not hasattr(param.parent.parent.parent, "parent"): +# return [param.name] +# else: +# return extract_param_path(param.parent) + [param.name] +# else: +# return [param.name] def extract_param_name(param): diff --git a/galaxy/converter.py b/galaxy/converter.py index 9abc3e15..9942ad17 100755 --- a/galaxy/converter.py +++ b/galaxy/converter.py @@ -646,13 +646,13 @@ def create_command(tool, model, **kwargs): # rationale: in the autogenerated tests the same file was used as input to multiple parameters # this leads to conflicts while linking... might also be better in general if param.type is _InFile: - param_cmd['preprocessing'].append("mkdir %s &&" % param.name) + param_cmd['preprocessing'].append("mkdir %s &&" % actual_parameter) if param.is_list: - param_cmd['preprocessing'].append("${ ' '.join([\"ln -s '%s' '" + param.name + "/%s.%s' &&\" % (_, re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $" + param.name + " if _]) }") - param_cmd['command'].append("${' '.join([\"'" + param.name + "/%s.%s'\"%(re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $" + param.name + " if _])}") + param_cmd['preprocessing'].append("${ ' '.join([\"ln -s '%s' '" + actual_parameter + "/%s.%s' &&\" % (_, re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $" + actual_parameter + " if _]) }") + param_cmd['command'].append("${' '.join([\"'" + actual_parameter + "/%s.%s'\"%(re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $" + actual_parameter + " if _])}") else: - param_cmd['preprocessing'].append("ln -s '$" + param.name + "' '" + param.name + "/${re.sub(\"[^\w\-_]\", \"_\", $" + param.name + ".element_identifier)}.$gxy2omsext($" + param.name + ".ext)' &&") - param_cmd['command'].append("'" + param.name + "/${re.sub(\"[^\w\-_]\", \"_\", $" + param.name + ".element_identifier)}.$gxy2omsext($" + param.name + ".ext)'") + param_cmd['preprocessing'].append("ln -s '$" + actual_parameter + "' '" + actual_parameter + "/${re.sub(\"[^\w\-_]\", \"_\", $" + actual_parameter + ".element_identifier)}.$gxy2omsext($" + actual_parameter + ".ext)' &&") + param_cmd['command'].append("'" + actual_parameter + "/${re.sub(\"[^\w\-_]\", \"_\", $" + actual_parameter + ".element_identifier)}.$gxy2omsext($" + actual_parameter + ".ext)'") elif param.type is _OutFile: # check if there is a parameter that sets the format # if so we add an extension to the generated files which will be used to @@ -674,7 +674,7 @@ def create_command(tool, model, **kwargs): else: type_param_name = None - param_cmd['preprocessing'].append("mkdir " + param.name + " &&") + param_cmd['preprocessing'].append("mkdir " + actual_parameter + " &&") # if there is only one format (the outoput node sets format using the format attribute of the data/discover node) # - single file: write to temp file with oms extension and move this to the actual result file @@ -683,12 +683,12 @@ def create_command(tool, model, **kwargs): fmt = formats.pop() if param.is_list: logger.info("1 fmt + list %s -> %s" % (param.name, actual_input_parameter), 1) - param_cmd['command'].append("${' '.join([\"'" + param.name + "/%s.%s'\"%(re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(\"" + fmt + "\")) for _ in $" + actual_input_parameter + " if _])}") - param_cmd['postprocessing'].append("${' '.join([\"&& mv -n '" + param.name + "/%(id)s.%(gext)s' '" + param.name + "/%(id)s'\"%{\"id\": re.sub('[^\w\-_]', '_', _.element_identifier), \"gext\": $gxy2omsext(\"" + fmt + "\")} for _ in $" + actual_input_parameter + " if _])}") + param_cmd['command'].append("${' '.join([\"'" + actual_parameter + "/%s.%s'\"%(re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(\"" + fmt + "\")) for _ in $" + actual_parameter + " if _])}") + param_cmd['postprocessing'].append("${' '.join([\"&& mv -n '" + actual_parameter + "/%(id)s.%(gext)s' '" + actual_parameter + "/%(id)s'\"%{\"id\": re.sub('[^\w\-_]', '_', _.element_identifier), \"gext\": $gxy2omsext(\"" + fmt + "\")} for _ in $" + actual_parameter + " if _])}") else: logger.info("1 fmt + dataset %s" % param.name, 1) - param_cmd['command'].append("'" + param.name + "/output.${gxy2omsext(\"" + fmt + "\")}'") - param_cmd['postprocessing'].append("&& mv '" + param.name + "/output.${gxy2omsext(\"" + fmt + "\")}' '$" + param.name + "'") + param_cmd['command'].append("'" + actual_parameter + "/output.${gxy2omsext(\"" + fmt + "\")}'") + param_cmd['postprocessing'].append("&& mv '" + actual_parameter + "/output.${gxy2omsext(\"" + fmt + "\")}' '$" + actual_parameter + "'") # if there is a type parameter then we use the type selected by the user # - single: write to temp file with the oms extension and mv it to the actual file output which is treated via change_format @@ -696,33 +696,33 @@ def create_command(tool, model, **kwargs): elif type_param_name is not None: if param.is_list: logger.info("type + list %s" % param.name, 1) - param_cmd['command'].append("${' '.join([\"'" + param.name + "/%s.%s'\"%(re.sub('[^\w\-_]', '_', _.element_identifier), " + type_param_name + ") for _ in $" + actual_input_parameter + " if _])}") - param_cmd['postprocessing'].append("${' '.join([\"&& mv -n '" + param.name + "/%(id)s.%(omsext)s' '" + param.name + "/%(id)s.%(gext)s'\"%{\"id\": re.sub('[^\w\-_]', '_', _.element_identifier), \"omsext\":" + type_param_name + ", \"gext\": oms2gxyext(\"" + type_param_name + "\")} for _ in $" + actual_input_parameter + " if _])}") + param_cmd['command'].append("${' '.join([\"'" + actual_parameter + "/%s.%s'\"%(re.sub('[^\w\-_]', '_', _.element_identifier), " + type_param_name + ") for _ in $" + actual_parameter + " if _])}") + param_cmd['postprocessing'].append("${' '.join([\"&& mv -n '" + actual_parameter + "/%(id)s.%(omsext)s' '" + actual_parameter + "/%(id)s.%(gext)s'\"%{\"id\": re.sub('[^\w\-_]', '_', _.element_identifier), \"omsext\":" + type_param_name + ", \"gext\": oms2gxyext(\"" + type_param_name + "\")} for _ in $" + actual_parameter + " if _])}") else: logger.info("type + dataset %s" % param.name, 1) # 1st create file with openms extension (often required by openms) # then move it to the actual place specified by the parameter # the format is then set by the tag using - param_cmd['command'].append("'" + param.name + "/output.${" + type_param_name + "}'") - param_cmd['postprocessing'].append("&& mv '" + param.name + "/output.${" + type_param_name + "}' '$" + actual_parameter + "'") + param_cmd['command'].append("'" + actual_parameter + "/output.${" + type_param_name + "}'") + param_cmd['postprocessing'].append("&& mv '" + actual_parameter + "/output.${" + type_param_name + "}' '$" + actual_parameter + "'") elif actual_input_parameter is not None: if param.is_list: logger.info("actual + list %s" % param.name, 1) - param_cmd['command'].append("${' '.join([\"'" + param.name + "/%s.%s'\"%(re.sub('[^\w\-_]', '_', _.element_identifier), _.ext) for _ in $" + actual_input_parameter + " if _])}") + param_cmd['command'].append("${' '.join([\"'" + actual_parameter + "/%s.%s'\"%(re.sub('[^\w\-_]', '_', _.element_identifier), _.ext) for _ in $" + actual_input_parameter + " if _])}") else: logger.info("actual + dataset %s %s %s" % (param.name, actual_input_parameter, corresponding_input.is_list), 1) if corresponding_input.is_list: - param_cmd['command'].append("'" + param.name + "/output.${" + actual_input_parameter + "[0].ext}'") - param_cmd['postprocessing'].append("&& mv '" + param.name + "/output.${" + actual_input_parameter + "[0].ext}' '$" + actual_parameter + "'") + param_cmd['command'].append("'" + actual_parameter + "/output.${" + actual_input_parameter + "[0].ext}'") + param_cmd['postprocessing'].append("&& mv '" + actual_parameter + "/output.${" + actual_input_parameter + "[0].ext}' '$" + actual_parameter + "'") else: - param_cmd['command'].append("'" + param.name + "/output.${" + actual_input_parameter + ".ext}'") - param_cmd['postprocessing'].append("&& mv '" + param.name + "/output.${" + actual_input_parameter + ".ext}' '$" + actual_parameter + "'") + param_cmd['command'].append("'" + actual_parameter + "/output.${" + actual_input_parameter + ".ext}'") + param_cmd['postprocessing'].append("&& mv '" + actual_parameter + "/output.${" + actual_input_parameter + ".ext}' '$" + actual_parameter + "'") else: if param.is_list: raise Exception("output parameter itemlist %s without corresponding input") else: logger.info("else + dataset %s" % param.name, 1) - param_cmd['command'].append("'$out_" + param.name + "'") + param_cmd['command'].append("'$" + actual_parameter + "'") # # select with multiple = true # elif is_selection_parameter(param) and param.is_list: @@ -818,7 +818,7 @@ def get_galaxy_parameter_path(param, inparam=False): p = get_galaxy_parameter_name(param, inparam) path = utils.extract_param_path(param) if len(path) > 1 and (param.type is not _OutFile or inparam): - return (".".join(["section_" + _ for _ in path[:-1]]) + "." + p).replace("-", "_") + return (".".join(path[:-1]) + "." + p).replace("-", "_") elif param.advanced and (param.type is not _OutFile or inparam): return ADVANCED_OPTIONS_NAME + "cond." + p else: @@ -1109,9 +1109,13 @@ def create_param_attribute_list(param_node, param, model, supported_file_formats # if is_out_type_param(param, model): # param.restrictions.choices = get_supported_file_types(param.restrictions.choices, supported_file_formats) - # # add a nothing selected option to mandatory options w/o default - # if param_node.attrib["optional"] == "False" and (param.default is None or param.default is _Null): - # param.restrictions.choices.insert(0, "select a value") + # add a nothing selected option to mandatory options w/o default + if param.default is None or param.default is _Null: + if param_node.attrib["optional"] == "true": + option_node = add_child_node(param_node, "option", OrderedDict([("value", "")]), text="default (nothing chosen)") +# else: +# option_node = add_child_node(param_node, "option", OrderedDict([("value", "")]), text="select a value") + # create as many diff --git a/tests/test-data/bool.xml b/tests/test-data/bool.xml index d5cb1fa8..46f324b3 100644 --- a/tests/test-data/bool.xml +++ b/tests/test-data/bool.xml @@ -16,8 +16,10 @@ ## Preprocessing ## Main program call + +export PYTHONPATH='$__tool_directory__/CTDopts' && @EXECUTABLE@ -write_ctd ./ && -python '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' && +python3 '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' && @EXECUTABLE@ -ini @EXECUTABLE@.ctd > $param_stdout @@ -30,6 +32,16 @@ python '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' && +
+ + + + + + + + +
@@ -46,6 +58,16 @@ python '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' && +
+ + + + + + + + +
diff --git a/tests/test-data/empty.xml b/tests/test-data/empty.xml index 47ed931c..3206ae45 100644 --- a/tests/test-data/empty.xml +++ b/tests/test-data/empty.xml @@ -16,8 +16,10 @@ ## Preprocessing ## Main program call + +export PYTHONPATH='$__tool_directory__/CTDopts' && @EXECUTABLE@ -write_ctd ./ && -python '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' && +python3 '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' && @EXECUTABLE@ -ini @EXECUTABLE@.ctd > $param_stdout diff --git a/tests/test-data/fill_ctd.py b/tests/test-data/fill_ctd.py index bbd45cbb..f1d748cd 100644 --- a/tests/test-data/fill_ctd.py +++ b/tests/test-data/fill_ctd.py @@ -3,7 +3,7 @@ import operator import sys -from CTDopts.CTDopts import CTDModel, _Null, _InFile +from CTDopts.CTDopts import CTDModel, _Null, _InFile, _NumericRange def getFromDict(dataDict, mapList): @@ -38,25 +38,16 @@ def qstring2list(qs): """ lst = list() qs = qs.split(" ") - quoted = False for p in qs: if p == "": continue - if not quoted: - if p.startswith('"'): - p = p[1:] if p.startswith('"') and p.endswith('"'): - p = p[1:-1] + lst.append(p[1:-1]) elif p.startswith('"'): - lst.append("") - quoted = True - p = p[1:]+" " + lst.append( p[1:]+" ") elif p.endswith('"'): - quoted = False - p = p[:-1] - if quoted: - lst[-1] += p + lst[-1] += p[:-1] else: lst.append(p) return lst @@ -77,9 +68,10 @@ def qstring2list(qs): # - optional data input parameters that have defaults and for which no # value is given are overwritten with the default for p in model.get_parameters(): - if p.is_list and p.restrictions is None: + if p.is_list and (p.restrictions is None or type(p.restrictions) is _NumericRange): v = getFromDict(args, p.get_lineage(name_only=True)) - setInDict(args, p.get_lineage(name_only=True), qstring2list(v)) + if type(v) is str: + setInDict(args, p.get_lineage(name_only=True), qstring2list(v)) if p.type is _InFile and p.default not in [None, _Null]: v = getFromDict(args, p.get_lineage(name_only=True)) if v in [[], ""]: diff --git a/tests/test-data/float.ctd b/tests/test-data/float.ctd index 5fc283fb..03eadb4d 100644 --- a/tests/test-data/float.ctd +++ b/tests/test-data/float.ctd @@ -41,6 +41,46 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/test-data/float.xml b/tests/test-data/float.xml index 682fb632..a79ab809 100644 --- a/tests/test-data/float.xml +++ b/tests/test-data/float.xml @@ -16,8 +16,10 @@ ## Preprocessing ## Main program call + +export PYTHONPATH='$__tool_directory__/CTDopts' && @EXECUTABLE@ -write_ctd ./ && -python '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' && +python3 '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' && @EXECUTABLE@ -ini @EXECUTABLE@.ctd > $param_stdout @@ -34,6 +36,24 @@ python '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' && +
+ + + + + + + + + + + + + + + + +
@@ -54,6 +74,16 @@ python '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' && +
+ + + + + + + + +
diff --git a/tests/test-data/ifile.ctd b/tests/test-data/ifile.ctd index c221a000..65779cbc 100644 --- a/tests/test-data/ifile.ctd +++ b/tests/test-data/ifile.ctd @@ -90,6 +90,93 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/test-data/ifile.xml b/tests/test-data/ifile.xml index e639bd96..ebbecd69 100644 --- a/tests/test-data/ifile.xml +++ b/tests/test-data/ifile.xml @@ -14,26 +14,20 @@ #import re ## Preprocessing -#if $mandatory_noformat: - mkdir mandatory_noformat && - ln -s '$mandatory_noformat' 'mandatory_noformat/${re.sub("[^\w\-_]", "_", $mandatory_noformat.element_identifier)}.$gxy2omsext($mandatory_noformat.ext)' && -#end if +mkdir mandatory_noformat && +ln -s '$mandatory_noformat' 'mandatory_noformat/${re.sub("[^\w\-_]", "_", $mandatory_noformat.element_identifier)}.$gxy2omsext($mandatory_noformat.ext)' && #if $optional_noformat: mkdir optional_noformat && ln -s '$optional_noformat' 'optional_noformat/${re.sub("[^\w\-_]", "_", $optional_noformat.element_identifier)}.$gxy2omsext($optional_noformat.ext)' && #end if -#if $mandatory_oneformat: - mkdir mandatory_oneformat && - ln -s '$mandatory_oneformat' 'mandatory_oneformat/${re.sub("[^\w\-_]", "_", $mandatory_oneformat.element_identifier)}.$gxy2omsext($mandatory_oneformat.ext)' && -#end if +mkdir mandatory_oneformat && +ln -s '$mandatory_oneformat' 'mandatory_oneformat/${re.sub("[^\w\-_]", "_", $mandatory_oneformat.element_identifier)}.$gxy2omsext($mandatory_oneformat.ext)' && #if $optional_oneformat: mkdir optional_oneformat && ln -s '$optional_oneformat' 'optional_oneformat/${re.sub("[^\w\-_]", "_", $optional_oneformat.element_identifier)}.$gxy2omsext($optional_oneformat.ext)' && #end if -#if $mandatory_multformat: - mkdir mandatory_multformat && - ln -s '$mandatory_multformat' 'mandatory_multformat/${re.sub("[^\w\-_]", "_", $mandatory_multformat.element_identifier)}.$gxy2omsext($mandatory_multformat.ext)' && -#end if +mkdir mandatory_multformat && +ln -s '$mandatory_multformat' 'mandatory_multformat/${re.sub("[^\w\-_]", "_", $mandatory_multformat.element_identifier)}.$gxy2omsext($mandatory_multformat.ext)' && #if $optional_multformat: mkdir optional_multformat && ln -s '$optional_multformat' 'optional_multformat/${re.sub("[^\w\-_]", "_", $optional_multformat.element_identifier)}.$gxy2omsext($optional_multformat.ext)' && @@ -62,26 +56,20 @@ mkdir default_optional_multformat && ln -s '$default_optional_multformat' 'default_optional_multformat/${re.sub("[^\w\-_]", "_", $default_optional_multformat.element_identifier)}.$gxy2omsext($default_optional_multformat.ext)' && #end if -#if $list_mandatory_noformat: - mkdir list_mandatory_noformat && - ${ ' '.join(["ln -s '%s' 'list_mandatory_noformat/%s.%s' &&" % (_, re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $list_mandatory_noformat if _]) } -#end if +mkdir list_mandatory_noformat && +${ ' '.join(["ln -s '%s' 'list_mandatory_noformat/%s.%s' &&" % (_, re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $list_mandatory_noformat if _]) } #if $list_optional_noformat: mkdir list_optional_noformat && ${ ' '.join(["ln -s '%s' 'list_optional_noformat/%s.%s' &&" % (_, re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $list_optional_noformat if _]) } #end if -#if $list_mandatory_oneformat: - mkdir list_mandatory_oneformat && - ${ ' '.join(["ln -s '%s' 'list_mandatory_oneformat/%s.%s' &&" % (_, re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $list_mandatory_oneformat if _]) } -#end if +mkdir list_mandatory_oneformat && +${ ' '.join(["ln -s '%s' 'list_mandatory_oneformat/%s.%s' &&" % (_, re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $list_mandatory_oneformat if _]) } #if $list_optional_oneformat: mkdir list_optional_oneformat && ${ ' '.join(["ln -s '%s' 'list_optional_oneformat/%s.%s' &&" % (_, re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $list_optional_oneformat if _]) } #end if -#if $list_mandatory_multformat: - mkdir list_mandatory_multformat && - ${ ' '.join(["ln -s '%s' 'list_mandatory_multformat/%s.%s' &&" % (_, re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $list_mandatory_multformat if _]) } -#end if +mkdir list_mandatory_multformat && +${ ' '.join(["ln -s '%s' 'list_mandatory_multformat/%s.%s' &&" % (_, re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $list_mandatory_multformat if _]) } #if $list_optional_multformat: mkdir list_optional_multformat && ${ ' '.join(["ln -s '%s' 'list_optional_multformat/%s.%s' &&" % (_, re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $list_optional_multformat if _]) } @@ -110,129 +98,281 @@ mkdir list_default_optional_multformat && ${ ' '.join(["ln -s '%s' 'list_default_optional_multformat/%s.%s' &&" % (_, re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $list_default_optional_multformat if _]) } #end if +mkdir test_section.sect_mandatory_noformat && +ln -s '$test_section.sect_mandatory_noformat' 'test_section.sect_mandatory_noformat/${re.sub("[^\w\-_]", "_", $test_section.sect_mandatory_noformat.element_identifier)}.$gxy2omsext($test_section.sect_mandatory_noformat.ext)' && +#if $test_section.sect_optional_noformat: + mkdir test_section.sect_optional_noformat && + ln -s '$test_section.sect_optional_noformat' 'test_section.sect_optional_noformat/${re.sub("[^\w\-_]", "_", $test_section.sect_optional_noformat.element_identifier)}.$gxy2omsext($test_section.sect_optional_noformat.ext)' && +#end if +mkdir test_section.sect_mandatory_oneformat && +ln -s '$test_section.sect_mandatory_oneformat' 'test_section.sect_mandatory_oneformat/${re.sub("[^\w\-_]", "_", $test_section.sect_mandatory_oneformat.element_identifier)}.$gxy2omsext($test_section.sect_mandatory_oneformat.ext)' && +#if $test_section.sect_optional_oneformat: + mkdir test_section.sect_optional_oneformat && + ln -s '$test_section.sect_optional_oneformat' 'test_section.sect_optional_oneformat/${re.sub("[^\w\-_]", "_", $test_section.sect_optional_oneformat.element_identifier)}.$gxy2omsext($test_section.sect_optional_oneformat.ext)' && +#end if +mkdir test_section.sect_mandatory_multformat && +ln -s '$test_section.sect_mandatory_multformat' 'test_section.sect_mandatory_multformat/${re.sub("[^\w\-_]", "_", $test_section.sect_mandatory_multformat.element_identifier)}.$gxy2omsext($test_section.sect_mandatory_multformat.ext)' && +#if $test_section.sect_optional_multformat: + mkdir test_section.sect_optional_multformat && + ln -s '$test_section.sect_optional_multformat' 'test_section.sect_optional_multformat/${re.sub("[^\w\-_]", "_", $test_section.sect_optional_multformat.element_identifier)}.$gxy2omsext($test_section.sect_optional_multformat.ext)' && +#end if +#if $test_section.sect_default_mandatory_noformat: + mkdir test_section.sect_default_mandatory_noformat && + ln -s '$test_section.sect_default_mandatory_noformat' 'test_section.sect_default_mandatory_noformat/${re.sub("[^\w\-_]", "_", $test_section.sect_default_mandatory_noformat.element_identifier)}.$gxy2omsext($test_section.sect_default_mandatory_noformat.ext)' && +#end if +#if $test_section.sect_default_optional_noformat: + mkdir test_section.sect_default_optional_noformat && + ln -s '$test_section.sect_default_optional_noformat' 'test_section.sect_default_optional_noformat/${re.sub("[^\w\-_]", "_", $test_section.sect_default_optional_noformat.element_identifier)}.$gxy2omsext($test_section.sect_default_optional_noformat.ext)' && +#end if +#if $test_section.sect_default_mandatory_oneformat: + mkdir test_section.sect_default_mandatory_oneformat && + ln -s '$test_section.sect_default_mandatory_oneformat' 'test_section.sect_default_mandatory_oneformat/${re.sub("[^\w\-_]", "_", $test_section.sect_default_mandatory_oneformat.element_identifier)}.$gxy2omsext($test_section.sect_default_mandatory_oneformat.ext)' && +#end if +#if $test_section.sect_default_optional_oneformat: + mkdir test_section.sect_default_optional_oneformat && + ln -s '$test_section.sect_default_optional_oneformat' 'test_section.sect_default_optional_oneformat/${re.sub("[^\w\-_]", "_", $test_section.sect_default_optional_oneformat.element_identifier)}.$gxy2omsext($test_section.sect_default_optional_oneformat.ext)' && +#end if +#if $test_section.sect_default_mandatory_multformat: + mkdir test_section.sect_default_mandatory_multformat && + ln -s '$test_section.sect_default_mandatory_multformat' 'test_section.sect_default_mandatory_multformat/${re.sub("[^\w\-_]", "_", $test_section.sect_default_mandatory_multformat.element_identifier)}.$gxy2omsext($test_section.sect_default_mandatory_multformat.ext)' && +#end if +#if $test_section.sect_default_optional_multformat: + mkdir test_section.sect_default_optional_multformat && + ln -s '$test_section.sect_default_optional_multformat' 'test_section.sect_default_optional_multformat/${re.sub("[^\w\-_]", "_", $test_section.sect_default_optional_multformat.element_identifier)}.$gxy2omsext($test_section.sect_default_optional_multformat.ext)' && +#end if +mkdir test_section.sect_list_mandatory_noformat && +${ ' '.join(["ln -s '%s' 'test_section.sect_list_mandatory_noformat/%s.%s' &&" % (_, re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $test_section.sect_list_mandatory_noformat if _]) } +#if $test_section.sect_list_optional_noformat: + mkdir test_section.sect_list_optional_noformat && + ${ ' '.join(["ln -s '%s' 'test_section.sect_list_optional_noformat/%s.%s' &&" % (_, re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $test_section.sect_list_optional_noformat if _]) } +#end if +mkdir test_section.sect_list_mandatory_oneformat && +${ ' '.join(["ln -s '%s' 'test_section.sect_list_mandatory_oneformat/%s.%s' &&" % (_, re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $test_section.sect_list_mandatory_oneformat if _]) } +#if $test_section.sect_list_optional_oneformat: + mkdir test_section.sect_list_optional_oneformat && + ${ ' '.join(["ln -s '%s' 'test_section.sect_list_optional_oneformat/%s.%s' &&" % (_, re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $test_section.sect_list_optional_oneformat if _]) } +#end if +mkdir test_section.sect_list_mandatory_multformat && +${ ' '.join(["ln -s '%s' 'test_section.sect_list_mandatory_multformat/%s.%s' &&" % (_, re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $test_section.sect_list_mandatory_multformat if _]) } +#if $test_section.sect_list_optional_multformat: + mkdir test_section.sect_list_optional_multformat && + ${ ' '.join(["ln -s '%s' 'test_section.sect_list_optional_multformat/%s.%s' &&" % (_, re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $test_section.sect_list_optional_multformat if _]) } +#end if +#if $test_section.sect_list_default_mandatory_noformat: + mkdir test_section.sect_list_default_mandatory_noformat && + ${ ' '.join(["ln -s '%s' 'test_section.sect_list_default_mandatory_noformat/%s.%s' &&" % (_, re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $test_section.sect_list_default_mandatory_noformat if _]) } +#end if +#if $test_section.sect_list_default_optional_noformat: + mkdir test_section.sect_list_default_optional_noformat && + ${ ' '.join(["ln -s '%s' 'test_section.sect_list_default_optional_noformat/%s.%s' &&" % (_, re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $test_section.sect_list_default_optional_noformat if _]) } +#end if +#if $test_section.sect_list_default_mandatory_oneformat: + mkdir test_section.sect_list_default_mandatory_oneformat && + ${ ' '.join(["ln -s '%s' 'test_section.sect_list_default_mandatory_oneformat/%s.%s' &&" % (_, re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $test_section.sect_list_default_mandatory_oneformat if _]) } +#end if +#if $test_section.sect_list_default_optional_oneformat: + mkdir test_section.sect_list_default_optional_oneformat && + ${ ' '.join(["ln -s '%s' 'test_section.sect_list_default_optional_oneformat/%s.%s' &&" % (_, re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $test_section.sect_list_default_optional_oneformat if _]) } +#end if +#if $test_section.sect_list_default_mandatory_multformat: + mkdir test_section.sect_list_default_mandatory_multformat && + ${ ' '.join(["ln -s '%s' 'test_section.sect_list_default_mandatory_multformat/%s.%s' &&" % (_, re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $test_section.sect_list_default_mandatory_multformat if _]) } +#end if +#if $test_section.sect_list_default_optional_multformat: + mkdir test_section.sect_list_default_optional_multformat && + ${ ' '.join(["ln -s '%s' 'test_section.sect_list_default_optional_multformat/%s.%s' &&" % (_, re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $test_section.sect_list_default_optional_multformat if _]) } +#end if #if $adv_opts_cond.adv_opts_selector=='advanced': - #if $adv_opts_cond.mandatory_noformat_adv: - mkdir mandatory_noformat_adv && - ln -s '$mandatory_noformat_adv' 'mandatory_noformat_adv/${re.sub("[^\w\-_]", "_", $mandatory_noformat_adv.element_identifier)}.$gxy2omsext($mandatory_noformat_adv.ext)' && - #end if + mkdir adv_opts_cond.mandatory_noformat_adv && + ln -s '$adv_opts_cond.mandatory_noformat_adv' 'adv_opts_cond.mandatory_noformat_adv/${re.sub("[^\w\-_]", "_", $adv_opts_cond.mandatory_noformat_adv.element_identifier)}.$gxy2omsext($adv_opts_cond.mandatory_noformat_adv.ext)' && #if $adv_opts_cond.optional_noformat_adv: - mkdir optional_noformat_adv && - ln -s '$optional_noformat_adv' 'optional_noformat_adv/${re.sub("[^\w\-_]", "_", $optional_noformat_adv.element_identifier)}.$gxy2omsext($optional_noformat_adv.ext)' && - #end if - #if $adv_opts_cond.mandatory_oneformat_adv: - mkdir mandatory_oneformat_adv && - ln -s '$mandatory_oneformat_adv' 'mandatory_oneformat_adv/${re.sub("[^\w\-_]", "_", $mandatory_oneformat_adv.element_identifier)}.$gxy2omsext($mandatory_oneformat_adv.ext)' && + mkdir adv_opts_cond.optional_noformat_adv && + ln -s '$adv_opts_cond.optional_noformat_adv' 'adv_opts_cond.optional_noformat_adv/${re.sub("[^\w\-_]", "_", $adv_opts_cond.optional_noformat_adv.element_identifier)}.$gxy2omsext($adv_opts_cond.optional_noformat_adv.ext)' && #end if + mkdir adv_opts_cond.mandatory_oneformat_adv && + ln -s '$adv_opts_cond.mandatory_oneformat_adv' 'adv_opts_cond.mandatory_oneformat_adv/${re.sub("[^\w\-_]", "_", $adv_opts_cond.mandatory_oneformat_adv.element_identifier)}.$gxy2omsext($adv_opts_cond.mandatory_oneformat_adv.ext)' && #if $adv_opts_cond.optional_oneformat_adv: - mkdir optional_oneformat_adv && - ln -s '$optional_oneformat_adv' 'optional_oneformat_adv/${re.sub("[^\w\-_]", "_", $optional_oneformat_adv.element_identifier)}.$gxy2omsext($optional_oneformat_adv.ext)' && - #end if - #if $adv_opts_cond.mandatory_multformat_adv: - mkdir mandatory_multformat_adv && - ln -s '$mandatory_multformat_adv' 'mandatory_multformat_adv/${re.sub("[^\w\-_]", "_", $mandatory_multformat_adv.element_identifier)}.$gxy2omsext($mandatory_multformat_adv.ext)' && + mkdir adv_opts_cond.optional_oneformat_adv && + ln -s '$adv_opts_cond.optional_oneformat_adv' 'adv_opts_cond.optional_oneformat_adv/${re.sub("[^\w\-_]", "_", $adv_opts_cond.optional_oneformat_adv.element_identifier)}.$gxy2omsext($adv_opts_cond.optional_oneformat_adv.ext)' && #end if + mkdir adv_opts_cond.mandatory_multformat_adv && + ln -s '$adv_opts_cond.mandatory_multformat_adv' 'adv_opts_cond.mandatory_multformat_adv/${re.sub("[^\w\-_]", "_", $adv_opts_cond.mandatory_multformat_adv.element_identifier)}.$gxy2omsext($adv_opts_cond.mandatory_multformat_adv.ext)' && #if $adv_opts_cond.optional_multformat_adv: - mkdir optional_multformat_adv && - ln -s '$optional_multformat_adv' 'optional_multformat_adv/${re.sub("[^\w\-_]", "_", $optional_multformat_adv.element_identifier)}.$gxy2omsext($optional_multformat_adv.ext)' && + mkdir adv_opts_cond.optional_multformat_adv && + ln -s '$adv_opts_cond.optional_multformat_adv' 'adv_opts_cond.optional_multformat_adv/${re.sub("[^\w\-_]", "_", $adv_opts_cond.optional_multformat_adv.element_identifier)}.$gxy2omsext($adv_opts_cond.optional_multformat_adv.ext)' && #end if #if $adv_opts_cond.default_mandatory_noformat_adv: - mkdir default_mandatory_noformat_adv && - ln -s '$default_mandatory_noformat_adv' 'default_mandatory_noformat_adv/${re.sub("[^\w\-_]", "_", $default_mandatory_noformat_adv.element_identifier)}.$gxy2omsext($default_mandatory_noformat_adv.ext)' && + mkdir adv_opts_cond.default_mandatory_noformat_adv && + ln -s '$adv_opts_cond.default_mandatory_noformat_adv' 'adv_opts_cond.default_mandatory_noformat_adv/${re.sub("[^\w\-_]", "_", $adv_opts_cond.default_mandatory_noformat_adv.element_identifier)}.$gxy2omsext($adv_opts_cond.default_mandatory_noformat_adv.ext)' && #end if #if $adv_opts_cond.default_optional_noformat_adv: - mkdir default_optional_noformat_adv && - ln -s '$default_optional_noformat_adv' 'default_optional_noformat_adv/${re.sub("[^\w\-_]", "_", $default_optional_noformat_adv.element_identifier)}.$gxy2omsext($default_optional_noformat_adv.ext)' && + mkdir adv_opts_cond.default_optional_noformat_adv && + ln -s '$adv_opts_cond.default_optional_noformat_adv' 'adv_opts_cond.default_optional_noformat_adv/${re.sub("[^\w\-_]", "_", $adv_opts_cond.default_optional_noformat_adv.element_identifier)}.$gxy2omsext($adv_opts_cond.default_optional_noformat_adv.ext)' && #end if #if $adv_opts_cond.default_mandatory_oneformat_adv: - mkdir default_mandatory_oneformat_adv && - ln -s '$default_mandatory_oneformat_adv' 'default_mandatory_oneformat_adv/${re.sub("[^\w\-_]", "_", $default_mandatory_oneformat_adv.element_identifier)}.$gxy2omsext($default_mandatory_oneformat_adv.ext)' && + mkdir adv_opts_cond.default_mandatory_oneformat_adv && + ln -s '$adv_opts_cond.default_mandatory_oneformat_adv' 'adv_opts_cond.default_mandatory_oneformat_adv/${re.sub("[^\w\-_]", "_", $adv_opts_cond.default_mandatory_oneformat_adv.element_identifier)}.$gxy2omsext($adv_opts_cond.default_mandatory_oneformat_adv.ext)' && #end if #if $adv_opts_cond.default_optional_oneformat_adv: - mkdir default_optional_oneformat_adv && - ln -s '$default_optional_oneformat_adv' 'default_optional_oneformat_adv/${re.sub("[^\w\-_]", "_", $default_optional_oneformat_adv.element_identifier)}.$gxy2omsext($default_optional_oneformat_adv.ext)' && + mkdir adv_opts_cond.default_optional_oneformat_adv && + ln -s '$adv_opts_cond.default_optional_oneformat_adv' 'adv_opts_cond.default_optional_oneformat_adv/${re.sub("[^\w\-_]", "_", $adv_opts_cond.default_optional_oneformat_adv.element_identifier)}.$gxy2omsext($adv_opts_cond.default_optional_oneformat_adv.ext)' && #end if #if $adv_opts_cond.default_mandatory_multformat_adv: - mkdir default_mandatory_multformat_adv && - ln -s '$default_mandatory_multformat_adv' 'default_mandatory_multformat_adv/${re.sub("[^\w\-_]", "_", $default_mandatory_multformat_adv.element_identifier)}.$gxy2omsext($default_mandatory_multformat_adv.ext)' && + mkdir adv_opts_cond.default_mandatory_multformat_adv && + ln -s '$adv_opts_cond.default_mandatory_multformat_adv' 'adv_opts_cond.default_mandatory_multformat_adv/${re.sub("[^\w\-_]", "_", $adv_opts_cond.default_mandatory_multformat_adv.element_identifier)}.$gxy2omsext($adv_opts_cond.default_mandatory_multformat_adv.ext)' && #end if #if $adv_opts_cond.default_optional_multformat_adv: - mkdir default_optional_multformat_adv && - ln -s '$default_optional_multformat_adv' 'default_optional_multformat_adv/${re.sub("[^\w\-_]", "_", $default_optional_multformat_adv.element_identifier)}.$gxy2omsext($default_optional_multformat_adv.ext)' && - #end if - #if $adv_opts_cond.list_mandatory_noformat_adv: - mkdir list_mandatory_noformat_adv && - ${ ' '.join(["ln -s '%s' 'list_mandatory_noformat_adv/%s.%s' &&" % (_, re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $list_mandatory_noformat_adv if _]) } + mkdir adv_opts_cond.default_optional_multformat_adv && + ln -s '$adv_opts_cond.default_optional_multformat_adv' 'adv_opts_cond.default_optional_multformat_adv/${re.sub("[^\w\-_]", "_", $adv_opts_cond.default_optional_multformat_adv.element_identifier)}.$gxy2omsext($adv_opts_cond.default_optional_multformat_adv.ext)' && #end if + mkdir adv_opts_cond.list_mandatory_noformat_adv && + ${ ' '.join(["ln -s '%s' 'adv_opts_cond.list_mandatory_noformat_adv/%s.%s' &&" % (_, re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $adv_opts_cond.list_mandatory_noformat_adv if _]) } #if $adv_opts_cond.list_optional_noformat_adv: - mkdir list_optional_noformat_adv && - ${ ' '.join(["ln -s '%s' 'list_optional_noformat_adv/%s.%s' &&" % (_, re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $list_optional_noformat_adv if _]) } - #end if - #if $adv_opts_cond.list_mandatory_oneformat_adv: - mkdir list_mandatory_oneformat_adv && - ${ ' '.join(["ln -s '%s' 'list_mandatory_oneformat_adv/%s.%s' &&" % (_, re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $list_mandatory_oneformat_adv if _]) } + mkdir adv_opts_cond.list_optional_noformat_adv && + ${ ' '.join(["ln -s '%s' 'adv_opts_cond.list_optional_noformat_adv/%s.%s' &&" % (_, re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $adv_opts_cond.list_optional_noformat_adv if _]) } #end if + mkdir adv_opts_cond.list_mandatory_oneformat_adv && + ${ ' '.join(["ln -s '%s' 'adv_opts_cond.list_mandatory_oneformat_adv/%s.%s' &&" % (_, re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $adv_opts_cond.list_mandatory_oneformat_adv if _]) } #if $adv_opts_cond.list_optional_oneformat_adv: - mkdir list_optional_oneformat_adv && - ${ ' '.join(["ln -s '%s' 'list_optional_oneformat_adv/%s.%s' &&" % (_, re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $list_optional_oneformat_adv if _]) } - #end if - #if $adv_opts_cond.list_mandatory_multformat_adv: - mkdir list_mandatory_multformat_adv && - ${ ' '.join(["ln -s '%s' 'list_mandatory_multformat_adv/%s.%s' &&" % (_, re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $list_mandatory_multformat_adv if _]) } + mkdir adv_opts_cond.list_optional_oneformat_adv && + ${ ' '.join(["ln -s '%s' 'adv_opts_cond.list_optional_oneformat_adv/%s.%s' &&" % (_, re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $adv_opts_cond.list_optional_oneformat_adv if _]) } #end if + mkdir adv_opts_cond.list_mandatory_multformat_adv && + ${ ' '.join(["ln -s '%s' 'adv_opts_cond.list_mandatory_multformat_adv/%s.%s' &&" % (_, re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $adv_opts_cond.list_mandatory_multformat_adv if _]) } #if $adv_opts_cond.list_optional_multformat_adv: - mkdir list_optional_multformat_adv && - ${ ' '.join(["ln -s '%s' 'list_optional_multformat_adv/%s.%s' &&" % (_, re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $list_optional_multformat_adv if _]) } + mkdir adv_opts_cond.list_optional_multformat_adv && + ${ ' '.join(["ln -s '%s' 'adv_opts_cond.list_optional_multformat_adv/%s.%s' &&" % (_, re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $adv_opts_cond.list_optional_multformat_adv if _]) } #end if #if $adv_opts_cond.list_default_mandatory_noformat_adv: - mkdir list_default_mandatory_noformat_adv && - ${ ' '.join(["ln -s '%s' 'list_default_mandatory_noformat_adv/%s.%s' &&" % (_, re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $list_default_mandatory_noformat_adv if _]) } + mkdir adv_opts_cond.list_default_mandatory_noformat_adv && + ${ ' '.join(["ln -s '%s' 'adv_opts_cond.list_default_mandatory_noformat_adv/%s.%s' &&" % (_, re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $adv_opts_cond.list_default_mandatory_noformat_adv if _]) } #end if #if $adv_opts_cond.list_default_optional_noformat_adv: - mkdir list_default_optional_noformat_adv && - ${ ' '.join(["ln -s '%s' 'list_default_optional_noformat_adv/%s.%s' &&" % (_, re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $list_default_optional_noformat_adv if _]) } + mkdir adv_opts_cond.list_default_optional_noformat_adv && + ${ ' '.join(["ln -s '%s' 'adv_opts_cond.list_default_optional_noformat_adv/%s.%s' &&" % (_, re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $adv_opts_cond.list_default_optional_noformat_adv if _]) } #end if #if $adv_opts_cond.list_default_mandatory_oneformat_adv: - mkdir list_default_mandatory_oneformat_adv && - ${ ' '.join(["ln -s '%s' 'list_default_mandatory_oneformat_adv/%s.%s' &&" % (_, re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $list_default_mandatory_oneformat_adv if _]) } + mkdir adv_opts_cond.list_default_mandatory_oneformat_adv && + ${ ' '.join(["ln -s '%s' 'adv_opts_cond.list_default_mandatory_oneformat_adv/%s.%s' &&" % (_, re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $adv_opts_cond.list_default_mandatory_oneformat_adv if _]) } #end if #if $adv_opts_cond.list_default_optional_oneformat_adv: - mkdir list_default_optional_oneformat_adv && - ${ ' '.join(["ln -s '%s' 'list_default_optional_oneformat_adv/%s.%s' &&" % (_, re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $list_default_optional_oneformat_adv if _]) } + mkdir adv_opts_cond.list_default_optional_oneformat_adv && + ${ ' '.join(["ln -s '%s' 'adv_opts_cond.list_default_optional_oneformat_adv/%s.%s' &&" % (_, re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $adv_opts_cond.list_default_optional_oneformat_adv if _]) } #end if #if $adv_opts_cond.list_default_mandatory_multformat_adv: - mkdir list_default_mandatory_multformat_adv && - ${ ' '.join(["ln -s '%s' 'list_default_mandatory_multformat_adv/%s.%s' &&" % (_, re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $list_default_mandatory_multformat_adv if _]) } + mkdir adv_opts_cond.list_default_mandatory_multformat_adv && + ${ ' '.join(["ln -s '%s' 'adv_opts_cond.list_default_mandatory_multformat_adv/%s.%s' &&" % (_, re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $adv_opts_cond.list_default_mandatory_multformat_adv if _]) } #end if #if $adv_opts_cond.list_default_optional_multformat_adv: - mkdir list_default_optional_multformat_adv && - ${ ' '.join(["ln -s '%s' 'list_default_optional_multformat_adv/%s.%s' &&" % (_, re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $list_default_optional_multformat_adv if _]) } + mkdir adv_opts_cond.list_default_optional_multformat_adv && + ${ ' '.join(["ln -s '%s' 'adv_opts_cond.list_default_optional_multformat_adv/%s.%s' &&" % (_, re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $adv_opts_cond.list_default_optional_multformat_adv if _]) } + #end if + mkdir test_section.sect_mandatory_noformat_adv && + ln -s '$test_section.sect_mandatory_noformat_adv' 'test_section.sect_mandatory_noformat_adv/${re.sub("[^\w\-_]", "_", $test_section.sect_mandatory_noformat_adv.element_identifier)}.$gxy2omsext($test_section.sect_mandatory_noformat_adv.ext)' && + #if $test_section.sect_optional_noformat_adv: + mkdir test_section.sect_optional_noformat_adv && + ln -s '$test_section.sect_optional_noformat_adv' 'test_section.sect_optional_noformat_adv/${re.sub("[^\w\-_]", "_", $test_section.sect_optional_noformat_adv.element_identifier)}.$gxy2omsext($test_section.sect_optional_noformat_adv.ext)' && + #end if + mkdir test_section.sect_mandatory_oneformat_adv && + ln -s '$test_section.sect_mandatory_oneformat_adv' 'test_section.sect_mandatory_oneformat_adv/${re.sub("[^\w\-_]", "_", $test_section.sect_mandatory_oneformat_adv.element_identifier)}.$gxy2omsext($test_section.sect_mandatory_oneformat_adv.ext)' && + #if $test_section.sect_optional_oneformat_adv: + mkdir test_section.sect_optional_oneformat_adv && + ln -s '$test_section.sect_optional_oneformat_adv' 'test_section.sect_optional_oneformat_adv/${re.sub("[^\w\-_]", "_", $test_section.sect_optional_oneformat_adv.element_identifier)}.$gxy2omsext($test_section.sect_optional_oneformat_adv.ext)' && + #end if + mkdir test_section.sect_mandatory_multformat_adv && + ln -s '$test_section.sect_mandatory_multformat_adv' 'test_section.sect_mandatory_multformat_adv/${re.sub("[^\w\-_]", "_", $test_section.sect_mandatory_multformat_adv.element_identifier)}.$gxy2omsext($test_section.sect_mandatory_multformat_adv.ext)' && + #if $test_section.sect_optional_multformat_adv: + mkdir test_section.sect_optional_multformat_adv && + ln -s '$test_section.sect_optional_multformat_adv' 'test_section.sect_optional_multformat_adv/${re.sub("[^\w\-_]", "_", $test_section.sect_optional_multformat_adv.element_identifier)}.$gxy2omsext($test_section.sect_optional_multformat_adv.ext)' && + #end if + #if $test_section.sect_default_mandatory_noformat_adv: + mkdir test_section.sect_default_mandatory_noformat_adv && + ln -s '$test_section.sect_default_mandatory_noformat_adv' 'test_section.sect_default_mandatory_noformat_adv/${re.sub("[^\w\-_]", "_", $test_section.sect_default_mandatory_noformat_adv.element_identifier)}.$gxy2omsext($test_section.sect_default_mandatory_noformat_adv.ext)' && + #end if + #if $test_section.sect_default_optional_noformat_adv: + mkdir test_section.sect_default_optional_noformat_adv && + ln -s '$test_section.sect_default_optional_noformat_adv' 'test_section.sect_default_optional_noformat_adv/${re.sub("[^\w\-_]", "_", $test_section.sect_default_optional_noformat_adv.element_identifier)}.$gxy2omsext($test_section.sect_default_optional_noformat_adv.ext)' && + #end if + #if $test_section.sect_default_mandatory_oneformat_adv: + mkdir test_section.sect_default_mandatory_oneformat_adv && + ln -s '$test_section.sect_default_mandatory_oneformat_adv' 'test_section.sect_default_mandatory_oneformat_adv/${re.sub("[^\w\-_]", "_", $test_section.sect_default_mandatory_oneformat_adv.element_identifier)}.$gxy2omsext($test_section.sect_default_mandatory_oneformat_adv.ext)' && + #end if + #if $test_section.sect_default_optional_oneformat_adv: + mkdir test_section.sect_default_optional_oneformat_adv && + ln -s '$test_section.sect_default_optional_oneformat_adv' 'test_section.sect_default_optional_oneformat_adv/${re.sub("[^\w\-_]", "_", $test_section.sect_default_optional_oneformat_adv.element_identifier)}.$gxy2omsext($test_section.sect_default_optional_oneformat_adv.ext)' && + #end if + #if $test_section.sect_default_mandatory_multformat_adv: + mkdir test_section.sect_default_mandatory_multformat_adv && + ln -s '$test_section.sect_default_mandatory_multformat_adv' 'test_section.sect_default_mandatory_multformat_adv/${re.sub("[^\w\-_]", "_", $test_section.sect_default_mandatory_multformat_adv.element_identifier)}.$gxy2omsext($test_section.sect_default_mandatory_multformat_adv.ext)' && + #end if + #if $test_section.sect_default_optional_multformat_adv: + mkdir test_section.sect_default_optional_multformat_adv && + ln -s '$test_section.sect_default_optional_multformat_adv' 'test_section.sect_default_optional_multformat_adv/${re.sub("[^\w\-_]", "_", $test_section.sect_default_optional_multformat_adv.element_identifier)}.$gxy2omsext($test_section.sect_default_optional_multformat_adv.ext)' && + #end if + mkdir test_section.sect_list_mandatory_noformat_adv && + ${ ' '.join(["ln -s '%s' 'test_section.sect_list_mandatory_noformat_adv/%s.%s' &&" % (_, re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $test_section.sect_list_mandatory_noformat_adv if _]) } + #if $test_section.sect_list_optional_noformat_adv: + mkdir test_section.sect_list_optional_noformat_adv && + ${ ' '.join(["ln -s '%s' 'test_section.sect_list_optional_noformat_adv/%s.%s' &&" % (_, re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $test_section.sect_list_optional_noformat_adv if _]) } + #end if + mkdir test_section.sect_list_mandatory_oneformat_adv && + ${ ' '.join(["ln -s '%s' 'test_section.sect_list_mandatory_oneformat_adv/%s.%s' &&" % (_, re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $test_section.sect_list_mandatory_oneformat_adv if _]) } + #if $test_section.sect_list_optional_oneformat_adv: + mkdir test_section.sect_list_optional_oneformat_adv && + ${ ' '.join(["ln -s '%s' 'test_section.sect_list_optional_oneformat_adv/%s.%s' &&" % (_, re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $test_section.sect_list_optional_oneformat_adv if _]) } + #end if + mkdir test_section.sect_list_mandatory_multformat_adv && + ${ ' '.join(["ln -s '%s' 'test_section.sect_list_mandatory_multformat_adv/%s.%s' &&" % (_, re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $test_section.sect_list_mandatory_multformat_adv if _]) } + #if $test_section.sect_list_optional_multformat_adv: + mkdir test_section.sect_list_optional_multformat_adv && + ${ ' '.join(["ln -s '%s' 'test_section.sect_list_optional_multformat_adv/%s.%s' &&" % (_, re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $test_section.sect_list_optional_multformat_adv if _]) } + #end if + #if $test_section.sect_list_default_mandatory_noformat_adv: + mkdir test_section.sect_list_default_mandatory_noformat_adv && + ${ ' '.join(["ln -s '%s' 'test_section.sect_list_default_mandatory_noformat_adv/%s.%s' &&" % (_, re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $test_section.sect_list_default_mandatory_noformat_adv if _]) } + #end if + #if $test_section.sect_list_default_optional_noformat_adv: + mkdir test_section.sect_list_default_optional_noformat_adv && + ${ ' '.join(["ln -s '%s' 'test_section.sect_list_default_optional_noformat_adv/%s.%s' &&" % (_, re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $test_section.sect_list_default_optional_noformat_adv if _]) } + #end if + #if $test_section.sect_list_default_mandatory_oneformat_adv: + mkdir test_section.sect_list_default_mandatory_oneformat_adv && + ${ ' '.join(["ln -s '%s' 'test_section.sect_list_default_mandatory_oneformat_adv/%s.%s' &&" % (_, re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $test_section.sect_list_default_mandatory_oneformat_adv if _]) } + #end if + #if $test_section.sect_list_default_optional_oneformat_adv: + mkdir test_section.sect_list_default_optional_oneformat_adv && + ${ ' '.join(["ln -s '%s' 'test_section.sect_list_default_optional_oneformat_adv/%s.%s' &&" % (_, re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $test_section.sect_list_default_optional_oneformat_adv if _]) } + #end if + #if $test_section.sect_list_default_mandatory_multformat_adv: + mkdir test_section.sect_list_default_mandatory_multformat_adv && + ${ ' '.join(["ln -s '%s' 'test_section.sect_list_default_mandatory_multformat_adv/%s.%s' &&" % (_, re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $test_section.sect_list_default_mandatory_multformat_adv if _]) } + #end if + #if $test_section.sect_list_default_optional_multformat_adv: + mkdir test_section.sect_list_default_optional_multformat_adv && + ${ ' '.join(["ln -s '%s' 'test_section.sect_list_default_optional_multformat_adv/%s.%s' &&" % (_, re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $test_section.sect_list_default_optional_multformat_adv if _]) } #end if #end if ## Main program call + +export PYTHONPATH='$__tool_directory__/CTDopts' && @EXECUTABLE@ -write_ctd ./ && -python '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' && +python3 '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' && @EXECUTABLE@ -ini @EXECUTABLE@.ctd -#if $mandatory_noformat: - -mandatory_noformat - 'mandatory_noformat/${re.sub("[^\w\-_]", "_", $mandatory_noformat.element_identifier)}.$gxy2omsext($mandatory_noformat.ext)' -#end if +-mandatory_noformat +'mandatory_noformat/${re.sub("[^\w\-_]", "_", $mandatory_noformat.element_identifier)}.$gxy2omsext($mandatory_noformat.ext)' #if $optional_noformat: -optional_noformat 'optional_noformat/${re.sub("[^\w\-_]", "_", $optional_noformat.element_identifier)}.$gxy2omsext($optional_noformat.ext)' #end if -#if $mandatory_oneformat: - -mandatory_oneformat - 'mandatory_oneformat/${re.sub("[^\w\-_]", "_", $mandatory_oneformat.element_identifier)}.$gxy2omsext($mandatory_oneformat.ext)' -#end if +-mandatory_oneformat +'mandatory_oneformat/${re.sub("[^\w\-_]", "_", $mandatory_oneformat.element_identifier)}.$gxy2omsext($mandatory_oneformat.ext)' #if $optional_oneformat: -optional_oneformat 'optional_oneformat/${re.sub("[^\w\-_]", "_", $optional_oneformat.element_identifier)}.$gxy2omsext($optional_oneformat.ext)' #end if -#if $mandatory_multformat: - -mandatory_multformat - 'mandatory_multformat/${re.sub("[^\w\-_]", "_", $mandatory_multformat.element_identifier)}.$gxy2omsext($mandatory_multformat.ext)' -#end if +-mandatory_multformat +'mandatory_multformat/${re.sub("[^\w\-_]", "_", $mandatory_multformat.element_identifier)}.$gxy2omsext($mandatory_multformat.ext)' #if $optional_multformat: -optional_multformat 'optional_multformat/${re.sub("[^\w\-_]", "_", $optional_multformat.element_identifier)}.$gxy2omsext($optional_multformat.ext)' @@ -261,26 +401,20 @@ python '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' && -default_optional_multformat 'default_optional_multformat/${re.sub("[^\w\-_]", "_", $default_optional_multformat.element_identifier)}.$gxy2omsext($default_optional_multformat.ext)' #end if -#if $list_mandatory_noformat: - -list_mandatory_noformat - ${' '.join(["'list_mandatory_noformat/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $list_mandatory_noformat if _])} -#end if +-list_mandatory_noformat +${' '.join(["'list_mandatory_noformat/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $list_mandatory_noformat if _])} #if $list_optional_noformat: -list_optional_noformat ${' '.join(["'list_optional_noformat/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $list_optional_noformat if _])} #end if -#if $list_mandatory_oneformat: - -list_mandatory_oneformat - ${' '.join(["'list_mandatory_oneformat/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $list_mandatory_oneformat if _])} -#end if +-list_mandatory_oneformat +${' '.join(["'list_mandatory_oneformat/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $list_mandatory_oneformat if _])} #if $list_optional_oneformat: -list_optional_oneformat ${' '.join(["'list_optional_oneformat/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $list_optional_oneformat if _])} #end if -#if $list_mandatory_multformat: - -list_mandatory_multformat - ${' '.join(["'list_mandatory_multformat/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $list_mandatory_multformat if _])} -#end if +-list_mandatory_multformat +${' '.join(["'list_mandatory_multformat/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $list_mandatory_multformat if _])} #if $list_optional_multformat: -list_optional_multformat ${' '.join(["'list_optional_multformat/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $list_optional_multformat if _])} @@ -309,102 +443,258 @@ python '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' && -list_default_optional_multformat ${' '.join(["'list_default_optional_multformat/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $list_default_optional_multformat if _])} #end if +-test_section:sect_mandatory_noformat +'test_section.sect_mandatory_noformat/${re.sub("[^\w\-_]", "_", $test_section.sect_mandatory_noformat.element_identifier)}.$gxy2omsext($test_section.sect_mandatory_noformat.ext)' +#if $test_section.sect_optional_noformat: + -test_section:sect_optional_noformat + 'test_section.sect_optional_noformat/${re.sub("[^\w\-_]", "_", $test_section.sect_optional_noformat.element_identifier)}.$gxy2omsext($test_section.sect_optional_noformat.ext)' +#end if +-test_section:sect_mandatory_oneformat +'test_section.sect_mandatory_oneformat/${re.sub("[^\w\-_]", "_", $test_section.sect_mandatory_oneformat.element_identifier)}.$gxy2omsext($test_section.sect_mandatory_oneformat.ext)' +#if $test_section.sect_optional_oneformat: + -test_section:sect_optional_oneformat + 'test_section.sect_optional_oneformat/${re.sub("[^\w\-_]", "_", $test_section.sect_optional_oneformat.element_identifier)}.$gxy2omsext($test_section.sect_optional_oneformat.ext)' +#end if +-test_section:sect_mandatory_multformat +'test_section.sect_mandatory_multformat/${re.sub("[^\w\-_]", "_", $test_section.sect_mandatory_multformat.element_identifier)}.$gxy2omsext($test_section.sect_mandatory_multformat.ext)' +#if $test_section.sect_optional_multformat: + -test_section:sect_optional_multformat + 'test_section.sect_optional_multformat/${re.sub("[^\w\-_]", "_", $test_section.sect_optional_multformat.element_identifier)}.$gxy2omsext($test_section.sect_optional_multformat.ext)' +#end if +#if $test_section.sect_default_mandatory_noformat: + -test_section:sect_default_mandatory_noformat + 'test_section.sect_default_mandatory_noformat/${re.sub("[^\w\-_]", "_", $test_section.sect_default_mandatory_noformat.element_identifier)}.$gxy2omsext($test_section.sect_default_mandatory_noformat.ext)' +#end if +#if $test_section.sect_default_optional_noformat: + -test_section:sect_default_optional_noformat + 'test_section.sect_default_optional_noformat/${re.sub("[^\w\-_]", "_", $test_section.sect_default_optional_noformat.element_identifier)}.$gxy2omsext($test_section.sect_default_optional_noformat.ext)' +#end if +#if $test_section.sect_default_mandatory_oneformat: + -test_section:sect_default_mandatory_oneformat + 'test_section.sect_default_mandatory_oneformat/${re.sub("[^\w\-_]", "_", $test_section.sect_default_mandatory_oneformat.element_identifier)}.$gxy2omsext($test_section.sect_default_mandatory_oneformat.ext)' +#end if +#if $test_section.sect_default_optional_oneformat: + -test_section:sect_default_optional_oneformat + 'test_section.sect_default_optional_oneformat/${re.sub("[^\w\-_]", "_", $test_section.sect_default_optional_oneformat.element_identifier)}.$gxy2omsext($test_section.sect_default_optional_oneformat.ext)' +#end if +#if $test_section.sect_default_mandatory_multformat: + -test_section:sect_default_mandatory_multformat + 'test_section.sect_default_mandatory_multformat/${re.sub("[^\w\-_]", "_", $test_section.sect_default_mandatory_multformat.element_identifier)}.$gxy2omsext($test_section.sect_default_mandatory_multformat.ext)' +#end if +#if $test_section.sect_default_optional_multformat: + -test_section:sect_default_optional_multformat + 'test_section.sect_default_optional_multformat/${re.sub("[^\w\-_]", "_", $test_section.sect_default_optional_multformat.element_identifier)}.$gxy2omsext($test_section.sect_default_optional_multformat.ext)' +#end if +-test_section:sect_list_mandatory_noformat +${' '.join(["'test_section.sect_list_mandatory_noformat/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $test_section.sect_list_mandatory_noformat if _])} +#if $test_section.sect_list_optional_noformat: + -test_section:sect_list_optional_noformat + ${' '.join(["'test_section.sect_list_optional_noformat/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $test_section.sect_list_optional_noformat if _])} +#end if +-test_section:sect_list_mandatory_oneformat +${' '.join(["'test_section.sect_list_mandatory_oneformat/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $test_section.sect_list_mandatory_oneformat if _])} +#if $test_section.sect_list_optional_oneformat: + -test_section:sect_list_optional_oneformat + ${' '.join(["'test_section.sect_list_optional_oneformat/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $test_section.sect_list_optional_oneformat if _])} +#end if +-test_section:sect_list_mandatory_multformat +${' '.join(["'test_section.sect_list_mandatory_multformat/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $test_section.sect_list_mandatory_multformat if _])} +#if $test_section.sect_list_optional_multformat: + -test_section:sect_list_optional_multformat + ${' '.join(["'test_section.sect_list_optional_multformat/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $test_section.sect_list_optional_multformat if _])} +#end if +#if $test_section.sect_list_default_mandatory_noformat: + -test_section:sect_list_default_mandatory_noformat + ${' '.join(["'test_section.sect_list_default_mandatory_noformat/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $test_section.sect_list_default_mandatory_noformat if _])} +#end if +#if $test_section.sect_list_default_optional_noformat: + -test_section:sect_list_default_optional_noformat + ${' '.join(["'test_section.sect_list_default_optional_noformat/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $test_section.sect_list_default_optional_noformat if _])} +#end if +#if $test_section.sect_list_default_mandatory_oneformat: + -test_section:sect_list_default_mandatory_oneformat + ${' '.join(["'test_section.sect_list_default_mandatory_oneformat/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $test_section.sect_list_default_mandatory_oneformat if _])} +#end if +#if $test_section.sect_list_default_optional_oneformat: + -test_section:sect_list_default_optional_oneformat + ${' '.join(["'test_section.sect_list_default_optional_oneformat/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $test_section.sect_list_default_optional_oneformat if _])} +#end if +#if $test_section.sect_list_default_mandatory_multformat: + -test_section:sect_list_default_mandatory_multformat + ${' '.join(["'test_section.sect_list_default_mandatory_multformat/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $test_section.sect_list_default_mandatory_multformat if _])} +#end if +#if $test_section.sect_list_default_optional_multformat: + -test_section:sect_list_default_optional_multformat + ${' '.join(["'test_section.sect_list_default_optional_multformat/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $test_section.sect_list_default_optional_multformat if _])} +#end if #if $adv_opts_cond.adv_opts_selector=='advanced': - #if $adv_opts_cond.mandatory_noformat_adv: - -mandatory_noformat_adv - 'mandatory_noformat_adv/${re.sub("[^\w\-_]", "_", $mandatory_noformat_adv.element_identifier)}.$gxy2omsext($mandatory_noformat_adv.ext)' - #end if + -mandatory_noformat_adv + 'adv_opts_cond.mandatory_noformat_adv/${re.sub("[^\w\-_]", "_", $adv_opts_cond.mandatory_noformat_adv.element_identifier)}.$gxy2omsext($adv_opts_cond.mandatory_noformat_adv.ext)' #if $adv_opts_cond.optional_noformat_adv: -optional_noformat_adv - 'optional_noformat_adv/${re.sub("[^\w\-_]", "_", $optional_noformat_adv.element_identifier)}.$gxy2omsext($optional_noformat_adv.ext)' - #end if - #if $adv_opts_cond.mandatory_oneformat_adv: - -mandatory_oneformat_adv - 'mandatory_oneformat_adv/${re.sub("[^\w\-_]", "_", $mandatory_oneformat_adv.element_identifier)}.$gxy2omsext($mandatory_oneformat_adv.ext)' + 'adv_opts_cond.optional_noformat_adv/${re.sub("[^\w\-_]", "_", $adv_opts_cond.optional_noformat_adv.element_identifier)}.$gxy2omsext($adv_opts_cond.optional_noformat_adv.ext)' #end if + -mandatory_oneformat_adv + 'adv_opts_cond.mandatory_oneformat_adv/${re.sub("[^\w\-_]", "_", $adv_opts_cond.mandatory_oneformat_adv.element_identifier)}.$gxy2omsext($adv_opts_cond.mandatory_oneformat_adv.ext)' #if $adv_opts_cond.optional_oneformat_adv: -optional_oneformat_adv - 'optional_oneformat_adv/${re.sub("[^\w\-_]", "_", $optional_oneformat_adv.element_identifier)}.$gxy2omsext($optional_oneformat_adv.ext)' - #end if - #if $adv_opts_cond.mandatory_multformat_adv: - -mandatory_multformat_adv - 'mandatory_multformat_adv/${re.sub("[^\w\-_]", "_", $mandatory_multformat_adv.element_identifier)}.$gxy2omsext($mandatory_multformat_adv.ext)' + 'adv_opts_cond.optional_oneformat_adv/${re.sub("[^\w\-_]", "_", $adv_opts_cond.optional_oneformat_adv.element_identifier)}.$gxy2omsext($adv_opts_cond.optional_oneformat_adv.ext)' #end if + -mandatory_multformat_adv + 'adv_opts_cond.mandatory_multformat_adv/${re.sub("[^\w\-_]", "_", $adv_opts_cond.mandatory_multformat_adv.element_identifier)}.$gxy2omsext($adv_opts_cond.mandatory_multformat_adv.ext)' #if $adv_opts_cond.optional_multformat_adv: -optional_multformat_adv - 'optional_multformat_adv/${re.sub("[^\w\-_]", "_", $optional_multformat_adv.element_identifier)}.$gxy2omsext($optional_multformat_adv.ext)' + 'adv_opts_cond.optional_multformat_adv/${re.sub("[^\w\-_]", "_", $adv_opts_cond.optional_multformat_adv.element_identifier)}.$gxy2omsext($adv_opts_cond.optional_multformat_adv.ext)' #end if #if $adv_opts_cond.default_mandatory_noformat_adv: -default_mandatory_noformat_adv - 'default_mandatory_noformat_adv/${re.sub("[^\w\-_]", "_", $default_mandatory_noformat_adv.element_identifier)}.$gxy2omsext($default_mandatory_noformat_adv.ext)' + 'adv_opts_cond.default_mandatory_noformat_adv/${re.sub("[^\w\-_]", "_", $adv_opts_cond.default_mandatory_noformat_adv.element_identifier)}.$gxy2omsext($adv_opts_cond.default_mandatory_noformat_adv.ext)' #end if #if $adv_opts_cond.default_optional_noformat_adv: -default_optional_noformat_adv - 'default_optional_noformat_adv/${re.sub("[^\w\-_]", "_", $default_optional_noformat_adv.element_identifier)}.$gxy2omsext($default_optional_noformat_adv.ext)' + 'adv_opts_cond.default_optional_noformat_adv/${re.sub("[^\w\-_]", "_", $adv_opts_cond.default_optional_noformat_adv.element_identifier)}.$gxy2omsext($adv_opts_cond.default_optional_noformat_adv.ext)' #end if #if $adv_opts_cond.default_mandatory_oneformat_adv: -default_mandatory_oneformat_adv - 'default_mandatory_oneformat_adv/${re.sub("[^\w\-_]", "_", $default_mandatory_oneformat_adv.element_identifier)}.$gxy2omsext($default_mandatory_oneformat_adv.ext)' + 'adv_opts_cond.default_mandatory_oneformat_adv/${re.sub("[^\w\-_]", "_", $adv_opts_cond.default_mandatory_oneformat_adv.element_identifier)}.$gxy2omsext($adv_opts_cond.default_mandatory_oneformat_adv.ext)' #end if #if $adv_opts_cond.default_optional_oneformat_adv: -default_optional_oneformat_adv - 'default_optional_oneformat_adv/${re.sub("[^\w\-_]", "_", $default_optional_oneformat_adv.element_identifier)}.$gxy2omsext($default_optional_oneformat_adv.ext)' + 'adv_opts_cond.default_optional_oneformat_adv/${re.sub("[^\w\-_]", "_", $adv_opts_cond.default_optional_oneformat_adv.element_identifier)}.$gxy2omsext($adv_opts_cond.default_optional_oneformat_adv.ext)' #end if #if $adv_opts_cond.default_mandatory_multformat_adv: -default_mandatory_multformat_adv - 'default_mandatory_multformat_adv/${re.sub("[^\w\-_]", "_", $default_mandatory_multformat_adv.element_identifier)}.$gxy2omsext($default_mandatory_multformat_adv.ext)' + 'adv_opts_cond.default_mandatory_multformat_adv/${re.sub("[^\w\-_]", "_", $adv_opts_cond.default_mandatory_multformat_adv.element_identifier)}.$gxy2omsext($adv_opts_cond.default_mandatory_multformat_adv.ext)' #end if #if $adv_opts_cond.default_optional_multformat_adv: -default_optional_multformat_adv - 'default_optional_multformat_adv/${re.sub("[^\w\-_]", "_", $default_optional_multformat_adv.element_identifier)}.$gxy2omsext($default_optional_multformat_adv.ext)' - #end if - #if $adv_opts_cond.list_mandatory_noformat_adv: - -list_mandatory_noformat_adv - ${' '.join(["'list_mandatory_noformat_adv/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $list_mandatory_noformat_adv if _])} + 'adv_opts_cond.default_optional_multformat_adv/${re.sub("[^\w\-_]", "_", $adv_opts_cond.default_optional_multformat_adv.element_identifier)}.$gxy2omsext($adv_opts_cond.default_optional_multformat_adv.ext)' #end if + -list_mandatory_noformat_adv + ${' '.join(["'adv_opts_cond.list_mandatory_noformat_adv/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $adv_opts_cond.list_mandatory_noformat_adv if _])} #if $adv_opts_cond.list_optional_noformat_adv: -list_optional_noformat_adv - ${' '.join(["'list_optional_noformat_adv/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $list_optional_noformat_adv if _])} - #end if - #if $adv_opts_cond.list_mandatory_oneformat_adv: - -list_mandatory_oneformat_adv - ${' '.join(["'list_mandatory_oneformat_adv/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $list_mandatory_oneformat_adv if _])} + ${' '.join(["'adv_opts_cond.list_optional_noformat_adv/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $adv_opts_cond.list_optional_noformat_adv if _])} #end if + -list_mandatory_oneformat_adv + ${' '.join(["'adv_opts_cond.list_mandatory_oneformat_adv/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $adv_opts_cond.list_mandatory_oneformat_adv if _])} #if $adv_opts_cond.list_optional_oneformat_adv: -list_optional_oneformat_adv - ${' '.join(["'list_optional_oneformat_adv/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $list_optional_oneformat_adv if _])} - #end if - #if $adv_opts_cond.list_mandatory_multformat_adv: - -list_mandatory_multformat_adv - ${' '.join(["'list_mandatory_multformat_adv/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $list_mandatory_multformat_adv if _])} + ${' '.join(["'adv_opts_cond.list_optional_oneformat_adv/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $adv_opts_cond.list_optional_oneformat_adv if _])} #end if + -list_mandatory_multformat_adv + ${' '.join(["'adv_opts_cond.list_mandatory_multformat_adv/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $adv_opts_cond.list_mandatory_multformat_adv if _])} #if $adv_opts_cond.list_optional_multformat_adv: -list_optional_multformat_adv - ${' '.join(["'list_optional_multformat_adv/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $list_optional_multformat_adv if _])} + ${' '.join(["'adv_opts_cond.list_optional_multformat_adv/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $adv_opts_cond.list_optional_multformat_adv if _])} #end if #if $adv_opts_cond.list_default_mandatory_noformat_adv: -list_default_mandatory_noformat_adv - ${' '.join(["'list_default_mandatory_noformat_adv/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $list_default_mandatory_noformat_adv if _])} + ${' '.join(["'adv_opts_cond.list_default_mandatory_noformat_adv/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $adv_opts_cond.list_default_mandatory_noformat_adv if _])} #end if #if $adv_opts_cond.list_default_optional_noformat_adv: -list_default_optional_noformat_adv - ${' '.join(["'list_default_optional_noformat_adv/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $list_default_optional_noformat_adv if _])} + ${' '.join(["'adv_opts_cond.list_default_optional_noformat_adv/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $adv_opts_cond.list_default_optional_noformat_adv if _])} #end if #if $adv_opts_cond.list_default_mandatory_oneformat_adv: -list_default_mandatory_oneformat_adv - ${' '.join(["'list_default_mandatory_oneformat_adv/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $list_default_mandatory_oneformat_adv if _])} + ${' '.join(["'adv_opts_cond.list_default_mandatory_oneformat_adv/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $adv_opts_cond.list_default_mandatory_oneformat_adv if _])} #end if #if $adv_opts_cond.list_default_optional_oneformat_adv: -list_default_optional_oneformat_adv - ${' '.join(["'list_default_optional_oneformat_adv/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $list_default_optional_oneformat_adv if _])} + ${' '.join(["'adv_opts_cond.list_default_optional_oneformat_adv/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $adv_opts_cond.list_default_optional_oneformat_adv if _])} #end if #if $adv_opts_cond.list_default_mandatory_multformat_adv: -list_default_mandatory_multformat_adv - ${' '.join(["'list_default_mandatory_multformat_adv/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $list_default_mandatory_multformat_adv if _])} + ${' '.join(["'adv_opts_cond.list_default_mandatory_multformat_adv/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $adv_opts_cond.list_default_mandatory_multformat_adv if _])} #end if #if $adv_opts_cond.list_default_optional_multformat_adv: -list_default_optional_multformat_adv - ${' '.join(["'list_default_optional_multformat_adv/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $list_default_optional_multformat_adv if _])} + ${' '.join(["'adv_opts_cond.list_default_optional_multformat_adv/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $adv_opts_cond.list_default_optional_multformat_adv if _])} + #end if + -test_section:sect_mandatory_noformat_adv + 'test_section.sect_mandatory_noformat_adv/${re.sub("[^\w\-_]", "_", $test_section.sect_mandatory_noformat_adv.element_identifier)}.$gxy2omsext($test_section.sect_mandatory_noformat_adv.ext)' + #if $test_section.sect_optional_noformat_adv: + -test_section:sect_optional_noformat_adv + 'test_section.sect_optional_noformat_adv/${re.sub("[^\w\-_]", "_", $test_section.sect_optional_noformat_adv.element_identifier)}.$gxy2omsext($test_section.sect_optional_noformat_adv.ext)' + #end if + -test_section:sect_mandatory_oneformat_adv + 'test_section.sect_mandatory_oneformat_adv/${re.sub("[^\w\-_]", "_", $test_section.sect_mandatory_oneformat_adv.element_identifier)}.$gxy2omsext($test_section.sect_mandatory_oneformat_adv.ext)' + #if $test_section.sect_optional_oneformat_adv: + -test_section:sect_optional_oneformat_adv + 'test_section.sect_optional_oneformat_adv/${re.sub("[^\w\-_]", "_", $test_section.sect_optional_oneformat_adv.element_identifier)}.$gxy2omsext($test_section.sect_optional_oneformat_adv.ext)' + #end if + -test_section:sect_mandatory_multformat_adv + 'test_section.sect_mandatory_multformat_adv/${re.sub("[^\w\-_]", "_", $test_section.sect_mandatory_multformat_adv.element_identifier)}.$gxy2omsext($test_section.sect_mandatory_multformat_adv.ext)' + #if $test_section.sect_optional_multformat_adv: + -test_section:sect_optional_multformat_adv + 'test_section.sect_optional_multformat_adv/${re.sub("[^\w\-_]", "_", $test_section.sect_optional_multformat_adv.element_identifier)}.$gxy2omsext($test_section.sect_optional_multformat_adv.ext)' + #end if + #if $test_section.sect_default_mandatory_noformat_adv: + -test_section:sect_default_mandatory_noformat_adv + 'test_section.sect_default_mandatory_noformat_adv/${re.sub("[^\w\-_]", "_", $test_section.sect_default_mandatory_noformat_adv.element_identifier)}.$gxy2omsext($test_section.sect_default_mandatory_noformat_adv.ext)' + #end if + #if $test_section.sect_default_optional_noformat_adv: + -test_section:sect_default_optional_noformat_adv + 'test_section.sect_default_optional_noformat_adv/${re.sub("[^\w\-_]", "_", $test_section.sect_default_optional_noformat_adv.element_identifier)}.$gxy2omsext($test_section.sect_default_optional_noformat_adv.ext)' + #end if + #if $test_section.sect_default_mandatory_oneformat_adv: + -test_section:sect_default_mandatory_oneformat_adv + 'test_section.sect_default_mandatory_oneformat_adv/${re.sub("[^\w\-_]", "_", $test_section.sect_default_mandatory_oneformat_adv.element_identifier)}.$gxy2omsext($test_section.sect_default_mandatory_oneformat_adv.ext)' + #end if + #if $test_section.sect_default_optional_oneformat_adv: + -test_section:sect_default_optional_oneformat_adv + 'test_section.sect_default_optional_oneformat_adv/${re.sub("[^\w\-_]", "_", $test_section.sect_default_optional_oneformat_adv.element_identifier)}.$gxy2omsext($test_section.sect_default_optional_oneformat_adv.ext)' + #end if + #if $test_section.sect_default_mandatory_multformat_adv: + -test_section:sect_default_mandatory_multformat_adv + 'test_section.sect_default_mandatory_multformat_adv/${re.sub("[^\w\-_]", "_", $test_section.sect_default_mandatory_multformat_adv.element_identifier)}.$gxy2omsext($test_section.sect_default_mandatory_multformat_adv.ext)' + #end if + #if $test_section.sect_default_optional_multformat_adv: + -test_section:sect_default_optional_multformat_adv + 'test_section.sect_default_optional_multformat_adv/${re.sub("[^\w\-_]", "_", $test_section.sect_default_optional_multformat_adv.element_identifier)}.$gxy2omsext($test_section.sect_default_optional_multformat_adv.ext)' + #end if + -test_section:sect_list_mandatory_noformat_adv + ${' '.join(["'test_section.sect_list_mandatory_noformat_adv/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $test_section.sect_list_mandatory_noformat_adv if _])} + #if $test_section.sect_list_optional_noformat_adv: + -test_section:sect_list_optional_noformat_adv + ${' '.join(["'test_section.sect_list_optional_noformat_adv/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $test_section.sect_list_optional_noformat_adv if _])} + #end if + -test_section:sect_list_mandatory_oneformat_adv + ${' '.join(["'test_section.sect_list_mandatory_oneformat_adv/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $test_section.sect_list_mandatory_oneformat_adv if _])} + #if $test_section.sect_list_optional_oneformat_adv: + -test_section:sect_list_optional_oneformat_adv + ${' '.join(["'test_section.sect_list_optional_oneformat_adv/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $test_section.sect_list_optional_oneformat_adv if _])} + #end if + -test_section:sect_list_mandatory_multformat_adv + ${' '.join(["'test_section.sect_list_mandatory_multformat_adv/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $test_section.sect_list_mandatory_multformat_adv if _])} + #if $test_section.sect_list_optional_multformat_adv: + -test_section:sect_list_optional_multformat_adv + ${' '.join(["'test_section.sect_list_optional_multformat_adv/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $test_section.sect_list_optional_multformat_adv if _])} + #end if + #if $test_section.sect_list_default_mandatory_noformat_adv: + -test_section:sect_list_default_mandatory_noformat_adv + ${' '.join(["'test_section.sect_list_default_mandatory_noformat_adv/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $test_section.sect_list_default_mandatory_noformat_adv if _])} + #end if + #if $test_section.sect_list_default_optional_noformat_adv: + -test_section:sect_list_default_optional_noformat_adv + ${' '.join(["'test_section.sect_list_default_optional_noformat_adv/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $test_section.sect_list_default_optional_noformat_adv if _])} + #end if + #if $test_section.sect_list_default_mandatory_oneformat_adv: + -test_section:sect_list_default_mandatory_oneformat_adv + ${' '.join(["'test_section.sect_list_default_mandatory_oneformat_adv/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $test_section.sect_list_default_mandatory_oneformat_adv if _])} + #end if + #if $test_section.sect_list_default_optional_oneformat_adv: + -test_section:sect_list_default_optional_oneformat_adv + ${' '.join(["'test_section.sect_list_default_optional_oneformat_adv/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $test_section.sect_list_default_optional_oneformat_adv if _])} + #end if + #if $test_section.sect_list_default_mandatory_multformat_adv: + -test_section:sect_list_default_mandatory_multformat_adv + ${' '.join(["'test_section.sect_list_default_mandatory_multformat_adv/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $test_section.sect_list_default_mandatory_multformat_adv if _])} + #end if + #if $test_section.sect_list_default_optional_multformat_adv: + -test_section:sect_list_default_optional_multformat_adv + ${' '.join(["'test_section.sect_list_default_optional_multformat_adv/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $test_section.sect_list_default_optional_multformat_adv if _])} #end if #end if > $param_stdout @@ -414,54 +704,104 @@ python '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' && - - - - - - - + + + + + + + - + - + - - - - - - - + + + + + + + - + - + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
- - - - - - - + + + + + + + - + - + - - - - - - - + + + + + + + - + - + @@ -470,7 +810,33 @@ python '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' && + + + + + + +
+ + + + + + + + + + + + +
+ + + + + + diff --git a/tests/test-data/integer.ctd b/tests/test-data/integer.ctd index d4eedcea..e05787c7 100644 --- a/tests/test-data/integer.ctd +++ b/tests/test-data/integer.ctd @@ -6,6 +6,7 @@ + @@ -41,7 +42,47 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/test-data/integer.xml b/tests/test-data/integer.xml index 8d542be9..6ee5dca5 100644 --- a/tests/test-data/integer.xml +++ b/tests/test-data/integer.xml @@ -16,8 +16,10 @@ ## Preprocessing ## Main program call + +export PYTHONPATH='$__tool_directory__/CTDopts' && @EXECUTABLE@ -write_ctd ./ && -python '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' && +python3 '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' && @EXECUTABLE@ -ini @EXECUTABLE@.ctd > $param_stdout @@ -34,6 +36,24 @@ python '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' && +
+ + + + + + + + + + + + + + + + +
@@ -54,6 +74,16 @@ python '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' && +
+ + + + + + + + +
diff --git a/tests/test-data/label-help.xml b/tests/test-data/label-help.xml index 8096552f..02d90123 100644 --- a/tests/test-data/label-help.xml +++ b/tests/test-data/label-help.xml @@ -16,8 +16,10 @@ ## Preprocessing ## Main program call + +export PYTHONPATH='$__tool_directory__/CTDopts' && @EXECUTABLE@ -write_ctd ./ && -python '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' && +python3 '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' && @EXECUTABLE@ -ini @EXECUTABLE@.ctd > $param_stdout @@ -27,35 +29,27 @@ python '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' && - - - - - - - - diff --git a/tests/test-data/mock.py b/tests/test-data/mock.py index c715ed4b..7de337ca 100755 --- a/tests/test-data/mock.py +++ b/tests/test-data/mock.py @@ -8,6 +8,7 @@ import shutil import sys +from CTDopts.CTDopts import CTDModel, _InFile, _OutFile # from argparse import ArgumentParser # parser = ArgumentParser(prog="mock.py", description="MOCK", add_help=True) @@ -17,22 +18,23 @@ # help="Process CTDFILE") # parser.add_argument('moreargs', metavar='ARGS', type=str, nargs='*', help='more arguments') # args = parser.parse_args() -print(sys.argv) wd = os.path.dirname(__file__) bn = os.path.splitext(os.path.basename(__file__))[0] if sys.argv[1] == "-write_ctd": + print("CP %s %s" %(os.path.join(wd, bn + ".ctd"), os.path.join(sys.argv[2], bn + ".ctd"))) shutil.copyfile(os.path.join(wd, bn + ".ctd"), os.path.join(sys.argv[2], bn + ".ctd")) elif sys.argv[1] == "-ini": fparam = {"input": set(), "output": set()} - with open(sys.argv[2]) as cf: - for line in cf: - m = re.search(r'type="(input|output)-file"', line) - if m is not None: - n = re.search(r'name="([^"]+)"', line) - fparam[m.group(1)].add(n.group(1)) - + model = CTDModel(from_file=sys.argv[2]) + for p in model.get_parameters(): + cli = ":".join(p.get_lineage(name_only=True)) + if p.type is _InFile: + fparam["input"].add(cli) + elif p.type is _OutFile: + fparam["output"].add(cli) + i = 3 while i < len(sys.argv): if sys.argv[i].startswith("-"): @@ -42,7 +44,6 @@ mode = "r" else: mode = "w" - while i + 1 < len(sys.argv): if sys.argv[i + 1].startswith("-"): break diff --git a/tests/test-data/ofile-corresponding-input.ctd b/tests/test-data/ofile-corresponding-input.ctd index d5043afb..170056c3 100644 --- a/tests/test-data/ofile-corresponding-input.ctd +++ b/tests/test-data/ofile-corresponding-input.ctd @@ -29,7 +29,26 @@ - + + + + + + + + + + + + + + + + + + + + diff --git a/tests/test-data/ofile-corresponding-input.xml b/tests/test-data/ofile-corresponding-input.xml index 8ccc6a2a..e7db7dbe 100644 --- a/tests/test-data/ofile-corresponding-input.xml +++ b/tests/test-data/ofile-corresponding-input.xml @@ -26,17 +26,23 @@ mkdir mandatory_multformat && #if $optional_multformat_FLAG: mkdir optional_multformat && #end if -#if $input_noformat: - mkdir input_noformat && - ln -s '$input_noformat' 'input_noformat/${re.sub("[^\w\-_]", "_", $input_noformat.element_identifier)}.$gxy2omsext($input_noformat.ext)' && +mkdir input_noformat && +ln -s '$input_noformat' 'input_noformat/${re.sub("[^\w\-_]", "_", $input_noformat.element_identifier)}.$gxy2omsext($input_noformat.ext)' && +mkdir input_oneformat && +ln -s '$input_oneformat' 'input_oneformat/${re.sub("[^\w\-_]", "_", $input_oneformat.element_identifier)}.$gxy2omsext($input_oneformat.ext)' && +mkdir input_multformat && +ln -s '$input_multformat' 'input_multformat/${re.sub("[^\w\-_]", "_", $input_multformat.element_identifier)}.$gxy2omsext($input_multformat.ext)' && +mkdir sect_mandatory_noformat && +#if $test_section.sect_optional_noformat_FLAG: + mkdir sect_optional_noformat && #end if -#if $input_oneformat: - mkdir input_oneformat && - ln -s '$input_oneformat' 'input_oneformat/${re.sub("[^\w\-_]", "_", $input_oneformat.element_identifier)}.$gxy2omsext($input_oneformat.ext)' && +mkdir sect_mandatory_oneformat && +#if $test_section.sect_optional_oneformat_FLAG: + mkdir sect_optional_oneformat && #end if -#if $input_multformat: - mkdir input_multformat && - ln -s '$input_multformat' 'input_multformat/${re.sub("[^\w\-_]", "_", $input_multformat.element_identifier)}.$gxy2omsext($input_multformat.ext)' && +mkdir sect_mandatory_multformat && +#if $test_section.sect_optional_multformat_FLAG: + mkdir sect_optional_multformat && #end if #if $adv_opts_cond.adv_opts_selector=='advanced': mkdir mandatory_noformat_adv && @@ -51,11 +57,25 @@ mkdir mandatory_multformat && #if $adv_opts_cond.optional_multformat_adv_FLAG: mkdir optional_multformat_adv && #end if + mkdir sect_mandatory_noformat_adv && + #if $test_section.sect_optional_noformat_adv_FLAG: + mkdir sect_optional_noformat_adv && + #end if + mkdir sect_mandatory_oneformat_adv && + #if $test_section.sect_optional_oneformat_adv_FLAG: + mkdir sect_optional_oneformat_adv && + #end if + mkdir sect_mandatory_multformat_adv && + #if $test_section.sect_optional_multformat_adv_FLAG: + mkdir sect_optional_multformat_adv && + #end if #end if ## Main program call + +export PYTHONPATH='$__tool_directory__/CTDopts' && @EXECUTABLE@ -write_ctd ./ && -python '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' && +python3 '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' && @EXECUTABLE@ -ini @EXECUTABLE@.ctd -mandatory_noformat 'mandatory_noformat/output.${gxy2omsext("txt")}' @@ -75,17 +95,29 @@ python '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' && -optional_multformat 'optional_multformat/output.${input_multformat.ext}' #end if -#if $input_noformat: - -input_noformat - 'input_noformat/${re.sub("[^\w\-_]", "_", $input_noformat.element_identifier)}.$gxy2omsext($input_noformat.ext)' +-input_noformat +'input_noformat/${re.sub("[^\w\-_]", "_", $input_noformat.element_identifier)}.$gxy2omsext($input_noformat.ext)' +-input_oneformat +'input_oneformat/${re.sub("[^\w\-_]", "_", $input_oneformat.element_identifier)}.$gxy2omsext($input_oneformat.ext)' +-input_multformat +'input_multformat/${re.sub("[^\w\-_]", "_", $input_multformat.element_identifier)}.$gxy2omsext($input_multformat.ext)' +-test_section:sect_mandatory_noformat +'sect_mandatory_noformat/output.${gxy2omsext("txt")}' +#if $test_section.sect_optional_noformat_FLAG: + -test_section:sect_optional_noformat + 'sect_optional_noformat/output.${gxy2omsext("txt")}' #end if -#if $input_oneformat: - -input_oneformat - 'input_oneformat/${re.sub("[^\w\-_]", "_", $input_oneformat.element_identifier)}.$gxy2omsext($input_oneformat.ext)' +-test_section:sect_mandatory_oneformat +'sect_mandatory_oneformat/output.${gxy2omsext("tsv")}' +#if $test_section.sect_optional_oneformat_FLAG: + -test_section:sect_optional_oneformat + 'sect_optional_oneformat/output.${gxy2omsext("tsv")}' #end if -#if $input_multformat: - -input_multformat - 'input_multformat/${re.sub("[^\w\-_]", "_", $input_multformat.element_identifier)}.$gxy2omsext($input_multformat.ext)' +-test_section:sect_mandatory_multformat +'sect_mandatory_multformat/output.${input_multformat.ext}' +#if $test_section.sect_optional_multformat_FLAG: + -test_section:sect_optional_multformat + 'sect_optional_multformat/output.${input_multformat.ext}' #end if #if $adv_opts_cond.adv_opts_selector=='advanced': -mandatory_noformat_adv @@ -106,6 +138,24 @@ python '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' && -optional_multformat_adv 'optional_multformat_adv/output.${input_multformat.ext}' #end if + -test_section:sect_mandatory_noformat_adv + 'sect_mandatory_noformat_adv/output.${gxy2omsext("txt")}' + #if $test_section.sect_optional_noformat_adv_FLAG: + -test_section:sect_optional_noformat_adv + 'sect_optional_noformat_adv/output.${gxy2omsext("txt")}' + #end if + -test_section:sect_mandatory_oneformat_adv + 'sect_mandatory_oneformat_adv/output.${gxy2omsext("tsv")}' + #if $test_section.sect_optional_oneformat_adv_FLAG: + -test_section:sect_optional_oneformat_adv + 'sect_optional_oneformat_adv/output.${gxy2omsext("tsv")}' + #end if + -test_section:sect_mandatory_multformat_adv + 'sect_mandatory_multformat_adv/output.${input_multformat.ext}' + #if $test_section.sect_optional_multformat_adv_FLAG: + -test_section:sect_optional_multformat_adv + 'sect_optional_multformat_adv/output.${input_multformat.ext}' + #end if #end if ## Postprocessing @@ -121,6 +171,18 @@ python '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' && #if $optional_multformat_FLAG: && mv 'optional_multformat/output.${input_multformat.ext}' '$optional_multformat' #end if +&& mv 'sect_mandatory_noformat/output.${gxy2omsext("txt")}' '$sect_mandatory_noformat' +#if $test_section.sect_optional_noformat_FLAG: + && mv 'sect_optional_noformat/output.${gxy2omsext("txt")}' '$sect_optional_noformat' +#end if +&& mv 'sect_mandatory_oneformat/output.${gxy2omsext("tsv")}' '$sect_mandatory_oneformat' +#if $test_section.sect_optional_oneformat_FLAG: + && mv 'sect_optional_oneformat/output.${gxy2omsext("tsv")}' '$sect_optional_oneformat' +#end if +&& mv 'sect_mandatory_multformat/output.${input_multformat.ext}' '$sect_mandatory_multformat' +#if $test_section.sect_optional_multformat_FLAG: + && mv 'sect_optional_multformat/output.${input_multformat.ext}' '$sect_optional_multformat' +#end if #if $adv_opts_cond.adv_opts_selector=='advanced': && mv 'mandatory_noformat_adv/output.${gxy2omsext("txt")}' '$mandatory_noformat_adv' #if $adv_opts_cond.optional_noformat_adv_FLAG: @@ -134,6 +196,18 @@ python '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' && #if $adv_opts_cond.optional_multformat_adv_FLAG: && mv 'optional_multformat_adv/output.${input_multformat.ext}' '$optional_multformat_adv' #end if + && mv 'sect_mandatory_noformat_adv/output.${gxy2omsext("txt")}' '$sect_mandatory_noformat_adv' + #if $test_section.sect_optional_noformat_adv_FLAG: + && mv 'sect_optional_noformat_adv/output.${gxy2omsext("txt")}' '$sect_optional_noformat_adv' + #end if + && mv 'sect_mandatory_oneformat_adv/output.${gxy2omsext("tsv")}' '$sect_mandatory_oneformat_adv' + #if $test_section.sect_optional_oneformat_adv_FLAG: + && mv 'sect_optional_oneformat_adv/output.${gxy2omsext("tsv")}' '$sect_optional_oneformat_adv' + #end if + && mv 'sect_mandatory_multformat_adv/output.${input_multformat.ext}' '$sect_mandatory_multformat_adv' + #if $test_section.sect_optional_multformat_adv_FLAG: + && mv 'sect_optional_multformat_adv/output.${input_multformat.ext}' '$sect_optional_multformat_adv' + #end if #end if]]> @@ -142,9 +216,17 @@ python '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' && - - - + + + +
+ + + + + + +
@@ -176,12 +258,47 @@ python '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' && adv_opts_cond['adv_opts_selector'] == 'advanced' and adv_opts_cond['optional_multformat_adv_FLAG'] + + + test_section["sect_optional_noformat_FLAG"] + + + + test_section["sect_optional_oneformat_FLAG"] + + + + test_section["sect_optional_multformat_FLAG"] + + + + test_section["sect_optional_noformat_adv_FLAG"] + + + + test_section["sect_optional_oneformat_adv_FLAG"] + + + + test_section["sect_optional_multformat_adv_FLAG"] + - + + + + +
+ + + + + + +
@@ -194,6 +311,12 @@ python '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' && + + + + + + --> + + + + + + + + + + + + + + diff --git a/tests/test-data/ofile-mult-typeparam.xml b/tests/test-data/ofile-mult-typeparam.xml index 01dbcc4a..749b1cc4 100644 --- a/tests/test-data/ofile-mult-typeparam.xml +++ b/tests/test-data/ofile-mult-typeparam.xml @@ -16,52 +16,85 @@ ## Preprocessing mkdir mandatory_mandatoryinput && mkdir mandatory_optionalinput && -#if $inp_optional_mandatoryinput: - mkdir inp_optional_mandatoryinput && - ${ ' '.join(["ln -s '%s' 'inp_optional_mandatoryinput/%s.%s' &&" % (_, re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $inp_optional_mandatoryinput if _]) } -#end if +mkdir inp_optional_mandatoryinput && +${ ' '.join(["ln -s '%s' 'inp_optional_mandatoryinput/%s.%s' &&" % (_, re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $inp_optional_mandatoryinput if _]) } #if $inp_mandatory_optionalinput: mkdir inp_mandatory_optionalinput && ${ ' '.join(["ln -s '%s' 'inp_mandatory_optionalinput/%s.%s' &&" % (_, re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $inp_mandatory_optionalinput if _]) } #end if +mkdir sect_mandatory_mandatoryinput && +mkdir sect_mandatory_optionalinput && +mkdir test_section.sect_inp_optional_mandatoryinput && +${ ' '.join(["ln -s '%s' 'test_section.sect_inp_optional_mandatoryinput/%s.%s' &&" % (_, re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $test_section.sect_inp_optional_mandatoryinput if _]) } +#if $test_section.sect_inp_mandatory_optionalinput: + mkdir test_section.sect_inp_mandatory_optionalinput && + ${ ' '.join(["ln -s '%s' 'test_section.sect_inp_mandatory_optionalinput/%s.%s' &&" % (_, re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $test_section.sect_inp_mandatory_optionalinput if _]) } +#end if ## Main program call + +export PYTHONPATH='$__tool_directory__/CTDopts' && @EXECUTABLE@ -write_ctd ./ && -python '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' && +python3 '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' && @EXECUTABLE@ -ini @EXECUTABLE@.ctd -mandatory_mandatoryinput -${' '.join(["'mandatory_mandatoryinput/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext("tsv")) for _ in $inp_optional_mandatoryinput if _])} +${' '.join(["'mandatory_mandatoryinput/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext("tsv")) for _ in $mandatory_mandatoryinput if _])} -mandatory_optionalinput -${' '.join(["'mandatory_optionalinput/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier), mandatory_optionalinput_type) for _ in $inp_mandatory_optionalinput if _])} -#if $inp_optional_mandatoryinput: - -inp_optional_mandatoryinput - ${' '.join(["'inp_optional_mandatoryinput/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $inp_optional_mandatoryinput if _])} -#end if +${' '.join(["'mandatory_optionalinput/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier), mandatory_optionalinput_type) for _ in $mandatory_optionalinput if _])} +-inp_optional_mandatoryinput +${' '.join(["'inp_optional_mandatoryinput/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $inp_optional_mandatoryinput if _])} #if $inp_mandatory_optionalinput: -inp_mandatory_optionalinput ${' '.join(["'inp_mandatory_optionalinput/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $inp_mandatory_optionalinput if _])} #end if +-test_section:sect_mandatory_mandatoryinput +${' '.join(["'sect_mandatory_mandatoryinput/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext("tsv")) for _ in $sect_mandatory_mandatoryinput if _])} +-test_section:sect_mandatory_optionalinput +${' '.join(["'sect_mandatory_optionalinput/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier), test_section.sect_mandatory_optionalinput_type) for _ in $sect_mandatory_optionalinput if _])} +-test_section:sect_inp_optional_mandatoryinput +${' '.join(["'test_section.sect_inp_optional_mandatoryinput/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $test_section.sect_inp_optional_mandatoryinput if _])} +#if $test_section.sect_inp_mandatory_optionalinput: + -test_section:sect_inp_mandatory_optionalinput + ${' '.join(["'test_section.sect_inp_mandatory_optionalinput/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $test_section.sect_inp_mandatory_optionalinput if _])} +#end if ## Postprocessing -${' '.join(["&& mv -n 'mandatory_mandatoryinput/%(id)s.%(gext)s' 'mandatory_mandatoryinput/%(id)s'"%{"id": re.sub('[^\w\-_]', '_', _.element_identifier), "gext": $gxy2omsext("tsv")} for _ in $inp_optional_mandatoryinput if _])} -${' '.join(["&& mv -n 'mandatory_optionalinput/%(id)s.%(omsext)s' 'mandatory_optionalinput/%(id)s.%(gext)s'"%{"id": re.sub('[^\w\-_]', '_', _.element_identifier), "omsext":mandatory_optionalinput_type, "gext": oms2gxyext("mandatory_optionalinput_type")} for _ in $inp_mandatory_optionalinput if _])}]]> +${' '.join(["&& mv -n 'mandatory_mandatoryinput/%(id)s.%(gext)s' 'mandatory_mandatoryinput/%(id)s'"%{"id": re.sub('[^\w\-_]', '_', _.element_identifier), "gext": $gxy2omsext("tsv")} for _ in $mandatory_mandatoryinput if _])} +${' '.join(["&& mv -n 'mandatory_optionalinput/%(id)s.%(omsext)s' 'mandatory_optionalinput/%(id)s.%(gext)s'"%{"id": re.sub('[^\w\-_]', '_', _.element_identifier), "omsext":mandatory_optionalinput_type, "gext": oms2gxyext("mandatory_optionalinput_type")} for _ in $mandatory_optionalinput if _])} +${' '.join(["&& mv -n 'sect_mandatory_mandatoryinput/%(id)s.%(gext)s' 'sect_mandatory_mandatoryinput/%(id)s'"%{"id": re.sub('[^\w\-_]', '_', _.element_identifier), "gext": $gxy2omsext("tsv")} for _ in $sect_mandatory_mandatoryinput if _])} +${' '.join(["&& mv -n 'sect_mandatory_optionalinput/%(id)s.%(omsext)s' 'sect_mandatory_optionalinput/%(id)s.%(gext)s'"%{"id": re.sub('[^\w\-_]', '_', _.element_identifier), "omsext":test_section.sect_mandatory_optionalinput_type, "gext": oms2gxyext("test_section.sect_mandatory_optionalinput_type")} for _ in $sect_mandatory_optionalinput if _])}]]> - - + + - - - + + + - - + + +
+ + + + + + + + + + + + + +
@@ -70,13 +103,27 @@ ${' '.join(["&& mv -n 'mandatory_optionalinput/%(id)s.%(omsext)s' 'mandatory_opt + + + + + + - + + +
+ + + +
+ + @@ -83,10 +77,10 @@ ${' '.join(["&& mv -n 'mandatory_mandatoryinput/%(id)s.%(gext)s' 'mandatory_mand - - - - + + + + @@ -108,6 +102,8 @@ ${' '.join(["&& mv -n 'mandatory_mandatoryinput/%(id)s.%(gext)s' 'mandatory_mand + + diff --git a/tests/test-data/ofile-typeparam.ctd b/tests/test-data/ofile-typeparam.ctd index 685a990f..9c9a755e 100644 --- a/tests/test-data/ofile-typeparam.ctd +++ b/tests/test-data/ofile-typeparam.ctd @@ -42,7 +42,39 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/test-data/ofile-typeparam.xml b/tests/test-data/ofile-typeparam.xml index 7b11acd9..73796a37 100644 --- a/tests/test-data/ofile-typeparam.xml +++ b/tests/test-data/ofile-typeparam.xml @@ -22,18 +22,26 @@ mkdir mandatory_multformat && #if $optional_multformat_FLAG: mkdir optional_multformat && #end if -#if $decoy_input_noformat: - mkdir decoy_input_noformat && - ln -s '$decoy_input_noformat' 'decoy_input_noformat/${re.sub("[^\w\-_]", "_", $decoy_input_noformat.element_identifier)}.$gxy2omsext($decoy_input_noformat.ext)' && +mkdir decoy_input_noformat && +ln -s '$decoy_input_noformat' 'decoy_input_noformat/${re.sub("[^\w\-_]", "_", $decoy_input_noformat.element_identifier)}.$gxy2omsext($decoy_input_noformat.ext)' && +mkdir decoy_input_oneformat && +ln -s '$decoy_input_oneformat' 'decoy_input_oneformat/${re.sub("[^\w\-_]", "_", $decoy_input_oneformat.element_identifier)}.$gxy2omsext($decoy_input_oneformat.ext)' && +mkdir decoy_input_multformat && +ln -s '$decoy_input_multformat' 'decoy_input_multformat/${re.sub("[^\w\-_]", "_", $decoy_input_multformat.element_identifier)}.$gxy2omsext($decoy_input_multformat.ext)' && +mkdir sect_mandatory_noformat && +#if $test_section.sect_optional_noformat_FLAG: + mkdir sect_optional_noformat && #end if -#if $decoy_input_oneformat: - mkdir decoy_input_oneformat && - ln -s '$decoy_input_oneformat' 'decoy_input_oneformat/${re.sub("[^\w\-_]", "_", $decoy_input_oneformat.element_identifier)}.$gxy2omsext($decoy_input_oneformat.ext)' && -#end if -#if $decoy_input_multformat: - mkdir decoy_input_multformat && - ln -s '$decoy_input_multformat' 'decoy_input_multformat/${re.sub("[^\w\-_]", "_", $decoy_input_multformat.element_identifier)}.$gxy2omsext($decoy_input_multformat.ext)' && +mkdir sect_mandatory_multformat && +#if $test_section.sect_optional_multformat_FLAG: + mkdir sect_optional_multformat && #end if +mkdir test_section.sect_decoy_input_noformat && +ln -s '$test_section.sect_decoy_input_noformat' 'test_section.sect_decoy_input_noformat/${re.sub("[^\w\-_]", "_", $test_section.sect_decoy_input_noformat.element_identifier)}.$gxy2omsext($test_section.sect_decoy_input_noformat.ext)' && +mkdir test_section.sect_decoy_input_oneformat && +ln -s '$test_section.sect_decoy_input_oneformat' 'test_section.sect_decoy_input_oneformat/${re.sub("[^\w\-_]", "_", $test_section.sect_decoy_input_oneformat.element_identifier)}.$gxy2omsext($test_section.sect_decoy_input_oneformat.ext)' && +mkdir test_section.sect_decoy_input_multformat && +ln -s '$test_section.sect_decoy_input_multformat' 'test_section.sect_decoy_input_multformat/${re.sub("[^\w\-_]", "_", $test_section.sect_decoy_input_multformat.element_identifier)}.$gxy2omsext($test_section.sect_decoy_input_multformat.ext)' && #if $adv_opts_cond.adv_opts_selector=='advanced': mkdir mandatory_noformat_adv && #if $adv_opts_cond.optional_noformat_adv_FLAG: @@ -43,11 +51,21 @@ mkdir mandatory_multformat && #if $adv_opts_cond.optional_multformat_adv_FLAG: mkdir optional_multformat_adv && #end if + mkdir sect_mandatory_noformat_adv && + #if $test_section.sect_optional_noformat_adv_FLAG: + mkdir sect_optional_noformat_adv && + #end if + mkdir sect_mandatory_multformat_adv && + #if $test_section.sect_optional_multformat_adv_FLAG: + mkdir sect_optional_multformat_adv && + #end if #end if ## Main program call + +export PYTHONPATH='$__tool_directory__/CTDopts' && @EXECUTABLE@ -write_ctd ./ && -python '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' && +python3 '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' && @EXECUTABLE@ -ini @EXECUTABLE@.ctd -mandatory_noformat 'mandatory_noformat/output.${gxy2omsext("txt")}' @@ -61,18 +79,30 @@ python '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' && -optional_multformat 'optional_multformat/output.${optional_multformat_type}' #end if -#if $decoy_input_noformat: - -decoy_input_noformat - 'decoy_input_noformat/${re.sub("[^\w\-_]", "_", $decoy_input_noformat.element_identifier)}.$gxy2omsext($decoy_input_noformat.ext)' -#end if -#if $decoy_input_oneformat: - -decoy_input_oneformat - 'decoy_input_oneformat/${re.sub("[^\w\-_]", "_", $decoy_input_oneformat.element_identifier)}.$gxy2omsext($decoy_input_oneformat.ext)' +-decoy_input_noformat +'decoy_input_noformat/${re.sub("[^\w\-_]", "_", $decoy_input_noformat.element_identifier)}.$gxy2omsext($decoy_input_noformat.ext)' +-decoy_input_oneformat +'decoy_input_oneformat/${re.sub("[^\w\-_]", "_", $decoy_input_oneformat.element_identifier)}.$gxy2omsext($decoy_input_oneformat.ext)' +-decoy_input_multformat +'decoy_input_multformat/${re.sub("[^\w\-_]", "_", $decoy_input_multformat.element_identifier)}.$gxy2omsext($decoy_input_multformat.ext)' +-test_section:sect_mandatory_noformat +'sect_mandatory_noformat/output.${gxy2omsext("txt")}' +#if $test_section.sect_optional_noformat_FLAG: + -test_section:sect_optional_noformat + 'sect_optional_noformat/output.${gxy2omsext("txt")}' #end if -#if $decoy_input_multformat: - -decoy_input_multformat - 'decoy_input_multformat/${re.sub("[^\w\-_]", "_", $decoy_input_multformat.element_identifier)}.$gxy2omsext($decoy_input_multformat.ext)' +-test_section:sect_mandatory_multformat +'sect_mandatory_multformat/output.${test_section.sect_mandatory_multformat_type}' +#if $test_section.sect_optional_multformat_FLAG: + -test_section:sect_optional_multformat + 'sect_optional_multformat/output.${test_section.sect_optional_multformat_type}' #end if +-test_section:sect_decoy_input_noformat +'test_section.sect_decoy_input_noformat/${re.sub("[^\w\-_]", "_", $test_section.sect_decoy_input_noformat.element_identifier)}.$gxy2omsext($test_section.sect_decoy_input_noformat.ext)' +-test_section:sect_decoy_input_oneformat +'test_section.sect_decoy_input_oneformat/${re.sub("[^\w\-_]", "_", $test_section.sect_decoy_input_oneformat.element_identifier)}.$gxy2omsext($test_section.sect_decoy_input_oneformat.ext)' +-test_section:sect_decoy_input_multformat +'test_section.sect_decoy_input_multformat/${re.sub("[^\w\-_]", "_", $test_section.sect_decoy_input_multformat.element_identifier)}.$gxy2omsext($test_section.sect_decoy_input_multformat.ext)' #if $adv_opts_cond.adv_opts_selector=='advanced': -mandatory_noformat_adv 'mandatory_noformat_adv/output.${gxy2omsext("txt")}' @@ -86,6 +116,18 @@ python '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' && -optional_multformat_adv 'optional_multformat_adv/output.${adv_opts_cond.optional_multformat_adv_type}' #end if + -test_section:sect_mandatory_noformat_adv + 'sect_mandatory_noformat_adv/output.${gxy2omsext("txt")}' + #if $test_section.sect_optional_noformat_adv_FLAG: + -test_section:sect_optional_noformat_adv + 'sect_optional_noformat_adv/output.${gxy2omsext("txt")}' + #end if + -test_section:sect_mandatory_multformat_adv + 'sect_mandatory_multformat_adv/output.${test_section.sect_mandatory_multformat_adv_type}' + #if $test_section.sect_optional_multformat_adv_FLAG: + -test_section:sect_optional_multformat_adv + 'sect_optional_multformat_adv/output.${test_section.sect_optional_multformat_adv_type}' + #end if #end if ## Postprocessing @@ -97,6 +139,14 @@ python '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' && #if $optional_multformat_FLAG: && mv 'optional_multformat/output.${optional_multformat_type}' '$optional_multformat' #end if +&& mv 'sect_mandatory_noformat/output.${gxy2omsext("txt")}' '$sect_mandatory_noformat' +#if $test_section.sect_optional_noformat_FLAG: + && mv 'sect_optional_noformat/output.${gxy2omsext("txt")}' '$sect_optional_noformat' +#end if +&& mv 'sect_mandatory_multformat/output.${test_section.sect_mandatory_multformat_type}' '$sect_mandatory_multformat' +#if $test_section.sect_optional_multformat_FLAG: + && mv 'sect_optional_multformat/output.${test_section.sect_optional_multformat_type}' '$sect_optional_multformat' +#end if #if $adv_opts_cond.adv_opts_selector=='advanced': && mv 'mandatory_noformat_adv/output.${gxy2omsext("txt")}' '$mandatory_noformat_adv' #if $adv_opts_cond.optional_noformat_adv_FLAG: @@ -106,64 +156,129 @@ python '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' && #if $adv_opts_cond.optional_multformat_adv_FLAG: && mv 'optional_multformat_adv/output.${adv_opts_cond.optional_multformat_adv_type}' '$optional_multformat_adv' #end if + && mv 'sect_mandatory_noformat_adv/output.${gxy2omsext("txt")}' '$sect_mandatory_noformat_adv' + #if $test_section.sect_optional_noformat_adv_FLAG: + && mv 'sect_optional_noformat_adv/output.${gxy2omsext("txt")}' '$sect_optional_noformat_adv' + #end if + && mv 'sect_mandatory_multformat_adv/output.${test_section.sect_mandatory_multformat_adv_type}' '$sect_mandatory_multformat_adv' + #if $test_section.sect_optional_multformat_adv_FLAG: + && mv 'sect_optional_multformat_adv/output.${test_section.sect_optional_multformat_adv_type}' '$sect_optional_multformat_adv' + #end if #end if]]> - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
- - - + + + - - - + + + - - - + + + - - - + + + @@ -208,15 +323,73 @@ python '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' && adv_opts_cond['adv_opts_selector'] == 'advanced' and adv_opts_cond['optional_multformat_adv_FLAG'] + + + test_section["sect_optional_noformat_FLAG"] + + + + + + + + + + + + + + + test_section["sect_optional_multformat_FLAG"] + + + + test_section["sect_optional_noformat_adv_FLAG"] + + + + + + + + + + + + + + + test_section["sect_optional_multformat_adv_FLAG"] + - + + + + +
+ + + + + + + + + + + + + + + +
@@ -230,6 +403,10 @@ python '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' && + + + + - + + + + + + + + + + + + + + + + + + + diff --git a/tests/test-data/ofile.xml b/tests/test-data/ofile.xml index 686d29cb..47146d35 100644 --- a/tests/test-data/ofile.xml +++ b/tests/test-data/ofile.xml @@ -26,6 +26,18 @@ mkdir mandatory_multformat && #if $optional_multformat_FLAG: mkdir optional_multformat && #end if +mkdir sect_mandatory_noformat && +#if $test_section.sect_optional_noformat_FLAG: + mkdir sect_optional_noformat && +#end if +mkdir sect_mandatory_oneformat && +#if $test_section.sect_optional_oneformat_FLAG: + mkdir sect_optional_oneformat && +#end if +mkdir sect_mandatory_multformat && +#if $test_section.sect_optional_multformat_FLAG: + mkdir sect_optional_multformat && +#end if #if $adv_opts_cond.adv_opts_selector=='advanced': mkdir mandatory_noformat_adv && #if $adv_opts_cond.optional_noformat_adv_FLAG: @@ -39,11 +51,25 @@ mkdir mandatory_multformat && #if $adv_opts_cond.optional_multformat_adv_FLAG: mkdir optional_multformat_adv && #end if + mkdir sect_mandatory_noformat_adv && + #if $test_section.sect_optional_noformat_adv_FLAG: + mkdir sect_optional_noformat_adv && + #end if + mkdir sect_mandatory_oneformat_adv && + #if $test_section.sect_optional_oneformat_adv_FLAG: + mkdir sect_optional_oneformat_adv && + #end if + mkdir sect_mandatory_multformat_adv && + #if $test_section.sect_optional_multformat_adv_FLAG: + mkdir sect_optional_multformat_adv && + #end if #end if ## Main program call + +export PYTHONPATH='$__tool_directory__/CTDopts' && @EXECUTABLE@ -write_ctd ./ && -python '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' && +python3 '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' && @EXECUTABLE@ -ini @EXECUTABLE@.ctd -mandatory_noformat 'mandatory_noformat/output.${gxy2omsext("txt")}' @@ -63,6 +89,24 @@ python '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' && -optional_multformat 'optional_multformat/output.${optional_multformat_type}' #end if +-test_section:sect_mandatory_noformat +'sect_mandatory_noformat/output.${gxy2omsext("txt")}' +#if $test_section.sect_optional_noformat_FLAG: + -test_section:sect_optional_noformat + 'sect_optional_noformat/output.${gxy2omsext("txt")}' +#end if +-test_section:sect_mandatory_oneformat +'sect_mandatory_oneformat/output.${gxy2omsext("tsv")}' +#if $test_section.sect_optional_oneformat_FLAG: + -test_section:sect_optional_oneformat + 'sect_optional_oneformat/output.${gxy2omsext("tsv")}' +#end if +-test_section:sect_mandatory_multformat +'sect_mandatory_multformat/output.${test_section.sect_mandatory_multformat_type}' +#if $test_section.sect_optional_multformat_FLAG: + -test_section:sect_optional_multformat + 'sect_optional_multformat/output.${test_section.sect_optional_multformat_type}' +#end if #if $adv_opts_cond.adv_opts_selector=='advanced': -mandatory_noformat_adv 'mandatory_noformat_adv/output.${gxy2omsext("txt")}' @@ -82,6 +126,24 @@ python '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' && -optional_multformat_adv 'optional_multformat_adv/output.${adv_opts_cond.optional_multformat_adv_type}' #end if + -test_section:sect_mandatory_noformat_adv + 'sect_mandatory_noformat_adv/output.${gxy2omsext("txt")}' + #if $test_section.sect_optional_noformat_adv_FLAG: + -test_section:sect_optional_noformat_adv + 'sect_optional_noformat_adv/output.${gxy2omsext("txt")}' + #end if + -test_section:sect_mandatory_oneformat_adv + 'sect_mandatory_oneformat_adv/output.${gxy2omsext("tsv")}' + #if $test_section.sect_optional_oneformat_adv_FLAG: + -test_section:sect_optional_oneformat_adv + 'sect_optional_oneformat_adv/output.${gxy2omsext("tsv")}' + #end if + -test_section:sect_mandatory_multformat_adv + 'sect_mandatory_multformat_adv/output.${test_section.sect_mandatory_multformat_adv_type}' + #if $test_section.sect_optional_multformat_adv_FLAG: + -test_section:sect_optional_multformat_adv + 'sect_optional_multformat_adv/output.${test_section.sect_optional_multformat_adv_type}' + #end if #end if ## Postprocessing @@ -97,6 +159,18 @@ python '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' && #if $optional_multformat_FLAG: && mv 'optional_multformat/output.${optional_multformat_type}' '$optional_multformat' #end if +&& mv 'sect_mandatory_noformat/output.${gxy2omsext("txt")}' '$sect_mandatory_noformat' +#if $test_section.sect_optional_noformat_FLAG: + && mv 'sect_optional_noformat/output.${gxy2omsext("txt")}' '$sect_optional_noformat' +#end if +&& mv 'sect_mandatory_oneformat/output.${gxy2omsext("tsv")}' '$sect_mandatory_oneformat' +#if $test_section.sect_optional_oneformat_FLAG: + && mv 'sect_optional_oneformat/output.${gxy2omsext("tsv")}' '$sect_optional_oneformat' +#end if +&& mv 'sect_mandatory_multformat/output.${test_section.sect_mandatory_multformat_type}' '$sect_mandatory_multformat' +#if $test_section.sect_optional_multformat_FLAG: + && mv 'sect_optional_multformat/output.${test_section.sect_optional_multformat_type}' '$sect_optional_multformat' +#end if #if $adv_opts_cond.adv_opts_selector=='advanced': && mv 'mandatory_noformat_adv/output.${gxy2omsext("txt")}' '$mandatory_noformat_adv' #if $adv_opts_cond.optional_noformat_adv_FLAG: @@ -110,6 +184,18 @@ python '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' && #if $adv_opts_cond.optional_multformat_adv_FLAG: && mv 'optional_multformat_adv/output.${adv_opts_cond.optional_multformat_adv_type}' '$optional_multformat_adv' #end if + && mv 'sect_mandatory_noformat_adv/output.${gxy2omsext("txt")}' '$sect_mandatory_noformat_adv' + #if $test_section.sect_optional_noformat_adv_FLAG: + && mv 'sect_optional_noformat_adv/output.${gxy2omsext("txt")}' '$sect_optional_noformat_adv' + #end if + && mv 'sect_mandatory_oneformat_adv/output.${gxy2omsext("tsv")}' '$sect_mandatory_oneformat_adv' + #if $test_section.sect_optional_oneformat_adv_FLAG: + && mv 'sect_optional_oneformat_adv/output.${gxy2omsext("tsv")}' '$sect_optional_oneformat_adv' + #end if + && mv 'sect_mandatory_multformat_adv/output.${test_section.sect_mandatory_multformat_adv_type}' '$sect_mandatory_multformat_adv' + #if $test_section.sect_optional_multformat_adv_FLAG: + && mv 'sect_optional_multformat_adv/output.${test_section.sect_optional_multformat_adv_type}' '$sect_optional_multformat_adv' + #end if #end if]]> @@ -126,6 +212,30 @@ python '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' && +
+ + + + + + + + + + + + + + + + + + + + + + +
@@ -165,14 +275,50 @@ python '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' && adv_opts_cond['adv_opts_selector'] == 'advanced' and adv_opts_cond['optional_multformat_adv_FLAG'] + + + test_section["sect_optional_noformat_FLAG"] + + + + test_section["sect_optional_oneformat_FLAG"] + + + + test_section["sect_optional_multformat_FLAG"] + + + + test_section["sect_optional_noformat_adv_FLAG"] + + + + test_section["sect_optional_oneformat_adv_FLAG"] + + + + test_section["sect_optional_multformat_adv_FLAG"] + - + +
+ + + + + + + + + + +
@@ -187,6 +333,12 @@ python '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' && + + + + + + - - - - Repeat parameter tests - - repeat - macros.xml - - - - $param_stdout - -## Postprocessing]]> - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/tests/test-data/repeat b/tests/test-data/repeat_test similarity index 100% rename from tests/test-data/repeat rename to tests/test-data/repeat_test diff --git a/tests/test-data/repeat.ctd b/tests/test-data/repeat_test.ctd similarity index 50% rename from tests/test-data/repeat.ctd rename to tests/test-data/repeat_test.ctd index 147d6fd2..5d50de5d 100644 --- a/tests/test-data/repeat.ctd +++ b/tests/test-data/repeat_test.ctd @@ -1,5 +1,5 @@ - + @@ -119,6 +119,121 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/test-data/repeat_test.xml b/tests/test-data/repeat_test.xml new file mode 100644 index 00000000..6cdfbe35 --- /dev/null +++ b/tests/test-data/repeat_test.xml @@ -0,0 +1,446 @@ + + + + + Repeat parameter tests + + repeat_test + macros.xml + + + + $param_stdout + +## Postprocessing]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + + + + + + + + + + + + + + + + + +
+ + + + + + + + + + + + + + + + + + + diff --git a/tests/test-data/select_test.ctd b/tests/test-data/select_test.ctd index 5b704b63..fe1f157e 100644 --- a/tests/test-data/select_test.ctd +++ b/tests/test-data/select_test.ctd @@ -71,6 +71,72 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/test-data/select_test.xml b/tests/test-data/select_test.xml index aba27bf5..9ab82b48 100644 --- a/tests/test-data/select_test.xml +++ b/tests/test-data/select_test.xml @@ -16,8 +16,10 @@ ## Preprocessing ## Main program call + +export PYTHONPATH='$__tool_directory__/CTDopts' && @EXECUTABLE@ -write_ctd ./ && -python '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' && +python3 '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' && @EXECUTABLE@ -ini @EXECUTABLE@.ctd > $param_stdout @@ -27,244 +29,486 @@ python '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' && - - - - + + + + - - + + - + - - - - + + + + - - + + - + - - - - - + + + + + - - + + - - + + - - - - - + + + + + - - + + - - + + - - - - + + + + - - - - + + + + - - + + - - + + - - - - - + + + + + - - - - - + + + + + - - + + - + - - + + - + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
- - - - + + + + - - + + - + - - - - + + + + - - + + - + - - - - - + + + + + - - + + - - + + - - - - - + + + + + - - + + - - + + - - - - + + + + - - - - + + + + - - + + - - + + - - - - - + + + + + - - - - - + + + + + - - + + - + - - + + - + @@ -278,6 +522,16 @@ python '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' && +
+ + + + + + + + +
diff --git a/tests/test-data/string.ctd b/tests/test-data/string.ctd index fc3d289c..b7c19e2b 100644 --- a/tests/test-data/string.ctd +++ b/tests/test-data/string.ctd @@ -14,6 +14,17 @@ + + + + + + + + + + + diff --git a/tests/test-data/string.xml b/tests/test-data/string.xml index 0053c51a..c1fdbdcf 100644 --- a/tests/test-data/string.xml +++ b/tests/test-data/string.xml @@ -16,8 +16,10 @@ ## Preprocessing ## Main program call + +export PYTHONPATH='$__tool_directory__/CTDopts' && @EXECUTABLE@ -write_ctd ./ && -python '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' && +python3 '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' && @EXECUTABLE@ -ini @EXECUTABLE@.ctd > $param_stdout @@ -27,36 +29,54 @@ python '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' && - - - - +
+ + + + + + + + + + + + + + + + + + + + + + + + +
- - - - @@ -68,6 +88,12 @@ python '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' && +
+ + + + +
diff --git a/tests/test-data/test-data/12test.ext b/tests/test-data/test-data/12test.ext new file mode 120000 index 00000000..769e1336 --- /dev/null +++ b/tests/test-data/test-data/12test.ext @@ -0,0 +1 @@ +test.ext \ No newline at end of file diff --git a/tests/test-data/test-data/12test2.ext b/tests/test-data/test-data/12test2.ext new file mode 120000 index 00000000..769e1336 --- /dev/null +++ b/tests/test-data/test-data/12test2.ext @@ -0,0 +1 @@ +test.ext \ No newline at end of file diff --git a/tests/test-data/test-data/13test.ext b/tests/test-data/test-data/13test.ext new file mode 120000 index 00000000..769e1336 --- /dev/null +++ b/tests/test-data/test-data/13test.ext @@ -0,0 +1 @@ +test.ext \ No newline at end of file diff --git a/tests/test-data/test-data/13test2.ext b/tests/test-data/test-data/13test2.ext new file mode 120000 index 00000000..769e1336 --- /dev/null +++ b/tests/test-data/test-data/13test2.ext @@ -0,0 +1 @@ +test.ext \ No newline at end of file diff --git a/tests/test-data/test-data/14test.ext b/tests/test-data/test-data/14test.ext new file mode 120000 index 00000000..769e1336 --- /dev/null +++ b/tests/test-data/test-data/14test.ext @@ -0,0 +1 @@ +test.ext \ No newline at end of file diff --git a/tests/test-data/test-data/14test2.ext b/tests/test-data/test-data/14test2.ext new file mode 120000 index 00000000..769e1336 --- /dev/null +++ b/tests/test-data/test-data/14test2.ext @@ -0,0 +1 @@ +test.ext \ No newline at end of file diff --git a/tests/test-data/test-data/15test.ext b/tests/test-data/test-data/15test.ext new file mode 120000 index 00000000..769e1336 --- /dev/null +++ b/tests/test-data/test-data/15test.ext @@ -0,0 +1 @@ +test.ext \ No newline at end of file diff --git a/tests/test-data/test-data/15test2.ext b/tests/test-data/test-data/15test2.ext new file mode 120000 index 00000000..769e1336 --- /dev/null +++ b/tests/test-data/test-data/15test2.ext @@ -0,0 +1 @@ +test.ext \ No newline at end of file diff --git a/tests/test-data/test-data/16test.ext b/tests/test-data/test-data/16test.ext new file mode 120000 index 00000000..769e1336 --- /dev/null +++ b/tests/test-data/test-data/16test.ext @@ -0,0 +1 @@ +test.ext \ No newline at end of file diff --git a/tests/test-data/test-data/16test2.ext b/tests/test-data/test-data/16test2.ext new file mode 120000 index 00000000..769e1336 --- /dev/null +++ b/tests/test-data/test-data/16test2.ext @@ -0,0 +1 @@ +test.ext \ No newline at end of file diff --git a/tests/test-data/test-data/17test.ext b/tests/test-data/test-data/17test.ext new file mode 120000 index 00000000..769e1336 --- /dev/null +++ b/tests/test-data/test-data/17test.ext @@ -0,0 +1 @@ +test.ext \ No newline at end of file diff --git a/tests/test-data/test-data/17test2.ext b/tests/test-data/test-data/17test2.ext new file mode 120000 index 00000000..769e1336 --- /dev/null +++ b/tests/test-data/test-data/17test2.ext @@ -0,0 +1 @@ +test.ext \ No newline at end of file diff --git a/tests/test-data/test-data/18test.ext b/tests/test-data/test-data/18test.ext new file mode 120000 index 00000000..769e1336 --- /dev/null +++ b/tests/test-data/test-data/18test.ext @@ -0,0 +1 @@ +test.ext \ No newline at end of file diff --git a/tests/test-data/test-data/18test2.ext b/tests/test-data/test-data/18test2.ext new file mode 120000 index 00000000..769e1336 --- /dev/null +++ b/tests/test-data/test-data/18test2.ext @@ -0,0 +1 @@ +test.ext \ No newline at end of file diff --git a/tests/test-data/test-data/19test.ext b/tests/test-data/test-data/19test.ext new file mode 120000 index 00000000..769e1336 --- /dev/null +++ b/tests/test-data/test-data/19test.ext @@ -0,0 +1 @@ +test.ext \ No newline at end of file diff --git a/tests/test-data/test-data/19test2.ext b/tests/test-data/test-data/19test2.ext new file mode 120000 index 00000000..769e1336 --- /dev/null +++ b/tests/test-data/test-data/19test2.ext @@ -0,0 +1 @@ +test.ext \ No newline at end of file diff --git a/tests/test-data/test-data/20test.ext b/tests/test-data/test-data/20test.ext new file mode 120000 index 00000000..769e1336 --- /dev/null +++ b/tests/test-data/test-data/20test.ext @@ -0,0 +1 @@ +test.ext \ No newline at end of file diff --git a/tests/test-data/test-data/20test2.ext b/tests/test-data/test-data/20test2.ext new file mode 120000 index 00000000..769e1336 --- /dev/null +++ b/tests/test-data/test-data/20test2.ext @@ -0,0 +1 @@ +test.ext \ No newline at end of file diff --git a/tests/test-data/test-data/21test.ext b/tests/test-data/test-data/21test.ext new file mode 120000 index 00000000..769e1336 --- /dev/null +++ b/tests/test-data/test-data/21test.ext @@ -0,0 +1 @@ +test.ext \ No newline at end of file diff --git a/tests/test-data/test-data/21test2.ext b/tests/test-data/test-data/21test2.ext new file mode 120000 index 00000000..769e1336 --- /dev/null +++ b/tests/test-data/test-data/21test2.ext @@ -0,0 +1 @@ +test.ext \ No newline at end of file diff --git a/tests/test-data/test-data/22test.ext b/tests/test-data/test-data/22test.ext new file mode 120000 index 00000000..769e1336 --- /dev/null +++ b/tests/test-data/test-data/22test.ext @@ -0,0 +1 @@ +test.ext \ No newline at end of file diff --git a/tests/test-data/test-data/22test2.ext b/tests/test-data/test-data/22test2.ext new file mode 120000 index 00000000..769e1336 --- /dev/null +++ b/tests/test-data/test-data/22test2.ext @@ -0,0 +1 @@ +test.ext \ No newline at end of file diff --git a/tests/test-data/test-data/23test.ext b/tests/test-data/test-data/23test.ext new file mode 120000 index 00000000..769e1336 --- /dev/null +++ b/tests/test-data/test-data/23test.ext @@ -0,0 +1 @@ +test.ext \ No newline at end of file diff --git a/tests/test-data/test-data/23test2.ext b/tests/test-data/test-data/23test2.ext new file mode 120000 index 00000000..769e1336 --- /dev/null +++ b/tests/test-data/test-data/23test2.ext @@ -0,0 +1 @@ +test.ext \ No newline at end of file From eac138ef284c58734f5eebd7e7fefea748bb96d6 Mon Sep 17 00:00:00 2001 From: M Bernt Date: Mon, 2 Mar 2020 17:34:07 +0100 Subject: [PATCH 086/121] fixes fixes fixes - parametrize unsniffable extensions test-unsniffable - file types file - simplified format - store as pairs instead of odict (to allow for ambiguities in both directions) - run test_only after treating hardcoded / blacklisted params - ITEMLISTS wo default: restore nothing chosen option - file type select option text is now `galaxy_extension (oms_extension)` if they are not the same (in lower case) - fix output file cheetah for cases with type param - bool flags - use true/falsevalue instead of checked - sections - always use section name in cheetah and output names (because there are cases of outputs with same name in different sections) - handling of stdout - always add output if all other outputs are optional and not selected - output with tee - proper label - ctd test files: add case for bool, add more file types --- common/utils.py | 5 +- galaxy/converter.py | 428 ++++++++++++------ galaxy/macros.xml | 30 +- tests/test-data/bool.ctd | 5 + tests/test-data/bool.xml | 13 +- tests/test-data/empty.xml | 9 +- tests/test-data/filetypes.txt | 12 +- tests/test-data/fill_ctd.py | 85 +++- tests/test-data/float.xml | 9 +- tests/test-data/ifile.xml | 9 +- tests/test-data/integer.xml | 9 +- tests/test-data/label-help.xml | 9 +- tests/test-data/mock.py | 3 +- tests/test-data/ofile-corresponding-input.xml | 135 +++--- tests/test-data/ofile-mult-typeparam.ctd | 12 +- tests/test-data/ofile-mult-typeparam.xml | 65 +-- tests/test-data/ofile-mult.ctd | 2 - tests/test-data/ofile-mult.xml | 15 +- tests/test-data/ofile-typeparam.xml | 187 ++++---- tests/test-data/ofile.xml | 191 ++++---- tests/test-data/repeat_test.xml | 9 +- tests/test-data/select_test.xml | 25 +- tests/test-data/string.xml | 9 +- 23 files changed, 761 insertions(+), 515 deletions(-) diff --git a/common/utils.py b/common/utils.py index 676f0081..b3e2595a 100644 --- a/common/utils.py +++ b/common/utils.py @@ -112,7 +112,10 @@ def validate_argument_is_valid_path(args, argument_name): paths_to_check.append(str(member_value).strip()) for path_to_check in paths_to_check: - validate_path_exists(path_to_check) + try: + validate_path_exists(path_to_check) + except ApplicationException: + raise ApplicationException("Argument %s: The provided output file name (%s) points to a directory." % (argument_name, path_to_check)) # taken from diff --git a/galaxy/converter.py b/galaxy/converter.py index 9942ad17..bfe4b1c2 100755 --- a/galaxy/converter.py +++ b/galaxy/converter.py @@ -35,11 +35,9 @@ def __init__(self, code_range="", level="", description=None): class DataType: - def __init__(self, extension, galaxy_extension=None, galaxy_type=None, mimetype=None): + def __init__(self, extension, galaxy_extension): self.extension = extension self.galaxy_extension = galaxy_extension - self.galaxy_type = galaxy_type - self.mimetype = mimetype def add_specific_args(parser): @@ -83,8 +81,14 @@ def add_specific_args(parser): action="append", required=None, help="The prefix of the macro name in the corresponding trest macros file") parser.add_argument("--test-test", dest="test_test", action='store_true', default=False, required=False, help="Generate a simple test for the internal unit tests.") + parser.add_argument("--test-only", dest="test_only", action='store_true', default=False, required=False, help="Generate only the test section.") + parser.add_argument("--test-unsniffable", dest="test_unsniffable", nargs="+", default=[], required=False, + help="File extensions that can't be sniffed in Galaxy." + "Needs to be the OpenMS extensions (1st column in --formats-file)." + "For testdata with such extensions ftype will be set in the tes according to the file extension") + parser.add_argument("--tool-version", dest="tool_version", required=False, default=None, help="Tool version to use (if not given its extracted from the CTD)") @@ -112,6 +116,7 @@ def convert_models(args, parsed_ctds): @param args command line arguments @param parsed_ctds the ctds """ + # validate and prepare the passed arguments validate_and_prepare_args(args, parsed_ctds[0].ctd_model) @@ -143,6 +148,7 @@ def convert_models(args, parsed_ctds): parameter_hardcoder=args.parameter_hardcoder, test_test=args.test_test, test_only=args.test_only, + test_unsniffable=args.test_unsniffable, test_macros_file_names=args.test_macros_files, test_macros_prefix=args.test_macros_prefix, tool_version=args.tool_version) @@ -303,7 +309,7 @@ def check_test_macros(test_macros_files, test_macros_prefix, parsed_ctds): def parse_file_formats(formats_file): """ """ - supported_formats = OrderedDict() + supported_formats = [] if formats_file is not None: line_number = 0 with open(formats_file) as f: @@ -312,27 +318,19 @@ def parse_file_formats(formats_file): if line is None or not line.strip() or line.strip().startswith("#"): # ignore (it'd be weird to have something like: # if line is not None and not (not line.strip()) ... - pass + continue + parsed_formats = line.strip().split() + # valid lines contain either one or two columns + if not (0 < len(parsed_formats) < 3): + logger.warning( + "Invalid line at line number %d of the given formats file. Line will be ignored:\n%s" % + (line_number, line), 0) + # ignore the line + continue + elif len(parsed_formats) == 1: + supported_formats.append(DataType(parsed_formats[0], parsed_formats[0])) else: - # not an empty line, no comment - # strip the line and split by whitespace - parsed_formats = line.strip().split() - # valid lines contain either one or four columns - if not (len(parsed_formats) == 1 or len(parsed_formats) == 3 or len(parsed_formats) == 4): - logger.warning( - "Invalid line at line number %d of the given formats file. Line will be ignored:\n%s" % - (line_number, line), 0) - # ignore the line - continue - elif len(parsed_formats) == 1: - supported_formats[parsed_formats[0]] = DataType(parsed_formats[0], parsed_formats[0]) - else: - mimetype = None - # check if mimetype was provided - if len(parsed_formats) == 4: - mimetype = parsed_formats[3] - supported_formats[parsed_formats[0]] = DataType(parsed_formats[0], parsed_formats[1], - parsed_formats[2], mimetype) + supported_formats.append(DataType(parsed_formats[0], parsed_formats[1])) return supported_formats @@ -344,9 +342,10 @@ def get_fileformat_maps(supported_formats): o2g = {} g2o = {} for s in supported_formats: - o2g[s] = supported_formats[s].galaxy_extension - if supported_formats[s].galaxy_extension not in g2o: - g2o[supported_formats[s].galaxy_extension] = s + if s.extension not in o2g: + o2g[s.extension] = s.galaxy_extension + if s.galaxy_extension not in g2o: + g2o[s.galaxy_extension] = s.extension return g2o, o2g @@ -406,15 +405,6 @@ def _convert_internal(parsed_ctds, **kwargs): @return a tuple containing the model, output destination, origin file """ - if "test_only" in kwargs and kwargs["test_only"]: - for parsed_ctd in parsed_ctds: - test = create_test_only(parsed_ctd.ctd_model, **kwargs) - tree = ElementTree(test) - output_file = parsed_ctd.suggested_output_file - logger.info("Writing to %s" % utils.get_filename(output_file), 1) - tree.write(output_file, encoding="UTF-8", xml_declaration=False, pretty_print=True) - return - parameter_hardcoder = kwargs["parameter_hardcoder"] for parsed_ctd in parsed_ctds: model = parsed_ctd.ctd_model @@ -437,7 +427,7 @@ def _convert_internal(parsed_ctds, **kwargs): if not hasattr(param, a): continue # logger.error("%s %s"%(param.name, type(getattr(param, a))) ) - if type(getattr(param, a)) is _FileFormat: + if type(getattr(param, a)) is _FileFormat or (param.type in [_InFile, _OutFile] and a == "restrictions"): setattr(param, a, _FileFormat(str(hardcoded_attributes[a]))) elif type(getattr(param, a)) is _Choices: # logger.error("SET CTD %s %s %s" %(param.name, a, hardcoded_attributes[a])) @@ -447,6 +437,14 @@ def _convert_internal(parsed_ctds, **kwargs): else: setattr(param, a, hardcoded_attributes[a]) + if "test_only" in kwargs and kwargs["test_only"]: + test = create_test_only(parsed_ctd.ctd_model, **kwargs) + tree = ElementTree(test) + output_file = parsed_ctd.suggested_output_file + logger.info("Writing to %s" % utils.get_filename(output_file), 1) + tree.write(output_file, encoding="UTF-8", xml_declaration=False, pretty_print=True) + continue + logger.info("Converting %s (source %s)" % (model.name, utils.get_filename(origin_file)), 0) tool = create_tool(model) write_header(tool, model) @@ -455,7 +453,7 @@ def _convert_internal(parsed_ctds, **kwargs): expand_macros(tool, kwargs["macros_to_expand"]) # command, inputs, outputs = create_cio(tool, model, **kwargs) create_command(tool, model, **kwargs) - create_configfile(tool, model, **kwargs) + create_configfile(tool, model) inputs = create_inputs(tool, model, **kwargs) outputs = create_outputs(tool, model, **kwargs) if kwargs["test_test"]: @@ -560,22 +558,19 @@ def create_description(tool, model): description.text = model.opt_attribs["description"] -def get_by_path(dataDict, mapList): - for k in mapList: - dataDict = dataDict[k] - return dataDict - - -def set_by_path(dic, keys, value): - for key in keys[:-1]: - dic = dic.setdefault(key, {}) - dic[keys[-1]] = value - - -def create_configfile(tool, model, **kwargs): +def create_configfile(tool, model): + """ + create section. + this will create a json file containing the tool parameter values that can + be accessed in cheetah with $args_json. + note that data_style="paths" (i.e. input data sets are included in the json) + is set even if input files are given on the CLI. reason is that in this way + default values in the CTD can be restored for optional input files. + """ configfiles_node = add_child_node(tool, "configfiles") - inputs_node = add_child_node(configfiles_node, "inputs", OrderedDict([("name", "args_json"), ("data_style", "paths")])) + inputs_node = add_child_node(configfiles_node, "inputs", + OrderedDict([("name", "args_json"), ("data_style", "paths")])) def create_command(tool, model, **kwargs): @@ -601,7 +596,8 @@ def create_command(tool, model, **kwargs): # TODO remove the PYTHONPATH export CTDopts should come from conda https://github.com/WorkflowConversion/CTDopts/pulls; also python3 -> python? final_cmd['command'].extend(["", "## Main program call"]) final_cmd['command'].append(""" -export PYTHONPATH='$__tool_directory__/CTDopts' && +set -o pipefail && +export PYTHONPATH='$__tool_directory__/CTDopts' && @EXECUTABLE@ -write_ctd ./ && python3 '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' && @EXECUTABLE@ -ini @EXECUTABLE@.ctd""") @@ -613,12 +609,10 @@ def create_command(tool, model, **kwargs): parameter_hardcoder = kwargs["parameter_hardcoder"] supported_file_formats = kwargs["supported_file_formats"] + g2o, o2g = get_fileformat_maps(supported_file_formats) - found_output_parameter = False for param in utils.extract_and_flatten_parameters(model): param = modify_param_for_galaxy(param) - if param.type is _OutFile: - found_output_parameter = True param_cmd = {'preprocessing': [], 'command': [], 'postprocessing': []} param_name = utils.extract_param_name(param) @@ -654,12 +648,13 @@ def create_command(tool, model, **kwargs): param_cmd['preprocessing'].append("ln -s '$" + actual_parameter + "' '" + actual_parameter + "/${re.sub(\"[^\w\-_]\", \"_\", $" + actual_parameter + ".element_identifier)}.$gxy2omsext($" + actual_parameter + ".ext)' &&") param_cmd['command'].append("'" + actual_parameter + "/${re.sub(\"[^\w\-_]\", \"_\", $" + actual_parameter + ".element_identifier)}.$gxy2omsext($" + actual_parameter + ".ext)'") elif param.type is _OutFile: + actual_parameter = get_galaxy_parameter_path(param, separator="_") # check if there is a parameter that sets the format # if so we add an extension to the generated files which will be used to # determine the format in the output tag # in all other cases (corresponding input / there is only one allowed format) # the format will be set in the output tag - formats = get_formats(param, supported_file_formats, TYPE_TO_GALAXY_TYPE[param.type]) + formats = get_formats(param, o2g, TYPE_TO_GALAXY_TYPE[param.type]) type_param = get_out_type_param(param, model, parameter_hardcoder) corresponding_input, fmt_from_corresponding = get_corresponding_input(param, model) if corresponding_input is not None: @@ -670,7 +665,7 @@ def create_command(tool, model, **kwargs): if type_param is not None: type_param_name = get_galaxy_parameter_path(type_param) elif len(formats) > 1 and (corresponding_input is None or not fmt_from_corresponding) and not param.is_list: - type_param_name = get_galaxy_parameter_path(param, "type") + type_param_name = get_galaxy_parameter_path(param, suffix="type") else: type_param_name = None @@ -682,11 +677,11 @@ def create_command(tool, model, **kwargs): if len(formats) == 1: fmt = formats.pop() if param.is_list: - logger.info("1 fmt + list %s -> %s" % (param.name, actual_input_parameter), 1) - param_cmd['command'].append("${' '.join([\"'" + actual_parameter + "/%s.%s'\"%(re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(\"" + fmt + "\")) for _ in $" + actual_parameter + " if _])}") - param_cmd['postprocessing'].append("${' '.join([\"&& mv -n '" + actual_parameter + "/%(id)s.%(gext)s' '" + actual_parameter + "/%(id)s'\"%{\"id\": re.sub('[^\w\-_]', '_', _.element_identifier), \"gext\": $gxy2omsext(\"" + fmt + "\")} for _ in $" + actual_parameter + " if _])}") + # logger.info("1 fmt + list %s -> %s" % (param.name, actual_input_parameter), 1) + param_cmd['command'].append("${' '.join([\"'" + actual_parameter + "/%s.%s'\"%(re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(\"" + fmt + "\")) for _ in $" + actual_input_parameter + " if _])}") + param_cmd['postprocessing'].append("${' '.join([\"&& mv -n '" + actual_input_parameter + "/%(id)s.%(gext)s' '" + actual_parameter + "/%(id)s'\"%{\"id\": re.sub('[^\w\-_]', '_', _.element_identifier), \"gext\": $gxy2omsext(\"" + fmt + "\")} for _ in $" + actual_parameter + " if _])}") else: - logger.info("1 fmt + dataset %s" % param.name, 1) + # logger.info("1 fmt + dataset %s" % param.name, 1) param_cmd['command'].append("'" + actual_parameter + "/output.${gxy2omsext(\"" + fmt + "\")}'") param_cmd['postprocessing'].append("&& mv '" + actual_parameter + "/output.${gxy2omsext(\"" + fmt + "\")}' '$" + actual_parameter + "'") @@ -695,11 +690,11 @@ def create_command(tool, model, **kwargs): # - list: let the command create output files with the oms extensions, postprocessing renames them to the galaxy extensions, output is then discover + __name_and_ext__ elif type_param_name is not None: if param.is_list: - logger.info("type + list %s" % param.name, 1) - param_cmd['command'].append("${' '.join([\"'" + actual_parameter + "/%s.%s'\"%(re.sub('[^\w\-_]', '_', _.element_identifier), " + type_param_name + ") for _ in $" + actual_parameter + " if _])}") - param_cmd['postprocessing'].append("${' '.join([\"&& mv -n '" + actual_parameter + "/%(id)s.%(omsext)s' '" + actual_parameter + "/%(id)s.%(gext)s'\"%{\"id\": re.sub('[^\w\-_]', '_', _.element_identifier), \"omsext\":" + type_param_name + ", \"gext\": oms2gxyext(\"" + type_param_name + "\")} for _ in $" + actual_parameter + " if _])}") + # logger.info("type + list %s" % param.name, 1) + param_cmd['command'].append("${' '.join([\"'" + actual_parameter + "/%s.%s'\"%(re.sub('[^\w\-_]', '_', _.element_identifier), $" + type_param_name + ") for _ in $" + actual_input_parameter + " if _])}") + param_cmd['postprocessing'].append("${' '.join([\"&& mv -n '" + actual_parameter + "/%(id)s.%(omsext)s' '" + actual_parameter + "/%(id)s.%(gext)s'\"%{\"id\": re.sub('[^\w\-_]', '_', _.element_identifier), \"omsext\":$" + type_param_name + ", \"gext\": $oms2gxyext(str($" + type_param_name + "))} for _ in $" + actual_input_parameter + " if _])}") else: - logger.info("type + dataset %s" % param.name, 1) + # logger.info("type + dataset %s" % param.name, 1) # 1st create file with openms extension (often required by openms) # then move it to the actual place specified by the parameter # the format is then set by the tag using @@ -707,10 +702,10 @@ def create_command(tool, model, **kwargs): param_cmd['postprocessing'].append("&& mv '" + actual_parameter + "/output.${" + type_param_name + "}' '$" + actual_parameter + "'") elif actual_input_parameter is not None: if param.is_list: - logger.info("actual + list %s" % param.name, 1) + # logger.info("actual + list %s" % param.name, 1) param_cmd['command'].append("${' '.join([\"'" + actual_parameter + "/%s.%s'\"%(re.sub('[^\w\-_]', '_', _.element_identifier), _.ext) for _ in $" + actual_input_parameter + " if _])}") else: - logger.info("actual + dataset %s %s %s" % (param.name, actual_input_parameter, corresponding_input.is_list), 1) + # logger.info("actual + dataset %s %s %s" % (param.name, actual_input_parameter, corresponding_input.is_list), 1) if corresponding_input.is_list: param_cmd['command'].append("'" + actual_parameter + "/output.${" + actual_input_parameter + "[0].ext}'") param_cmd['postprocessing'].append("&& mv '" + actual_parameter + "/output.${" + actual_input_parameter + "[0].ext}' '$" + actual_parameter + "'") @@ -721,7 +716,7 @@ def create_command(tool, model, **kwargs): if param.is_list: raise Exception("output parameter itemlist %s without corresponding input") else: - logger.info("else + dataset %s" % param.name, 1) + # logger.info("else + dataset %s" % param.name, 1) param_cmd['command'].append("'$" + actual_parameter + "'") # # select with multiple = true @@ -742,7 +737,7 @@ def create_command(tool, model, **kwargs): # need no if (otherwise the empty string could not be provided) if not (param.required or is_boolean_parameter(param) or (param.type is str and param.restrictions is None)): # and not(param.type is _InFile and param.is_list): - actual_parameter = get_galaxy_parameter_path(param, "FLAG") + actual_parameter = get_galaxy_parameter_path(param, suffix="FLAG") for stage in param_cmd: if len(param_cmd[stage]) == 0: continue @@ -768,8 +763,12 @@ def create_command(tool, model, **kwargs): advanced_cmd[stage] = [advanced_command_start] + utils.indent(advanced_cmd[stage]) + [advanced_command_end] final_cmd[stage].extend(advanced_cmd[stage]) - if not found_output_parameter: - final_cmd['command'].append("> $param_stdout") + optout, noout = all_outputs_optional(model, parameter_hardcoder) + if len(optout) > 0 or noout: + stdout = ["| tee '$stdout'"] + if len(optout) > 0: + stdout = ["#if %s:" % " and ".join(["not($%s)" % get_galaxy_parameter_path(_, suffix="FLAG") for _ in optout])] + utils.indent(stdout) + ["#end if"] + final_cmd['command'].extend(stdout) command_node = add_child_node(tool, "command") command_node.attrib["detect_errors"] = "exit_code" @@ -811,36 +810,40 @@ def expand_macros(node, macros_to_expand): expand_node.attrib["macro"] = expand_macro -def get_galaxy_parameter_path(param, inparam=False): +def get_galaxy_parameter_path(param, separator=".", suffix=None): """ - get the complete cheetah path for a parameter + Get the complete path for a parameter as a string where the path + components are joined by the given separator. A given suffix can + be appended. + + """ - p = get_galaxy_parameter_name(param, inparam) + p = get_galaxy_parameter_name(param, suffix) path = utils.extract_param_path(param) - if len(path) > 1 and (param.type is not _OutFile or inparam): - return (".".join(path[:-1]) + "." + p).replace("-", "_") - elif param.advanced and (param.type is not _OutFile or inparam): + if len(path) > 1: + return (separator.join(path[:-1]) + separator + p).replace("-", "_") + elif param.advanced and (param.type is not _OutFile or suffix): return ADVANCED_OPTIONS_NAME + "cond." + p else: return p -def get_galaxy_parameter_name(param, inparam=False): +def get_galaxy_parameter_name(param, suffix=None): """ get the name of the parameter used in the galaxy tool - replace : and - by _ - - add value inpram for output parameters if not False + - add suffix for output parameters if not None - the idea of inparam is to be used for optional outputs (out_x) for + the idea of suffix is to be used for optional outputs (out_x) for which an additional boolean input (out_x_FLAG) exists @param param the parameter - @param inparam get the name of the corresponding input + @param suffix suffix to append @return the name used for the parameter in the tool form """ p = param.name.replace("-", "_") - if param.type is _OutFile and inparam: - return "%s_%s" % (p, inparam) + if param.type is _OutFile and suffix is not None: + return "%s_%s" % (p, suffix) else: return "%s" % p @@ -937,6 +940,8 @@ def create_inputs(tool, model, **kwargs): # some suites (such as OpenMS) need some advanced options when handling inputs advanced_node = None parameter_hardcoder = kwargs["parameter_hardcoder"] + supported_file_formats = kwargs["supported_file_formats"] + g2o, o2g = get_fileformat_maps(supported_file_formats) # treat all non output-file/advanced/blacklisted/hardcoded parameters as inputs for param in utils.extract_and_flatten_parameters(model, True): @@ -949,7 +954,7 @@ def create_inputs(tool, model, **kwargs): param = modify_param_for_galaxy(param) # no need to show hardcoded parameters hardcoded_value = parameter_hardcoder.get_hardcoded_value(utils.extract_param_name(param), model.name) - if parameter_hardcoder.get_blacklist(utils.extract_param_name(param), model.name) or hardcoded_value is not None: + if parameter_hardcoder.get_blacklist(utils.extract_param_name(param), model.name): continue if utils.extract_param_name(param.parent) in section_nodes: @@ -966,20 +971,29 @@ def create_inputs(tool, model, **kwargs): if param.type is _OutFile: # if there are multiple possible output formats, but no parameter to choose the type or a # corresponding input then add a selection parameter - formats = get_formats(param, kwargs["supported_file_formats"], TYPE_TO_GALAXY_TYPE[_OutFile]) + formats = get_formats(param, o2g, TYPE_TO_GALAXY_TYPE[_OutFile]) type_param = get_out_type_param(param, model, parameter_hardcoder) corresponding_input, fmt_from_corresponding = get_corresponding_input(param, model) if len(formats) > 1 and type_param is None and (corresponding_input is None or not fmt_from_corresponding) and not param.is_list: fmt_select = add_child_node(parent_node, "param", OrderedDict([("name", param.name + "_type"), ("type", "select"), ("optional", "false"), ("label", "File type of output %s (%s)" % (param.name, param.description))])) g2o, o2g = get_fileformat_maps(kwargs["supported_file_formats"]) - for f in formats: - option_node = add_child_node(fmt_select, "option", OrderedDict([("value", g2o[f])]), f) - +# for f in formats: +# option_node = add_child_node(fmt_select, "option", OrderedDict([("value", g2o[f])]), f) + for choice in param.restrictions.formats: + option_node = add_child_node(fmt_select, "option", OrderedDict([("value", str(choice))])) + option_node.text = o2g[str(choice)] + if choice.lower() != o2g[str(choice)]: + option_node.text += " (%s)" % choice # create an additional bool input which is used to filter for the output # mandatory outpiles: no input node needed # inputs: create the input param if not param.required: - add_child_node(parent_node, "param", OrderedDict([("type", "boolean"), ("name", param.name + "_FLAG"), ("optional", "false"), ("checked", "false"), ("label", "Generate output %s (%s)" % (param.name, param.description))])) + add_child_node(parent_node, "param", OrderedDict([("type", "boolean"), + ("name", param.name + "_FLAG"), + ("checked", "false"), + ("truevalue", "true"), + ("falsevalue", "false"), + ("label", "Generate output %s (%s)" % (param.name, param.description))])) # else: # add_child_node(parent_node, "param", OrderedDict([("type", "hidden"), ("name", param.name)])) continue @@ -988,10 +1002,21 @@ def create_inputs(tool, model, **kwargs): param_node = add_child_node(parent_node, "param") create_param_attribute_list(param_node, param, model, kwargs["supported_file_formats"]) + # set hardcoded values as hidden parameters + if hardcoded_value is not None: + param_node.attrib["value"] = hardcoded_value + param_node.attrib["type"] = "hidden" + for a in param_node.attrib.keys(): + if a not in ["argument", "name", "value", "type"]: + del param_node.attrib[a] + san_node = add_child_node(param_node, "sanitizer") + add_child_node(san_node, "valid", OrderedDict([("initial", "string.printable")])) + hardcoded_attributes = parameter_hardcoder.get_hardcoded_attributes(param.name, model.name) if hardcoded_attributes is not None: for a in hardcoded_attributes: - param_node.attrib[a] = str(hardcoded_attributes[a]) + if a in param_node.attrib: + param_node.attrib[a] = str(hardcoded_attributes[a]) for sn in section_nodes: if utils.extract_param_name(section_params[sn].parent) in section_nodes: @@ -1012,12 +1037,12 @@ def is_default(value, param): return param.default == value or (type(param.default) is list and value in param.default) -def get_formats(param, supported_file_formats, default=None): +def get_formats(param, o2g, default=None): """ - determine format attribute + determine format attribute (i.e. list of allowed Galaxy extensions) from + the CTD restictions (i.e. the OpenMS extensions) - check if all listed possible formats are supported in Galaxy - if there is a single one, then take this - - otherwise try to determine an input with the same restrictions and use this as format source """ formats = set() if param.restrictions is not None: @@ -1025,14 +1050,14 @@ def get_formats(param, supported_file_formats, default=None): # check if there are formats that have not been registered yet... output = list() for format_name in param.restrictions.formats: - if format_name not in supported_file_formats.keys(): + if format_name not in o2g: output.append(str(format_name)) + else: + formats.add(o2g[format_name]) # warn only if there's about to complain if output: logger.warning("Parameter " + param.name + " has the following unsupported format(s):" + ','.join(output), 1) - - formats = get_supported_file_types(param.restrictions.formats, supported_file_formats) else: raise InvalidModelException("Unrecognized restriction type [%(type)s] " "for [%(name)s]" % {"type": type(param.restrictions), @@ -1061,7 +1086,12 @@ def create_param_attribute_list(param_node, param, model, supported_file_formats # set the name, argument and a first guess for the type (which will be over written # in some cases .. see below) - param_node.attrib["name"] = param.name + # even if the conversion relies on the fact that the param names are identical + # to the ctd ITEM names we replace dashes by underscores because input and output + # parameters need to be treated in cheetah. variable names are currently fixed back + # to dashes in fill_ctd.py. currently there seems to be only a single tool + # requiring this https://github.com/OpenMS/OpenMS/pull/4529 + param_node.attrib["name"] = get_galaxy_parameter_name(param) param_node.attrib["argument"] = "-%s" % utils.extract_param_name(param) param_type = TYPE_TO_GALAXY_TYPE[param.type] if param_type is None: @@ -1085,7 +1115,7 @@ def create_param_attribute_list(param_node, param, model, supported_file_formats if param.type is _InFile: # assume it's just text unless restrictions are provided param_node.attrib["type"] = "data" - param_node.attrib["format"] = ",".join(get_formats(param, supported_file_formats, TYPE_TO_GALAXY_TYPE[_InFile])) + param_node.attrib["format"] = ",".join(get_formats(param, o2g, TYPE_TO_GALAXY_TYPE[_InFile])) # in the case of multiple input set multiple flag if param.is_list: param_node.attrib["multiple"] = "true" @@ -1110,17 +1140,19 @@ def create_param_attribute_list(param_node, param, model, supported_file_formats # param.restrictions.choices = get_supported_file_types(param.restrictions.choices, supported_file_formats) # add a nothing selected option to mandatory options w/o default - if param.default is None or param.default is _Null: + if param.default is None or type(param.default) is _Null: if param_node.attrib["optional"] == "true": option_node = add_child_node(param_node, "option", OrderedDict([("value", "")]), text="default (nothing chosen)") # else: # option_node = add_child_node(param_node, "option", OrderedDict([("value", "")]), text="select a value") - + # create as many - + + + @@ -75,7 +80,7 @@ python3 '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' && - + $param_stdout +| tee '$stdout' ## Postprocessing]]> @@ -29,11 +30,11 @@ python3 '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' && - + - + recurse - True/False (bool objects) -> "true"/"false" (lowercase string) + - data inputs with multiple and optional true give [None] if no file is given -> [] - None -> "" (empty string) + - replace bash expressions: + - environment variables (need to consist capital letters and _) by their value + - expressions """ - for k in d: - if type(d[k]) is bool: - d[k] = str(d[k]).lower() - elif type(d[k]) is list and len(d[k]) == 1 and d[k][0] is None: + for k in d.keys(): +# if type(d[k]) is bool: +# d[k] = str(d[k]).lower() +# el + if type(d[k]) is list and len(d[k]) == 1 and d[k][0] is None: d[k] = [] elif d[k] is None: d[k] = "" + elif type(d[k]) is str and d[k].startswith("$"): + m = re.fullmatch("\$([A-Z_]+)", d[k]) + if m: + d[k] = os.environ.get(m.group(1), "") + continue + m = re.fullmatch("\$(\{[A-Z_]+):-(.*)\}", d[k]) + if m: + d[k] = os.environ.get(m.group(1), m.group(2)) + continue + + try: + p = subprocess.run("echo %s" % d[k], shell=True, check=True, stdout=subprocess.PIPE, encoding="utf8") + d[k] = p.stdout.strip() + except subprocess.CalledProcessError: + sys.stderr.write("fill_ctd error: Could not evaluate %s" % d[k]) + continue return d @@ -38,20 +68,26 @@ def qstring2list(qs): """ lst = list() qs = qs.split(" ") + quoted = False for p in qs: if p == "": continue - if p.startswith('"') and p.endswith('"'): lst.append(p[1:-1]) elif p.startswith('"'): - lst.append( p[1:]+" ") + quoted = True + lst.append(p[1:] + " ") elif p.endswith('"'): + quoted = False lst[-1] += p[:-1] else: - lst.append(p) + if quoted: + lst[-1] += p + " " + else: + lst.append(p) return lst + input_ctd = sys.argv[1] with open(sys.argv[2]) as fh: args = json.load(fh, object_hook=_json_object_hook) @@ -62,17 +98,36 @@ def qstring2list(qs): model = CTDModel(from_file=input_ctd) -# transform values from json that -# - correspond to unrestricted ITEMLIST which are represented as strings -# ("=quoted and space separated) in Galaxy to lists +# transform values from json that correspond to +# - old style booleans (string + restrictions) -> transformed to a str +# - unrestricted ITEMLIST which are represented as strings +# ("=quoted and space separated) in Galaxy -> transform to lists # - optional data input parameters that have defaults and for which no -# value is given are overwritten with the default +# value is given -> overwritte with the default for p in model.get_parameters(): - if p.is_list and (p.restrictions is None or type(p.restrictions) is _NumericRange): + + # few tools use dashes in parameters which are automatically replaced + # by underscores by Galaxy. in these cases the dictionary needs to be + # updates + # TODO might be removed later https://github.com/OpenMS/OpenMS/pull/4529 + try: + getFromDict(args, p.get_lineage(name_only=True)) + except KeyError: + try: + jl = [_.replace("-", "_") for _ in p.get_lineage(name_only=True)] + setInDict(args, p.get_lineage(name_only=True), getFromDict(args, jl)) + except KeyError: + pass + + if p.type is str and type(p.restrictions) is _Choices and set(p.restrictions.choices) == set(["true","false"]): + v = getFromDict(args, p.get_lineage(name_only=True)) + setInDict(args, p.get_lineage(name_only=True), str(v).lower()) + + elif p.is_list and (p.restrictions is None or type(p.restrictions) is _NumericRange): v = getFromDict(args, p.get_lineage(name_only=True)) if type(v) is str: setInDict(args, p.get_lineage(name_only=True), qstring2list(v)) - if p.type is _InFile and p.default not in [None, _Null]: + elif p.type is _InFile and not (p.default is None or type(p.default) is _Null): v = getFromDict(args, p.get_lineage(name_only=True)) if v in [[], ""]: setInDict(args, p.get_lineage(name_only=True), p.default) diff --git a/tests/test-data/float.xml b/tests/test-data/float.xml index a79ab809..a003050c 100644 --- a/tests/test-data/float.xml +++ b/tests/test-data/float.xml @@ -17,11 +17,12 @@ ## Main program call -export PYTHONPATH='$__tool_directory__/CTDopts' && +set -o pipefail && +export PYTHONPATH='$__tool_directory__/CTDopts' && @EXECUTABLE@ -write_ctd ./ && python3 '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' && @EXECUTABLE@ -ini @EXECUTABLE@.ctd -> $param_stdout +| tee '$stdout' ## Postprocessing]]> @@ -66,7 +67,7 @@ python3 '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' && - + @@ -91,7 +92,7 @@ python3 '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' && - + $param_stdout +| tee '$stdout' ## Postprocessing]]> @@ -806,7 +807,7 @@ ${' '.join(["'test_section.sect_list_mandatory_multformat/%s.%s'"%(re.sub('[^\w\ - + @@ -839,7 +840,7 @@ ${' '.join(["'test_section.sect_list_mandatory_multformat/%s.%s'"%(re.sub('[^\w\ - + $param_stdout +| tee '$stdout' ## Postprocessing]]> @@ -66,7 +67,7 @@ python3 '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' && - + @@ -91,7 +92,7 @@ python3 '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' && - + $param_stdout +| tee '$stdout' ## Postprocessing]]> @@ -54,7 +55,7 @@ python3 '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' && - + @@ -66,7 +67,7 @@ python3 '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' && - + - - - + + +
- - - - - - + + + + + +
- - - + + + @@ -258,28 +259,28 @@ python3 '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' && adv_opts_cond['adv_opts_selector'] == 'advanced' and adv_opts_cond['optional_multformat_adv_FLAG'] - - + + test_section["sect_optional_noformat_FLAG"] - - + + test_section["sect_optional_oneformat_FLAG"] - - + + test_section["sect_optional_multformat_FLAG"] - - + + test_section["sect_optional_noformat_adv_FLAG"] - - + + test_section["sect_optional_oneformat_adv_FLAG"] - - + + test_section["sect_optional_multformat_adv_FLAG"] @@ -311,12 +312,12 @@ python3 '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' && - - - - - - + + + + + + - - + + - - + + - + - + diff --git a/tests/test-data/ofile-mult-typeparam.xml b/tests/test-data/ofile-mult-typeparam.xml index 749b1cc4..c9a8f362 100644 --- a/tests/test-data/ofile-mult-typeparam.xml +++ b/tests/test-data/ofile-mult-typeparam.xml @@ -22,8 +22,8 @@ ${ ' '.join(["ln -s '%s' 'inp_optional_mandatoryinput/%s.%s' &&" % (_, re.sub('[ mkdir inp_mandatory_optionalinput && ${ ' '.join(["ln -s '%s' 'inp_mandatory_optionalinput/%s.%s' &&" % (_, re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $inp_mandatory_optionalinput if _]) } #end if -mkdir sect_mandatory_mandatoryinput && -mkdir sect_mandatory_optionalinput && +mkdir test_section_sect_mandatory_mandatoryinput && +mkdir test_section_sect_mandatory_optionalinput && mkdir test_section.sect_inp_optional_mandatoryinput && ${ ' '.join(["ln -s '%s' 'test_section.sect_inp_optional_mandatoryinput/%s.%s' &&" % (_, re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $test_section.sect_inp_optional_mandatoryinput if _]) } #if $test_section.sect_inp_mandatory_optionalinput: @@ -33,14 +33,15 @@ ${ ' '.join(["ln -s '%s' 'test_section.sect_inp_optional_mandatoryinput/%s.%s' & ## Main program call -export PYTHONPATH='$__tool_directory__/CTDopts' && +set -o pipefail && +export PYTHONPATH='$__tool_directory__/CTDopts' && @EXECUTABLE@ -write_ctd ./ && python3 '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' && @EXECUTABLE@ -ini @EXECUTABLE@.ctd -mandatory_mandatoryinput -${' '.join(["'mandatory_mandatoryinput/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext("tsv")) for _ in $mandatory_mandatoryinput if _])} +${' '.join(["'mandatory_mandatoryinput/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext("tsv")) for _ in $inp_optional_mandatoryinput if _])} -mandatory_optionalinput -${' '.join(["'mandatory_optionalinput/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier), mandatory_optionalinput_type) for _ in $mandatory_optionalinput if _])} +${' '.join(["'mandatory_optionalinput/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier), $mandatory_optionalinput_type) for _ in $inp_mandatory_optionalinput if _])} -inp_optional_mandatoryinput ${' '.join(["'inp_optional_mandatoryinput/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $inp_optional_mandatoryinput if _])} #if $inp_mandatory_optionalinput: @@ -48,9 +49,9 @@ ${' '.join(["'inp_optional_mandatoryinput/%s.%s'"%(re.sub('[^\w\-_]', '_', _.ele ${' '.join(["'inp_mandatory_optionalinput/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $inp_mandatory_optionalinput if _])} #end if -test_section:sect_mandatory_mandatoryinput -${' '.join(["'sect_mandatory_mandatoryinput/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext("tsv")) for _ in $sect_mandatory_mandatoryinput if _])} +${' '.join(["'test_section_sect_mandatory_mandatoryinput/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier), $test_section.sect_mandatory_mandatoryinput_type) for _ in $test_section.sect_inp_optional_mandatoryinput if _])} -test_section:sect_mandatory_optionalinput -${' '.join(["'sect_mandatory_optionalinput/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier), test_section.sect_mandatory_optionalinput_type) for _ in $sect_mandatory_optionalinput if _])} +${' '.join(["'test_section_sect_mandatory_optionalinput/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier), $test_section.sect_mandatory_optionalinput_type) for _ in $test_section.sect_inp_mandatory_optionalinput if _])} -test_section:sect_inp_optional_mandatoryinput ${' '.join(["'test_section.sect_inp_optional_mandatoryinput/%s.%s'"%(re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $test_section.sect_inp_optional_mandatoryinput if _])} #if $test_section.sect_inp_mandatory_optionalinput: @@ -59,41 +60,43 @@ ${' '.join(["'test_section.sect_inp_optional_mandatoryinput/%s.%s'"%(re.sub('[^\ #end if ## Postprocessing -${' '.join(["&& mv -n 'mandatory_mandatoryinput/%(id)s.%(gext)s' 'mandatory_mandatoryinput/%(id)s'"%{"id": re.sub('[^\w\-_]', '_', _.element_identifier), "gext": $gxy2omsext("tsv")} for _ in $mandatory_mandatoryinput if _])} -${' '.join(["&& mv -n 'mandatory_optionalinput/%(id)s.%(omsext)s' 'mandatory_optionalinput/%(id)s.%(gext)s'"%{"id": re.sub('[^\w\-_]', '_', _.element_identifier), "omsext":mandatory_optionalinput_type, "gext": oms2gxyext("mandatory_optionalinput_type")} for _ in $mandatory_optionalinput if _])} -${' '.join(["&& mv -n 'sect_mandatory_mandatoryinput/%(id)s.%(gext)s' 'sect_mandatory_mandatoryinput/%(id)s'"%{"id": re.sub('[^\w\-_]', '_', _.element_identifier), "gext": $gxy2omsext("tsv")} for _ in $sect_mandatory_mandatoryinput if _])} -${' '.join(["&& mv -n 'sect_mandatory_optionalinput/%(id)s.%(omsext)s' 'sect_mandatory_optionalinput/%(id)s.%(gext)s'"%{"id": re.sub('[^\w\-_]', '_', _.element_identifier), "omsext":test_section.sect_mandatory_optionalinput_type, "gext": oms2gxyext("test_section.sect_mandatory_optionalinput_type")} for _ in $sect_mandatory_optionalinput if _])}]]> +${' '.join(["&& mv -n 'inp_optional_mandatoryinput/%(id)s.%(gext)s' 'mandatory_mandatoryinput/%(id)s'"%{"id": re.sub('[^\w\-_]', '_', _.element_identifier), "gext": $gxy2omsext("tsv")} for _ in $mandatory_mandatoryinput if _])} +${' '.join(["&& mv -n 'mandatory_optionalinput/%(id)s.%(omsext)s' 'mandatory_optionalinput/%(id)s.%(gext)s'"%{"id": re.sub('[^\w\-_]', '_', _.element_identifier), "omsext":$mandatory_optionalinput_type, "gext": $oms2gxyext(str($mandatory_optionalinput_type))} for _ in $inp_mandatory_optionalinput if _])} +${' '.join(["&& mv -n 'test_section_sect_mandatory_mandatoryinput/%(id)s.%(omsext)s' 'test_section_sect_mandatory_mandatoryinput/%(id)s.%(gext)s'"%{"id": re.sub('[^\w\-_]', '_', _.element_identifier), "omsext":$test_section.sect_mandatory_mandatoryinput_type, "gext": $oms2gxyext(str($test_section.sect_mandatory_mandatoryinput_type))} for _ in $test_section.sect_inp_optional_mandatoryinput if _])} +${' '.join(["&& mv -n 'test_section_sect_mandatory_optionalinput/%(id)s.%(omsext)s' 'test_section_sect_mandatory_optionalinput/%(id)s.%(gext)s'"%{"id": re.sub('[^\w\-_]', '_', _.element_identifier), "omsext":$test_section.sect_mandatory_optionalinput_type, "gext": $oms2gxyext(str($test_section.sect_mandatory_optionalinput_type))} for _ in $test_section.sect_inp_mandatory_optionalinput if _])}]]> - - + + - - - + + +
- - + + + - - - + + + + - - + +
@@ -103,11 +106,11 @@ ${' '.join(["&& mv -n 'sect_mandatory_optionalinput/%(id)s.%(omsext)s' 'sect_man - - + + - - + + @@ -116,14 +119,14 @@ ${' '.join(["&& mv -n 'sect_mandatory_optionalinput/%(id)s.%(omsext)s' 'sect_man
- - + +
- - + + - - - 3.7 + 3.8 0 python - + ctdopts diff --git a/tests/test-data/bool.xml b/tests/test-data/bool.xml index 215efdb8..055f8ef1 100644 --- a/tests/test-data/bool.xml +++ b/tests/test-data/bool.xml @@ -68,7 +68,7 @@ python3 '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' '$hard - + diff --git a/tests/test-data/empty.xml b/tests/test-data/empty.xml index 9eed92d7..87ba915f 100644 --- a/tests/test-data/empty.xml +++ b/tests/test-data/empty.xml @@ -47,7 +47,7 @@ python3 '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' '$hard - + diff --git a/tests/test-data/float.xml b/tests/test-data/float.xml index 155271a9..23d2a99a 100644 --- a/tests/test-data/float.xml +++ b/tests/test-data/float.xml @@ -82,7 +82,7 @@ python3 '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' '$hard - + diff --git a/tests/test-data/ifile.xml b/tests/test-data/ifile.xml index c8db4862..3bb949eb 100644 --- a/tests/test-data/ifile.xml +++ b/tests/test-data/ifile.xml @@ -870,7 +870,7 @@ ${' '.join(["'test_section.sect_list_mandatory_multformat/%s/%s.%s'"%(os.path.ba - + diff --git a/tests/test-data/integer.xml b/tests/test-data/integer.xml index 5c5dc618..fa638691 100644 --- a/tests/test-data/integer.xml +++ b/tests/test-data/integer.xml @@ -82,7 +82,7 @@ python3 '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' '$hard - + diff --git a/tests/test-data/label-help.xml b/tests/test-data/label-help.xml index c41289f0..dffd5f25 100644 --- a/tests/test-data/label-help.xml +++ b/tests/test-data/label-help.xml @@ -71,7 +71,7 @@ python3 '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' '$hard - + diff --git a/tests/test-data/repeat_test.xml b/tests/test-data/repeat_test.xml index 1a7643ed..7dd9bb1d 100644 --- a/tests/test-data/repeat_test.xml +++ b/tests/test-data/repeat_test.xml @@ -402,7 +402,7 @@ python3 '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' '$hard - + diff --git a/tests/test-data/select_test.xml b/tests/test-data/select_test.xml index 7810d66d..3709d3a4 100644 --- a/tests/test-data/select_test.xml +++ b/tests/test-data/select_test.xml @@ -546,7 +546,7 @@ python3 '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' '$hard - + diff --git a/tests/test-data/string.xml b/tests/test-data/string.xml index 941b2ac3..50413d74 100644 --- a/tests/test-data/string.xml +++ b/tests/test-data/string.xml @@ -98,7 +98,7 @@ python3 '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' '$hard - +
diff --git a/tests/test_galaxy_cli.py b/tests/test_galaxy_cli.py index 31381684..66c65ccf 100644 --- a/tests/test_galaxy_cli.py +++ b/tests/test_galaxy_cli.py @@ -28,7 +28,7 @@ def _compare_cli_output(self, fileprefix): # out_file = to_test_data('{}.xml'.format(fileprefix)) my_env = os.environ.copy() my_env["PATH"] = fileprefix + ":" + my_env["PATH"] - cmd = ['CTDConverter', 'galaxy', '-i', in_pth, '-o', out_file, '-f', ftypes_pth, '-m', macro_pth, '--test-test', '-p', hcparam_pth, "--tool-version", "5.0.011"] + cmd = ['CTDConverter', 'galaxy', '-i', in_pth, '-o', out_file, '-f', ftypes_pth, '-m', macro_pth, '--test-test', '-p', hcparam_pth, "--tool-version", "3.8"] # print("cmd %s" % cmd) popen = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=my_env) From b044b082659ffb1de672dd1e5e6378b529137a20 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Mon, 8 Feb 2021 19:36:42 +0100 Subject: [PATCH 116/121] change format only for non defaults if the format of an output is determined by a select then set the select-default as format of the output and add `` only for the other choices --- galaxy/converter.py | 10 ++++++--- tests/test-data/ofile-typeparam.xml | 32 +++++++++++------------------ tests/test-data/ofile.xml | 32 +++++++++++------------------ 3 files changed, 31 insertions(+), 43 deletions(-) diff --git a/galaxy/converter.py b/galaxy/converter.py index cd4166f3..072f7b2a 100755 --- a/galaxy/converter.py +++ b/galaxy/converter.py @@ -1635,13 +1635,17 @@ def create_output_node(parent, param, model, supported_file_formats, parameter_h elif type_param_name is not None: logger.info("OUTPUT %s type" % param.name, 1) if not param.is_list: - change_node = add_child_node(data_node, "change_format") - for r in type_param_choices: + if len(type_param_choices) > 1: + change_node = add_child_node(data_node, "change_format") + for i, r in enumerate(type_param_choices): f = o2g.get(r, None) # TODO this should not happen for fully specified fileformats file if f is None: f = r - add_child_node(change_node, "when", OrderedDict([("input", type_param_name), ("value", r), ("format", f)])) + if i == 0: + data_node.attrib["format"] = f + else: + add_child_node(change_node, "when", OrderedDict([("input", type_param_name), ("value", r), ("format", f)])) else: discover_node.attrib["pattern"] = "__name_and_ext__" elif corresponding_input is not None: diff --git a/tests/test-data/ofile-typeparam.xml b/tests/test-data/ofile-typeparam.xml index 725657e1..568ebe07 100644 --- a/tests/test-data/ofile-typeparam.xml +++ b/tests/test-data/ofile-typeparam.xml @@ -294,16 +294,14 @@ python3 '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' '$hard OPTIONAL_OUTPUTS is not None and "optional_noformat_FLAG" in OPTIONAL_OUTPUTS - + - - + - @@ -313,16 +311,14 @@ python3 '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' '$hard OPTIONAL_OUTPUTS is not None and "optional_noformat_adv_FLAG" in OPTIONAL_OUTPUTS - + - - + - @@ -332,16 +328,14 @@ python3 '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' '$hard OPTIONAL_OUTPUTS is not None and "sect_optional_noformat_FLAG" in OPTIONAL_OUTPUTS - + - - + - @@ -351,16 +345,14 @@ python3 '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' '$hard OPTIONAL_OUTPUTS is not None and "sect_optional_noformat_adv_FLAG" in OPTIONAL_OUTPUTS - + - - + - @@ -400,13 +392,13 @@ python3 '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' '$hard - + - + - + - + OPTIONAL_OUTPUTS is not None and "optional_oneformat_FLAG" in OPTIONAL_OUTPUTS - + - - + - @@ -295,16 +293,14 @@ python3 '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' '$hard OPTIONAL_OUTPUTS is not None and "optional_oneformat_adv_FLAG" in OPTIONAL_OUTPUTS - + - - + - @@ -318,16 +314,14 @@ python3 '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' '$hard OPTIONAL_OUTPUTS is not None and "sect_optional_oneformat_FLAG" in OPTIONAL_OUTPUTS - + - - + - @@ -341,16 +335,14 @@ python3 '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' '$hard OPTIONAL_OUTPUTS is not None and "sect_optional_oneformat_adv_FLAG" in OPTIONAL_OUTPUTS - + - - + - @@ -377,16 +369,16 @@ python3 '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' '$hard - + - + - + - + Date: Wed, 10 Feb 2021 11:15:03 +0100 Subject: [PATCH 117/121] exclude some unsupported/nonsense test cases --- tests/test-data/ofile-mult-typeparam.ctd | 28 +++---- tests/test-data/ofile-mult-typeparam.xml | 95 ++++++------------------ tests/test-data/ofile-mult.ctd | 9 ++- tests/test-data/ofile-mult.xml | 43 +---------- tests/test-data/ofile-typeparam.ctd | 14 ++-- tests/test-data/ofile-typeparam.xml | 24 +++--- tests/test-data/ofile.xml | 24 +++--- 7 files changed, 74 insertions(+), 163 deletions(-) diff --git a/tests/test-data/ofile-mult-typeparam.ctd b/tests/test-data/ofile-mult-typeparam.ctd index 8283894b..5e77871a 100644 --- a/tests/test-data/ofile-mult-typeparam.ctd +++ b/tests/test-data/ofile-mult-typeparam.ctd @@ -14,33 +14,33 @@ - - - - - + + - + --> - - - - - + + diff --git a/tests/test-data/ofile-mult-typeparam.xml b/tests/test-data/ofile-mult-typeparam.xml index fd7c3734..e8bb0e07 100644 --- a/tests/test-data/ofile-mult-typeparam.xml +++ b/tests/test-data/ofile-mult-typeparam.xml @@ -16,29 +16,15 @@ ## Preprocessing mkdir mandatory_mandatoryinput && -mkdir ${' '.join(["'mandatory_mandatoryinput/%s'"%(os.path.basename(str(_))[:-4]) for _ in $inp_optional_mandatoryinput if _])} && -mkdir mandatory_optionalinput && -mkdir ${' '.join(["'mandatory_optionalinput/%s'"%(os.path.basename(str(_))[:-4]) for _ in $inp_mandatory_optionalinput if _])} && -mkdir inp_optional_mandatoryinput && -mkdir ${' '.join(["'inp_optional_mandatoryinput/%s'"%(os.path.basename(str(_))[:-4]) for _ in $inp_optional_mandatoryinput if _])} && -${' '.join(["ln -s '%s' 'inp_optional_mandatoryinput/%s/%s.%s' && " % (_, os.path.basename(str(_))[:-4], re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $inp_optional_mandatoryinput if _])} -#if $inp_mandatory_optionalinput: - mkdir inp_mandatory_optionalinput && - mkdir ${' '.join(["'inp_mandatory_optionalinput/%s'"%(os.path.basename(str(_))[:-4]) for _ in $inp_mandatory_optionalinput if _])} && - ${' '.join(["ln -s '%s' 'inp_mandatory_optionalinput/%s/%s.%s' && " % (_, os.path.basename(str(_))[:-4], re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $inp_mandatory_optionalinput if _])} -#end if +mkdir ${' '.join(["'mandatory_mandatoryinput/%s'"%(os.path.basename(str(_))[:-4]) for _ in $inp_mandatory_mandatoryinput if _])} && +mkdir inp_mandatory_mandatoryinput && +mkdir ${' '.join(["'inp_mandatory_mandatoryinput/%s'"%(os.path.basename(str(_))[:-4]) for _ in $inp_mandatory_mandatoryinput if _])} && +${' '.join(["ln -s '%s' 'inp_mandatory_mandatoryinput/%s/%s.%s' && " % (_, os.path.basename(str(_))[:-4], re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $inp_mandatory_mandatoryinput if _])} mkdir test_section_sect_mandatory_mandatoryinput && -mkdir ${' '.join(["'test_section_sect_mandatory_mandatoryinput/%s'"%(os.path.basename(str(_))[:-4]) for _ in $test_section.sect_inp_optional_mandatoryinput if _])} && -mkdir test_section_sect_mandatory_optionalinput && -mkdir ${' '.join(["'test_section_sect_mandatory_optionalinput/%s'"%(os.path.basename(str(_))[:-4]) for _ in $test_section.sect_inp_mandatory_optionalinput if _])} && -mkdir test_section.sect_inp_optional_mandatoryinput && -mkdir ${' '.join(["'test_section.sect_inp_optional_mandatoryinput/%s'"%(os.path.basename(str(_))[:-4]) for _ in $test_section.sect_inp_optional_mandatoryinput if _])} && -${' '.join(["ln -s '%s' 'test_section.sect_inp_optional_mandatoryinput/%s/%s.%s' && " % (_, os.path.basename(str(_))[:-4], re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $test_section.sect_inp_optional_mandatoryinput if _])} -#if $test_section.sect_inp_mandatory_optionalinput: - mkdir test_section.sect_inp_mandatory_optionalinput && - mkdir ${' '.join(["'test_section.sect_inp_mandatory_optionalinput/%s'"%(os.path.basename(str(_))[:-4]) for _ in $test_section.sect_inp_mandatory_optionalinput if _])} && - ${' '.join(["ln -s '%s' 'test_section.sect_inp_mandatory_optionalinput/%s/%s.%s' && " % (_, os.path.basename(str(_))[:-4], re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $test_section.sect_inp_mandatory_optionalinput if _])} -#end if +mkdir ${' '.join(["'test_section_sect_mandatory_mandatoryinput/%s'"%(os.path.basename(str(_))[:-4]) for _ in $test_section.sect_inp_mandatory_mandatoryinput if _])} && +mkdir test_section.sect_inp_mandatory_mandatoryinput && +mkdir ${' '.join(["'test_section.sect_inp_mandatory_mandatoryinput/%s'"%(os.path.basename(str(_))[:-4]) for _ in $test_section.sect_inp_mandatory_mandatoryinput if _])} && +${' '.join(["ln -s '%s' 'test_section.sect_inp_mandatory_mandatoryinput/%s/%s.%s' && " % (_, os.path.basename(str(_))[:-4], re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $test_section.sect_inp_mandatory_mandatoryinput if _])} ## Main program call @@ -47,31 +33,17 @@ set -o pipefail && python3 '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' '$hardcoded_json' && @EXECUTABLE@ -ini @EXECUTABLE@.ctd -mandatory_mandatoryinput -${' '.join(["'mandatory_mandatoryinput/%s/%s.%s'"%(os.path.basename(str(_))[:-4], re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext("tsv")) for _ in $inp_optional_mandatoryinput if _])} --mandatory_optionalinput -${' '.join(["'mandatory_optionalinput/%s/%s.%s'"%(os.path.basename(str(_))[:-4], re.sub('[^\w\-_]', '_', _.element_identifier), $mandatory_optionalinput_type) for _ in $inp_mandatory_optionalinput if _])} --inp_optional_mandatoryinput -${' '.join(["'inp_optional_mandatoryinput/%s/%s.%s'"%(os.path.basename(str(_))[:-4], re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $inp_optional_mandatoryinput if _])} -#if $inp_mandatory_optionalinput: - -inp_mandatory_optionalinput - ${' '.join(["'inp_mandatory_optionalinput/%s/%s.%s'"%(os.path.basename(str(_))[:-4], re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $inp_mandatory_optionalinput if _])} -#end if +${' '.join(["'mandatory_mandatoryinput/%s/%s.%s'"%(os.path.basename(str(_))[:-4], re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext("tsv")) for _ in $inp_mandatory_mandatoryinput if _])} +-inp_mandatory_mandatoryinput +${' '.join(["'inp_mandatory_mandatoryinput/%s/%s.%s'"%(os.path.basename(str(_))[:-4], re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $inp_mandatory_mandatoryinput if _])} -test_section:sect_mandatory_mandatoryinput -${' '.join(["'test_section_sect_mandatory_mandatoryinput/%s/%s.%s'"%(os.path.basename(str(_))[:-4], re.sub('[^\w\-_]', '_', _.element_identifier), $test_section.sect_mandatory_mandatoryinput_type) for _ in $test_section.sect_inp_optional_mandatoryinput if _])} --test_section:sect_mandatory_optionalinput -${' '.join(["'test_section_sect_mandatory_optionalinput/%s/%s.%s'"%(os.path.basename(str(_))[:-4], re.sub('[^\w\-_]', '_', _.element_identifier), $test_section.sect_mandatory_optionalinput_type) for _ in $test_section.sect_inp_mandatory_optionalinput if _])} --test_section:sect_inp_optional_mandatoryinput -${' '.join(["'test_section.sect_inp_optional_mandatoryinput/%s/%s.%s'"%(os.path.basename(str(_))[:-4], re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $test_section.sect_inp_optional_mandatoryinput if _])} -#if $test_section.sect_inp_mandatory_optionalinput: - -test_section:sect_inp_mandatory_optionalinput - ${' '.join(["'test_section.sect_inp_mandatory_optionalinput/%s/%s.%s'"%(os.path.basename(str(_))[:-4], re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $test_section.sect_inp_mandatory_optionalinput if _])} -#end if +${' '.join(["'test_section_sect_mandatory_mandatoryinput/%s/%s.%s'"%(os.path.basename(str(_))[:-4], re.sub('[^\w\-_]', '_', _.element_identifier), $test_section.sect_mandatory_mandatoryinput_type) for _ in $test_section.sect_inp_mandatory_mandatoryinput if _])} +-test_section:sect_inp_mandatory_mandatoryinput +${' '.join(["'test_section.sect_inp_mandatory_mandatoryinput/%s/%s.%s'"%(os.path.basename(str(_))[:-4], re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $test_section.sect_inp_mandatory_mandatoryinput if _])} ## Postprocessing -${' '.join(["&& mv -n 'mandatory_mandatoryinput/%(bn)s/%(id)s.%(gext)s' 'mandatory_mandatoryinput/%(bn)s/%(id)s'"%{"bn": os.path.basename(str(_))[:-4], "id": re.sub('[^\w\-_]', '_', _.element_identifier), "gext": $gxy2omsext("tsv")} for _ in $inp_optional_mandatoryinput if _])} -${' '.join(["&& mv -n 'mandatory_optionalinput/%(bn)s/%(id)s.%(omsext)s' 'mandatory_optionalinput/%(bn)s/%(id)s.%(gext)s'"%{"bn": os.path.basename(str(_))[:-4], "id": re.sub('[^\w\-_]', '_', _.element_identifier), "omsext":$mandatory_optionalinput_type, "gext": $oms2gxyext(str($mandatory_optionalinput_type))} for _ in $inp_mandatory_optionalinput if _])} -${' '.join(["&& mv -n 'test_section_sect_mandatory_mandatoryinput/%(bn)s/%(id)s.%(omsext)s' 'test_section_sect_mandatory_mandatoryinput/%(bn)s/%(id)s.%(gext)s'"%{"bn": os.path.basename(str(_))[:-4], "id": re.sub('[^\w\-_]', '_', _.element_identifier), "omsext":$test_section.sect_mandatory_mandatoryinput_type, "gext": $oms2gxyext(str($test_section.sect_mandatory_mandatoryinput_type))} for _ in $test_section.sect_inp_optional_mandatoryinput if _])} -${' '.join(["&& mv -n 'test_section_sect_mandatory_optionalinput/%(bn)s/%(id)s.%(omsext)s' 'test_section_sect_mandatory_optionalinput/%(bn)s/%(id)s.%(gext)s'"%{"bn": os.path.basename(str(_))[:-4], "id": re.sub('[^\w\-_]', '_', _.element_identifier), "omsext":$test_section.sect_mandatory_optionalinput_type, "gext": $oms2gxyext(str($test_section.sect_mandatory_optionalinput_type))} for _ in $test_section.sect_inp_mandatory_optionalinput if _])} +${' '.join(["&& mv -n 'mandatory_mandatoryinput/%(bn)s/%(id)s.%(gext)s' 'mandatory_mandatoryinput/%(bn)s/%(id)s'"%{"bn": os.path.basename(str(_))[:-4], "id": re.sub('[^\w\-_]', '_', _.element_identifier), "gext": $gxy2omsext("tsv")} for _ in $inp_mandatory_mandatoryinput if _])} +${' '.join(["&& mv -n 'test_section_sect_mandatory_mandatoryinput/%(bn)s/%(id)s.%(omsext)s' 'test_section_sect_mandatory_mandatoryinput/%(bn)s/%(id)s.%(gext)s'"%{"bn": os.path.basename(str(_))[:-4], "id": re.sub('[^\w\-_]', '_', _.element_identifier), "omsext":$test_section.sect_mandatory_mandatoryinput_type, "gext": $oms2gxyext(str($test_section.sect_mandatory_mandatoryinput_type))} for _ in $test_section.sect_inp_mandatory_mandatoryinput if _])} #if "ctd_out_FLAG" in $OPTIONAL_OUTPUTS && mv '@EXECUTABLE@.ctd' '$ctd_out' #end if]]> @@ -85,14 +57,7 @@ ${' '.join(["&& mv -n 'test_section_sect_mandatory_optionalinput/%(bn)s/%(id)s.% - - - - - - - - +
@@ -100,15 +65,7 @@ ${' '.join(["&& mv -n 'test_section_sect_mandatory_optionalinput/%(bn)s/%(id)s.% - - - - - - - - - +
@@ -119,36 +76,26 @@ ${' '.join(["&& mv -n 'test_section_sect_mandatory_optionalinput/%(bn)s/%(id)s.% - - - - - - OPTIONAL_OUTPUTS is not None and "ctd_out_FLAG" in OPTIONAL_OUTPUTS - + - - +
- - +
- - + + - + --> diff --git a/tests/test-data/ofile-mult.xml b/tests/test-data/ofile-mult.xml index 71f81abf..bd376e82 100644 --- a/tests/test-data/ofile-mult.xml +++ b/tests/test-data/ofile-mult.xml @@ -21,28 +21,12 @@ mkdir ${' '.join(["'mandatory_mandatoryinput/%s'"%(os.path.basename(str(_))[:-4] mkdir optional_mandatoryinput && mkdir ${' '.join(["'optional_mandatoryinput/%s'"%(os.path.basename(str(_))[:-4]) for _ in $inp_optional_mandatoryinput if _])} && #end if -mkdir mandatory_optionalinput && -mkdir ${' '.join(["'mandatory_optionalinput/%s'"%(os.path.basename(str(_))[:-4]) for _ in $inp_mandatory_optionalinput if _])} && -#if "optional_optionalinput_FLAG" in str($OPTIONAL_OUTPUTS).split(',') - mkdir optional_optionalinput && - mkdir ${' '.join(["'optional_optionalinput/%s'"%(os.path.basename(str(_))[:-4]) for _ in $inp_optional_optionalinput if _])} && -#end if mkdir inp_mandatory_mandatoryinput && mkdir ${' '.join(["'inp_mandatory_mandatoryinput/%s'"%(os.path.basename(str(_))[:-4]) for _ in $inp_mandatory_mandatoryinput if _])} && ${' '.join(["ln -s '%s' 'inp_mandatory_mandatoryinput/%s/%s.%s' && " % (_, os.path.basename(str(_))[:-4], re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $inp_mandatory_mandatoryinput if _])} mkdir inp_optional_mandatoryinput && mkdir ${' '.join(["'inp_optional_mandatoryinput/%s'"%(os.path.basename(str(_))[:-4]) for _ in $inp_optional_mandatoryinput if _])} && ${' '.join(["ln -s '%s' 'inp_optional_mandatoryinput/%s/%s.%s' && " % (_, os.path.basename(str(_))[:-4], re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $inp_optional_mandatoryinput if _])} -#if $inp_mandatory_optionalinput: - mkdir inp_mandatory_optionalinput && - mkdir ${' '.join(["'inp_mandatory_optionalinput/%s'"%(os.path.basename(str(_))[:-4]) for _ in $inp_mandatory_optionalinput if _])} && - ${' '.join(["ln -s '%s' 'inp_mandatory_optionalinput/%s/%s.%s' && " % (_, os.path.basename(str(_))[:-4], re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $inp_mandatory_optionalinput if _])} -#end if -#if $inp_optional_optionalinput: - mkdir inp_optional_optionalinput && - mkdir ${' '.join(["'inp_optional_optionalinput/%s'"%(os.path.basename(str(_))[:-4]) for _ in $inp_optional_optionalinput if _])} && - ${' '.join(["ln -s '%s' 'inp_optional_optionalinput/%s/%s.%s' && " % (_, os.path.basename(str(_))[:-4], re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $inp_optional_optionalinput if _])} -#end if ## Main program call @@ -56,24 +40,10 @@ ${' '.join(["'mandatory_mandatoryinput/%s/%s.%s'"%(os.path.basename(str(_))[:-4] -optional_mandatoryinput ${' '.join(["'optional_mandatoryinput/%s/%s.%s'"%(os.path.basename(str(_))[:-4], re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext("tsv")) for _ in $inp_optional_mandatoryinput if _])} #end if --mandatory_optionalinput -${' '.join(["'mandatory_optionalinput/%s/%s.%s'"%(os.path.basename(str(_))[:-4], re.sub('[^\w\-_]', '_', _.element_identifier), _.ext) for _ in $inp_mandatory_optionalinput if _])} -#if "optional_optionalinput_FLAG" in str($OPTIONAL_OUTPUTS).split(',') - -optional_optionalinput - ${' '.join(["'optional_optionalinput/%s/%s.%s'"%(os.path.basename(str(_))[:-4], re.sub('[^\w\-_]', '_', _.element_identifier), _.ext) for _ in $inp_optional_optionalinput if _])} -#end if -inp_mandatory_mandatoryinput ${' '.join(["'inp_mandatory_mandatoryinput/%s/%s.%s'"%(os.path.basename(str(_))[:-4], re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $inp_mandatory_mandatoryinput if _])} -inp_optional_mandatoryinput ${' '.join(["'inp_optional_mandatoryinput/%s/%s.%s'"%(os.path.basename(str(_))[:-4], re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $inp_optional_mandatoryinput if _])} -#if $inp_mandatory_optionalinput: - -inp_mandatory_optionalinput - ${' '.join(["'inp_mandatory_optionalinput/%s/%s.%s'"%(os.path.basename(str(_))[:-4], re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $inp_mandatory_optionalinput if _])} -#end if -#if $inp_optional_optionalinput: - -inp_optional_optionalinput - ${' '.join(["'inp_optional_optionalinput/%s/%s.%s'"%(os.path.basename(str(_))[:-4], re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $inp_optional_optionalinput if _])} -#end if #if len(str($OPTIONAL_OUTPUTS).split(',')) == 0 | tee '$stdout' #end if @@ -93,12 +63,9 @@ ${' '.join(["&& mv -n 'mandatory_mandatoryinput/%(bn)s/%(id)s.%(gext)s' 'mandato - - - @@ -110,26 +77,18 @@ ${' '.join(["&& mv -n 'mandatory_mandatoryinput/%(bn)s/%(id)s.%(gext)s' 'mandato OPTIONAL_OUTPUTS is not None and "optional_mandatoryinput_FLAG" in OPTIONAL_OUTPUTS - - - - - - OPTIONAL_OUTPUTS is not None and "optional_optionalinput_FLAG" in OPTIONAL_OUTPUTS - OPTIONAL_OUTPUTS is not None and "ctd_out_FLAG" in OPTIONAL_OUTPUTS - + - - + @@ -52,7 +52,7 @@ - + @@ -64,7 +64,7 @@ - + diff --git a/tests/test-data/ofile-typeparam.xml b/tests/test-data/ofile-typeparam.xml index 568ebe07..0fb61691 100644 --- a/tests/test-data/ofile-typeparam.xml +++ b/tests/test-data/ofile-typeparam.xml @@ -294,13 +294,13 @@ python3 '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' '$hard OPTIONAL_OUTPUTS is not None and "optional_noformat_FLAG" in OPTIONAL_OUTPUTS - + - + @@ -311,13 +311,13 @@ python3 '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' '$hard OPTIONAL_OUTPUTS is not None and "optional_noformat_adv_FLAG" in OPTIONAL_OUTPUTS - + - + @@ -328,13 +328,13 @@ python3 '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' '$hard OPTIONAL_OUTPUTS is not None and "sect_optional_noformat_FLAG" in OPTIONAL_OUTPUTS - + - + @@ -345,13 +345,13 @@ python3 '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' '$hard OPTIONAL_OUTPUTS is not None and "sect_optional_noformat_adv_FLAG" in OPTIONAL_OUTPUTS - + - + @@ -392,13 +392,13 @@ python3 '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' '$hard - + - + - + - + OPTIONAL_OUTPUTS is not None and "optional_oneformat_FLAG" in OPTIONAL_OUTPUTS - + - + @@ -293,13 +293,13 @@ python3 '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' '$hard OPTIONAL_OUTPUTS is not None and "optional_oneformat_adv_FLAG" in OPTIONAL_OUTPUTS - + - + @@ -314,13 +314,13 @@ python3 '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' '$hard OPTIONAL_OUTPUTS is not None and "sect_optional_oneformat_FLAG" in OPTIONAL_OUTPUTS - + - + @@ -335,13 +335,13 @@ python3 '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' '$hard OPTIONAL_OUTPUTS is not None and "sect_optional_oneformat_adv_FLAG" in OPTIONAL_OUTPUTS - + - + @@ -369,16 +369,16 @@ python3 '$__tool_directory__/fill_ctd.py' '@EXECUTABLE@.ctd' '$args_json' '$hard - + - + - + - + Date: Wed, 10 Feb 2021 11:20:19 +0100 Subject: [PATCH 118/121] import os --- galaxy/converter.py | 2 +- tests/test-data/bool.xml | 2 +- tests/test-data/empty.xml | 2 +- tests/test-data/float.xml | 2 +- tests/test-data/ifile.xml | 2 +- tests/test-data/integer.xml | 2 +- tests/test-data/label-help.xml | 2 +- tests/test-data/ofile-corresponding-input.xml | 2 +- tests/test-data/ofile-mult-typeparam.xml | 2 +- tests/test-data/ofile-mult.xml | 2 +- tests/test-data/ofile-typeparam.xml | 2 +- tests/test-data/ofile.xml | 2 +- tests/test-data/repeat_test.xml | 2 +- tests/test-data/select_test.xml | 2 +- tests/test-data/string.xml | 2 +- 15 files changed, 15 insertions(+), 15 deletions(-) diff --git a/galaxy/converter.py b/galaxy/converter.py index 072f7b2a..901aba20 100755 --- a/galaxy/converter.py +++ b/galaxy/converter.py @@ -658,7 +658,7 @@ def create_command(tool, model, **kwargs): final_cmd = OrderedDict([('preprocessing', []), ('command', []), ('postprocessing', [])]) advanced_cmd = {'preprocessing': [], 'command': [], 'postprocessing': []} - final_cmd['preprocessing'].extend(["@QUOTE_FOO@", "@EXT_FOO@", "#import re", "#import os.path", "", "## Preprocessing"]) + final_cmd['preprocessing'].extend(["@QUOTE_FOO@", "@EXT_FOO@", "#import re", "#import os", "", "## Preprocessing"]) # - call the executable with -write_ctd to write the ctd file (with defaults) # - use fill_ctd.py to overwrite the defaults in the ctd file with the diff --git a/tests/test-data/bool.xml b/tests/test-data/bool.xml index 055f8ef1..536b7dfc 100644 --- a/tests/test-data/bool.xml +++ b/tests/test-data/bool.xml @@ -12,7 +12,7 @@ Date: Wed, 10 Feb 2021 12:35:43 +0100 Subject: [PATCH 119/121] update docs --- README.md | 88 +++++++++++++++++++++++++------------ convert.py | 110 ++--------------------------------------------- galaxy/README.md | 31 +++++-------- 3 files changed, 75 insertions(+), 154 deletions(-) diff --git a/README.md b/README.md index 0639b019..8b3c5db5 100644 --- a/README.md +++ b/README.md @@ -123,17 +123,71 @@ Several inputs are given. The output is the already existent folder, `/data/wrap Please note that the output file name is **not** taken from the name of the input file, rather from the name of the tool, that is, from the `name` attribute in the `` element in its corresponding CTD. By convention, the name of the CTD file and the name of the tool match. -### Blacklisting Parameters -* Purpose: Some parameters present in the CTD are not to be exposed on the output files. Think of parameters such as `--help`, `--debug` that might won't make much sense to be exposed to final users in a workflow management system. -* Short/long version: `-b` / `--blacklist-parameters` +### Exclusion, hardcoding, and modification of Parameters +* Purpose: Some parameters present in the CTD are not to be exposed on the output files (e.g. parameters such as `--help`, `--debug` that might won't make much sense to be exposed to users in a workflow management system), other parameters should be hardcoded (i.e. parameters that should not be exposed to the user but still set to a fixed value on the generated command line), and for other parameters it might be necessary to modify attributes of the input CTD or the generated output. +* Short/long version: `-p` / `--hardcoded-parameters` * Required: no. -* Taken values: A list of parameters to be blacklisted. +* Taken values: A json file defining: exclusion, hardcoded, and modifications of parameters Example: - $ pythonconvert.py [FORMAT] ... -b h help quiet - -In this case, `CTDConverter` will not process any of the parameters named `h`, `help`, or `quiet`, that is, they will not appear in the generated output files. + $ pythonconvert.py [FORMAT] ... -p JSON_FILE + +The json defines a mapping from parameter names to a list of modifications: + +``` +{ + "parameter1": [MODIFICATION1, ...], + "parameter2": [MODIFICATION1, ...] + ... +} +``` + +where each modification is a mapping as defined below. + + +#### Hardcoding parameters + +If a parameter should always be set on the command line using a fixed value, +i.e. the user can to choose the value, this can be done as follows: + +`"parameter": [{"value":"HARDCODED_VALUE"}]` + +#### Excluding parameters + +In order to exclude a parameter, it will not appear in the generated tool or +the generated command line, the same syntax as for hardcoding is used but a +special reserved value is used: + +`"parameter": [{"value": "@"}]` + +#### Modifying parameters + +It's possible to modify attributes of the input CTD definition of a parameter +as well as attributes of the generated Galaxy XML tags. + +``` + "test": [{ + "CTD:type": "text", + "XML:type": "hidden" + }], +``` + +### Restricting modifications to a subset of the tools + +Its possible to specify modifications to a parameter for only a subset of the tools +by specifying a list of tools as follows: + +``` + "output_files": [{ + "CTD:required": true, + "tools": ["OpenSwathDIAPreScoring"] + }, { + "CTD:restrictions": "txt,tsv,pep.xml,pepXML,html", + "tools": ["SpectraSTSearchAdapter"] + + }] +``` ### Schema Validation * Purpose: Provide validation of input CTDs against a schema file (i.e, a XSD file). @@ -147,26 +201,6 @@ If a schema is provided, all input CTDs will be validated against it. **NOTE:** Please make sure to read the [section on issues with schema validation](#issues-with-libxml2-and-schema-validation) if you require validation of CTDs against a schema. -### Hardcoding Parameters -* Purpose: Fixing the value of a parameter and hide it from the end user. -* Short/long version: `-p` / `--hardcoded-parameters` -* Required: no. -* Taken values: The path of a file containing the mapping between parameter names and hardcoded values to use. - -It is sometimes required that parameters are hidden from the end user in workflow systems and that they take a predetermined, fixed value. Allowing end users to control parameters similar to `--verbosity`, `--threads`, etc., might create more problems than solving them. For this purpose, the parameter `-p`/`--hardcoded-parameters` takes the path of a file that contains up to three columns separated by whitespace that map parameter names to the hardcoded value. The first column contains the name of the parameter and the second one the hardcoded value. Only the first two columns are mandatory. - -If the parameter is to be hardcoded only for certain tools, a third column containing a comma separated list of tool names for which the hardcoding will apply can be added. - -Lines starting with `#` will be ignored. The following is an example of a valid file: - - # Parameter name # Value # Tool(s) - threads 8 - mode quiet - xtandem_executable xtandem XTandemAdapter - verbosity high Foo, Bar - -The parameters `threads` and `mode` will be set to `8` and `quiet`, respectively, for all parsed CTDs. However, the `xtandem_executable` parameter will be set to `xtandem` only for the `XTandemAdapter` tool. Similarly, the parameter `verbosity` will be set to `high` for the `Foo` and `Bar` tools only. - ### Providing a default executable Path * Purpose: Help workflow engines locate tools by providing a path. * Short/long version: `-x` / `--default-executable-path` diff --git a/convert.py b/convert.py index ec23474a..7ee7c2a9 100644 --- a/convert.py +++ b/convert.py @@ -38,109 +38,7 @@ $ python converter.py [FORMAT] -i [INPUT_FILES] -o [OUTPUT_DIRECTORY] -III - Hardcoding parameters - - It is possible to hardcode parameters. This makes sense if you want to set a tool in 'quiet' mode or if your tools - support multi-threading and accept the number of threads via a parameter, without giving end users the chance to - change the values for these parameters. - - In order to generate hardcoded parameters, you need to provide a simple file. Each line of this file contains - two or three columns separated by tabs. Any line starting with a '#' will be ignored. The first column contains - the name of the parameter, the second column contains the value that will always be set for this parameter. Only the - first two columns are mandatory. - - If the parameter is to be hardcoded only for a set of tools, then a third column can be added. This column contains - a comma-separated list of tool names for which the parameter will be hardcoded. If a third column is not present, - then all processed tools containing the given parameter will get a hardcoded value for it. - - The following is an example of a valid file: - - ##################################### HARDCODED PARAMETERS example ##################################### - # Every line starting with a # will be handled as a comment and will not be parsed. - # The first column is the name of the parameter and the second column is the value that will be used. - - # Parameter name # Value # Tool(s) - threads 8 - mode quiet - xtandem_executable xtandem XTandemAdapter - verbosity high Foo, Bar - - ######################################################################################################### - - Using the above file will produce a command-line similar to: - - [TOOL] ... -threads 8 -mode quiet ... - - for all tools. For XTandemAdapter, however, the command-line will look like: - - XtandemAdapter ... -threads 8 -mode quiet -xtandem_executable xtandem ... - - And for tools Foo and Bar, the command-line will be similar to: - - Foo -threads 8 -mode quiet -verbosity high ... - - - IV - Engine-specific parameters - - i - Galaxy - - a. Providing file formats, mimetypes - - Galaxy supports the concept of file format in order to connect compatible ports, that is, input ports of a - certain data format will be able to receive data from a port from the same format. This converter allows you - to provide a personalized file in which you can relate the CTD data formats with supported Galaxy data formats. - The layout of this file consists of lines, each of either one or four columns separated by any amount of - whitespace. The content of each column is as follows: - - * 1st column: file extension - * 2nd column: data type, as listed in Galaxy - * 3rd column: full-named Galaxy data type, as it will appear on datatypes_conf.xml - * 4th column: mimetype (optional) - - The following is an example of a valid "file formats" file: - - ########################################## FILE FORMATS example ########################################## - # Every line starting with a # will be handled as a comment and will not be parsed. - # The first column is the file format as given in the CTD and second column is the Galaxy data format. The - # second, third, fourth and fifth columns can be left empty if the data type has already been registered - # in Galaxy, otherwise, all but the mimetype must be provided. - - # CTD type # Galaxy type # Long Galaxy data type # Mimetype - csv tabular galaxy.datatypes.data:Text - fasta - ini txt galaxy.datatypes.data:Text - txt - idxml txt galaxy.datatypes.xml:GenericXml application/xml - options txt galaxy.datatypes.data:Text - grid grid galaxy.datatypes.data:Grid - ########################################################################################################## - - Note that each line consists precisely of either one, three or four columns. In the case of data types already - registered in Galaxy (such as fasta and txt in the above example), only the first column is needed. In the - case of data types that haven't been yet registered in Galaxy, the first three columns are needed - (mimetype is optional). - - For information about Galaxy data types and subclasses, see the following page: - https://wiki.galaxyproject.org/Admin/Datatypes/Adding%20Datatypes - - - b. Finer control over which tools will be converted - - Sometimes only a subset of CTDs needs to be converted. It is possible to either explicitly specify which tools - will be converted or which tools will not be converted. - - The value of the -s/--skip-tools parameter is a file in which each line will be interpreted as the name of a - tool that will not be converted. Conversely, the value of the -r/--required-tools is a file in which each line - will be interpreted as a tool that is required. Only one of these parameters can be specified at a given time. - - The format of both files is exactly the same. As stated before, each line will be interpreted as the name of a - tool. Any line starting with a '#' will be ignored. - - - ii - CWL - - There are, for now, no CWL-specific parameters or options. - +For more detailed help see README.md in the root folder as well as `galaxy/README.md` or `cwl/README.md`. ''' program_license = '''%(short_description)s @@ -188,9 +86,9 @@ def main(argv=None): from cwl import converter elif target == 'galaxy': from galaxy import converter - elif target == '-h' or target == '--help' or target == '--h' or target == 'help': - print(program_license) - return 0 +# elif target == '-h' or target == '--help' or target == '--h' or target == 'help': +# print(program_license) +# return 0 else: utils.logger.error("Unrecognized target engine. Supported targets are 'cwl' and 'galaxy'.") return 1 diff --git a/galaxy/README.md b/galaxy/README.md index fa728103..56ab0998 100644 --- a/galaxy/README.md +++ b/galaxy/README.md @@ -77,6 +77,16 @@ There are some macros that are required, namely `stdio`, `requirements` and `adv Please note that the used macros files **must** be copied to your Galaxy installation on the same location in which you place the generated *ToolConfig* files, otherwise Galaxy will not be able to parse the generated *ToolConfig* files! +## Including additional Test Macros files +* Purpose: Include macros containing tests +* `--test-macros` and `--test-macros-prefix` +* Required: no. +* Taken values: List of paths and corresponding prefixes + +This allows to specify macro file(s) containing tests. The macros should be +named `` where `PREFIX` is the value specified +with `--test-macros-prefix`. + ## Generating a `datatypes_conf.xml` File * Purpose: Specify the destination of a generated `datatypes_conf.xml` file. * Short/long version: `-d` / `--datatypes-destination` @@ -113,27 +123,6 @@ Note that each line consists of either one, three or four columns. In the case o For information about Galaxy data types and subclasses, consult the following page: https://wiki.galaxyproject.org/Admin/Datatypes/Adding%20Datatypes -## Remarks about some of the *OpenMS* Tools -* Most of the tools can be generated automatically. However, some of the tools need some extra work (for now). -* The following adapters need to be changed, such that you provide the path to the executable: - * FidoAdapter (add `-exe fido` in the command tag, delete the `$param_exe` in the command tag, delete the parameter from the input list). - * MSGFPlusAdapter (add `-executable msgfplus.jar` in the command tag, delete the `$param_executable` in the command tag, delete the parameter from the input list). - * MyriMatchAdapter (add `-myrimatch_executable myrimatch` in the command tag, delete the `$param_myrimatch_executable` in the command tag, delete the parameter from the input list). - * OMSSAAdapter (add `-omssa_executable omssa` in the command tag, delete the `$param_omssa_executable` in the command tag, delete the parameter from the input list). - * PepNovoAdapter (add `-pepnovo_executable pepnovo` in the command tag, delete the `$param_pepnovo_executable` in the command tag, delete the parameter from the input list). - * XTandemAdapter (add `-xtandem_executable xtandem` in the command tag, delete the $param_xtandem_executable in the command tag, delete the parameter from the input list). - * To avoid the deletion in the inputs you can also add these parameters to the blacklist - - $ python convert.py galaxy -b exe executable myrimatch_excutable omssa_executable pepnovo_executable xtandem_executable - -* The following tools have multiple outputs (number of inputs = number of outputs) which is not yet supported in Galaxy-stable: - * SeedListGenerator - * SpecLibSearcher - * MapAlignerIdentification - * MapAlignerPoseClustering - * MapAlignerSpectrum - * MapAlignerRTTransformer - [CTDopts]: https://github.com/genericworkflownodes/CTDopts [macros.xml]: https://github.com/WorkflowConversion/CTDConverter/blob/master/galaxy/macros.xml [CTDSchema]: https://github.com/genericworkflownodes/CTDSchema \ No newline at end of file From 966007fbe0bbd5fc8ff725ec1308cce69751bca0 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Wed, 10 Feb 2021 13:14:30 +0100 Subject: [PATCH 120/121] flake8 fixes --- common/utils.py | 2 +- convert.py | 6 +- cwl/converter.py | 4 +- galaxy/converter.py | 175 ++++++++++-------------------------- tests/test-data/fill_ctd.py | 53 ++++++++--- tests/test-data/mock.py | 1 - tests/test_galaxy_cli.py | 3 +- 7 files changed, 97 insertions(+), 147 deletions(-) diff --git a/common/utils.py b/common/utils.py index a870e7f5..481d30b2 100644 --- a/common/utils.py +++ b/common/utils.py @@ -441,6 +441,6 @@ def indent(s, indentation=" "): def getFromDict(dataDict, mapList): return reduce(operator.getitem, mapList, dataDict) + def setInDict(dataDict, mapList, value): getFromDict(dataDict, mapList[:-1])[mapList[-1]] = value - diff --git a/convert.py b/convert.py index 7ee7c2a9..84ed1204 100644 --- a/convert.py +++ b/convert.py @@ -72,9 +72,9 @@ def main(argv=None): # converter will register its own parameters after we've registered the basic ones... we have to do it old school if len(argv) < 2: utils.logger.error("Not enough arguments provided") - print("\nUsage: $ python convert.py [TARGET] [ARGUMENTS]\n\n" + - "Where:\n" + - " target: one of 'cwl' or 'galaxy'\n\n" + + print("\nUsage: $ python convert.py [TARGET] [ARGUMENTS]\n\n" + "Where:\n" + " target: one of 'cwl' or 'galaxy'\n\n" "Run again using the -h/--help option to print more detailed help.\n") return 1 diff --git a/cwl/converter.py b/cwl/converter.py index 08fad015..587ccebd 100755 --- a/cwl/converter.py +++ b/cwl/converter.py @@ -11,7 +11,7 @@ import ruamel.yaml as yaml -from CTDopts.CTDopts import _InFile, _OutFile, ParameterGroup, _Choices, _NumericRange, _FileFormat, ModelError, _Null +from CTDopts.CTDopts import _InFile, _OutFile, _Choices, _Null from common import utils, logger # all cwl-related properties are defined here @@ -72,7 +72,7 @@ def convert_models(args, parsed_ctds): logger.info("Writing to %s" % utils.get_filename(output_file), 1) - stream = file(output_file, 'w') + stream = open(output_file, 'w') stream.write(CWL_SHEBANG + '\n\n') stream.write("# This CWL file was automatically generated using CTDConverter.\n") stream.write("# Visit https://github.com/WorkflowConversion/CTDConverter for more information.\n\n") diff --git a/galaxy/converter.py b/galaxy/converter.py index 901aba20..baed60cf 100755 --- a/galaxy/converter.py +++ b/galaxy/converter.py @@ -4,19 +4,17 @@ import os import os.path import re -import string import sys from collections import OrderedDict import copy -from io import StringIO from lxml import etree from lxml.etree import CDATA, SubElement, Element, ElementTree, ParseError, parse, strip_elements from common import utils, logger from common.exceptions import ApplicationException, InvalidModelException -from CTDopts.CTDopts import _InFile, _OutFile, _OutPrefix, ParameterGroup, _Choices, _NumericRange, _FileFormat, ModelError, _Null, CTDTYPE_TO_TYPE, ParameterGroup +from CTDopts.CTDopts import _InFile, _OutFile, _OutPrefix, ParameterGroup, _Choices, _NumericRange, _FileFormat, ModelError, _Null # mapping to CTD types to Galaxy types @@ -41,7 +39,7 @@ def __init__(self, code_range="", level="", description=None): class DataType: - def __init__(self, extension, galaxy_extension, composite = None): + def __init__(self, extension, galaxy_extension, composite=None): self.extension = extension self.galaxy_extension = galaxy_extension self.composite = composite @@ -117,7 +115,7 @@ def modify_param_for_galaxy(param): # need to be taken care by hardcoded values and the other cases # are chosen automatically if not specified on the command line) if param.required and not (param.default is None or type(param.default) is _Null): - logger.warning("Data parameter %s with default (%s)" %(param.name, param.default), 1) + logger.warning("Data parameter %s with default (%s)" % (param.name, param.default), 1) param.required = False param.default = _Null() return param @@ -170,16 +168,6 @@ def convert_models(args, parsed_ctds): tool_version=args.tool_version, tool_profile=args.tool_profile, bump=bump) - # generation of galaxy stubs is ready... now, let's see if we need to generate a tool_conf.xml - # TODO remove tool conf .. deprecated -# if args.tool_conf_destination is not None: -# generate_tool_conf(parsed_ctds, args.tool_conf_destination, -# args.galaxy_tool_path, args.default_category) - - # TODO remove datatypes_conf generation - # generate datatypes_conf.xml -# if args.data_types_destination is not None: -# generate_data_type_conf(supported_file_formats, args.data_types_destination) def parse_bump_file(bump_file): @@ -189,7 +177,6 @@ def parse_bump_file(bump_file): return json.load(fp) - def parse_tools_list_file(tools_list_file): """ """ @@ -224,12 +211,10 @@ def parse_macros_files(macros_file_names, tool_version, supported_file_types, re logger.info("Macro %s found" % name, 1) macros_to_expand.append(name) except ParseError as e: - raise ApplicationException("The macros file " + macros_file_name + " could not be parsed. Cause: " + - str(e)) + raise ApplicationException("The macros file " + macros_file_name + " could not be parsed. Cause: " + str(e)) except IOError as e: - raise ApplicationException("The macros file " + macros_file_name + " could not be opened. Cause: " + - str(e)) + raise ApplicationException("The macros file " + macros_file_name + " could not be opened. Cause: " + str(e)) else: macros_file.close() @@ -315,11 +300,9 @@ def check_test_macros(test_macros_files, test_macros_prefix, parsed_ctds): macro_ids.add(name) except ParseError as e: - raise ApplicationException("The macros file " + mf + " could not be parsed. Cause: " + - str(e)) + raise ApplicationException("The macros file " + mf + " could not be parsed. Cause: " + str(e)) except IOError as e: - raise ApplicationException("The macros file " + mf + " could not be opened. Cause: " + - str(e)) + raise ApplicationException("The macros file " + mf + " could not be opened. Cause: " + str(e)) for t in tool_ids - macro_ids: logger.error("missing %s" % t) add_child_node(root, "xml", OrderedDict([("name", mp + t)])) @@ -356,10 +339,7 @@ def parse_file_formats(formats_file): parsed_formats[1], composite)) else: - logger.warning( - "Invalid line at line number %d of the given formats file. Line will be ignored:\n%s" % - (line_number, line), 0) - + logger.warning("Invalid line at line number %d of the given formats file. Line will be ignored:\n%s" % (line_number, line), 0) return supported_formats @@ -458,8 +438,8 @@ def _convert_internal(parsed_ctds, **kwargs): if a == "type": try: t = GALAXY_TYPE_TO_TYPE[hardcoded_attributes[a]] - except: - logger.error("Could not set hardcoded attribute %s=%s for %s"%(a, hardcoded_attributes[a], param.name)) + except KeyError: + logger.error("Could not set hardcoded attribute %s=%s for %s" % (a, hardcoded_attributes[a], param.name)) sys.exit(1) setattr(param, a, t) elif type(getattr(param, a)) is _FileFormat or (param.type in [_InFile, _OutFile, _OutPrefix] and a == "restrictions"): @@ -519,61 +499,6 @@ def write_header(tool, model): tool.addprevious(etree.Comment('Proposed Tool Section: [%s]' % model.opt_attribs.get("category", ""))) -def generate_tool_conf(parsed_ctds, tool_conf_destination, galaxy_tool_path, default_category): - # for each category, we keep a list of models corresponding to it - categories_to_tools = dict() - for parsed_ctd in parsed_ctds: - category = strip(parsed_ctd.ctd_model.opt_attribs.get("category", "")) - if not category.strip(): - category = default_category - if category not in categories_to_tools: - categories_to_tools[category] = [] - categories_to_tools[category].append(utils.get_filename(parsed_ctd.suggested_output_file)) - - # at this point, we should have a map for all categories->tools - toolbox_node = Element("toolbox") - - if galaxy_tool_path is not None and not galaxy_tool_path.strip().endswith("/"): - galaxy_tool_path = galaxy_tool_path.strip() + "/" - if galaxy_tool_path is None: - galaxy_tool_path = "" - - for category, file_names in categories_to_tools.iteritems(): - section_node = add_child_node(toolbox_node, "section") - section_node.attrib["id"] = "section-id-" + "".join(category.split()) - section_node.attrib["name"] = category - - for filename in file_names: - tool_node = add_child_node(section_node, "tool") - tool_node.attrib["file"] = galaxy_tool_path + filename - - toolconf_tree = ElementTree(toolbox_node) - toolconf_tree.write(open(tool_conf_destination, 'w'), encoding="UTF-8", xml_declaration=True, pretty_print=True) - logger.info("Generated Galaxy tool_conf.xml in %s" % tool_conf_destination, 0) - - -def generate_data_type_conf(supported_file_formats, data_types_destination): - data_types_node = Element("datatypes") - registration_node = add_child_node(data_types_node, "registration") - registration_node.attrib["converters_path"] = "lib/galaxy/datatypes/converters" - registration_node.attrib["display_path"] = "display_applications" - - for format_name in supported_file_formats: - data_type = supported_file_formats[format_name] - # add only if it's a data type that does not exist in Galaxy - if data_type.galaxy_type is not None: - data_type_node = add_child_node(registration_node, "datatype") - # we know galaxy_extension is not None - data_type_node.attrib["extension"] = data_type.galaxy_extension - data_type_node.attrib["type"] = data_type.galaxy_type - if data_type.mimetype is not None: - data_type_node.attrib["mimetype"] = data_type.mimetype - - data_types_tree = ElementTree(data_types_node) - data_types_tree.write(open(data_types_destination, 'w'), encoding="UTF-8", xml_declaration=True, pretty_print=True) - logger.info("Generated Galaxy datatypes_conf.xml in %s" % data_types_destination, 0) - - def create_tool(model, profile, bump): """ initialize the tool @@ -593,7 +518,7 @@ def create_tool(model, profile, bump): attrib = OrderedDict([("id", tool_id), ("name", model.name), - ("version", "@TOOL_VERSION@+galaxy"+gxy_version)]) + ("version", "@TOOL_VERSION@+galaxy" + gxy_version)]) if profile is not None: attrib["profile"] = profile return Element("tool", attrib) @@ -626,8 +551,8 @@ def create_configfiles(tool, model, **kwargs): """ configfiles_node = add_child_node(tool, "configfiles") - inputs_node = add_child_node(configfiles_node, "inputs", - OrderedDict([("name", "args_json"), ("data_style", "paths")])) + add_child_node(configfiles_node, "inputs", + OrderedDict([("name", "args_json"), ("data_style", "paths")])) parameter_hardcoder = kwargs.get("parameter_hardcoder") hc_dict = dict() @@ -638,15 +563,16 @@ def create_configfiles(tool, model, **kwargs): path = utils.extract_param_path(param) for i, v in enumerate(path[:-1]): try: - utils.getFromDict(hc_dict, path[:i+1]) + utils.getFromDict(hc_dict, path[:i + 1]) except KeyError: - utils.setInDict(hc_dict, path[:i+1], {}) + utils.setInDict(hc_dict, path[:i + 1], {}) utils.setInDict(hc_dict, path, hardcoded_value) hc_node = add_child_node(configfiles_node, "configfile", - OrderedDict([("name", "hardcoded_json")])) - hc_node.text = CDATA(json.dumps(hc_dict).replace('$', '\$')) + OrderedDict([("name", "hardcoded_json")])) + hc_node.text = CDATA(json.dumps(hc_dict).replace('$', r'\$')) # print(json.dumps(hc_dict)) + def create_command(tool, model, **kwargs): """ @param tool the Galaxy tool @@ -687,7 +613,6 @@ def create_command(tool, model, **kwargs): param = modify_param_for_galaxy(param) param_cmd = {'preprocessing': [], 'command': [], 'postprocessing': []} - param_name = utils.extract_param_name(param) command_line_prefix = utils.extract_command_line_prefix(param, model) # TODO use utils.extract_param_name(param).replace(":", "_")? Then hardcoding ctd variables (with :) and tool variables (with _) can be distinguished @@ -745,7 +670,7 @@ def create_command(tool, model, **kwargs): if type_param is not None: type_param_name = get_galaxy_parameter_path(type_param) elif len(formats) > 1 and (corresponding_input is None or not - fmt_from_corresponding): # and not param.is_list: + fmt_from_corresponding): # and not param.is_list: type_param_name = get_galaxy_parameter_path(param, suffix="type") else: type_param_name = None @@ -859,7 +784,7 @@ def create_command(tool, model, **kwargs): final_cmd['command'].extend(stdout) ctd_out = ["#if \"ctd_out_FLAG\" in $OPTIONAL_OUTPUTS"] + utils.indent(["&& mv '@EXECUTABLE@.ctd' '$ctd_out'"]) + ["#end if"] - final_cmd['postprocessing'].extend( ctd_out ) + final_cmd['postprocessing'].extend(ctd_out) command_node = add_child_node(tool, "command") command_node.attrib["detect_errors"] = "exit_code" @@ -955,6 +880,7 @@ def get_out_type_param(out_param, model, parameter_hardcoder): def is_in_type_param(param, model): return is_type_param(param, model, [_InFile]) + def is_out_type_param(param, model): """ check if the parameter is output_type parameter @@ -963,6 +889,7 @@ def is_out_type_param(param, model): """ return is_type_param(param, model, [_OutFile, _OutPrefix]) + def is_type_param(param, model, tpe): """ check if the parameter is _type parameter of an in/output @@ -978,6 +905,7 @@ def is_type_param(param, model, tpe): return True return False + def get_corresponding_input(out_param, model): """ get the input parameter corresponding to the given output @@ -1073,7 +1001,6 @@ def create_inputs(tool, model, **kwargs): parent_node = inputs_node # sometimes special inputs are needed for outfiles: - add_formats = False if param.type is _OutFile or param.type is _OutPrefix: # if there are multiple possible output formats, but no parameter to choose the type or a # corresponding input then add a selection parameter @@ -1081,7 +1008,7 @@ def create_inputs(tool, model, **kwargs): type_param = get_out_type_param(param, model, parameter_hardcoder) corresponding_input, fmt_from_corresponding = get_corresponding_input(param, model) if len(formats) > 1 and type_param is None and (corresponding_input is None or not - fmt_from_corresponding):# and not param.is_list: + fmt_from_corresponding): # and not param.is_list: fmt_select = add_child_node(parent_node, "param", OrderedDict([("name", param.name + "_type"), ("type", "select"), ("optional", "false"), ("label", "File type of output %s (%s)" % (param.name, param.description))])) g2o, o2g = get_fileformat_maps(kwargs["supported_file_formats"]) # for f in formats: @@ -1102,8 +1029,7 @@ def create_inputs(tool, model, **kwargs): for a in hardcoded_attributes: param_node.attrib[a] = str(hardcoded_attributes[a]) - - section_parents = [ utils.extract_param_name(section_params[sn].parent) for sn in section_nodes ] + section_parents = [utils.extract_param_name(section_params[sn].parent) for sn in section_nodes] for sn in section_nodes: if len(section_nodes[sn]) == 0 and sn not in section_parents: continue @@ -1116,7 +1042,7 @@ def create_inputs(tool, model, **kwargs): # Add select for optional outputs out, optout = all_outputs(model, parameter_hardcoder) - attrib = OrderedDict([("name","OPTIONAL_OUTPUTS"), + attrib = OrderedDict([("name", "OPTIONAL_OUTPUTS"), ("type", "select"), ("optional", "true"), ("multiple", "true"), @@ -1129,11 +1055,11 @@ def create_inputs(tool, model, **kwargs): for o in optout: title, help_text = generate_label_and_help(o.description) option_node = add_child_node(param_node, "option", - OrderedDict([("value", o.name+"_FLAG")]), - text = "%s (%s)" % (o.name, title)) + OrderedDict([("value", o.name + "_FLAG")]), + text="%s (%s)" % (o.name, title)) option_node = add_child_node(param_node, "option", OrderedDict([("value", "ctd_out_FLAG")]), - text = "Output used ctd (ini) configuration file") + text="Output used ctd (ini) configuration file") return inputs_node @@ -1190,6 +1116,7 @@ def get_galaxy_formats(param, model, o2g, default=None): "name": param.name}) return sorted(gxy_formats) + def create_param_attribute_list(param_node, param, model, supported_file_formats): """ get the attributes of input parameters @@ -1276,9 +1203,9 @@ def create_param_attribute_list(param_node, param, model, supported_file_formats option_node.attrib["selected"] = "true" else: for choice in param.restrictions.choices: - option_node = add_child_node(param_node, "option", + option_node = add_child_node(param_node, "option", OrderedDict([("value", str(choice))]), - text = str(choice)) + text=str(choice)) if is_default(choice, param): option_node.attrib["selected"] = "true" @@ -1513,17 +1440,16 @@ def create_boolean_parameter(param_node, param): param_node.attrib["falsevalue"] = "false" param_node.attrib["checked"] = str(param.default).lower() - if param_node.attrib.has_key("optional"): + if "optional" in param_node.attrib: del param_node.attrib["optional"] def all_outputs(model, parameter_hardcoder): """ - return lists of reqired and optional output parameters + return lists of reqired and optional output parameters """ out = [] optout = [] - nout = 0 for param in utils.extract_and_flatten_parameters(model): hardcoded_value = parameter_hardcoder.get_hardcoded_value(utils.extract_param_name(param), model.name) if parameter_hardcoder.get_blacklist(utils.extract_param_name(param), model.name) or hardcoded_value: @@ -1579,12 +1505,11 @@ def create_outputs(parent, model, **kwargs): OrderedDict([("name", "stdout"), ("format", "txt"), ("label", "${tool.name} on ${on_string}: stdout"), ("format", "txt")])) - filter_node = add_child_node(stdout, "filter", - text="OPTIONAL_OUTPUTS is None") + add_child_node(stdout, "filter", text="OPTIONAL_OUTPUTS is None") # manually add output for the ctd file - ctd_out = add_child_node(outputs_node, "data", OrderedDict([("name","ctd_out"), ("format", "xml"), ("label", "${tool.name} on ${on_string}: ctd")])) - ctd_filter = add_child_node(ctd_out, "filter", text='OPTIONAL_OUTPUTS is not None and "ctd_out_FLAG" in OPTIONAL_OUTPUTS') + ctd_out = add_child_node(outputs_node, "data", OrderedDict([("name", "ctd_out"), ("format", "xml"), ("label", "${tool.name} on ${on_string}: ctd")])) + add_child_node(ctd_out, "filter", text='OPTIONAL_OUTPUTS is not None and "ctd_out_FLAG" in OPTIONAL_OUTPUTS') return outputs_node @@ -1594,7 +1519,7 @@ def create_output_node(parent, param, model, supported_file_formats, parameter_h # add a data node / collection + discover_datasets # in the former case we just set the discover_node equal to the data node # then we can just use this to set the common format attribute - if not param.is_list and not param.type is _OutPrefix: + if not param.is_list and param.type is not _OutPrefix: data_node = add_child_node(parent, "data") discover_node = data_node else: @@ -1612,9 +1537,9 @@ def create_output_node(parent, param, model, supported_file_formats, parameter_h corresponding_input, fmt_from_corresponding = get_corresponding_input(param, model) if type_param is not None: type_param_name = get_galaxy_parameter_path(type_param) - type_param_choices = get_formats(param, model, o2g)# [_ for _ in type_param.restrictions.choices] + type_param_choices = get_formats(param, model, o2g) # [_ for _ in type_param.restrictions.choices] elif len(formats) > 1 and (corresponding_input is None or not - fmt_from_corresponding): # and not param.is_list: + fmt_from_corresponding): # and not param.is_list: type_param_name = get_galaxy_parameter_path(param, suffix="type") type_param_choices = get_formats(param, model, o2g) else: @@ -1628,7 +1553,7 @@ def create_output_node(parent, param, model, supported_file_formats, parameter_h if param.is_list: discover_node.attrib["pattern"] = "__name__" elif param.type is _OutPrefix: - discover_node.attrib["pattern"] = "_?(?P.*)\.[^.]*" + discover_node.attrib["pattern"] = r"_?(?P.*)\.[^.]*" # if there is another parameter where the user selects the format # then this format was added as file extension on the CLI, now we can discover this @@ -1809,7 +1734,7 @@ def create_test_only(model, **kwargs): test = Element("test") advanced = add_child_node(test, "conditional", OrderedDict([("name", "adv_opts_cond")])) - adv_sel = add_child_node(advanced, "param", OrderedDict([("name", "adv_opts_selector"), ("value", "advanced")])) + add_child_node(advanced, "param", OrderedDict([("name", "adv_opts_selector"), ("value", "advanced")])) optout = ["ctd_out_FLAG"] outcnt = 1 @@ -1871,16 +1796,14 @@ def create_test_only(model, **kwargs): ext = formats[0] if len(formats) > 1 and (corresponding_input is None or not - fmt_from_corresponding): # and not param.is_list: + fmt_from_corresponding): # and not param.is_list: if type_param is None: try: print("%s -> %s" % (ext, g2o[ext])) attrib = OrderedDict([("name", param.name + "_type"), ("value", g2o[ext])]) - fmt_select = add_child_node(parent, "param", attrib) - except: - raise Exception("parent %s name %s ext %s" % - (parent, param.name, - ext)) + add_child_node(parent, "param", attrib) + except KeyError: + raise Exception("parent %s name %s ext %s" % (parent, param.name, ext)) if type_param is not None and type(type_param.default) is _Null: if ext is not None: type_param.default = ext @@ -1948,15 +1871,15 @@ def create_test_only(model, **kwargs): ext = os.path.splitext(value)[1][1:] if ext in unsniffable and ext in o2g: nd.attrib["ftype"] = o2g[ext] - - add_child_node(test, "param", OrderedDict([("name", "OPTIONAL_OUTPUTS"), + + add_child_node(test, "param", OrderedDict([("name", "OPTIONAL_OUTPUTS"), ("value", ",".join(optout))])) ctd_out = add_child_node(test, "output", OrderedDict([("name", "ctd_out"), ("ftype", "xml")])) ctd_assert = add_child_node(ctd_out, "assert_contents") add_child_node(ctd_assert, "is_valid_xml") if outcnt == 0: - outcnt += 1 + outcnt += 1 nd = add_child_node(test, "output", OrderedDict([("name", "stdout"), ("value", "stdout.txt"), ("compare", "sim_size")])) diff --git a/tests/test-data/fill_ctd.py b/tests/test-data/fill_ctd.py index dd90e7de..03e749b3 100644 --- a/tests/test-data/fill_ctd.py +++ b/tests/test-data/fill_ctd.py @@ -1,18 +1,18 @@ import collections -from functools import reduce # forward compatibility for Python 3 import json import operator import os import re import subprocess import sys +from functools import reduce # forward compatibility for Python 3 from CTDopts.CTDopts import ( - CTDModel, _Choices, - _Null, _InFile, - _NumericRange + _Null, + _NumericRange, + CTDModel ) @@ -30,17 +30,19 @@ def mergeDicts(d, e): no values of d are overwritten """ for k, v in e.items(): - if (k in d and isinstance(d[k], dict) and isinstance(e[k], collections.Mapping)): + if (k in d and isinstance(d[k], dict) and isinstance(e[k], collections.abc.Mapping)): mergeDicts(d[k], e[k]) - elif k not in d and not isinstance(e[k], collections.Mapping): + elif k not in d and not isinstance(e[k], collections.abc.Mapping): d[k] = e[k] else: sys.stderr.write("fill_ctd.py: could not merge key %s for %s in %s" % (k, d, e)) sys.exit(1) + def _json_object_hook_noenvlookup(d): return _json_object_hook(d, envlookup=False) + def _json_object_hook(d, envlookup=True): """ wee helper to transform the json written by galaxy @@ -105,6 +107,21 @@ def qstring2list(qs): return lst +def fix_underscores(args): + if type(args) is dict: + for k in list(args.keys()): + v = args[k] + if type(v) is dict: + fix_underscores(args[k]) + if k.startswith("_"): + args[k[1:]] = v + del args[k] + elif type(args) is list: + for i, v in enumerate(args): + if type(v) is dict: + fix_underscores(args[i]) + + input_ctd = sys.argv[1] # load user specified parameters from json @@ -118,15 +135,23 @@ def qstring2list(qs): # insert the hc_args into the args mergeDicts(args, hc_args) - if "adv_opts_cond" in args: args.update(args["adv_opts_cond"]) del args["adv_opts_cond"] +# IDMapper has in and spectra:in params, in is used in out as format_source", +# which does not work in Galaxy: https://github.com/galaxyproject/galaxy/pull/9493" +# therefore hardcoded params change the name of spectra:in to spectra:_in +# which is corrected here again +# TODO remove once PR is in and adapt profile accordingly +fix_underscores(args) + model = CTDModel(from_file=input_ctd) # transform values from json that correspond to # - old style booleans (string + restrictions) -> transformed to a str +# - new style booleans that get a string (happens for hidden parameters [-test]) +# are transformed to a bool # - unrestricted ITEMLIST which are represented as strings # ("=quoted and space separated) in Galaxy -> transform to lists # - optional data input parameters that have defaults and for which no @@ -141,20 +166,24 @@ def qstring2list(qs): except KeyError: # few tools use dashes in parameters which are automatically replaced # by underscores by Galaxy. in these cases the dictionary needs to be - # updated + # updated (better: then dash and the underscore variant are in the dict) # TODO might be removed later https://github.com/OpenMS/OpenMS/pull/4529 try: - jl = [_.replace("-", "_") for _ in p.get_lineage(name_only=True)] - getFromDict(args, jl) + lineage = [_.replace("-", "_") for _ in p.get_lineage(name_only=True)] + val = getFromDict(args, lineage) except KeyError: continue else: - setInDict(args, p.get_lineage(name_only=True), jl) + setInDict(args, p.get_lineage(name_only=True), val) if p.type is str and type(p.restrictions) is _Choices and set(p.restrictions.choices) == set(["true", "false"]): v = getFromDict(args, p.get_lineage(name_only=True)) setInDict(args, p.get_lineage(name_only=True), str(v).lower()) - + elif p.type is bool: + v = getFromDict(args, p.get_lineage(name_only=True)) + if isinstance(v, str): + v = (v.lower() == "true") + setInDict(args, p.get_lineage(name_only=True), v) elif p.is_list and (p.restrictions is None or type(p.restrictions) is _NumericRange): v = getFromDict(args, p.get_lineage(name_only=True)) if type(v) is str: diff --git a/tests/test-data/mock.py b/tests/test-data/mock.py index 64094cf9..442dae11 100755 --- a/tests/test-data/mock.py +++ b/tests/test-data/mock.py @@ -4,7 +4,6 @@ # i.e. to create them import os -import re import shutil import sys diff --git a/tests/test_galaxy_cli.py b/tests/test_galaxy_cli.py index 66c65ccf..bb3d9b03 100644 --- a/tests/test_galaxy_cli.py +++ b/tests/test_galaxy_cli.py @@ -1,8 +1,7 @@ -import unittest import os -from shutil import copyfile import subprocess import tempfile +import unittest def to_test_data(*args): From 9247ce5d2397143a0683e179755be5e083b9d054 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Wed, 10 Feb 2021 14:58:37 +0100 Subject: [PATCH 121/121] use continuous integers instead of dataset id simpler and nicer --- galaxy/converter.py | 24 +- tests/test-data/bool.xml | 1 - tests/test-data/empty.xml | 1 - tests/test-data/float.xml | 1 - tests/test-data/ifile.xml | 289 +++++++++--------- tests/test-data/integer.xml | 1 - tests/test-data/label-help.xml | 1 - tests/test-data/ofile-corresponding-input.xml | 1 - tests/test-data/ofile-mult-typeparam.xml | 25 +- tests/test-data/ofile-mult.xml | 25 +- tests/test-data/ofile-typeparam.xml | 1 - tests/test-data/ofile.xml | 1 - tests/test-data/repeat_test.xml | 1 - tests/test-data/select_test.xml | 1 - tests/test-data/string.xml | 1 - 15 files changed, 180 insertions(+), 194 deletions(-) diff --git a/galaxy/converter.py b/galaxy/converter.py index baed60cf..be0136cb 100755 --- a/galaxy/converter.py +++ b/galaxy/converter.py @@ -584,7 +584,7 @@ def create_command(tool, model, **kwargs): final_cmd = OrderedDict([('preprocessing', []), ('command', []), ('postprocessing', [])]) advanced_cmd = {'preprocessing': [], 'command': [], 'postprocessing': []} - final_cmd['preprocessing'].extend(["@QUOTE_FOO@", "@EXT_FOO@", "#import re", "#import os", "", "## Preprocessing"]) + final_cmd['preprocessing'].extend(["@QUOTE_FOO@", "@EXT_FOO@", "#import re", "", "## Preprocessing"]) # - call the executable with -write_ctd to write the ctd file (with defaults) # - use fill_ctd.py to overwrite the defaults in the ctd file with the @@ -639,9 +639,9 @@ def create_command(tool, model, **kwargs): if param.type is _InFile: param_cmd['preprocessing'].append("mkdir %s &&" % actual_parameter) if param.is_list: - param_cmd['preprocessing'].append("mkdir ${' '.join([\"'" + actual_parameter + "/%s'\"%(os.path.basename(str(_))[:-4]) for _ in $" + _actual_parameter + " if _])} && ") - param_cmd['preprocessing'].append("${' '.join([\"ln -s '%s' '" + actual_parameter + "/%s/%s.%s' && \" % (_, os.path.basename(str(_))[:-4], re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $" + _actual_parameter + " if _])}") - param_cmd['command'].append("${' '.join([\"'" + actual_parameter + "/%s/%s.%s'\"%(os.path.basename(str(_))[:-4], re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(_.ext)) for _ in $" + _actual_parameter + " if _])}") + param_cmd['preprocessing'].append("mkdir ${' '.join([\"'" + actual_parameter + "/%s'\" % (i) for i, f in enumerate($" + _actual_parameter + ") if f])} && ") + param_cmd['preprocessing'].append("${' '.join([\"ln -s '%s' '" + actual_parameter + "/%s/%s.%s' && \" % (f, i, re.sub('[^\w\-_]', '_', f.element_identifier), $gxy2omsext(f.ext)) for i, f in enumerate($" + _actual_parameter + ") if f])}") + param_cmd['command'].append("${' '.join([\"'" + actual_parameter + "/%s/%s.%s'\"%(i, re.sub('[^\w\-_]', '_', f.element_identifier), $gxy2omsext(f.ext)) for i, f in enumerate($" + _actual_parameter + ") if f])}") else: param_cmd['preprocessing'].append("ln -s '$" + _actual_parameter + "' '" + actual_parameter + "/${re.sub(\"[^\w\-_]\", \"_\", $" + _actual_parameter + ".element_identifier)}.$gxy2omsext($" + _actual_parameter + ".ext)' &&") param_cmd['command'].append("'" + actual_parameter + "/${re.sub(\"[^\w\-_]\", \"_\", $" + _actual_parameter + ".element_identifier)}.$gxy2omsext($" + _actual_parameter + ".ext)'") @@ -685,9 +685,9 @@ def create_command(tool, model, **kwargs): fmt = formats.pop() if param.is_list: logger.info("1 fmt + list %s -> %s" % (param.name, actual_input_parameter), 1) - param_cmd['preprocessing'].append("mkdir ${' '.join([\"'" + actual_parameter + "/%s'\"%(os.path.basename(str(_))[:-4]) for _ in $" + actual_input_parameter + " if _])} && ") - param_cmd['command'].append("${' '.join([\"'" + actual_parameter + "/%s/%s.%s'\"%(os.path.basename(str(_))[:-4], re.sub('[^\w\-_]', '_', _.element_identifier), $gxy2omsext(\"" + fmt + "\")) for _ in $" + actual_input_parameter + " if _])}") - param_cmd['postprocessing'].append("${' '.join([\"&& mv -n '" + actual_parameter + "/%(bn)s/%(id)s.%(gext)s' '" + _actual_parameter + "/%(bn)s/%(id)s'\"%{\"bn\": os.path.basename(str(_))[:-4], \"id\": re.sub('[^\w\-_]', '_', _.element_identifier), \"gext\": $gxy2omsext(\"" + fmt + "\")} for _ in $" + actual_input_parameter + " if _])}") + param_cmd['preprocessing'].append("mkdir ${' '.join([\"'" + actual_parameter + "/%s'\" % (i) for i, f in enumerate($" + actual_input_parameter + ") if f])} && ") + param_cmd['command'].append("${' '.join([\"'" + actual_parameter + "/%s/%s.%s'\"%(i, re.sub('[^\w\-_]', '_', f.element_identifier), $gxy2omsext(\"" + fmt + "\")) for i, f in enumerate($" + actual_input_parameter + ") if f])}") + param_cmd['postprocessing'].append("${' '.join([\"&& mv -n '" + actual_parameter + "/%(bn)s/%(id)s.%(gext)s' '" + _actual_parameter + "/%(bn)s/%(id)s'\"%{\"bn\": i, \"id\": re.sub('[^\w\-_]', '_', f.element_identifier), \"gext\": $gxy2omsext(\"" + fmt + "\")} for i, f in enumerate($" + actual_input_parameter + ") if f])}") else: logger.info("1 fmt + dataset %s" % param.name, 1) param_cmd['command'].append("'" + actual_parameter + "/output.${gxy2omsext(\"" + fmt + "\")}'") @@ -699,9 +699,9 @@ def create_command(tool, model, **kwargs): elif type_param_name is not None: if param.is_list: logger.info("type + list %s" % param.name, 1) - param_cmd['preprocessing'].append("mkdir ${' '.join([\"'" + actual_parameter + "/%s'\"%(os.path.basename(str(_))[:-4]) for _ in $" + actual_input_parameter + " if _])} && ") - param_cmd['command'].append("${' '.join([\"'" + actual_parameter + "/%s/%s.%s'\"%(os.path.basename(str(_))[:-4], re.sub('[^\w\-_]', '_', _.element_identifier), $" + type_param_name + ") for _ in $" + actual_input_parameter + " if _])}") - param_cmd['postprocessing'].append("${' '.join([\"&& mv -n '" + actual_parameter + "/%(bn)s/%(id)s.%(omsext)s' '" + actual_parameter + "/%(bn)s/%(id)s.%(gext)s'\"%{\"bn\": os.path.basename(str(_))[:-4], \"id\": re.sub('[^\w\-_]', '_', _.element_identifier), \"omsext\":$" + type_param_name + ", \"gext\": $oms2gxyext(str($" + type_param_name + "))} for _ in $" + actual_input_parameter + " if _])}") + param_cmd['preprocessing'].append("mkdir ${' '.join([\"'" + actual_parameter + "/%s'\" % (i) for i, f in enumerate($" + actual_input_parameter + ") if f])} && ") + param_cmd['command'].append("${' '.join([\"'" + actual_parameter + "/%s/%s.%s'\"%(i, re.sub('[^\w\-_]', '_', f.element_identifier), $" + type_param_name + ") for i, f in enumerate($" + actual_input_parameter + ") if f])}") + param_cmd['postprocessing'].append("${' '.join([\"&& mv -n '" + actual_parameter + "/%(bn)s/%(id)s.%(omsext)s' '" + actual_parameter + "/%(bn)s/%(id)s.%(gext)s'\"%{\"bn\": i, \"id\": re.sub('[^\w\-_]', '_', f.element_identifier), \"omsext\":$" + type_param_name + ", \"gext\": $oms2gxyext(str($" + type_param_name + "))} for i, f in enumerate($" + actual_input_parameter + ") if f])}") else: logger.info("type + dataset %s" % param.name, 1) # 1st create file with openms extension (often required by openms) @@ -712,8 +712,8 @@ def create_command(tool, model, **kwargs): elif actual_input_parameter is not None: if param.is_list: logger.info("actual + list %s" % param.name, 1) - param_cmd['preprocessing'].append("mkdir ${' '.join([\"'" + actual_parameter + "/%s'\"%(os.path.basename(str(_))[:-4]) for _ in $" + actual_input_parameter + " if _])} && ") - param_cmd['command'].append("${' '.join([\"'" + actual_parameter + "/%s/%s.%s'\"%(os.path.basename(str(_))[:-4], re.sub('[^\w\-_]', '_', _.element_identifier), _.ext) for _ in $" + actual_input_parameter + " if _])}") + param_cmd['preprocessing'].append("mkdir ${' '.join([\"'" + actual_parameter + "/%s'\" % (i) for i, f in enumerate($" + actual_input_parameter + ") if f])} && ") + param_cmd['command'].append("${' '.join([\"'" + actual_parameter + "/%s/%s.%s'\"%(i, re.sub('[^\w\-_]', '_', f.element_identifier), f.ext) for i, f in enumerate($" + actual_input_parameter + ") if f])}") else: logger.info("actual + dataset %s %s %s" % (param.name, actual_input_parameter, corresponding_input.is_list), 1) if corresponding_input.is_list: diff --git a/tests/test-data/bool.xml b/tests/test-data/bool.xml index 536b7dfc..89f1ed08 100644 --- a/tests/test-data/bool.xml +++ b/tests/test-data/bool.xml @@ -12,7 +12,6 @@ diff --git a/tests/test-data/ofile-mult.xml b/tests/test-data/ofile-mult.xml index 6a36dc91..ae4ae589 100644 --- a/tests/test-data/ofile-mult.xml +++ b/tests/test-data/ofile-mult.xml @@ -12,21 +12,20 @@