From 10e9b19d1357160771f254937560ee4a6c9505b2 Mon Sep 17 00:00:00 2001 From: Nalini Ganapati Date: Wed, 18 Dec 2024 23:19:14 -0800 Subject: [PATCH 1/6] React to added genomicsdb api changes --- .github/scripts/install_prereqs.sh | 3 ++- examples/genomicsdb_cache | 5 +++-- examples/genomicsdb_query | 4 ++-- src/genomicsdb.pxd | 2 ++ src/genomicsdb.pyx | 15 ++++++++++++--- test/test_filesystem_api.py | 8 ++++++++ 6 files changed, 29 insertions(+), 8 deletions(-) diff --git a/.github/scripts/install_prereqs.sh b/.github/scripts/install_prereqs.sh index b90ec0c..6b612dc 100755 --- a/.github/scripts/install_prereqs.sh +++ b/.github/scripts/install_prereqs.sh @@ -206,7 +206,8 @@ if [[ $1 == "release" ]]; then echo "DYLD_LIBRARY_PATH=${DYLD_LIBRARY_PATH}" echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}" # For Debugging - git clone https://github.com/GenomicsDB/GenomicsDB.git -b develop $NATIVE_BUILD_DIR + #git clone https://github.com/GenomicsDB/GenomicsDB.git -b develop $NATIVE_BUILD_DIR + it clone https://github.com/GenomicsDB/GenomicsDB.git -b ng_errmsg $NATIVE_BUILD_DIR pushd $NATIVE_BUILD_DIR mkdir build && pushd build && diff --git a/examples/genomicsdb_cache b/examples/genomicsdb_cache index cd1a879..e8dd56f 100755 --- a/examples/genomicsdb_cache +++ b/examples/genomicsdb_cache @@ -93,7 +93,7 @@ def main(): args = parser.parse_args() workspace = genomicsdb_common.normalize_path(args.workspace) - if not genomicsdb.is_file(workspace + "/__tiledb_workspace.tdb"): + if not genomicsdb.workspace_exists(workspace): raise RuntimeError(f"workspace({workspace}) not found") callset_file = args.callset @@ -128,7 +128,8 @@ def main(): for array in genomicsdb_common.get_arrays(contigs_map, intervals, partitions): print(f"Caching fragments for array {array}") - genomicsdb.cache_array_metadata(workspace, array) + if genomicsdb.array_exists(workspace, array): + genomicsdb.cache_array_metadata(workspace, array) if __name__ == "__main__": diff --git a/examples/genomicsdb_query b/examples/genomicsdb_query index 8bd57ac..5015c02 100755 --- a/examples/genomicsdb_query +++ b/examples/genomicsdb_query @@ -204,7 +204,7 @@ def setup_gdb(): args = parser.parse_args() workspace = genomicsdb_common.normalize_path(args.workspace) - if not genomicsdb.is_file(workspace + "/__tiledb_workspace.tdb"): + if not genomicsdb.workspace_exists(workspace): raise RuntimeError(f"workspace({workspace}) not found") callset_file = args.callset if not callset_file: @@ -346,7 +346,7 @@ def main(): print(f"Using {args.max_arrow_byte_size} number of bytes as hint for writing out parquet files") for idx, array in enumerate(arrays): - if not genomicsdb.is_file(workspace + "/" + array + "/__array_schema.tdb"): + if not genomicsdb.array_exists(workspace, array): print(f"\tArray({array}) not imported into workspace({workspace}) for interval({interval})") continue query_config = query_pb.QueryConfiguration() diff --git a/src/genomicsdb.pxd b/src/genomicsdb.pxd index 5f1bd85..72293bc 100644 --- a/src/genomicsdb.pxd +++ b/src/genomicsdb.pxd @@ -252,6 +252,8 @@ cdef extern from "genomicsdb_utils.h": cdef bint c_is_file "genomicsdb::is_file"(string) cdef ssize_t c_file_size "genomicsdb::file_size"(string) cdef int c_read_entire_file "genomicsdb::read_entire_file"(string, void**, size_t*) + cdef bint c_workspace_exists "genomicsdb::workspace_exists"(string) + cdef bint c_array_exists "genomicsdb::array_exists"(string, string) cdef vector[string] c_get_array_names "genomicsdb::get_array_names"(string) cdef int c_cache_fragment_metadata "genomicsdb::cache_fragment_metadata"(string, string) pass diff --git a/src/genomicsdb.pyx b/src/genomicsdb.pyx index e55eb05..67bc055 100644 --- a/src/genomicsdb.pyx +++ b/src/genomicsdb.pyx @@ -497,6 +497,15 @@ def read_entire_file(filename): return contents_string -def cache_array_metadata(workspace, array): - if c_cache_fragment_metadata(as_string(workspace), as_string(array)) != 0: - print(f"Could not cache fragment metadata for array={array} in {workspace}") +def workspace_exists(workspace): + return c_workspace_exists(as_string(workspace)) + + +def array_exists(workspace, array_name): + return c_array_exists(as_string(workspace), as_string(array_name)) + + +def cache_metadata(workspace): + for array in c_get_array_names(as_string(workspace)): + if c_cache_fragment_metadata(as_string(workspace), array) != 0: + print(f"Could not cache fragment metadata for array={array.decode()} in {workspace}") diff --git a/test/test_filesystem_api.py b/test/test_filesystem_api.py index 345dc52..b1dcb56 100644 --- a/test/test_filesystem_api.py +++ b/test/test_filesystem_api.py @@ -19,3 +19,11 @@ def test_filesystem_api(tmpdir): read_text = genomicsdb.read_entire_file(hello_file) assert read_text == text + + assert not genomicsdb.workspace_exists("non-existent-ws") + assert not genomicsdb.workspace_exists("az://non-existent-container/ws") + assert not genomicsdb.workspace_exists("az://non-existent-container@non-existent-account.blob/ws") + + assert not genomicsdb.array_exists("non-existent-ws", "non-existent-array") + assert not genomicsdb.array_exists("az://non-existent-container/ws", "non-existent-array") + assert not genomicsdb.array_exists("az://non-existent-container@non-existent-account.blob/ws", "non-existent-array") From 756d46d52ee12e878843d9dbe4db19f0f2af3aff Mon Sep 17 00:00:00 2001 From: Nalini Ganapati Date: Thu, 19 Dec 2024 08:16:30 -0800 Subject: [PATCH 2/6] Move to ng_errmsg for new api --- .github/workflows/basic.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/basic.yml b/.github/workflows/basic.yml index 7e18d4a..67f8bbf 100644 --- a/.github/workflows/basic.yml +++ b/.github/workflows/basic.yml @@ -15,7 +15,7 @@ on: env: GENOMICSDB_BUILD_DIR: ${{github.workspace}} GENOMICSDB_HOME: ${{github.workspace}}/install - GENOMICSDB_BRANCH: develop + GENOMICSDB_BRANCH: ng_errmsg jobs: native-build: From 3f7bd1f0a637f0bd983bb4e70cf3644c2f49e476 Mon Sep 17 00:00:00 2001 From: Nalini Ganapati Date: Thu, 19 Dec 2024 09:14:55 -0800 Subject: [PATCH 3/6] Fix cache_array_metadata --- src/genomicsdb.pyx | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/genomicsdb.pyx b/src/genomicsdb.pyx index 67bc055..0af77eb 100644 --- a/src/genomicsdb.pyx +++ b/src/genomicsdb.pyx @@ -501,11 +501,10 @@ def workspace_exists(workspace): return c_workspace_exists(as_string(workspace)) -def array_exists(workspace, array_name): - return c_array_exists(as_string(workspace), as_string(array_name)) +def array_exists(workspace, array): + return c_array_exists(as_string(workspace), as_string(array)) -def cache_metadata(workspace): - for array in c_get_array_names(as_string(workspace)): - if c_cache_fragment_metadata(as_string(workspace), array) != 0: - print(f"Could not cache fragment metadata for array={array.decode()} in {workspace}") +def cache_array_metadata(workspace, array): + if c_cache_fragment_metadata(as_string(workspace), as_string(array)) != 0: + print(f"Could not cache fragment metadata for array={array.decode()} in {workspace}") From f7e59e6a1e9cf0ad0ded68e6a84540303bbc895c Mon Sep 17 00:00:00 2001 From: Nalini Ganapati Date: Thu, 19 Dec 2024 09:29:35 -0800 Subject: [PATCH 4/6] Revert to develop --- .github/scripts/install_prereqs.sh | 3 +-- .github/workflows/basic.yml | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/scripts/install_prereqs.sh b/.github/scripts/install_prereqs.sh index 6b612dc..b90ec0c 100755 --- a/.github/scripts/install_prereqs.sh +++ b/.github/scripts/install_prereqs.sh @@ -206,8 +206,7 @@ if [[ $1 == "release" ]]; then echo "DYLD_LIBRARY_PATH=${DYLD_LIBRARY_PATH}" echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}" # For Debugging - #git clone https://github.com/GenomicsDB/GenomicsDB.git -b develop $NATIVE_BUILD_DIR - it clone https://github.com/GenomicsDB/GenomicsDB.git -b ng_errmsg $NATIVE_BUILD_DIR + git clone https://github.com/GenomicsDB/GenomicsDB.git -b develop $NATIVE_BUILD_DIR pushd $NATIVE_BUILD_DIR mkdir build && pushd build && diff --git a/.github/workflows/basic.yml b/.github/workflows/basic.yml index 67f8bbf..7e18d4a 100644 --- a/.github/workflows/basic.yml +++ b/.github/workflows/basic.yml @@ -15,7 +15,7 @@ on: env: GENOMICSDB_BUILD_DIR: ${{github.workspace}} GENOMICSDB_HOME: ${{github.workspace}}/install - GENOMICSDB_BRANCH: ng_errmsg + GENOMICSDB_BRANCH: develop jobs: native-build: From 793a8636dd785d99cb02aba793a1b7701dc3beff Mon Sep 17 00:00:00 2001 From: Nalini Ganapati Date: Thu, 19 Dec 2024 11:50:59 -0800 Subject: [PATCH 5/6] Move to macos-13 for release --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 281826d..de6ca4a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -28,7 +28,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [macos-12, ubuntu-22.04] + os: [macos-13, ubuntu-22.04] env: INSTALL_PREFIX: ${{ github.workspace }}/prereqs MACOSX_DEPLOYMENT_TARGET: 12.1 From dd7404cfc1f4a306d9220555d0a12bf0c322014a Mon Sep 17 00:00:00 2001 From: Nalini Ganapati Date: Thu, 19 Dec 2024 12:16:18 -0800 Subject: [PATCH 6/6] Release.yml --- .github/workflows/release.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index de6ca4a..45791c6 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -45,7 +45,7 @@ jobs: CIBW_BEFORE_BUILD: "pip install -r {package}/requirements_dev.txt && cd {package}" CIBW_BUILD: "cp39-* cp310-* cp311-* cp312-*" CIBW_SKIP: "*musllinux*" - CIBW_CONFIG_SETTINGS: "--build-option=--with-version=${{ needs.version_number.outputs.version }} --build-option=--with-genomicsdb=${{ env.INSTALL_PREFIX }}" + CIBW_CONFIG_SETTINGS: "--global-option=--with-version=${{ needs.version_number.outputs.version }} --build-option=--with-genomicsdb=${{ env.INSTALL_PREFIX }}" CIBW_ENVIRONMENT: CMAKE_PREFIX_PATH=${{ env.INSTALL_PREFIX }} CIBW_ENVIRONMENT_MACOS: DYLD_LIBRARY_PATH=${{ env.INSTALL_PREFIX }}/lib CIBW_ENVIRONMENT_LINUX: > @@ -66,7 +66,7 @@ jobs: build_macos_arm64_wheels: name: Build wheels for macOS arm64 needs: version_number - runs-on: macos-12 + runs-on: macos-13 env: INSTALL_PREFIX: ${{ github.workspace }}/prereqs_arm64 MACOSX_DEPLOYMENT_TARGET: 12.1 @@ -81,7 +81,7 @@ jobs: CIBW_BEFORE_ALL: "OSX_ARCH='arm64' INSTALL_PREFIX=${{ env.INSTALL_PREFIX }} .github/scripts/install_prereqs.sh release" CIBW_BEFORE_BUILD: "pip install -r {package}/requirements_dev.txt && cd {package}" CIBW_BUILD: "cp39-* cp310-* cp311-* cp312-*" - CIBW_CONFIG_SETTINGS: "--build-option=--with-version=${{ needs.version_number.outputs.version }} --build-option=--with-genomicsdb=${{ env.INSTALL_PREFIX }}" + CIBW_CONFIG_SETTINGS: "--global-option=--with-version=${{ needs.version_number.outputs.version }} --build-option=--with-genomicsdb=${{ env.INSTALL_PREFIX }}" CIBW_ENVIRONMENT: DYLD_LIBRARY_PATH=${{ env.INSTALL_PREFIX }}/lib - uses: actions/upload-artifact@v4