Add support for genomicsdb api changes to filesystem/workspace utilities

.github/workflows/release.yml

@@ -28,7 +28,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [macos-12, ubuntu-22.04]
+        os: [macos-13, ubuntu-22.04]
     env:
        INSTALL_PREFIX: ${{ github.workspace }}/prereqs
        MACOSX_DEPLOYMENT_TARGET: 12.1
@@ -45,7 +45,7 @@ jobs:
           CIBW_BEFORE_BUILD: "pip install -r {package}/requirements_dev.txt && cd {package}"
           CIBW_BUILD: "cp39-* cp310-* cp311-* cp312-*"
           CIBW_SKIP: "*musllinux*"
-          CIBW_CONFIG_SETTINGS: "--build-option=--with-version=${{ needs.version_number.outputs.version }} --build-option=--with-genomicsdb=${{ env.INSTALL_PREFIX }}"
+          CIBW_CONFIG_SETTINGS: "--global-option=--with-version=${{ needs.version_number.outputs.version }} --build-option=--with-genomicsdb=${{ env.INSTALL_PREFIX }}"
           CIBW_ENVIRONMENT: CMAKE_PREFIX_PATH=${{ env.INSTALL_PREFIX }}
           CIBW_ENVIRONMENT_MACOS: DYLD_LIBRARY_PATH=${{ env.INSTALL_PREFIX }}/lib
           CIBW_ENVIRONMENT_LINUX: >
@@ -66,7 +66,7 @@ jobs:
   build_macos_arm64_wheels:
     name: Build wheels for macOS arm64
     needs: version_number
-    runs-on: macos-12
+    runs-on: macos-13
     env:
       INSTALL_PREFIX: ${{ github.workspace }}/prereqs_arm64
       MACOSX_DEPLOYMENT_TARGET: 12.1
@@ -81,7 +81,7 @@ jobs:
           CIBW_BEFORE_ALL: "OSX_ARCH='arm64' INSTALL_PREFIX=${{ env.INSTALL_PREFIX }} .github/scripts/install_prereqs.sh release"
           CIBW_BEFORE_BUILD: "pip install -r {package}/requirements_dev.txt && cd {package}"
           CIBW_BUILD: "cp39-* cp310-* cp311-* cp312-*"
-          CIBW_CONFIG_SETTINGS: "--build-option=--with-version=${{ needs.version_number.outputs.version }} --build-option=--with-genomicsdb=${{ env.INSTALL_PREFIX }}"
+          CIBW_CONFIG_SETTINGS: "--global-option=--with-version=${{ needs.version_number.outputs.version }} --build-option=--with-genomicsdb=${{ env.INSTALL_PREFIX }}"
           CIBW_ENVIRONMENT: DYLD_LIBRARY_PATH=${{ env.INSTALL_PREFIX }}/lib
 
       - uses: actions/upload-artifact@v4

examples/genomicsdb_cache

@@ -93,7 +93,7 @@ def main():
     args = parser.parse_args()
 
     workspace = genomicsdb_common.normalize_path(args.workspace)
-    if not genomicsdb.is_file(workspace + "/__tiledb_workspace.tdb"):
+    if not genomicsdb.workspace_exists(workspace):
         raise RuntimeError(f"workspace({workspace}) not found")
 
     callset_file = args.callset
@@ -128,7 +128,8 @@ def main():
 
         for array in genomicsdb_common.get_arrays(contigs_map, intervals, partitions):
             print(f"Caching fragments for array {array}")
-            genomicsdb.cache_array_metadata(workspace, array)
+            if genomicsdb.array_exists(workspace, array):
+                genomicsdb.cache_array_metadata(workspace, array)
 
 
 if __name__ == "__main__":

examples/genomicsdb_query

@@ -204,7 +204,7 @@ def setup_gdb():
     args = parser.parse_args()
 
     workspace = genomicsdb_common.normalize_path(args.workspace)
-    if not genomicsdb.is_file(workspace + "/__tiledb_workspace.tdb"):
+    if not genomicsdb.workspace_exists(workspace):
         raise RuntimeError(f"workspace({workspace}) not found")
     callset_file = args.callset
     if not callset_file:
@@ -346,7 +346,7 @@ def main():
             print(f"Using {args.max_arrow_byte_size} number of bytes as hint for writing out parquet files")
 
         for idx, array in enumerate(arrays):
-            if not genomicsdb.is_file(workspace + "/" + array + "/__array_schema.tdb"):
+            if not genomicsdb.array_exists(workspace, array):
                 print(f"\tArray({array}) not imported into workspace({workspace}) for interval({interval})")
                 continue
             query_config = query_pb.QueryConfiguration()

src/genomicsdb.pxd

@@ -252,6 +252,8 @@ cdef extern from "genomicsdb_utils.h":
     cdef bint c_is_file "genomicsdb::is_file"(string)
     cdef ssize_t c_file_size "genomicsdb::file_size"(string)
     cdef int c_read_entire_file "genomicsdb::read_entire_file"(string, void**, size_t*)
+    cdef bint c_workspace_exists "genomicsdb::workspace_exists"(string)
+    cdef bint c_array_exists "genomicsdb::array_exists"(string, string)
     cdef vector[string] c_get_array_names "genomicsdb::get_array_names"(string)
     cdef int c_cache_fragment_metadata "genomicsdb::cache_fragment_metadata"(string, string)
     pass

src/genomicsdb.pyx

@@ -497,6 +497,14 @@ def read_entire_file(filename):
         return contents_string
 
 
+def workspace_exists(workspace):
+    return c_workspace_exists(as_string(workspace))
+
+
+def array_exists(workspace, array):
+    return c_array_exists(as_string(workspace), as_string(array))
+
+
 def cache_array_metadata(workspace, array):
     if c_cache_fragment_metadata(as_string(workspace), as_string(array)) != 0:
-        print(f"Could not cache fragment metadata for array={array} in {workspace}")
+        print(f"Could not cache fragment metadata for array={array.decode()} in {workspace}")

test/test_filesystem_api.py

@@ -19,3 +19,11 @@ def test_filesystem_api(tmpdir):
 
     read_text = genomicsdb.read_entire_file(hello_file)
     assert read_text == text
+
+    assert not genomicsdb.workspace_exists("non-existent-ws")
+    assert not genomicsdb.workspace_exists("az://non-existent-container/ws")
+    assert not genomicsdb.workspace_exists("az://non-existent-container@non-existent-account.blob/ws")
+
+    assert not genomicsdb.array_exists("non-existent-ws", "non-existent-array")
+    assert not genomicsdb.array_exists("az://non-existent-container/ws", "non-existent-array")
+    assert not genomicsdb.array_exists("az://non-existent-container@non-existent-account.blob/ws", "non-existent-array")