Implement list_filled_paths for the HDF5 backend

include/al_backend.h

@@ -175,6 +175,17 @@ class IMAS_CORE_LIBRARY_API Backend
   virtual void get_occurrences(Context* ctx, const char* ids_name, int** occurrences_list, int* size) = 0;
 
 
+  /**
+     Get a list of all Data Dictionary paths for which some data is filled.
+     This function is only implemented for tensorizing backends (i.e. the HDF5 backend).
+     @param[in] dataobjectname IDS name and occurrence
+     @param[in,out] path_list list of c-style strings (ending with a null byte)
+     @param[in,out] size specify the size of the array (number of elements)
+     @throw BackendException
+  */
+  virtual void list_filled_paths(Context* ctx, const char* dataobjectname, char*** path_list, int* size) = 0;
+
+
   /**
     Returns true if the backend performs time data interpolation (e.g time slices operations or IMAS-3885 with data resampling), false otherwise.
   **/

include/al_lowlevel.h

@@ -490,6 +490,20 @@ extern "C"
 
   IMAS_CORE_LIBRARY_API al_status_t al_get_occurrences(int pctxID, const char* ids_name, int** occurrences_list, int* size);
 
+  /**
+     Get a list of Data Dictionary paths (without indices) containing filled data in the backend.
+
+     Notes:
+     1. This function is only implemented for tensorizing backends (i.e. the HDF5 backend).
+     2. The paths may appear in any order.
+     3. The caller is responsible for freeing the list and all strings in it.
+
+     @param[in] dataobjectname IDS name and occurrence, e.g. "core_profiles", "equilibrium/1"
+     @param[in,out] path_list list of c-style strings (ending with a null byte)
+     @param[in,out] size specify the size of the array (number of elements)
+   */
+  IMAS_CORE_LIBRARY_API al_status_t al_list_filled_paths(int pctxID, const char* dataobjectname, char*** path_list, int* size);
+
   //IMAS_CORE_LIBRARY_API al_status_t al_close_pulse(int pctxID, int mode, const char *options);
 
   //HLI wrappers for plugins API

python/imas_core/_al_lowlevel.pyx

@@ -1227,3 +1227,22 @@ def al_get_occurrences(ctx, ids_name):
 def get_al_version():
     version = ll.getALVersion()
     return version.decode('UTF-8')
+
+
+def al_list_filled_paths(ctx: int, dataobjectname: str):
+    cdef char** path_list
+    cdef int cSize
+
+    al_status = ll.al_list_filled_paths(ctx, dataobjectname.encode(), &path_list, &cSize)
+    if al_status.code < 0:
+        raise get_proper_exception_class(f'Error while calling al_list_filled_paths: {al_status.message.decode()}', al_status.code)
+
+    # Create python list of strings from the C-style path_list and clean up memory
+    result = []
+    for i in range(cSize):
+        result.append(path_list[i].decode())
+        free(path_list[i])
+    if cSize > 0:
+        free(path_list)
+
+    return al_status.code, result

python/imas_core/al_lowlevel_interface.pxd

@@ -54,4 +54,6 @@ cdef extern from "al_lowlevel.h":
 
     al_status_t al_get_occurrences(int ctx, const char* ids_name, int **occurrences_list, int *size)
 
+    al_status_t al_list_filled_paths(int ctx, const char* dataobjectname, char*** path_list, int *size)
+
     const char* getALVersion()

src/al_lowlevel.cpp

@@ -1775,6 +1775,30 @@ al_status_t al_get_occurrences(int pctxID, const char* ids_name, int** occurrenc
   return status;
 }
 
+al_status_t al_list_filled_paths(int pctxID, const char* dataobjectname, char*** path_list, int* size) {
+  al_status_t status;
+
+  status.code = 0;
+  try {
+    LLenv lle = Lowlevel::getLLenv(pctxID);
+    lle.backend->list_filled_paths(lle.context, dataobjectname, path_list, size);
+  }
+  catch (const ALBackendException& e) {
+    status.code = alerror::backend_err;
+    ALException::registerStatus(status.message, __func__, e);
+  }
+  catch (const ALLowlevelException& e) {
+    status.code = alerror::lowlevel_err;
+    ALException::registerStatus(status.message, __func__, e);
+  }
+  catch (const std::exception& e) {
+    status.code = alerror::unknown_err;
+    ALException::registerStatus(status.message, __func__, e);
+  }
+
+  return status;
+}
+
 
 al_status_t al_setvalue_parameter_plugin(const char* parameter_name, int datatype, int dim, int *size, void *data, const char* pluginName) {
     al_status_t status;

src/ascii_backend.cpp

@@ -708,4 +708,7 @@ void AsciiBackend::get_occurrences(Context* ctx, const  char* ids_name, int** oc
     (*occurrences_list)[i] = occurrences[i];
 }
 
+void AsciiBackend::list_filled_paths(Context* ctx, const char* dataobjectname, char*** path_list, int* size) {
+    throw ALBackendException("list_filled_paths is not implemented in the ASCII Backend", LOG);
+}
 

src/ascii_backend.h

@@ -91,6 +91,7 @@ class IMAS_CORE_LIBRARY_API AsciiBackend : public Backend
   std::pair<int,int> getVersion(DataEntryContext *ctx) override;
 
   void get_occurrences(Context* ctx, const  char* ids_name, int** occurrences_list, int* size) override;
+  void list_filled_paths(Context* ctx, const char* dataobjectname, char*** path_list, int* size) override;
 
   bool supportsTimeDataInterpolation() override {
     return false;

src/flexbuffers_backend.cpp

@@ -398,6 +398,15 @@ void FlexbuffersBackend::get_occurrences(
     throw ALBackendException("get_occurrences is not implemented in the Serialize Backend", LOG);
 }
 
+void FlexbuffersBackend::list_filled_paths(
+    Context* ctx,
+    const char* dataobjectname,
+    char*** path_list,
+    int* size
+) {
+    throw ALBackendException("list_filled_paths is not implemented in the Serialize Backend", LOG);
+}
+
 void FlexbuffersBackend::_start_vector() {
     _vector_starts.push(_builder->StartVector());
 }

src/flexbuffers_backend.h

@@ -48,6 +48,7 @@ class IMAS_CORE_LIBRARY_API FlexbuffersBackend : public Backend
     void deleteData(OperationContext *ctx, std::string path) override;
     void beginArraystructAction(ArraystructContext *ctx, int *size) override;
     void get_occurrences(Context* ctx, const char* ids_name, int** occurrences_list, int* size) override;
+    void list_filled_paths(Context* ctx, const char* dataobjectname, char*** path_list, int* size) override;
     // timerange (get_sample) API is not supported:
     bool supportsTimeDataInterpolation() override { return false; }
     bool supportsTimeRangeOperation() override { return false; }

src/hdf5/hdf5_backend.cpp

@@ -162,3 +162,11 @@ void HDF5Backend::get_occurrences(Context* ctx, const  char* ids_name, int** occ
         throw ALBackendException("HDF5Backend: master file not opened while calling HDF5Backend::get_occurrences()", LOG); 
     hdf5Reader->get_occurrences(ids_name, occurrences_list, size, file_id);
 }
+
+void HDF5Backend::list_filled_paths(Context* ctx, const char* dataobjectname, char*** path_list, int* size)
+{
+    if (file_id == -1) // master file not opened
+        throw ALBackendException("HDF5Backend: master file not opened while calling HDF5Backend::list_filled_paths()", LOG); 
+
+    hdf5Reader->list_filled_paths(dataobjectname, path_list, size, file_id, opened_IDS_files, files_directory, relative_file_path);
+}

src/hdf5/hdf5_backend.h

@@ -131,6 +131,7 @@ class HDF5Backend:public Backend {
     void beginAction(OperationContext * ctx) override;
 
     void get_occurrences(Context* ctx, const char* ids_name, int** occurrences_list, int* size) override;
+    void list_filled_paths(Context* ctx, const char* dataobjectname, char*** path_list, int* size) override;
 
     bool supportsTimeDataInterpolation() override {
       return true;

src/hdf5/hdf5_reader.cpp

@@ -1568,6 +1568,49 @@ void HDF5Reader::get_occurrences(const char *ids_name, int **occurrences_list, i
         p[i] = occurrences[i];
 }
 
+void HDF5Reader::list_filled_paths(const char* dataobjectname, char*** path_list, int* size, hid_t file_id, std::unordered_map < std::string, hid_t > &opened_IDS_files, std::string & files_directory, std::string & relative_file_path)
+{
+    HDF5Utils hdf5_utils;
+    // Create temporary OperationContext to allow reusing existing logic from hdf5_utils:
+    OperationContext ctx(NULL, dataobjectname, "", READ_OP);
+    hid_t gid = -1;
+    hdf5_utils.open_IDS_group(&ctx, file_id, opened_IDS_files, files_directory, relative_file_path, &gid);
+    if (gid <= 0) {
+        // Group does not exist => nothing is filled
+        *size = 0;
+        return;
+    }
+
+    // Create a vector of all link names in the hdf5 group
+    std::vector<std::string> variables;
+    H5L_iterate_t iterate_callback = [](hid_t group_id, const char* cname, const H5L_info_t* info, void* op_data) -> herr_t {
+        std::string name(cname);
+        // Skip if name ends with _SHAPE
+        if (name.size() < 6 || name.substr(name.size() - 6) != "_SHAPE")
+            static_cast<std::vector<std::string>*>(op_data)->push_back(name);
+        return 0; // success
+    };
+    herr_t status = H5Literate(gid, H5_INDEX_NAME, H5_ITER_NATIVE, NULL, iterate_callback, &variables);
+    if (status != 0)
+        throw ALBackendException("HDF5Backend: H5Literate has failed in HDF5Reader::get_occurrences()", LOG);
+
+    // Convert to DD path: remove AoS brackets ([]) and use / to separate paths
+    for (auto &str : variables) {
+        // Remove square brackets
+        str.erase(std::remove(str.begin(), str.end(), '['), str.end());
+        str.erase(std::remove(str.begin(), str.end(), ']'), str.end());
+        // Replace '&' with '/'
+        std::replace(str.begin(), str.end(), '&', '/');
+    }
+
+    // Create the C-style array:
+    *size = variables.size();
+    *path_list = (char**) malloc(*size * sizeof(char*));
+    for (int i=0; i < *size; ++i) {
+        (*path_list)[i] = strdup(variables[i].c_str());
+    }
+}
+
 herr_t HDF5Reader::iterate_callback(hid_t loc_id, const char *name, const H5L_info_t *info, void *callback_data)
 {
     std::vector<std::string> *op = reinterpret_cast<std::vector<std::string> *>(callback_data);

src/hdf5/hdf5_reader.h

@@ -89,6 +89,7 @@ class HDF5Reader {
     virtual int read_ND_Data(Context * ctx, std::string & att_name, std::string & timebasename, int datatype, void **data, int *dim, int *size);
     virtual void beginReadArraystructAction(ArraystructContext * ctx, int *size);
     virtual void get_occurrences(const char* ids_name, int** occurrences_list, int* size, hid_t master_file_id);
+    void list_filled_paths(const char* dataobjectname, char*** path_list, int* size, hid_t file_id, std::unordered_map < std::string, hid_t > &opened_IDS_files, std::string & files_directory, std::string & relative_file_path);
 
     void open_IDS_group(OperationContext * ctx, hid_t file_id, std::unordered_map < std::string, hid_t > &opened_IDS_files, std::string & files_directory, std::string & relative_file_path);
     void close_file_handler(std::string external_link_name, std::unordered_map < std::string, hid_t > &opened_IDS_files);

src/mdsplus/mdsplus_backend.cpp

@@ -4932,6 +4932,10 @@ void MDSplusBackend::get_occurrences(Context* ctx, const char* ids_name, int** o
 	*size = occurrences.size();
 }
 
+void MDSplusBackend::list_filled_paths(Context* ctx, const char* dataobjectname, char*** path_list, int* size) {
+    throw ALBackendException("list_filled_paths is not implemented in the MDSplus Backend", LOG);
+}
+
  void MDSplusBackend::fullPath(Context *ctx, std::string &path) {
    if (ctx->getType() == CTX_PULSE_TYPE) {
        path = "";

src/mdsplus/mdsplus_backend.h

@@ -250,6 +250,7 @@ class IMAS_CORE_LIBRARY_API MDSplusBackend:public Backend
     std::pair<int,int> getVersion(DataEntryContext *ctx) override;
 
     void get_occurrences(Context* ctx, const char* ids_name, int** occurrences_list, int* size) override;
+    void list_filled_paths(Context* ctx, const char* dataobjectname, char*** path_list, int* size) override;
 
     bool supportsTimeDataInterpolation() {
       return true;

src/memory_backend.cpp

@@ -1148,7 +1148,10 @@ else
 		throw ALBackendException(message, LOG);
 	}
 
-
+	void MemoryBackend::list_filled_paths(Context* ctx, const char* dataobjectname, char*** path_list, int* size) {
+		std::string message("list_filled_paths() is not implemented in the MemoryBackend");
+		throw ALBackendException(message, LOG);
+	}
 
 //////////////////////////////////////////////////////////////////////////
 

src/memory_backend.h

@@ -673,6 +673,7 @@ class IMAS_CORE_LIBRARY_API MemoryBackend:public Backend
     ALData *getAlSlice(ArraystructContext *ctx, ALData &inData, double time, std::vector<double> timebaseV);
 
 	void get_occurrences(Context* ctx, const char* ids_name, int** occurrences_list, int* size) override;
+    void list_filled_paths(Context* ctx, const char* dataobjectname, char*** path_list, int* size) override;
 
 	bool supportsTimeDataInterpolation() override {
       return false;

src/no_backend.cpp

@@ -85,3 +85,11 @@ void NoBackend::get_occurrences(Context* ctx, const char* ids_name, int** occurr
     *size = 0;
 }
 
+void NoBackend::list_filled_paths(Context* ctx, const char* dataobjectname, char*** path_list, int* size)
+{
+  if (verbose) {
+      std::string message("list_filled_paths() is not implemented in the NoBackend");
+      throw ALBackendException(message, LOG);
+  }
+  *size = 0;
+}

src/no_backend.h

@@ -64,6 +64,8 @@ class IMAS_CORE_LIBRARY_API NoBackend : public Backend
 
   void get_occurrences(Context* ctx, const  char* ids_name, int** occurrences_list, int* size) override;
 
+  void list_filled_paths(Context* ctx, const char* dataobjectname, char*** path_list, int* size) override;
+
   bool supportsTimeDataInterpolation() override {
       return false;
     }

src/uda/uda_backend.cpp

@@ -1310,3 +1310,6 @@ void UDABackend::get_occurrences(Context* ctx, const char* ids_name, int** occur
     }
 }
 
+void UDABackend::list_filled_paths(Context* ctx, const char* dataobjectname, char*** path_list, int* size) {
+    throw ALBackendException("list_filled_paths is not implemented in the UDA Backend", LOG);
+}

src/uda/uda_backend.h

@@ -183,6 +183,7 @@ class IMAS_CORE_LIBRARY_API UDABackend : public Backend
     std::pair<int,int> getVersion(DataEntryContext *ctx) override;
 
     void get_occurrences(Context* ctx, const char* ids_name, int** occurrences_list, int* size) override;
+    void list_filled_paths(Context* ctx, const char* dataobjectname, char*** path_list, int* size) override;
 
     bool supportsTimeDataInterpolation() override;