diff --git a/include/al_backend.h b/include/al_backend.h index afa1cdc8..aec9860f 100644 --- a/include/al_backend.h +++ b/include/al_backend.h @@ -175,6 +175,17 @@ class IMAS_CORE_LIBRARY_API Backend virtual void get_occurrences(Context* ctx, const char* ids_name, int** occurrences_list, int* size) = 0; + /** + Get a list of all Data Dictionary paths for which some data is filled. + This function is only implemented for tensorizing backends (i.e. the HDF5 backend). + @param[in] dataobjectname IDS name and occurrence + @param[in,out] path_list list of c-style strings (ending with a null byte) + @param[in,out] size specify the size of the array (number of elements) + @throw BackendException + */ + virtual void list_filled_paths(Context* ctx, const char* dataobjectname, char*** path_list, int* size) = 0; + + /** Returns true if the backend performs time data interpolation (e.g time slices operations or IMAS-3885 with data resampling), false otherwise. **/ diff --git a/include/al_lowlevel.h b/include/al_lowlevel.h index e2dea3a2..e3bd79be 100644 --- a/include/al_lowlevel.h +++ b/include/al_lowlevel.h @@ -490,6 +490,20 @@ extern "C" IMAS_CORE_LIBRARY_API al_status_t al_get_occurrences(int pctxID, const char* ids_name, int** occurrences_list, int* size); + /** + Get a list of Data Dictionary paths (without indices) containing filled data in the backend. + + Notes: + 1. This function is only implemented for tensorizing backends (i.e. the HDF5 backend). + 2. The paths may appear in any order. + 3. The caller is responsible for freeing the list and all strings in it. + + @param[in] dataobjectname IDS name and occurrence, e.g. "core_profiles", "equilibrium/1" + @param[in,out] path_list list of c-style strings (ending with a null byte) + @param[in,out] size specify the size of the array (number of elements) + */ + IMAS_CORE_LIBRARY_API al_status_t al_list_filled_paths(int pctxID, const char* dataobjectname, char*** path_list, int* size); + //IMAS_CORE_LIBRARY_API al_status_t al_close_pulse(int pctxID, int mode, const char *options); //HLI wrappers for plugins API diff --git a/python/imas_core/_al_lowlevel.pyx b/python/imas_core/_al_lowlevel.pyx index 562940d3..5eb16a80 100644 --- a/python/imas_core/_al_lowlevel.pyx +++ b/python/imas_core/_al_lowlevel.pyx @@ -1227,3 +1227,22 @@ def al_get_occurrences(ctx, ids_name): def get_al_version(): version = ll.getALVersion() return version.decode('UTF-8') + + +def al_list_filled_paths(ctx: int, dataobjectname: str): + cdef char** path_list + cdef int cSize + + al_status = ll.al_list_filled_paths(ctx, dataobjectname.encode(), &path_list, &cSize) + if al_status.code < 0: + raise get_proper_exception_class(f'Error while calling al_list_filled_paths: {al_status.message.decode()}', al_status.code) + + # Create python list of strings from the C-style path_list and clean up memory + result = [] + for i in range(cSize): + result.append(path_list[i].decode()) + free(path_list[i]) + if cSize > 0: + free(path_list) + + return al_status.code, result diff --git a/python/imas_core/al_lowlevel_interface.pxd b/python/imas_core/al_lowlevel_interface.pxd index b590795e..4ac9c78a 100644 --- a/python/imas_core/al_lowlevel_interface.pxd +++ b/python/imas_core/al_lowlevel_interface.pxd @@ -54,4 +54,6 @@ cdef extern from "al_lowlevel.h": al_status_t al_get_occurrences(int ctx, const char* ids_name, int **occurrences_list, int *size) + al_status_t al_list_filled_paths(int ctx, const char* dataobjectname, char*** path_list, int *size) + const char* getALVersion() diff --git a/src/al_lowlevel.cpp b/src/al_lowlevel.cpp index fa7f42ed..93b80798 100644 --- a/src/al_lowlevel.cpp +++ b/src/al_lowlevel.cpp @@ -1775,6 +1775,30 @@ al_status_t al_get_occurrences(int pctxID, const char* ids_name, int** occurrenc return status; } +al_status_t al_list_filled_paths(int pctxID, const char* dataobjectname, char*** path_list, int* size) { + al_status_t status; + + status.code = 0; + try { + LLenv lle = Lowlevel::getLLenv(pctxID); + lle.backend->list_filled_paths(lle.context, dataobjectname, path_list, size); + } + catch (const ALBackendException& e) { + status.code = alerror::backend_err; + ALException::registerStatus(status.message, __func__, e); + } + catch (const ALLowlevelException& e) { + status.code = alerror::lowlevel_err; + ALException::registerStatus(status.message, __func__, e); + } + catch (const std::exception& e) { + status.code = alerror::unknown_err; + ALException::registerStatus(status.message, __func__, e); + } + + return status; +} + al_status_t al_setvalue_parameter_plugin(const char* parameter_name, int datatype, int dim, int *size, void *data, const char* pluginName) { al_status_t status; diff --git a/src/ascii_backend.cpp b/src/ascii_backend.cpp index 40d943ed..2fa9be03 100644 --- a/src/ascii_backend.cpp +++ b/src/ascii_backend.cpp @@ -708,4 +708,7 @@ void AsciiBackend::get_occurrences(Context* ctx, const char* ids_name, int** oc (*occurrences_list)[i] = occurrences[i]; } +void AsciiBackend::list_filled_paths(Context* ctx, const char* dataobjectname, char*** path_list, int* size) { + throw ALBackendException("list_filled_paths is not implemented in the ASCII Backend", LOG); +} diff --git a/src/ascii_backend.h b/src/ascii_backend.h index fe884e5c..8ccbd26d 100644 --- a/src/ascii_backend.h +++ b/src/ascii_backend.h @@ -91,6 +91,7 @@ class IMAS_CORE_LIBRARY_API AsciiBackend : public Backend std::pair getVersion(DataEntryContext *ctx) override; void get_occurrences(Context* ctx, const char* ids_name, int** occurrences_list, int* size) override; + void list_filled_paths(Context* ctx, const char* dataobjectname, char*** path_list, int* size) override; bool supportsTimeDataInterpolation() override { return false; diff --git a/src/flexbuffers_backend.cpp b/src/flexbuffers_backend.cpp index 32f71811..75494803 100644 --- a/src/flexbuffers_backend.cpp +++ b/src/flexbuffers_backend.cpp @@ -398,6 +398,15 @@ void FlexbuffersBackend::get_occurrences( throw ALBackendException("get_occurrences is not implemented in the Serialize Backend", LOG); } +void FlexbuffersBackend::list_filled_paths( + Context* ctx, + const char* dataobjectname, + char*** path_list, + int* size +) { + throw ALBackendException("list_filled_paths is not implemented in the Serialize Backend", LOG); +} + void FlexbuffersBackend::_start_vector() { _vector_starts.push(_builder->StartVector()); } diff --git a/src/flexbuffers_backend.h b/src/flexbuffers_backend.h index 5517cb2e..7471be24 100644 --- a/src/flexbuffers_backend.h +++ b/src/flexbuffers_backend.h @@ -48,6 +48,7 @@ class IMAS_CORE_LIBRARY_API FlexbuffersBackend : public Backend void deleteData(OperationContext *ctx, std::string path) override; void beginArraystructAction(ArraystructContext *ctx, int *size) override; void get_occurrences(Context* ctx, const char* ids_name, int** occurrences_list, int* size) override; + void list_filled_paths(Context* ctx, const char* dataobjectname, char*** path_list, int* size) override; // timerange (get_sample) API is not supported: bool supportsTimeDataInterpolation() override { return false; } bool supportsTimeRangeOperation() override { return false; } diff --git a/src/hdf5/hdf5_backend.cpp b/src/hdf5/hdf5_backend.cpp index 2e4112e7..cb95f539 100644 --- a/src/hdf5/hdf5_backend.cpp +++ b/src/hdf5/hdf5_backend.cpp @@ -162,3 +162,11 @@ void HDF5Backend::get_occurrences(Context* ctx, const char* ids_name, int** occ throw ALBackendException("HDF5Backend: master file not opened while calling HDF5Backend::get_occurrences()", LOG); hdf5Reader->get_occurrences(ids_name, occurrences_list, size, file_id); } + +void HDF5Backend::list_filled_paths(Context* ctx, const char* dataobjectname, char*** path_list, int* size) +{ + if (file_id == -1) // master file not opened + throw ALBackendException("HDF5Backend: master file not opened while calling HDF5Backend::list_filled_paths()", LOG); + + hdf5Reader->list_filled_paths(dataobjectname, path_list, size, file_id, opened_IDS_files, files_directory, relative_file_path); +} diff --git a/src/hdf5/hdf5_backend.h b/src/hdf5/hdf5_backend.h index 26bdc001..96cc1e60 100644 --- a/src/hdf5/hdf5_backend.h +++ b/src/hdf5/hdf5_backend.h @@ -131,6 +131,7 @@ class HDF5Backend:public Backend { void beginAction(OperationContext * ctx) override; void get_occurrences(Context* ctx, const char* ids_name, int** occurrences_list, int* size) override; + void list_filled_paths(Context* ctx, const char* dataobjectname, char*** path_list, int* size) override; bool supportsTimeDataInterpolation() override { return true; diff --git a/src/hdf5/hdf5_reader.cpp b/src/hdf5/hdf5_reader.cpp index 4dde86c7..34881cb2 100644 --- a/src/hdf5/hdf5_reader.cpp +++ b/src/hdf5/hdf5_reader.cpp @@ -1568,6 +1568,49 @@ void HDF5Reader::get_occurrences(const char *ids_name, int **occurrences_list, i p[i] = occurrences[i]; } +void HDF5Reader::list_filled_paths(const char* dataobjectname, char*** path_list, int* size, hid_t file_id, std::unordered_map < std::string, hid_t > &opened_IDS_files, std::string & files_directory, std::string & relative_file_path) +{ + HDF5Utils hdf5_utils; + // Create temporary OperationContext to allow reusing existing logic from hdf5_utils: + OperationContext ctx(NULL, dataobjectname, "", READ_OP); + hid_t gid = -1; + hdf5_utils.open_IDS_group(&ctx, file_id, opened_IDS_files, files_directory, relative_file_path, &gid); + if (gid <= 0) { + // Group does not exist => nothing is filled + *size = 0; + return; + } + + // Create a vector of all link names in the hdf5 group + std::vector variables; + H5L_iterate_t iterate_callback = [](hid_t group_id, const char* cname, const H5L_info_t* info, void* op_data) -> herr_t { + std::string name(cname); + // Skip if name ends with _SHAPE + if (name.size() < 6 || name.substr(name.size() - 6) != "_SHAPE") + static_cast*>(op_data)->push_back(name); + return 0; // success + }; + herr_t status = H5Literate(gid, H5_INDEX_NAME, H5_ITER_NATIVE, NULL, iterate_callback, &variables); + if (status != 0) + throw ALBackendException("HDF5Backend: H5Literate has failed in HDF5Reader::get_occurrences()", LOG); + + // Convert to DD path: remove AoS brackets ([]) and use / to separate paths + for (auto &str : variables) { + // Remove square brackets + str.erase(std::remove(str.begin(), str.end(), '['), str.end()); + str.erase(std::remove(str.begin(), str.end(), ']'), str.end()); + // Replace '&' with '/' + std::replace(str.begin(), str.end(), '&', '/'); + } + + // Create the C-style array: + *size = variables.size(); + *path_list = (char**) malloc(*size * sizeof(char*)); + for (int i=0; i < *size; ++i) { + (*path_list)[i] = strdup(variables[i].c_str()); + } +} + herr_t HDF5Reader::iterate_callback(hid_t loc_id, const char *name, const H5L_info_t *info, void *callback_data) { std::vector *op = reinterpret_cast *>(callback_data); diff --git a/src/hdf5/hdf5_reader.h b/src/hdf5/hdf5_reader.h index 80ebeaba..a00874cd 100644 --- a/src/hdf5/hdf5_reader.h +++ b/src/hdf5/hdf5_reader.h @@ -89,6 +89,7 @@ class HDF5Reader { virtual int read_ND_Data(Context * ctx, std::string & att_name, std::string & timebasename, int datatype, void **data, int *dim, int *size); virtual void beginReadArraystructAction(ArraystructContext * ctx, int *size); virtual void get_occurrences(const char* ids_name, int** occurrences_list, int* size, hid_t master_file_id); + void list_filled_paths(const char* dataobjectname, char*** path_list, int* size, hid_t file_id, std::unordered_map < std::string, hid_t > &opened_IDS_files, std::string & files_directory, std::string & relative_file_path); void open_IDS_group(OperationContext * ctx, hid_t file_id, std::unordered_map < std::string, hid_t > &opened_IDS_files, std::string & files_directory, std::string & relative_file_path); void close_file_handler(std::string external_link_name, std::unordered_map < std::string, hid_t > &opened_IDS_files); diff --git a/src/mdsplus/mdsplus_backend.cpp b/src/mdsplus/mdsplus_backend.cpp index a0c73f1b..a8fe928a 100644 --- a/src/mdsplus/mdsplus_backend.cpp +++ b/src/mdsplus/mdsplus_backend.cpp @@ -4932,6 +4932,10 @@ void MDSplusBackend::get_occurrences(Context* ctx, const char* ids_name, int** o *size = occurrences.size(); } +void MDSplusBackend::list_filled_paths(Context* ctx, const char* dataobjectname, char*** path_list, int* size) { + throw ALBackendException("list_filled_paths is not implemented in the MDSplus Backend", LOG); +} + void MDSplusBackend::fullPath(Context *ctx, std::string &path) { if (ctx->getType() == CTX_PULSE_TYPE) { path = ""; diff --git a/src/mdsplus/mdsplus_backend.h b/src/mdsplus/mdsplus_backend.h index 75617518..a9dcb524 100644 --- a/src/mdsplus/mdsplus_backend.h +++ b/src/mdsplus/mdsplus_backend.h @@ -250,6 +250,7 @@ class IMAS_CORE_LIBRARY_API MDSplusBackend:public Backend std::pair getVersion(DataEntryContext *ctx) override; void get_occurrences(Context* ctx, const char* ids_name, int** occurrences_list, int* size) override; + void list_filled_paths(Context* ctx, const char* dataobjectname, char*** path_list, int* size) override; bool supportsTimeDataInterpolation() { return true; diff --git a/src/memory_backend.cpp b/src/memory_backend.cpp index 9b380fb3..8009d20e 100644 --- a/src/memory_backend.cpp +++ b/src/memory_backend.cpp @@ -1148,7 +1148,10 @@ else throw ALBackendException(message, LOG); } - + void MemoryBackend::list_filled_paths(Context* ctx, const char* dataobjectname, char*** path_list, int* size) { + std::string message("list_filled_paths() is not implemented in the MemoryBackend"); + throw ALBackendException(message, LOG); + } ////////////////////////////////////////////////////////////////////////// diff --git a/src/memory_backend.h b/src/memory_backend.h index e74bcb92..c0d22ec9 100644 --- a/src/memory_backend.h +++ b/src/memory_backend.h @@ -673,6 +673,7 @@ class IMAS_CORE_LIBRARY_API MemoryBackend:public Backend ALData *getAlSlice(ArraystructContext *ctx, ALData &inData, double time, std::vector timebaseV); void get_occurrences(Context* ctx, const char* ids_name, int** occurrences_list, int* size) override; + void list_filled_paths(Context* ctx, const char* dataobjectname, char*** path_list, int* size) override; bool supportsTimeDataInterpolation() override { return false; diff --git a/src/no_backend.cpp b/src/no_backend.cpp index 2b3807cb..222f21c2 100644 --- a/src/no_backend.cpp +++ b/src/no_backend.cpp @@ -85,3 +85,11 @@ void NoBackend::get_occurrences(Context* ctx, const char* ids_name, int** occurr *size = 0; } +void NoBackend::list_filled_paths(Context* ctx, const char* dataobjectname, char*** path_list, int* size) +{ + if (verbose) { + std::string message("list_filled_paths() is not implemented in the NoBackend"); + throw ALBackendException(message, LOG); + } + *size = 0; +} diff --git a/src/no_backend.h b/src/no_backend.h index 8b46e6b2..2162632a 100644 --- a/src/no_backend.h +++ b/src/no_backend.h @@ -64,6 +64,8 @@ class IMAS_CORE_LIBRARY_API NoBackend : public Backend void get_occurrences(Context* ctx, const char* ids_name, int** occurrences_list, int* size) override; + void list_filled_paths(Context* ctx, const char* dataobjectname, char*** path_list, int* size) override; + bool supportsTimeDataInterpolation() override { return false; } diff --git a/src/uda/uda_backend.cpp b/src/uda/uda_backend.cpp index b1ccb81b..74a538b3 100644 --- a/src/uda/uda_backend.cpp +++ b/src/uda/uda_backend.cpp @@ -1310,3 +1310,6 @@ void UDABackend::get_occurrences(Context* ctx, const char* ids_name, int** occur } } +void UDABackend::list_filled_paths(Context* ctx, const char* dataobjectname, char*** path_list, int* size) { + throw ALBackendException("list_filled_paths is not implemented in the UDA Backend", LOG); +} diff --git a/src/uda/uda_backend.h b/src/uda/uda_backend.h index 0869aa3c..115e8d5b 100644 --- a/src/uda/uda_backend.h +++ b/src/uda/uda_backend.h @@ -183,6 +183,7 @@ class IMAS_CORE_LIBRARY_API UDABackend : public Backend std::pair getVersion(DataEntryContext *ctx) override; void get_occurrences(Context* ctx, const char* ids_name, int** occurrences_list, int* size) override; + void list_filled_paths(Context* ctx, const char* dataobjectname, char*** path_list, int* size) override; bool supportsTimeDataInterpolation() override;