From 5cb4e5aaa53a2747ee8616bd95cbdfff8826a5f4 Mon Sep 17 00:00:00 2001 From: JoshuaSBrown Date: Thu, 5 Feb 2026 12:58:26 -0500 Subject: [PATCH 01/14] refactor: python package to be compatible with proto3 envelope --- python/datafed_pkg/datafed/CMakeLists.txt | 26 ++++------ python/datafed_pkg/datafed/Connection.py | 63 ++++++++++++++++++----- python/datafed_pkg/datafed/MessageLib.py | 20 ++++--- python/pyproto_add_msg_idx.py | 61 ---------------------- 4 files changed, 71 insertions(+), 99 deletions(-) delete mode 100755 python/pyproto_add_msg_idx.py diff --git a/python/datafed_pkg/datafed/CMakeLists.txt b/python/datafed_pkg/datafed/CMakeLists.txt index 9e066deea..da7b92b73 100644 --- a/python/datafed_pkg/datafed/CMakeLists.txt +++ b/python/datafed_pkg/datafed/CMakeLists.txt @@ -6,12 +6,13 @@ foreach(file ${SrcFiles}) configure_file(${file} ${CMAKE_CURRENT_BINARY_DIR} COPYONLY ) endforeach() -# Collect top-level proto files as dependencies -file( GLOB ProtoFiles ${DataFed_SOURCE_DIR}/common/proto/common/*.proto ) +# Collect proto files from the new 1-1-1 directory structure +file( GLOB_RECURSE ProtoFiles ${DataFed_SOURCE_DIR}/common/proto/common/*.proto ) # OBJECT - is needed because we don't want to compile to a binary # because we are dealing with python add_library(protobuf-target-py OBJECT ${ProtoFiles}) + protobuf_generate( LANGUAGE python TARGET protobuf-target-py @@ -22,21 +23,16 @@ protobuf_generate( add_custom_target( pydatafed_src DEPENDS protobuf-target-py ) -# By default this will output the proto py files in the CMAKE BINARY DIR +# Fix relative imports in generated pb2 files +# No longer need pyproto_add_msg_idx.py — message type IDs are derived +# at runtime from envelope field numbers via registerEnvelope() add_custom_command( TARGET pydatafed_src POST_BUILD COMMAND sed -i -r 's:^import.*_pb2:from . \\0:' ${protobuf-generated-files-py} - COMMAND ${DataFed_SOURCE_DIR}/python/pyproto_add_msg_idx.py ${DataFed_SOURCE_DIR}/common/proto/common/SDMS_Anon.proto ${CMAKE_CURRENT_BINARY_DIR}/SDMS_Anon_pb2.py - COMMAND ${DataFed_SOURCE_DIR}/python/pyproto_add_msg_idx.py ${DataFed_SOURCE_DIR}/common/proto/common/SDMS_Auth.proto ${CMAKE_CURRENT_BINARY_DIR}/SDMS_Auth_pb2.py -) - -# Crea#te copies of the files so they show up in the source folder as well +) + +# Create copies of the files so they show up in the source folder as well # for the purpose of testing add_custom_target( pydatafed_proto_src DEPENDS pydatafed_src ) -add_custom_command( TARGET pydatafed_proto_src POST_BUILD pydatafed_src - COMMAND cp ${CMAKE_CURRENT_BINARY_DIR}/SDMS_Auth_pb2.py ${CMAKE_CURRENT_SOURCE_DIR}/ - COMMAND cp ${CMAKE_CURRENT_BINARY_DIR}/SDMS_pb2.py ${CMAKE_CURRENT_SOURCE_DIR}/ - COMMAND cp ${CMAKE_CURRENT_BINARY_DIR}/Version_pb2.py ${CMAKE_CURRENT_SOURCE_DIR}/ - COMMAND cp ${CMAKE_CURRENT_BINARY_DIR}/SDMS_Anon_pb2.py ${CMAKE_CURRENT_SOURCE_DIR}/ +add_custom_command( TARGET pydatafed_proto_src POST_BUILD pydatafed_src + COMMAND cp ${CMAKE_CURRENT_BINARY_DIR}/envelope_pb2.py ${CMAKE_CURRENT_SOURCE_DIR}/ ) - - diff --git a/python/datafed_pkg/datafed/Connection.py b/python/datafed_pkg/datafed/Connection.py index d34dd1e1c..4ee0e8885 100644 --- a/python/datafed_pkg/datafed/Connection.py +++ b/python/datafed_pkg/datafed/Connection.py @@ -6,13 +6,11 @@ # unserialized, and custom framing is generated to efficiently convey message # type, size, and a re-association context value. # -# The Google protobuf library does not provide a mechanism for identifying -# message types numerically (only by string), so a build-time custom tool -# (pyproto_add_msg_idx.py) is used to generate the mappings from message -# names to message index (and vice versa) and appends this information as -# dictionaries to the compiled proto files (xxxx_pb2.py). The -# registerProtocol() method then loads uses this information to create -# consistent message type framing for python send/recv methods. +# Message type identification is derived at runtime from the Envelope proto +# message's field descriptors. Each message type has a stable field number +# in the Envelope, which serves as its wire-format type ID. This replaces +# the previous build-time pyproto_add_msg_idx.py hack that assigned type +# IDs based on message declaration order within proto files. from google.protobuf.message_factory import GetMessageClass import logging @@ -116,19 +114,60 @@ def __del__(self): self._zmq_ctxt.destroy() ## - # @brief Register a protobuf module + # @brief Register message types from the Envelope proto message + # + # This method derives message type mappings at runtime by inspecting the + # Envelope message's field descriptors. Each field in the Envelope that + # wraps a message type has a stable field number, which becomes the + # message type ID used in wire framing. This replaces the old + # registerProtocol() approach that relied on build-time generated + # _msg_name_to_type / _msg_type_to_name dicts. + # + # @param envelope_module - The compiled envelope_pb2 module + # @param envelope_class_name - Name of the envelope message (default: "Envelope") + # + def registerEnvelope(self, envelope_module, envelope_class_name="Envelope"): + envelope_class = getattr(envelope_module, envelope_class_name) + envelope_desc = envelope_class.DESCRIPTOR + + for field in envelope_desc.fields: + if field.message_type is None: + # Skip non-message fields (e.g. scalars) if any exist + continue + + msg_type = field.number + desc = field.message_type + + self._msg_desc_by_type[msg_type] = desc + self._msg_desc_by_name[desc.name] = desc + self._msg_type_by_desc[desc] = msg_type + + self._logger.debug( + "Registered %d message types from %s", + len(self._msg_desc_by_type), + envelope_class_name, + ) + + ## + # @brief Register a protobuf module (DEPRECATED - use registerEnvelope) # # This method registers an imported protobuf module (_pb2 file) for use # with the Connection class. Registration is required for proper message # framing and serialization. # + # This relies on build-time generated _msg_name_to_type dicts appended + # to _pb2 files by pyproto_add_msg_idx.py. Prefer registerEnvelope() + # which derives mappings from envelope field numbers at runtime. + # # @param msg_module - Protobuf module (imported *_pb2 module) # def registerProtocol(self, msg_module): - # Message descriptors are stored by name created by protobuf compiler - # A custom post-proc tool generates and appends _msg_name_to_type with - # defined DataFed-sepcific numer message types - + import warnings + warnings.warn( + "registerProtocol() is deprecated, use registerEnvelope() instead", + DeprecationWarning, + stacklevel=2, + ) for name, desc in sorted(msg_module.DESCRIPTOR.message_types_by_name.items()): msg_t = msg_module._msg_name_to_type[name] self._msg_desc_by_type[msg_t] = desc diff --git a/python/datafed_pkg/datafed/MessageLib.py b/python/datafed_pkg/datafed/MessageLib.py index 62354c0eb..69cb2c9d7 100644 --- a/python/datafed_pkg/datafed/MessageLib.py +++ b/python/datafed_pkg/datafed/MessageLib.py @@ -13,8 +13,7 @@ import zmq from . import Version_pb2 -from . import SDMS_Anon_pb2 as anon -from . import SDMS_Auth_pb2 as auth +from . import envelope_pb2 as proto from . import Connection from . import VERSION @@ -166,8 +165,7 @@ def __init__( server_host, server_port, _server_pub_key, _client_pub_key, _client_priv_key ) - self._conn.registerProtocol(anon) - self._conn.registerProtocol(auth) + self._conn.registerEnvelope(proto) # Make a request to pypi package_name = "datafed" # Replace with the package name you want to check @@ -191,7 +189,7 @@ def __init__( self.new_client_avail = latest_version_on_pypi # Check for compatible protocol versions - reply, mt = self.sendRecv(anon.VersionRequest(), 10000) + reply, mt = self.sendRecv(proto.VersionRequest(), 10000) if reply is None: raise Exception( "Timeout waiting for server connection. Make sure" @@ -223,7 +221,7 @@ def __init__( self.manualAuthByToken(client_token) else: # Check if server authenticated based on keys - reply, mt = self.sendRecv(anon.GetAuthStatusRequest(), 10000) + reply, mt = self.sendRecv(proto.GetAuthStatusRequest(), 10000) self._auth = reply.auth self._uid = reply.uid @@ -263,7 +261,7 @@ def getAuthStatus(self): # @exception Exception: On communication timeout or authentication failure. # def manualAuthByPassword(self, uid, password): - msg = anon.AuthenticateByPasswordRequest() + msg = proto.AuthenticateByPasswordRequest() msg.uid = uid msg.password = password a, b = self.sendRecv(msg) @@ -272,7 +270,7 @@ def manualAuthByPassword(self, uid, password): self._conn.reset() # Test auth status - reply, mt = self.sendRecv(anon.GetAuthStatusRequest()) + reply, mt = self.sendRecv(proto.GetAuthStatusRequest()) if not reply.auth: raise Exception("Password authentication failed.") @@ -280,7 +278,7 @@ def manualAuthByPassword(self, uid, password): self._uid = reply.uid def manualAuthByToken(self, token): - msg = anon.AuthenticateByTokenRequest() + msg = proto.AuthenticateByTokenRequest() msg.token = token self.sendRecv(msg) @@ -288,7 +286,7 @@ def manualAuthByToken(self, token): self._conn.reset() # Test auth status - reply, mt = self.sendRecv(anon.GetAuthStatusRequest()) + reply, mt = self.sendRecv(proto.GetAuthStatusRequest()) if not reply.auth: raise Exception("Token authentication failed") @@ -332,7 +330,7 @@ def getDefaultTimeout(self): def getDailyMessage(self): # Get daily message, if set - reply, mt = self.sendRecv(anon.DailyMessageRequest(), 10000) + reply, mt = self.sendRecv(proto.DailyMessageRequest(), 10000) if reply is None: raise Exception("Timeout waiting for server connection.") diff --git a/python/pyproto_add_msg_idx.py b/python/pyproto_add_msg_idx.py deleted file mode 100755 index 233f83b33..000000000 --- a/python/pyproto_add_msg_idx.py +++ /dev/null @@ -1,61 +0,0 @@ -#!/usr/bin/env python3 - -""" -Protobuf processing to generate message ID maps for C++, Python, and JS -""" - -import sys -import re - -print("args", sys.argv) - -pf_in = open(sys.argv[1], "r") -pf_out = open(sys.argv[2], "a") - -while True: - line = pf_in.readline() - if len(line) == 0: - sys.exit(-1) - parts = re.split(r"\W+", line.strip()) - # print( line, parts ) - try: - idx = parts.index("ID") - # print( "ID:", parts[idx+1] ) - msg_type = int(parts[idx + 1]) << 8 - break - except BaseException: - pass - -# msg_type = 0 - -by_type = [] -idx = 0 - -pf_out.write("\n_msg_name_to_type = {\n") - -while True: - line = pf_in.readline() - if len(line) == 0: - break - - if line.startswith("message "): - msg_name = line.split()[1] - by_type.append(msg_name) - # print( msg_name, msg_type ) - if idx > 0: - pf_out.write(",\n") - pf_out.write(" '{}' : {}".format(msg_name, msg_type | idx)) - idx += 1 - -pf_out.write("\n}\n\n_msg_type_to_name = {\n") - -idx = 0 -for name in by_type: - if idx > 0: - pf_out.write(",\n") - pf_out.write(" {} : '{}'".format(msg_type | idx, name)) - idx += 1 - -pf_out.write("\n}\n") - -sys.exit(0) From 32ef6a2ec4ada1ca0e72300a35de24dfc4cb4ee7 Mon Sep 17 00:00:00 2001 From: JoshuaSBrown Date: Thu, 5 Feb 2026 16:49:16 -0500 Subject: [PATCH 02/14] refactor: improve docs, address double wrapping env, moderate test timeouts. --- common/include/common/IMessage.hpp | 204 ++++++++++++++++++++--- common/include/common/ProtoBufMap.hpp | 150 +++++++++++++++-- common/source/Frame.hpp | 114 +++++++++++-- common/source/ProtoBufMap.cpp | 6 + common/tests/unit/test_Proxy.cpp | 4 +- common/tests/unit/test_ProxyBasicZMQ.cpp | 4 +- 6 files changed, 425 insertions(+), 57 deletions(-) diff --git a/common/include/common/IMessage.hpp b/common/include/common/IMessage.hpp index 679d054ec..71aa5bbc7 100644 --- a/common/include/common/IMessage.hpp +++ b/common/include/common/IMessage.hpp @@ -17,16 +17,45 @@ class Message; namespace SDMS { -enum class MessageType { GOOGLE_PROTOCOL_BUFFER, STRING }; +/** + * @enum MessageType + * @brief Discriminator for the payload type carried by an IMessage. + */ +enum class MessageType { + GOOGLE_PROTOCOL_BUFFER, ///< Payload is a google::protobuf::Message. + STRING ///< Payload is a plain std::string. +}; /** - * The message is on its way to a server this it is a REQUEST - * The message is on its way from a server then it is a RESPONSE - **/ + * @enum MessageState + * @brief Indicates the directional state of a message relative to a server. + * + * - @c REQUEST — the message is traveling *toward* a server. + * - @c RESPONSE — the message is traveling *from* a server. + */ enum class MessageState { REQUEST, RESPONSE }; -enum class MessageAttribute { ID, KEY, STATE, CORRELATION_ID }; +/** + * @enum MessageAttribute + * @brief Named keys for the core metadata attributes stored on a message. + */ +enum class MessageAttribute { + ID, ///< Unique message identifier. + KEY, ///< Routing or lookup key. + STATE, ///< Current MessageState (REQUEST or RESPONSE). + CORRELATION_ID ///< Identifier used to correlate requests with responses. +}; +/** + * @brief Convert a MessageAttribute to its human-readable string form. + * + * @param[in] attribute The attribute to stringify. + * @return A string representation, or @c "unsupported_toString_print" if + * the attribute does not have a defined conversion. + * + * @note Only ID and KEY are currently supported; all other values fall + * through to the unsupported case. + */ inline const std::string toString(const MessageAttribute attribute) { if (attribute == MessageAttribute::ID) { return std::string("ID"); @@ -37,65 +66,186 @@ inline const std::string toString(const MessageAttribute attribute) { } } +/// @brief Well-known string constants used throughout the messaging layer. namespace constants { namespace message { namespace google { -/// Supported dynamic arguments +/// @brief Key for the serialized frame size of a protobuf message. const std::string FRAME_SIZE = "frame_size"; +/// @brief Key for the integral message-type identifier (e.g. envelope field number). const std::string MSG_TYPE = "msg_type"; +/// @brief Key for the security/session context attached to a message. const std::string CONTEXT = "context"; } // namespace google } // namespace message } // namespace constants +/** + * @class IMessage + * @brief Abstract interface for all messages exchanged through the SDMS + * messaging infrastructure. + * + * IMessage defines a uniform contract for: + * - **Payload management** — carrying either a protobuf Message or a raw + * string, with ownership semantics enforced by the implementation. + * - **Routing** — an ordered list of route identifiers that determines + * how a message is forwarded through the system. + * - **Metadata** — core attributes (ID, KEY, STATE, CORRELATION_ID) plus + * arbitrary named attributes with small-integer values. + * + * Implementations are expected to own the payload and manage its lifetime. + * Callers retrieve payload data via non-owning raw pointers or copies. + */ class IMessage { public: - + /// @brief Virtual destructor. virtual ~IMessage() {}; - virtual bool exists(MessageAttribute) const = 0; - virtual bool exists(const std::string &) const = 0; /** + * @brief Check whether a core metadata attribute has been set. + * + * @param[in] attribute The attribute to query. + * @return @c true if the attribute is present on this message. + */ + virtual bool exists(MessageAttribute attribute) const = 0; + + /** + * @brief Check whether a named (dynamic) attribute has been set. + * + * @param[in] attribute_name The name of the dynamic attribute to query. + * @return @c true if the named attribute is present on this message. + */ + virtual bool exists(const std::string &attribute_name) const = 0; + + /*--------------------------------------------------------------------- * Setters - **/ + *-------------------------------------------------------------------*/ + + /** + * @brief Set the message payload. + * + * The implementation must take ownership of the supplied payload. For + * protobuf payloads the unique_ptr is moved in; for string payloads the + * string is copied or moved. After this call the IMessage instance is + * solely responsible for the lifetime of the payload data. + * + * @param[in] payload A variant holding either a protobuf message + * (via @c std::unique_ptr) or a plain string. + */ + virtual void setPayload( + std::variant, + std::string> + payload) = 0; /** - * Adding a payload should make a copy and store internally. It should - * Imply ownership of the payload and it's memory management. - **/ - virtual void - setPayload(std::variant, - std::string>) = 0; + * @brief Append a single route to the end of the routing list. + * + * @param[in] route The route identifier to append. + */ virtual void addRoute(const std::string &route) = 0; + /** + * @brief Replace the entire routing list. + * + * @param[in] routes The new ordered list of route identifiers. + */ virtual void setRoutes(const std::list &routes) = 0; - virtual void set(MessageAttribute, const std::string &) = 0; - virtual void set(MessageAttribute, MessageState) = 0; + /** + * @brief Set a core metadata attribute to a string value. + * + * Applicable to attributes such as @c ID, @c KEY, and + * @c CORRELATION_ID. + * + * @param[in] attribute The attribute to set. + * @param[in] value The string value to assign. + */ + virtual void set(MessageAttribute attribute, const std::string &value) = 0; - virtual void set(std::string attribute_name, - std::variant) = 0; /** + * @brief Set a core metadata attribute to a MessageState value. + * + * Intended for the @c STATE attribute. + * + * @param[in] attribute The attribute to set (expected: @c STATE). + * @param[in] state The MessageState value to assign. + */ + virtual void set(MessageAttribute attribute, MessageState state) = 0; + + /** + * @brief Set a named dynamic attribute to a small unsigned integer. + * + * Dynamic attributes are identified by string keys (e.g. the constants + * in @c SDMS::constants::message::google). + * + * @param[in] attribute_name The name of the dynamic attribute. + * @param[in] value The value, stored as one of uint8/16/32. + */ + virtual void set(std::string attribute_name, + std::variant value) = 0; + + /*--------------------------------------------------------------------- * Getters - **/ + *-------------------------------------------------------------------*/ /** - * The correlation ID is assigned to a message when it is created and is - *extremely important for tracing a message in the logs. - **/ + * @brief Retrieve a core metadata attribute. + * + * The returned variant holds a @c std::string for most attributes, or a + * @c MessageState when querying @c STATE. + * + * @param[in] attribute The attribute to retrieve. + * @return A variant containing either the string value or the + * MessageState, depending on the attribute. + */ virtual std::variant - get(MessageAttribute) const = 0; + get(MessageAttribute attribute) const = 0; + + /** + * @brief Get a const reference to the ordered routing list. + * + * @return Const reference to the internal route list. + */ virtual const std::list &getRoutes() const = 0; + + /** + * @brief Get a mutable reference to the ordered routing list. + * + * @return Mutable reference to the internal route list. + */ virtual std::list &getRoutes() = 0; + + /** + * @brief Return the payload type carried by this message. + * + * @return The MessageType discriminator. + */ virtual MessageType type() const noexcept = 0; + + /** + * @brief Retrieve a named dynamic attribute. + * + * @param[in] attribute_name The name of the dynamic attribute. + * @return The value stored as a @c uint8_t, @c uint16_t, or @c uint32_t + * variant. + */ virtual std::variant get(const std::string &attribute_name) const = 0; - /// Note not returning a unique_ptr but a raw pointer because the message - // should stil have ownership of the object. + /** + * @brief Retrieve a non-owning handle to the message payload. + * + * Ownership remains with the IMessage instance. For protobuf payloads + * a raw pointer is returned (not a @c unique_ptr) to make this + * explicit. For string payloads the string is returned by value (copy). + * + * @return A variant holding either a non-owning + * @c google::protobuf::Message* or a @c std::string. + */ virtual std::variant getPayload() = 0; }; } // namespace SDMS + #endif // MESSAGE_HPP diff --git a/common/include/common/ProtoBufMap.hpp b/common/include/common/ProtoBufMap.hpp index b1da59161..e30f62b90 100644 --- a/common/include/common/ProtoBufMap.hpp +++ b/common/include/common/ProtoBufMap.hpp @@ -1,3 +1,14 @@ +/** + * @file ProtoBufMap.hpp + * @brief Provides message type mapping and envelope wrap/unwrap for the SDMS + * protobuf messaging layer. + * + * ProtoBufMap maintains bidirectional mappings between envelope field numbers + * (used as stable message type identifiers) and protobuf Descriptor objects. + * It also handles serialization boundary concerns via the Envelope message + * pattern, wrapping outgoing messages and unwrapping incoming ones. + */ + #ifndef PROTOBUFMAP_HPP #define PROTOBUFMAP_HPP #pragma once @@ -17,47 +28,154 @@ namespace SDMS { +/** + * @class ProtoBufMap + * @brief Bidirectional message type registry and envelope serialization layer. + * + * Maps between uint16_t message type identifiers (derived from envelope field + * numbers) and protobuf Descriptor/FileDescriptor objects. Implements the + * IMessageMapper interface and provides envelope wrap/unwrap operations for + * wire-boundary serialization. + * + * Message type identifiers are organized by category: + * - 10–29: Anonymous (no-auth) messages + * - 200–249: Record messages + * - etc. + * + * @see IMessageMapper + * @see SDMS::Envelope + */ class ProtoBufMap : public IMessageMapper { public: + /** @brief Maps a file index to its protobuf FileDescriptor. */ typedef std::map FileDescriptorMap; + + /** @brief Maps a message type ID (envelope field number) to its Descriptor. */ typedef std::map DescriptorMap; + + /** @brief Reverse mapping from Descriptor back to message type ID. */ typedef std::map MsgTypeMap; private: - FileDescriptorMap m_file_descriptor_map; - DescriptorMap m_descriptor_map; - MsgTypeMap m_msg_type_map; + FileDescriptorMap m_file_descriptor_map; ///< Registered file descriptors. + DescriptorMap m_descriptor_map; ///< Type ID → Descriptor lookup. + MsgTypeMap m_msg_type_map; ///< Descriptor → type ID lookup. public: + /** + * @brief Constructs the map and populates all mappings from the Envelope + * descriptor's oneof fields. + * + * Iterates the Envelope message descriptor via reflection to register every + * message field, using each field's number as the stable type identifier. + */ ProtoBufMap(); + /** + * @brief Retrieves the Descriptor for a given message type ID. + * + * @param message_type Envelope field number identifying the message type. + * @return Pointer to the Descriptor, or nullptr if not found. + */ const ::google::protobuf::Descriptor * getDescriptorType(uint16_t message_type) const; + + /** + * @brief Checks whether a message type ID is registered. + * + * @param message_type Envelope field number to look up. + * @return True if the type is registered, false otherwise. + */ bool exists(uint16_t message_type) const; - uint16_t getMessageType(const ::google::protobuf::Message& msg) const; + + /** + * @brief Resolves the message type ID for a concrete protobuf message. + * + * @param msg A protobuf message instance. + * @return The envelope field number corresponding to the message's type. + * @throws std::runtime_error If the message type is not registered. + */ + uint16_t getMessageType(const ::google::protobuf::Message &msg) const; + + /** + * @brief Returns a human-readable name for a message type ID. + * + * @param MessageType Envelope field number identifying the message type. + * @return The full protobuf type name, or an error string if not found. + */ std::string toString(uint16_t MessageType) const; - virtual uint16_t getMessageType(const std::string& message_name) const final; - - // Envelope wrap/unwrap for wire boundary + + /** + * @brief Resolves a message type ID from a protobuf message type name string. + * + * @param message_name Fully-qualified or short protobuf message name. + * @return The corresponding envelope field number. + * @throws std::runtime_error If the name does not match any registered type. + */ + virtual uint16_t + getMessageType(const std::string &message_name) const final; + + /** + * @brief Wraps an inner message into an Envelope for wire transmission. + * + * Resolves the inner message's type ID via getMessageType(), then uses + * reflection to set the corresponding oneof field in a new Envelope by + * copying from the inner message. + * + * @param inner The message to wrap. + * @return A populated Envelope ready for serialization. + * @throws EC_INVALID_PARAM If the resolved field number does not exist + * in the Envelope descriptor. + */ std::unique_ptr - wrapInEnvelope(const ::google::protobuf::Message& inner) const; + wrapInEnvelope(const ::google::protobuf::Message &inner) const; + /** + * @brief Extracts the inner message from a received Envelope. + * + * Uses reflection to determine which oneof field is set, then calls + * ReleaseMessage() to transfer ownership of the inner message out of + * the envelope. After this call, the released field in @p envelope is + * cleared. + * + * @param envelope The received Envelope to unwrap. Modified in place — + * the released field is no longer owned by the envelope. + * @return The extracted inner message. Caller takes ownership. + * @throws EC_INVALID_PARAM If the envelope's message type cannot be + * resolved to a valid field. + */ std::unique_ptr<::google::protobuf::Message> - unwrapFromEnvelope(SDMS::Envelope& envelope) const; + unwrapFromEnvelope(SDMS::Envelope &envelope) const; - // Check if message type requires authentication - virtual bool requiresAuth(const std::string& msg_type) const final { + /** + * @brief Determines whether a message type requires authentication. + * + * Messages in the anonymous set (e.g., version handshake, authentication + * requests) do not require a prior authenticated session. All other + * message types do. + * + * @param msg_type Short message type name (e.g., "VersionRequest"). + * @return True if the message requires an authenticated session, false if + * it is in the anonymous (no-auth) set. + */ + virtual bool requiresAuth(const std::string &msg_type) const final { static const std::unordered_set anon_types = { - "AckReply", "NackReply", "VersionRequest", "VersionReply", - "GetAuthStatusRequest", "AuthenticateByPasswordRequest", - "AuthenticateByTokenRequest", "AuthStatusReply", - "DailyMessageRequest", "DailyMessageReply" - }; + "AckReply", + "NackReply", + "VersionRequest", + "VersionReply", + "GetAuthStatusRequest", + "AuthenticateByPasswordRequest", + "AuthenticateByTokenRequest", + "AuthStatusReply", + "DailyMessageRequest", + "DailyMessageReply"}; return anon_types.count(msg_type) == 0; } }; + } // namespace SDMS #endif // PROTOBUFMAP_HPP diff --git a/common/source/Frame.hpp b/common/source/Frame.hpp index 5c68a2a25..aaa263ac4 100644 --- a/common/source/Frame.hpp +++ b/common/source/Frame.hpp @@ -1,3 +1,14 @@ +/** + * @file Frame.hpp + * @brief Wire-level frame header and conversion utilities for the SDMS + * messaging transport layer. + * + * Defines the fixed-size Frame header that prefixes every message on the wire, + * along with FrameConverter (for copying between Frame and ZMQ/IMessage + * representations) and FrameFactory (for constructing Frame headers from + * various message sources). + */ + #ifndef FRAME_HPP #define FRAME_HPP #pragma once @@ -16,36 +27,119 @@ namespace SDMS { // Forward declarations class ProtoBufMap; +/** + * @struct Frame + * @brief Fixed-size 8-byte header that precedes every message on the wire. + * + * Contains the serialized payload size, the message type identifier (envelope + * field number), and an optional application-defined context value. Fields are + * converted to/from network byte order at the ZMQ serialization boundary. + */ struct Frame { + uint32_t size = 0; ///< Size of the serialized payload in bytes. + uint16_t msg_type = 0; ///< Message type ID (envelope field number). + uint16_t context = 0; ///< Optional application-defined context value. - uint32_t size = 0; ///< Size of buffer in bytes - uint16_t msg_type = 0; - uint16_t context = 0; ///< Optional context value - + /** + * @brief Resets all fields to zero. + */ void clear() { size = 0; msg_type = 0; context = 0; } - }; +/** + * @class FrameConverter + * @brief Copies data between Frame headers and ZMQ or IMessage representations. + * + * Provides bidirectional conversion for ZMQ messages. The IMessage overload + * only supports the FROM_FRAME direction (Frame → IMessage); the reverse + * is unsupported and will throw. + */ class FrameConverter { public: - /** - * Make sure that zmq_msg_init is not called on this message before it - * is passed in. - **/ - enum class CopyDirection { TO_FRAME, FROM_FRAME }; + /** @brief Specifies the direction of a copy operation. */ + enum class CopyDirection { + TO_FRAME, ///< Deserialize: copy from source into Frame. + FROM_FRAME ///< Serialize: copy from Frame into destination. + }; + /** + * @brief Copies between a raw ZMQ message buffer and a Frame. + * + * The ZMQ message must be pre-initialized and sized to exactly 8 bytes + * (sizeof(Frame)) before calling. For the FROM_FRAME direction, use: + * @code + * zmq_msg_init_size(&zmq_msg, 8); + * @endcode + * + * @param copy Direction of the copy operation. + * @param zmq_msg Pre-initialized ZMQ message of exactly 8 bytes. + * @param frame Frame to populate (TO_FRAME) or read from (FROM_FRAME). + * + * @throws TraceException if zmq_msg size != sizeof(Frame). + */ void copy(CopyDirection copy, zmq_msg_t &zmq_msg, Frame &frame); + + /** + * @brief Copies frame fields from a Frame into an IMessage. + * + * @note Only CopyDirection::FROM_FRAME is supported. Passing TO_FRAME + * will throw. + * + * @param copy Must be CopyDirection::FROM_FRAME. + * @param msg IMessage to populate with frame metadata. + * @param frame Source Frame to read from. + * + * @throws TraceException if direction is TO_FRAME. + */ void copy(CopyDirection copy, IMessage &msg, const Frame &frame); }; +/** + * @class FrameFactory + * @brief Constructs Frame headers from various message representations. + * + * Each overload computes the payload size and resolves the message type ID + * appropriate to the source type. The context field is only populated when + * constructing from an IMessage that carries it. + */ class FrameFactory { public: + /** + * @brief Creates a Frame from a protobuf Message, resolving the type ID + * via ProtoBufMap. + * + * The context field is left at its default (0). + * + * @param a_msg Protobuf message (ByteSizeLong() determines frame size). + * @param proto_map Registry used to look up the message type ID. + * @return A Frame with size and msg_type populated. + */ Frame create(::google::protobuf::Message &a_msg, ProtoBufMap &proto_map); + + /** + * @brief Creates a Frame by extracting metadata from an IMessage. + * + * All three fields (FRAME_SIZE, MSG_TYPE, CONTEXT) must be present in the + * IMessage or the call will throw. + * + * @param msg The IMessage containing frame metadata. + * @return A fully populated Frame. + * @throws TraceException if any required field is missing from the IMessage. + */ Frame create(const IMessage &msg); + + /** + * @brief Creates a Frame by deserializing a raw ZMQ message. + * + * Returns a zeroed Frame if the ZMQ message is empty. + * + * @param zmq_msg Raw ZMQ message containing serialized frame bytes. + * @return The deserialized Frame, or a zeroed Frame if empty. + */ Frame create(zmq_msg_t &zmq_msg); }; diff --git a/common/source/ProtoBufMap.cpp b/common/source/ProtoBufMap.cpp index a24b7b47e..731009370 100644 --- a/common/source/ProtoBufMap.cpp +++ b/common/source/ProtoBufMap.cpp @@ -16,6 +16,12 @@ ProtoBufMap::ProtoBufMap() {} std::unique_ptr ProtoBufMap::wrapInEnvelope(const proto::Message& inner) const { + // If already an Envelope, return a copy — no double-wrapping + const auto* env = dynamic_cast(&inner); + if (env) { + return std::make_unique(*env); + } + uint16_t field_number = getMessageType(inner); auto envelope = std::make_unique(); const auto* field_desc = diff --git a/common/tests/unit/test_Proxy.cpp b/common/tests/unit/test_Proxy.cpp index 62721e2d4..b4ab4c4fe 100644 --- a/common/tests/unit/test_Proxy.cpp +++ b/common/tests/unit/test_Proxy.cpp @@ -228,7 +228,7 @@ BOOST_AUTO_TEST_CASE(testing_Proxy) { std::move(incoming_operators), log_context_proxy); std::chrono::duration duration = - std::chrono::milliseconds(2000); + std::chrono::milliseconds(1000); proxy.setRunDuration(duration); proxy.run(); @@ -448,7 +448,7 @@ BOOST_AUTO_TEST_CASE(testing_Proxy2) { log_context_proxy_middle); std::chrono::duration duration = - std::chrono::milliseconds(800); + std::chrono::milliseconds(400); proxy.setRunDuration(duration); proxy.run(); diff --git a/common/tests/unit/test_ProxyBasicZMQ.cpp b/common/tests/unit/test_ProxyBasicZMQ.cpp index 90f49a454..a5e319ddd 100644 --- a/common/tests/unit/test_ProxyBasicZMQ.cpp +++ b/common/tests/unit/test_ProxyBasicZMQ.cpp @@ -218,7 +218,7 @@ BOOST_AUTO_TEST_CASE(testing_ProxyBasicZMQ) { log_context1); std::chrono::duration duration = - std::chrono::milliseconds(2000); + std::chrono::milliseconds(1000); proxy.setRunDuration(duration); proxy.run(); @@ -261,7 +261,7 @@ BOOST_AUTO_TEST_CASE(testing_ProxyBasicZMQ) { ICommunicator::Response response = server->receive(MessageType::GOOGLE_PROTOCOL_BUFFER); - std::chrono::duration duration = std::chrono::milliseconds(800); + std::chrono::duration duration = std::chrono::milliseconds(400); auto end_time = std::chrono::steady_clock::now() + duration; while (response.time_out and end_time > std::chrono::steady_clock::now()) { From 63c62a4486b8ec66ea69c553610e86549270c20e Mon Sep 17 00:00:00 2001 From: JoshuaSBrown Date: Thu, 5 Feb 2026 11:29:00 -0500 Subject: [PATCH 03/14] refactor: Core server to support proto3 --- core/server/ClientWorker.cpp | 310 ++++++-------- core/server/ClientWorker.hpp | 2 +- core/server/Config.hpp | 2 +- core/server/DatabaseAPI.cpp | 428 +++++++++---------- core/server/DatabaseAPI.hpp | 438 ++++++++++---------- core/server/GlobusAPI.cpp | 53 +-- core/server/GlobusAPI.hpp | 1 - core/server/TaskMgr.cpp | 2 +- core/server/TaskMgr.hpp | 2 - core/server/TaskWorker.cpp | 32 +- core/server/Version.hpp.in | 16 + core/server/main.cpp | 17 +- core/server/tests/unit/test_DatabaseAPI.cpp | 10 + 13 files changed, 649 insertions(+), 664 deletions(-) diff --git a/core/server/ClientWorker.cpp b/core/server/ClientWorker.cpp index be85bae71..5691f73df 100644 --- a/core/server/ClientWorker.cpp +++ b/core/server/ClientWorker.cpp @@ -1,4 +1,3 @@ - // Local DataFed includes #include "ClientWorker.hpp" #include "TaskMgr.hpp" @@ -14,10 +13,7 @@ #include "common/libjson.hpp" // Proto files -#include "common/SDMS.pb.h" -#include "common/SDMS_Anon.pb.h" -#include "common/SDMS_Auth.pb.h" -#include "common/Version.pb.h" +#include "common/envelope.pb.h" // Third party includes #include @@ -30,9 +26,6 @@ using namespace std; namespace SDMS { -using namespace SDMS::Anon; -using namespace SDMS::Auth; - namespace Core { map ClientWorker::m_msg_handlers; @@ -81,20 +74,19 @@ void ClientWorker::wait() { } } -#define SET_MSG_HANDLER(proto_id, msg, func) \ - m_msg_handlers[m_msg_mapper->getMessageType(proto_id, #msg)] = func -#define SET_MSG_HANDLER_DB(proto_id, rq, rp, func) \ - m_msg_handlers[m_msg_mapper->getMessageType(proto_id, #rq)] = \ +#define SET_MSG_HANDLER(msg, func) \ + m_msg_handlers[m_msg_mapper->getMessageType(#msg)] = func +#define SET_MSG_HANDLER_DB(rq, rp, func) \ + m_msg_handlers[m_msg_mapper->getMessageType(#rq)] = \ &ClientWorker::dbPassThrough /** * This method configures message handling by creating a map from message type - * to handler function. There are currently two protocol levels: anonymous and - * authenticated. Each is supported by a Google protobuf interface (in - * /common/proto). Most requests can be handled directly by the DB (via - * DatabaseAPI class), but some require local processing. This method maps the - * two classes of requests using the macros SET_MSG_HANDLER (for local) and - * SET_MSG_HANDLER_DB (for DB only). + * (envelope field number) to handler function. Message types are identified + * by their field number in the Envelope proto message. Most requests can be + * handled directly by the DB (via DatabaseAPI class), but some require local + * processing. This method maps the two classes of requests using the macros + * SET_MSG_HANDLER (for local) and SET_MSG_HANDLER_DB (for DB only). */ void ClientWorker::setupMsgHandlers() { static std::atomic_flag lock = ATOMIC_FLAG_INIT; @@ -105,192 +97,158 @@ void ClientWorker::setupMsgHandlers() { return; try { - // Register and setup handlers for the Anonymous interface - - uint8_t proto_id = m_msg_mapper->getProtocolID( - MessageProtocol::GOOGLE_ANONONYMOUS); // REG_PROTO( SDMS::Anon ); - // Requests that require the server to take action - SET_MSG_HANDLER(proto_id, VersionRequest, - &ClientWorker::procVersionRequest); - SET_MSG_HANDLER(proto_id, AuthenticateByPasswordRequest, + // Anonymous interface handlers + SET_MSG_HANDLER(VersionRequest, &ClientWorker::procVersionRequest); + SET_MSG_HANDLER(AuthenticateByPasswordRequest, &ClientWorker::procAuthenticateByPasswordRequest); - SET_MSG_HANDLER(proto_id, AuthenticateByTokenRequest, + SET_MSG_HANDLER(AuthenticateByTokenRequest, &ClientWorker::procAuthenticateByTokenRequest); - SET_MSG_HANDLER(proto_id, GetAuthStatusRequest, + SET_MSG_HANDLER(GetAuthStatusRequest, &ClientWorker::procGetAuthStatusRequest); + SET_MSG_HANDLER_DB(DailyMessageRequest, DailyMessageReply, dailyMessage); - // Requests that can be handled by DB client directly - SET_MSG_HANDLER_DB(proto_id, DailyMessageRequest, DailyMessageReply, - dailyMessage); - - // Register and setup handlers for the Authenticated interface - proto_id = m_msg_mapper->getProtocolID(MessageProtocol::GOOGLE_AUTHORIZED); - - // Requests that require the server to take action - SET_MSG_HANDLER(proto_id, GenerateCredentialsRequest, + // Authenticated interface handlers + SET_MSG_HANDLER(GenerateCredentialsRequest, &ClientWorker::procGenerateCredentialsRequest); - SET_MSG_HANDLER(proto_id, RevokeCredentialsRequest, + SET_MSG_HANDLER(RevokeCredentialsRequest, &ClientWorker::procRevokeCredentialsRequest); - SET_MSG_HANDLER(proto_id, DataGetRequest, - &ClientWorker::procDataGetRequest); - SET_MSG_HANDLER(proto_id, DataPutRequest, - &ClientWorker::procDataPutRequest); - SET_MSG_HANDLER(proto_id, RecordCreateRequest, + SET_MSG_HANDLER(DataGetRequest, &ClientWorker::procDataGetRequest); + SET_MSG_HANDLER(DataPutRequest, &ClientWorker::procDataPutRequest); + SET_MSG_HANDLER(RecordCreateRequest, &ClientWorker::procRecordCreateRequest); - SET_MSG_HANDLER(proto_id, RecordUpdateRequest, + SET_MSG_HANDLER(RecordUpdateRequest, &ClientWorker::procRecordUpdateRequest); - SET_MSG_HANDLER(proto_id, RecordUpdateBatchRequest, + SET_MSG_HANDLER(RecordUpdateBatchRequest, &ClientWorker::procRecordUpdateBatchRequest); - SET_MSG_HANDLER(proto_id, RecordDeleteRequest, + SET_MSG_HANDLER(RecordDeleteRequest, &ClientWorker::procRecordDeleteRequest); - SET_MSG_HANDLER(proto_id, RecordAllocChangeRequest, + SET_MSG_HANDLER(RecordAllocChangeRequest, &ClientWorker::procRecordAllocChangeRequest); - SET_MSG_HANDLER(proto_id, RecordOwnerChangeRequest, + SET_MSG_HANDLER(RecordOwnerChangeRequest, &ClientWorker::procRecordOwnerChangeRequest); - SET_MSG_HANDLER(proto_id, ProjectSearchRequest, + SET_MSG_HANDLER(ProjectSearchRequest, &ClientWorker::procProjectSearchRequest); - SET_MSG_HANDLER(proto_id, CollDeleteRequest, + SET_MSG_HANDLER(CollDeleteRequest, &ClientWorker::procCollectionDeleteRequest); - SET_MSG_HANDLER(proto_id, ProjectDeleteRequest, + SET_MSG_HANDLER(ProjectDeleteRequest, &ClientWorker::procProjectDeleteRequest); - SET_MSG_HANDLER(proto_id, RepoAuthzRequest, - &ClientWorker::procRepoAuthzRequest); - SET_MSG_HANDLER(proto_id, RepoAllocationCreateRequest, + SET_MSG_HANDLER(RepoAuthzRequest, &ClientWorker::procRepoAuthzRequest); + SET_MSG_HANDLER(RepoAllocationCreateRequest, &ClientWorker::procRepoAllocationCreateRequest); - SET_MSG_HANDLER(proto_id, RepoAllocationDeleteRequest, + SET_MSG_HANDLER(RepoAllocationDeleteRequest, &ClientWorker::procRepoAllocationDeleteRequest); - SET_MSG_HANDLER(proto_id, UserGetAccessTokenRequest, + SET_MSG_HANDLER(UserGetAccessTokenRequest, &ClientWorker::procUserGetAccessTokenRequest); - SET_MSG_HANDLER(proto_id, SchemaCreateRequest, + SET_MSG_HANDLER(SchemaCreateRequest, &ClientWorker::procSchemaCreateRequest); - SET_MSG_HANDLER(proto_id, SchemaReviseRequest, + SET_MSG_HANDLER(SchemaReviseRequest, &ClientWorker::procSchemaReviseRequest); - SET_MSG_HANDLER(proto_id, SchemaUpdateRequest, + SET_MSG_HANDLER(SchemaUpdateRequest, &ClientWorker::procSchemaUpdateRequest); - SET_MSG_HANDLER(proto_id, MetadataValidateRequest, + SET_MSG_HANDLER(MetadataValidateRequest, &ClientWorker::procMetadataValidateRequest); // Requires updating repo cache - SET_MSG_HANDLER(proto_id, RepoCreateRequest, &ClientWorker::procRepoCreate); - SET_MSG_HANDLER(proto_id, RepoUpdateRequest, &ClientWorker::procRepoUpdate); - SET_MSG_HANDLER(proto_id, RepoDeleteRequest, &ClientWorker::procRepoDelete); + SET_MSG_HANDLER(RepoCreateRequest, &ClientWorker::procRepoCreate); + SET_MSG_HANDLER(RepoUpdateRequest, &ClientWorker::procRepoUpdate); + SET_MSG_HANDLER(RepoDeleteRequest, &ClientWorker::procRepoDelete); // Requests that can be handled by DB client directly - SET_MSG_HANDLER_DB(proto_id, CheckPermsRequest, CheckPermsReply, - checkPerms); - SET_MSG_HANDLER_DB(proto_id, GetPermsRequest, GetPermsReply, getPerms); - SET_MSG_HANDLER_DB(proto_id, UserViewRequest, UserDataReply, userView); - SET_MSG_HANDLER_DB(proto_id, UserSetAccessTokenRequest, AckReply, + SET_MSG_HANDLER_DB(CheckPermsRequest, CheckPermsReply, checkPerms); + SET_MSG_HANDLER_DB(GetPermsRequest, GetPermsReply, getPerms); + SET_MSG_HANDLER_DB(UserViewRequest, UserDataReply, userView); + SET_MSG_HANDLER_DB(UserSetAccessTokenRequest, AckReply, userSetAccessToken); - SET_MSG_HANDLER_DB(proto_id, UserCreateRequest, UserDataReply, userCreate); - SET_MSG_HANDLER_DB(proto_id, UserUpdateRequest, UserDataReply, userUpdate); - SET_MSG_HANDLER_DB(proto_id, UserListAllRequest, UserDataReply, - userListAll); - SET_MSG_HANDLER_DB(proto_id, UserListCollabRequest, UserDataReply, - userListCollab); - SET_MSG_HANDLER_DB(proto_id, UserFindByUUIDsRequest, UserDataReply, + SET_MSG_HANDLER_DB(UserCreateRequest, UserDataReply, userCreate); + SET_MSG_HANDLER_DB(UserUpdateRequest, UserDataReply, userUpdate); + SET_MSG_HANDLER_DB(UserListAllRequest, UserDataReply, userListAll); + SET_MSG_HANDLER_DB(UserListCollabRequest, UserDataReply, userListCollab); + SET_MSG_HANDLER_DB(UserFindByUUIDsRequest, UserDataReply, userFindByUUIDs); - SET_MSG_HANDLER_DB(proto_id, UserFindByNameUIDRequest, UserDataReply, + SET_MSG_HANDLER_DB(UserFindByNameUIDRequest, UserDataReply, userFindByNameUID); - SET_MSG_HANDLER_DB(proto_id, UserGetRecentEPRequest, UserGetRecentEPReply, + SET_MSG_HANDLER_DB(UserGetRecentEPRequest, UserGetRecentEPReply, userGetRecentEP); - SET_MSG_HANDLER_DB(proto_id, UserSetRecentEPRequest, AckReply, - userSetRecentEP); - SET_MSG_HANDLER_DB(proto_id, ProjectViewRequest, ProjectDataReply, - projView); - SET_MSG_HANDLER_DB(proto_id, ProjectCreateRequest, ProjectDataReply, - projCreate); - SET_MSG_HANDLER_DB(proto_id, ProjectUpdateRequest, ProjectDataReply, - projUpdate); - SET_MSG_HANDLER_DB(proto_id, ProjectListRequest, ListingReply, projList); - SET_MSG_HANDLER_DB(proto_id, ProjectGetRoleRequest, ProjectGetRoleReply, + SET_MSG_HANDLER_DB(UserSetRecentEPRequest, AckReply, userSetRecentEP); + SET_MSG_HANDLER_DB(ProjectViewRequest, ProjectDataReply, projView); + SET_MSG_HANDLER_DB(ProjectCreateRequest, ProjectDataReply, projCreate); + SET_MSG_HANDLER_DB(ProjectUpdateRequest, ProjectDataReply, projUpdate); + SET_MSG_HANDLER_DB(ProjectListRequest, ListingReply, projList); + SET_MSG_HANDLER_DB(ProjectGetRoleRequest, ProjectGetRoleReply, projGetRole); - SET_MSG_HANDLER_DB(proto_id, RecordViewRequest, RecordDataReply, - recordView); - SET_MSG_HANDLER_DB(proto_id, RecordCreateBatchRequest, RecordDataReply, + SET_MSG_HANDLER_DB(RecordViewRequest, RecordDataReply, recordView); + SET_MSG_HANDLER_DB(RecordCreateBatchRequest, RecordDataReply, recordCreateBatch); - SET_MSG_HANDLER_DB(proto_id, RecordExportRequest, RecordExportReply, - recordExport); - SET_MSG_HANDLER_DB(proto_id, RecordLockRequest, ListingReply, recordLock); - SET_MSG_HANDLER_DB(proto_id, RecordListByAllocRequest, ListingReply, + SET_MSG_HANDLER_DB(RecordExportRequest, RecordExportReply, recordExport); + SET_MSG_HANDLER_DB(RecordLockRequest, ListingReply, recordLock); + SET_MSG_HANDLER_DB(RecordListByAllocRequest, ListingReply, recordListByAlloc); - SET_MSG_HANDLER_DB(proto_id, RecordGetDependencyGraphRequest, ListingReply, + SET_MSG_HANDLER_DB(RecordGetDependencyGraphRequest, ListingReply, recordGetDependencyGraph); - SET_MSG_HANDLER_DB(proto_id, SearchRequest, ListingReply, generalSearch); - SET_MSG_HANDLER_DB(proto_id, DataPathRequest, DataPathReply, dataPath); - SET_MSG_HANDLER_DB(proto_id, CollViewRequest, CollDataReply, collView); - SET_MSG_HANDLER_DB(proto_id, CollReadRequest, ListingReply, collRead); - SET_MSG_HANDLER_DB(proto_id, CollListPublishedRequest, ListingReply, + SET_MSG_HANDLER_DB(SearchRequest, ListingReply, generalSearch); + SET_MSG_HANDLER_DB(DataPathRequest, DataPathReply, dataPath); + SET_MSG_HANDLER_DB(CollViewRequest, CollDataReply, collView); + SET_MSG_HANDLER_DB(CollReadRequest, ListingReply, collRead); + SET_MSG_HANDLER_DB(CollListPublishedRequest, ListingReply, collListPublished); - SET_MSG_HANDLER_DB(proto_id, CollCreateRequest, CollDataReply, collCreate); - SET_MSG_HANDLER_DB(proto_id, CollUpdateRequest, CollDataReply, collUpdate); - SET_MSG_HANDLER_DB(proto_id, CollWriteRequest, ListingReply, collWrite); - SET_MSG_HANDLER_DB(proto_id, CollMoveRequest, AckReply, collMove); - SET_MSG_HANDLER_DB(proto_id, CollGetParentsRequest, CollPathReply, - collGetParents); - SET_MSG_HANDLER_DB(proto_id, CollGetOffsetRequest, CollGetOffsetReply, + SET_MSG_HANDLER_DB(CollCreateRequest, CollDataReply, collCreate); + SET_MSG_HANDLER_DB(CollUpdateRequest, CollDataReply, collUpdate); + SET_MSG_HANDLER_DB(CollWriteRequest, ListingReply, collWrite); + SET_MSG_HANDLER_DB(CollMoveRequest, AckReply, collMove); + SET_MSG_HANDLER_DB(CollGetParentsRequest, CollPathReply, collGetParents); + SET_MSG_HANDLER_DB(CollGetOffsetRequest, CollGetOffsetReply, collGetOffset); - SET_MSG_HANDLER_DB(proto_id, QueryListRequest, ListingReply, queryList); - SET_MSG_HANDLER_DB(proto_id, QueryViewRequest, QueryDataReply, queryView); - SET_MSG_HANDLER_DB(proto_id, QueryExecRequest, ListingReply, queryExec); - SET_MSG_HANDLER_DB(proto_id, QueryCreateRequest, QueryDataReply, - queryCreate); - SET_MSG_HANDLER_DB(proto_id, QueryUpdateRequest, QueryDataReply, - queryUpdate); - SET_MSG_HANDLER_DB(proto_id, QueryDeleteRequest, AckReply, queryDelete); - SET_MSG_HANDLER_DB(proto_id, NoteViewRequest, NoteDataReply, noteView); - SET_MSG_HANDLER_DB(proto_id, NoteListBySubjectRequest, NoteDataReply, + SET_MSG_HANDLER_DB(QueryListRequest, ListingReply, queryList); + SET_MSG_HANDLER_DB(QueryViewRequest, QueryDataReply, queryView); + SET_MSG_HANDLER_DB(QueryExecRequest, ListingReply, queryExec); + SET_MSG_HANDLER_DB(QueryCreateRequest, QueryDataReply, queryCreate); + SET_MSG_HANDLER_DB(QueryUpdateRequest, QueryDataReply, queryUpdate); + SET_MSG_HANDLER_DB(QueryDeleteRequest, AckReply, queryDelete); + SET_MSG_HANDLER_DB(NoteViewRequest, NoteDataReply, noteView); + SET_MSG_HANDLER_DB(NoteListBySubjectRequest, NoteDataReply, noteListBySubject); - SET_MSG_HANDLER_DB(proto_id, NoteCreateRequest, NoteDataReply, noteCreate); - SET_MSG_HANDLER_DB(proto_id, NoteUpdateRequest, NoteDataReply, noteUpdate); - SET_MSG_HANDLER_DB(proto_id, NoteCommentEditRequest, NoteDataReply, + SET_MSG_HANDLER_DB(NoteCreateRequest, NoteDataReply, noteCreate); + SET_MSG_HANDLER_DB(NoteUpdateRequest, NoteDataReply, noteUpdate); + SET_MSG_HANDLER_DB(NoteCommentEditRequest, NoteDataReply, noteCommentEdit); - SET_MSG_HANDLER_DB(proto_id, TaskListRequest, TaskDataReply, taskList); - SET_MSG_HANDLER_DB(proto_id, TaskViewRequest, TaskDataReply, taskView); - SET_MSG_HANDLER_DB(proto_id, ACLViewRequest, ACLDataReply, aclView); - SET_MSG_HANDLER_DB(proto_id, ACLUpdateRequest, ACLDataReply, aclUpdate); - SET_MSG_HANDLER_DB(proto_id, ACLSharedListRequest, ListingReply, - aclSharedList); - SET_MSG_HANDLER_DB(proto_id, ACLSharedListItemsRequest, ListingReply, + SET_MSG_HANDLER_DB(TaskListRequest, TaskDataReply, taskList); + SET_MSG_HANDLER_DB(TaskViewRequest, TaskDataReply, taskView); + SET_MSG_HANDLER_DB(ACLViewRequest, ACLDataReply, aclView); + SET_MSG_HANDLER_DB(ACLUpdateRequest, ACLDataReply, aclUpdate); + SET_MSG_HANDLER_DB(ACLSharedListRequest, ListingReply, aclSharedList); + SET_MSG_HANDLER_DB(ACLSharedListItemsRequest, ListingReply, aclSharedListItems); - SET_MSG_HANDLER_DB(proto_id, GroupCreateRequest, GroupDataReply, - groupCreate); - SET_MSG_HANDLER_DB(proto_id, GroupUpdateRequest, GroupDataReply, - groupUpdate); - SET_MSG_HANDLER_DB(proto_id, GroupDeleteRequest, AckReply, groupDelete); - SET_MSG_HANDLER_DB(proto_id, GroupListRequest, GroupDataReply, groupList); - SET_MSG_HANDLER_DB(proto_id, GroupViewRequest, GroupDataReply, groupView); - SET_MSG_HANDLER_DB(proto_id, RepoListRequest, RepoDataReply, repoList); - SET_MSG_HANDLER_DB(proto_id, RepoViewRequest, RepoDataReply, repoView); - SET_MSG_HANDLER_DB(proto_id, RepoCalcSizeRequest, RepoCalcSizeReply, - repoCalcSize); - SET_MSG_HANDLER_DB(proto_id, RepoListAllocationsRequest, - RepoAllocationsReply, repoListAllocations); - SET_MSG_HANDLER_DB(proto_id, RepoListSubjectAllocationsRequest, + SET_MSG_HANDLER_DB(GroupCreateRequest, GroupDataReply, groupCreate); + SET_MSG_HANDLER_DB(GroupUpdateRequest, GroupDataReply, groupUpdate); + SET_MSG_HANDLER_DB(GroupDeleteRequest, AckReply, groupDelete); + SET_MSG_HANDLER_DB(GroupListRequest, GroupDataReply, groupList); + SET_MSG_HANDLER_DB(GroupViewRequest, GroupDataReply, groupView); + SET_MSG_HANDLER_DB(RepoListRequest, RepoDataReply, repoList); + SET_MSG_HANDLER_DB(RepoViewRequest, RepoDataReply, repoView); + SET_MSG_HANDLER_DB(RepoCalcSizeRequest, RepoCalcSizeReply, repoCalcSize); + SET_MSG_HANDLER_DB(RepoListAllocationsRequest, RepoAllocationsReply, + repoListAllocations); + SET_MSG_HANDLER_DB(RepoListSubjectAllocationsRequest, RepoAllocationsReply, repoListSubjectAllocations); - SET_MSG_HANDLER_DB(proto_id, RepoListObjectAllocationsRequest, + SET_MSG_HANDLER_DB(RepoListObjectAllocationsRequest, RepoAllocationsReply, repoListObjectAllocations); - SET_MSG_HANDLER_DB(proto_id, RepoViewAllocationRequest, - RepoAllocationsReply, repoViewAllocation); - SET_MSG_HANDLER_DB(proto_id, RepoAllocationSetRequest, AckReply, + SET_MSG_HANDLER_DB(RepoViewAllocationRequest, RepoAllocationsReply, + repoViewAllocation); + SET_MSG_HANDLER_DB(RepoAllocationSetRequest, AckReply, repoAllocationSet); - SET_MSG_HANDLER_DB(proto_id, RepoAllocationSetDefaultRequest, AckReply, + SET_MSG_HANDLER_DB(RepoAllocationSetDefaultRequest, AckReply, repoAllocationSetDefault); - SET_MSG_HANDLER_DB(proto_id, RepoAllocationStatsRequest, - RepoAllocationStatsReply, repoAllocationStats); - SET_MSG_HANDLER_DB(proto_id, SchemaSearchRequest, SchemaDataReply, - schemaSearch); - SET_MSG_HANDLER_DB(proto_id, SchemaViewRequest, SchemaDataReply, - schemaView); - SET_MSG_HANDLER_DB(proto_id, SchemaDeleteRequest, AckReply, schemaDelete); - SET_MSG_HANDLER_DB(proto_id, TagSearchRequest, TagDataReply, tagSearch); - SET_MSG_HANDLER_DB(proto_id, TagListByCountRequest, TagDataReply, - tagListByCount); - SET_MSG_HANDLER_DB(proto_id, TopicListTopicsRequest, TopicDataReply, + SET_MSG_HANDLER_DB(RepoAllocationStatsRequest, RepoAllocationStatsReply, + repoAllocationStats); + SET_MSG_HANDLER_DB(SchemaSearchRequest, SchemaDataReply, schemaSearch); + SET_MSG_HANDLER_DB(SchemaViewRequest, SchemaDataReply, schemaView); + SET_MSG_HANDLER_DB(SchemaDeleteRequest, AckReply, schemaDelete); + SET_MSG_HANDLER_DB(TagSearchRequest, TagDataReply, tagSearch); + SET_MSG_HANDLER_DB(TagListByCountRequest, TagDataReply, tagListByCount); + SET_MSG_HANDLER_DB(TopicListTopicsRequest, TopicDataReply, topicListTopics); - SET_MSG_HANDLER_DB(proto_id, TopicViewRequest, TopicDataReply, topicView); - SET_MSG_HANDLER_DB(proto_id, TopicSearchRequest, TopicDataReply, - topicSearch); + SET_MSG_HANDLER_DB(TopicViewRequest, TopicDataReply, topicView); + SET_MSG_HANDLER_DB(TopicSearchRequest, TopicDataReply, topicSearch); } catch (TraceException &e) { DL_ERROR(m_log_context, "exception: " << e.toString()); throw; @@ -335,7 +293,7 @@ void ClientWorker::workerThread(LogContext log_context) { }(); ProtoBufMap proto_map; - uint16_t task_list_msg_type = proto_map.getMessageType(2, "TaskListRequest"); + uint16_t task_list_msg_type = proto_map.getMessageType("TaskListRequest"); DL_DEBUG(log_context, "W" << m_tid << " m_run " << m_run); @@ -368,7 +326,7 @@ void ClientWorker::workerThread(LogContext log_context) { << " [" << uid << "]"); } - if (uid.compare("anon") == 0 && msg_type > 0x1FF) { + if (uid.compare("anon") == 0 && proto_map.requiresAuth(proto_map.toString(msg_type))) { DL_WARNING(message_log_context, "W" << m_tid << " unauthorized access attempt from anon user"); @@ -376,8 +334,8 @@ void ClientWorker::workerThread(LogContext log_context) { // I know this is not great... allocating memory here slow // This will need to be fixed - auto nack = std::make_unique(); - nack->set_err_code(ID_AUTHN_REQUIRED); + auto nack = std::make_unique(); + nack->set_err_code(AUTHN_REQUIRED); nack->set_err_msg("Authentication required"); response_msg->setPayload(std::move(nack)); client->send(*response_msg); @@ -471,7 +429,7 @@ void ClientWorker::workerThread(LogContext log_context) { if (send_reply) { \ auto msg_reply = m_msg_factory.createResponseEnvelope(*msg_request); \ auto nack = std::make_unique(); \ - nack->set_err_code(ID_INTERNAL_ERROR); \ + nack->set_err_code(INTERNAL_ERROR); \ nack->set_err_msg(e.what()); \ msg_reply->setPayload(std::move(nack)); \ return msg_reply; \ @@ -483,7 +441,7 @@ void ClientWorker::workerThread(LogContext log_context) { if (send_reply) { \ auto msg_reply = m_msg_factory.createResponseEnvelope(*msg_request); \ auto nack = std::make_unique(); \ - nack->set_err_code(ID_INTERNAL_ERROR); \ + nack->set_err_code(INTERNAL_ERROR); \ nack->set_err_msg("Unknown exception type"); \ msg_reply->setPayload(std::move(nack)); \ return msg_reply; \ @@ -502,7 +460,7 @@ void ClientWorker::workerThread(LogContext log_context) { "unregistered msg type)."); \ auto msg_reply = m_msg_factory.createResponseEnvelope(*msg_request); \ auto nack = std::make_unique(); \ - nack->set_err_code(ID_BAD_REQUEST); \ + nack->set_err_code(BAD_REQUEST); \ nack->set_err_msg( \ "Message parse failed (malformed or unregistered msg type)"); \ msg_reply->setPayload(std::move(nack)); \ @@ -587,15 +545,15 @@ ClientWorker::procVersionRequest(const std::string &a_uid, (void)a_uid; DL_TRACE(log_context, "Version request"); - reply.set_release_year(DATAFED_RELEASE_YEAR); - reply.set_release_month(DATAFED_RELEASE_MONTH); - reply.set_release_day(DATAFED_RELEASE_DAY); - reply.set_release_hour(DATAFED_RELEASE_HOUR); - reply.set_release_minute(DATAFED_RELEASE_MINUTE); + reply.set_release_year( release::YEAR); + reply.set_release_month( release::MONTH); + reply.set_release_day( release::DAY); + reply.set_release_hour( release::HOUR); + reply.set_release_minute(release::MINUTE); - reply.set_api_major(DATAFED_COMMON_PROTOCOL_API_MAJOR); - reply.set_api_minor(DATAFED_COMMON_PROTOCOL_API_MINOR); - reply.set_api_patch(DATAFED_COMMON_PROTOCOL_API_PATCH); + reply.set_api_major(protocol::version::MAJOR); + reply.set_api_minor(protocol::version::MINOR); + reply.set_api_patch(protocol::version::PATCH); reply.set_component_major(core::version::MAJOR); reply.set_component_minor(core::version::MINOR); @@ -692,7 +650,7 @@ std::unique_ptr ClientWorker::procGenerateCredentialsRequest( char secret_key[41]; if (zmq_curve_keypair(public_key, secret_key) != 0) - EXCEPT_PARAM(ID_SERVICE_ERROR, + EXCEPT_PARAM(SERVICE_ERROR, "Key generation failed: " << zmq_strerror(errno)); pub_key = public_key; diff --git a/core/server/ClientWorker.hpp b/core/server/ClientWorker.hpp index cee872363..1da7af103 100644 --- a/core/server/ClientWorker.hpp +++ b/core/server/ClientWorker.hpp @@ -182,7 +182,7 @@ class ClientWorker : public nlohmann::json_schema::basic_error_handler { void schemaEnforceRequiredProperties(const nlohmann::json &a_schema); void recordCollectionDelete(const std::vector &a_ids, - Auth::TaskDataReply &a_reply, + SDMS::TaskDataReply &a_reply, LogContext log_context); void handleTaskResponse(libjson::Value &a_result, LogContext log_context); diff --git a/core/server/Config.hpp b/core/server/Config.hpp index ffc19fb92..f1170bbbe 100644 --- a/core/server/Config.hpp +++ b/core/server/Config.hpp @@ -8,7 +8,7 @@ // DataFed Common public includes #include "common/DynaLog.hpp" #include "common/ICredentials.hpp" -#include "common/SDMS.pb.h" +#include "common/envelope.pb.h" // Standard includes #include diff --git a/core/server/DatabaseAPI.cpp b/core/server/DatabaseAPI.cpp index aa0d35905..7363d4c18 100644 --- a/core/server/DatabaseAPI.cpp +++ b/core/server/DatabaseAPI.cpp @@ -4,9 +4,12 @@ // Local public includes #include "common/DynaLog.hpp" -#include "common/SDMS.pb.h" +#include "common/envelope.pb.h" #include "common/TraceException.hpp" #include "common/Util.hpp" +#include "common/envelope.pb.h" +#include "common/enums/access_token_type.pb.h" +#include "common/enums/search_mode.pb.h" // Third party includes #include @@ -26,7 +29,6 @@ using namespace std; namespace SDMS { namespace Core { -using namespace SDMS::Auth; using namespace libjson; #define TRANSLATE_BEGIN() try { @@ -44,7 +46,7 @@ DatabaseAPI::DatabaseAPI(const std::string &a_db_url, : m_client(0), m_db_url(a_db_url) { m_curl = curl_easy_init(); if (!m_curl) - EXCEPT(ID_INTERNAL_ERROR, "libcurl init failed"); + EXCEPT(INTERNAL_ERROR, "libcurl init failed"); setClient(""); @@ -146,7 +148,7 @@ long DatabaseAPI::dbGet(const char *a_url_path, a_result.fromString(res_json); } catch (libjson::ParseError &e) { DL_DEBUG(log_context, "PARSE [" << res_json << "]"); - EXCEPT_PARAM(ID_SERVICE_ERROR, + EXCEPT_PARAM(SERVICE_ERROR, "Invalid JSON returned from DB: " << e.toString()); } } @@ -155,14 +157,14 @@ long DatabaseAPI::dbGet(const char *a_url_path, return http_code; } else { if (res_json.size() && a_result.asObject().has("errorMessage")) { - EXCEPT_PARAM(ID_BAD_REQUEST, a_result.asObject().asString()); + EXCEPT_PARAM(BAD_REQUEST, a_result.asObject().asString()); } else { - EXCEPT_PARAM(ID_BAD_REQUEST, "SDMS DB service call failed. Code: " + EXCEPT_PARAM(BAD_REQUEST, "SDMS DB service call failed. Code: " << http_code << ", err: " << error); } } } else { - EXCEPT_PARAM(ID_SERVICE_ERROR, "SDMS DB interface failed. error: " + EXCEPT_PARAM(SERVICE_ERROR, "SDMS DB interface failed. error: " << error << ", " << curl_easy_strerror(res)); } @@ -237,7 +239,7 @@ long DatabaseAPI::dbPost(const char *a_url_path, a_result.fromString(res_json); } catch (libjson::ParseError &e) { DL_DEBUG(log_context, "PARSE [" << res_json << "]"); - EXCEPT_PARAM(ID_SERVICE_ERROR, + EXCEPT_PARAM(SERVICE_ERROR, "Invalid JSON returned from DB: " << e.toString()); } } @@ -250,14 +252,14 @@ long DatabaseAPI::dbPost(const char *a_url_path, << (a_body ? *a_body : "") << "]"); - EXCEPT_PARAM(ID_BAD_REQUEST, a_result.asObject().asString()); + EXCEPT_PARAM(BAD_REQUEST, a_result.asObject().asString()); } else { - EXCEPT_PARAM(ID_BAD_REQUEST, "SDMS DB service call failed. Code: " + EXCEPT_PARAM(BAD_REQUEST, "SDMS DB service call failed. Code: " << http_code << ", err: " << error); } } } else { - EXCEPT_PARAM(ID_SERVICE_ERROR, "SDMS DB interface failed. error: " + EXCEPT_PARAM(SERVICE_ERROR, "SDMS DB interface failed. error: " << error << ", " << curl_easy_strerror(res)); } @@ -270,7 +272,7 @@ void DatabaseAPI::serverPing(LogContext log_context) { } void DatabaseAPI::clientAuthenticateByPassword(const std::string &a_password, - Anon::AuthStatusReply &a_reply, + SDMS::AuthStatusReply &a_reply, LogContext log_context) { Value result; @@ -279,7 +281,7 @@ void DatabaseAPI::clientAuthenticateByPassword(const std::string &a_password, } void DatabaseAPI::clientAuthenticateByToken(const std::string &a_token, - Anon::AuthStatusReply &a_reply, + SDMS::AuthStatusReply &a_reply, LogContext log_context) { Value result; @@ -287,7 +289,7 @@ void DatabaseAPI::clientAuthenticateByToken(const std::string &a_token, setAuthStatus(a_reply, result); } -void DatabaseAPI::setAuthStatus(Anon::AuthStatusReply &a_reply, +void DatabaseAPI::setAuthStatus(SDMS::AuthStatusReply &a_reply, const Value &a_result) { const Value::Object &obj = a_result.asObject(); a_reply.set_uid(obj.getString("uid")); @@ -386,7 +388,7 @@ void DatabaseAPI::userSetAccessToken(const std::string &a_acc_tok, {"access", a_acc_tok}, {"refresh", a_ref_tok}, {"expires_in", to_string(a_expires_in)}}; - if (token_type != SDMS::AccessTokenType::ACCESS_SENTINEL) { + if (token_type != SDMS::AccessTokenType::TOKEN_UNSPECIFIED) { params.push_back({"type", to_string(token_type)}); } if (!other_token_data.empty()) { @@ -406,7 +408,7 @@ void DatabaseAPI::userSetAccessToken(const std::string &a_access_token, } void DatabaseAPI::userSetAccessToken( - const Auth::UserSetAccessTokenRequest &a_request, Anon::AckReply &a_reply, + const SDMS::UserSetAccessTokenRequest &a_request, SDMS::AckReply &a_reply, LogContext log_context) { (void)a_reply; userSetAccessToken(a_request.access(), a_request.expires_in(), @@ -451,8 +453,8 @@ void DatabaseAPI::purgeTransferRecords(size_t age) { dbGetRaw(url, result); } -void DatabaseAPI::userCreate(const Auth::UserCreateRequest &a_request, - Auth::UserDataReply &a_reply, +void DatabaseAPI::userCreate(const SDMS::UserCreateRequest &a_request, + SDMS::UserDataReply &a_reply, LogContext log_context) { DL_DEBUG(log_context, "DataFed user create - uid: " << a_request.uid() @@ -488,8 +490,8 @@ void DatabaseAPI::userCreate(const Auth::UserCreateRequest &a_request, setUserData(a_reply, result, log_context); } -void DatabaseAPI::userView(const Auth::UserViewRequest &a_request, - Auth::UserDataReply &a_reply, +void DatabaseAPI::userView(const SDMS::UserViewRequest &a_request, + SDMS::UserDataReply &a_reply, LogContext log_context) { vector> params; params.push_back({"subject", a_request.uid()}); @@ -503,7 +505,7 @@ void DatabaseAPI::userView(const Auth::UserViewRequest &a_request, } void DatabaseAPI::userUpdate(const UserUpdateRequest &a_request, - Auth::UserDataReply &a_reply, + SDMS::UserDataReply &a_reply, LogContext log_context) { Value result; @@ -522,7 +524,7 @@ void DatabaseAPI::userUpdate(const UserUpdateRequest &a_request, } void DatabaseAPI::userListAll(const UserListAllRequest &a_request, - Auth::UserDataReply &a_reply, + SDMS::UserDataReply &a_reply, LogContext log_context) { vector> params; if (a_request.has_offset() && a_request.has_count()) { @@ -537,7 +539,7 @@ void DatabaseAPI::userListAll(const UserListAllRequest &a_request, } void DatabaseAPI::userListCollab(const UserListCollabRequest &a_request, - Auth::UserDataReply &a_reply, + SDMS::UserDataReply &a_reply, LogContext log_context) { Value result; vector> params; @@ -550,8 +552,8 @@ void DatabaseAPI::userListCollab(const UserListCollabRequest &a_request, setUserData(a_reply, result, log_context); } -void DatabaseAPI::userFindByUUIDs(const Auth::UserFindByUUIDsRequest &a_request, - Auth::UserDataReply &a_reply, +void DatabaseAPI::userFindByUUIDs(const SDMS::UserFindByUUIDsRequest &a_request, + SDMS::UserDataReply &a_reply, LogContext log_context) { string uuids = "["; @@ -570,8 +572,8 @@ void DatabaseAPI::userFindByUUIDs(const Auth::UserFindByUUIDsRequest &a_request, } void DatabaseAPI::userFindByNameUID( - const Auth::UserFindByNameUIDRequest &a_request, - Auth::UserDataReply &a_reply, LogContext log_context) { + const SDMS::UserFindByNameUIDRequest &a_request, + SDMS::UserDataReply &a_reply, LogContext log_context) { Value result; vector> params; params.push_back({"name_uid", a_request.name_uid()}); @@ -585,8 +587,8 @@ void DatabaseAPI::userFindByNameUID( setUserData(a_reply, result, log_context); } -void DatabaseAPI::userGetRecentEP(const Auth::UserGetRecentEPRequest &a_request, - Auth::UserGetRecentEPReply &a_reply, +void DatabaseAPI::userGetRecentEP(const SDMS::UserGetRecentEPRequest &a_request, + SDMS::UserGetRecentEPReply &a_reply, LogContext log_context) { (void)a_request; Value result; @@ -604,8 +606,8 @@ void DatabaseAPI::userGetRecentEP(const Auth::UserGetRecentEPRequest &a_request, TRANSLATE_END(result, log_context) } -void DatabaseAPI::userSetRecentEP(const Auth::UserSetRecentEPRequest &a_request, - Anon::AckReply &a_reply, +void DatabaseAPI::userSetRecentEP(const SDMS::UserSetRecentEPRequest &a_request, + SDMS::AckReply &a_reply, LogContext log_context) { (void)a_reply; Value result; @@ -621,7 +623,7 @@ void DatabaseAPI::userSetRecentEP(const Auth::UserSetRecentEPRequest &a_request, dbGet("usr/ep/set", {{"eps", eps}}, result, log_context); } -void DatabaseAPI::setUserData(Auth::UserDataReply &a_reply, +void DatabaseAPI::setUserData(SDMS::UserDataReply &a_reply, const Value &a_result, LogContext log_context) { UserData *user; Value::ArrayConstIter k; @@ -676,8 +678,8 @@ void DatabaseAPI::setUserData(Auth::UserDataReply &a_reply, TRANSLATE_END(a_result, log_context) } -void DatabaseAPI::projCreate(const Auth::ProjectCreateRequest &a_request, - Auth::ProjectDataReply &a_reply, +void DatabaseAPI::projCreate(const SDMS::ProjectCreateRequest &a_request, + SDMS::ProjectDataReply &a_reply, LogContext log_context) { Value result; vector> params; @@ -715,8 +717,8 @@ void DatabaseAPI::projCreate(const Auth::ProjectCreateRequest &a_request, setProjectData(a_reply, result, log_context); } -void DatabaseAPI::projUpdate(const Auth::ProjectUpdateRequest &a_request, - Auth::ProjectDataReply &a_reply, +void DatabaseAPI::projUpdate(const SDMS::ProjectUpdateRequest &a_request, + SDMS::ProjectDataReply &a_reply, LogContext log_context) { Value result; vector> params; @@ -756,8 +758,8 @@ void DatabaseAPI::projUpdate(const Auth::ProjectUpdateRequest &a_request, setProjectData(a_reply, result, log_context); } -void DatabaseAPI::projView(const Auth::ProjectViewRequest &a_request, - Auth::ProjectDataReply &a_reply, +void DatabaseAPI::projView(const SDMS::ProjectViewRequest &a_request, + SDMS::ProjectDataReply &a_reply, LogContext log_context) { Value result; dbGet("prj/view", {{"id", a_request.id()}}, result, log_context); @@ -765,8 +767,8 @@ void DatabaseAPI::projView(const Auth::ProjectViewRequest &a_request, setProjectData(a_reply, result, log_context); } -void DatabaseAPI::projList(const Auth::ProjectListRequest &a_request, - Auth::ListingReply &a_reply, +void DatabaseAPI::projList(const SDMS::ProjectListRequest &a_request, + SDMS::ListingReply &a_reply, LogContext log_context) { Value result; vector> params; @@ -792,8 +794,8 @@ void DatabaseAPI::projList(const Auth::ProjectListRequest &a_request, setListingDataReply(a_reply, result, log_context); } -void DatabaseAPI::projGetRole(const Auth::ProjectGetRoleRequest &a_request, - Auth::ProjectGetRoleReply &a_reply, +void DatabaseAPI::projGetRole(const SDMS::ProjectGetRoleRequest &a_request, + SDMS::ProjectGetRoleReply &a_reply, LogContext log_context) { Value result; vector> params; @@ -808,7 +810,7 @@ void DatabaseAPI::projGetRole(const Auth::ProjectGetRoleRequest &a_request, } void DatabaseAPI::projSearch(const std::string &a_query, - Auth::ProjectDataReply &a_reply, + SDMS::ProjectDataReply &a_reply, LogContext log_context) { Value result; @@ -817,7 +819,7 @@ void DatabaseAPI::projSearch(const std::string &a_query, setProjectData(a_reply, result, log_context); } -void DatabaseAPI::setProjectData(Auth::ProjectDataReply &a_reply, +void DatabaseAPI::setProjectData(SDMS::ProjectDataReply &a_reply, const Value &a_result, LogContext log_context) { ProjectData *proj; @@ -872,8 +874,8 @@ void DatabaseAPI::setProjectData(Auth::ProjectDataReply &a_reply, } void DatabaseAPI::recordListByAlloc( - const Auth::RecordListByAllocRequest &a_request, - Auth::ListingReply &a_reply, LogContext log_context) { + const SDMS::RecordListByAllocRequest &a_request, + SDMS::ListingReply &a_reply, LogContext log_context) { Value result; vector> params; params.push_back({"repo", a_request.repo()}); @@ -888,8 +890,8 @@ void DatabaseAPI::recordListByAlloc( setListingDataReply(a_reply, result, log_context); } -void DatabaseAPI::recordView(const Auth::RecordViewRequest &a_request, - Auth::RecordDataReply &a_reply, +void DatabaseAPI::recordView(const SDMS::RecordViewRequest &a_request, + SDMS::RecordDataReply &a_reply, LogContext log_context) { Value result; @@ -898,8 +900,8 @@ void DatabaseAPI::recordView(const Auth::RecordViewRequest &a_request, setRecordData(a_reply, result, log_context); } -void DatabaseAPI::recordCreate(const Auth::RecordCreateRequest &a_request, - Auth::RecordDataReply &a_reply, +void DatabaseAPI::recordCreate(const SDMS::RecordCreateRequest &a_request, + SDMS::RecordDataReply &a_reply, LogContext log_context) { Value result; nlohmann::json payload; @@ -968,8 +970,8 @@ void DatabaseAPI::recordCreate(const Auth::RecordCreateRequest &a_request, } void DatabaseAPI::recordCreateBatch( - const Auth::RecordCreateBatchRequest &a_request, - Auth::RecordDataReply &a_reply, LogContext log_context) { + const SDMS::RecordCreateBatchRequest &a_request, + SDMS::RecordDataReply &a_reply, LogContext log_context) { Value result; dbPost("dat/create/batch", {}, &a_request.records(), result, log_context); @@ -977,8 +979,8 @@ void DatabaseAPI::recordCreateBatch( setRecordData(a_reply, result, log_context); } -void DatabaseAPI::recordUpdate(const Auth::RecordUpdateRequest &a_request, - Auth::RecordDataReply &a_reply, +void DatabaseAPI::recordUpdate(const SDMS::RecordUpdateRequest &a_request, + SDMS::RecordDataReply &a_reply, libjson::Value &result, LogContext log_context) { nlohmann::json payload; payload["id"] = a_request.id(); @@ -1051,8 +1053,8 @@ void DatabaseAPI::recordUpdate(const Auth::RecordUpdateRequest &a_request, } void DatabaseAPI::recordUpdateBatch( - const Auth::RecordUpdateBatchRequest &a_request, - Auth::RecordDataReply &a_reply, libjson::Value &result, + const SDMS::RecordUpdateBatchRequest &a_request, + SDMS::RecordDataReply &a_reply, libjson::Value &result, LogContext log_context) { // "records" field is a JSON document - send directly to DB dbPost("dat/update/batch", {}, &a_request.records(), result, log_context); @@ -1060,7 +1062,7 @@ void DatabaseAPI::recordUpdateBatch( setRecordData(a_reply, result, log_context); } -void DatabaseAPI::recordUpdateSize(const Auth::RepoDataSizeReply &a_size_rep, +void DatabaseAPI::recordUpdateSize(const SDMS::RepoDataSizeReply &a_size_rep, LogContext log_context) { libjson::Value result; @@ -1091,8 +1093,8 @@ void DatabaseAPI::recordUpdateSchemaError(const std::string &a_rec_id, log_context); } -void DatabaseAPI::recordExport(const Auth::RecordExportRequest &a_request, - Auth::RecordExportReply &a_reply, +void DatabaseAPI::recordExport(const SDMS::RecordExportRequest &a_request, + SDMS::RecordExportReply &a_reply, LogContext log_context) { Value result; @@ -1117,8 +1119,8 @@ void DatabaseAPI::recordExport(const Auth::RecordExportRequest &a_request, TRANSLATE_END(result, log_context) } -void DatabaseAPI::recordLock(const Auth::RecordLockRequest &a_request, - Auth::ListingReply &a_reply, +void DatabaseAPI::recordLock(const SDMS::RecordLockRequest &a_request, + SDMS::ListingReply &a_reply, LogContext log_context) { Value result; string ids; @@ -1143,8 +1145,8 @@ void DatabaseAPI::recordLock(const Auth::RecordLockRequest &a_request, } void DatabaseAPI::recordGetDependencyGraph( - const Auth::RecordGetDependencyGraphRequest &a_request, - Auth::ListingReply &a_reply, LogContext log_context) { + const SDMS::RecordGetDependencyGraphRequest &a_request, + SDMS::ListingReply &a_reply, LogContext log_context) { Value result; dbGet("dat/dep/graph/get", {{"id", a_request.id()}}, result, log_context); @@ -1152,7 +1154,7 @@ void DatabaseAPI::recordGetDependencyGraph( setListingDataReply(a_reply, result, log_context); } -void DatabaseAPI::setRecordData(Auth::RecordDataReply &a_reply, +void DatabaseAPI::setRecordData(SDMS::RecordDataReply &a_reply, const Value &a_result, LogContext log_context) { RecordData *rec; DependencyData *deps; @@ -1268,8 +1270,8 @@ void DatabaseAPI::setRecordData(Auth::RecordDataReply &a_reply, TRANSLATE_END(a_result, log_context) } -void DatabaseAPI::dataPath(const Auth::DataPathRequest &a_request, - Auth::DataPathReply &a_reply, +void DatabaseAPI::dataPath(const SDMS::DataPathRequest &a_request, + SDMS::DataPathReply &a_reply, LogContext log_context) { Value result; @@ -1293,8 +1295,8 @@ void DatabaseAPI::dataPath(const Auth::DataPathRequest &a_request, * depending on scope. The DB relies on either tha "dataview" or "collview" * Arango search views for execution of the query. */ -void DatabaseAPI::generalSearch(const Auth::SearchRequest &a_request, - Auth::ListingReply &a_reply, +void DatabaseAPI::generalSearch(const SDMS::SearchRequest &a_request, + SDMS::ListingReply &a_reply, LogContext log_context) { Value result; string qry_begin, qry_end, qry_filter, params; @@ -1321,8 +1323,8 @@ void DatabaseAPI::generalSearch(const Auth::SearchRequest &a_request, } void DatabaseAPI::collListPublished( - const Auth::CollListPublishedRequest &a_request, - Auth::ListingReply &a_reply, LogContext log_context) { + const SDMS::CollListPublishedRequest &a_request, + SDMS::ListingReply &a_reply, LogContext log_context) { Value result; vector> params; @@ -1338,8 +1340,8 @@ void DatabaseAPI::collListPublished( setListingDataReply(a_reply, result, log_context); } -void DatabaseAPI::collCreate(const Auth::CollCreateRequest &a_request, - Auth::CollDataReply &a_reply, +void DatabaseAPI::collCreate(const SDMS::CollCreateRequest &a_request, + SDMS::CollDataReply &a_reply, LogContext log_context) { Value result; nlohmann::json payload; @@ -1376,8 +1378,8 @@ void DatabaseAPI::collCreate(const Auth::CollCreateRequest &a_request, setCollData(a_reply, result, log_context); } -void DatabaseAPI::collUpdate(const Auth::CollUpdateRequest &a_request, - Auth::CollDataReply &a_reply, +void DatabaseAPI::collUpdate(const SDMS::CollUpdateRequest &a_request, + SDMS::CollDataReply &a_reply, LogContext log_context) { Value result; nlohmann::json payload; @@ -1414,8 +1416,8 @@ void DatabaseAPI::collUpdate(const Auth::CollUpdateRequest &a_request, setCollData(a_reply, result, log_context); } -void DatabaseAPI::collView(const Auth::CollViewRequest &a_request, - Auth::CollDataReply &a_reply, +void DatabaseAPI::collView(const SDMS::CollViewRequest &a_request, + SDMS::CollDataReply &a_reply, LogContext log_context) { Value result; @@ -1424,8 +1426,8 @@ void DatabaseAPI::collView(const Auth::CollViewRequest &a_request, setCollData(a_reply, result, log_context); } -void DatabaseAPI::collRead(const Auth::CollReadRequest &a_request, - Auth::ListingReply &a_reply, +void DatabaseAPI::collRead(const SDMS::CollReadRequest &a_request, + SDMS::ListingReply &a_reply, LogContext log_context) { Value result; vector> params; @@ -1440,8 +1442,8 @@ void DatabaseAPI::collRead(const Auth::CollReadRequest &a_request, setListingDataReply(a_reply, result, log_context); } -void DatabaseAPI::collWrite(const Auth::CollWriteRequest &a_request, - Auth::ListingReply &a_reply, +void DatabaseAPI::collWrite(const SDMS::CollWriteRequest &a_request, + SDMS::ListingReply &a_reply, LogContext log_context) { string add_list, rem_list; vector> params; @@ -1479,8 +1481,8 @@ void DatabaseAPI::collWrite(const Auth::CollWriteRequest &a_request, setListingDataReply(a_reply, result, log_context); } -void DatabaseAPI::collMove(const Auth::CollMoveRequest &a_request, - Anon::AckReply &a_reply, LogContext log_context) { +void DatabaseAPI::collMove(const SDMS::CollMoveRequest &a_request, + SDMS::AckReply &a_reply, LogContext log_context) { (void)a_reply; if (a_request.item_size() == 0) @@ -1504,8 +1506,8 @@ void DatabaseAPI::collMove(const Auth::CollMoveRequest &a_request, result, log_context); } -void DatabaseAPI::collGetParents(const Auth::CollGetParentsRequest &a_request, - Auth::CollPathReply &a_reply, +void DatabaseAPI::collGetParents(const SDMS::CollGetParentsRequest &a_request, + SDMS::CollPathReply &a_reply, LogContext log_context) { Value result; vector> params; @@ -1518,8 +1520,8 @@ void DatabaseAPI::collGetParents(const Auth::CollGetParentsRequest &a_request, setCollPathData(a_reply, result, log_context); } -void DatabaseAPI::collGetOffset(const Auth::CollGetOffsetRequest &a_request, - Auth::CollGetOffsetReply &a_reply, +void DatabaseAPI::collGetOffset(const SDMS::CollGetOffsetRequest &a_request, + SDMS::CollGetOffsetReply &a_reply, LogContext log_context) { Value result; @@ -1534,7 +1536,7 @@ void DatabaseAPI::collGetOffset(const Auth::CollGetOffsetRequest &a_request, a_reply.set_offset(result.asObject().getNumber("offset")); } -void DatabaseAPI::setCollData(Auth::CollDataReply &a_reply, +void DatabaseAPI::setCollData(SDMS::CollDataReply &a_reply, const libjson::Value &a_result, LogContext log_context) { CollData *coll; @@ -1637,7 +1639,7 @@ void DatabaseAPI::setCollPathData(CollPathReply &a_reply, TRANSLATE_END(a_result, log_context) } -void DatabaseAPI::setListingDataReply(Auth::ListingReply &a_reply, +void DatabaseAPI::setListingDataReply(SDMS::ListingReply &a_reply, const libjson::Value &a_result, LogContext log_context) { Value::ObjectConstIter j; @@ -1730,8 +1732,8 @@ void DatabaseAPI::setListingData(ListingData *a_item, } } -void DatabaseAPI::queryList(const Auth::QueryListRequest &a_request, - Auth::ListingReply &a_reply, +void DatabaseAPI::queryList(const SDMS::QueryListRequest &a_request, + SDMS::ListingReply &a_reply, LogContext log_context) { Value result; vector> params; @@ -1745,8 +1747,8 @@ void DatabaseAPI::queryList(const Auth::QueryListRequest &a_request, setListingDataReply(a_reply, result, log_context); } -void DatabaseAPI::queryCreate(const Auth::QueryCreateRequest &a_request, - Auth::QueryDataReply &a_reply, +void DatabaseAPI::queryCreate(const SDMS::QueryCreateRequest &a_request, + SDMS::QueryDataReply &a_reply, LogContext log_context) { Value result; // vector> params; @@ -1783,8 +1785,8 @@ void DatabaseAPI::queryCreate(const Auth::QueryCreateRequest &a_request, setQueryData(a_reply, result, log_context); } -void DatabaseAPI::queryUpdate(const Auth::QueryUpdateRequest &a_request, - Auth::QueryDataReply &a_reply, +void DatabaseAPI::queryUpdate(const SDMS::QueryUpdateRequest &a_request, + SDMS::QueryDataReply &a_reply, LogContext log_context) { Value result; nlohmann::json payload; @@ -1827,8 +1829,8 @@ void DatabaseAPI::queryUpdate(const Auth::QueryUpdateRequest &a_request, } // DatabaseAPI::queryDelete( const std::string & a_id ) -void DatabaseAPI::queryDelete(const Auth::QueryDeleteRequest &a_request, - Anon::AckReply &a_reply, LogContext log_context) { +void DatabaseAPI::queryDelete(const SDMS::QueryDeleteRequest &a_request, + SDMS::AckReply &a_reply, LogContext log_context) { (void)a_reply; Value result; string ids = "["; @@ -1844,8 +1846,8 @@ void DatabaseAPI::queryDelete(const Auth::QueryDeleteRequest &a_request, dbGet("qry/delete", {{"ids", ids}}, result, log_context); } -void DatabaseAPI::queryView(const Auth::QueryViewRequest &a_request, - Auth::QueryDataReply &a_reply, +void DatabaseAPI::queryView(const SDMS::QueryViewRequest &a_request, + SDMS::QueryDataReply &a_reply, LogContext log_context) { Value result; @@ -1854,8 +1856,8 @@ void DatabaseAPI::queryView(const Auth::QueryViewRequest &a_request, setQueryData(a_reply, result, log_context); } -void DatabaseAPI::queryExec(const Auth::QueryExecRequest &a_request, - Auth::ListingReply &a_reply, +void DatabaseAPI::queryExec(const SDMS::QueryExecRequest &a_request, + SDMS::ListingReply &a_reply, LogContext log_context) { Value result; vector> params; @@ -1893,8 +1895,8 @@ void DatabaseAPI::setQueryData(QueryDataReply &a_reply, TRANSLATE_END(a_result, log_context) } -void DatabaseAPI::aclView(const Auth::ACLViewRequest &a_request, - Auth::ACLDataReply &a_reply, LogContext log_context) { +void DatabaseAPI::aclView(const SDMS::ACLViewRequest &a_request, + SDMS::ACLDataReply &a_reply, LogContext log_context) { libjson::Value result; dbGet("acl/view", {{"id", a_request.id()}}, result, log_context); @@ -1902,8 +1904,8 @@ void DatabaseAPI::aclView(const Auth::ACLViewRequest &a_request, setACLData(a_reply, result, log_context); } -void DatabaseAPI::aclUpdate(const Auth::ACLUpdateRequest &a_request, - Auth::ACLDataReply &a_reply, +void DatabaseAPI::aclUpdate(const SDMS::ACLUpdateRequest &a_request, + SDMS::ACLDataReply &a_reply, LogContext log_context) { Value result; vector> params; @@ -1916,8 +1918,8 @@ void DatabaseAPI::aclUpdate(const Auth::ACLUpdateRequest &a_request, setACLData(a_reply, result, log_context); } -void DatabaseAPI::aclSharedList(const Auth::ACLSharedListRequest &a_request, - Auth::ListingReply &a_reply, +void DatabaseAPI::aclSharedList(const SDMS::ACLSharedListRequest &a_request, + SDMS::ListingReply &a_reply, LogContext log_context) { Value result; vector> params; @@ -1934,8 +1936,8 @@ void DatabaseAPI::aclSharedList(const Auth::ACLSharedListRequest &a_request, } void DatabaseAPI::aclSharedListItems( - const Auth::ACLSharedListItemsRequest &a_request, - Auth::ListingReply &a_reply, LogContext log_context) { + const SDMS::ACLSharedListItemsRequest &a_request, + SDMS::ListingReply &a_reply, LogContext log_context) { Value result; vector> params; @@ -1971,8 +1973,8 @@ void DatabaseAPI::setACLData(ACLDataReply &a_reply, TRANSLATE_END(a_result, log_context) } -void DatabaseAPI::groupCreate(const Auth::GroupCreateRequest &a_request, - Auth::GroupDataReply &a_reply, +void DatabaseAPI::groupCreate(const SDMS::GroupCreateRequest &a_request, + SDMS::GroupDataReply &a_reply, LogContext log_context) { Value result; @@ -2000,8 +2002,8 @@ void DatabaseAPI::groupCreate(const Auth::GroupCreateRequest &a_request, setGroupData(a_reply, result, log_context); } -void DatabaseAPI::groupUpdate(const Auth::GroupUpdateRequest &a_request, - Auth::GroupDataReply &a_reply, +void DatabaseAPI::groupUpdate(const SDMS::GroupUpdateRequest &a_request, + SDMS::GroupDataReply &a_reply, LogContext log_context) { Value result; @@ -2039,8 +2041,8 @@ void DatabaseAPI::groupUpdate(const Auth::GroupUpdateRequest &a_request, setGroupData(a_reply, result, log_context); } -void DatabaseAPI::groupDelete(const Auth::GroupDeleteRequest &a_request, - Anon::AckReply &a_reply, LogContext log_context) { +void DatabaseAPI::groupDelete(const SDMS::GroupDeleteRequest &a_request, + SDMS::AckReply &a_reply, LogContext log_context) { (void)a_reply; Value result; @@ -2052,8 +2054,8 @@ void DatabaseAPI::groupDelete(const Auth::GroupDeleteRequest &a_request, dbGet("grp/delete", params, result, log_context); } -void DatabaseAPI::groupList(const Auth::GroupListRequest &a_request, - Auth::GroupDataReply &a_reply, +void DatabaseAPI::groupList(const SDMS::GroupListRequest &a_request, + SDMS::GroupDataReply &a_reply, LogContext log_context) { (void)a_request; @@ -2067,8 +2069,8 @@ void DatabaseAPI::groupList(const Auth::GroupListRequest &a_request, setGroupData(a_reply, result, log_context); } -void DatabaseAPI::groupView(const Auth::GroupViewRequest &a_request, - Auth::GroupDataReply &a_reply, +void DatabaseAPI::groupView(const SDMS::GroupViewRequest &a_request, + SDMS::GroupDataReply &a_reply, LogContext log_context) { Value result; vector> params; @@ -2117,8 +2119,8 @@ void DatabaseAPI::setGroupData(GroupDataReply &a_reply, TRANSLATE_END(a_result, log_context) } -void DatabaseAPI::repoList(const Auth::RepoListRequest &a_request, - Auth::RepoDataReply &a_reply, +void DatabaseAPI::repoList(const SDMS::RepoListRequest &a_request, + SDMS::RepoDataReply &a_reply, LogContext log_context) { Value result; @@ -2158,8 +2160,8 @@ void DatabaseAPI::repoView(std::vector &a_repos, } } -void DatabaseAPI::repoView(const Auth::RepoViewRequest &a_request, - Auth::RepoDataReply &a_reply, +void DatabaseAPI::repoView(const SDMS::RepoViewRequest &a_request, + SDMS::RepoDataReply &a_reply, LogContext log_context) { Value result; @@ -2169,8 +2171,8 @@ void DatabaseAPI::repoView(const Auth::RepoViewRequest &a_request, setRepoData(&a_reply, temp, result, log_context); } -void DatabaseAPI::repoCreate(const Auth::RepoCreateRequest &a_request, - Auth::RepoDataReply &a_reply, +void DatabaseAPI::repoCreate(const SDMS::RepoCreateRequest &a_request, + SDMS::RepoDataReply &a_reply, LogContext log_context) { Value result; @@ -2189,13 +2191,13 @@ void DatabaseAPI::repoCreate(const Auth::RepoCreateRequest &a_request, }; // List of optional fields to check - add_if_present(&Auth::RepoCreateRequest::has_path, &Auth::RepoCreateRequest::path, "path"); - add_if_present(&Auth::RepoCreateRequest::has_pub_key, &Auth::RepoCreateRequest::pub_key, "pub_key"); - add_if_present(&Auth::RepoCreateRequest::has_address, &Auth::RepoCreateRequest::address, "address"); - add_if_present(&Auth::RepoCreateRequest::has_endpoint, &Auth::RepoCreateRequest::endpoint, "endpoint"); - add_if_present(&Auth::RepoCreateRequest::has_desc, &Auth::RepoCreateRequest::desc, "desc"); - add_if_present(&Auth::RepoCreateRequest::has_domain, &Auth::RepoCreateRequest::domain, "domain"); - add_if_present(&Auth::RepoCreateRequest::has_exp_path, &Auth::RepoCreateRequest::exp_path, "exp_path"); + add_if_present(&SDMS::RepoCreateRequest::has_path, &SDMS::RepoCreateRequest::path, "path"); + add_if_present(&SDMS::RepoCreateRequest::has_pub_key, &SDMS::RepoCreateRequest::pub_key, "pub_key"); + add_if_present(&SDMS::RepoCreateRequest::has_address, &SDMS::RepoCreateRequest::address, "address"); + add_if_present(&SDMS::RepoCreateRequest::has_endpoint, &SDMS::RepoCreateRequest::endpoint, "endpoint"); + add_if_present(&SDMS::RepoCreateRequest::has_desc, &SDMS::RepoCreateRequest::desc, "desc"); + add_if_present(&SDMS::RepoCreateRequest::has_domain, &SDMS::RepoCreateRequest::domain, "domain"); + add_if_present(&SDMS::RepoCreateRequest::has_exp_path, &SDMS::RepoCreateRequest::exp_path, "exp_path"); if (a_request.admin_size() > 0) { nlohmann::json admins = nlohmann::json::array(); @@ -2212,8 +2214,8 @@ void DatabaseAPI::repoCreate(const Auth::RepoCreateRequest &a_request, setRepoData(&a_reply, temp, result, log_context); } -void DatabaseAPI::repoUpdate(const Auth::RepoUpdateRequest &a_request, - Auth::RepoDataReply &a_reply, +void DatabaseAPI::repoUpdate(const SDMS::RepoUpdateRequest &a_request, + SDMS::RepoDataReply &a_reply, LogContext log_context) { Value result; nlohmann::json payload; @@ -2260,16 +2262,16 @@ void DatabaseAPI::repoUpdate(const Auth::RepoUpdateRequest &a_request, setRepoData(&a_reply, temp, result, log_context); } -void DatabaseAPI::repoDelete(const Auth::RepoDeleteRequest &a_request, - Anon::AckReply &a_reply, LogContext log_context) { +void DatabaseAPI::repoDelete(const SDMS::RepoDeleteRequest &a_request, + SDMS::AckReply &a_reply, LogContext log_context) { (void)a_reply; Value result; dbGet("repo/delete", {{"id", a_request.id()}}, result, log_context); } -void DatabaseAPI::repoCalcSize(const Auth::RepoCalcSizeRequest &a_request, - Auth::RepoCalcSizeReply &a_reply, +void DatabaseAPI::repoCalcSize(const SDMS::RepoCalcSizeRequest &a_request, + SDMS::RepoCalcSizeReply &a_reply, LogContext log_context) { Value result; @@ -2298,7 +2300,7 @@ void DatabaseAPI::repoCalcSize(const Auth::RepoCalcSizeRequest &a_request, TRANSLATE_END(result, log_context) } -void DatabaseAPI::setRepoData(Auth::RepoDataReply *a_reply, +void DatabaseAPI::setRepoData(SDMS::RepoDataReply *a_reply, std::vector &a_repos, const libjson::Value &a_result, LogContext log_context) { @@ -2371,8 +2373,8 @@ void DatabaseAPI::setRepoData(Auth::RepoDataReply *a_reply, } void DatabaseAPI::repoListAllocations( - const Auth::RepoListAllocationsRequest &a_request, - Auth::RepoAllocationsReply &a_reply, LogContext log_context) { + const SDMS::RepoListAllocationsRequest &a_request, + SDMS::RepoAllocationsReply &a_reply, LogContext log_context) { Value result; dbGet("repo/alloc/list/by_repo", {{"repo", a_request.id()}}, result, @@ -2382,8 +2384,8 @@ void DatabaseAPI::repoListAllocations( } void DatabaseAPI::repoListSubjectAllocations( - const Auth::RepoListSubjectAllocationsRequest &a_request, - Auth::RepoAllocationsReply &a_reply, LogContext log_context) { + const SDMS::RepoListSubjectAllocationsRequest &a_request, + SDMS::RepoAllocationsReply &a_reply, LogContext log_context) { Value result; vector> params; if (a_request.has_subject()) @@ -2399,8 +2401,8 @@ void DatabaseAPI::repoListSubjectAllocations( } void DatabaseAPI::repoListObjectAllocations( - const Auth::RepoListObjectAllocationsRequest &a_request, - Auth::RepoAllocationsReply &a_reply, LogContext log_context) { + const SDMS::RepoListObjectAllocationsRequest &a_request, + SDMS::RepoAllocationsReply &a_reply, LogContext log_context) { Value result; dbGet("repo/alloc/list/by_object", {{"object", a_request.id()}}, result, @@ -2409,7 +2411,7 @@ void DatabaseAPI::repoListObjectAllocations( setAllocData(a_reply, result, log_context); } -void DatabaseAPI::setAllocData(Auth::RepoAllocationsReply &a_reply, +void DatabaseAPI::setAllocData(SDMS::RepoAllocationsReply &a_reply, const libjson::Value &a_result, LogContext log_context) { TRANSLATE_BEGIN() @@ -2444,8 +2446,8 @@ void DatabaseAPI::setAllocData(AllocData *a_alloc, } void DatabaseAPI::repoViewAllocation( - const Auth::RepoViewAllocationRequest &a_request, - Auth::RepoAllocationsReply &a_reply, LogContext log_context) { + const SDMS::RepoViewAllocationRequest &a_request, + SDMS::RepoAllocationsReply &a_reply, LogContext log_context) { Value result; vector> params; params.push_back({"repo", a_request.repo()}); @@ -2458,8 +2460,8 @@ void DatabaseAPI::repoViewAllocation( } void DatabaseAPI::repoAllocationStats( - const Auth::RepoAllocationStatsRequest &a_request, - Auth::RepoAllocationStatsReply &a_reply, LogContext log_context) { + const SDMS::RepoAllocationStatsRequest &a_request, + SDMS::RepoAllocationStatsReply &a_reply, LogContext log_context) { Value result; vector> params; params.push_back({"repo", a_request.repo()}); @@ -2496,7 +2498,7 @@ void DatabaseAPI::setAllocStatsData(AllocStatsData &a_stats, } void DatabaseAPI::repoAllocationSet( - const Auth::RepoAllocationSetRequest &a_request, Anon::AckReply &a_reply, + const SDMS::RepoAllocationSetRequest &a_request, SDMS::AckReply &a_reply, LogContext log_context) { (void)a_reply; Value result; @@ -2510,8 +2512,8 @@ void DatabaseAPI::repoAllocationSet( } void DatabaseAPI::repoAllocationSetDefault( - const Auth::RepoAllocationSetDefaultRequest &a_request, - Anon::AckReply &a_reply, LogContext log_context) { + const SDMS::RepoAllocationSetDefaultRequest &a_request, + SDMS::AckReply &a_reply, LogContext log_context) { (void)a_reply; Value result; @@ -2556,8 +2558,8 @@ void DatabaseAPI::getPerms(const GetPermsRequest &a_request, TRANSLATE_END(result, log_context) } -void DatabaseAPI::repoAuthz(const Auth::RepoAuthzRequest &a_request, - Anon::AckReply &a_reply, LogContext log_context) { +void DatabaseAPI::repoAuthz(const SDMS::RepoAuthzRequest &a_request, + SDMS::AckReply &a_reply, LogContext log_context) { (void)a_reply; Value result; @@ -2571,8 +2573,8 @@ void DatabaseAPI::repoAuthz(const Auth::RepoAuthzRequest &a_request, result, log_context); } -void DatabaseAPI::topicListTopics(const Auth::TopicListTopicsRequest &a_request, - Auth::TopicDataReply &a_reply, +void DatabaseAPI::topicListTopics(const SDMS::TopicListTopicsRequest &a_request, + SDMS::TopicDataReply &a_reply, LogContext log_context) { Value result; vector> params; @@ -2588,8 +2590,8 @@ void DatabaseAPI::topicListTopics(const Auth::TopicListTopicsRequest &a_request, setTopicDataReply(a_reply, result, log_context); } -void DatabaseAPI::topicView(const Auth::TopicViewRequest &a_request, - Auth::TopicDataReply &a_reply, +void DatabaseAPI::topicView(const SDMS::TopicViewRequest &a_request, + SDMS::TopicDataReply &a_reply, LogContext log_context) { Value result; @@ -2598,8 +2600,8 @@ void DatabaseAPI::topicView(const Auth::TopicViewRequest &a_request, setTopicDataReply(a_reply, result, log_context); } -void DatabaseAPI::topicSearch(const Auth::TopicSearchRequest &a_request, - Auth::TopicDataReply &a_reply, +void DatabaseAPI::topicSearch(const SDMS::TopicSearchRequest &a_request, + SDMS::TopicDataReply &a_reply, LogContext log_context) { Value result; @@ -2608,7 +2610,7 @@ void DatabaseAPI::topicSearch(const Auth::TopicSearchRequest &a_request, setTopicDataReply(a_reply, result, log_context); } -void DatabaseAPI::setTopicDataReply(Auth::TopicDataReply &a_reply, +void DatabaseAPI::setTopicDataReply(SDMS::TopicDataReply &a_reply, const libjson::Value &a_result, LogContext log_context) { TRANSLATE_BEGIN() @@ -2659,7 +2661,7 @@ void DatabaseAPI::setTopicDataReply(Auth::TopicDataReply &a_reply, } void DatabaseAPI::noteCreate(const NoteCreateRequest &a_request, - Auth::NoteDataReply &a_reply, + SDMS::NoteDataReply &a_reply, LogContext log_context) { DL_DEBUG(log_context, "NoteCreate"); @@ -2677,7 +2679,7 @@ void DatabaseAPI::noteCreate(const NoteCreateRequest &a_request, } void DatabaseAPI::noteUpdate(const NoteUpdateRequest &a_request, - Auth::NoteDataReply &a_reply, + SDMS::NoteDataReply &a_reply, LogContext log_context) { DL_DEBUG(log_context, "NoteUpdate"); @@ -2697,8 +2699,8 @@ void DatabaseAPI::noteUpdate(const NoteUpdateRequest &a_request, setNoteDataReply(a_reply, result, log_context); } -void DatabaseAPI::noteCommentEdit(const Auth::NoteCommentEditRequest &a_request, - Auth::NoteDataReply &a_reply, +void DatabaseAPI::noteCommentEdit(const SDMS::NoteCommentEditRequest &a_request, + SDMS::NoteDataReply &a_reply, LogContext log_context) { Value result; vector> params; @@ -2711,8 +2713,8 @@ void DatabaseAPI::noteCommentEdit(const Auth::NoteCommentEditRequest &a_request, setNoteDataReply(a_reply, result, log_context); } -void DatabaseAPI::noteView(const Auth::NoteViewRequest &a_request, - Auth::NoteDataReply &a_reply, +void DatabaseAPI::noteView(const SDMS::NoteViewRequest &a_request, + SDMS::NoteDataReply &a_reply, LogContext log_context) { Value result; @@ -2722,8 +2724,8 @@ void DatabaseAPI::noteView(const Auth::NoteViewRequest &a_request, } void DatabaseAPI::noteListBySubject( - const Auth::NoteListBySubjectRequest &a_request, - Auth::NoteDataReply &a_reply, LogContext log_context) { + const SDMS::NoteListBySubjectRequest &a_request, + SDMS::NoteDataReply &a_reply, LogContext log_context) { Value result; dbGet("note/list/by_subject", {{"subject", a_request.subject()}}, result, @@ -2738,7 +2740,7 @@ void DatabaseAPI::notePurge(uint32_t a_age_sec, LogContext log_context) { dbGet("note/purge", {{"age_sec", to_string(a_age_sec)}}, result, log_context); } -void DatabaseAPI::setNoteDataReply(Auth::NoteDataReply &a_reply, +void DatabaseAPI::setNoteDataReply(SDMS::NoteDataReply &a_reply, const libjson::Value &a_result, LogContext log_context) { Value::ArrayConstIter i; @@ -2803,8 +2805,8 @@ void DatabaseAPI::setNoteData(NoteData *a_note, } } -void DatabaseAPI::tagSearch(const Auth::TagSearchRequest &a_request, - Auth::TagDataReply &a_reply, +void DatabaseAPI::tagSearch(const SDMS::TagSearchRequest &a_request, + SDMS::TagDataReply &a_reply, LogContext log_context) { Value result; vector> params; @@ -2821,8 +2823,8 @@ void DatabaseAPI::tagSearch(const Auth::TagSearchRequest &a_request, setTagDataReply(a_reply, result, log_context); } -void DatabaseAPI::tagListByCount(const Auth::TagListByCountRequest &a_request, - Auth::TagDataReply &a_reply, +void DatabaseAPI::tagListByCount(const SDMS::TagListByCountRequest &a_request, + SDMS::TagDataReply &a_reply, LogContext log_context) { Value result; vector> params; @@ -2837,7 +2839,7 @@ void DatabaseAPI::tagListByCount(const Auth::TagListByCountRequest &a_request, setTagDataReply(a_reply, result, log_context); } -void DatabaseAPI::setTagDataReply(Auth::TagDataReply &a_reply, +void DatabaseAPI::setTagDataReply(SDMS::TagDataReply &a_reply, const Value &a_result, LogContext log_context) { Value::ObjectConstIter j; @@ -2871,8 +2873,8 @@ void DatabaseAPI::setTagData(TagData *a_tag, a_tag->set_count(a_obj.getNumber("count")); } -void DatabaseAPI::schemaSearch(const Auth::SchemaSearchRequest &a_request, - Auth::SchemaDataReply &a_reply, +void DatabaseAPI::schemaSearch(const SDMS::SchemaSearchRequest &a_request, + SDMS::SchemaDataReply &a_reply, LogContext log_context) { libjson::Value result; vector> params; @@ -2896,8 +2898,8 @@ void DatabaseAPI::schemaSearch(const Auth::SchemaSearchRequest &a_request, setSchemaDataReply(a_reply, result, log_context); } -void DatabaseAPI::schemaView(const Auth::SchemaViewRequest &a_request, - Auth::SchemaDataReply &a_reply, +void DatabaseAPI::schemaView(const SDMS::SchemaViewRequest &a_request, + SDMS::SchemaDataReply &a_reply, LogContext log_context) { libjson::Value result; vector> params; @@ -2910,7 +2912,7 @@ void DatabaseAPI::schemaView(const Auth::SchemaViewRequest &a_request, setSchemaDataReply(a_reply, result, log_context); } -void DatabaseAPI::schemaCreate(const Auth::SchemaCreateRequest &a_request, +void DatabaseAPI::schemaCreate(const SDMS::SchemaCreateRequest &a_request, LogContext log_context) { libjson::Value result; @@ -2926,7 +2928,7 @@ void DatabaseAPI::schemaCreate(const Auth::SchemaCreateRequest &a_request, dbPost("schema/create", {}, &body, result, log_context); } -void DatabaseAPI::schemaRevise(const Auth::SchemaReviseRequest &a_request, +void DatabaseAPI::schemaRevise(const SDMS::SchemaReviseRequest &a_request, LogContext log_context) { libjson::Value result; @@ -2952,7 +2954,7 @@ void DatabaseAPI::schemaRevise(const Auth::SchemaReviseRequest &a_request, dbPost("schema/revise", {{"id", a_request.id()}}, &body, result, log_context); } -void DatabaseAPI::schemaUpdate(const Auth::SchemaUpdateRequest &a_request, +void DatabaseAPI::schemaUpdate(const SDMS::SchemaUpdateRequest &a_request, LogContext log_context) { libjson::Value result; @@ -2981,8 +2983,8 @@ void DatabaseAPI::schemaUpdate(const Auth::SchemaUpdateRequest &a_request, dbPost("schema/update", {{"id", a_request.id()}}, &body, result, log_context); } -void DatabaseAPI::schemaDelete(const Auth::SchemaDeleteRequest &a_request, - Anon::AckReply &a_reply, +void DatabaseAPI::schemaDelete(const SDMS::SchemaDeleteRequest &a_request, + SDMS::AckReply &a_reply, LogContext log_context) { (void)a_reply; libjson::Value result; @@ -2990,7 +2992,7 @@ void DatabaseAPI::schemaDelete(const Auth::SchemaDeleteRequest &a_request, dbPost("schema/delete", {{"id", a_request.id()}}, 0, result, log_context); } -void DatabaseAPI::setSchemaDataReply(Auth::SchemaDataReply &a_reply, +void DatabaseAPI::setSchemaDataReply(SDMS::SchemaDataReply &a_reply, const libjson::Value &a_result, LogContext log_context) { Value::ObjectConstIter j; @@ -3078,8 +3080,8 @@ void DatabaseAPI::schemaView(const std::string &a_id, libjson::Value &a_result, dbGet("schema/view", {{"id", a_id}}, a_result, log_context); } -void DatabaseAPI::dailyMessage(const Anon::DailyMessageRequest &a_request, - Anon::DailyMessageReply &a_reply, +void DatabaseAPI::dailyMessage(const SDMS::DailyMessageRequest &a_request, + SDMS::DailyMessageReply &a_reply, LogContext log_context) { (void)a_request; // Not used libjson::Value result; @@ -3129,8 +3131,8 @@ void DatabaseAPI::taskAbort(const std::string &a_task_id, log_context); } -void DatabaseAPI::taskInitDataGet(const Auth::DataGetRequest &a_request, - Auth::DataGetReply &a_reply, +void DatabaseAPI::taskInitDataGet(const SDMS::DataGetRequest &a_request, + SDMS::DataGetReply &a_reply, libjson::Value &a_result, LogContext log_context) { nlohmann::json payload; @@ -3172,7 +3174,7 @@ void DatabaseAPI::taskInitDataGet(const Auth::DataGetRequest &a_request, setDataGetReply(a_reply, a_result, log_context); } -void DatabaseAPI::setDataGetReply(Auth::DataGetReply &a_reply, +void DatabaseAPI::setDataGetReply(SDMS::DataGetReply &a_reply, const libjson::Value &a_result, LogContext log_context) { Value::ObjectIter t; @@ -3203,8 +3205,8 @@ void DatabaseAPI::setDataGetReply(Auth::DataGetReply &a_reply, TRANSLATE_END(a_result, log_context) } -void DatabaseAPI::taskInitDataPut(const Auth::DataPutRequest &a_request, - Auth::DataPutReply &a_reply, +void DatabaseAPI::taskInitDataPut(const SDMS::DataPutRequest &a_request, + SDMS::DataPutReply &a_reply, libjson::Value &a_result, LogContext log_context) { nlohmann::json payload; @@ -3240,7 +3242,7 @@ void DatabaseAPI::taskInitDataPut(const Auth::DataPutRequest &a_request, setDataPutReply(a_reply, a_result, log_context); } -void DatabaseAPI::setDataPutReply(Auth::DataPutReply &a_reply, +void DatabaseAPI::setDataPutReply(SDMS::DataPutReply &a_reply, const libjson::Value &a_result, LogContext log_context) { Value::ObjectIter t; @@ -3252,7 +3254,7 @@ void DatabaseAPI::setDataPutReply(Auth::DataPutReply &a_reply, Value::ArrayConstIter j; if (!obj.has("glob_data") || obj.value().size() != 1) - EXCEPT_PARAM(ID_BAD_REQUEST, "Invalid or missing upload target"); + EXCEPT_PARAM(BAD_REQUEST, "Invalid or missing upload target"); const Value::Array &arr = obj.asArray(); const Value::Object &rec = arr.begin()->asObject(); @@ -3289,8 +3291,8 @@ void DatabaseAPI::taskInitRecordCollectionDelete( } void DatabaseAPI::taskInitRecordAllocChange( - const Auth::RecordAllocChangeRequest &a_request, - Auth::RecordAllocChangeReply &a_reply, libjson::Value &a_result, + const SDMS::RecordAllocChangeRequest &a_request, + SDMS::RecordAllocChangeReply &a_reply, libjson::Value &a_result, LogContext log_context) { nlohmann::json payload; nlohmann::json ids = nlohmann::json::array(); @@ -3329,8 +3331,8 @@ void DatabaseAPI::taskInitRecordAllocChange( } void DatabaseAPI::taskInitRecordOwnerChange( - const Auth::RecordOwnerChangeRequest &a_request, - Auth::RecordOwnerChangeReply &a_reply, libjson::Value &a_result, + const SDMS::RecordOwnerChangeRequest &a_request, + SDMS::RecordOwnerChangeReply &a_reply, libjson::Value &a_result, LogContext log_context) { nlohmann::json payload; nlohmann::json ids = nlohmann::json::array(); @@ -3374,7 +3376,7 @@ void DatabaseAPI::taskInitRecordOwnerChange( } void DatabaseAPI::taskInitProjectDelete( - const Auth::ProjectDeleteRequest &a_request, Auth::TaskDataReply &a_reply, + const SDMS::ProjectDeleteRequest &a_request, SDMS::TaskDataReply &a_reply, libjson::Value &a_result, LogContext log_context) { nlohmann::json payload; nlohmann::json ids = nlohmann::json::array(); @@ -3392,8 +3394,8 @@ void DatabaseAPI::taskInitProjectDelete( } void DatabaseAPI::taskInitRepoAllocationCreate( - const Auth::RepoAllocationCreateRequest &a_request, - Auth::TaskDataReply &a_reply, libjson::Value &a_result, + const SDMS::RepoAllocationCreateRequest &a_request, + SDMS::TaskDataReply &a_reply, libjson::Value &a_result, LogContext log_context) { dbGet("repo/alloc/create", {{"subject", a_request.subject()}, @@ -3406,8 +3408,8 @@ void DatabaseAPI::taskInitRepoAllocationCreate( } void DatabaseAPI::taskInitRepoAllocationDelete( - const Auth::RepoAllocationDeleteRequest &a_request, - Auth::TaskDataReply &a_reply, libjson::Value &a_result, + const SDMS::RepoAllocationDeleteRequest &a_request, + SDMS::TaskDataReply &a_reply, libjson::Value &a_result, LogContext log_context) { dbGet("repo/alloc/delete", {{"subject", a_request.subject()}, {"repo", a_request.repo()}}, @@ -3516,7 +3518,7 @@ void DatabaseAPI::setTaskData(TaskData *a_task, * method removes tasks that are nor in READY status from the original JSON * input - this is to. */ -void DatabaseAPI::setTaskDataReply(Auth::TaskDataReply &a_reply, +void DatabaseAPI::setTaskDataReply(SDMS::TaskDataReply &a_reply, const libjson::Value &a_result, LogContext log_context) { TRANSLATE_BEGIN() @@ -3536,7 +3538,7 @@ void DatabaseAPI::setTaskDataReply(Auth::TaskDataReply &a_reply, * * JSON contains an array of task objects containing task fields. */ -void DatabaseAPI::setTaskDataReplyArray(Auth::TaskDataReply &a_reply, +void DatabaseAPI::setTaskDataReplyArray(SDMS::TaskDataReply &a_reply, const libjson::Value &a_result, LogContext log_context) { TRANSLATE_BEGIN() @@ -3596,8 +3598,8 @@ void DatabaseAPI::taskFinalize(const std::string &a_task_id, bool a_succeeded, dbPost("task/finalize", params, 0, a_result, log_context); } -void DatabaseAPI::taskList(const Auth::TaskListRequest &a_request, - Auth::TaskDataReply &a_reply, +void DatabaseAPI::taskList(const SDMS::TaskListRequest &a_request, + SDMS::TaskDataReply &a_reply, LogContext log_context) { vector> params; @@ -3629,8 +3631,8 @@ void DatabaseAPI::taskList(const Auth::TaskListRequest &a_request, setTaskDataReplyArray(a_reply, result, log_context); } -void DatabaseAPI::taskView(const Auth::TaskViewRequest &a_request, - Auth::TaskDataReply &a_reply, +void DatabaseAPI::taskView(const SDMS::TaskViewRequest &a_request, + SDMS::TaskDataReply &a_reply, LogContext log_context) { libjson::Value result; @@ -3790,13 +3792,13 @@ void DatabaseAPI::metricsPurge(uint32_t a_timestamp, LogContext log_context) { log_context); } -uint32_t DatabaseAPI::parseSearchRequest(const Auth::SearchRequest &a_request, +uint32_t DatabaseAPI::parseSearchRequest(const SDMS::SearchRequest &a_request, std::string &a_qry_begin, std::string &a_qry_end, std::string &a_qry_filter, std::string &a_params, LogContext log_context) { - string view = (a_request.mode() == SM_DATA ? "dataview" : "collview"); + string view = (a_request.mode() == SDMS::SM_DATA ? "dataview" : "collview"); if (a_request.has_published() && a_request.published()) { a_qry_begin = string("for i in ") + view + " search i.public == true"; @@ -3860,7 +3862,7 @@ uint32_t DatabaseAPI::parseSearchRequest(const Auth::SearchRequest &a_request, } // Data-only search options - if (a_request.mode() == SM_DATA) { + if (a_request.mode() == SDMS::SM_DATA) { if (a_request.has_sch_id() > 0) { a_qry_begin += " and i.sch_id == @sch"; a_params += ",\"sch_id\":\"" + a_request.sch_id() + "\""; @@ -3937,7 +3939,7 @@ uint32_t DatabaseAPI::parseSearchRequest(const Auth::SearchRequest &a_request, string(" return distinct " "{_id:i._id,title:i.title,'desc':i['desc'],owner:i.owner,owner_" "name:name,alias:i.alias") + - (a_request.mode() == SM_DATA ? ",size:i.size,md_err:i.md_err" : "") + "}"; + (a_request.mode() == SDMS::SM_DATA ? ",size:i.size,md_err:i.md_err" : "") + "}"; a_qry_begin = a_qry_begin; a_qry_end = a_qry_end; diff --git a/core/server/DatabaseAPI.hpp b/core/server/DatabaseAPI.hpp index bf94ce90b..f0df10a51 100644 --- a/core/server/DatabaseAPI.hpp +++ b/core/server/DatabaseAPI.hpp @@ -4,9 +4,7 @@ // Local public includes #include "common/DynaLog.hpp" -#include "common/SDMS.pb.h" -#include "common/SDMS_Anon.pb.h" -#include "common/SDMS_Auth.pb.h" +#include "common/envelope.pb.h" #include "common/libjson.hpp" // Third party includes @@ -38,10 +36,10 @@ class DatabaseAPI { void setClient(const std::string &a_client); void clientAuthenticateByPassword(const std::string &a_password, - Anon::AuthStatusReply &a_reply, + SDMS::AuthStatusReply &a_reply, LogContext log_context); void clientAuthenticateByToken(const std::string &a_token, - Anon::AuthStatusReply &a_reply, + SDMS::AuthStatusReply &a_reply, LogContext log_context); void clientLinkIdentity(const std::string &a_identity, LogContext log_context); @@ -72,186 +70,186 @@ class DatabaseAPI { std::vector &a_expiring_tokens, LogContext log_context); void purgeTransferRecords(size_t age); - void checkPerms(const Auth::CheckPermsRequest &a_request, - Auth::CheckPermsReply &a_reply, LogContext log_context); - void getPerms(const Auth::GetPermsRequest &a_request, - Auth::GetPermsReply &a_reply, LogContext log_context); - - void userSetAccessToken(const Auth::UserSetAccessTokenRequest &a_request, - Anon::AckReply &a_reply, LogContext log_context); - void userCreate(const Auth::UserCreateRequest &a_request, - Auth::UserDataReply &a_reply, LogContext log_context); - void userView(const Auth::UserViewRequest &a_request, - Auth::UserDataReply &a_reply, LogContext log_context); - void userUpdate(const Auth::UserUpdateRequest &a_request, - Auth::UserDataReply &a_reply, LogContext log_context); - void userListAll(const Auth::UserListAllRequest &a_request, - Auth::UserDataReply &a_reply, LogContext log_context); - void userListCollab(const Auth::UserListCollabRequest &a_request, - Auth::UserDataReply &a_reply, LogContext log_context); - void userFindByUUIDs(const Auth::UserFindByUUIDsRequest &a_request, - Auth::UserDataReply &a_reply, LogContext log_context); - void userFindByNameUID(const Auth::UserFindByNameUIDRequest &a_request, - Auth::UserDataReply &a_reply, LogContext log_context); - void userGetRecentEP(const Auth::UserGetRecentEPRequest &a_request, - Auth::UserGetRecentEPReply &a_reply, + void checkPerms(const SDMS::CheckPermsRequest &a_request, + SDMS::CheckPermsReply &a_reply, LogContext log_context); + void getPerms(const SDMS::GetPermsRequest &a_request, + SDMS::GetPermsReply &a_reply, LogContext log_context); + + void userSetAccessToken(const SDMS::UserSetAccessTokenRequest &a_request, + SDMS::AckReply &a_reply, LogContext log_context); + void userCreate(const SDMS::UserCreateRequest &a_request, + SDMS::UserDataReply &a_reply, LogContext log_context); + void userView(const SDMS::UserViewRequest &a_request, + SDMS::UserDataReply &a_reply, LogContext log_context); + void userUpdate(const SDMS::UserUpdateRequest &a_request, + SDMS::UserDataReply &a_reply, LogContext log_context); + void userListAll(const SDMS::UserListAllRequest &a_request, + SDMS::UserDataReply &a_reply, LogContext log_context); + void userListCollab(const SDMS::UserListCollabRequest &a_request, + SDMS::UserDataReply &a_reply, LogContext log_context); + void userFindByUUIDs(const SDMS::UserFindByUUIDsRequest &a_request, + SDMS::UserDataReply &a_reply, LogContext log_context); + void userFindByNameUID(const SDMS::UserFindByNameUIDRequest &a_request, + SDMS::UserDataReply &a_reply, LogContext log_context); + void userGetRecentEP(const SDMS::UserGetRecentEPRequest &a_request, + SDMS::UserGetRecentEPReply &a_reply, LogContext log_context); - void userSetRecentEP(const Auth::UserSetRecentEPRequest &a_request, - Anon::AckReply &a_reply, LogContext log_context); - - void projCreate(const Auth::ProjectCreateRequest &a_request, - Auth::ProjectDataReply &a_reply, LogContext log_context); - void projUpdate(const Auth::ProjectUpdateRequest &a_request, - Auth::ProjectDataReply &a_reply, LogContext log_context); - void projView(const Auth::ProjectViewRequest &a_request, - Auth::ProjectDataReply &a_reply, LogContext log_context); - void projList(const Auth::ProjectListRequest &a_request, - Auth::ListingReply &a_reply, LogContext log_context); - void projSearch(const std::string &a_query, Auth::ProjectDataReply &a_reply, + void userSetRecentEP(const SDMS::UserSetRecentEPRequest &a_request, + SDMS::AckReply &a_reply, LogContext log_context); + + void projCreate(const SDMS::ProjectCreateRequest &a_request, + SDMS::ProjectDataReply &a_reply, LogContext log_context); + void projUpdate(const SDMS::ProjectUpdateRequest &a_request, + SDMS::ProjectDataReply &a_reply, LogContext log_context); + void projView(const SDMS::ProjectViewRequest &a_request, + SDMS::ProjectDataReply &a_reply, LogContext log_context); + void projList(const SDMS::ProjectListRequest &a_request, + SDMS::ListingReply &a_reply, LogContext log_context); + void projSearch(const std::string &a_query, SDMS::ProjectDataReply &a_reply, LogContext log_context); - void projGetRole(const Auth::ProjectGetRoleRequest &a_request, - Auth::ProjectGetRoleReply &a_reply, LogContext log_context); - - void recordView(const Auth::RecordViewRequest &a_request, - Auth::RecordDataReply &a_reply, LogContext log_context); - void recordCreate(const Auth::RecordCreateRequest &a_request, - Auth::RecordDataReply &a_reply, LogContext log_context); - void recordCreateBatch(const Auth::RecordCreateBatchRequest &a_request, - Auth::RecordDataReply &a_reply, + void projGetRole(const SDMS::ProjectGetRoleRequest &a_request, + SDMS::ProjectGetRoleReply &a_reply, LogContext log_context); + + void recordView(const SDMS::RecordViewRequest &a_request, + SDMS::RecordDataReply &a_reply, LogContext log_context); + void recordCreate(const SDMS::RecordCreateRequest &a_request, + SDMS::RecordDataReply &a_reply, LogContext log_context); + void recordCreateBatch(const SDMS::RecordCreateBatchRequest &a_request, + SDMS::RecordDataReply &a_reply, LogContext log_context); - void recordUpdate(const Auth::RecordUpdateRequest &a_request, - Auth::RecordDataReply &a_reply, libjson::Value &result, + void recordUpdate(const SDMS::RecordUpdateRequest &a_request, + SDMS::RecordDataReply &a_reply, libjson::Value &result, LogContext log_context); - void recordUpdateBatch(const Auth::RecordUpdateBatchRequest &a_request, - Auth::RecordDataReply &a_reply, libjson::Value &result, + void recordUpdateBatch(const SDMS::RecordUpdateBatchRequest &a_request, + SDMS::RecordDataReply &a_reply, libjson::Value &result, LogContext log_context); - void recordUpdateSize(const Auth::RepoDataSizeReply &a_sizes, + void recordUpdateSize(const SDMS::RepoDataSizeReply &a_sizes, LogContext log_context); void recordUpdateSchemaError(const std::string &a_rec_id, const std::string &a_err_msg, LogContext log_context); - void recordExport(const Auth::RecordExportRequest &a_request, - Auth::RecordExportReply &a_reply, LogContext log_context); - void recordLock(const Auth::RecordLockRequest &a_request, - Auth::ListingReply &a_reply, LogContext log_context); - void recordListByAlloc(const Auth::RecordListByAllocRequest &a_request, - Auth::ListingReply &a_reply, LogContext log_context); + void recordExport(const SDMS::RecordExportRequest &a_request, + SDMS::RecordExportReply &a_reply, LogContext log_context); + void recordLock(const SDMS::RecordLockRequest &a_request, + SDMS::ListingReply &a_reply, LogContext log_context); + void recordListByAlloc(const SDMS::RecordListByAllocRequest &a_request, + SDMS::ListingReply &a_reply, LogContext log_context); void recordGetDependencyGraph( - const Auth::RecordGetDependencyGraphRequest &a_request, - Auth::ListingReply &a_reply, LogContext log_context); - - void generalSearch(const Auth::SearchRequest &a_request, - Auth::ListingReply &a_reply, LogContext log_context); - - void dataPath(const Auth::DataPathRequest &a_request, - Auth::DataPathReply &a_reply, LogContext log_context); - - void collListPublished(const Auth::CollListPublishedRequest &a_request, - Auth::ListingReply &a_reply, LogContext log_context); - void collCreate(const Auth::CollCreateRequest &a_request, - Auth::CollDataReply &a_reply, LogContext log_context); - void collUpdate(const Auth::CollUpdateRequest &a_request, - Auth::CollDataReply &a_reply, LogContext log_context); - void collView(const Auth::CollViewRequest &a_request, - Auth::CollDataReply &a_reply, LogContext log_context); - void collRead(const Auth::CollReadRequest &a_request, - Auth::ListingReply &a_reply, LogContext log_context); - void collWrite(const Auth::CollWriteRequest &a_request, - Auth::ListingReply &a_reply, LogContext log_context); - void collMove(const Auth::CollMoveRequest &a_request, Anon::AckReply &a_reply, + const SDMS::RecordGetDependencyGraphRequest &a_request, + SDMS::ListingReply &a_reply, LogContext log_context); + + void generalSearch(const SDMS::SearchRequest &a_request, + SDMS::ListingReply &a_reply, LogContext log_context); + + void dataPath(const SDMS::DataPathRequest &a_request, + SDMS::DataPathReply &a_reply, LogContext log_context); + + void collListPublished(const SDMS::CollListPublishedRequest &a_request, + SDMS::ListingReply &a_reply, LogContext log_context); + void collCreate(const SDMS::CollCreateRequest &a_request, + SDMS::CollDataReply &a_reply, LogContext log_context); + void collUpdate(const SDMS::CollUpdateRequest &a_request, + SDMS::CollDataReply &a_reply, LogContext log_context); + void collView(const SDMS::CollViewRequest &a_request, + SDMS::CollDataReply &a_reply, LogContext log_context); + void collRead(const SDMS::CollReadRequest &a_request, + SDMS::ListingReply &a_reply, LogContext log_context); + void collWrite(const SDMS::CollWriteRequest &a_request, + SDMS::ListingReply &a_reply, LogContext log_context); + void collMove(const SDMS::CollMoveRequest &a_request, SDMS::AckReply &a_reply, LogContext log_context); - void collGetParents(const Auth::CollGetParentsRequest &a_request, - Auth::CollPathReply &a_reply, LogContext log_context); - void collGetOffset(const Auth::CollGetOffsetRequest &a_request, - Auth::CollGetOffsetReply &a_reply, LogContext log_context); - - void queryList(const Auth::QueryListRequest &a_request, - Auth::ListingReply &a_reply, LogContext log_context); - void queryCreate(const Auth::QueryCreateRequest &a_request, - Auth::QueryDataReply &a_reply, LogContext log_context); - void queryUpdate(const Auth::QueryUpdateRequest &a_request, - Auth::QueryDataReply &a_reply, LogContext log_context); - void queryDelete(const Auth::QueryDeleteRequest &a_request, - Anon::AckReply &a_reply, LogContext log_context); - void queryView(const Auth::QueryViewRequest &a_request, - Auth::QueryDataReply &a_reply, LogContext log_context); - void queryExec(const Auth::QueryExecRequest &a_request, - Auth::ListingReply &a_reply, LogContext log_context); - - void aclView(const Auth::ACLViewRequest &a_request, - Auth::ACLDataReply &a_reply, LogContext log_context); - void aclUpdate(const Auth::ACLUpdateRequest &a_request, - Auth::ACLDataReply &a_reply, LogContext log_context); - void aclSharedList(const Auth::ACLSharedListRequest &a_request, - Auth::ListingReply &a_reply, LogContext log_context); - void aclSharedListItems(const Auth::ACLSharedListItemsRequest &a_request, - Auth::ListingReply &a_reply, LogContext log_context); - - void groupCreate(const Auth::GroupCreateRequest &a_request, - Auth::GroupDataReply &a_reply, LogContext log_context); - void groupUpdate(const Auth::GroupUpdateRequest &a_request, - Auth::GroupDataReply &a_reply, LogContext log_context); - void groupDelete(const Auth::GroupDeleteRequest &a_request, - Anon::AckReply &a_reply, LogContext log_context); - void groupList(const Auth::GroupListRequest &a_request, - Auth::GroupDataReply &a_reply, LogContext log_context); - void groupView(const Auth::GroupViewRequest &a_request, - Auth::GroupDataReply &a_reply, LogContext log_context); + void collGetParents(const SDMS::CollGetParentsRequest &a_request, + SDMS::CollPathReply &a_reply, LogContext log_context); + void collGetOffset(const SDMS::CollGetOffsetRequest &a_request, + SDMS::CollGetOffsetReply &a_reply, LogContext log_context); + + void queryList(const SDMS::QueryListRequest &a_request, + SDMS::ListingReply &a_reply, LogContext log_context); + void queryCreate(const SDMS::QueryCreateRequest &a_request, + SDMS::QueryDataReply &a_reply, LogContext log_context); + void queryUpdate(const SDMS::QueryUpdateRequest &a_request, + SDMS::QueryDataReply &a_reply, LogContext log_context); + void queryDelete(const SDMS::QueryDeleteRequest &a_request, + SDMS::AckReply &a_reply, LogContext log_context); + void queryView(const SDMS::QueryViewRequest &a_request, + SDMS::QueryDataReply &a_reply, LogContext log_context); + void queryExec(const SDMS::QueryExecRequest &a_request, + SDMS::ListingReply &a_reply, LogContext log_context); + + void aclView(const SDMS::ACLViewRequest &a_request, + SDMS::ACLDataReply &a_reply, LogContext log_context); + void aclUpdate(const SDMS::ACLUpdateRequest &a_request, + SDMS::ACLDataReply &a_reply, LogContext log_context); + void aclSharedList(const SDMS::ACLSharedListRequest &a_request, + SDMS::ListingReply &a_reply, LogContext log_context); + void aclSharedListItems(const SDMS::ACLSharedListItemsRequest &a_request, + SDMS::ListingReply &a_reply, LogContext log_context); + + void groupCreate(const SDMS::GroupCreateRequest &a_request, + SDMS::GroupDataReply &a_reply, LogContext log_context); + void groupUpdate(const SDMS::GroupUpdateRequest &a_request, + SDMS::GroupDataReply &a_reply, LogContext log_context); + void groupDelete(const SDMS::GroupDeleteRequest &a_request, + SDMS::AckReply &a_reply, LogContext log_context); + void groupList(const SDMS::GroupListRequest &a_request, + SDMS::GroupDataReply &a_reply, LogContext log_context); + void groupView(const SDMS::GroupViewRequest &a_request, + SDMS::GroupDataReply &a_reply, LogContext log_context); void repoList(std::vector &a_repos, LogContext log_context); - void repoList(const Auth::RepoListRequest &a_request, - Auth::RepoDataReply &a_reply, LogContext log_context); + void repoList(const SDMS::RepoListRequest &a_request, + SDMS::RepoDataReply &a_reply, LogContext log_context); void repoView(std::vector &a_repos, LogContext log_context); - void repoView(const Auth::RepoViewRequest &a_request, - Auth::RepoDataReply &a_reply, LogContext log_context); - void repoCreate(const Auth::RepoCreateRequest &a_request, - Auth::RepoDataReply &a_reply, LogContext log_context); - void repoUpdate(const Auth::RepoUpdateRequest &a_request, - Auth::RepoDataReply &a_reply, LogContext log_context); - void repoDelete(const Auth::RepoDeleteRequest &a_request, - Anon::AckReply &a_reply, LogContext log_context); - void repoCalcSize(const Auth::RepoCalcSizeRequest &a_request, - Auth::RepoCalcSizeReply &a_reply, LogContext log_context); - void repoListAllocations(const Auth::RepoListAllocationsRequest &a_request, - Auth::RepoAllocationsReply &a_reply, + void repoView(const SDMS::RepoViewRequest &a_request, + SDMS::RepoDataReply &a_reply, LogContext log_context); + void repoCreate(const SDMS::RepoCreateRequest &a_request, + SDMS::RepoDataReply &a_reply, LogContext log_context); + void repoUpdate(const SDMS::RepoUpdateRequest &a_request, + SDMS::RepoDataReply &a_reply, LogContext log_context); + void repoDelete(const SDMS::RepoDeleteRequest &a_request, + SDMS::AckReply &a_reply, LogContext log_context); + void repoCalcSize(const SDMS::RepoCalcSizeRequest &a_request, + SDMS::RepoCalcSizeReply &a_reply, LogContext log_context); + void repoListAllocations(const SDMS::RepoListAllocationsRequest &a_request, + SDMS::RepoAllocationsReply &a_reply, LogContext log_context); void repoListSubjectAllocations( - const Auth::RepoListSubjectAllocationsRequest &a_request, - Auth::RepoAllocationsReply &a_reply, LogContext log_context); + const SDMS::RepoListSubjectAllocationsRequest &a_request, + SDMS::RepoAllocationsReply &a_reply, LogContext log_context); void repoListObjectAllocations( - const Auth::RepoListObjectAllocationsRequest &a_request, - Auth::RepoAllocationsReply &a_reply, LogContext log_context); - void repoViewAllocation(const Auth::RepoViewAllocationRequest &a_request, - Auth::RepoAllocationsReply &a_reply, + const SDMS::RepoListObjectAllocationsRequest &a_request, + SDMS::RepoAllocationsReply &a_reply, LogContext log_context); + void repoViewAllocation(const SDMS::RepoViewAllocationRequest &a_request, + SDMS::RepoAllocationsReply &a_reply, LogContext log_context); - void repoAllocationStats(const Auth::RepoAllocationStatsRequest &a_request, - Auth::RepoAllocationStatsReply &a_reply, + void repoAllocationStats(const SDMS::RepoAllocationStatsRequest &a_request, + SDMS::RepoAllocationStatsReply &a_reply, LogContext log_context); - void repoAllocationSet(const Auth::RepoAllocationSetRequest &a_request, - Anon::AckReply &a_reply, LogContext log_context); + void repoAllocationSet(const SDMS::RepoAllocationSetRequest &a_request, + SDMS::AckReply &a_reply, LogContext log_context); void repoAllocationSetDefault( - const Auth::RepoAllocationSetDefaultRequest &a_request, - Anon::AckReply &a_reply, LogContext log_context); - void repoAuthz(const Auth::RepoAuthzRequest &a_request, - Anon::AckReply &a_reply, LogContext log_context); - - void topicListTopics(const Auth::TopicListTopicsRequest &a_request, - Auth::TopicDataReply &a_reply, LogContext log_context); - void topicView(const Auth::TopicViewRequest &a_request, - Auth::TopicDataReply &a_reply, LogContext log_context); - void topicSearch(const Auth::TopicSearchRequest &a_request, - Auth::TopicDataReply &a_reply, LogContext log_context); - - void noteCreate(const Auth::NoteCreateRequest &a_request, - Auth::NoteDataReply &a_reply, LogContext log_context); - void noteUpdate(const Auth::NoteUpdateRequest &a_request, - Auth::NoteDataReply &a_reply, LogContext log_context); - void noteCommentEdit(const Auth::NoteCommentEditRequest &a_request, - Auth::NoteDataReply &a_reply, LogContext log_context); - void noteView(const Auth::NoteViewRequest &a_request, - Auth::NoteDataReply &a_reply, LogContext log_context); - void noteListBySubject(const Auth::NoteListBySubjectRequest &a_request, - Auth::NoteDataReply &a_reply, LogContext log_context); + const SDMS::RepoAllocationSetDefaultRequest &a_request, + SDMS::AckReply &a_reply, LogContext log_context); + void repoAuthz(const SDMS::RepoAuthzRequest &a_request, + SDMS::AckReply &a_reply, LogContext log_context); + + void topicListTopics(const SDMS::TopicListTopicsRequest &a_request, + SDMS::TopicDataReply &a_reply, LogContext log_context); + void topicView(const SDMS::TopicViewRequest &a_request, + SDMS::TopicDataReply &a_reply, LogContext log_context); + void topicSearch(const SDMS::TopicSearchRequest &a_request, + SDMS::TopicDataReply &a_reply, LogContext log_context); + + void noteCreate(const SDMS::NoteCreateRequest &a_request, + SDMS::NoteDataReply &a_reply, LogContext log_context); + void noteUpdate(const SDMS::NoteUpdateRequest &a_request, + SDMS::NoteDataReply &a_reply, LogContext log_context); + void noteCommentEdit(const SDMS::NoteCommentEditRequest &a_request, + SDMS::NoteDataReply &a_reply, LogContext log_context); + void noteView(const SDMS::NoteViewRequest &a_request, + SDMS::NoteDataReply &a_reply, LogContext log_context); + void noteListBySubject(const SDMS::NoteListBySubjectRequest &a_request, + SDMS::NoteDataReply &a_reply, LogContext log_context); void notePurge(uint32_t a_age_sec, LogContext log_context); void taskLoadReady(libjson::Value &a_result, LogContext log_context); @@ -261,34 +259,34 @@ class DatabaseAPI { void taskAbort(const std::string &a_task_id, const std::string &a_msg, libjson::Value &a_task_reply, LogContext log_context); - void taskInitDataGet(const Auth::DataGetRequest &a_request, - Auth::DataGetReply &a_reply, libjson::Value &a_result, + void taskInitDataGet(const SDMS::DataGetRequest &a_request, + SDMS::DataGetReply &a_reply, libjson::Value &a_result, LogContext log_context); - void taskInitDataPut(const Auth::DataPutRequest &a_request, - Auth::DataPutReply &a_reply, libjson::Value &a_result, + void taskInitDataPut(const SDMS::DataPutRequest &a_request, + SDMS::DataPutReply &a_reply, libjson::Value &a_result, LogContext log_context); void taskInitRecordCollectionDelete(const std::vector &a_ids, - Auth::TaskDataReply &a_reply, + SDMS::TaskDataReply &a_reply, libjson::Value &a_result, LogContext log_context); void - taskInitRecordAllocChange(const Auth::RecordAllocChangeRequest &a_request, - Auth::RecordAllocChangeReply &a_reply, + taskInitRecordAllocChange(const SDMS::RecordAllocChangeRequest &a_request, + SDMS::RecordAllocChangeReply &a_reply, libjson::Value &a_result, LogContext log_context); void - taskInitRecordOwnerChange(const Auth::RecordOwnerChangeRequest &a_request, - Auth::RecordOwnerChangeReply &a_reply, + taskInitRecordOwnerChange(const SDMS::RecordOwnerChangeRequest &a_request, + SDMS::RecordOwnerChangeReply &a_reply, libjson::Value &a_result, LogContext log_context); void taskInitRepoAllocationCreate( - const Auth::RepoAllocationCreateRequest &a_request, - Auth::TaskDataReply &a_reply, libjson::Value &a_result, + const SDMS::RepoAllocationCreateRequest &a_request, + SDMS::TaskDataReply &a_reply, libjson::Value &a_result, LogContext log_context); void taskInitRepoAllocationDelete( - const Auth::RepoAllocationDeleteRequest &a_request, - Auth::TaskDataReply &a_reply, libjson::Value &a_result, + const SDMS::RepoAllocationDeleteRequest &a_request, + SDMS::TaskDataReply &a_reply, libjson::Value &a_result, LogContext log_context); - void taskInitProjectDelete(const Auth::ProjectDeleteRequest &a_request, - Auth::TaskDataReply &a_reply, + void taskInitProjectDelete(const SDMS::ProjectDeleteRequest &a_request, + SDMS::TaskDataReply &a_reply, libjson::Value &a_result, LogContext log_context); void taskStart(const std::string &a_task_id, libjson::Value &a_result, LogContext log_context); @@ -298,34 +296,34 @@ class DatabaseAPI { void taskFinalize(const std::string &a_task_id, bool a_succeeded, const std::string &a_msg, libjson::Value &a_result, LogContext log_context); - void taskList(const Auth::TaskListRequest &a_request, - Auth::TaskDataReply &a_reply, LogContext log_context); - void taskView(const Auth::TaskViewRequest &a_request, - Auth::TaskDataReply &a_reply, LogContext log_context); + void taskList(const SDMS::TaskListRequest &a_request, + SDMS::TaskDataReply &a_reply, LogContext log_context); + void taskView(const SDMS::TaskViewRequest &a_request, + SDMS::TaskDataReply &a_reply, LogContext log_context); void taskPurge(uint32_t a_age_sec, LogContext log_context); - void tagSearch(const Auth::TagSearchRequest &a_request, - Auth::TagDataReply &a_reply, LogContext log_context); - void tagListByCount(const Auth::TagListByCountRequest &a_request, - Auth::TagDataReply &a_reply, LogContext log_context); + void tagSearch(const SDMS::TagSearchRequest &a_request, + SDMS::TagDataReply &a_reply, LogContext log_context); + void tagListByCount(const SDMS::TagListByCountRequest &a_request, + SDMS::TagDataReply &a_reply, LogContext log_context); - void schemaSearch(const Auth::SchemaSearchRequest &a_request, - Auth::SchemaDataReply &a_reply, LogContext log_context); - void schemaView(const Auth::SchemaViewRequest &a_request, - Auth::SchemaDataReply &a_reply, LogContext log_context); + void schemaSearch(const SDMS::SchemaSearchRequest &a_request, + SDMS::SchemaDataReply &a_reply, LogContext log_context); + void schemaView(const SDMS::SchemaViewRequest &a_request, + SDMS::SchemaDataReply &a_reply, LogContext log_context); void schemaView(const std::string &a_id, libjson::Value &a_result, LogContext log_context); - void schemaCreate(const Auth::SchemaCreateRequest &a_request, + void schemaCreate(const SDMS::SchemaCreateRequest &a_request, LogContext log_context); - void schemaRevise(const Auth::SchemaReviseRequest &a_request, + void schemaRevise(const SDMS::SchemaReviseRequest &a_request, LogContext log_context); - void schemaUpdate(const Auth::SchemaUpdateRequest &a_request, + void schemaUpdate(const SDMS::SchemaUpdateRequest &a_request, LogContext log_context); - void schemaDelete(const Auth::SchemaDeleteRequest &a_request, - Anon::AckReply &a_reply, LogContext log_context); + void schemaDelete(const SDMS::SchemaDeleteRequest &a_request, + SDMS::AckReply &a_reply, LogContext log_context); - void dailyMessage(const Anon::DailyMessageRequest &a_request, - Anon::DailyMessageReply &a_reply, LogContext log_context); + void dailyMessage(const SDMS::DailyMessageRequest &a_request, + SDMS::DailyMessageReply &a_reply, LogContext log_context); void metricsUpdateMsgCounts( uint32_t a_timestamp, uint32_t a_total, @@ -342,66 +340,66 @@ class DatabaseAPI { const std::vector> &a_params, const std::string *a_body, libjson::Value &a_result, LogContext); - void setAuthStatus(Anon::AuthStatusReply &a_reply, + void setAuthStatus(SDMS::AuthStatusReply &a_reply, const libjson::Value &a_result); - void setUserData(Auth::UserDataReply &a_reply, const libjson::Value &a_result, + void setUserData(SDMS::UserDataReply &a_reply, const libjson::Value &a_result, LogContext log_context); - void setProjectData(Auth::ProjectDataReply &a_reply, + void setProjectData(SDMS::ProjectDataReply &a_reply, const libjson::Value &a_result, LogContext log_context); - void setRecordData(Auth::RecordDataReply &a_reply, + void setRecordData(SDMS::RecordDataReply &a_reply, const libjson::Value &a_result, LogContext log_context); - void setCollData(Auth::CollDataReply &a_reply, const libjson::Value &a_result, + void setCollData(SDMS::CollDataReply &a_reply, const libjson::Value &a_result, LogContext log_context); - void setCollPathData(Auth::CollPathReply &a_reply, + void setCollPathData(SDMS::CollPathReply &a_reply, const libjson::Value &a_result, LogContext log_context); - void setQueryData(Auth::QueryDataReply &a_reply, + void setQueryData(SDMS::QueryDataReply &a_reply, const libjson::Value &a_result, LogContext log_context); - void setListingDataReply(Auth::ListingReply &a_reply, + void setListingDataReply(SDMS::ListingReply &a_reply, const libjson::Value &a_result, LogContext log_context); void setListingData(ListingData *a_item, const libjson::Value::Object &a_obj, LogContext log_context); - void setGroupData(Auth::GroupDataReply &a_reply, + void setGroupData(SDMS::GroupDataReply &a_reply, const libjson::Value &a_result, LogContext log_context); - void setACLData(Auth::ACLDataReply &a_reply, const libjson::Value &a_result, + void setACLData(SDMS::ACLDataReply &a_reply, const libjson::Value &a_result, LogContext log_context); - void setAllocData(Auth::RepoAllocationsReply &a_reply, + void setAllocData(SDMS::RepoAllocationsReply &a_reply, const libjson::Value &a_result, LogContext log_context); void setAllocData(AllocData *a_alloc, const libjson::Value::Object &a_obj, LogContext log_context); - void setRepoData(Auth::RepoDataReply *a_reply, std::vector &a_repos, + void setRepoData(SDMS::RepoDataReply *a_reply, std::vector &a_repos, const libjson::Value &a_result, LogContext log_context); void setAllocStatsData(AllocStatsData &a_stats, const libjson::Value::Object &a_object, LogContext log_context); - void setNoteDataReply(Auth::NoteDataReply &a_reply, + void setNoteDataReply(SDMS::NoteDataReply &a_reply, const libjson::Value &a_result, LogContext log_context); void setNoteData(NoteData *a_item, const libjson::Value::Object &a_obj, LogContext log_context); - void setTaskDataReply(Auth::TaskDataReply &a_reply, + void setTaskDataReply(SDMS::TaskDataReply &a_reply, const libjson::Value &a_result, LogContext log_context); - void setTaskDataReplyArray(Auth::TaskDataReply &a_reply, + void setTaskDataReplyArray(SDMS::TaskDataReply &a_reply, const libjson::Value &a_result, LogContext log_context); void setTaskData(TaskData *a_task, const libjson::Value &a_task_json, LogContext log_context); - void setDataGetReply(Auth::DataGetReply &a_reply, + void setDataGetReply(SDMS::DataGetReply &a_reply, const libjson::Value &a_result, LogContext log_context); - void setDataPutReply(Auth::DataPutReply &a_reply, + void setDataPutReply(SDMS::DataPutReply &a_reply, const libjson::Value &a_result, LogContext log_context); - void setTagDataReply(Auth::TagDataReply &a_reply, + void setTagDataReply(SDMS::TagDataReply &a_reply, const libjson::Value &a_result, LogContext log_context); void setTagData(TagData *a_tag, const libjson::Value::Object &a_obj, LogContext log_context); - void setTopicDataReply(Auth::TopicDataReply &a_reply, + void setTopicDataReply(SDMS::TopicDataReply &a_reply, const libjson::Value &a_result, LogContext log_context); - void setSchemaDataReply(Auth::SchemaDataReply &a_reply, + void setSchemaDataReply(SDMS::SchemaDataReply &a_reply, const libjson::Value &a_result, LogContext log_context); void setSchemaData(SchemaData *a_schema, const libjson::Value::Object &a_obj); - uint32_t parseSearchRequest(const Auth::SearchRequest &a_request, + uint32_t parseSearchRequest(const SDMS::SearchRequest &a_request, std::string &a_qry_begin, std::string &a_qry_end, std::string &a_filter, std::string &a_params, LogContext log_context); diff --git a/core/server/GlobusAPI.cpp b/core/server/GlobusAPI.cpp index 7cc42b4cd..b0834206a 100644 --- a/core/server/GlobusAPI.cpp +++ b/core/server/GlobusAPI.cpp @@ -5,6 +5,7 @@ #include "common/DynaLog.hpp" #include "common/TraceException.hpp" #include "common/Util.hpp" +#include "common/envelope.pb.h" // Standard includes #include @@ -226,7 +227,7 @@ std::string GlobusAPI::getSubmissionID(const std::string &a_acc_token) { try { if (!raw_result.size()) - EXCEPT_PARAM(ID_SERVICE_ERROR, "Empty response. Code: " << code); + EXCEPT_PARAM(SERVICE_ERROR, "Empty response. Code: " << code); Value result; @@ -239,7 +240,7 @@ std::string GlobusAPI::getSubmissionID(const std::string &a_acc_token) { return resp_obj.getString("value"); } catch (libjson::ParseError &e) { DL_DEBUG(m_log_context, "PARSE FAILED! " << raw_result); - EXCEPT_PARAM(ID_SERVICE_ERROR, + EXCEPT_PARAM(SERVICE_ERROR, "Globus submission API call returned invalid JSON."); } catch (TraceException &e) { DL_DEBUG(m_log_context, raw_result); @@ -247,7 +248,7 @@ std::string GlobusAPI::getSubmissionID(const std::string &a_acc_token) { throw; } catch (...) { DL_DEBUG(m_log_context, "UNEXPECTED/MISSING JSON! " << raw_result); - EXCEPT_PARAM(ID_SERVICE_ERROR, + EXCEPT_PARAM(SERVICE_ERROR, "Globus submission API call returned unexpected content"); } } @@ -292,7 +293,7 @@ string GlobusAPI::transfer( try { if (!raw_result.size()) - EXCEPT_PARAM(ID_SERVICE_ERROR, "Empty response. Code: " << code); + EXCEPT_PARAM(SERVICE_ERROR, "Empty response. Code: " << code); Value result; @@ -307,14 +308,14 @@ string GlobusAPI::transfer( string &code = resp_obj.getString("code"); if (code.compare("Accepted") != 0) - EXCEPT_PARAM(ID_SERVICE_ERROR, "Request not accepted (" << code << ")"); + EXCEPT_PARAM(SERVICE_ERROR, "Request not accepted (" << code << ")"); string &task_id = resp_obj.getString("task_id"); return task_id; } catch (libjson::ParseError &e) { DL_ERROR(m_log_context, "PARSE FAILED! " << raw_result); - EXCEPT_PARAM(ID_SERVICE_ERROR, + EXCEPT_PARAM(SERVICE_ERROR, "Globus transfer API call returned invalid JSON."); } catch (TraceException &e) { DL_ERROR(m_log_context, raw_result); @@ -322,7 +323,7 @@ string GlobusAPI::transfer( throw; } catch (...) { DL_ERROR(m_log_context, "UNEXPECTED EXCEPTION " << raw_result); - EXCEPT_PARAM(ID_SERVICE_ERROR, + EXCEPT_PARAM(SERVICE_ERROR, "Globus transfer API call returned unexpected content"); } } @@ -344,7 +345,7 @@ bool GlobusAPI::checkTransferStatus(const std::string &a_task_id, try { if (!raw_result.size()) { - EXCEPT_PARAM(ID_SERVICE_ERROR, "Empty response. Code: " << code); + EXCEPT_PARAM(SERVICE_ERROR, "Empty response. Code: " << code); } Value result; @@ -400,7 +401,7 @@ bool GlobusAPI::checkTransferStatus(const std::string &a_task_id, } } catch (libjson::ParseError &e) { DL_ERROR(m_log_context, "PARSE FAILED! " << raw_result); - EXCEPT_PARAM(ID_SERVICE_ERROR, + EXCEPT_PARAM(SERVICE_ERROR, "Globus task view API call returned invalid JSON."); } catch (TraceException &e) { DL_ERROR(m_log_context, raw_result); @@ -408,7 +409,7 @@ bool GlobusAPI::checkTransferStatus(const std::string &a_task_id, throw; } catch (...) { DL_ERROR(m_log_context, "UNEXPECTED/MISSING JSON! " << raw_result); - EXCEPT_PARAM(ID_SERVICE_ERROR, + EXCEPT_PARAM(SERVICE_ERROR, "Globus task view API call returned unexpected content"); } @@ -421,7 +422,7 @@ bool GlobusAPI::checkTransferStatus(const std::string &a_task_id, try { if (!raw_result.size()) - EXCEPT_PARAM(ID_SERVICE_ERROR, "Empty response. Code: " << code); + EXCEPT_PARAM(SERVICE_ERROR, "Empty response. Code: " << code); Value result; @@ -434,7 +435,7 @@ bool GlobusAPI::checkTransferStatus(const std::string &a_task_id, string &data_type = resp_obj.getString("DATA_TYPE"); if (data_type.compare("event_list") != 0) - EXCEPT(ID_SERVICE_ERROR, "Invalid DATA_TYPE field."); + EXCEPT(SERVICE_ERROR, "Invalid DATA_TYPE field."); vector events; @@ -457,7 +458,7 @@ bool GlobusAPI::checkTransferStatus(const std::string &a_task_id, return eventsHaveErrors(events, a_status, a_err_msg); } catch (libjson::ParseError &e) { DL_ERROR(m_log_context, "PARSE FAILED! " << raw_result); - EXCEPT_PARAM(ID_SERVICE_ERROR, + EXCEPT_PARAM(SERVICE_ERROR, "Globus task event list API call returned invalid JSON."); } catch (TraceException &e) { DL_ERROR(m_log_context, raw_result); @@ -465,7 +466,7 @@ bool GlobusAPI::checkTransferStatus(const std::string &a_task_id, throw; } catch (...) { DL_ERROR(m_log_context, "UNEXPECTED/MISSING JSON! " << raw_result); - EXCEPT_PARAM(ID_SERVICE_ERROR, + EXCEPT_PARAM(SERVICE_ERROR, "Globus task event list API call returned unexpected content"); } } @@ -480,7 +481,7 @@ void GlobusAPI::cancelTask(const std::string &a_task_id, try { if (!raw_result.size()) - EXCEPT_PARAM(ID_SERVICE_ERROR, "Empty response. Code: " << code); + EXCEPT_PARAM(SERVICE_ERROR, "Empty response. Code: " << code); Value result; @@ -493,11 +494,11 @@ void GlobusAPI::cancelTask(const std::string &a_task_id, string &resp_code = resp_obj.getString("code"); if (resp_code != "Canceled") - EXCEPT_PARAM(ID_SERVICE_ERROR, + EXCEPT_PARAM(SERVICE_ERROR, "Unexpected 'code' value returned: " << resp_code); } catch (libjson::ParseError &e) { DL_ERROR(m_log_context, "PARSE FAILED! " << raw_result); - EXCEPT_PARAM(ID_SERVICE_ERROR, + EXCEPT_PARAM(SERVICE_ERROR, "Globus cancel task API call returned invalid JSON."); } catch (TraceException &e) { DL_ERROR(m_log_context, raw_result); @@ -505,7 +506,7 @@ void GlobusAPI::cancelTask(const std::string &a_task_id, throw; } catch (...) { DL_ERROR(m_log_context, "UNEXPECTED/MISSING JSON! " << raw_result); - EXCEPT_PARAM(ID_SERVICE_ERROR, + EXCEPT_PARAM(SERVICE_ERROR, "Globus cancel task API call returned unexpected content"); } } @@ -555,7 +556,7 @@ void GlobusAPI::getEndpointInfo(const std::string &a_ep_id, Value result; try { if (!raw_result.size()) - EXCEPT_PARAM(ID_SERVICE_ERROR, "Empty response. Code: " << code); + EXCEPT_PARAM(SERVICE_ERROR, "Empty response. Code: " << code); result.fromString(raw_result); @@ -599,7 +600,7 @@ void GlobusAPI::getEndpointInfo(const std::string &a_ep_id, } } catch (libjson::ParseError &e) { DL_ERROR(m_log_context, "PARSE FAILED! " << raw_result); - EXCEPT_PARAM(ID_SERVICE_ERROR, + EXCEPT_PARAM(SERVICE_ERROR, "Globus endpoint API call returned invalid JSON."); } catch (TraceException &e) { DL_ERROR(m_log_context, raw_result); @@ -608,7 +609,7 @@ void GlobusAPI::getEndpointInfo(const std::string &a_ep_id, throw; } catch (exception &e) { DL_ERROR(m_log_context, "UNEXPECTED/MISSING JSON! " << raw_result); - EXCEPT_PARAM(ID_SERVICE_ERROR, + EXCEPT_PARAM(SERVICE_ERROR, "Globus endpoint API call returned unexpected content"); } } @@ -625,7 +626,7 @@ void GlobusAPI::refreshAccessToken(const std::string &a_ref_tok, if (!raw_result.size()) { EXCEPT_PARAM( - ID_SERVICE_ERROR, + SERVICE_ERROR, "Globus token API call returned empty response. Code: " << code); } @@ -643,7 +644,7 @@ void GlobusAPI::refreshAccessToken(const std::string &a_ref_tok, } catch (libjson::ParseError &e) { DL_ERROR(m_log_context, "PARSE FAILED! Globus token API call returned invalid JSON"); - EXCEPT_PARAM(ID_SERVICE_ERROR, + EXCEPT_PARAM(SERVICE_ERROR, "Globus token API call returned invalid JSON."); } catch (TraceException &e) { DL_ERROR(m_log_context, raw_result); @@ -651,7 +652,7 @@ void GlobusAPI::refreshAccessToken(const std::string &a_ref_tok, throw; } catch (exception &e) { DL_ERROR(m_log_context, "UNEXPECTED/MISSING JSON! " << raw_result); - EXCEPT_PARAM(ID_SERVICE_ERROR, + EXCEPT_PARAM(SERVICE_ERROR, "Globus token API call returned unexpected content"); } } @@ -661,9 +662,9 @@ void GlobusAPI::checkResponsCode(long a_code, if (a_code < 200 || a_code > 202) { libjson::Value::ObjectIter i = a_body.find("message"); if (i == a_body.end()) - EXCEPT_PARAM(ID_SERVICE_ERROR, "Request failed, code: " << a_code); + EXCEPT_PARAM(SERVICE_ERROR, "Request failed, code: " << a_code); else - EXCEPT_PARAM(ID_SERVICE_ERROR, + EXCEPT_PARAM(SERVICE_ERROR, "Request failed, code: " << a_code << ", reason: " << i->second.asString()); } diff --git a/core/server/GlobusAPI.hpp b/core/server/GlobusAPI.hpp index c571a1e4a..20c856411 100644 --- a/core/server/GlobusAPI.hpp +++ b/core/server/GlobusAPI.hpp @@ -7,7 +7,6 @@ // Local public includes #include "common/DynaLog.hpp" -#include "common/SDMS.pb.h" #include "common/libjson.hpp" // Third party includes diff --git a/core/server/TaskMgr.cpp b/core/server/TaskMgr.cpp index ef8003b6d..0d284e348 100644 --- a/core/server/TaskMgr.cpp +++ b/core/server/TaskMgr.cpp @@ -7,9 +7,9 @@ // Local public includes #include "common/DynaLog.hpp" -#include "common/SDMS.pb.h" #include "common/TraceException.hpp" #include "common/libjson.hpp" +#include "common/envelope.pb.h" // Standard includes #include diff --git a/core/server/TaskMgr.hpp b/core/server/TaskMgr.hpp index 52aacf4dd..f24c1dc52 100644 --- a/core/server/TaskMgr.hpp +++ b/core/server/TaskMgr.hpp @@ -8,8 +8,6 @@ #include "ITaskWorker.hpp" // Local public includes -#include "common/SDMS.pb.h" -#include "common/SDMS_Auth.pb.h" #include "common/libjson.hpp" // Standard includes diff --git a/core/server/TaskWorker.cpp b/core/server/TaskWorker.cpp index 1278db017..9da9c9423 100644 --- a/core/server/TaskWorker.cpp +++ b/core/server/TaskWorker.cpp @@ -11,8 +11,12 @@ #include "common/ICommunicator.hpp" #include "common/IMessage.hpp" #include "common/MessageFactory.hpp" -#include "common/SDMS.pb.h" +#include "common/envelope.pb.h" #include "common/SocketOptions.hpp" +#include "common/enums/task_command.pb.h" +#include "common/enums/encryption.pb.h" +#include "common/enums/task_type.pb.h" +#include "common/enums/access_token_type.pb.h" // Standard includes #include "common/TraceException.hpp" @@ -40,11 +44,11 @@ TaskWorker::TaskWorker(ITaskMgr &a_mgr, uint32_t a_worker_id, m_thread = std::make_unique(&TaskWorker::workerThread, this, log_context); - m_execute[TC_RAW_DATA_TRANSFER] = &cmdRawDataTransfer; - m_execute[TC_RAW_DATA_DELETE] = &cmdRawDataDelete; - m_execute[TC_RAW_DATA_UPDATE_SIZE] = &cmdRawDataUpdateSize; - m_execute[TC_ALLOC_CREATE] = &cmdAllocCreate; - m_execute[TC_ALLOC_DELETE] = &cmdAllocDelete; + m_execute[::SDMS::TC_RAW_DATA_TRANSFER] = &cmdRawDataTransfer; + m_execute[::SDMS::TC_RAW_DATA_DELETE] = &cmdRawDataDelete; + m_execute[::SDMS::TC_RAW_DATA_UPDATE_SIZE] = &cmdRawDataUpdateSize; + m_execute[::SDMS::TC_ALLOC_CREATE] = &cmdAllocCreate; + m_execute[::SDMS::TC_ALLOC_DELETE] = &cmdAllocDelete; } TaskWorker::~TaskWorker() { @@ -101,7 +105,7 @@ void TaskWorker::workerThread(LogContext log_context) { if (obj.has("step")) { step = obj.asNumber(); - } else if (cmd != TC_STOP) { + } else if (cmd != SDMS::TC_STOP) { EXCEPT(1, "Reply missing step value"); } @@ -111,7 +115,7 @@ void TaskWorker::workerThread(LogContext log_context) { "TASK_ID: " << m_task->task_id << ", Step: " << step); response = m_execute[cmd](*this, params, log_context); - } else if (cmd == TC_STOP) { + } else if (cmd == SDMS::TC_STOP) { DL_DEBUG(log_context, "TASK_ID: " << m_task->task_id << ", STOP at step: " << step); m_mgr.newTasks(params, log_context); @@ -344,7 +348,7 @@ ICommunicator::Response TaskWorker::cmdRawDataDelete(TaskWorker &me, auto message_req = msg_factory.create(MessageType::GOOGLE_PROTOCOL_BUFFER); auto del_req = - std::make_unique(); // del_req; + std::make_unique(); // del_req; for (; i < j; i++, id++) { RecordDataLocation *loc = del_req->add_loc(); loc->set_id(id->asString()); @@ -371,7 +375,7 @@ TaskWorker::cmdRawDataUpdateSize(TaskWorker &me, const Value &a_task_params, const string &repo_id = obj.getString("repo_id"); const string &path = obj.getString("repo_path"); const Value::Array &ids = obj.getArray("ids"); - auto size_req = std::make_unique(); // sz_req; + auto size_req = std::make_unique(); // sz_req; // RecordDataLocation * loc; MessageFactory msg_factory; @@ -399,7 +403,7 @@ TaskWorker::cmdRawDataUpdateSize(TaskWorker &me, const Value &a_task_params, } auto proto_msg = std::get(response.message->getPayload()); - auto size_reply = dynamic_cast(proto_msg); + auto size_reply = dynamic_cast(proto_msg); if (size_reply != 0) { if (size_reply->size_size() != (int)ids.size()) { DL_ERROR(log_context, @@ -433,7 +437,7 @@ ICommunicator::Response TaskWorker::cmdAllocCreate(TaskWorker &me, MessageFactory msg_factory; auto message = msg_factory.create(MessageType::GOOGLE_PROTOCOL_BUFFER); - auto req = std::make_unique(); + auto req = std::make_unique(); req->set_path(path); message->setPayload(std::move(req)); @@ -455,7 +459,7 @@ ICommunicator::Response TaskWorker::cmdAllocDelete(TaskWorker &me, MessageFactory msg_factory; auto message = msg_factory.create(MessageType::GOOGLE_PROTOCOL_BUFFER); - auto req = std::make_unique(); + auto req = std::make_unique(); req->set_path(path); message->setPayload(std::move(req)); log_context.correlation_id = @@ -653,7 +657,7 @@ TaskWorker::repoSendRecv(const string &a_repo_id, auto proto_msg = std::get(response.message->getPayload()); - auto nack = dynamic_cast(proto_msg); + auto nack = dynamic_cast(proto_msg); if (nack != 0) { ErrorCode code = nack->err_code(); string msg = diff --git a/core/server/Version.hpp.in b/core/server/Version.hpp.in index be239e85f..b186730d3 100644 --- a/core/server/Version.hpp.in +++ b/core/server/Version.hpp.in @@ -18,6 +18,22 @@ namespace SDMS { constexpr int PATCH = @DATAFED_FOXX_API_PATCH@; } } + + namespace protocol { + namespace version { + constexpr int MAJOR = @DATAFED_COMMON_PROTOCOL_API_MAJOR@; + constexpr int MINOR = @DATAFED_COMMON_PROTOCOL_API_MINOR@; + constexpr int PATCH = @DATAFED_COMMON_PROTOCOL_API_PATCH@; + } + } + + namespace release { + constexpr int YEAR = @DATAFED_RELEASE_YEAR@; + constexpr int MONTH = @DATAFED_RELEASE_MONTH@; + constexpr int DAY = @DATAFED_RELEASE_DAY@; + constexpr int HOUR = @DATAFED_RELEASE_HOUR@; + constexpr int MINUTE = @DATAFED_RELEASE_MINUTE@; + } } #endif // CORE_VERSION_HPP diff --git a/core/server/main.cpp b/core/server/main.cpp index 92cf985ee..48cca3b5b 100644 --- a/core/server/main.cpp +++ b/core/server/main.cpp @@ -7,8 +7,7 @@ #include "common/DynaLog.hpp" #include "common/TraceException.hpp" #include "common/Util.hpp" -// messaging version -#include "common/Version.pb.h" +#include "Version.hpp" // Third party includes #include @@ -105,13 +104,13 @@ int main(int a_argc, char **a_argv) { } if (opt_map.count("version")) { - cout << "Release Version: " << DATAFED_RELEASE_YEAR << "." - << DATAFED_RELEASE_MONTH << "." << DATAFED_RELEASE_DAY << "." - << DATAFED_RELEASE_HOUR << "." << DATAFED_RELEASE_MINUTE + cout << "Release Version: " << release::YEAR << "." + << release::MONTH << "." << release::DAY << "." + << release::HOUR << "." << release::MINUTE << std::endl; - cout << "Messaging API: " << DATAFED_COMMON_PROTOCOL_API_MAJOR << "." - << DATAFED_COMMON_PROTOCOL_API_MINOR << "." - << DATAFED_COMMON_PROTOCOL_API_PATCH << endl; + cout << "Messaging API: " << protocol::version::MAJOR << "." + << protocol::version::MINOR << "." + << protocol::version::PATCH << endl; cout << "Core Server: " << core::version::MAJOR << "." << core::version::MINOR << "." << core::version::PATCH << endl; return 0; @@ -120,7 +119,7 @@ int main(int a_argc, char **a_argv) { if (cfg_file.size()) { ifstream optfile(cfg_file.c_str()); if (!optfile.is_open()) - EXCEPT_PARAM(ID_CLIENT_ERROR, + EXCEPT_PARAM(CLIENT_ERROR, "Could not open config file: " << cfg_file); po::store(po::parse_config_file(optfile, opts, false), opt_map); diff --git a/core/server/tests/unit/test_DatabaseAPI.cpp b/core/server/tests/unit/test_DatabaseAPI.cpp index 23812af2b..1c302eed4 100644 --- a/core/server/tests/unit/test_DatabaseAPI.cpp +++ b/core/server/tests/unit/test_DatabaseAPI.cpp @@ -34,6 +34,13 @@ class DatabaseAPITestHelper : public DatabaseAPI { } }; +struct GlobalProtobufTeardown { + ~GlobalProtobufTeardown() { + // This is the teardown function that runs once at the end + google::protobuf::ShutdownProtobufLibrary(); + } +}; + struct CurlGlobalFixture { CurlGlobalFixture() { curl_global_init(CURL_GLOBAL_DEFAULT); } @@ -43,6 +50,9 @@ struct CurlGlobalFixture { // Register fixture to run once per test module BOOST_TEST_GLOBAL_CONFIGURATION(CurlGlobalFixture); +// Declare a global fixture instance +BOOST_GLOBAL_FIXTURE(GlobalProtobufTeardown); + const std::string url("https://localhost:8529"); const std::string user("bob"); const std::string pass("open_sesame"); From 62224ed4df4563832323a627c3722f697fa69542 Mon Sep 17 00:00:00 2001 From: JoshuaSBrown Date: Thu, 5 Feb 2026 16:57:48 -0500 Subject: [PATCH 04/14] refactor: address duplicate include --- core/server/DatabaseAPI.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/core/server/DatabaseAPI.cpp b/core/server/DatabaseAPI.cpp index 7363d4c18..7a297b21e 100644 --- a/core/server/DatabaseAPI.cpp +++ b/core/server/DatabaseAPI.cpp @@ -7,7 +7,6 @@ #include "common/envelope.pb.h" #include "common/TraceException.hpp" #include "common/Util.hpp" -#include "common/envelope.pb.h" #include "common/enums/access_token_type.pb.h" #include "common/enums/search_mode.pb.h" From 188830f7f91b0644fd4337ff60b7930935a78ea0 Mon Sep 17 00:00:00 2001 From: JoshuaSBrown Date: Thu, 5 Feb 2026 12:21:33 -0500 Subject: [PATCH 05/14] refactor: use proto3 modern library for repo server. --- repository/server/Config.hpp | 4 +- repository/server/RepoServer.cpp | 25 ++++------ repository/server/RequestWorker.cpp | 73 ++++++++++++++--------------- repository/server/Version.hpp.in | 16 +++++++ repository/server/main.cpp | 15 +++--- 5 files changed, 68 insertions(+), 65 deletions(-) diff --git a/repository/server/Config.hpp b/repository/server/Config.hpp index 887d47996..2c954692e 100644 --- a/repository/server/Config.hpp +++ b/repository/server/Config.hpp @@ -5,11 +5,9 @@ // Common public includes #include "common/ICredentials.hpp" -// Proto includes -#include "common/SDMS.pb.h" - // Standard includes #include +#include #include #include diff --git a/repository/server/RepoServer.cpp b/repository/server/RepoServer.cpp index 36b33dbbf..915673168 100644 --- a/repository/server/RepoServer.cpp +++ b/repository/server/RepoServer.cpp @@ -15,10 +15,7 @@ #include "common/Util.hpp" // Proto includes -#include "common/SDMS.pb.h" -#include "common/SDMS_Anon.pb.h" -#include "common/SDMS_Auth.pb.h" -#include "common/Version.pb.h" +#include "common/envelope.pb.h" // Standard includes #include @@ -33,8 +30,6 @@ ((_T1.tv_sec - _T0.tv_sec) + ((_T1.tv_nsec - _T0.tv_nsec) / 1.0e9)) using namespace std; -using namespace SDMS::Anon; -using namespace SDMS::Auth; namespace { std::string randomAlphaNumericCode() { @@ -212,22 +207,22 @@ void Server::checkServerVersion() { << ver_reply->api_patch() << ")"); } bool new_release_available = false; - if (ver_reply->release_year() > Version::DATAFED_RELEASE_YEAR) { + if (ver_reply->release_year() > release::YEAR) { new_release_available = true; - } else if (ver_reply->release_year() == Version::DATAFED_RELEASE_YEAR) { - if (ver_reply->release_month() > Version::DATAFED_RELEASE_MONTH) { + } else if (ver_reply->release_year() == release::YEAR) { + if (ver_reply->release_month() > release::MONTH) { new_release_available = true; } else if (ver_reply->release_month() == - Version::DATAFED_RELEASE_MONTH) { - if (ver_reply->release_day() > Version::DATAFED_RELEASE_DAY) { + release::MONTH) { + if (ver_reply->release_day() > release::DAY) { new_release_available = true; - } else if (ver_reply->release_day() == Version::DATAFED_RELEASE_DAY) { - if (ver_reply->release_hour() > Version::DATAFED_RELEASE_HOUR) { + } else if (ver_reply->release_day() == release::DAY) { + if (ver_reply->release_hour() > release::HOUR) { new_release_available = true; } else if (ver_reply->release_hour() == - Version::DATAFED_RELEASE_HOUR) { + release::HOUR) { if (ver_reply->release_minute() > - Version::DATAFED_RELEASE_MINUTE) { + release::MINUTE) { new_release_available = true; } } diff --git a/repository/server/RequestWorker.cpp b/repository/server/RequestWorker.cpp index 49bb9186e..8e3c00772 100644 --- a/repository/server/RequestWorker.cpp +++ b/repository/server/RequestWorker.cpp @@ -12,11 +12,8 @@ #include "common/TraceException.hpp" #include "common/Util.hpp" -// Proto includes -#include "common/SDMS.pb.h" -#include "common/SDMS_Anon.pb.h" -#include "common/SDMS_Auth.pb.h" -#include "common/Version.pb.h" +// Proto files +#include "common/envelope.pb.h" // Third party includes #include @@ -29,9 +26,6 @@ using namespace std; namespace SDMS { -using namespace SDMS::Anon; -using namespace SDMS::Auth; - namespace Repo { map RequestWorker::m_msg_handlers; @@ -180,9 +174,14 @@ void RequestWorker::wait() { } } -#define SET_MSG_HANDLER(proto_id, msg, func) \ - m_msg_handlers[m_msg_mapper->getMessageType(proto_id, #msg)] = func +#define SET_MSG_HANDLER(msg, func) \ + m_msg_handlers[m_msg_mapper->getMessageType(#msg)] = func +/** + * This method configures message handling by creating a map from message type + * (envelope field number) to handler function. Message types are identified + * by their field number in the Envelope proto message. + */ void RequestWorker::setupMsgHandlers() { static std::atomic_flag lock = ATOMIC_FLAG_INIT; @@ -190,22 +189,17 @@ void RequestWorker::setupMsgHandlers() { return; try { + // Anonymous interface handlers + SET_MSG_HANDLER(VersionRequest, &RequestWorker::procVersionRequest); - uint8_t proto_id = - m_msg_mapper->getProtocolID(MessageProtocol::GOOGLE_ANONONYMOUS); - - SET_MSG_HANDLER(proto_id, VersionRequest, - &RequestWorker::procVersionRequest); - - proto_id = m_msg_mapper->getProtocolID(MessageProtocol::GOOGLE_AUTHORIZED); - - SET_MSG_HANDLER(proto_id, RepoDataDeleteRequest, + // Authenticated interface handlers + SET_MSG_HANDLER(RepoDataDeleteRequest, &RequestWorker::procDataDeleteRequest); - SET_MSG_HANDLER(proto_id, RepoDataGetSizeRequest, + SET_MSG_HANDLER(RepoDataGetSizeRequest, &RequestWorker::procDataGetSizeRequest); - SET_MSG_HANDLER(proto_id, RepoPathCreateRequest, + SET_MSG_HANDLER(RepoPathCreateRequest, &RequestWorker::procPathCreateRequest); - SET_MSG_HANDLER(proto_id, RepoPathDeleteRequest, + SET_MSG_HANDLER(RepoPathDeleteRequest, &RequestWorker::procPathDeleteRequest); } catch (TraceException &e) { DL_ERROR(m_log_context, @@ -252,6 +246,8 @@ void RequestWorker::workerThread(LogContext log_context) { timeout_on_poll); }(repo_thread_id); + ProtoBufMap proto_map; + DL_TRACE(log_context, "Listening on address " << client->address()); while (m_run) { @@ -280,7 +276,8 @@ void RequestWorker::workerThread(LogContext log_context) { uint16_t msg_type = std::get( message.get(constants::message::google::MSG_TYPE)); - DL_TRACE(message_log_context, "Received msg of type: " << msg_type); + DL_TRACE(message_log_context, "Received msg of type: " + << proto_map.toString(msg_type)); if (m_msg_handlers.count(msg_type)) { map::iterator handler = @@ -348,7 +345,7 @@ void RequestWorker::workerThread(LogContext log_context) { DL_ERROR(message_log_context, "Error: " << e.what()); \ auto msg_reply = m_msg_factory.createResponseEnvelope(*msg_request); \ auto nack = std::make_unique(); \ - nack->set_err_code(ID_INTERNAL_ERROR); \ + nack->set_err_code(INTERNAL_ERROR); \ nack->set_err_msg(e.what()); \ msg_reply->setPayload(std::move(nack)); \ return msg_reply; \ @@ -358,7 +355,7 @@ void RequestWorker::workerThread(LogContext log_context) { "Error unkown exception while processing message!"); \ auto msg_reply = m_msg_factory.createResponseEnvelope(*msg_request); \ auto nack = std::make_unique(); \ - nack->set_err_code(ID_INTERNAL_ERROR); \ + nack->set_err_code(INTERNAL_ERROR); \ nack->set_err_msg("Unknown exception type"); \ msg_reply->setPayload(std::move(nack)); \ return msg_reply; \ @@ -373,7 +370,7 @@ void RequestWorker::workerThread(LogContext log_context) { "Message parse failed (malformed or unregistered msg type)."); \ auto msg_reply = m_msg_factory.createResponseEnvelope(*msg_request); \ auto nack = std::make_unique(); \ - nack->set_err_code(ID_BAD_REQUEST); \ + nack->set_err_code(BAD_REQUEST); \ nack->set_err_msg( \ "Message parse failed (malformed or unregistered msg type)"); \ msg_reply->setPayload(std::move(nack)); \ @@ -388,15 +385,15 @@ RequestWorker::procVersionRequest(std::unique_ptr &&msg_request) { DL_DEBUG(message_log_context, "Version request."); - reply.set_release_year(Version::DATAFED_RELEASE_YEAR); - reply.set_release_month(Version::DATAFED_RELEASE_MONTH); - reply.set_release_day(Version::DATAFED_RELEASE_DAY); - reply.set_release_hour(Version::DATAFED_RELEASE_HOUR); - reply.set_release_minute(Version::DATAFED_RELEASE_MINUTE); + reply.set_release_year(SDMS::release::YEAR); + reply.set_release_month(SDMS::release::MONTH); + reply.set_release_day(SDMS::release::DAY); + reply.set_release_hour(SDMS::release::HOUR); + reply.set_release_minute(SDMS::release::MINUTE); - reply.set_api_major(Version::DATAFED_COMMON_PROTOCOL_API_MAJOR); - reply.set_api_minor(Version::DATAFED_COMMON_PROTOCOL_API_MINOR); - reply.set_api_patch(Version::DATAFED_COMMON_PROTOCOL_API_PATCH); + reply.set_api_major(SDMS::protocol::version::MAJOR); + reply.set_api_minor(SDMS::protocol::version::MINOR); + reply.set_api_patch(SDMS::protocol::version::PATCH); reply.set_component_major(SDMS::repository::version::MAJOR); reply.set_component_minor(SDMS::repository::version::MINOR); @@ -407,7 +404,7 @@ RequestWorker::procVersionRequest(std::unique_ptr &&msg_request) { std::unique_ptr RequestWorker::procDataDeleteRequest(std::unique_ptr &&msg_request) { - PROC_MSG_BEGIN(Auth::RepoDataDeleteRequest, Anon::AckReply) + PROC_MSG_BEGIN(RepoDataDeleteRequest, AckReply) if (request->loc_size()) { @@ -427,7 +424,7 @@ RequestWorker::procDataDeleteRequest(std::unique_ptr &&msg_request) { std::unique_ptr RequestWorker::procDataGetSizeRequest(std::unique_ptr &&msg_request) { - PROC_MSG_BEGIN(Auth::RepoDataGetSizeRequest, Auth::RepoDataSizeReply) + PROC_MSG_BEGIN(RepoDataGetSizeRequest, RepoDataSizeReply) DL_DEBUG(message_log_context, "Data get size."); @@ -462,7 +459,7 @@ RequestWorker::procDataGetSizeRequest(std::unique_ptr &&msg_request) { std::unique_ptr RequestWorker::procPathCreateRequest(std::unique_ptr &&msg_request) { - PROC_MSG_BEGIN(Auth::RepoPathCreateRequest, Anon::AckReply) + PROC_MSG_BEGIN(RepoPathCreateRequest, AckReply) std::string local_path = createSanitizedPath(request->path()); @@ -480,7 +477,7 @@ RequestWorker::procPathCreateRequest(std::unique_ptr &&msg_request) { std::unique_ptr RequestWorker::procPathDeleteRequest(std::unique_ptr &&msg_request) { - PROC_MSG_BEGIN(Auth::RepoPathDeleteRequest, Anon::AckReply) + PROC_MSG_BEGIN(RepoPathDeleteRequest, AckReply) DL_DEBUG(message_log_context, "Relative path delete request: " << request->path()); diff --git a/repository/server/Version.hpp.in b/repository/server/Version.hpp.in index 322bb4c5c..617a414e9 100644 --- a/repository/server/Version.hpp.in +++ b/repository/server/Version.hpp.in @@ -10,6 +10,22 @@ namespace SDMS { constexpr int PATCH = @DATAFED_REPO_PATCH@; } } + + namespace protocol { + namespace version { + constexpr int MAJOR = @DATAFED_COMMON_PROTOCOL_API_MAJOR@; + constexpr int MINOR = @DATAFED_COMMON_PROTOCOL_API_MINOR@; + constexpr int PATCH = @DATAFED_COMMON_PROTOCOL_API_PATCH@; + } + } + + namespace release { + constexpr int YEAR = @DATAFED_RELEASE_YEAR@; + constexpr int MONTH = @DATAFED_RELEASE_MONTH@; + constexpr int DAY = @DATAFED_RELEASE_DAY@; + constexpr int HOUR = @DATAFED_RELEASE_HOUR@; + constexpr int MINUTE = @DATAFED_RELEASE_MINUTE@; + } } #endif // REPO_VERSION_HPP diff --git a/repository/server/main.cpp b/repository/server/main.cpp index a5c00d1f0..0e5d2e287 100644 --- a/repository/server/main.cpp +++ b/repository/server/main.cpp @@ -9,9 +9,6 @@ #include "common/TraceException.hpp" #include "common/Util.hpp" -// Protocol includes -#include "common/Version.pb.h" - // Third party includes #include @@ -78,13 +75,13 @@ int main(int a_argc, char **a_argv) { } if (opt_map.count("version")) { - cout << "Release Version: " << DATAFED_RELEASE_YEAR << "." - << DATAFED_RELEASE_MONTH << "." << DATAFED_RELEASE_DAY << "." - << DATAFED_RELEASE_HOUR << "." << DATAFED_RELEASE_MINUTE + cout << "Release Version: " << release::YEAR << "." + << release::MONTH << "." << release::DAY << "." + << release::HOUR << "." << release::MINUTE << std::endl; - cout << "Messaging API: " << DATAFED_COMMON_PROTOCOL_API_MAJOR << "." - << DATAFED_COMMON_PROTOCOL_API_MINOR << "." - << DATAFED_COMMON_PROTOCOL_API_PATCH << endl; + cout << "Messaging API: " << protocol::version::MAJOR << "." + << protocol::version::MINOR << "." + << protocol::version::PATCH << endl; cout << "Repo Server: " << repository::version::MAJOR << "." << repository::version::MINOR << "." << repository::version::PATCH << endl; From 3abf148436ae0613522ee4c6b5f191f36fb85b65 Mon Sep 17 00:00:00 2001 From: JoshuaSBrown Date: Thu, 5 Feb 2026 12:58:26 -0500 Subject: [PATCH 06/14] refactor: python package to be compatible with proto3 envelope --- python/datafed_pkg/datafed/CMakeLists.txt | 26 ++++------ python/datafed_pkg/datafed/Connection.py | 63 ++++++++++++++++++----- python/datafed_pkg/datafed/MessageLib.py | 20 ++++--- python/pyproto_add_msg_idx.py | 61 ---------------------- 4 files changed, 71 insertions(+), 99 deletions(-) delete mode 100755 python/pyproto_add_msg_idx.py diff --git a/python/datafed_pkg/datafed/CMakeLists.txt b/python/datafed_pkg/datafed/CMakeLists.txt index 9e066deea..da7b92b73 100644 --- a/python/datafed_pkg/datafed/CMakeLists.txt +++ b/python/datafed_pkg/datafed/CMakeLists.txt @@ -6,12 +6,13 @@ foreach(file ${SrcFiles}) configure_file(${file} ${CMAKE_CURRENT_BINARY_DIR} COPYONLY ) endforeach() -# Collect top-level proto files as dependencies -file( GLOB ProtoFiles ${DataFed_SOURCE_DIR}/common/proto/common/*.proto ) +# Collect proto files from the new 1-1-1 directory structure +file( GLOB_RECURSE ProtoFiles ${DataFed_SOURCE_DIR}/common/proto/common/*.proto ) # OBJECT - is needed because we don't want to compile to a binary # because we are dealing with python add_library(protobuf-target-py OBJECT ${ProtoFiles}) + protobuf_generate( LANGUAGE python TARGET protobuf-target-py @@ -22,21 +23,16 @@ protobuf_generate( add_custom_target( pydatafed_src DEPENDS protobuf-target-py ) -# By default this will output the proto py files in the CMAKE BINARY DIR +# Fix relative imports in generated pb2 files +# No longer need pyproto_add_msg_idx.py — message type IDs are derived +# at runtime from envelope field numbers via registerEnvelope() add_custom_command( TARGET pydatafed_src POST_BUILD COMMAND sed -i -r 's:^import.*_pb2:from . \\0:' ${protobuf-generated-files-py} - COMMAND ${DataFed_SOURCE_DIR}/python/pyproto_add_msg_idx.py ${DataFed_SOURCE_DIR}/common/proto/common/SDMS_Anon.proto ${CMAKE_CURRENT_BINARY_DIR}/SDMS_Anon_pb2.py - COMMAND ${DataFed_SOURCE_DIR}/python/pyproto_add_msg_idx.py ${DataFed_SOURCE_DIR}/common/proto/common/SDMS_Auth.proto ${CMAKE_CURRENT_BINARY_DIR}/SDMS_Auth_pb2.py -) - -# Crea#te copies of the files so they show up in the source folder as well +) + +# Create copies of the files so they show up in the source folder as well # for the purpose of testing add_custom_target( pydatafed_proto_src DEPENDS pydatafed_src ) -add_custom_command( TARGET pydatafed_proto_src POST_BUILD pydatafed_src - COMMAND cp ${CMAKE_CURRENT_BINARY_DIR}/SDMS_Auth_pb2.py ${CMAKE_CURRENT_SOURCE_DIR}/ - COMMAND cp ${CMAKE_CURRENT_BINARY_DIR}/SDMS_pb2.py ${CMAKE_CURRENT_SOURCE_DIR}/ - COMMAND cp ${CMAKE_CURRENT_BINARY_DIR}/Version_pb2.py ${CMAKE_CURRENT_SOURCE_DIR}/ - COMMAND cp ${CMAKE_CURRENT_BINARY_DIR}/SDMS_Anon_pb2.py ${CMAKE_CURRENT_SOURCE_DIR}/ +add_custom_command( TARGET pydatafed_proto_src POST_BUILD pydatafed_src + COMMAND cp ${CMAKE_CURRENT_BINARY_DIR}/envelope_pb2.py ${CMAKE_CURRENT_SOURCE_DIR}/ ) - - diff --git a/python/datafed_pkg/datafed/Connection.py b/python/datafed_pkg/datafed/Connection.py index d34dd1e1c..4ee0e8885 100644 --- a/python/datafed_pkg/datafed/Connection.py +++ b/python/datafed_pkg/datafed/Connection.py @@ -6,13 +6,11 @@ # unserialized, and custom framing is generated to efficiently convey message # type, size, and a re-association context value. # -# The Google protobuf library does not provide a mechanism for identifying -# message types numerically (only by string), so a build-time custom tool -# (pyproto_add_msg_idx.py) is used to generate the mappings from message -# names to message index (and vice versa) and appends this information as -# dictionaries to the compiled proto files (xxxx_pb2.py). The -# registerProtocol() method then loads uses this information to create -# consistent message type framing for python send/recv methods. +# Message type identification is derived at runtime from the Envelope proto +# message's field descriptors. Each message type has a stable field number +# in the Envelope, which serves as its wire-format type ID. This replaces +# the previous build-time pyproto_add_msg_idx.py hack that assigned type +# IDs based on message declaration order within proto files. from google.protobuf.message_factory import GetMessageClass import logging @@ -116,19 +114,60 @@ def __del__(self): self._zmq_ctxt.destroy() ## - # @brief Register a protobuf module + # @brief Register message types from the Envelope proto message + # + # This method derives message type mappings at runtime by inspecting the + # Envelope message's field descriptors. Each field in the Envelope that + # wraps a message type has a stable field number, which becomes the + # message type ID used in wire framing. This replaces the old + # registerProtocol() approach that relied on build-time generated + # _msg_name_to_type / _msg_type_to_name dicts. + # + # @param envelope_module - The compiled envelope_pb2 module + # @param envelope_class_name - Name of the envelope message (default: "Envelope") + # + def registerEnvelope(self, envelope_module, envelope_class_name="Envelope"): + envelope_class = getattr(envelope_module, envelope_class_name) + envelope_desc = envelope_class.DESCRIPTOR + + for field in envelope_desc.fields: + if field.message_type is None: + # Skip non-message fields (e.g. scalars) if any exist + continue + + msg_type = field.number + desc = field.message_type + + self._msg_desc_by_type[msg_type] = desc + self._msg_desc_by_name[desc.name] = desc + self._msg_type_by_desc[desc] = msg_type + + self._logger.debug( + "Registered %d message types from %s", + len(self._msg_desc_by_type), + envelope_class_name, + ) + + ## + # @brief Register a protobuf module (DEPRECATED - use registerEnvelope) # # This method registers an imported protobuf module (_pb2 file) for use # with the Connection class. Registration is required for proper message # framing and serialization. # + # This relies on build-time generated _msg_name_to_type dicts appended + # to _pb2 files by pyproto_add_msg_idx.py. Prefer registerEnvelope() + # which derives mappings from envelope field numbers at runtime. + # # @param msg_module - Protobuf module (imported *_pb2 module) # def registerProtocol(self, msg_module): - # Message descriptors are stored by name created by protobuf compiler - # A custom post-proc tool generates and appends _msg_name_to_type with - # defined DataFed-sepcific numer message types - + import warnings + warnings.warn( + "registerProtocol() is deprecated, use registerEnvelope() instead", + DeprecationWarning, + stacklevel=2, + ) for name, desc in sorted(msg_module.DESCRIPTOR.message_types_by_name.items()): msg_t = msg_module._msg_name_to_type[name] self._msg_desc_by_type[msg_t] = desc diff --git a/python/datafed_pkg/datafed/MessageLib.py b/python/datafed_pkg/datafed/MessageLib.py index 62354c0eb..69cb2c9d7 100644 --- a/python/datafed_pkg/datafed/MessageLib.py +++ b/python/datafed_pkg/datafed/MessageLib.py @@ -13,8 +13,7 @@ import zmq from . import Version_pb2 -from . import SDMS_Anon_pb2 as anon -from . import SDMS_Auth_pb2 as auth +from . import envelope_pb2 as proto from . import Connection from . import VERSION @@ -166,8 +165,7 @@ def __init__( server_host, server_port, _server_pub_key, _client_pub_key, _client_priv_key ) - self._conn.registerProtocol(anon) - self._conn.registerProtocol(auth) + self._conn.registerEnvelope(proto) # Make a request to pypi package_name = "datafed" # Replace with the package name you want to check @@ -191,7 +189,7 @@ def __init__( self.new_client_avail = latest_version_on_pypi # Check for compatible protocol versions - reply, mt = self.sendRecv(anon.VersionRequest(), 10000) + reply, mt = self.sendRecv(proto.VersionRequest(), 10000) if reply is None: raise Exception( "Timeout waiting for server connection. Make sure" @@ -223,7 +221,7 @@ def __init__( self.manualAuthByToken(client_token) else: # Check if server authenticated based on keys - reply, mt = self.sendRecv(anon.GetAuthStatusRequest(), 10000) + reply, mt = self.sendRecv(proto.GetAuthStatusRequest(), 10000) self._auth = reply.auth self._uid = reply.uid @@ -263,7 +261,7 @@ def getAuthStatus(self): # @exception Exception: On communication timeout or authentication failure. # def manualAuthByPassword(self, uid, password): - msg = anon.AuthenticateByPasswordRequest() + msg = proto.AuthenticateByPasswordRequest() msg.uid = uid msg.password = password a, b = self.sendRecv(msg) @@ -272,7 +270,7 @@ def manualAuthByPassword(self, uid, password): self._conn.reset() # Test auth status - reply, mt = self.sendRecv(anon.GetAuthStatusRequest()) + reply, mt = self.sendRecv(proto.GetAuthStatusRequest()) if not reply.auth: raise Exception("Password authentication failed.") @@ -280,7 +278,7 @@ def manualAuthByPassword(self, uid, password): self._uid = reply.uid def manualAuthByToken(self, token): - msg = anon.AuthenticateByTokenRequest() + msg = proto.AuthenticateByTokenRequest() msg.token = token self.sendRecv(msg) @@ -288,7 +286,7 @@ def manualAuthByToken(self, token): self._conn.reset() # Test auth status - reply, mt = self.sendRecv(anon.GetAuthStatusRequest()) + reply, mt = self.sendRecv(proto.GetAuthStatusRequest()) if not reply.auth: raise Exception("Token authentication failed") @@ -332,7 +330,7 @@ def getDefaultTimeout(self): def getDailyMessage(self): # Get daily message, if set - reply, mt = self.sendRecv(anon.DailyMessageRequest(), 10000) + reply, mt = self.sendRecv(proto.DailyMessageRequest(), 10000) if reply is None: raise Exception("Timeout waiting for server connection.") diff --git a/python/pyproto_add_msg_idx.py b/python/pyproto_add_msg_idx.py deleted file mode 100755 index 233f83b33..000000000 --- a/python/pyproto_add_msg_idx.py +++ /dev/null @@ -1,61 +0,0 @@ -#!/usr/bin/env python3 - -""" -Protobuf processing to generate message ID maps for C++, Python, and JS -""" - -import sys -import re - -print("args", sys.argv) - -pf_in = open(sys.argv[1], "r") -pf_out = open(sys.argv[2], "a") - -while True: - line = pf_in.readline() - if len(line) == 0: - sys.exit(-1) - parts = re.split(r"\W+", line.strip()) - # print( line, parts ) - try: - idx = parts.index("ID") - # print( "ID:", parts[idx+1] ) - msg_type = int(parts[idx + 1]) << 8 - break - except BaseException: - pass - -# msg_type = 0 - -by_type = [] -idx = 0 - -pf_out.write("\n_msg_name_to_type = {\n") - -while True: - line = pf_in.readline() - if len(line) == 0: - break - - if line.startswith("message "): - msg_name = line.split()[1] - by_type.append(msg_name) - # print( msg_name, msg_type ) - if idx > 0: - pf_out.write(",\n") - pf_out.write(" '{}' : {}".format(msg_name, msg_type | idx)) - idx += 1 - -pf_out.write("\n}\n\n_msg_type_to_name = {\n") - -idx = 0 -for name in by_type: - if idx > 0: - pf_out.write(",\n") - pf_out.write(" {} : '{}'".format(msg_type | idx, name)) - idx += 1 - -pf_out.write("\n}\n") - -sys.exit(0) From 5df7ac8e963d75ccc4ce425e170303e21e7b120c Mon Sep 17 00:00:00 2001 From: JoshuaSBrown Date: Thu, 5 Feb 2026 17:09:12 -0500 Subject: [PATCH 07/14] fix: corrected path to proto3 files in CMakeLists.txt --- python/datafed_pkg/datafed/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/datafed_pkg/datafed/CMakeLists.txt b/python/datafed_pkg/datafed/CMakeLists.txt index da7b92b73..a66b1f79a 100644 --- a/python/datafed_pkg/datafed/CMakeLists.txt +++ b/python/datafed_pkg/datafed/CMakeLists.txt @@ -7,7 +7,7 @@ foreach(file ${SrcFiles}) endforeach() # Collect proto files from the new 1-1-1 directory structure -file( GLOB_RECURSE ProtoFiles ${DataFed_SOURCE_DIR}/common/proto/common/*.proto ) +file( GLOB_RECURSE ProtoFiles ${DataFed_SOURCE_DIR}/common/proto3/common/*.proto ) # OBJECT - is needed because we don't want to compile to a binary # because we are dealing with python @@ -16,7 +16,7 @@ add_library(protobuf-target-py OBJECT ${ProtoFiles}) protobuf_generate( LANGUAGE python TARGET protobuf-target-py - IMPORT_DIRS "${DataFed_SOURCE_DIR}/common/proto/common" + IMPORT_DIRS "${DataFed_SOURCE_DIR}/common/proto3/common" OUT_VAR protobuf-generated-files-py PROTOC_OUT_DIR "${CMAKE_CURRENT_BINARY_DIR}" ) From 87a4c46ab0f3fd649734a83673b76113644792bd Mon Sep 17 00:00:00 2001 From: JoshuaSBrown Date: Fri, 6 Feb 2026 00:44:57 -0500 Subject: [PATCH 08/14] refactor: fixing remaining _pb2 SDMS anon proto2 files --- python/datafed_pkg/datafed/CLI.py | 18 ++-- python/datafed_pkg/datafed/CommandLib.py | 101 +++++++++++------------ python/datafed_pkg/datafed/MessageLib.py | 9 +- python/datafed_pkg/datafed/VERSION.py.in | 8 ++ python/datafed_pkg/test/security.py | 4 +- 5 files changed, 72 insertions(+), 68 deletions(-) diff --git a/python/datafed_pkg/datafed/CLI.py b/python/datafed_pkg/datafed/CLI.py index 949d535ae..27918efba 100644 --- a/python/datafed_pkg/datafed/CLI.py +++ b/python/datafed_pkg/datafed/CLI.py @@ -34,8 +34,6 @@ from prompt_toolkit.history import FileHistory from prompt_toolkit.auto_suggest import AutoSuggestFromHistory -# from . import SDMS_Auth_pb2 as auth -from . import Version_pb2 from . import CommandLib from . import Config from . import VERSION @@ -162,14 +160,14 @@ def run(): except _NoCommand as e: # Be nice and switch to interactive when no command given if _interactive and _first: - api_version = f"{Version_pb2.DATAFED_COMMON_PROTOCOL_API_MAJOR}." - api_version += f"{Version_pb2.DATAFED_COMMON_PROTOCOL_API_MINOR}." - api_version += f"{Version_pb2.DATAFED_COMMON_PROTOCOL_API_PATCH}" - release_version = f"{Version_pb2.DATAFED_RELEASE_YEAR}." - release_version += f"{Version_pb2.DATAFED_RELEASE_MONTH}." - release_version += f"{Version_pb2.DATAFED_RELEASE_DAY}." - release_version += f"{Version_pb2.DATAFED_RELEASE_HOUR}." - release_version += f"{Version_pb2.DATAFED_RELEASE_MINUTE}" + api_version = f"{VERSION.DATAFED_COMMON_PROTOCOL_API_MAJOR}." + api_version += f"{VERSION.DATAFED_COMMON_PROTOCOL_API_MINOR}." + api_version += f"{VERSION.DATAFED_COMMON_PROTOCOL_API_PATCH}" + release_version = f"{VERSION.DATAFED_RELEASE_YEAR}." + release_version += f"{VERSION.DATAFED_RELEASE_MONTH}." + release_version += f"{VERSION.DATAFED_RELEASE_DAY}." + release_version += f"{VERSION.DATAFED_RELEASE_HOUR}." + release_version += f"{VERSION.DATAFED_RELEASE_MINUTE}" _print_msg(1, f"Welcome to DataFed CLI, version {VERSION.__version__}") _print_msg( 1, " Release, version {}".format(release_version) diff --git a/python/datafed_pkg/datafed/CommandLib.py b/python/datafed_pkg/datafed/CommandLib.py index 3f4ad8098..7d2e14b7f 100644 --- a/python/datafed_pkg/datafed/CommandLib.py +++ b/python/datafed_pkg/datafed/CommandLib.py @@ -14,8 +14,7 @@ import time import pathlib import requests -from . import SDMS_Auth_pb2 as auth -from . import SDMS_pb2 as sdms +from . import envelope_pb2 as sdms from . import MessageLib from . import Config @@ -168,7 +167,7 @@ def generateCredentials(self): ------ Exception: On communication or server error """ - msg = auth.GenerateCredentialsRequest() + msg = sdms.GenerateCredentialsRequest() return self._mapi.sendRecv(msg) @@ -236,7 +235,7 @@ def repoCreate( ------ Exception : On communication or server error """ - msg = auth.RepoCreateRequest() + msg = sdms.RepoCreateRequest() msg.id = repo_id msg.title = title msg.desc = desc @@ -260,7 +259,7 @@ def repoList(self, list_all: bool = False): By default will only list the repos associated with the user. """ - msg = auth.RepoListRequest() + msg = sdms.RepoListRequest() msg.all = list_all return self._mapi.sendRecv(msg) @@ -281,7 +280,7 @@ def repoDelete(self, repo_id): ------ Exception : On communication or server error """ - msg = auth.RepoDeleteRequest() + msg = sdms.RepoDeleteRequest() msg.id = repo_id return self._mapi.sendRecv(msg) @@ -289,7 +288,7 @@ def repoAllocationCreate(self, repo_id, subject, data_limit, rec_limit): if not repo_id.startswith("repo/"): repo_id = "repo/" + repo_id - msg = auth.RepoAllocationCreateRequest() + msg = sdms.RepoAllocationCreateRequest() msg.repo = repo_id msg.subject = subject msg.data_limit = data_limit @@ -300,14 +299,14 @@ def repoListAllocations(self, repo_id): if not repo_id.startswith("repo/"): repo_id = "repo/" + repo_id - msg = auth.RepoListAllocationsRequest() + msg = sdms.RepoListAllocationsRequest() msg.id = repo_id return self._mapi.sendRecv(msg) def repoAllocationDelete(self, repo_id, subject): if not repo_id.startswith("repo/"): repo_id = "repo/" + repo_id - msg = auth.RepoAllocationDeleteRequest() + msg = sdms.RepoAllocationDeleteRequest() msg.repo = repo_id msg.subject = subject return self._mapi.sendRecv(msg) @@ -341,7 +340,7 @@ def dataView(self, data_id, details=False, context=None): ------ Exception : On communication or server error """ - msg = auth.RecordViewRequest() + msg = sdms.RecordViewRequest() msg.id = self._resolve_id(data_id, context) msg.details = details @@ -436,7 +435,7 @@ def dataCreate( if metadata and metadata_file: raise Exception("Cannot specify both metadata and metadata-file options.") - msg = auth.RecordCreateRequest() + msg = sdms.RecordCreateRequest() msg.title = title msg.parent_id = self._resolve_id(parent_id, context) @@ -579,7 +578,7 @@ def dataUpdate( if metadata and metadata_file: raise Exception("Cannot specify both metadata and metadata-file options.") - msg = auth.RecordUpdateRequest() + msg = sdms.RecordUpdateRequest() msg.id = self._resolve_id(data_id, context) if title is not None: @@ -673,7 +672,7 @@ def dataDelete(self, data_id, context=None): ------ Exception : On invalid options or communication / server error """ - msg = auth.RecordDeleteRequest() + msg = sdms.RecordDeleteRequest() if isinstance(data_id, list): for i in data_id: @@ -740,7 +739,7 @@ def dataGet( # Request server to map specified IDs into a list of specific record IDs. # This accounts for download of collections. - msg = auth.DataGetRequest() + msg = sdms.DataGetRequest() msg.check = True if isinstance(item_id, str): @@ -761,7 +760,7 @@ def dataGet( if len(glob_ids) > 0: # Globus transfers - msg = auth.DataGetRequest() + msg = sdms.DataGetRequest() msg.id.extend(glob_ids) msg.path = self._resolvePathForGlobus(path, False) msg.encrypt = encrypt @@ -770,7 +769,7 @@ def dataGet( reply = self._mapi.sendRecv(msg) if reply[0].task and wait: - msg2 = auth.TaskViewRequest() + msg2 = sdms.TaskViewRequest() msg2.task_id = reply[0].task.id elapsed = 0 @@ -849,7 +848,7 @@ def dataPut( ------ Exception : On invalid options or communication / server error. """ - msg = auth.DataPutRequest() + msg = sdms.DataPutRequest() msg.id = self._resolve_id(data_id, context) msg.path = self._resolvePathForGlobus(path, False) msg.encrypt = encrypt @@ -859,7 +858,7 @@ def dataPut( reply = self._mapi.sendRecv(msg) if (reply[0].HasField("task")) and wait: - msg2 = auth.TaskViewRequest() + msg2 = sdms.TaskViewRequest() msg2.task_id = reply[0].task.id elapsed = 0 @@ -944,7 +943,7 @@ def dataBatchCreate(self, file, coll_id=None, context=None): payload.extend(records) - msg = auth.RecordCreateBatchRequest() + msg = sdms.RecordCreateBatchRequest() msg.records = jsonlib.dumps(payload) return self._mapi.sendRecv(msg) @@ -998,7 +997,7 @@ def dataBatchUpdate(self, file): else: payload.extend(records) - msg = auth.RecordUpdateBatchRequest() + msg = sdms.RecordUpdateBatchRequest() msg.records = jsonlib.dumps(payload) return self._mapi.sendRecv(msg) @@ -1029,7 +1028,7 @@ def collectionView(self, coll_id, context=None): ------ Exception : On invalid options or communication / server error. """ - msg = auth.CollViewRequest() + msg = sdms.CollViewRequest() msg.id = self._resolve_id(coll_id, context) # msg.id = self._resolve_coll_id( coll_id, context ) @@ -1083,7 +1082,7 @@ def collectionCreate( ------ Exception : On communication or server error """ - msg = auth.CollCreateRequest() + msg = sdms.CollCreateRequest() msg.title = title if alias: @@ -1149,7 +1148,7 @@ def collectionUpdate( ------ Exception : On communication or server error """ - msg = auth.CollUpdateRequest() + msg = sdms.CollUpdateRequest() msg.id = self._resolve_id(coll_id, context) if title is not None: @@ -1197,7 +1196,7 @@ def collectionDelete(self, coll_id, context=None): ------ Exception : On communication or server error """ - msg = auth.CollDeleteRequest() + msg = sdms.CollDeleteRequest() if isinstance(coll_id, list): for i in coll_id: @@ -1234,7 +1233,7 @@ def collectionItemsList(self, coll_id, offset=0, count=20, context=None): Exception : On communication or server error Exception : On invalid options """ - msg = auth.CollReadRequest() + msg = sdms.CollReadRequest() msg.count = count msg.offset = offset msg.id = self._resolve_id(coll_id, context) @@ -1276,7 +1275,7 @@ def collectionItemsUpdate(self, coll_id, add_ids=None, rem_ids=None, context=Non Exception : On communication or server error Exception : On invalid options """ - msg = auth.CollWriteRequest() + msg = sdms.CollWriteRequest() msg.id = self._resolve_id(coll_id, context) if isinstance(add_ids, list): @@ -1317,7 +1316,7 @@ def collectionGetParents(self, coll_id, inclusive=False, context=None): Exception : On communication or server error Exception : On invalid options """ - msg = auth.CollGetParentsRequest() + msg = sdms.CollGetParentsRequest() msg.id = self._resolve_id(coll_id, context) msg.inclusive = inclusive @@ -1348,7 +1347,7 @@ def queryList(self, offset=0, count=20): Exception : On communication or server error Exception : On invalid options """ - msg = auth.QueryListRequest() + msg = sdms.QueryListRequest() msg.offset = offset msg.count = count @@ -1371,7 +1370,7 @@ def queryView(self, query_id): ------ Exception : On communication or server error """ - msg = auth.QueryViewRequest() + msg = sdms.QueryViewRequest() msg.id = query_id return self._mapi.sendRecv(msg) @@ -1422,7 +1421,7 @@ def queryCreate( Exception : On communication or server error Exception : On invalid options """ - msg = auth.QueryCreateRequest() + msg = sdms.QueryCreateRequest() msg.title = title self._buildSearchRequest( @@ -1487,7 +1486,7 @@ def queryUpdate( Exception : On invalid options """ - msg = auth.QueryUpdateRequest() + msg = sdms.QueryUpdateRequest() msg.id = query_id if title is not None: @@ -1532,7 +1531,7 @@ def queryDelete(self, query_id): ------ Exception : On communication or server error """ - msg = auth.QueryDeleteRequest() + msg = sdms.QueryDeleteRequest() msg.id.append(query_id) return self._mapi.sendRecv(msg) @@ -1560,7 +1559,7 @@ def queryExec(self, query_id, offset=0, count=20): Exception : On communication or server error Exception : On invalid options """ - msg = auth.QueryExecRequest() + msg = sdms.QueryExecRequest() msg.id = query_id msg.offset = offset msg.count = count @@ -1612,7 +1611,7 @@ def queryDirect( Exception : On communication or server error Exception : On invalid options """ - msg = auth.SearchRequest() + msg = sdms.SearchRequest() self._buildSearchRequest( msg, @@ -1776,7 +1775,7 @@ def userListCollaborators(self, offset=0, count=20): Exception : On communication or server error Exception : On invalid options """ - msg = auth.UserListCollabRequest() + msg = sdms.UserListCollabRequest() msg.offset = offset msg.count = count @@ -1802,7 +1801,7 @@ def userListAll(self, offset=0, count=20): Exception : On communication or server error Exception : On invalid options """ - msg = auth.UserListAllRequest() + msg = sdms.UserListAllRequest() msg.offset = offset msg.count = count @@ -1826,7 +1825,7 @@ def userView(self, uid): Exception : On communication or server error Exception : On invalid options """ - msg = auth.UserViewRequest() + msg = sdms.UserViewRequest() msg.uid = uid return self._mapi.sendRecv(msg) @@ -1868,7 +1867,7 @@ def projectList(self, owned=True, admin=True, member=True, offset=0, count=20): Exception : On communication or server error Exception : On invalid options """ - msg = auth.ProjectListRequest() + msg = sdms.ProjectListRequest() msg.as_owner = owned msg.as_admin = admin msg.as_member = member @@ -1895,7 +1894,7 @@ def projectView(self, project_id): Exception : On communication or server error Exception : On invalid options """ - msg = auth.ProjectViewRequest() + msg = sdms.ProjectViewRequest() msg.id = project_id return self._mapi.sendRecv(msg) @@ -1918,7 +1917,7 @@ def projectGetRole(self, project_id): Exception : On communication or server error Exception : On invalid options """ - msg = auth.ProjectGetRoleRequest() + msg = sdms.ProjectGetRoleRequest() msg.id = project_id reply = self._mapi.sendRecv(msg) @@ -1951,7 +1950,7 @@ def sharedList(self, inc_users=None, inc_projects=None, subject=None): Exception : On communication or server error Exception : On invalid options """ - msg = auth.ACLSharedListRequest() + msg = sdms.ACLSharedListRequest() if inc_users is not None: msg.inc_users = inc_users @@ -1981,7 +1980,7 @@ def sharedUsersList( self ): Exception : On communication or server error Exception : On invalid options """ - msg = auth.ACLByUserRequest() + msg = sdms.ACLByUserRequest() return self._mapi.sendRecv( msg ) @@ -1999,7 +1998,7 @@ def sharedProjectsList( self ): Exception : On communication or server error Exception : On invalid options """ - msg = auth.ACLByProjRequest() + msg = sdms.ACLByProjRequest() return self._mapi.sendRecv( msg ) ''' @@ -2031,7 +2030,7 @@ def sharedListItems(self, owner_id, context=None, offset=None, count=None): """ # TODO add support for offset & count - msg = auth.ACLSharedListItemsRequest() + msg = sdms.ACLSharedListItemsRequest() msg.owner = owner_id.lower() if context is not None: msg.subject = context.lower() @@ -2081,7 +2080,7 @@ def taskList( if since is not None and (time_from is not None or time_to is not None): raise Exception("Cannot specify 'since' and 'from'/'to' ranges.") - msg = auth.TaskListRequest() + msg = sdms.TaskListRequest() if time_from is not None: ts = self.strToTimestamp(time_from) @@ -2191,12 +2190,12 @@ def taskView(self, task_id=None): Exception : On invalid options """ if task_id: - msg = auth.TaskViewRequest() + msg = sdms.TaskViewRequest() msg.task_id = task_id reply = self._mapi.sendRecv(msg) else: - msg = auth.TaskListRequest() + msg = sdms.TaskListRequest() msg.offset = 0 msg.count = 1 @@ -2221,7 +2220,7 @@ def endpointListRecent(self): ------ Exception : On communication or server error """ - msg = auth.UserGetRecentEPRequest() + msg = sdms.UserGetRecentEPRequest() return self._mapi.sendRecv(msg) @@ -2305,7 +2304,7 @@ def setupCredentials(self): "Client configuration directory and/or client key files not configured" ) - msg = auth.GenerateCredentialsRequest() + msg = sdms.GenerateCredentialsRequest() reply = self._mapi.sendRecv(msg) @@ -2352,7 +2351,7 @@ def setContext(self, item_id=None): id2 = item_id if id2[0:2] == "p/": - msg = auth.ProjectViewRequest() + msg = sdms.ProjectViewRequest() msg.id = id2 else: if id2[0:2] != "u/": @@ -2364,7 +2363,7 @@ def setContext(self, item_id=None): ) id2 = "u/" + id2 - msg = auth.UserViewRequest() + msg = sdms.UserViewRequest() msg.uid = id2 # Don't need reply - just using to throw an except if id/uid is diff --git a/python/datafed_pkg/datafed/MessageLib.py b/python/datafed_pkg/datafed/MessageLib.py index 69cb2c9d7..f0964e7d7 100644 --- a/python/datafed_pkg/datafed/MessageLib.py +++ b/python/datafed_pkg/datafed/MessageLib.py @@ -12,7 +12,6 @@ import zmq -from . import Version_pb2 from . import envelope_pb2 as proto from . import Connection from . import VERSION @@ -196,7 +195,7 @@ def __init__( "the right ports are open." ) - if reply.api_major != Version_pb2.DATAFED_COMMON_PROTOCOL_API_MAJOR: + if reply.api_major != VERSION.DATAFED_COMMON_PROTOCOL_API_MAJOR: error_msg = ( "Incompatible server api detected {}.{}.{}, you are running " "{}.{}.{} consider " @@ -204,9 +203,9 @@ def __init__( reply.api_major, reply.api_minor, reply.api_patch, - Version_pb2.DATAFED_COMMON_PROTOCOL_API_MAJOR, - Version_pb2.DATAFED_COMMON_PROTOCOL_API_MINOR, - Version_pb2.DATAFED_COMMON_PROTOCOL_API_PATCH, + VERSION.DATAFED_COMMON_PROTOCOL_API_MAJOR, + VERSION.DATAFED_COMMON_PROTOCOL_API_MINOR, + VERSION.DATAFED_COMMON_PROTOCOL_API_PATCH, ) ) if self.new_client_avail: diff --git a/python/datafed_pkg/datafed/VERSION.py.in b/python/datafed_pkg/datafed/VERSION.py.in index fc9c6a3b3..03ae7c5f5 100644 --- a/python/datafed_pkg/datafed/VERSION.py.in +++ b/python/datafed_pkg/datafed/VERSION.py.in @@ -1 +1,9 @@ __version__="@DATAFED_PYTHON_CLIENT_MAJOR@.@DATAFED_PYTHON_CLIENT_MINOR@.@DATAFED_PYTHON_CLIENT_PATCH@@DATAFED_PYTHON_CLIENT_RELEASE_TYPE@@DATAFED_PYTHON_CLIENT_PRE_RELEASE_IDENTIFER@" +DATAFED_COMMON_PROTOCOL_API_MAJOR=@DATAFED_COMMON_PROTOCOL_API_MAJOR@ +DATAFED_COMMON_PROTOCOL_API_MINOR=@DATAFED_COMMON_PROTOCOL_API_MINOR@ +DATAFED_COMMON_PROTOCOL_API_PATCH=@DATAFED_COMMON_PROTOCOL_API_PATCH@ +DATAFED_RELEASE_YEAR=@DATAFED_RELEASE_YEAR@ +DATAFED_RELEASE_MONTH=@DATAFED_RELEASE_MONTH@ +DATAFED_RELEASE_DAY=@DATAFED_RELEASE_DAY@ +DATAFED_RELEASE_HOUR=@DATAFED_RELEASE_HOUR@ +DATAFED_RELEASE_MINUTE=@DATAFED_RELEASE_MINUTE@ diff --git a/python/datafed_pkg/test/security.py b/python/datafed_pkg/test/security.py index d9b479c1a..a8e42c027 100755 --- a/python/datafed_pkg/test/security.py +++ b/python/datafed_pkg/test/security.py @@ -2,7 +2,7 @@ import getpass import datafed.CommandLib -import datafed.SDMS_Auth_pb2 as auth +import datafed.envelope_pb2 as sdms opts = {} @@ -15,7 +15,7 @@ api.loginByPassword(uid, password) -msg = auth.UserCreateRequest() +msg = sdms.UserCreateRequest() msg.uid = "newuser" msg.password = "temptemp" msg.name = "New User" From f3baad39d2eb898c33b9dc3fcbdccab04b230383 Mon Sep 17 00:00:00 2001 From: JoshuaSBrown Date: Fri, 6 Feb 2026 14:16:36 -0500 Subject: [PATCH 09/14] refactor: Python CMakeLists.txt file --- python/datafed_pkg/datafed/CMakeLists.txt | 65 +++++++++++++++++------ 1 file changed, 49 insertions(+), 16 deletions(-) diff --git a/python/datafed_pkg/datafed/CMakeLists.txt b/python/datafed_pkg/datafed/CMakeLists.txt index a66b1f79a..81c536dc6 100644 --- a/python/datafed_pkg/datafed/CMakeLists.txt +++ b/python/datafed_pkg/datafed/CMakeLists.txt @@ -1,13 +1,13 @@ -cmake_minimum_required (VERSION 3.17.0) +cmake_minimum_required(VERSION 3.17.0) # Copy py source to build package source dir -file( GLOB SrcFiles ${CMAKE_CURRENT_SOURCE_DIR}/*.py ) +file(GLOB SrcFiles ${CMAKE_CURRENT_SOURCE_DIR}/*.py) foreach(file ${SrcFiles}) - configure_file(${file} ${CMAKE_CURRENT_BINARY_DIR} COPYONLY ) + configure_file(${file} ${CMAKE_CURRENT_BINARY_DIR} COPYONLY) endforeach() # Collect proto files from the new 1-1-1 directory structure -file( GLOB_RECURSE ProtoFiles ${DataFed_SOURCE_DIR}/common/proto3/common/*.proto ) +file(GLOB_RECURSE ProtoFiles ${DataFed_SOURCE_DIR}/common/proto3/common/*.proto) # OBJECT - is needed because we don't want to compile to a binary # because we are dealing with python @@ -19,20 +19,53 @@ protobuf_generate( IMPORT_DIRS "${DataFed_SOURCE_DIR}/common/proto3/common" OUT_VAR protobuf-generated-files-py PROTOC_OUT_DIR "${CMAKE_CURRENT_BINARY_DIR}" - ) +) + +add_custom_target(pydatafed_src DEPENDS protobuf-target-py) -add_custom_target( pydatafed_src DEPENDS protobuf-target-py ) +# Proto subdirectories that protoc generates imports for +set(PROTO_SUBDIRS anon auth enums messages) -# Fix relative imports in generated pb2 files -# No longer need pyproto_add_msg_idx.py — message type IDs are derived -# at runtime from envelope field numbers via registerEnvelope() -add_custom_command( TARGET pydatafed_src POST_BUILD - COMMAND sed -i -r 's:^import.*_pb2:from . \\0:' ${protobuf-generated-files-py} +# Fix imports in generated pb2 files to use relative imports within the package. +# protoc generates absolute imports like: +# from anon import ack_reply_pb2 as ... +# from enums import error_code_pb2 as ... +# import envelope_pb2 as ... +# These must become relative imports: +# from .anon import ack_reply_pb2 as ... +# from .enums import error_code_pb2 as ... +# from . import envelope_pb2 as ... +add_custom_command(TARGET pydatafed_src POST_BUILD + COMMAND find ${CMAKE_CURRENT_BINARY_DIR} -name "*_pb2.py" -exec + sed -i -r + -e "s:^from (anon|auth|enums|messages) import:from .\\1 import:g" + -e "s:^from (anon|auth|enums|messages)\\.:from .\\1.:g" + -e "s:^import ([a-zA-Z0-9_]*_pb2):from . import \\1:g" + {} + + COMMENT "Rewriting protobuf imports to relative" ) -# Create copies of the files so they show up in the source folder as well -# for the purpose of testing -add_custom_target( pydatafed_proto_src DEPENDS pydatafed_src ) -add_custom_command( TARGET pydatafed_proto_src POST_BUILD pydatafed_src - COMMAND cp ${CMAKE_CURRENT_BINARY_DIR}/envelope_pb2.py ${CMAKE_CURRENT_SOURCE_DIR}/ +# Ensure __init__.py exists in each generated subdirectory +foreach(subdir ${PROTO_SUBDIRS}) + add_custom_command(TARGET pydatafed_src POST_BUILD + COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_CURRENT_BINARY_DIR}/${subdir} + COMMAND ${CMAKE_COMMAND} -E touch ${CMAKE_CURRENT_BINARY_DIR}/${subdir}/__init__.py + ) +endforeach() + +# Copy generated files back to source tree for testing +add_custom_target(pydatafed_proto_src DEPENDS pydatafed_src) +add_custom_command(TARGET pydatafed_proto_src POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy + ${CMAKE_CURRENT_BINARY_DIR}/envelope_pb2.py + ${CMAKE_CURRENT_SOURCE_DIR}/ + # Copy subdirectories back to source for testing + COMMAND ${CMAKE_COMMAND} -E copy_directory + ${CMAKE_CURRENT_BINARY_DIR}/anon ${CMAKE_CURRENT_SOURCE_DIR}/anon + COMMAND ${CMAKE_COMMAND} -E copy_directory + ${CMAKE_CURRENT_BINARY_DIR}/auth ${CMAKE_CURRENT_SOURCE_DIR}/auth + COMMAND ${CMAKE_COMMAND} -E copy_directory + ${CMAKE_CURRENT_BINARY_DIR}/enums ${CMAKE_CURRENT_SOURCE_DIR}/enums + COMMAND ${CMAKE_COMMAND} -E copy_directory + ${CMAKE_CURRENT_BINARY_DIR}/messages ${CMAKE_CURRENT_SOURCE_DIR}/messages ) From 27e124edb4f2d307358d144e8c93a6d31e8f3abe Mon Sep 17 00:00:00 2001 From: JoshuaSBrown Date: Fri, 6 Feb 2026 15:27:08 -0500 Subject: [PATCH 10/14] fix: ( are not handled correctly in CMake --- python/datafed_pkg/datafed/CMakeLists.txt | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/python/datafed_pkg/datafed/CMakeLists.txt b/python/datafed_pkg/datafed/CMakeLists.txt index 81c536dc6..36beb2123 100644 --- a/python/datafed_pkg/datafed/CMakeLists.txt +++ b/python/datafed_pkg/datafed/CMakeLists.txt @@ -38,8 +38,14 @@ set(PROTO_SUBDIRS anon auth enums messages) add_custom_command(TARGET pydatafed_src POST_BUILD COMMAND find ${CMAKE_CURRENT_BINARY_DIR} -name "*_pb2.py" -exec sed -i -r - -e "s:^from (anon|auth|enums|messages) import:from .\\1 import:g" - -e "s:^from (anon|auth|enums|messages)\\.:from .\\1.:g" + -e "s:^from anon import:from .anon import:g" + -e "s:^from anon\\.:from .anon.:g" + -e "s:^from auth import:from .auth import:g" + -e "s:^from auth\\.:from .auth.:g" + -e "s:^from enums import:from .enums import:g" + -e "s:^from enums\\.:from .enums.:g" + -e "s:^from messages import:from .messages import:g" + -e "s:^from messages\\.:from .messages.:g" -e "s:^import ([a-zA-Z0-9_]*_pb2):from . import \\1:g" {} + COMMENT "Rewriting protobuf imports to relative" From 125adc3a8b928b9094eb424f0a45833812c79717 Mon Sep 17 00:00:00 2001 From: JoshuaSBrown Date: Fri, 6 Feb 2026 23:57:09 -0500 Subject: [PATCH 11/14] refactor: python working with message envelope and proto3 --- python/datafed_pkg/CMakeLists.txt | 2 +- python/datafed_pkg/datafed/CMakeLists.txt | 23 +----- python/datafed_pkg/datafed/CommandLib.py | 3 +- python/datafed_pkg/datafed/Connection.py | 79 ++++++++++++------- .../datafed_pkg/scripts/fix_proto_imports.sh | 65 +++++++++++++++ python/datafed_pkg/setup.py | 9 +++ 6 files changed, 130 insertions(+), 51 deletions(-) create mode 100755 python/datafed_pkg/scripts/fix_proto_imports.sh diff --git a/python/datafed_pkg/CMakeLists.txt b/python/datafed_pkg/CMakeLists.txt index 730fbed68..ac834383e 100644 --- a/python/datafed_pkg/CMakeLists.txt +++ b/python/datafed_pkg/CMakeLists.txt @@ -35,4 +35,4 @@ endforeach() add_subdirectory( datafed ) add_custom_target( pydatafed ) -add_dependencies( pydatafed pydatafed_src) +add_dependencies( pydatafed pydatafed_proto_src) diff --git a/python/datafed_pkg/datafed/CMakeLists.txt b/python/datafed_pkg/datafed/CMakeLists.txt index 36beb2123..900b953cd 100644 --- a/python/datafed_pkg/datafed/CMakeLists.txt +++ b/python/datafed_pkg/datafed/CMakeLists.txt @@ -35,30 +35,13 @@ set(PROTO_SUBDIRS anon auth enums messages) # from .anon import ack_reply_pb2 as ... # from .enums import error_code_pb2 as ... # from . import envelope_pb2 as ... + +# Create the import fixup script add_custom_command(TARGET pydatafed_src POST_BUILD - COMMAND find ${CMAKE_CURRENT_BINARY_DIR} -name "*_pb2.py" -exec - sed -i -r - -e "s:^from anon import:from .anon import:g" - -e "s:^from anon\\.:from .anon.:g" - -e "s:^from auth import:from .auth import:g" - -e "s:^from auth\\.:from .auth.:g" - -e "s:^from enums import:from .enums import:g" - -e "s:^from enums\\.:from .enums.:g" - -e "s:^from messages import:from .messages import:g" - -e "s:^from messages\\.:from .messages.:g" - -e "s:^import ([a-zA-Z0-9_]*_pb2):from . import \\1:g" - {} + + COMMAND sh ${DataFed_SOURCE_DIR}/python/datafed_pkg/scripts/fix_proto_imports.sh ${CMAKE_CURRENT_BINARY_DIR} COMMENT "Rewriting protobuf imports to relative" ) -# Ensure __init__.py exists in each generated subdirectory -foreach(subdir ${PROTO_SUBDIRS}) - add_custom_command(TARGET pydatafed_src POST_BUILD - COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_CURRENT_BINARY_DIR}/${subdir} - COMMAND ${CMAKE_COMMAND} -E touch ${CMAKE_CURRENT_BINARY_DIR}/${subdir}/__init__.py - ) -endforeach() - # Copy generated files back to source tree for testing add_custom_target(pydatafed_proto_src DEPENDS pydatafed_src) add_custom_command(TARGET pydatafed_proto_src POST_BUILD diff --git a/python/datafed_pkg/datafed/CommandLib.py b/python/datafed_pkg/datafed/CommandLib.py index 7d2e14b7f..7b9140678 100644 --- a/python/datafed_pkg/datafed/CommandLib.py +++ b/python/datafed_pkg/datafed/CommandLib.py @@ -14,10 +14,9 @@ import time import pathlib import requests -from . import envelope_pb2 as sdms from . import MessageLib from . import Config - +from . import envelope_pb2 as sdms class API: """ diff --git a/python/datafed_pkg/datafed/Connection.py b/python/datafed_pkg/datafed/Connection.py index 4ee0e8885..87546f528 100644 --- a/python/datafed_pkg/datafed/Connection.py +++ b/python/datafed_pkg/datafed/Connection.py @@ -67,6 +67,10 @@ def __init__( self._msg_desc_by_type = {} self._msg_desc_by_name = {} self._msg_type_by_desc = {} + self._field_by_msg_desc = {} + + self._envelope_class = None + self._envelope_desc = None self._address = "tcp://{0}:{1}".format(server_host, server_port) # init zeromq @@ -130,6 +134,10 @@ def registerEnvelope(self, envelope_module, envelope_class_name="Envelope"): envelope_class = getattr(envelope_module, envelope_class_name) envelope_desc = envelope_class.DESCRIPTOR + # Store for envelope wrapping/unwrapping + self._envelope_class = envelope_class + self._envelope_desc = envelope_desc + for field in envelope_desc.fields: if field.message_type is None: # Skip non-message fields (e.g. scalars) if any exist @@ -141,6 +149,7 @@ def registerEnvelope(self, envelope_module, envelope_class_name="Envelope"): self._msg_desc_by_type[msg_type] = desc self._msg_desc_by_name[desc.name] = desc self._msg_type_by_desc[desc] = msg_type + self._field_by_msg_desc[desc] = field self._logger.debug( "Registered %d message types from %s", @@ -177,15 +186,15 @@ def registerProtocol(self, msg_module): ## # @brief Receive a message # - # Receive a protobuf message with timeout. This method automatically - # parses and creates a new protobuf message class based on received - # framing. The new message object, the message name (defined in the - # associated proto file), and re-association context are returned as - # a tuple. On timeout, (None,None,None) is returned. + # Receive a protobuf message with timeout. The wire payload is an + # Envelope message; this method deserializes the Envelope and extracts + # the inner message via the oneof payload field. The inner message + # object, its name, and re-association context are returned as a tuple. + # On timeout, (None, None, None) is returned. # # @param timeout - Timeout in milliseconds - # @return Tuple of message, message type, and re-association context - # @retval (object,str,int) or (None,None,None) on timeout + # @return Tuple of message, message name, and re-association context + # @retval (object, str, int) or (None, None, None) on timeout # @exception Exception: if unregistered message type is received. # def recv(self, a_timeout=1000): @@ -219,38 +228,46 @@ def recv(self, a_timeout=1000): # client self._socket.recv_string(0) - # receive custom frame header and unpack + # Receive frame: 8 bytes = uint32 size + uint16 msg_type + uint16 context frame_data = self._socket.recv(0) - frame_values = struct.unpack(">LBBH", frame_data) - msg_type = (frame_values[1] << 8) | frame_values[2] + frame_values = struct.unpack(">LHH", frame_data) + body_size = frame_values[0] + msg_type = frame_values[1] + ctxt = frame_values[2] - # find message descriptor based on type (descriptor index) - - if not (msg_type in self._msg_desc_by_type): + if msg_type not in self._msg_desc_by_type: raise Exception( "received unregistered message type: {}".format(msg_type) ) - desc = self._msg_desc_by_type[msg_type] + data = self._socket.recv(0) - if frame_values[0] > 0: - # Create message by parsing content - data = self._socket.recv(0) - reply = GetMessageClass(desc)() - reply.ParseFromString(data) + if body_size > 0: + # Deserialize as Envelope + envelope = self._envelope_class() + envelope.ParseFromString(data) + + # Extract inner message from the oneof + payload_field = envelope.WhichOneof("payload") + if payload_field is None: + raise Exception("Received Envelope with no payload set") + reply = getattr(envelope, payload_field) else: - # No content, just create message instance - data = self._socket.recv(0) + # Zero-size body: create empty message instance from type + desc = self._msg_desc_by_type[msg_type] reply = GetMessageClass(desc)() - return reply, desc.name, frame_values[3] + return reply, reply.DESCRIPTOR.name, ctxt else: return None, None, None ## # @brief Send a message # - # Serializes and sends framing and message payload over connection. + # Wraps the inner message in an Envelope, serializes it, and sends + # framing and payload over the connection. The frame header carries the + # message type (Envelope field number) for efficient routing on the + # server side. # # @param message - The protobuf message object to be sent # @param ctxt - Reply re-association value (int) @@ -258,9 +275,15 @@ def recv(self, a_timeout=1000): # def send(self, message, ctxt): # Find msg type by descriptor look-up - if not (message.DESCRIPTOR in self._msg_type_by_desc): + if message.DESCRIPTOR not in self._msg_type_by_desc: raise Exception("Attempt to send unregistered message type.") + msg_type = self._msg_type_by_desc[message.DESCRIPTOR] + field = self._field_by_msg_desc[message.DESCRIPTOR] + + # Wrap inner message in Envelope + envelope = self._envelope_class() + getattr(envelope, field.name).CopyFrom(message) # Initial Null frame self._socket.send_string("BEGIN_DATAFED", zmq.SNDMORE) @@ -274,12 +297,12 @@ def send(self, message, ctxt): self._socket.send_string(self._pub_key, zmq.SNDMORE) self._socket.send_string("no_user", zmq.SNDMORE) - # Serialize - data = message.SerializeToString() + # Serialize the Envelope (not the inner message) + data = envelope.SerializeToString() data_sz = len(data) - # Build the message frame, to match C-struct MessageFrame - frame = struct.pack(">LBBH", data_sz, msg_type >> 8, msg_type & 0xFF, ctxt) + # Build the message frame: uint32 size + uint16 msg_type + uint16 context + frame = struct.pack(">LHH", data_sz, msg_type, ctxt) if data_sz > 0: # Send frame and payload diff --git a/python/datafed_pkg/scripts/fix_proto_imports.sh b/python/datafed_pkg/scripts/fix_proto_imports.sh new file mode 100755 index 000000000..ecde0ce3c --- /dev/null +++ b/python/datafed_pkg/scripts/fix_proto_imports.sh @@ -0,0 +1,65 @@ +#!/bin/sh +set -e + +PROTO_DIR="$1" +ROOT_DIR="${2:-$1}" + +if [ -z "$PROTO_DIR" ]; then + echo "Usage: fix_proto_imports.sh [root_dir]" + echo " proto_output_dir: directory to find and fix _pb2.py files" + echo " root_dir: package root for computing relative depth (defaults to proto_output_dir)" + exit 1 +fi + +find "$PROTO_DIR" -name '*_pb2.py' | while read f; do + relpath=$(realpath --relative-to="$ROOT_DIR" "$f") + case "$relpath" in + */*) + sed -i \ + -e 's:^from anon import:from ..anon import:g' \ + -e 's:^from anon\.:from ..anon.:g' \ + -e 's:^from auth import:from ..auth import:g' \ + -e 's:^from auth\.:from ..auth.:g' \ + -e 's:^from enums import:from ..enums import:g' \ + -e 's:^from enums\.:from ..enums.:g' \ + -e 's:^from messages import:from ..messages import:g' \ + -e 's:^from messages\.:from ..messages.:g' \ + -e 's:^import \(.*_pb2\):from . import \1:g' \ + "$f" + ;; + *) + sed -i \ + -e 's:^from anon import:from .anon import:g' \ + -e 's:^from anon\.:from .anon.:g' \ + -e 's:^from auth import:from .auth import:g' \ + -e 's:^from auth\.:from .auth.:g' \ + -e 's:^from enums import:from .enums import:g' \ + -e 's:^from enums\.:from .enums.:g' \ + -e 's:^from messages import:from .messages import:g' \ + -e 's:^from messages\.:from .messages.:g' \ + -e 's:^import \(.*_pb2\):from . import \1:g' \ + "$f" + ;; + esac +done + +for subdir in anon auth enums messages; do + if [ -d "$ROOT_DIR/$subdir" ]; then + touch "$ROOT_DIR/$subdir/__init__.py" + fi +done + +# Append re-exports to envelope_pb2.py for backward compatibility +# Connection.py uses getattr(envelope_module, class_name) for dynamic dispatch +echo "" >> "$ROOT_DIR/envelope_pb2.py" +echo "# Re-export all message and enum classes for dynamic lookup" >> "$ROOT_DIR/envelope_pb2.py" + +for subdir in anon auth enums messages; do + if [ -d "$ROOT_DIR/$subdir" ]; then + for f in "$ROOT_DIR/$subdir"/*_pb2.py; do + [ -f "$f" ] || continue + module=$(basename "$f" .py) + echo "from .$subdir.$module import *" >> "$ROOT_DIR/envelope_pb2.py" + done + fi +done diff --git a/python/datafed_pkg/setup.py b/python/datafed_pkg/setup.py index 21abefdcc..55ebd76d9 100644 --- a/python/datafed_pkg/setup.py +++ b/python/datafed_pkg/setup.py @@ -22,6 +22,13 @@ long_description_content_type="text/markdown", url="https://github.com/ORNL/DataFed", packages=setuptools.find_packages(), + package_data={ + "datafed": ["*.py"], + "datafed.anon": ["*.py"], + "datafed.auth": ["*.py"], + "datafed.enums": ["*.py"], + "datafed.messages": ["*.py"], + }, setup_requires=["setuptools"], install_requires=install_requires, entry_points={"console_scripts": ["datafed = datafed.CLI:run"]}, @@ -31,3 +38,5 @@ "Operating System :: OS Independent", ], ) + + From 1c5d904ebdccbb1491cbb0ee935852748bd925aa Mon Sep 17 00:00:00 2001 From: JoshuaSBrown Date: Fri, 6 Feb 2026 23:59:45 -0500 Subject: [PATCH 12/14] fix: fix cmake copy command for requirements.txt --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index d6f03811f..098465a9f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -199,7 +199,7 @@ endif() if( BUILD_PYTHON_CLIENT ) # make target = pydatafed - file(COPY ${PROJECT_SOURCE_DIR}/external/DataFedDependencies/python/datafed_pkg/requirements.txt DESTINATION ${PROJECT_SOURCE_DIR}/python/datafed_pkg/requirements.txt) + file(COPY ${PROJECT_SOURCE_DIR}/external/DataFedDependencies/python/datafed_pkg/requirements.txt DESTINATION ${PROJECT_SOURCE_DIR}/python/datafed_pkg) add_subdirectory( python EXCLUDE_FROM_ALL ) endif() From 951ac7854a71a665074198b4ceaeeccf96f9242b Mon Sep 17 00:00:00 2001 From: JoshuaSBrown Date: Sat, 7 Feb 2026 05:00:27 +0000 Subject: [PATCH 13/14] chore: Auto-format shell scripts with shfmt --- .../datafed_pkg/scripts/fix_proto_imports.sh | 90 +++++++++---------- 1 file changed, 45 insertions(+), 45 deletions(-) diff --git a/python/datafed_pkg/scripts/fix_proto_imports.sh b/python/datafed_pkg/scripts/fix_proto_imports.sh index ecde0ce3c..06b10a1a0 100755 --- a/python/datafed_pkg/scripts/fix_proto_imports.sh +++ b/python/datafed_pkg/scripts/fix_proto_imports.sh @@ -5,61 +5,61 @@ PROTO_DIR="$1" ROOT_DIR="${2:-$1}" if [ -z "$PROTO_DIR" ]; then - echo "Usage: fix_proto_imports.sh [root_dir]" - echo " proto_output_dir: directory to find and fix _pb2.py files" - echo " root_dir: package root for computing relative depth (defaults to proto_output_dir)" - exit 1 + echo "Usage: fix_proto_imports.sh [root_dir]" + echo " proto_output_dir: directory to find and fix _pb2.py files" + echo " root_dir: package root for computing relative depth (defaults to proto_output_dir)" + exit 1 fi find "$PROTO_DIR" -name '*_pb2.py' | while read f; do - relpath=$(realpath --relative-to="$ROOT_DIR" "$f") - case "$relpath" in - */*) - sed -i \ - -e 's:^from anon import:from ..anon import:g' \ - -e 's:^from anon\.:from ..anon.:g' \ - -e 's:^from auth import:from ..auth import:g' \ - -e 's:^from auth\.:from ..auth.:g' \ - -e 's:^from enums import:from ..enums import:g' \ - -e 's:^from enums\.:from ..enums.:g' \ - -e 's:^from messages import:from ..messages import:g' \ - -e 's:^from messages\.:from ..messages.:g' \ - -e 's:^import \(.*_pb2\):from . import \1:g' \ - "$f" - ;; - *) - sed -i \ - -e 's:^from anon import:from .anon import:g' \ - -e 's:^from anon\.:from .anon.:g' \ - -e 's:^from auth import:from .auth import:g' \ - -e 's:^from auth\.:from .auth.:g' \ - -e 's:^from enums import:from .enums import:g' \ - -e 's:^from enums\.:from .enums.:g' \ - -e 's:^from messages import:from .messages import:g' \ - -e 's:^from messages\.:from .messages.:g' \ - -e 's:^import \(.*_pb2\):from . import \1:g' \ - "$f" - ;; - esac + relpath=$(realpath --relative-to="$ROOT_DIR" "$f") + case "$relpath" in + */*) + sed -i \ + -e 's:^from anon import:from ..anon import:g' \ + -e 's:^from anon\.:from ..anon.:g' \ + -e 's:^from auth import:from ..auth import:g' \ + -e 's:^from auth\.:from ..auth.:g' \ + -e 's:^from enums import:from ..enums import:g' \ + -e 's:^from enums\.:from ..enums.:g' \ + -e 's:^from messages import:from ..messages import:g' \ + -e 's:^from messages\.:from ..messages.:g' \ + -e 's:^import \(.*_pb2\):from . import \1:g' \ + "$f" + ;; + *) + sed -i \ + -e 's:^from anon import:from .anon import:g' \ + -e 's:^from anon\.:from .anon.:g' \ + -e 's:^from auth import:from .auth import:g' \ + -e 's:^from auth\.:from .auth.:g' \ + -e 's:^from enums import:from .enums import:g' \ + -e 's:^from enums\.:from .enums.:g' \ + -e 's:^from messages import:from .messages import:g' \ + -e 's:^from messages\.:from .messages.:g' \ + -e 's:^import \(.*_pb2\):from . import \1:g' \ + "$f" + ;; + esac done for subdir in anon auth enums messages; do - if [ -d "$ROOT_DIR/$subdir" ]; then - touch "$ROOT_DIR/$subdir/__init__.py" - fi + if [ -d "$ROOT_DIR/$subdir" ]; then + touch "$ROOT_DIR/$subdir/__init__.py" + fi done # Append re-exports to envelope_pb2.py for backward compatibility # Connection.py uses getattr(envelope_module, class_name) for dynamic dispatch -echo "" >> "$ROOT_DIR/envelope_pb2.py" -echo "# Re-export all message and enum classes for dynamic lookup" >> "$ROOT_DIR/envelope_pb2.py" +echo "" >>"$ROOT_DIR/envelope_pb2.py" +echo "# Re-export all message and enum classes for dynamic lookup" >>"$ROOT_DIR/envelope_pb2.py" for subdir in anon auth enums messages; do - if [ -d "$ROOT_DIR/$subdir" ]; then - for f in "$ROOT_DIR/$subdir"/*_pb2.py; do - [ -f "$f" ] || continue - module=$(basename "$f" .py) - echo "from .$subdir.$module import *" >> "$ROOT_DIR/envelope_pb2.py" - done - fi + if [ -d "$ROOT_DIR/$subdir" ]; then + for f in "$ROOT_DIR/$subdir"/*_pb2.py; do + [ -f "$f" ] || continue + module=$(basename "$f" .py) + echo "from .$subdir.$module import *" >>"$ROOT_DIR/envelope_pb2.py" + done + fi done From c8bb534dd5019a5885fe3ad133f2e7b8a9a26d2d Mon Sep 17 00:00:00 2001 From: Joshua S Brown Date: Sat, 7 Feb 2026 00:58:04 -0500 Subject: [PATCH 14/14] [DAPS-1848] - refactor: update authz files to use proto3. (#1851) * [DAPS-1850] - refactor web server proto3 (#1852) --- CMakeLists.txt | 6 +- docker/Dockerfile.runtime | 1 + .../globus5/authz/source/AuthzWorker.cpp | 30 +- .../globus5/authz/source/Version.hpp.in | 17 ++ .../authz/tests/unit/test_AuthzWorker.cpp | 6 +- web/datafed-ws.js | 261 +++++++++--------- web/docker/Dockerfile | 5 +- web/static/api.js | 52 ---- web/version.js.in | 7 +- 9 files changed, 174 insertions(+), 211 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 098465a9f..cb3a1fb3d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -128,8 +128,8 @@ endif() if( BUILD_WEB_SERVER ) include(./cmake/Web.cmake) - file(COPY "${CMAKE_CURRENT_SOURCE_DIR}/common/proto/common/" - DESTINATION "${CMAKE_CURRENT_SOURCE_DIR}/web/proto/") + file(COPY "${CMAKE_CURRENT_SOURCE_DIR}/common/proto3/common/" + DESTINATION "${CMAKE_CURRENT_SOURCE_DIR}/web/proto3/") if( ENABLE_UNIT_TESTS ) add_test(NAME unit_tests_web COMMAND "${DEPENDENCY_INSTALL_PATH}/nvm/versions/node/${LOCAL_NODE_VERSION}/bin/npm" "test") @@ -227,9 +227,9 @@ if( INSTALL_CORE_SERVER ) endif() if( INSTALL_WEB_SERVER ) - install( FILES ${ProtoFiles} DESTINATION ${DATAFED_INSTALL_PATH}/web ) install( DIRECTORY ${PROJECT_SOURCE_DIR}/web/static DESTINATION ${DATAFED_INSTALL_PATH}/web ) install( DIRECTORY ${PROJECT_SOURCE_DIR}/web/views DESTINATION ${DATAFED_INSTALL_PATH}/web ) + install( DIRECTORY ${PROJECT_SOURCE_DIR}/web/proto3 DESTINATION ${DATAFED_INSTALL_PATH}/web ) install( FILES ${PROJECT_SOURCE_DIR}/web/version.js DESTINATION ${DATAFED_INSTALL_PATH}/web ) endif() diff --git a/docker/Dockerfile.runtime b/docker/Dockerfile.runtime index eb564dcc9..b8c8bfff1 100644 --- a/docker/Dockerfile.runtime +++ b/docker/Dockerfile.runtime @@ -27,6 +27,7 @@ COPY ./scripts/dependency_versions.sh ${BUILD_DIR}/scripts/ RUN mkdir -p ${DATAFED_DIR} RUN mkdir -p /opt/datafed RUN mkdir -p /var/log/datafed +RUN mkdir -p /opt/datafed/logs RUN chown -R datafed:root /opt/datafed RUN chown -R datafed:root /var/log/datafed RUN chown -R datafed:root ${DATAFED_DIR} diff --git a/repository/gridftp/globus5/authz/source/AuthzWorker.cpp b/repository/gridftp/globus5/authz/source/AuthzWorker.cpp index 234c1e166..9050bb45b 100644 --- a/repository/gridftp/globus5/authz/source/AuthzWorker.cpp +++ b/repository/gridftp/globus5/authz/source/AuthzWorker.cpp @@ -14,11 +14,8 @@ #include "common/TraceException.hpp" #include "common/Util.hpp" -// Protobuf includes -#include "common/SDMS.pb.h" -#include "common/SDMS_Anon.pb.h" -#include "common/SDMS_Auth.pb.h" -#include "common/Version.pb.h" +// Proto files +#include "common/envelope.pb.h" // Standard includes #include @@ -28,9 +25,8 @@ #include #include +using namespace SDMS; using namespace std; -using namespace SDMS::Anon; -using namespace SDMS::Auth; namespace { @@ -507,7 +503,7 @@ int AuthzWorker::processResponse(ICommunicator::Response &response) { auto payload = std::get(response.message->getPayload()); - Anon::NackReply *nack = dynamic_cast(payload); + NackReply *nack = dynamic_cast(payload); if (!nack) { return 0; } else { @@ -581,7 +577,7 @@ int AuthzWorker::checkAuth(char *client_id, char *path, char *action) { return 0; } - auto auth_req = std::make_unique(); + auto auth_req = std::make_unique(); auth_req->set_repo(m_config->repo_id); auth_req->set_client(client_id); @@ -619,19 +615,19 @@ const char *getVersion() { const char *getAPIVersion() { static std::string ver_str = - std::to_string(DATAFED_COMMON_PROTOCOL_API_MAJOR) + "." + - std::to_string(DATAFED_COMMON_PROTOCOL_API_MINOR) + "." + - std::to_string(DATAFED_COMMON_PROTOCOL_API_PATCH); + std::to_string(protocol::version::MAJOR) + "." + + std::to_string(protocol::version::MINOR) + "." + + std::to_string(protocol::version::PATCH); return ver_str.c_str(); } const char *getReleaseVersion() { - static std::string ver_str = std::to_string(DATAFED_RELEASE_YEAR) + "." + - std::to_string(DATAFED_RELEASE_MONTH) + "." + - std::to_string(DATAFED_RELEASE_DAY) + "." + - std::to_string(DATAFED_RELEASE_HOUR) + "." + - std::to_string(DATAFED_RELEASE_MINUTE); + static std::string ver_str = std::to_string(release::YEAR) + "." + + std::to_string(release::MONTH) + "." + + std::to_string(release::DAY) + "." + + std::to_string(release::HOUR) + "." + + std::to_string(release::MINUTE); return ver_str.c_str(); } diff --git a/repository/gridftp/globus5/authz/source/Version.hpp.in b/repository/gridftp/globus5/authz/source/Version.hpp.in index cdd5e35e0..f698e5b7a 100644 --- a/repository/gridftp/globus5/authz/source/Version.hpp.in +++ b/repository/gridftp/globus5/authz/source/Version.hpp.in @@ -10,6 +10,23 @@ namespace SDMS { constexpr int PATCH = @DATAFED_AUTHZ_PATCH@; } } + + namespace protocol { + namespace version { + constexpr int MAJOR = @DATAFED_COMMON_PROTOCOL_API_MAJOR@; + constexpr int MINOR = @DATAFED_COMMON_PROTOCOL_API_MINOR@; + constexpr int PATCH = @DATAFED_COMMON_PROTOCOL_API_PATCH@; + } + } + + namespace release { + constexpr int YEAR = @DATAFED_RELEASE_YEAR@; + constexpr int MONTH = @DATAFED_RELEASE_MONTH@; + constexpr int DAY = @DATAFED_RELEASE_DAY@; + constexpr int HOUR = @DATAFED_RELEASE_HOUR@; + constexpr int MINUTE = @DATAFED_RELEASE_MINUTE@; + } + } #endif // AUTHZ_VERSION_HPP diff --git a/repository/gridftp/globus5/authz/tests/unit/test_AuthzWorker.cpp b/repository/gridftp/globus5/authz/tests/unit/test_AuthzWorker.cpp index ba51d8d08..5b03f8466 100644 --- a/repository/gridftp/globus5/authz/tests/unit/test_AuthzWorker.cpp +++ b/repository/gridftp/globus5/authz/tests/unit/test_AuthzWorker.cpp @@ -15,7 +15,7 @@ #include "common/ICommunicator.hpp" #include "common/IMessage.hpp" #include "common/MessageFactory.hpp" -#include "common/SDMS_Anon.pb.h" +#include "common/envelope.pb.h" #include "common/TraceException.hpp" extern "C" { @@ -397,7 +397,7 @@ BOOST_AUTO_TEST_CASE(ProcessResponseWithValidMessage) { SDMS::MessageState::REQUEST); response.message->set(SDMS::constants::message::google::CONTEXT, context); auto auth_by_token_req = - std::make_unique(); + std::make_unique(); std::string token = "golden_chest"; auth_by_token_req->set_token(token); @@ -429,7 +429,7 @@ BOOST_AUTO_TEST_CASE(ProcessResponseWithNackReply) { response.message->set(SDMS::MessageAttribute::STATE, SDMS::MessageState::REQUEST); response.message->set(SDMS::constants::message::google::CONTEXT, context); - auto nack = std::make_unique(); + auto nack = std::make_unique(); response.message->setPayload(std::move(nack)); diff --git a/web/datafed-ws.js b/web/datafed-ws.js index d4373c707..b52b23c31 100755 --- a/web/datafed-ws.js +++ b/web/datafed-ws.js @@ -15,7 +15,7 @@ if (process.argv.length != 3) { throw "Invalid arguments, usage: datafed-ws config-file"; } -import web_version from "./version.js"; +import version from "./version.js"; import express from "express"; // For REST api import session from "express-session"; import sanitizeHtml from "sanitize-html"; @@ -55,6 +55,7 @@ var g_host, g_test, g_msg_by_id = {}, g_msg_by_name = {}, + g_envelope_type, g_core_sock = zmq.socket("dealer"), g_core_serv_addr, g_globus_auth, @@ -64,7 +65,7 @@ var g_host, g_ctx_next = 0, g_client_id, g_client_secret, - g_ready_start = 4, + g_ready_start = 3, g_version, g_ver_release_year, g_ver_release_month, @@ -145,6 +146,25 @@ class Logger { const logger = new Logger(LogLevel.INFO); +g_ver_release_year = version.DATAFED_RELEASE_YEAR; +g_ver_release_month = version.DATAFED_RELEASE_MONTH; +g_ver_release_day = version.DATAFED_RELEASE_DAY; +g_ver_release_hour = version.DATAFED_RELEASE_HOUR; +g_ver_release_minute = version.DATAFED_RELEASE_MINUTE; + +g_version = + g_ver_release_year + + "." + + g_ver_release_month + + "." + + g_ver_release_day + + "." + + g_ver_release_hour + + "." + + g_ver_release_minute; + +if (--g_ready_start == 0) startServer(); + function getCurrentLineNumber() { const stackTrace = new Error().stack; const lineMatches = stackTrace.match(/:\d+:\d+/g); @@ -1108,23 +1128,6 @@ app.get("/api/dat/lock", (a_req, a_resp) => { ); }); -app.get("/api/dat/lock/toggle", (a_req, a_resp) => { - sendMessage("RecordLockToggleRequest", { id: a_req.query.id }, a_req, a_resp, function (reply) { - a_resp.send(reply); - }); -}); - -app.get("/api/dat/copy", (a_req, a_resp) => { - var params = { - sourceId: a_req.query.src, - destId: a_req.query.dst, - }; - - sendMessage("DataCopyRequest", params, a_req, a_resp, function (reply) { - a_resp.send(reply); - }); -}); - app.get("/api/dat/delete", (a_req, a_resp) => { sendMessage( "RecordDeleteRequest", @@ -1211,18 +1214,6 @@ app.get("/api/dat/put", (a_req, a_resp) => { }); }); -app.get("/api/dat/dep/get", (a_req, a_resp) => { - sendMessage( - "RecordGetDependenciesRequest", - { id: a_req.query.ids }, - a_req, - a_resp, - function (reply) { - a_resp.send(reply); - }, - ); -}); - app.get("/api/dat/dep/graph/get", (a_req, a_resp) => { sendMessage( "RecordGetDependencyGraphRequest", @@ -1569,12 +1560,6 @@ app.get("/api/col/published/list", (a_req, a_resp) => { }); }); -app.post("/api/cat/search", (a_req, a_resp) => { - sendMessage("CatalogSearchRequest", a_req.body, a_req, a_resp, function (reply) { - a_resp.send(reply); - }); -}); - app.get("/api/globus/consent_url", storeCollectionId, (a_req, a_resp) => { const { requested_scopes, state, refresh_tokens, query_params } = a_req.query; @@ -1590,12 +1575,6 @@ app.get("/api/globus/consent_url", storeCollectionId, (a_req, a_resp) => { a_resp.json({ consent_url }); }); -app.post("/api/col/pub/search/data", (a_req, a_resp) => { - sendMessage("RecordSearchPublishedRequest", a_req.body, a_req, a_resp, function (reply) { - a_resp.send(reply); - }); -}); - app.get("/api/repo/list", (a_req, a_resp) => { var params = {}; if (a_req.query.all) params.all = a_req.query.all; @@ -1772,18 +1751,6 @@ app.get("/api/top/list/topics", (a_req, a_resp) => { }); }); -app.get("/api/top/list/coll", (a_req, a_resp) => { - var par = { topicId: a_req.query.id }; - if (a_req.query.offset != undefined && a_req.query.count != undefined) { - par.offset = a_req.query.offset; - par.count = a_req.query.count; - } - - sendMessage("TopicListCollectionsRequest", par, a_req, a_resp, function (reply) { - a_resp.json(reply); - }); -}); - app.get("/api/top/view", (a_req, a_resp) => { sendMessage("TopicViewRequest", { id: a_req.query.id }, a_req, a_resp, function (reply) { a_resp.json(reply); @@ -2065,15 +2032,17 @@ function sendMessage(a_msg_name, a_msg_data, a_req, a_resp, a_cb, a_anon) { a_resp.setHeader("Content-Type", "application/json"); allocRequestContext(a_resp, function (ctx) { - var msg = g_msg_by_name[a_msg_name]; - if (!msg) throw "Invalid message type: " + a_msg_name; + var msg_info = g_msg_by_name[a_msg_name]; + if (!msg_info) throw "Invalid message type: " + a_msg_name; - var msg_buf = msg.encode(a_msg_data).finish(); + // Wrap inner message data in an Envelope (matches C++ sendBody wrapInEnvelope) + var envelope_data = {}; + envelope_data[msg_info.field_name] = a_msg_data; + var msg_buf = g_envelope_type.encode(envelope_data).finish(); var frame = Buffer.alloc(8); frame.writeUInt32BE(msg_buf.length, 0); - frame.writeUInt8(msg._pid, 4); - frame.writeUInt8(msg._mid, 5); + frame.writeUInt16BE(msg_info.field_id, 4); frame.writeUInt16BE(ctx, 6); g_ctx[ctx] = function (a_reply) { @@ -2121,7 +2090,10 @@ function sendMessage(a_msg_name, a_msg_data, a_req, a_resp, a_cb, a_anon) { sendMessage.name, getCurrentLineNumber(), "MsgType is: " + - msg._msg_type + + msg_info.field_id + + " (" + + a_msg_name + + ")" + " Writing ctx to frame, " + ctx + " buffer size " + @@ -2142,7 +2114,10 @@ function sendMessage(a_msg_name, a_msg_data, a_req, a_resp, a_cb, a_anon) { sendMessage.name, getCurrentLineNumber(), "MsgType is: " + - msg._msg_type + + msg_info.field_id + + " (" + + a_msg_name + + ")" + " Writing ctx to frame, " + ctx + " buffer size " + @@ -2154,17 +2129,19 @@ function sendMessage(a_msg_name, a_msg_data, a_req, a_resp, a_cb, a_anon) { } function sendMessageDirect(a_msg_name, a_client, a_msg_data, a_cb) { - var msg = g_msg_by_name[a_msg_name]; - if (!msg) throw "Invalid message type: " + a_msg_name; + var msg_info = g_msg_by_name[a_msg_name]; + if (!msg_info) throw "Invalid message type: " + a_msg_name; allocRequestContext(null, function (ctx) { - var msg_buf = msg.encode(a_msg_data).finish(); + // Wrap inner message data in an Envelope (matches C++ sendBody wrapInEnvelope) + var envelope_data = {}; + envelope_data[msg_info.field_name] = a_msg_data; + var msg_buf = g_envelope_type.encode(envelope_data).finish(); var frame = Buffer.alloc(8); // A protobuf message doesn't have to have a payload frame.writeUInt32BE(msg_buf.length, 0); - frame.writeUInt8(msg._pid, 4); - frame.writeUInt8(msg._mid, 5); + frame.writeUInt16BE(msg_info.field_id, 4); frame.writeUInt16BE(ctx, 6); g_ctx[ctx] = a_cb; @@ -2187,7 +2164,10 @@ function sendMessageDirect(a_msg_name, a_client, a_msg_data, a_cb) { sendMessageDirect.name, getCurrentLineNumber(), "MsgType is: " + - msg._msg_type + + msg_info.field_id + + " (" + + a_msg_name + + ")" + " Direct Writing ctx to frame, " + ctx + " buffer size " + @@ -2208,7 +2188,10 @@ function sendMessageDirect(a_msg_name, a_client, a_msg_data, a_cb) { sendMessageDirect.name, getCurrentLineNumber(), "MsgType is: " + - msg._msg_type + + msg_info.field_id + + " (" + + a_msg_name + + ")" + " Direct Writing ctx to frame, " + ctx + " buffer size " + @@ -2219,71 +2202,65 @@ function sendMessageDirect(a_msg_name, a_client, a_msg_data, a_cb) { }); } -function processProtoFile(msg) { - //var mlist = msg.parent.order; - var i, - msg_list = []; - for (i in msg.parent.nested) msg_list.push(msg.parent.nested[i]); - - //msg_list.sort(); - - var pid = msg.values.ID; +/** + * Processes the proto3 Envelope message to build message type maps. + * + * Instead of the old proto2 approach that derived message types from Protocol enum IDs + * and file ordering (pid << 8 | mid), this uses the envelope's oneof field numbers + * as stable message type identifiers. + * + * Each map entry stores: + * - type: the protobufjs Type (for encode/decode of the inner message) + * - field_name: the envelope oneof field name (e.g. "version_request") + * - field_id: the envelope field number (used as msg_type in the frame) + * + * @param {protobuf.Root} root - The loaded protobuf root containing SDMS.Envelope + */ +function processEnvelope(root) { + g_envelope_type = root.lookupType("SDMS.Envelope"); + var payloadOneof = g_envelope_type.oneofs.payload; - for (i = 1; i < msg_list.length; i++) { - msg = msg_list[i]; - msg._pid = pid; - msg._mid = i - 1; - msg._msg_type = (pid << 8) | (i - 1); + if (!payloadOneof) throw "Missing 'payload' oneof in SDMS.Envelope"; - g_msg_by_id[msg._msg_type] = msg; - g_msg_by_name[msg.name] = msg; - } -} + payloadOneof.fieldsArray.forEach(function (field) { + var msgType = field.resolvedType; + if (!msgType) { + logger.warning( + processEnvelope.name, + getCurrentLineNumber(), + "Unresolved type for envelope field: " + field.name, + ); + return; + } -protobuf.load("Version.proto", function (err, root) { - if (err) throw err; + var entry = { + type: msgType, // protobufjs Type for encode/decode + field_name: field.name, // envelope oneof field name + field_id: field.id, // envelope field number = msg_type in frame + }; - var msg = root.lookupEnum("Version"); - if (!msg) throw "Missing Version enum in Version.Anon proto file"; - - g_ver_release_year = msg.values.DATAFED_RELEASE_YEAR; - g_ver_release_month = msg.values.DATAFED_RELEASE_MONTH; - g_ver_release_day = msg.values.DATAFED_RELEASE_DAY; - g_ver_release_hour = msg.values.DATAFED_RELEASE_HOUR; - g_ver_release_minute = msg.values.DATAFED_RELEASE_MINUTE; - - g_version = - g_ver_release_year + - "." + - g_ver_release_month + - "." + - g_ver_release_day + - "." + - g_ver_release_hour + - "." + - g_ver_release_minute; - - logger.info("protobuf.load", getCurrentLineNumber(), "Running Version: " + g_version); - if (--g_ready_start == 0) startServer(); -}); + g_msg_by_id[field.id] = entry; + g_msg_by_name[msgType.name] = entry; + }); -protobuf.load("SDMS_Anon.proto", function (err, root) { - if (err) throw err; + logger.info( + processEnvelope.name, + getCurrentLineNumber(), + "Loaded " + Object.keys(g_msg_by_id).length + " message types from envelope", + ); +} - var msg = root.lookupEnum("SDMS.Anon.Protocol"); - if (!msg) throw "Missing Protocol enum in SDMS.Anon proto file"; +var protobufRoot = new protobuf.Root(); - processProtoFile(msg); - if (--g_ready_start == 0) startServer(); -}); +protobufRoot.resolvePath = function (origin, target) { + return "proto3/" + target; +}; -protobuf.load("SDMS_Auth.proto", function (err, root) { +protobufRoot.load("envelope.proto", function (err, root) { if (err) throw err; - var msg = root.lookupEnum("SDMS.Auth.Protocol"); - if (!msg) throw "Missing Protocol enum in SDMS.Auth proto file"; - - processProtoFile(msg); + root.resolveAll(); + processEnvelope(root); if (--g_ready_start == 0) startServer(); }); @@ -2304,20 +2281,34 @@ g_core_sock.on( var mtype = (frame.readUInt8(4) << 8) | frame.readUInt8(5); var ctx = frame.readUInt16BE(6); - var msg_class = g_msg_by_id[mtype]; + var msg_info = g_msg_by_id[mtype]; var msg; + var msg_name = msg_info ? msg_info.type.name : "unknown(" + mtype + ")"; - if (msg_class) { + if (msg_info) { // Only try to decode if there is a payload if (msg_buf && msg_buf.length) { try { - // This is unserializing the protocol message - msg = msg_class.decode(msg_buf); + // Decode as Envelope (matches C++ receiveBody unwrapFromEnvelope) + var envelope = g_envelope_type.decode(msg_buf); + var which_field = envelope.payload; // oneof discriminator: field name that is set + if (which_field) { + msg = envelope[which_field]; + msg_name = which_field; + } else { + logger.warning( + "g_core_sock.on", + getCurrentLineNumber(), + "Envelope decoded but no payload field set, correlation_id: " + + correlation_id, + ); + msg = msg_info.type.create({}); + } if (!msg) { logger.error( "g_core_sock.on", getCurrentLineNumber(), - "ERROR: msg decode failed: no reason, correlation_id: " + + "ERROR: envelope decode produced null msg, correlation_id: " + correlation_id, ); } @@ -2325,11 +2316,15 @@ g_core_sock.on( logger.error( "g_core_sock.on", getCurrentLineNumber(), - "ERROR: msg decode failed: " + err + " correlation_id: " + correlation_id, + "ERROR: envelope decode failed: " + + err + + " correlation_id: " + + correlation_id, ); } } else { - msg = msg_class; + // No payload body - create empty message instance + msg = msg_info.type.create({}); } } else { logger.error( @@ -2345,7 +2340,7 @@ g_core_sock.on( logger.info( "g_core_sock.on", getCurrentLineNumber(), - "freed ctx: " + ctx + " for msg: " + msg_class.name, + "freed ctx: " + ctx + " for msg: " + msg_name, correlation_id, ); g_ctx_next = ctx; @@ -2360,7 +2355,7 @@ g_core_sock.on( " - msg type: " + mtype + ", name: " + - msg_class.name + + msg_name + " correlation_id: " + correlation_id, ); diff --git a/web/docker/Dockerfile b/web/docker/Dockerfile index 5fafb6b08..15cbbc2b1 100644 --- a/web/docker/Dockerfile +++ b/web/docker/Dockerfile @@ -39,7 +39,7 @@ COPY ./scripts/generate_ws_config.sh ${BUILD_DIR}/scripts/ COPY ./scripts/install_ws.sh ${BUILD_DIR}/scripts/ COPY ./scripts/export_dependency_version.sh ${BUILD_DIR}/scripts/ COPY ./cmake ${BUILD_DIR}/cmake -COPY ./common/proto ${BUILD_DIR}/common/proto +COPY ./common/proto3 ${BUILD_DIR}/common/proto3 COPY ./web ${BUILD_DIR}/web RUN ${DATAFED_DEPENDENCIES_ROOT}/scripts/generate_dependencies_config.sh && \ @@ -83,6 +83,7 @@ WORKDIR ${DATAFED_DIR} USER datafed +RUN mkdir -p ${DATAFED_DEFAULT_LOG_PATH}; chown root:datafed ${DATAFED_DEFAULT_LOG_PATH}; chmod -R g+rw ${DATAFED_DEFAULT_LOG_PATH} COPY --from=ws-build --chown=datafed:root ${DATAFED_DEPENDENCIES_ROOT}/scripts/ {DATAFED_DEPENDENCIES_ROOT}/scripts/ COPY --chown=datafed:root ./scripts/generate_datafed.sh ${BUILD_DIR}/scripts/generate_datafed.sh @@ -97,7 +98,7 @@ COPY --from=ws-build --chown=datafed:root ${DATAFED_DEPENDENCIES_ROOT}/scripts $ COPY --from=ws-build --chown=datafed:root ${DATAFED_INSTALL_PATH}/web ${DATAFED_INSTALL_PATH}/web COPY --from=ws-build --chown=datafed:root /usr/bin/curl /usr/bin/curl -WORKDIR ${BUILD_DIR} +WORKDIR ${BUILD_DIR}/web USER root diff --git a/web/static/api.js b/web/static/api.js index 3adf056f5..72af70138 100644 --- a/web/static/api.js +++ b/web/static/api.js @@ -184,17 +184,6 @@ export function dataPutCheck(a_id, a_cb) { _asyncGet("/api/dat/put?id=" + encodeURIComponent(a_id) + "&check=true", null, a_cb); } -export function dataGetDeps(a_ids, a_cb) { - _asyncGet("/api/dat/dep/get?ids=" + encodeURIComponent(a_ids), null, function (ok, data) { - if (ok) { - a_cb(data); - } else { - util.setStatusText("Get Dependencies Error: " + data, true); - a_cb(); - } - }); -} - export function dataGetDepGraph(a_id, a_cb) { _asyncGet("/api/dat/dep/graph/get?id=" + encodeURIComponent(a_id), null, function (ok, data) { if (ok) { @@ -240,25 +229,10 @@ export function sendDataDelete(a_ids, a_cb) { _asyncGet("/api/dat/delete?ids=" + encodeURIComponent(JSON.stringify(a_ids)), null, a_cb); } -export function copyData(a_src_id, a_dst_id, a_cb) { - _asyncGet( - "/api/dat/copy?src=" + - encodeURIComponent(a_src_id) + - "&dst=" + - encodeURIComponent(a_dst_id), - null, - a_cb, - ); -} - export function dataSearch(a_query, a_callback) { _asyncPost("/api/dat/search", a_query, a_callback); } -export function dataPubSearch(a_query, a_cb) { - _asyncPost("/api/col/pub/search/data", a_query, a_cb); -} - export function sendDataLock(a_ids, a_lock, a_cb) { _asyncGet( "/api/dat/lock?lock=" + a_lock + "&ids=" + encodeURIComponent(JSON.stringify(a_ids)), @@ -334,13 +308,6 @@ export function collDelete(a_ids, a_cb) { _asyncGet("/api/col/delete?ids=" + encodeURIComponent(JSON.stringify(a_ids)), null, a_cb); } -export function catalogSearch(a_query, a_cb) { - _asyncPost("/api/cat/search", a_query, a_cb); - /*_asyncPost( "/api/col/pub/search", a_query, function( ok, data ){ - setTimeout( function(){ a_cb( ok, data ); }, 2000 ); - });*/ -} - export function projList_url(a_owned, a_admin, a_member, a_sort, a_offset, a_count) { return ( "/api/prj/list?owner=" + @@ -852,25 +819,6 @@ export function topicListTopics(a_id, a_offset, a_count, a_cb) { if (!a_cb) return; _asyncGet(topicListTopics_url(a_id, a_offset, a_count), null, a_cb); - /*_asyncGet( topicListTopics_url( a_id, a_offset, a_count ), null, function( ok, data ){ - setTimeout( function(){ a_cb( ok, data ); }, 2000 ); - });*/ -} - -export function topicListColl_url(a_id, a_offset, a_count) { - return ( - "/api/top/list/coll?id=" + - a_id + - (a_offset != undefined && a_count != undefined - ? "&offset=" + a_offset + "&count=" + a_count - : "") - ); -} - -export function topicListColl(a_id, a_offset, a_count, a_cb) { - if (!a_cb) return; - - _asyncGet(topicListColl_url(a_id, a_offset, a_count), null, a_cb); } export function topicSearch_url(a_phrase) { diff --git a/web/version.js.in b/web/version.js.in index 69ec65aca..ca3eb3b97 100644 --- a/web/version.js.in +++ b/web/version.js.in @@ -2,5 +2,10 @@ const MAJOR = @DATAFED_WEB_MAJOR@; const MINOR = @DATAFED_WEB_MINOR@; const PATCH = @DATAFED_WEB_PATCH@; +const RELEASE_YEAR = @DATAFED_RELEASE_YEAR@; +const RELEASE_MONTH = @DATAFED_RELEASE_MONTH@; +const RELEASE_DAY = @DATAFED_RELEASE_DAY@; +const RELEASE_HOUR = @DATAFED_RELEASE_HOUR@; +const RELEASE_MINUTE = @DATAFED_RELEASE_MINUTE@; -export default { MAJOR, MINOR, PATCH }; +export default { MAJOR, MINOR, PATCH, RELEASE_YEAR, RELEASE_MONTH, RELEASE_DAY, RELEASE_HOUR, RELEASE_MINUTE }