diff --git a/Jenkinsfile b/Jenkinsfile index 856b13e..fa490ef 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,6 +1,6 @@ import java.text.SimpleDateFormat jobName = "metadata-exporter" -version = "0.1.66" +version = "0.1.68" build_dir = "build" buildPackageName = "meta-exporter" @@ -11,7 +11,7 @@ node { git branch: 'master', url: 'git@github.com:MONROE-PROJECT/Utilities.git' checkout([$class: 'GitSCM', - branches: [[name: 'nne-iot']], + branches: [[name: 'test-merge-ubus']], doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'metadata-exporter-alt']], submoduleCfg: [], @@ -33,39 +33,50 @@ node { userRemoteConfigs: [[url: 'git@github.com:Celerway/celerway-jenkins.git']]]) } - docker.withRegistry('http://registry:5000') { - docker.image('registry:5000/jenkins-slave:monroe').inside('-u jenkins') { + withDockerRegistry(credentialsId: 'gcr:nimbus-tools-gcr', url: 'http://eu.gcr.io/nimbus-tools') { + docker.image('eu.gcr.io/nimbus-tools/monroe-builder:stretch').inside('-u jenkins') { + stage ('Install libubox') { + copyArtifacts filter: '**/*.deb', fingerprintArtifacts: true, projectName: 'monroe/libubox', selector: lastSuccessful() + sh '''sudo find . -name "*.deb" -exec dpkg -i {} \\;''' + sh "rm libubox_*_amd64.deb" + } - stage ('Build') { - dir(build_dir) { - sh "cmake ../metadata-exporter-alt -DNNE=1 -DSQLITE3=1 -DZEROMQ_INPUT=1 -DZEROMQ_WRITER=1 -DZEROMQ_RELAY=1 -DGPS_NSB=1 -DMUNIN=1 -DSYSEVENT=1 -DNEAT=1 && make && make package" - } - sh "chmod +x versionize/versionize.sh; cp versionize/versionize.sh build/" - sh "chmod -R g-s ${build_dir}" - dir(build_dir) { - sh "mv meta_exporter-0.1.0-Linux.deb meta-exporter-0.1.0-Linux.deb" - sh "./versionize.sh meta-exporter-0.1.0-Linux.deb ${buildPackageName} ${version} ${shortCommit} || true" - sh "rm meta-exporter-0.1.0-Linux.deb" - } - } + stage ('Install libubus') { + step([$class: 'CopyArtifact', filter: '**/*.deb', fingerprintArtifacts: true, projectName: 'monroe/ubus', selector: [$class: 'StatusBuildSelector', stable: false]]) + sh '''sudo find . -name "*.deb" -exec dpkg -i {} \\;''' + sh "rm ubusd_*_amd64.deb" + } - stage ('Configure') { - dir(build_dir) { - sh """echo `cat pk_${buildPackageName}/DEBIAN/control |grep Version|sed -e 's/Version: //'` > pkgver""" - } - sh """cp -an metadata-exporter/* build/pk_${buildPackageName}/""" - dir(build_dir) { - sh """sed -i -e 's/${buildPackageName}/metadata-exporter/g' pk_${buildPackageName}/DEBIAN/md5sums pk_${buildPackageName}/DEBIAN/control""" - sh """mkdir -p pk_${buildPackageName}/usr/sbin && mv pk_${buildPackageName}/usr/sbin/meta_exporter pk_${buildPackageName}/usr/sbin/metadata-exporter""" - sh "chmod -R g-s pk_meta-exporter" - sh '''PKGVER=`cat pkgver` ;dpkg -b pk_meta-exporter metadata-exporter-${PKGVER}-Linux.deb''' - sh "rm meta-exporter*.deb" - } - } + stage ('Build') { + dir(build_dir) { + sh "cmake ../metadata-exporter-alt/src -DNNE=1 -DSQLITE3=1 -DZEROMQ_INPUT=1 -DZEROMQ_WRITER=1 -DZEROMQ_RELAY=1 -DGPS_NSB=1 -DMUNIN=1 -DSYSEVENT=1 -DNEAT=1 -DUBUS=1 && make && make package" + } + sh "chmod +x versionize/versionize.sh; cp versionize/versionize.sh build/" + sh "chmod -R g-s ${build_dir}" + dir(build_dir) { + sh "mv meta_exporter-0.1.0-Linux.deb meta-exporter-0.1.0-Linux.deb" + sh "./versionize.sh meta-exporter-0.1.0-Linux.deb ${buildPackageName} ${version} ${shortCommit} || true" + sh "rm meta-exporter-0.1.0-Linux.deb" + } + } - stage ('Archive artifacts') { - archiveArtifacts "${build_dir}/*.deb" - } + stage ('Configure') { + dir(build_dir) { + sh """echo `cat pk_${buildPackageName}/DEBIAN/control |grep Version|sed -e 's/Version: //'` > pkgver""" + } + sh """cp -an metadata-exporter/* build/pk_${buildPackageName}/""" + dir(build_dir) { + sh """sed -i -e 's/${buildPackageName}/metadata-exporter/g' pk_${buildPackageName}/DEBIAN/md5sums pk_${buildPackageName}/DEBIAN/control""" + sh """mkdir -p pk_${buildPackageName}/usr/sbin && mv pk_${buildPackageName}/usr/sbin/meta_exporter pk_${buildPackageName}/usr/sbin/metadata-exporter""" + sh "chmod -R g-s pk_meta-exporter" + sh '''PKGVER=`cat pkgver` ;dpkg -b pk_meta-exporter metadata-exporter-${PKGVER}-Linux.deb''' + sh "rm meta-exporter*.deb" + } + } + + stage ('Archive artifacts') { + archiveArtifacts "${build_dir}/*.deb" + } } } } diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..0e9d311 --- /dev/null +++ b/Makefile @@ -0,0 +1,47 @@ +include $(TOPDIR)/rules.mk + +PKG_NAME:=data-exporter +PKG_VERSION:=2021-12-13 +PKG_RELEASE=1.4 + +CMAKE_INSTALL:=1 + +PKG_LICENSE:=LGPLv2.1 +PKG_LICENSE_FILES:= + +PKG_MAINTAINER:=Lukasz Baj +PKG_BUILD_PARALLEL:=1 + +COMMIT_NO := $(shell git log --pretty=%H -1) +CMAKE_OPTIONS += -DVER=$(COMMIT_NO) + +include $(INCLUDE_DIR)/package.mk +include $(INCLUDE_DIR)/cmake.mk + +define Build/Prepare + mkdir -p $(PKG_BUILD_DIR) + cp -r src/* $(PKG_BUILD_DIR)/ +endef + +define Package/data-exporter + SECTION:=celerway + CATEGORY:=Celerway + DEPENDS:=+libjson-c +libmnl +libsqlite3 +libpthread +libuci +libubus +libubox + TITLE:=Utility for exporting metadata +endef + +TARGET_CFLAGS += -I$(STAGING_DIR)/usr/include +CMAKE_OPTIONS += -DTARGET_OWRT=1 -DSQLITE3=1 -DFILE=1 -DUBUS=1 + +define Package/data-exporter/install + $(INSTALL_DIR) $(1)/bin + $(CP) $(PKG_INSTALL_DIR)/usr/bin/meta_exporter $(1)/bin/ + $(INSTALL_BIN) ./files/metadata_exporter_timeloop.sh $(1)/bin/ + $(INSTALL_DIR) $(1)/etc/init.d + $(INSTALL_BIN) ./files/metadata.init $(1)/etc/init.d/metadata + $(INSTALL_BIN) ./files/metadataloop.init $(1)/etc/init.d/metadataloop + $(INSTALL_DIR) $(1)/etc/config + $(INSTALL_BIN) ./files/metadata.config $(1)/etc/config/metadata +endef + +$(eval $(call BuildPackage,data-exporter)) diff --git a/files/metadata.config b/files/metadata.config new file mode 100644 index 0000000..1d0596d --- /dev/null +++ b/files/metadata.config @@ -0,0 +1,3 @@ +config section metadata + option version '2' + option disabled '0' diff --git a/files/metadata.init b/files/metadata.init new file mode 100755 index 0000000..40bbb79 --- /dev/null +++ b/files/metadata.init @@ -0,0 +1,113 @@ +#!/bin/sh /etc/rc.common + +. /lib/config/uci.sh +. /usr/share/libubox/jshn.sh + +START=96 + +METADATAPATH="/bin" +METADATANAME="meta_exporter" +METADATADB="/tmp/metadata.db" +METADATAPREFIX="/tmp/metatemp" +USAGEPREFIX="/tmp/usage" +GPSPREFIX="/tmp/metadata-gps" +CONF_PATH="/tmp/meta_exporter.conf" +SESSION_ID_PATH="/tmp/bootcnt" +TSTAMP_DUMP_PATH="/tmp/last_conn_tstamp" +USE_PROCD=1 + +create_config() +{ + METADATANODEID=$(uci_get backend inventory id) + API_VERSION=$(uci_get backend inventory api_version) + SQLITE_INTERVAL=$(uci_get backend metadata sqlite_interval) + SQLITE_EVENTS=$(uci_get backend metadata sqlite_events) + EXPORT_GPS=$(uci_get cwy_router_settings metadata export_gps 1) + #The reason we use the default board name here, is that we need to separate between old and new Arcus. The new Arcus + #have a valid identifier stored in the DMI + BOARD_NAME=`cat /tmp/sysinfo/board_name` + + if [ -z "$METADATANODEID" ]; then + METADATANODEID=0 + fi + + if [ -z "$API_VERSION" ]; + then + API_VERSION=2 + fi + + json_init + + #Add zmq options + json_add_object "ubus_input" + json_add_boolean "conn" 1 + + if [ "$EXPORT_GPS" -eq 1 ]; + then + json_add_boolean "pos" 1 + fi + json_close_object + + json_add_object "sqlite" + json_add_string "database" "$METADATADB" + json_add_int "nodeid" "$METADATANODEID" + json_add_string "meta_prefix" "$METADATAPREFIX" + json_add_string "usage_prefix" "$USAGEPREFIX" + + if [ "$EXPORT_GPS" -eq 1 ]; + then + json_add_string "gps_prefix" "$GPSPREFIX" + fi + + json_add_string "session_id" "$SESSION_ID_PATH" + json_add_int "api_version" "$API_VERSION" + + if [ -n "$SQLITE_INTERVAL" ]; + then + json_add_int "interval" "$SQLITE_INTERVAL" + fi + + if [ -n "$SQLITE_EVENTS" ]; + then + json_add_int "events" "$SQLITE_EVENTS" + fi + + json_add_string "last_conn_tstamp_path" "$TSTAMP_DUMP_PATH" + json_add_string "output_format" "json" + json_add_string "ntp_fix_file" "/tmp/ntp-fix-obtained" + + json_close_object + + if [ "$EXPORT_GPS" -eq 1 ]; + then + if [ "$BOARD_NAME" == "celerway-arcus" ]; + then + json_add_object "gpsd" + json_add_string "address" "127.0.0.1" + json_add_string "port" "2947" + json_close_object + fi + fi + + echo "$(json_dump)" > "$CONF_PATH" +} + +start_service() +{ + DISABLED=$(uci_get cwy_router_settings metadata disabled); + + #Unless there is anywhere to store data, then don't run metadata collector. + #The reason for this check, is that we can avoid having to install metadata + #later. All we need to provide is a backend-config + if [ ! -f "/etc/config/backend" -o $DISABLED -eq 1 ]; + then + exit 0; + fi + + create_config + + procd_open_instance + procd_set_param command "$METADATAPATH/$METADATANAME" -c "$CONF_PATH" + procd_set_param respawn 0 5 0 + procd_close_instance +} diff --git a/files/metadata_exporter_timeloop.sh b/files/metadata_exporter_timeloop.sh new file mode 100755 index 0000000..c420a28 --- /dev/null +++ b/files/metadata_exporter_timeloop.sh @@ -0,0 +1,149 @@ +#!/bin/sh + +. /lib/config/uci.sh + +INTVL=30; +SLEEP_TIME=0; +NUM_USAGE_LIMIT=100; +NUM_USAGE_DELETE=; +NUM_METADATA_LIMIT=100; +NUM_METADATA_DELETE=; +NUM_GPS_LIMIT=100; +NUM_GPS_DELETE=; +RSYNC_PATH="/usr/bin/rsync"; +RSYNC_META_LOCAL="/tmp/metatemp*.json"; +RSYNC_USAGE_LOCAL="/tmp/usage*.json"; +RSYNC_GPS_LOCAL="/tmp/metadata-gps*.json"; +RSYNC_LOCAL="/tmp/metatemp*.json /tmp/usage*.json /tmp/metadata-gps*.json"; +RSYNC_SERVER=$(uci_get backend metadata server); +RSYNC_USER="$(uci_get backend metadata user)-cert"; +RSYNC_DST_PORT=$(uci_get backend metadata dst_port); +SSH_KEY_FILE="/tmp/ssh-keys/id_rsa" +SSH_KNOWN_HOSTS="/tmp/ssh-keys/known_hosts" + +RSYNC_REMOTE_PREFIX="$RSYNC_USER@$RSYNC_SERVER:/home/metadata/data"; + +while true; +do + #This $VAR:-text is som fancy Bash parameter expansion. It means that if + #variable is not set, then "return" the string. See: + #http://www.gnu.org/software/bash/manual/bash.html#Shell-Parameter-Expansion + CWY_VER=$(uci_get cwy_router_settings metadata version) + if [ "${CWY_VER:-error_1}" = "error_1" ] + then + echo "CWY_VER must be set"; + logger -t metadata_exporter "CWY_VER must be set"; + sleep $INTVL; + continue; + fi + + CWY_NODEID=$(uci_get backend inventory id) + if [ "${CWY_NODEID:-error_2}" = "error_2" ] + then + echo "CWY_NODEID must be set"; + logger -t metadata_exporter "CWY_NODEID must be set"; + sleep $INTVL; + continue; + fi + + NUM_METADATA_FILES=$(/bin/ls -l $RSYNC_META_LOCAL | wc -l) + if [ "$NUM_METADATA_FILES" -gt "$NUM_METADATA_LIMIT" ]; + then + NUM_METADATA_DELETE=$(( NUM_METADATA_FILES - NUM_METADATA_LIMIT )) + logger -t metadata_exporter "Found $NUM_METADATA_FILES number of metadata files, will remove $NUM_METADATA_DELETE"; + #Delete the X most recent usage files, as they are the least relevant + find $RSYNC_META_LOCAL -maxdepth 1 -exec stat -c " %Y %n" {} \; | sort -n | head -"$NUM_METADATA_DELETE" | cut -d " " -f 3 | xargs -- rm + fi + + NUM_USAGE_FILES=$(/bin/ls -l $RSYNC_USAGE_LOCAL | wc -l) + if [ "$NUM_USAGE_FILES" -gt "$NUM_USAGE_LIMIT" ]; + then + NUM_USAGE_DELETE=$(( NUM_USAGE_FILES - NUM_USAGE_LIMIT )) + logger -t metadata_exporter "Found $NUM_USAGE_FILES number of usage files, will remove $NUM_USAGE_DELETE"; + #Delete the X most recent usage files, as they are the least relevant + find $RSYNC_USAGE_LOCAL -maxdepth 1 -exec stat -c " %Y %n" {} \; | sort -n | head -"$NUM_USAGE_DELETE" | cut -d " " -f 3 | xargs -- rm + fi + + NUM_GPS_FILES=$(/bin/ls -l $RSYNC_GPS_LOCAL | wc -l) + if [ "$NUM_GPS_FILES" -gt "$NUM_GPS_LIMIT" ]; + then + NUM_GPS_DELETE=$(( NUM_GPS_FILES - NUM_GPS_LIMIT )) + logger -t metadata_exporter "Found $NUM_GPS_FILES number of usage files, will remove $NUM_GPS_DELETE"; + #Delete the X most recent usage files, as they are the least relevant + find $RSYNC_GPS_LOCAL -maxdepth 1 -exec stat -c " %Y %n" {} \; | sort -n | head -"$NUM_GPS_DELETE" | cut -d " " -f 3 | xargs -- rm + fi + + #TODO: Consider wrapping in a conditional + #sed -i "s/xxxxxx/$CWY_NODEID/g" $RSYNC_LOCAL + NUM_FILES=$(/bin/ls -l $RSYNC_LOCAL | wc -l) + + if [ "$NUM_FILES" -eq 0 ] + then + sleep 5 + continue + fi + + #rsync deletes files as they are copied (batch), so we don't risk copying (all) + #the same data multiple times if a copy fails + #use -t to preseve timestamp, so that we can sort files correctly on server. + #This is a good enough way to ensure that we get queries in order (we need to do + #INSERT before UPDATE). We might get problems if the same router reboots many + #times and we dont have a proper clock, but we will be saved by update + BEFORE_TIME=$(date '+%s'); + timeout -s KILL 40 "$RSYNC_PATH" -t --remove-source-files --stats --timeout=30 --perms --chmod=Dg=rw,Fg=rwx \ + -e "ssh -p $RSYNC_DST_PORT -i ${SSH_KEY_FILE} -o UserKnownHostsFile=${SSH_KNOWN_HOSTS} -o ConnectTimeout=10 -o PreferredAuthentications=publickey -o PasswordAuthentication=no -o PubkeyAuthentication=yes -S '/tmp/metassh-inv-%r@%h:%p' " \ + $RSYNC_LOCAL "$RSYNC_REMOTE_PREFIX/$CWY_VER/$CWY_NODEID/"; + RETVAL=$? + AFTER_TIME=$(date '+%s'); + + if [ "$RETVAL" -eq 30 ]; + then + logger -t metadata_exporter "metadata exporting timed out, kill SSH connection"; + metassh_pid=$(pgrep -f metassh); + + if [ -n "$metassh_pid" ]; + then + kill -9 "$metassh_pid" + fi + elif [ "$RETVAL" -ne 0 ] && [ "$RETVAL" -ne 23 ] + then + #23 is partial success, which we count as success since it most of the + #time will be because of missing metadata (gps) files + logger -t metadata_exporter "Copying metadata failed with code $RETVAL"; + /etc/init.d/ssh_key_manager restart + + #Kill SSH connection in case of failure too, might be that we end up + #with this weird I/O error that only seems to be resolved by re-establishing + #connection + metassh_pid=$(pgrep -f metassh); + if [ -n "$metassh_pid" ]; + then + kill -9 "$metassh_pid" + fi + else + logger -t metadata_exporter "Done copying files to server"; + date +%s > /tmp/last_meta_export + fi + + #Time can move backwards. Not very likely, but handle it in case of failure + #+ ntp happening at the same time + if [ "$AFTER_TIME" -lt "$BEFORE_TIME" ] + then + SLEEP_TIME="$INTVL"; + else + TDIFF=$(( AFTER_TIME - BEFORE_TIME )); + + #Make sure we don't end up in inifinte loop by always sleeping a minimum + #amount of time + if [ "$TDIFF" -ge "$INTVL" ] + then + SLEEP_TIME=1; + else + SLEEP_TIME=$(( INTVL - TDIFF )); + fi + fi + + sleep "$SLEEP_TIME"; + +done; +exit 0; diff --git a/files/metadataloop.init b/files/metadataloop.init new file mode 100755 index 0000000..5b57ea7 --- /dev/null +++ b/files/metadataloop.init @@ -0,0 +1,27 @@ +#!/bin/sh /etc/rc.common + +. /lib/config/uci.sh + +START=96 +USE_PROCD=1 + +METADATALOOPPATH=/bin +METADATALOOPNAME=metadata_exporter_timeloop.sh + +start_service() +{ + DISABLED=$(uci_get cwy_router_settings metadata disabled); + + #Unless there is anywhere to store data, then don't run metadata collector. + #The reason for this check, is that we can avoid having to install metadata + #later. All we need to provide is a backend-config + if [ ! -f "/etc/config/backend" -o $DISABLED -eq 1 ]; + then + exit 0; + fi + + procd_open_instance + procd_set_param command "$METADATALOOPPATH/$METADATALOOPNAME" + procd_set_param respawn 0 5 0 + procd_close_instance +} diff --git a/CMakeLists.txt b/src/CMakeLists.txt similarity index 94% rename from CMakeLists.txt rename to src/CMakeLists.txt index 6c500f4..8da5c36 100644 --- a/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -10,7 +10,6 @@ set(SOURCE backend_event_loop.c metadata_utils.c system_helpers.c - metadata_input_nl_zmq_common.c lib/minmea.c) @@ -74,7 +73,8 @@ endif() if (ZEROMQ_INPUT) set(LIBS ${LIBS} zmq) set(SOURCE ${SOURCE} - metadata_input_zeromq.c) + metadata_input_zeromq.c + metadata_input_nl_zmq_common.c) add_definitions("-DZEROMQ_SUPPORT_INPUT") endif() @@ -110,6 +110,13 @@ if (GPS_NSB) add_definitions("-DGPS_NSB_SUPPORT") endif() +if (UBUS) + set(LIBS ${LIBS} ubus ubox) + set(SOURCE ${SOURCE} + metadata_input_ubus.c) + add_definitions("-DUBUS_INPUT_SUPPORT") +endif() + set(CPACK_GENERATOR "DEB") set(CPACK_PACKAGE_VERSION_MAJOR "0") set(CPACK_PACKAGE_VERSION_MINOR "1") diff --git a/backend_event_loop.c b/src/backend_event_loop.c similarity index 100% rename from backend_event_loop.c rename to src/backend_event_loop.c diff --git a/backend_event_loop.h b/src/backend_event_loop.h similarity index 95% rename from backend_event_loop.h rename to src/backend_event_loop.h index b211587..6470c4c 100644 --- a/backend_event_loop.h +++ b/src/backend_event_loop.h @@ -30,6 +30,11 @@ #include #include +#define SQ_LIST_HEAD(name, type) \ +struct name { \ + struct type *lh_first; /* first element */ \ +} + #define MAX_EPOLL_EVENTS 10 //Any resource used by the callback is stored in the implementing "class". @@ -58,7 +63,7 @@ struct backend_timeout_handle{ struct backend_event_loop{ int32_t efd; - LIST_HEAD(timeout, backend_timeout_handle) timeout_list; + SQ_LIST_HEAD(timeout, backend_timeout_handle) timeout_list; backend_itr_cb itr_cb; void *itr_data; }; diff --git a/lib/minmea.c b/src/lib/minmea.c similarity index 100% rename from lib/minmea.c rename to src/lib/minmea.c diff --git a/lib/minmea.h b/src/lib/minmea.h similarity index 100% rename from lib/minmea.h rename to src/lib/minmea.h diff --git a/metadata_exporter.c b/src/metadata_exporter.c similarity index 94% rename from metadata_exporter.c rename to src/metadata_exporter.c index c870d3a..4dbf0d5 100644 --- a/metadata_exporter.c +++ b/src/metadata_exporter.c @@ -72,10 +72,12 @@ #ifdef NEAT_SUPPORT #include "metadata_writer_neat.h" #endif - #ifdef FILE_SUPPORT #include "metadata_writer_file.h" #endif +#ifdef UBUS_INPUT_SUPPORT + #include "metadata_input_ubus.h" +#endif #include "backend_event_loop.h" #include "metadata_exporter_log.h" @@ -138,12 +140,12 @@ void mde_start_timer(struct backend_event_loop *event_loop, } //TODO: Find a place to catch termination signals, and call this. -void mde_destroy(struct md_exporter *mde) +void mde_destroy(struct md_exporter *mde) { int i; - for (i=0; i<=MD_INPUT_MAX; i++) - if (mde->md_inputs[i] != NULL) - if (mde->md_inputs[i]->destroy != NULL) + for (i=0; i<=MD_INPUT_MAX; i++) + if (mde->md_inputs[i] != NULL) + if (mde->md_inputs[i]->destroy != NULL) mde->md_inputs[i]->destroy(mde->md_inputs[i]); } @@ -195,6 +197,9 @@ static void print_usage() #endif #ifdef SYSEVENT_SUPPORT md_sysevent_usage(); +#endif +#ifdef UBUS_INPUT_SUPPORT + md_ubus_input_usage(); #endif fprintf(stderr, "WRITERS:\n"); #ifdef NEAT_SUPPORT @@ -219,7 +224,7 @@ void read_config(char* config_file, json_object** config_obj) FILE* config = fopen(config_file, "r"); - if (config == NULL) { + if (config == NULL) { fprintf(stderr, "Could not open configuration file.\n"); exit(EXIT_FAILURE); } @@ -269,7 +274,7 @@ int main(int argc, char *argv[]) break; } else if (i == 'c') { read_config(optarg, &config); - } else if (i == 'h') { + } else if (i == 'h') { print_usage(); exit(EXIT_SUCCESS); } @@ -282,7 +287,7 @@ int main(int argc, char *argv[]) json_object_object_foreach(config, key, val) { if (!strcmp(key, "logfile")) { - logfile_path = json_object_get_string(val); + logfile_path = json_object_get_string(val); } else if (!strcmp(key, "syslog")) { mde->use_syslog = json_object_get_int(val); } @@ -325,6 +330,19 @@ int main(int argc, char *argv[]) num_inputs++; } #endif +#ifdef UBUS_INPUT_SUPPORT + else if (!strcmp(key, "ubus_input")) { + mde->md_inputs[MD_INPUT_UBUS] = calloc(sizeof(struct md_input_ubus), 1); + + if (mde->md_inputs[MD_INPUT_UBUS] == NULL) { + META_PRINT_SYSLOG(mde, LOG_ERR, "Could not allocate UBUS input\n"); + exit(EXIT_FAILURE); + } + + md_ubus_input_setup(mde, (struct md_input_ubus*) mde->md_inputs[MD_INPUT_UBUS]); + num_inputs++; + } +#endif #ifdef NNE_SUPPORT else if (!strcmp(key, "nne")) { mde->md_writers[MD_WRITER_NNE] = calloc(sizeof(struct md_writer_nne), 1); @@ -388,8 +406,8 @@ int main(int argc, char *argv[]) md_sysevent_setup(mde, (struct md_input_sysevent*) mde->md_inputs[MD_INPUT_SYSEVENT]); num_inputs++; - } -#endif + } +#endif #ifdef SQLITE_SUPPORT else if (!strcmp(key, "sqlite")) { mde->md_writers[MD_WRITER_SQLITE] = calloc(sizeof(struct md_writer_sqlite), 1); diff --git a/metadata_exporter.h b/src/metadata_exporter.h similarity index 99% rename from metadata_exporter.h rename to src/metadata_exporter.h index 4226fdb..ac333c6 100644 --- a/metadata_exporter.h +++ b/src/metadata_exporter.h @@ -138,6 +138,7 @@ enum md_inputs { MD_INPUT_SYSEVENT, MD_INPUT_ZEROMQ, MD_INPUT_ZEROMQ_RELAY, + MD_INPUT_UBUS, __MD_INPUT_MAX }; diff --git a/metadata_exporter_log.h b/src/metadata_exporter_log.h similarity index 100% rename from metadata_exporter_log.h rename to src/metadata_exporter_log.h diff --git a/metadata_input_gps_nsb.c b/src/metadata_input_gps_nsb.c similarity index 100% rename from metadata_input_gps_nsb.c rename to src/metadata_input_gps_nsb.c diff --git a/metadata_input_gps_nsb.h b/src/metadata_input_gps_nsb.h similarity index 100% rename from metadata_input_gps_nsb.h rename to src/metadata_input_gps_nsb.h diff --git a/metadata_input_gpsd.c b/src/metadata_input_gpsd.c similarity index 100% rename from metadata_input_gpsd.c rename to src/metadata_input_gpsd.c diff --git a/metadata_input_gpsd.h b/src/metadata_input_gpsd.h similarity index 100% rename from metadata_input_gpsd.h rename to src/metadata_input_gpsd.h diff --git a/metadata_input_iface_test.h b/src/metadata_input_iface_test.h similarity index 100% rename from metadata_input_iface_test.h rename to src/metadata_input_iface_test.h diff --git a/metadata_input_munin.c b/src/metadata_input_munin.c similarity index 100% rename from metadata_input_munin.c rename to src/metadata_input_munin.c diff --git a/metadata_input_munin.h b/src/metadata_input_munin.h similarity index 100% rename from metadata_input_munin.h rename to src/metadata_input_munin.h diff --git a/metadata_input_nl_zmq_common.c b/src/metadata_input_nl_zmq_common.c similarity index 100% rename from metadata_input_nl_zmq_common.c rename to src/metadata_input_nl_zmq_common.c diff --git a/metadata_input_nl_zmq_common.h b/src/metadata_input_nl_zmq_common.h similarity index 100% rename from metadata_input_nl_zmq_common.h rename to src/metadata_input_nl_zmq_common.h diff --git a/metadata_input_sysevent.c b/src/metadata_input_sysevent.c similarity index 100% rename from metadata_input_sysevent.c rename to src/metadata_input_sysevent.c diff --git a/metadata_input_sysevent.h b/src/metadata_input_sysevent.h similarity index 100% rename from metadata_input_sysevent.h rename to src/metadata_input_sysevent.h diff --git a/src/metadata_input_ubus.c b/src/metadata_input_ubus.c new file mode 100644 index 0000000..7cb4bab --- /dev/null +++ b/src/metadata_input_ubus.c @@ -0,0 +1,507 @@ +/* Copyright (c) 2021, Celerway, Lukasz Baj + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include + +#include "backend_event_loop.h" +#include "metadata_exporter_log.h" +#include "metadata_input_ubus.h" + +struct md_input_ubus *miu; + +static uint8_t md_input_ubus_subscribe(struct ubus_subscriber *subscriber, const char *obj_name); + + // Policy for the object add event +enum +{ + PATH, + __UBUS_OBJ_ADD_MAX +}; + +static struct blobmsg_policy ubus_obj_add_policy[__UBUS_OBJ_ADD_MAX] = { + [PATH] = {.name = "path", .type = BLOBMSG_TYPE_STRING} +}; + +// Metaupdate object policy +enum +{ + TIMESTAMP = 0, + EVENT_TYPE, + INTERFACE_ID_TYPE, + INTERFACE_ID, + INTERFACE_NAME, + IMEI, + IMSI, + INTERFACE_TYPE, + NETWORK_ADDRESS_FAMILY, + NETWORK_ADDRESS, + NETWORK_PROVIDER, + EVENT_PARAM, + EVENT_VALUE, + L3_SESSION_ID, + L4_SESSION_ID, + SIGNAL_STRENGTH, + RX_BYTES, + TX_BYTES, + HAS_IP, + CONNECTIVITY, + CONNECTION_MODE, + QUALITY, + __METADATA_MAX +}; + +static struct blobmsg_policy metadata_policy[__METADATA_MAX] = { + [TIMESTAMP] = {.name = "timestamp", .type = BLOBMSG_TYPE_INT64}, + [EVENT_TYPE] = {.name = "event_type", .type = BLOBMSG_TYPE_INT8}, + [INTERFACE_ID_TYPE] = {.name = "interface_id_type", .type = BLOBMSG_TYPE_INT8}, + [INTERFACE_ID] = {.name = "interface_id", .type = BLOBMSG_TYPE_STRING}, + [INTERFACE_NAME] = {.name = "interface_name", .type = BLOBMSG_TYPE_STRING}, + [IMEI] = {.name = "imei", .type = BLOBMSG_TYPE_STRING}, + [IMSI] = {.name = "imsi", .type = BLOBMSG_TYPE_STRING}, + [INTERFACE_TYPE] = {.name = "interface_type", .type = BLOBMSG_TYPE_INT8}, + [NETWORK_ADDRESS_FAMILY] = {.name = "network_address_family", .type = BLOBMSG_TYPE_INT8}, + [NETWORK_ADDRESS] = {.name = "network_address", .type = BLOBMSG_TYPE_STRING}, + [NETWORK_PROVIDER] = {.name = "network_provider", .type = BLOBMSG_TYPE_INT32}, + [EVENT_PARAM] = {.name = "event_param", .type = BLOBMSG_TYPE_INT8}, + [EVENT_VALUE] = {.name = "event_value", .type = BLOBMSG_TYPE_INT32}, + [L3_SESSION_ID] = {.name = "l3_session_id", .type = BLOBMSG_TYPE_INT64}, + [L4_SESSION_ID] = {.name = "l4_session_id", .type = BLOBMSG_TYPE_INT64}, + [SIGNAL_STRENGTH] = {.name = "signal_strength", .type = BLOBMSG_TYPE_INT8}, + [RX_BYTES] = {.name = "rx_bytes", .type = BLOBMSG_TYPE_INT64}, + [TX_BYTES] = {.name = "tx_bytes", .type = BLOBMSG_TYPE_INT64}, + [HAS_IP] = {.name = "has_ip", .type = BLOBMSG_TYPE_INT8}, + [CONNECTIVITY] = {.name = "connectivity", .type = BLOBMSG_TYPE_INT8}, + [CONNECTION_MODE] = {.name = "connection_mode", .type = BLOBMSG_TYPE_INT8}, + [QUALITY] = {.name = "quality", .type = BLOBMSG_TYPE_INT16}, +}; + +enum +{ + LATITUDE, + LONGITUDE, + ELEVATION, + SPEED, + SATELLITES, + __GPS_UPDATE_MAX +}; + +static struct blobmsg_policy gps_update_policy [__GPS_UPDATE_MAX] = { + [LATITUDE] = {.name = "latitude", .type = BLOBMSG_TYPE_STRING}, + [LONGITUDE] = {.name = "longitude", .type = BLOBMSG_TYPE_STRING}, + [ELEVATION] = {.name = "elevation", .type = BLOBMSG_TYPE_STRING}, + [SPEED] = {.name = "speed", .type = BLOBMSG_TYPE_STRING}, + [SATELLITES] = {.name = "satellites", .type = BLOBMSG_TYPE_STRING} +}; + +static void md_input_ubus_handle_conn_event(struct blob_attr *tb[]) +{ + struct md_conn_event *mce = miu->mce; + + memset(mce, 0, sizeof(struct md_conn_event)); + + mce->md_type = META_TYPE_CONNECTION; + //255 is reserved value used to indicate that there is no value to export + //(look at writers) + mce->event_value = UINT8_MAX; + + //We are inserting a sequence number. Sequence is so that we can + //see the order in which events arrived at the + //metadata exporter, making it easier to correlate events between + //applications. The different applications publishing data might also insert + //their own sequence number + mce->sequence = mde_inc_seq(miu->parent); + + if (tb[TIMESTAMP]) { + mce->tstamp = blobmsg_get_u64(tb[TIMESTAMP]); + } + + if (tb[EVENT_PARAM]) { + mce->event_param = blobmsg_get_u8(tb[EVENT_PARAM]); + } + + if (tb[EVENT_VALUE]) { + mce->event_value = (uint8_t)blobmsg_get_u32(tb[EVENT_VALUE]); + } + + if (tb[EVENT_TYPE]) { + mce->event_type = blobmsg_get_u8(tb[EVENT_TYPE]); + } + + if (tb[INTERFACE_ID_TYPE]) { + mce->interface_id_type = blobmsg_get_u8(tb[INTERFACE_ID_TYPE]); + } + + if (tb[INTERFACE_ID]) { + mce->interface_id = blobmsg_get_string(tb[INTERFACE_ID]); + } + + if (tb[IMEI]) { + mce->imei = blobmsg_get_string(tb[IMEI]); + } + + if (tb[IMSI]) { + mce->imsi = blobmsg_get_string(tb[IMSI]); + } + + if (tb[INTERFACE_NAME]) { + mce->interface_name = blobmsg_get_string(tb[INTERFACE_NAME]); + } + + if (tb[INTERFACE_TYPE]) { + mce->interface_type = blobmsg_get_u8(tb[INTERFACE_TYPE]); + } + + if (tb[NETWORK_ADDRESS_FAMILY]) { + mce->network_address_family = blobmsg_get_u8(tb[NETWORK_ADDRESS_FAMILY]); + } + + if (tb[NETWORK_ADDRESS]) { + mce->network_address = blobmsg_get_string(tb[NETWORK_ADDRESS]); + } + + if (tb[NETWORK_PROVIDER]) { + mce->network_provider = blobmsg_get_u32(tb[NETWORK_PROVIDER]); + } + + if (tb[L3_SESSION_ID]) { + mce->l3_session_id = blobmsg_get_u64(tb[L3_SESSION_ID]); + } + + if (tb[L4_SESSION_ID]) { + mce->l4_session_id = blobmsg_get_u64(tb[L4_SESSION_ID]); + } + + if (tb[SIGNAL_STRENGTH]) { + mce->signal_strength = (int8_t) blobmsg_get_u8(tb[SIGNAL_STRENGTH]); + } + + if (tb[RX_BYTES]) { + mce->rx_bytes = blobmsg_get_u64(tb[RX_BYTES]); + } + + if (tb[TX_BYTES]) { + mce->tx_bytes = blobmsg_get_u64(tb[TX_BYTES]); + } + + if (tb[HAS_IP]) { + mce->has_ip = (int8_t) blobmsg_get_u8(tb[HAS_IP]); + } + + if (tb[CONNECTIVITY]) { + mce->connectivity = (int8_t) blobmsg_get_u8(tb[CONNECTIVITY]); + } + + if (tb[CONNECTION_MODE]) { + mce->connection_mode = (int8_t) blobmsg_get_u8(tb[CONNECTION_MODE]); + } + + if (tb[QUALITY]) { + mce->quality = (uint8_t) blobmsg_get_u16(tb[QUALITY]); + } + + if (mce->event_param == CONN_EVENT_DATA_USAGE_UPDATE) { + if (!mce->tstamp || !mce->event_param || !mce->interface_id || (mce->imei && !mce->imsi) || + (mce->imsi && !mce->imei)) { + META_PRINT_SYSLOG(miu->parent, LOG_ERR, "Missing required argument in usage UBUS message\n"); + return; + } else { + mde_publish_event_obj(miu->parent, (struct md_event*) mce); + return; + } + } + + if (!mce->tstamp || !mce->sequence || + !mce->l3_session_id || !mce->event_param || + !mce->interface_type || !mce->network_address_family || + !mce->network_address || !mce->interface_id || + !mce->interface_id_type) { + META_PRINT_SYSLOG(miu->parent, LOG_ERR, "Missing required argument in UBUS message\n"); + return; + } + + mde_publish_event_obj(miu->parent, (struct md_event*) mce); +} + +static int md_input_ubus_dlb_subscriber_cb(struct ubus_context *ctx, struct ubus_object *obj, struct ubus_request_data *req, + const char *method, struct blob_attr *msg) +{ + struct blob_attr* tb[__METADATA_MAX] = {0}; + uint8_t event_type = 0; + + // We parse metadata and datausage messages only + META_PRINT_SYSLOG(miu->parent, LOG_INFO, "Got ubus method: %s\n", method); + if (strcmp(method, "metadata") && strcmp(method, "datausage")) { + return UBUS_STATUS_OK; + } + + if (blobmsg_parse(metadata_policy, __METADATA_MAX, tb, blobmsg_data(msg), blobmsg_data_len(msg))) { + META_PRINT_SYSLOG(miu->parent, LOG_ERR, "Can't parse %s message\n", method); + return UBUS_STATUS_NO_DATA; + } + + if (!tb[EVENT_TYPE]) { + META_PRINT_SYSLOG(miu->parent, LOG_ERR, "Event type is missing\n"); + return UBUS_STATUS_OK; + } + + event_type = blobmsg_get_u8(tb[EVENT_TYPE]); + if (!(event_type & miu->md_ubus_mask)) { + return UBUS_STATUS_OK; + } + + switch (event_type) + { + case META_TYPE_CONNECTION: + md_input_ubus_handle_conn_event(tb); + break; + + default: + META_PRINT_SYSLOG(miu->parent, LOG_ERR, "Unknown event type\n"); + break; + } + + return UBUS_STATUS_OK; +} + +static int md_input_ubus_gps_subscriber_cb(struct ubus_context *ctx, struct ubus_object *obj, struct ubus_request_data *req, + const char *method, struct blob_attr *msg) +{ + struct blob_attr* tb[__GPS_UPDATE_MAX] = {0}; + struct md_gps_event *mgps = miu->gps_event; + + if (blobmsg_parse(gps_update_policy, __GPS_UPDATE_MAX, tb, blobmsg_data(msg), blobmsg_data_len(msg))) { + META_PRINT_SYSLOG(miu->parent, LOG_ERR, "Can't parse GPS data\n"); + return UBUS_STATUS_NO_DATA; + } + + if (!tb[LONGITUDE] || !tb[LATITUDE] || !tb[SATELLITES]) { + return UBUS_STATUS_NO_DATA; + } + + memset(mgps, 0, sizeof(struct md_gps_event)); + + gettimeofday(&mgps->tstamp_tv, NULL); + mgps->md_type = META_TYPE_POS; + mgps->minmea_id = MINMEA_UNKNOWN; + mgps->sequence = mde_inc_seq(miu->parent); + + if (tb[LONGITUDE]) { + mgps->longitude = atof(blobmsg_get_string(tb[LONGITUDE])); + } + + if (tb[LATITUDE]) { + mgps->latitude = atof(blobmsg_get_string(tb[LATITUDE])); + } + + if (tb[SATELLITES]) { + mgps->satellites_tracked = atoi(blobmsg_get_string(tb[SATELLITES])); + } + + if (tb[ELEVATION]) { + mgps->altitude = atof(blobmsg_get_string(tb[ELEVATION])); + } + + if (tb[SPEED]) { + mgps->speed = atof(blobmsg_get_string(tb[SPEED])); + } + + mde_publish_event_obj(miu->parent, (struct md_event *) mgps); + + return UBUS_STATUS_OK; +} + +static void md_input_ubus_event_handler(struct ubus_context *ctx, struct ubus_event_handler *ev, const char *type, struct blob_attr *msg) +{ + struct blob_attr* tb[__UBUS_OBJ_ADD_MAX] = {0}; + + if (blobmsg_parse(ubus_obj_add_policy, __UBUS_OBJ_ADD_MAX, tb, blobmsg_data(msg), blobmsg_data_len(msg))) { + META_PRINT_SYSLOG(miu->parent, LOG_ERR, "Can't parse ubus.object.add event\n"); + return; + } + + if (!tb[PATH]) { + META_PRINT_SYSLOG(miu->parent, LOG_ERR, "No path in the object\n"); + return; + } + + // Resubscribe to the UBUS object if needed + if (!strcmp(blobmsg_get_string(tb[PATH]), "celerway.dlb")) { + md_input_ubus_subscribe(&miu->ubus_dlb_subscriber, "celerway.dlb"); + } else if (!strcmp(blobmsg_get_string(tb[PATH]), "gps")) { + md_input_ubus_subscribe(&miu->ubus_gps_subscriber, "gps"); + } +} + +static void md_input_ubus_handle_event(void *ptr, int32_t fd, uint32_t events) +{ + struct md_input_ubus *miu = (struct md_input_ubus *)ptr; + + ubus_handle_event(miu->ubus_ctx); +} + +static uint8_t md_input_ubus_connect() +{ + struct sigaction sig_action_int, sig_action_term; + + //save old sighandlers + if (sigaction(SIGINT, NULL, &sig_action_int) == -1 || sigaction(SIGTERM, NULL, &sig_action_term) == -1) { + META_PRINT_SYSLOG(miu->parent, LOG_ERR, "Could not replace signal handlers\n"); + return RETVAL_FAILURE; + } + + if (!(miu->ubus_ctx = ubus_connect(NULL))) { + META_PRINT_SYSLOG(miu->parent, LOG_ERR, "Could not connect to ubus\n"); + return 0; + } + + if (fcntl(miu->ubus_ctx->sock.fd, F_SETFD, FD_CLOEXEC) == -1) { + META_PRINT_SYSLOG(miu->parent, LOG_ERR, "Could not set FD_CLOEXEC to ubus fd\n"); + return RETVAL_FAILURE; + } + + //restore original sighandlers + if (sigaction(SIGINT, &sig_action_int, NULL) == -1 || sigaction(SIGTERM, &sig_action_term, NULL) == -1) { + META_PRINT_SYSLOG(miu->parent, LOG_ERR, "Could not replace signal handlers\n"); + return RETVAL_FAILURE; + } + + META_PRINT_SYSLOG(miu->parent, LOG_INFO, "Ubus connection has been initialized: %d\n", miu->ubus_ctx->sock.fd); + return RETVAL_SUCCESS; +} + +static uint8_t md_input_ubus_subscribe(struct ubus_subscriber *subscriber, const char *obj_name) +{ + uint32_t id; + + if (ubus_lookup_id(miu->ubus_ctx, obj_name, &id)) { + return RETVAL_FAILURE; + } + + // Only call register once per object + if (!subscriber->obj.id && ubus_register_subscriber(miu->ubus_ctx, subscriber)) { + return RETVAL_FAILURE; + } + + if (ubus_subscribe(miu->ubus_ctx, subscriber, id)) { + return RETVAL_FAILURE; + } + + return RETVAL_SUCCESS; +} + +static uint8_t md_input_ubus_register_event_handler() +{ + if (ubus_register_event_handler(miu->ubus_ctx, &miu->ubus_event_handler, "ubus.object.add")) { + return RETVAL_FAILURE; + } + + return RETVAL_SUCCESS; +} + +static uint8_t md_input_ubus_config() +{ + if (md_input_ubus_connect() != RETVAL_SUCCESS) { + return RETVAL_FAILURE; + } + + if(!(miu->event_handle = backend_create_epoll_handle(miu, miu->ubus_ctx->sock.fd, md_input_ubus_handle_event))) { + return RETVAL_FAILURE; + } + + backend_event_loop_update(miu->parent->event_loop, EPOLLIN, EPOLL_CTL_ADD, miu->ubus_ctx->sock.fd, miu->event_handle); + + memset(&miu->ubus_event_handler, 0, sizeof(struct ubus_event_handler)); + miu->ubus_event_handler.cb = md_input_ubus_event_handler; + if (md_input_ubus_register_event_handler() != RETVAL_SUCCESS) { + META_PRINT_SYSLOG(miu->parent, LOG_INFO, "Failed to register UBUS event handler\n"); + return RETVAL_FAILURE; + } + + if (miu->md_ubus_mask & META_TYPE_CONNECTION) { + memset(&miu->ubus_dlb_subscriber, 0, sizeof(struct ubus_subscriber)); + miu->ubus_dlb_subscriber.cb = md_input_ubus_dlb_subscriber_cb; + if (md_input_ubus_subscribe(&miu->ubus_dlb_subscriber, "celerway.dlb") != RETVAL_SUCCESS) { + META_PRINT_SYSLOG(miu->parent, LOG_ERR, "Failed to subscribe to the celerway.dlb object. Will retry on UBUS add event.\n"); + } + } + + if (miu->md_ubus_mask & META_TYPE_POS) { + memset(&miu->ubus_gps_subscriber, 0, sizeof(struct ubus_subscriber)); + miu->ubus_gps_subscriber.cb = md_input_ubus_gps_subscriber_cb; + if (md_input_ubus_subscribe(&miu->ubus_gps_subscriber, "gps") != RETVAL_SUCCESS) { + META_PRINT_SYSLOG(miu->parent, LOG_ERR, "Failed to subscribe to the gps object. Will retry on UBUS add event.\n"); + } + } + + return RETVAL_SUCCESS; +} + +static uint8_t md_input_ubus_init(void *ptr, json_object* config) +{ + struct md_input_ubus *miu = ptr; + miu->md_ubus_mask = 0; + + json_object* subconfig; + if (json_object_object_get_ex(config, "ubus_input", &subconfig)) { + json_object_object_foreach(subconfig, key, val) { + if (!strcmp(key, "conn")) { + miu->md_ubus_mask |= META_TYPE_CONNECTION; + miu->mce = calloc(sizeof(struct md_conn_event), 1); + if (miu->mce == NULL) { + META_PRINT_SYSLOG(miu->parent, LOG_ERR, "Can't allocate conn event struct\n"); + return RETVAL_FAILURE; + } + } else if (!strcmp(key, "pos")) { + miu->md_ubus_mask |= META_TYPE_POS; + miu->gps_event = calloc(sizeof(struct md_gps_event), 1); + if (miu->gps_event == NULL) { + META_PRINT_SYSLOG(miu->parent, LOG_ERR, "Can't allocate gps event struct\n"); + return RETVAL_FAILURE; + } + } + } + } + + if (!miu->md_ubus_mask) { + META_PRINT_SYSLOG(miu->parent, LOG_ERR, "At least one UBUS event type must be present\n"); + return RETVAL_FAILURE; + } + + return md_input_ubus_config(miu); +} + +void md_ubus_input_usage() +{ + +} + +void md_ubus_input_setup(struct md_exporter *mde, struct md_input_ubus *input) +{ + miu = input; + miu->parent = mde; + miu->init = md_input_ubus_init; +} \ No newline at end of file diff --git a/src/metadata_input_ubus.h b/src/metadata_input_ubus.h new file mode 100644 index 0000000..1f8035c --- /dev/null +++ b/src/metadata_input_ubus.h @@ -0,0 +1,51 @@ +/* Copyright (c) 2021, Celerway, Lukasz Baj + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef METADATA_INPUT_UBUS +#define METADATA_INPUT_UBUS + +#include + +#include "metadata_exporter.h" + +struct backend_epoll_handle; + +struct md_input_ubus { + MD_INPUT; + struct backend_epoll_handle *event_handle; + uint32_t md_ubus_mask; + struct ubus_context *ubus_ctx; + struct ubus_subscriber ubus_dlb_subscriber; + struct ubus_subscriber ubus_gps_subscriber; + struct ubus_event_handler ubus_event_handler; + struct md_conn_event *mce; + struct md_gps_event *gps_event; +}; + +void md_ubus_input_usage(); +void md_ubus_input_setup(struct md_exporter *mde, struct md_input_ubus *input); + +#endif \ No newline at end of file diff --git a/metadata_input_zeromq.c b/src/metadata_input_zeromq.c similarity index 100% rename from metadata_input_zeromq.c rename to src/metadata_input_zeromq.c diff --git a/metadata_input_zeromq.h b/src/metadata_input_zeromq.h similarity index 100% rename from metadata_input_zeromq.h rename to src/metadata_input_zeromq.h diff --git a/metadata_input_zeromq_relay.c b/src/metadata_input_zeromq_relay.c similarity index 100% rename from metadata_input_zeromq_relay.c rename to src/metadata_input_zeromq_relay.c diff --git a/metadata_input_zeromq_relay.h b/src/metadata_input_zeromq_relay.h similarity index 100% rename from metadata_input_zeromq_relay.h rename to src/metadata_input_zeromq_relay.h diff --git a/metadata_utils.c b/src/metadata_utils.c similarity index 100% rename from metadata_utils.c rename to src/metadata_utils.c diff --git a/metadata_utils.h b/src/metadata_utils.h similarity index 100% rename from metadata_utils.h rename to src/metadata_utils.h diff --git a/metadata_writer_file.c b/src/metadata_writer_file.c similarity index 100% rename from metadata_writer_file.c rename to src/metadata_writer_file.c diff --git a/metadata_writer_file.h b/src/metadata_writer_file.h similarity index 100% rename from metadata_writer_file.h rename to src/metadata_writer_file.h diff --git a/metadata_writer_inventory_conn.c b/src/metadata_writer_inventory_conn.c similarity index 100% rename from metadata_writer_inventory_conn.c rename to src/metadata_writer_inventory_conn.c diff --git a/metadata_writer_inventory_conn.h b/src/metadata_writer_inventory_conn.h similarity index 100% rename from metadata_writer_inventory_conn.h rename to src/metadata_writer_inventory_conn.h diff --git a/metadata_writer_inventory_gps.c b/src/metadata_writer_inventory_gps.c similarity index 100% rename from metadata_writer_inventory_gps.c rename to src/metadata_writer_inventory_gps.c diff --git a/metadata_writer_inventory_gps.h b/src/metadata_writer_inventory_gps.h similarity index 100% rename from metadata_writer_inventory_gps.h rename to src/metadata_writer_inventory_gps.h diff --git a/metadata_writer_inventory_system.c b/src/metadata_writer_inventory_system.c similarity index 100% rename from metadata_writer_inventory_system.c rename to src/metadata_writer_inventory_system.c diff --git a/metadata_writer_inventory_system.h b/src/metadata_writer_inventory_system.h similarity index 100% rename from metadata_writer_inventory_system.h rename to src/metadata_writer_inventory_system.h diff --git a/metadata_writer_json_helpers.c b/src/metadata_writer_json_helpers.c similarity index 100% rename from metadata_writer_json_helpers.c rename to src/metadata_writer_json_helpers.c diff --git a/metadata_writer_json_helpers.h b/src/metadata_writer_json_helpers.h similarity index 100% rename from metadata_writer_json_helpers.h rename to src/metadata_writer_json_helpers.h diff --git a/metadata_writer_neat.c b/src/metadata_writer_neat.c similarity index 100% rename from metadata_writer_neat.c rename to src/metadata_writer_neat.c diff --git a/metadata_writer_neat.h b/src/metadata_writer_neat.h similarity index 100% rename from metadata_writer_neat.h rename to src/metadata_writer_neat.h diff --git a/metadata_writer_nne.c b/src/metadata_writer_nne.c similarity index 95% rename from metadata_writer_nne.c rename to src/metadata_writer_nne.c index d337f57..6cf2247 100644 --- a/metadata_writer_nne.c +++ b/src/metadata_writer_nne.c @@ -271,18 +271,61 @@ static struct nne_value md_iface_parse_oper(struct nne_modem *modem, struct md_i return value; } +// quick and dirty storage for IP address entries from CONN_EVENTS +typedef struct { + char iface[100]; + char ip_addr[100]; +} ip_addr_cache_entry; + +static ip_addr_cache_entry ip_lookup[16]; + +static void insert_ip_cache(const char* iface, const char* ip_addr) +{ + int i; + for (i = 0; i < 16; i++) { + if ((ip_lookup[i].iface[0] == '\0') || (strncmp(ip_lookup[i].iface, iface, 100) == 0)) { + strncpy(ip_lookup[i].iface, iface, 100); + strncpy(ip_lookup[i].ip_addr, ip_addr, 100); + break; + } + } +} +static char* get_ip_cache(const char* iface) +{ + int i; + for (i = 0; i < 16; i++) { + if (strncmp(ip_lookup[i].iface, iface, 100) == 0) { + return ip_lookup[i].ip_addr; + } + } + return NULL; +} +static void delete_ip_cache(const char* iface) +{ + int i; + for (i = 0; i < 16; i++) { + if (strncmp(ip_lookup[i].iface, iface, 100) == 0) { + ip_lookup[i].iface[0] = '\0'; + ip_lookup[i].ip_addr[0] = '\0'; + break; + } + } +} + static struct nne_value md_iface_parse_ipaddr(struct nne_modem *modem, struct md_iface_event *mie, char *extra, size_t extra_len) { struct nne_value value; value.type = NNE_TYPE_STRING; - if (mie->ip_addr != NULL) { + char* ip_addr = get_ip_cache(mie->iccid); + + if (ip_addr != NULL) { value.u.v_str = strdup("UP"); // We need to pass ip address to the event message in the extra field; // the problem is, though, that it must be the address as seen inside the // experiment container, so we construct the address here the same way // as nne-lxc-network-manager: 192.168.. // In the current implementation we have only one container with id 1. - snprintf(extra, extra_len, "192.168.%d.101", modem->network_id); + snprintf(extra, extra_len, "%s", ip_addr); } else { value.u.v_str = strdup("DOWN"); @@ -771,6 +814,25 @@ static uint32_t md_find_network_id(uint32_t imsi_mccmnc, const char *iccid, cons return network_id; } + +static void md_nne_handle_conn_event(struct md_writer_nne *mwn, + struct md_conn_event *mce) { + if (mce->event_param != CONN_EVENT_META_UPDATE) { + return; + } + //only parse ifname and ip_addr, attention: interface_id is still iccid + if ((mce->interface_id) && (mce->has_ip)) { + META_PRINT_SYSLOG(mwn->parent, LOG_INFO, "NNE writer: inserting from CONN_EVENT: " + "ifname=%s, " + "ip_addr=%s\n", + mce->interface_id, + mce->network_address); + insert_ip_cache(mce->interface_id, mce->network_address); + } else if ((mce->interface_id) && (!mce->has_ip)) { + delete_ip_cache(mce->interface_id); + } +} + static void md_nne_handle_iface_event(struct md_writer_nne *mwn, struct md_iface_event *mie) { @@ -791,6 +853,8 @@ static void md_nne_handle_iface_event(struct md_writer_nne *mwn, "IFACE_EVENT_CELEVEL_CHANGE", }; + char *ip_addr = get_ip_cache(mie->iccid); + META_PRINT_SYSLOG(mwn->parent, LOG_ERR, "NNE writer: %s: " "ip_addr=%s, " "ifname=%s, " @@ -809,7 +873,7 @@ static void md_nne_handle_iface_event(struct md_writer_nne *mwn, "device_mode=%d, " "device_submode=%d\n", iface_event_type[mie->event_param], - mie->ip_addr, + ip_addr, mie->ifname, mie->iccid, mie->imsi, @@ -1207,6 +1271,10 @@ static int32_t md_nne_init(void *ptr, json_object* config) LIST_INIT(&(mwn->modem_list)); mwn->timeout_tstamp = 0; + for (int i=0; i<16; i++) { + ip_lookup[i].ip_addr[0] = '\0'; + } + json_object* subconfig; if (json_object_object_get_ex(config, "nne", &subconfig)) { json_object_object_foreach(subconfig, key, val) { @@ -1298,6 +1366,9 @@ static void md_nne_handle(struct md_writer *writer, struct md_event *event) case META_TYPE_INTERFACE: md_nne_handle_iface_event(mwn, (struct md_iface_event*) event); break; + case META_TYPE_CONNECTION: + md_nne_handle_conn_event(mwn, (struct md_conn_event*) event); + break; case META_TYPE_RADIO: md_nne_handle_radio(mwn, (struct md_radio_event*) event); break; diff --git a/metadata_writer_nne.h b/src/metadata_writer_nne.h similarity index 100% rename from metadata_writer_nne.h rename to src/metadata_writer_nne.h diff --git a/metadata_writer_sqlite.c b/src/metadata_writer_sqlite.c similarity index 100% rename from metadata_writer_sqlite.c rename to src/metadata_writer_sqlite.c diff --git a/metadata_writer_sqlite.h b/src/metadata_writer_sqlite.h similarity index 100% rename from metadata_writer_sqlite.h rename to src/metadata_writer_sqlite.h diff --git a/metadata_writer_sqlite_helpers.c b/src/metadata_writer_sqlite_helpers.c similarity index 100% rename from metadata_writer_sqlite_helpers.c rename to src/metadata_writer_sqlite_helpers.c diff --git a/metadata_writer_sqlite_helpers.h b/src/metadata_writer_sqlite_helpers.h similarity index 100% rename from metadata_writer_sqlite_helpers.h rename to src/metadata_writer_sqlite_helpers.h diff --git a/metadata_writer_sqlite_monitor.c b/src/metadata_writer_sqlite_monitor.c similarity index 100% rename from metadata_writer_sqlite_monitor.c rename to src/metadata_writer_sqlite_monitor.c diff --git a/metadata_writer_sqlite_monitor.h b/src/metadata_writer_sqlite_monitor.h similarity index 100% rename from metadata_writer_sqlite_monitor.h rename to src/metadata_writer_sqlite_monitor.h diff --git a/metadata_writer_zeromq.c b/src/metadata_writer_zeromq.c similarity index 100% rename from metadata_writer_zeromq.c rename to src/metadata_writer_zeromq.c diff --git a/metadata_writer_zeromq.h b/src/metadata_writer_zeromq.h similarity index 100% rename from metadata_writer_zeromq.h rename to src/metadata_writer_zeromq.h diff --git a/metadata_writer_zeromq_monroe.c b/src/metadata_writer_zeromq_monroe.c similarity index 100% rename from metadata_writer_zeromq_monroe.c rename to src/metadata_writer_zeromq_monroe.c diff --git a/metadata_writer_zeromq_nne.c b/src/metadata_writer_zeromq_nne.c similarity index 100% rename from metadata_writer_zeromq_nne.c rename to src/metadata_writer_zeromq_nne.c diff --git a/system_helpers.c b/src/system_helpers.c similarity index 100% rename from system_helpers.c rename to src/system_helpers.c diff --git a/system_helpers.h b/src/system_helpers.h similarity index 100% rename from system_helpers.h rename to src/system_helpers.h