fix: QA and tests running until timeout because of missing count in while lo…

.gitlab/common.yml

@@ -161,41 +161,6 @@
         reports:
             dotenv: build.env
 
-#.image_check:
-#    tags:
-#        - docker
-#    script:
-#        - |
-#            BRANCH_LOWER=$(echo "$CI_COMMIT_REF_NAME" | tr '[:upper:]' '[:lower:]')
-#            docker login "${REGISTRY}" -u "${HARBOR_USER}" -p "${HARBOR_DATAFED_GITLAB_CI_REGISTRY_TOKEN}"
-#            FORCE_BUILD="FALSE"
-#            set +e
-#            docker pull --quiet "${REGISTRY}/${PROJECT}/${COMPONENT}-${BRANCH_LOWER}:latest"
-#            if [ $? -eq 0 ]; then echo "Image exists"; else FORCE_BUILD="TRUE"; fi;
-#            set -e
-#            if [ "${BUILD_INTERMEDIATE}" == "TRUE" ]
-#            then
-#              set +e
-#              docker pull --quiet "${REGISTRY}/${PROJECT}/${COMPONENT}-${INTERMEDIATE_LAYER_NAME}-${BRANCH_LOWER}:latest"
-#              if [ $? -eq 0 ]; then echo "Image exists"; else FORCE_BUILD="TRUE"; fi;
-#             set -e
-#            fi
-#            if [ "$FORCE_BUILD" == "TRUE" ]
-#            then
-#              cp .gitlab/build/force_build_${COMPONENT}_image.yml ${COMPONENT}_image.yml
-#            else
-#              cp .gitlab/build/build_${COMPONENT}_image.yml ${COMPONENT}_image.yml
-#            fi
-#            echo "REGISTRY=${REGISTRY}" >> build.env
-#            echo "HARBOR_USER=${HARBOR_USER}" >> build.env
-#            echo "HARBOR_DATAFED_GITLAB_CI_REGISTRY_TOKEN=${HARBOR_DATAFED_GITLAB_CI_REGISTRY_TOKEN}" >> build.env
-#            sed -i 's/\(HARBOR_USER=.*\)\$/\1$$/g' build.env
-#    artifacts:
-#        paths:
-#            - ${COMPONENT}_image.yml
-#        reports:
-#            dotenv: build.env
-
 # The purpose of this anchor is to check that an image has been uploaded correctly
 # to the registry and if it has not attempt to upload it again.
 #

common/proto3/common/auth/query_update_request.proto

@@ -10,4 +10,5 @@ message QueryUpdateRequest {
     string id = 1;
     optional string title = 2;
     SearchRequest query = 3;
+    bool replace_query = 4;
 }

common/proto3/common/enums/dependency_dir.proto

@@ -5,6 +5,6 @@ package SDMS;
 option cc_enable_arenas = true;
 
 enum DependencyDir {
-    DEPENDENCY_DIR_IN = 0;
-    DEPENDENCY_DIR_OUT = 1;
+    DIR_IN = 0;
+    DIR_OUT = 1;
 }

common/proto3/common/enums/dependency_type.proto

@@ -5,7 +5,7 @@ package SDMS;
 option cc_enable_arenas = true;
 
 enum DependencyType {
-    DEPENDENCY_TYPE_IS_DERIVED_FROM = 0;
-    DEPENDENCY_TYPE_IS_COMPONENT_OF = 1;
-    DEPENDENCY_TYPE_IS_NEW_VERSION_OF = 2;
+    DEP_IS_DERIVED_FROM = 0;
+    DEP_IS_COMPONENT_OF = 1;
+    DEP_IS_NEW_VERSION_OF = 2;
 }

common/proto3/common/enums/execution_method.proto

@@ -5,7 +5,7 @@ package SDMS;
 option cc_enable_arenas = true;
 
 enum ExecutionMethod {
-    EXECUTION_METHOD_UNSPECIFIED = 0;
-    EXECUTION_METHOD_DIRECT = 1;
-    EXECUTION_METHOD_DEFERRED = 2;
+    EXEC_UNSPECIFIED = 0;
+    DIRECT = 1;
+    DEFERRED = 2;
 }

core/database/foxx/api/acl_router.js

@@ -234,7 +234,7 @@ router
 
 router
     .get("/view", function (req, res) {
-        let rules = null;
+        let rules = [];
         try {
             logger.logRequestStarted({
                 client: req.queryParams.client,
@@ -296,7 +296,7 @@ router
     .description("View current ACL on an object (data record or collection)");
 router
     .get("/shared/list", function (req, res) {
-        let result = null;
+        let result = [];
         try {
             const client = g_lib.getUserFromClientID(req.queryParams.client);
             result = g_lib.getACLOwnersBySubject(

core/database/foxx/api/admin_router.js

@@ -58,7 +58,7 @@ router
         let result = null;
         try {
             logger.logRequestStarted({
-                client: "N/A",
+                client: req.queryParams.client,
                 correlationId: req.headers["x-correlation-id"],
                 httpVerb: "GET",
                 routePath: basePath + "/test",
@@ -82,7 +82,7 @@ router
                 time: (t2 - t1) / 1000,
             });
             logger.logRequestSuccess({
-                client: "N/A",
+                client: req.queryParams.client,
                 correlationId: req.headers["x-correlation-id"],
                 httpVerb: "GET",
                 routePath: basePath + "/test",
@@ -92,13 +92,13 @@ router
             });
         } catch (e) {
             logger.logRequestFailure({
-                client: "N/A",
+                client: req.queryParams.client,
                 correlationId: req.headers["x-correlation-id"],
                 httpVerb: "GET",
                 routePath: basePath + "/test",
                 status: "Failure",
                 description: "Do perf test",
-                extra: { execution_time_seconds: (t2 - t1) / 1000 },
+                extra: { execution_time_seconds: 0 },
                 error: e,
             });
             g_lib.handleException(e, res);

core/database/foxx/api/coll_router.js

@@ -43,10 +43,11 @@ router
                         var owner = client,
                             parent_id;
 
+                        let owner_id = owner._id;
                         if (req.body.parent) {
                             parent_id = g_lib.resolveCollID(req.body.parent, client);
 
-                            var owner_id = g_db.owner.firstExample({
+                            owner_id = g_db.owner.firstExample({
                                 _from: parent_id,
                             })._to;
                             if (owner_id != client._id) {

core/database/foxx/api/config_router.js

@@ -48,7 +48,7 @@ router
                 routePath: basePath + "/msg/daily",
                 status: "Failure",
                 description: "Get message of the day",
-                extra: (msg.msg || "").substring(0, 10),
+                extra: ((msg && msg.msg) || "").substring(0, 10),
                 error: e,
             });
 

core/database/foxx/api/data_router.js

@@ -515,7 +515,8 @@ function recordUpdate(client, record, result) {
             perms |= permissions.PERM_WR_REC;
         }
 
-        if (data.locked || !g_lib.hasPermissions(client, data, perms)) throw error.ERR_PERM_DENIED;
+        if (data.locked || !permissions.hasPermissions(client, data, perms))
+            throw error.ERR_PERM_DENIED;
     }
 
     var owner_id = g_db.owner.firstExample({
@@ -646,14 +647,14 @@ function recordUpdate(client, record, result) {
 
             for (i in data.tags) {
                 tag = data.tags[i];
-                if (!(tag in record.tags)) {
+                if (!record.tags.includes(tag)) {
                     rem_tags.push(tag);
                 }
             }
 
             for (i in record.tags) {
                 tag = record.tags[i];
-                if (!(tag in data.tags)) {
+                if (!data.tags.includes(tag)) {
                     add_tags.push(tag);
                 }
             }
@@ -707,7 +708,7 @@ function recordUpdate(client, record, result) {
         }
     }
 
-    if (record.deps != undefined && (record.deps_add != undefined || record.deps_rem != undefined))
+    if (record.deps != undefined && (record.dep_add != undefined || record.dep_rem != undefined))
         throw [error.ERR_INVALID_PARAM, "Cannot use both dependency set and add/remove."];
 
     var dep,
@@ -1049,7 +1050,7 @@ router
                 httpVerb: "POST",
                 routePath: basePath + "/update/batch",
                 status: "Failure",
-                description: `Update a batch of existing data record. RecordIDs: ${displayIds}`,
+                description: `Update a batch of existing data record. RecordIDs: ${displayedIds}`,
                 extra: {
                     count: totalCount,
                 },
@@ -1515,7 +1516,7 @@ router
 
 router
     .get("/dep/graph/get", function (req, res) {
-        let result = null;
+        let result = [];
         try {
             logger.logRequestStarted({
                 client: req.queryParams.client,
@@ -1541,7 +1542,6 @@ router
                 notes,
                 gen = 0;
 
-            result = [];
             // Get Ancestors
 
             //console.log("get ancestors");
@@ -1825,6 +1825,7 @@ router
  */
 router
     .get("/path", function (req, res) {
+        let path = null;
         try {
             logger.logRequestStarted({
                 client: req.queryParams.client,
@@ -1856,7 +1857,7 @@ router
                     "Can only access data from '" + repo.domain + "' domain",
                 ];
 
-            var path = g_lib.computeDataPath(loc, true);
+            path = g_lib.computeDataPath(loc, true);
             res.send({
                 path: path,
             });
@@ -2389,6 +2390,15 @@ router
 router
     .post("/delete", function (req, res) {
         var retry = 10;
+        let ids = [];
+        logger.logRequestStarted({
+            client: req.queryParams.client,
+            correlationId: req.headers["x-correlation-id"],
+            httpVerb: "POST",
+            routePath: basePath + "/delete",
+            status: "Started",
+            description: `Attempting to delete a total of: ${req.body.ids.length}`,
+        });
 
         for (;;) {
             try {
@@ -2416,10 +2426,11 @@ router
                     },
                     action: function () {
                         const client = g_lib.getUserFromClientID(req.queryParams.client);
-                        var i,
-                            id,
-                            ids = [];
+                        var i, id;
 
+                        // Needs to be reinitialized to an empty array to avoid
+                        // accumulating content from retries
+                        ids = [];
                         for (i in req.body.ids) {
                             id = g_lib.resolveDataCollID(req.body.ids[i], client);
                             ids.push(id);
@@ -2430,8 +2441,30 @@ router
                         res.send(result);
                     },
                 });
+                const preview = ids.slice(0, 5).join(", ");
+                const idSummary = ids.length > 5 ? `${preview}, ...` : preview;
+                logger.logRequestSuccess({
+                    client: req.queryParams.client,
+                    correlationId: req.headers["x-correlation-id"],
+                    httpVerb: "POST",
+                    routePath: basePath + "/delete",
+                    status: "Success",
+                    description: `Delete data items: ${idSummary}...`,
+                    extra: { count: ids.length },
+                });
+
                 break;
             } catch (e) {
+                logger.logRequestFailure({
+                    client: req.queryParams.client,
+                    correlationId: req.headers["x-correlation-id"],
+                    httpVerb: "POST",
+                    routePath: basePath + "/delete",
+                    status: "Failure",
+                    description: `Attempting to delete a total of: ${req.body.ids.length}`,
+                    extra: { retry_attempt: retry },
+                    error: e,
+                });
                 if (--retry == 0 || !e.errorNum || e.errorNum != 1200) {
                     g_lib.handleException(e, res);
                 }

core/database/foxx/api/group_router.js

@@ -440,7 +440,6 @@ router
                 extra: logExtra,
             });
         } catch (e) {
-            res.send(groups);
             logger.logRequestFailure({
                 client: client?._id,
                 correlationId: req.headers["x-correlation-id"],

core/database/foxx/api/metrics_router.js

@@ -73,7 +73,7 @@ router
 router
     .get("/msg_count", function (req, res) {
         let client = null;
-        let result = null;
+        let result = [];
         try {
             client = g_lib.getUserFromClientID(req.queryParams.client);
             logger.logRequestStarted({
@@ -147,7 +147,7 @@ router
 router
     .get("/users/active", function (req, res) {
         let client = null;
-        let cnt = null;
+        let cnt = {};
         try {
             client = req.queryParams.client
                 ? g_lib.getUserFromClientID(req.queryParams.client)
@@ -161,7 +161,6 @@ router
                 description: "Get recently active users from metrics",
             });
 
-            cnt = {};
             var u,
                 r,
                 qryres = g_db

core/database/foxx/api/note_router.js

@@ -180,9 +180,10 @@ router
                         }),
                         old_state = note.state,
                         old_type = note.type,
-                        doc = g_db._document(ne._from),
                         updates = {};
 
+                    doc = g_db._document(ne._from);
+
                     /* Permissions to update: Currently any admin of the subject and the creator of the annotation may
                     make edits to the annotation. This approach is optimistic in assuming that conflicts will not arise
                     and all parties are ethical. Eventually a mechanism will be put in place to deal with conflicts and
@@ -412,7 +413,7 @@ router
                     req.queryParams.id +
                     " Comment ID:" +
                     req.queryParams.comment_idx,
-                extra: note.new,
+                extra: note,
                 error: e,
             });
             g_lib.handleException(e, res);