chore: Refactor CatchAllExecutor to improve error handling and validation

Author: tacxou

src/backend-runner/_executors/catch-all.executor.ts

@@ -6,17 +6,23 @@ import { Job } from 'bullmq';
 import { BackendConfigV1Dto } from '../_dto/backend-config-v1.dto';
 import { BackendResultInfoInterface, BackendResultInterface } from '../_interfaces/backend-result.interface';
 import { BackendCodesEnum } from '../_interfaces/backend-codes.enum';
-import { validateOrReject } from 'class-validator';
+import { ValidationError, validateOrReject } from 'class-validator';
 import { BackendResultInfoDto } from '../_dto/backend-result-info.dto';
 import { plainToInstance } from 'class-transformer';
 
+interface ValidationRecursive {
+  [key: string]: string;
+}
+
 export class CatchAllExecutor implements ExecutorInterface {
-  public constructor(public service: BackendRunnerService) {}
+  public constructor(public service: BackendRunnerService) { }
 
   public async execute({ job }): Promise<ExecutorExecuteResponseInterface> {
     let status = 0;
     const data = [];
 
+    const numberOfBackends = this.service.backendsConfig.backendsConfigData.length;
+
     for await (const backend of this.service.backendsConfig.backendsConfigData) {
       if (!backend.active) {
         this.service.logger.warn(`backend ${backend.name} is not active`);
@@ -32,6 +38,8 @@ export class CatchAllExecutor implements ExecutorInterface {
         this.service.logger.log('stop on Error');
         break;
       }
+
+      await job.updateProgress(100 / numberOfBackends);
     }
 
     return {
@@ -49,7 +57,7 @@ export class CatchAllExecutor implements ExecutorInterface {
     try {
       if (process.status !== 0) {
         this.service.logger.error(`Error executing backend ${backend.name}`);
-        const error = this.extractLastJsonImproved(process.error);
+        const error = this.extractLastJsonImproved(process.error || process.output);
         const errorSchema = plainToInstance(BackendResultInfoDto, error);
         await validateOrReject(errorSchema);
 
@@ -70,18 +78,34 @@ export class CatchAllExecutor implements ExecutorInterface {
         status: process.status,
         output,
       };
-    } catch (e) {
-      this.service.logger.error(`Error parsing JSON output from backend ${backend.name}`);
+    } catch (errors) {
+      console.log('errors', errors)
+      let validations: ValidationRecursive = {};
+      for (const error of errors) {
+        validations = { ...validations, ...this.validationRecursive(error) };
+      }
+
+      this.service.logger.error(`Invalid JSON response from backend ${backend.name}, erreur de validation : ${Object.keys(validations).join(', ')}`.trim());
 
       return {
         backend: backend.name,
         status: BackendCodesEnum.INVALID_JSON_RESPONSE,
-        message: `Invalid JSON response from backend: ${process.error || process.output}`,
+        message: `Erreur de validation : ${Object.keys(validations).join(', ')}`.trim(),
+        error: {
+          status: BackendCodesEnum.INVALID_JSON_RESPONSE,
+          message: `Erreur de validation : ${Object.keys(validations).join(', ')}`.trim(),
+          data: validations,
+        },
       };
     }
   }
 
   private extractLastJsonImproved(stdout: string): BackendResultInfoInterface {
+    if (!stdout) return {
+      status: BackendCodesEnum.INVALID_JSON_RESPONSE,
+      message: 'No output',
+    }
+
     const jsonCandidates = [];
     let braceCount = 0,
       inString = false,
@@ -113,6 +137,32 @@ export class CatchAllExecutor implements ExecutorInterface {
       }
     }
 
+    if (jsonCandidates.length === 0) return {
+      status: BackendCodesEnum.INVALID_JSON_RESPONSE,
+      message: 'No JSON output',
+    }
+
     return JSON.parse(jsonCandidates[jsonCandidates.length - 1]);
   }
+
+  private validationRecursive(error: ValidationError, prefix = ''): ValidationRecursive {
+    let validations = {};
+    if (error.constraints) {
+      validations[`${prefix + error.property}`] = Object.values(error.constraints)[0];
+    }
+    if (error.children.length > 0) {
+      for (const errorChild of error.children) {
+        if (errorChild.constraints) {
+          validations[`${prefix + error.property}.${errorChild.property}`] = Object.values(errorChild.constraints)[0];
+        }
+        if (errorChild.children.length > 0) {
+          validations = {
+            ...validations,
+            ...this.validationRecursive(errorChild, `${prefix + error.property}.${errorChild.property}.`),
+          };
+        }
+      }
+    }
+    return validations;
+  }
 }

src/backend-runner/backend-runner.service.ts

@@ -1,7 +1,7 @@
 import { InjectRedis } from '@nestjs-modules/ioredis';
 import { Injectable, Logger, OnApplicationBootstrap, OnModuleInit } from '@nestjs/common';
 import { ConfigService } from '@nestjs/config';
-import { Worker } from 'bullmq';
+import { UnrecoverableError, Worker } from 'bullmq';
 import { CatchAllExecutor } from './_executors/catch-all.executor';
 import { ListBackendsExecutor } from './_executors/list-backends.executor';
 import { ExecutorExecuteResponseInterface, ExecutorInterface } from './executors.interface';
@@ -50,7 +50,19 @@ export class BackendRunnerService implements OnApplicationBootstrap, OnModuleIni
         this.logger.log(`Job ${job.name} received. Try to execute...`);
 
         const result = await this.executors.get(jobName).execute({ job });
-        await job.updateProgress(100);
+        this.logger.verbose(`Job ${job.name} executed with status ${result.status}`);
+
+        if (result.status !== 0) {
+          this.logger.error(`Job ${job.name} failed with status ${result.status}`);
+          const errMsg = []
+          for (const data of result.data) {
+            errMsg.push(data?.error?.message || 'No error message');
+          }
+
+          throw new UnrecoverableError(`Job ${job.name} failed with status ${result.status}: ${errMsg.join(', ')}`);
+        }
+
+        this.logger.log(`Job ${job.name} success with status ${result.status}`);
         return result;
       },
       {

src/backend-runner/executors.interface.ts

@@ -8,7 +8,7 @@ export interface ExecutorExecuteOptionsInterface {
 export interface ExecutorExecuteResponseInterface {
   jobId: string;
   status: number;
-  data: object;
+  data: Array<any>;
 }
 
 export interface ExecutorInterface {