From fa64913eda8209a22dcbd1eb9a2ceb7616cfc285 Mon Sep 17 00:00:00 2001 From: RenzoPrats Date: Fri, 3 May 2024 17:13:41 -0300 Subject: [PATCH 01/27] feat: added possibility to set prefix key in user to service api configuration, started to create possibility to import kobo forms in add form modal --- src/routes/proxy/index.ts | 4 +++- src/utils/proxy/authManagement.ts | 7 ++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/routes/proxy/index.ts b/src/routes/proxy/index.ts index 89c9dca3b..38d5ea250 100644 --- a/src/routes/proxy/index.ts +++ b/src/routes/proxy/index.ts @@ -66,7 +66,8 @@ const proxyAPIRequest = async ( url, method: req.method, headers: { - Authorization: `Bearer ${token}`, + // if user-to-service token may contain prefix so it's get directly in getToken + Authorization: api.authType == 'user-to-service' ? token : `Bearer ${token}`, 'Content-Type': 'application/json', }, maxRedirects: 35, @@ -75,6 +76,7 @@ const proxyAPIRequest = async ( }), }) .then(async ({ data, status }) => { + console.log(data); // We are only caching the results of requests that are not user-dependent. // Otherwise, unwanted users could access cached data of other users. // As an improvement, we could include a stringified unique property of the user to the cache-key to enable user-specific cache. diff --git a/src/utils/proxy/authManagement.ts b/src/utils/proxy/authManagement.ts index 386935ff6..22614c443 100644 --- a/src/utils/proxy/authManagement.ts +++ b/src/utils/proxy/authManagement.ts @@ -99,7 +99,7 @@ export const getToken = async ( } if (apiConfiguration.authType === authType.userToService) { // Retrieve access token from settings, store it and return it - const settings: { token: string } = ping + const settings: { token: string, tokenPrefix: string } = ping ? apiConfiguration.settings : JSON.parse( CryptoJS.AES.decrypt( @@ -107,8 +107,9 @@ export const getToken = async ( config.get('encryption.key') ).toString(CryptoJS.enc.Utf8) ); - cache.set(tokenID, settings.token, 3570); - return settings.token; + const token = (settings.tokenPrefix ? settings.tokenPrefix : 'Bearer') + ' ' + settings.token; + cache.set(tokenID, token, 3570); + return token; } if (apiConfiguration.authType === authType.authorizationCode) { // Making sure to return only string, as access token is typed string | string[] From df15809c84cec8e89301c27d2e200c46c50046bb Mon Sep 17 00:00:00 2001 From: RenzoPrats Date: Fri, 3 May 2024 17:15:23 -0300 Subject: [PATCH 02/27] lint fixed --- src/routes/proxy/index.ts | 3 ++- src/utils/proxy/authManagement.ts | 5 +++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/routes/proxy/index.ts b/src/routes/proxy/index.ts index 38d5ea250..316893617 100644 --- a/src/routes/proxy/index.ts +++ b/src/routes/proxy/index.ts @@ -67,7 +67,8 @@ const proxyAPIRequest = async ( method: req.method, headers: { // if user-to-service token may contain prefix so it's get directly in getToken - Authorization: api.authType == 'user-to-service' ? token : `Bearer ${token}`, + Authorization: + api.authType == 'user-to-service' ? token : `Bearer ${token}`, 'Content-Type': 'application/json', }, maxRedirects: 35, diff --git a/src/utils/proxy/authManagement.ts b/src/utils/proxy/authManagement.ts index 22614c443..16c8774a4 100644 --- a/src/utils/proxy/authManagement.ts +++ b/src/utils/proxy/authManagement.ts @@ -99,7 +99,7 @@ export const getToken = async ( } if (apiConfiguration.authType === authType.userToService) { // Retrieve access token from settings, store it and return it - const settings: { token: string, tokenPrefix: string } = ping + const settings: { token: string; tokenPrefix: string } = ping ? apiConfiguration.settings : JSON.parse( CryptoJS.AES.decrypt( @@ -107,7 +107,8 @@ export const getToken = async ( config.get('encryption.key') ).toString(CryptoJS.enc.Utf8) ); - const token = (settings.tokenPrefix ? settings.tokenPrefix : 'Bearer') + ' ' + settings.token; + const token = + (settings.tokenPrefix ? settings.tokenPrefix : 'Bearer') + ' ' + settings.token; cache.set(tokenID, token, 3570); return token; } From f4503138e30023ac360ad3efebbfb178af495951 Mon Sep 17 00:00:00 2001 From: RenzoPrats Date: Fri, 10 May 2024 15:28:19 -0300 Subject: [PATCH 03/27] feat: started import kobo questions --- src/schema/mutation/addForm.mutation.ts | 39 ++++++++- src/utils/form/extractKoboFields.ts | 104 ++++++++++++++++++++++++ src/utils/form/index.ts | 1 + 3 files changed, 142 insertions(+), 2 deletions(-) create mode 100644 src/utils/form/extractKoboFields.ts diff --git a/src/schema/mutation/addForm.mutation.ts b/src/schema/mutation/addForm.mutation.ts index ca4769ed6..35090d598 100644 --- a/src/schema/mutation/addForm.mutation.ts +++ b/src/schema/mutation/addForm.mutation.ts @@ -5,7 +5,8 @@ import { GraphQLError, } from 'graphql'; import { validateGraphQLTypeName } from '@utils/validators'; -import { Resource, Form, Role, ReferenceData } from '@models'; +import { extractKoboFields } from '@utils/form'; +import { Resource, Form, Role, ReferenceData, ApiConfiguration } from '@models'; import { FormType } from '../types'; import { AppAbility } from '@security/defineUserAbility'; import { status } from '@const/enumTypes'; @@ -13,12 +14,15 @@ import { logger } from '@services/logger.service'; import { graphQLAuthCheck } from '@schema/shared'; import { Types } from 'mongoose'; import { Context } from '@server/apollo/context'; +import axios from 'axios'; /** Arguments for the addForm mutation */ type AddFormArgs = { name: string; resource?: string | Types.ObjectId; template?: string | Types.ObjectId; + apiConfiguration?: string | Types.ObjectId; + kobo?: string; }; /** @@ -31,10 +35,13 @@ export default { name: { type: new GraphQLNonNull(GraphQLString) }, resource: { type: GraphQLID }, template: { type: GraphQLID }, + apiConfiguration: { type: GraphQLID }, + kobo: { type: GraphQLString }, }, async resolve(parent, args: AddFormArgs, context: Context) { graphQLAuthCheck(context); try { + console.log(args); // Check authentication const user = context.user; const ability: AppAbility = user.ability; @@ -73,7 +80,35 @@ export default { canDeleteRecords: [], }; try { - if (!args.resource) { + + if (args.apiConfiguration) { + const apiConfiguration = await ApiConfiguration.findById(args.apiConfiguration); + + const url = apiConfiguration.endpoint + `assets/${args.kobo}?format=json`; + const response = await axios.get(url); + const survey = response.data.content.survey; + + const structure = extractKoboFields(survey); + + // create resource + const resource = new Resource({ + name: args.name, + //createdAt: new Date(), + permissions: defaultResourcePermissions, + }); + await resource.save(); + const form = new Form({ + name: args.name, + graphQLTypeName, + status: status.pending, + resource, + core: true, + permissions: defaultFormPermissions, + structure: structure + }); + await form.save(); + return form; + } else if (!args.resource) { // Check permission to create resource if (ability.cannot('create', 'Resource')) { throw new GraphQLError( diff --git a/src/utils/form/extractKoboFields.ts b/src/utils/form/extractKoboFields.ts new file mode 100644 index 000000000..3f31a9451 --- /dev/null +++ b/src/utils/form/extractKoboFields.ts @@ -0,0 +1,104 @@ +export const extractKoboFields = (survey: any) => { + survey.map((question: any) => { + console.log(question); + }) + // { + // "pages": [ + // { + // "name": "page1", + // "elements": [ + // { + // "type": "text", + // "name": "question_1", + // "title": "Question 1", + // "description": "What is the most famous rockstar?", + // "valueName": "question_1" + // }, + // { + // "type": "text", + // "name": "question2", + // "description": "What is the best singer in rock?", + // "valueName": "question2" + // }, + // { + // "type": "dropdown", + // "name": "choose_your_favorite_rock_genre", + // "title": "Choose your favorite rock genre", + // "valueName": "choose_your_favorite_rock_genre", + // "choices": [ + // { + // "value": "item1", + // "text": "Progressive" + // }, + // { + // "value": "item2", + // "text": "Psicodelic" + // }, + // { + // "value": "item3", + // "text": "Heavy metal" + // } + // ] + // }, + // { + // "type": "checkbox", + // "name": "which_is_the_better_rock_band_of_all_time", + // "title": "Which is the better rock band of all time", + // "valueName": "which_is_the_better_rock_band_of_all_time", + // "choices": [ + // { + // "value": "item1", + // "text": "The Beatles" + // }, + // { + // "value": "item2", + // "text": "The Rolling Stones" + // }, + // { + // "value": "item3", + // "text": "Queen" + // }, + // { + // "value": "item4", + // "text": "Pink Floyd" + // } + // ] + // }, + // { + // "type": "geospatial", + // "name": "question1", + // "valueName": "question1", + // "geoFields": [ + // { + // "value": "city", + // "label": "City" + // } + // ] + // }, + // { + // "type": "geospatial", + // "name": "question3", + // "valueName": "question3" + // }, + // { + // "type": "geospatial", + // "name": "question4", + // "valueName": "question4", + // "geoFields": [ + // { + // "value": "coordinates", + // "label": "coordenadas" + // }, + // { + // "value": "city", + // "label": "cidade" + // } + // ] + // } + // ] + // } + // ], + // "showQuestionNumbers": "off" + // } + return ""; +}; \ No newline at end of file diff --git a/src/utils/form/index.ts b/src/utils/form/index.ts index f7d424a8c..ddad6c659 100644 --- a/src/utils/form/index.ts +++ b/src/utils/form/index.ts @@ -4,6 +4,7 @@ export * from './addField'; export * from './replaceField'; export * from './findDuplicateFields'; export * from './extractFields'; +export * from './extractKoboFields'; export * from './getFieldType'; // === RECORDS === From 46cd45f2cb6ea46f2fce6462846278f17c251b21 Mon Sep 17 00:00:00 2001 From: RenzoPrats Date: Fri, 10 May 2024 15:44:23 -0300 Subject: [PATCH 04/27] lint fixed --- src/schema/mutation/addForm.mutation.ts | 14 +- src/utils/form/extractKoboFields.ts | 206 ++++++++++++------------ src/utils/proxy/authManagement.ts | 4 +- 3 files changed, 114 insertions(+), 110 deletions(-) diff --git a/src/schema/mutation/addForm.mutation.ts b/src/schema/mutation/addForm.mutation.ts index 35090d598..40fd5d2e0 100644 --- a/src/schema/mutation/addForm.mutation.ts +++ b/src/schema/mutation/addForm.mutation.ts @@ -80,14 +80,16 @@ export default { canDeleteRecords: [], }; try { - if (args.apiConfiguration) { - const apiConfiguration = await ApiConfiguration.findById(args.apiConfiguration); - - const url = apiConfiguration.endpoint + `assets/${args.kobo}?format=json`; + const apiConfiguration = await ApiConfiguration.findById( + args.apiConfiguration + ); + + const url = + apiConfiguration.endpoint + `assets/${args.kobo}?format=json`; const response = await axios.get(url); const survey = response.data.content.survey; - + const structure = extractKoboFields(survey); // create resource @@ -104,7 +106,7 @@ export default { resource, core: true, permissions: defaultFormPermissions, - structure: structure + structure: structure, }); await form.save(); return form; diff --git a/src/utils/form/extractKoboFields.ts b/src/utils/form/extractKoboFields.ts index 3f31a9451..bf8fb61a1 100644 --- a/src/utils/form/extractKoboFields.ts +++ b/src/utils/form/extractKoboFields.ts @@ -1,104 +1,104 @@ export const extractKoboFields = (survey: any) => { - survey.map((question: any) => { - console.log(question); - }) - // { - // "pages": [ - // { - // "name": "page1", - // "elements": [ - // { - // "type": "text", - // "name": "question_1", - // "title": "Question 1", - // "description": "What is the most famous rockstar?", - // "valueName": "question_1" - // }, - // { - // "type": "text", - // "name": "question2", - // "description": "What is the best singer in rock?", - // "valueName": "question2" - // }, - // { - // "type": "dropdown", - // "name": "choose_your_favorite_rock_genre", - // "title": "Choose your favorite rock genre", - // "valueName": "choose_your_favorite_rock_genre", - // "choices": [ - // { - // "value": "item1", - // "text": "Progressive" - // }, - // { - // "value": "item2", - // "text": "Psicodelic" - // }, - // { - // "value": "item3", - // "text": "Heavy metal" - // } - // ] - // }, - // { - // "type": "checkbox", - // "name": "which_is_the_better_rock_band_of_all_time", - // "title": "Which is the better rock band of all time", - // "valueName": "which_is_the_better_rock_band_of_all_time", - // "choices": [ - // { - // "value": "item1", - // "text": "The Beatles" - // }, - // { - // "value": "item2", - // "text": "The Rolling Stones" - // }, - // { - // "value": "item3", - // "text": "Queen" - // }, - // { - // "value": "item4", - // "text": "Pink Floyd" - // } - // ] - // }, - // { - // "type": "geospatial", - // "name": "question1", - // "valueName": "question1", - // "geoFields": [ - // { - // "value": "city", - // "label": "City" - // } - // ] - // }, - // { - // "type": "geospatial", - // "name": "question3", - // "valueName": "question3" - // }, - // { - // "type": "geospatial", - // "name": "question4", - // "valueName": "question4", - // "geoFields": [ - // { - // "value": "coordinates", - // "label": "coordenadas" - // }, - // { - // "value": "city", - // "label": "cidade" - // } - // ] - // } - // ] - // } - // ], - // "showQuestionNumbers": "off" - // } - return ""; -}; \ No newline at end of file + survey.map((question: any) => { + console.log(question); + }); + // { + // "pages": [ + // { + // "name": "page1", + // "elements": [ + // { + // "type": "text", + // "name": "question_1", + // "title": "Question 1", + // "description": "What is the most famous rockstar?", + // "valueName": "question_1" + // }, + // { + // "type": "text", + // "name": "question2", + // "description": "What is the best singer in rock?", + // "valueName": "question2" + // }, + // { + // "type": "dropdown", + // "name": "choose_your_favorite_rock_genre", + // "title": "Choose your favorite rock genre", + // "valueName": "choose_your_favorite_rock_genre", + // "choices": [ + // { + // "value": "item1", + // "text": "Progressive" + // }, + // { + // "value": "item2", + // "text": "Psicodelic" + // }, + // { + // "value": "item3", + // "text": "Heavy metal" + // } + // ] + // }, + // { + // "type": "checkbox", + // "name": "which_is_the_better_rock_band_of_all_time", + // "title": "Which is the better rock band of all time", + // "valueName": "which_is_the_better_rock_band_of_all_time", + // "choices": [ + // { + // "value": "item1", + // "text": "The Beatles" + // }, + // { + // "value": "item2", + // "text": "The Rolling Stones" + // }, + // { + // "value": "item3", + // "text": "Queen" + // }, + // { + // "value": "item4", + // "text": "Pink Floyd" + // } + // ] + // }, + // { + // "type": "geospatial", + // "name": "question1", + // "valueName": "question1", + // "geoFields": [ + // { + // "value": "city", + // "label": "City" + // } + // ] + // }, + // { + // "type": "geospatial", + // "name": "question3", + // "valueName": "question3" + // }, + // { + // "type": "geospatial", + // "name": "question4", + // "valueName": "question4", + // "geoFields": [ + // { + // "value": "coordinates", + // "label": "coordenadas" + // }, + // { + // "value": "city", + // "label": "cidade" + // } + // ] + // } + // ] + // } + // ], + // "showQuestionNumbers": "off" + // } + return ''; +}; diff --git a/src/utils/proxy/authManagement.ts b/src/utils/proxy/authManagement.ts index 16c8774a4..432507985 100644 --- a/src/utils/proxy/authManagement.ts +++ b/src/utils/proxy/authManagement.ts @@ -108,7 +108,9 @@ export const getToken = async ( ).toString(CryptoJS.enc.Utf8) ); const token = - (settings.tokenPrefix ? settings.tokenPrefix : 'Bearer') + ' ' + settings.token; + (settings.tokenPrefix ? settings.tokenPrefix : 'Bearer') + + ' ' + + settings.token; cache.set(tokenID, token, 3570); return token; } From f312098c0911394d3a3780d02c232cf03bdafe67 Mon Sep 17 00:00:00 2001 From: RenzoPrats Date: Fri, 17 May 2024 17:06:48 -0300 Subject: [PATCH 05/27] feat: started translate kobo questions to survey ones --- src/schema/mutation/addForm.mutation.ts | 21 +- src/utils/form/extractKoboFields.ts | 367 +++++++++++++++++------- 2 files changed, 287 insertions(+), 101 deletions(-) diff --git a/src/schema/mutation/addForm.mutation.ts b/src/schema/mutation/addForm.mutation.ts index 40fd5d2e0..c068afcee 100644 --- a/src/schema/mutation/addForm.mutation.ts +++ b/src/schema/mutation/addForm.mutation.ts @@ -15,6 +15,8 @@ import { graphQLAuthCheck } from '@schema/shared'; import { Types } from 'mongoose'; import { Context } from '@server/apollo/context'; import axios from 'axios'; +import config from 'config'; +import * as CryptoJS from 'crypto-js'; /** Arguments for the addForm mutation */ type AddFormArgs = { @@ -84,18 +86,29 @@ export default { const apiConfiguration = await ApiConfiguration.findById( args.apiConfiguration ); - const url = apiConfiguration.endpoint + `assets/${args.kobo}?format=json`; - const response = await axios.get(url); + + const settings = JSON.parse( + CryptoJS.AES.decrypt( + apiConfiguration.settings, + config.get('encryption.key') + ).toString(CryptoJS.enc.Utf8) + ); + const response = await axios.get(url, { + headers: { + 'Authorization': `${settings.tokenPrefix} ${settings.token}` + } + }); const survey = response.data.content.survey; + const choices = response.data.content.choices; + const title = response.data.name; - const structure = extractKoboFields(survey); + const structure = JSON.stringify(extractKoboFields(survey, title, choices)); // create resource const resource = new Resource({ name: args.name, - //createdAt: new Date(), permissions: defaultResourcePermissions, }); await resource.save(); diff --git a/src/utils/form/extractKoboFields.ts b/src/utils/form/extractKoboFields.ts index bf8fb61a1..f40b59970 100644 --- a/src/utils/form/extractKoboFields.ts +++ b/src/utils/form/extractKoboFields.ts @@ -1,104 +1,277 @@ -export const extractKoboFields = (survey: any) => { +const AVAILABLE_TYPES = [ + 'decimal', + 'geopoint', + // 'select_multiple', + 'date', + 'note', + // 'begin_score', + // 'score__row', + // 'end_score', + 'text', + 'time', + 'file', + 'integer', + 'datetime', + 'acknowledge', + // 'begin_rank', + // 'rank__level', + // 'end_rank', + 'range', + + +] +export const extractKoboFields = (survey: any, title: string, choices: any) => { + const questions = { + "title": title, + "pages" : [ + { + "name": "page1", + "elements": [ + ] + } + ], + "showQuestionNumbers": "off" + }; + survey.map((question: any) => { - console.log(question); + if (AVAILABLE_TYPES.includes(question.type)) { + switch (question.type) { + case 'decimal' : { + const newQuestion = { + 'type': 'text', + 'name': question.$autoname.toLowerCase(), + 'title': question.label[0], + 'valueName': question.$autoname.toLowerCase(), + "isRequired": question.required, + 'inputType': 'number' + }; + questions.pages[0].elements.push(newQuestion); + break; + } + case 'geopoint' : { + const newQuestion = { + 'type': 'geospatial', + 'name': question.$autoname.toLowerCase(), + 'title': question.label[0], + 'valueName': question.$autoname.toLowerCase(), + "isRequired": question.required, + }; + questions.pages[0].elements.push(newQuestion); + break; + } + case 'date' : { + const newQuestion = { + 'type': 'text', + 'name': question.$autoname.toLowerCase(), + 'title': question.label[0], + "isRequired": question.required, + 'valueName': question.$autoname.toLowerCase(), + 'inputType': 'date' + }; + questions.pages[0].elements.push(newQuestion); + break; + } + case 'note' : { + const newQuestion = { + 'type': 'expression', + 'name': question.$autoname.toLowerCase(), + 'title': question.label[0], + 'valueName': question.$autoname.toLowerCase(), + }; + questions.pages[0].elements.push(newQuestion); + break; + } + case 'text' : { + const newQuestion = { + 'type': 'text', + 'name': question.$autoname.toLowerCase(), + 'title': question.label[0], + "isRequired": question.required, + 'valueName': question.$autoname.toLowerCase(), + }; + questions.pages[0].elements.push(newQuestion); + break; + } + case 'time' : { + const newQuestion = { + 'type': 'text', + 'name': question.$autoname.toLowerCase(), + 'title': question.label[0], + "isRequired": question.required, + 'valueName': question.$autoname.toLowerCase(), + "inputType": "time" + }; + questions.pages[0].elements.push(newQuestion); + break; + } + case 'file' : { + const newQuestion = { + 'type': 'file', + 'name': question.$autoname.toLowerCase(), + 'title': question.label[0], + "isRequired": question.required, + 'valueName': question.$autoname.toLowerCase(), + "storeDataAsText": false, + "maxSize": 7340032 + }; + questions.pages[0].elements.push(newQuestion); + break; + } + case 'integer' : { + const newQuestion = { + 'type': 'text', + 'name': question.$autoname.toLowerCase(), + 'title': question.label[0], + "isRequired": question.required, + 'valueName': question.$autoname.toLowerCase(), + 'inputType': 'number' + }; + questions.pages[0].elements.push(newQuestion); + break; + } + case 'datetime' : { + const newQuestion = { + 'type': 'text', + 'name': question.$autoname.toLowerCase(), + 'title': question.label[0], + "isRequired": question.required, + 'valueName': question.$autoname.toLowerCase(), + 'inputType': 'datetime-local' + }; + questions.pages[0].elements.push(newQuestion); + break; + } + case 'acknowledge' : { + const newQuestion = { + 'type': 'boolean', + 'name': question.$autoname.toLowerCase(), + 'title': question.label[0], + 'valueName': question.$autoname.toLowerCase(), + }; + questions.pages[0].elements.push(newQuestion); + break; + } + case 'range' : { + const newQuestion = { + 'type': 'text', + 'name': question.$autoname.toLowerCase(), + 'title': question.label[0], + 'valueName': question.$autoname.toLowerCase(), + "inputType": "range", + 'step': question.parameters.split('step=1')[1] + }; + questions.pages[0].elements.push(newQuestion); + break; + } + } + } + }); - // { - // "pages": [ + // "logoPosition": "right", + // "pages": [ + // { + // "name": "page1", + // "elements": [ // { - // "name": "page1", - // "elements": [ - // { - // "type": "text", - // "name": "question_1", - // "title": "Question 1", - // "description": "What is the most famous rockstar?", - // "valueName": "question_1" - // }, - // { - // "type": "text", - // "name": "question2", - // "description": "What is the best singer in rock?", - // "valueName": "question2" - // }, - // { - // "type": "dropdown", - // "name": "choose_your_favorite_rock_genre", - // "title": "Choose your favorite rock genre", - // "valueName": "choose_your_favorite_rock_genre", - // "choices": [ - // { - // "value": "item1", - // "text": "Progressive" - // }, - // { - // "value": "item2", - // "text": "Psicodelic" - // }, - // { - // "value": "item3", - // "text": "Heavy metal" - // } - // ] - // }, - // { - // "type": "checkbox", - // "name": "which_is_the_better_rock_band_of_all_time", - // "title": "Which is the better rock band of all time", - // "valueName": "which_is_the_better_rock_band_of_all_time", - // "choices": [ - // { - // "value": "item1", - // "text": "The Beatles" - // }, - // { - // "value": "item2", - // "text": "The Rolling Stones" - // }, - // { - // "value": "item3", - // "text": "Queen" - // }, - // { - // "value": "item4", - // "text": "Pink Floyd" - // } - // ] - // }, - // { - // "type": "geospatial", - // "name": "question1", - // "valueName": "question1", - // "geoFields": [ - // { - // "value": "city", - // "label": "City" - // } - // ] - // }, - // { - // "type": "geospatial", - // "name": "question3", - // "valueName": "question3" - // }, - // { - // "type": "geospatial", - // "name": "question4", - // "valueName": "question4", - // "geoFields": [ - // { - // "value": "coordinates", - // "label": "coordenadas" - // }, - // { - // "value": "city", - // "label": "cidade" - // } - // ] - // } + // "type": "text", + // "name": "decimal", + // "title": "decimal", + // "valueName": "decimal", + // "inputType": "number" + // }, + // { + // "type": "geospatial", + // "name": "ponto", + // "title": "Ponto", + // "valueName": "ponto" + // }, + // { + // "type": "checkbox", + // "name": "question2", + // "title": "Selecionar Multiplos", + // "valueName": "question2", + // "choices": [ + // "Item 1", + // "Item 2" + // ], + // "showSelectAllItem": true + // }, + // { + // "type": "text", + // "name": "data", + // "title": "Data", + // "valueName": "data", + // "inputType": "date" + // }, + // { + // "type": "expression", + // "name": "nota", + // "title": "Nota", + // "valueName": "nota" + // }, + // { + // "type": "radiogroup", + // "name": "avaliacao", + // "title": "Avaliação", + // "valueName": "avaliacao", + // "choices": [ + // "Item 1", + // "Item 2", + // "Item 3" // ] + // }, + // { + // "type": "text", + // "name": "texto", + // "title": "texto", + // "valueName": "texto" + // }, + // { + // "type": "text", + // "name": "horario", + // "title": "horario", + // "valueName": "horario", + // "inputType": "time" + // }, + // { + // "type": "file", + // "name": "arquivo", + // "title": "arquivo", + // "valueName": "arquivo", + // "storeDataAsText": false, + // "maxSize": 7340032 + // }, + // { + // "type": "text", + // "name": "numero", + // "title": "numero", + // "valueName": "numero", + // "inputType": "number" + // }, + // { + // "type": "text", + // "name": "datetime", + // "title": "datetime", + // "valueName": "datetime", + // "inputType": "datetime-local" + // }, + // { + // "type": "boolean", + // "name": "reconhece", + // "title": "reconhece", + // "valueName": "reconhece" + // }, + // { + // "type": "text", + // "name": "intervalo", + // "title": "Intervalo", + // "valueName": "intervalo", + // "inputType": "range" // } - // ], - // "showQuestionNumbers": "off" + // ] // } - return ''; + // ], + // "showQuestionNumbers": "off" + // } + return questions; }; From 4900d89257754408807a6bcf4820a6f351b0172e Mon Sep 17 00:00:00 2001 From: RenzoPrats Date: Fri, 17 May 2024 17:23:17 -0300 Subject: [PATCH 06/27] lint fixed --- src/schema/mutation/addForm.mutation.ts | 9 +- src/utils/form/extractKoboFields.ts | 162 ++++++++++++------------ 2 files changed, 85 insertions(+), 86 deletions(-) diff --git a/src/schema/mutation/addForm.mutation.ts b/src/schema/mutation/addForm.mutation.ts index c068afcee..8dbd4ca7f 100644 --- a/src/schema/mutation/addForm.mutation.ts +++ b/src/schema/mutation/addForm.mutation.ts @@ -88,7 +88,6 @@ export default { ); const url = apiConfiguration.endpoint + `assets/${args.kobo}?format=json`; - const settings = JSON.parse( CryptoJS.AES.decrypt( apiConfiguration.settings, @@ -97,14 +96,16 @@ export default { ); const response = await axios.get(url, { headers: { - 'Authorization': `${settings.tokenPrefix} ${settings.token}` - } + Authorization: `${settings.tokenPrefix} ${settings.token}`, + }, }); const survey = response.data.content.survey; const choices = response.data.content.choices; const title = response.data.name; - const structure = JSON.stringify(extractKoboFields(survey, title, choices)); + const structure = JSON.stringify( + extractKoboFields(survey, title, choices) + ); // create resource const resource = new Resource({ diff --git a/src/utils/form/extractKoboFields.ts b/src/utils/form/extractKoboFields.ts index f40b59970..fbc535592 100644 --- a/src/utils/form/extractKoboFields.ts +++ b/src/utils/form/extractKoboFields.ts @@ -17,155 +17,153 @@ const AVAILABLE_TYPES = [ // 'rank__level', // 'end_rank', 'range', +]; - -] export const extractKoboFields = (survey: any, title: string, choices: any) => { + console.log(choices); const questions = { - "title": title, - "pages" : [ + title: title, + pages: [ { - "name": "page1", - "elements": [ - ] - } + name: 'page1', + elements: [], + }, ], - "showQuestionNumbers": "off" + showQuestionNumbers: 'off', }; survey.map((question: any) => { if (AVAILABLE_TYPES.includes(question.type)) { switch (question.type) { - case 'decimal' : { + case 'decimal': { const newQuestion = { - 'type': 'text', - 'name': question.$autoname.toLowerCase(), - 'title': question.label[0], - 'valueName': question.$autoname.toLowerCase(), - "isRequired": question.required, - 'inputType': 'number' + type: 'text', + name: question.$autoname.toLowerCase(), + title: question.label[0], + valueName: question.$autoname.toLowerCase(), + isRequired: question.required, + inputType: 'number', }; questions.pages[0].elements.push(newQuestion); break; } - case 'geopoint' : { + case 'geopoint': { const newQuestion = { - 'type': 'geospatial', - 'name': question.$autoname.toLowerCase(), - 'title': question.label[0], - 'valueName': question.$autoname.toLowerCase(), - "isRequired": question.required, + type: 'geospatial', + name: question.$autoname.toLowerCase(), + title: question.label[0], + valueName: question.$autoname.toLowerCase(), + isRequired: question.required, }; questions.pages[0].elements.push(newQuestion); break; } - case 'date' : { + case 'date': { const newQuestion = { - 'type': 'text', - 'name': question.$autoname.toLowerCase(), - 'title': question.label[0], - "isRequired": question.required, - 'valueName': question.$autoname.toLowerCase(), - 'inputType': 'date' + type: 'text', + name: question.$autoname.toLowerCase(), + title: question.label[0], + isRequired: question.required, + valueName: question.$autoname.toLowerCase(), + inputType: 'date', }; questions.pages[0].elements.push(newQuestion); break; } - case 'note' : { + case 'note': { const newQuestion = { - 'type': 'expression', - 'name': question.$autoname.toLowerCase(), - 'title': question.label[0], - 'valueName': question.$autoname.toLowerCase(), + type: 'expression', + name: question.$autoname.toLowerCase(), + title: question.label[0], + valueName: question.$autoname.toLowerCase(), }; questions.pages[0].elements.push(newQuestion); break; } - case 'text' : { + case 'text': { const newQuestion = { - 'type': 'text', - 'name': question.$autoname.toLowerCase(), - 'title': question.label[0], - "isRequired": question.required, - 'valueName': question.$autoname.toLowerCase(), + type: 'text', + name: question.$autoname.toLowerCase(), + title: question.label[0], + isRequired: question.required, + valueName: question.$autoname.toLowerCase(), }; questions.pages[0].elements.push(newQuestion); break; } - case 'time' : { + case 'time': { const newQuestion = { - 'type': 'text', - 'name': question.$autoname.toLowerCase(), - 'title': question.label[0], - "isRequired": question.required, - 'valueName': question.$autoname.toLowerCase(), - "inputType": "time" + type: 'text', + name: question.$autoname.toLowerCase(), + title: question.label[0], + isRequired: question.required, + valueName: question.$autoname.toLowerCase(), + inputType: 'time', }; questions.pages[0].elements.push(newQuestion); break; } - case 'file' : { + case 'file': { const newQuestion = { - 'type': 'file', - 'name': question.$autoname.toLowerCase(), - 'title': question.label[0], - "isRequired": question.required, - 'valueName': question.$autoname.toLowerCase(), - "storeDataAsText": false, - "maxSize": 7340032 + type: 'file', + name: question.$autoname.toLowerCase(), + title: question.label[0], + isRequired: question.required, + valueName: question.$autoname.toLowerCase(), + storeDataAsText: false, + maxSize: 7340032, }; questions.pages[0].elements.push(newQuestion); break; } - case 'integer' : { + case 'integer': { const newQuestion = { - 'type': 'text', - 'name': question.$autoname.toLowerCase(), - 'title': question.label[0], - "isRequired": question.required, - 'valueName': question.$autoname.toLowerCase(), - 'inputType': 'number' + type: 'text', + name: question.$autoname.toLowerCase(), + title: question.label[0], + isRequired: question.required, + valueName: question.$autoname.toLowerCase(), + inputType: 'number', }; questions.pages[0].elements.push(newQuestion); break; } - case 'datetime' : { + case 'datetime': { const newQuestion = { - 'type': 'text', - 'name': question.$autoname.toLowerCase(), - 'title': question.label[0], - "isRequired": question.required, - 'valueName': question.$autoname.toLowerCase(), - 'inputType': 'datetime-local' + type: 'text', + name: question.$autoname.toLowerCase(), + title: question.label[0], + isRequired: question.required, + valueName: question.$autoname.toLowerCase(), + inputType: 'datetime-local', }; questions.pages[0].elements.push(newQuestion); break; } - case 'acknowledge' : { + case 'acknowledge': { const newQuestion = { - 'type': 'boolean', - 'name': question.$autoname.toLowerCase(), - 'title': question.label[0], - 'valueName': question.$autoname.toLowerCase(), + type: 'boolean', + name: question.$autoname.toLowerCase(), + title: question.label[0], + valueName: question.$autoname.toLowerCase(), }; questions.pages[0].elements.push(newQuestion); break; } - case 'range' : { + case 'range': { const newQuestion = { - 'type': 'text', - 'name': question.$autoname.toLowerCase(), - 'title': question.label[0], - 'valueName': question.$autoname.toLowerCase(), - "inputType": "range", - 'step': question.parameters.split('step=1')[1] + type: 'text', + name: question.$autoname.toLowerCase(), + title: question.label[0], + valueName: question.$autoname.toLowerCase(), + inputType: 'range', + step: question.parameters.split('step=1')[1], }; questions.pages[0].elements.push(newQuestion); break; } } } - }); // "logoPosition": "right", // "pages": [ From a3d69f4fc7d74d126b54ca6d74b59ad8f81e5f09 Mon Sep 17 00:00:00 2001 From: RenzoPrats Date: Thu, 23 May 2024 15:30:18 -0300 Subject: [PATCH 07/27] last fix --- src/routes/proxy/index.ts | 1 - src/schema/mutation/addForm.mutation.ts | 3 +- src/utils/form/extractKoboFields.ts | 189 ++++++++++-------------- 3 files changed, 77 insertions(+), 116 deletions(-) diff --git a/src/routes/proxy/index.ts b/src/routes/proxy/index.ts index 316893617..a44cf554d 100644 --- a/src/routes/proxy/index.ts +++ b/src/routes/proxy/index.ts @@ -77,7 +77,6 @@ const proxyAPIRequest = async ( }), }) .then(async ({ data, status }) => { - console.log(data); // We are only caching the results of requests that are not user-dependent. // Otherwise, unwanted users could access cached data of other users. // As an improvement, we could include a stringified unique property of the user to the cache-key to enable user-specific cache. diff --git a/src/schema/mutation/addForm.mutation.ts b/src/schema/mutation/addForm.mutation.ts index 8dbd4ca7f..e762b1739 100644 --- a/src/schema/mutation/addForm.mutation.ts +++ b/src/schema/mutation/addForm.mutation.ts @@ -43,7 +43,6 @@ export default { async resolve(parent, args: AddFormArgs, context: Context) { graphQLAuthCheck(context); try { - console.log(args); // Check authentication const user = context.user; const ability: AppAbility = user.ability; @@ -94,6 +93,7 @@ export default { config.get('encryption.key') ).toString(CryptoJS.enc.Utf8) ); + // get kobo form data const response = await axios.get(url, { headers: { Authorization: `${settings.tokenPrefix} ${settings.token}`, @@ -103,6 +103,7 @@ export default { const choices = response.data.content.choices; const title = response.data.name; + // get structure const structure = JSON.stringify( extractKoboFields(survey, title, choices) ); diff --git a/src/utils/form/extractKoboFields.ts b/src/utils/form/extractKoboFields.ts index fbc535592..fdd137414 100644 --- a/src/utils/form/extractKoboFields.ts +++ b/src/utils/form/extractKoboFields.ts @@ -1,26 +1,35 @@ +/** + * available fields types in kobo that are compatible with oort + */ const AVAILABLE_TYPES = [ 'decimal', 'geopoint', - // 'select_multiple', + 'select_multiple', 'date', 'note', - // 'begin_score', - // 'score__row', - // 'end_score', + 'begin_score', + 'score__row', 'text', 'time', 'file', 'integer', 'datetime', 'acknowledge', - // 'begin_rank', - // 'rank__level', - // 'end_rank', + 'begin_rank', + 'rank__level', 'range', + 'image' ]; +/** + * Extract kobo form fields and convert to oort fields + * + * @param survey survey structure + * @param title title + * @param choices choices + * @returns oort survey + */ export const extractKoboFields = (survey: any, title: string, choices: any) => { - console.log(choices); const questions = { title: title, pages: [ @@ -32,6 +41,9 @@ export const extractKoboFields = (survey: any, title: string, choices: any) => { showQuestionNumbers: 'off', }; + let scoreChoiceId = ''; + let rankChoiceId = ''; + survey.map((question: any) => { if (AVAILABLE_TYPES.includes(question.type)) { switch (question.type) { @@ -58,6 +70,22 @@ export const extractKoboFields = (survey: any, title: string, choices: any) => { questions.pages[0].elements.push(newQuestion); break; } + case 'select_multiple': { + const newQuestion = { + type: 'checkbox', + name: question.$autoname.toLowerCase(), + title: question.label[0], + valueName: question.$autoname.toLowerCase(), + isRequired: question.required, + choices: + choices + .filter(choice => question.select_from_list_name === choice.list_name) + .map(choice => ({ value: choice.$autovalue, text: choice.label[0] })), + showSelectAllItem: true + }; + questions.pages[0].elements.push(newQuestion); + break; + } case 'date': { const newQuestion = { type: 'text', @@ -80,6 +108,44 @@ export const extractKoboFields = (survey: any, title: string, choices: any) => { questions.pages[0].elements.push(newQuestion); break; } + case 'begin_score': { + scoreChoiceId = question['kobo--score-choices']; + break; + } + case 'score__row': { + const newQuestion = { + type: 'radiogroup', + name: question.$autoname.toLowerCase(), + title: question.label[0], + valueName: question.$autoname.toLowerCase(), + isRequired: question.required, + choices: + choices + .filter(choice => scoreChoiceId === choice.list_name) + .map(choice => ({ value: choice.$autovalue, text: choice.label[0] })), + }; + questions.pages[0].elements.push(newQuestion); + break; + } + case 'begin_rank': { + rankChoiceId = question['kobo--rank-items']; + break; + } + case 'rank__level': { + const newQuestion = { + type: 'dropdown', + name: question.$autoname.toLowerCase(), + title: question.label[0], + valueName: question.$autoname.toLowerCase(), + isRequired: question.required, + choices: + choices + .filter(choice => rankChoiceId === choice.list_name) + .map(choice => ({ value: choice.$autovalue, text: choice.label[0] })), + }; + questions.pages[0].elements.push(newQuestion); + break; + } case 'text': { const newQuestion = { type: 'text', @@ -103,6 +169,7 @@ export const extractKoboFields = (survey: any, title: string, choices: any) => { questions.pages[0].elements.push(newQuestion); break; } + case 'image': case 'file': { const newQuestion = { type: 'file', @@ -165,111 +232,5 @@ export const extractKoboFields = (survey: any, title: string, choices: any) => { } } }); - // "logoPosition": "right", - // "pages": [ - // { - // "name": "page1", - // "elements": [ - // { - // "type": "text", - // "name": "decimal", - // "title": "decimal", - // "valueName": "decimal", - // "inputType": "number" - // }, - // { - // "type": "geospatial", - // "name": "ponto", - // "title": "Ponto", - // "valueName": "ponto" - // }, - // { - // "type": "checkbox", - // "name": "question2", - // "title": "Selecionar Multiplos", - // "valueName": "question2", - // "choices": [ - // "Item 1", - // "Item 2" - // ], - // "showSelectAllItem": true - // }, - // { - // "type": "text", - // "name": "data", - // "title": "Data", - // "valueName": "data", - // "inputType": "date" - // }, - // { - // "type": "expression", - // "name": "nota", - // "title": "Nota", - // "valueName": "nota" - // }, - // { - // "type": "radiogroup", - // "name": "avaliacao", - // "title": "Avaliação", - // "valueName": "avaliacao", - // "choices": [ - // "Item 1", - // "Item 2", - // "Item 3" - // ] - // }, - // { - // "type": "text", - // "name": "texto", - // "title": "texto", - // "valueName": "texto" - // }, - // { - // "type": "text", - // "name": "horario", - // "title": "horario", - // "valueName": "horario", - // "inputType": "time" - // }, - // { - // "type": "file", - // "name": "arquivo", - // "title": "arquivo", - // "valueName": "arquivo", - // "storeDataAsText": false, - // "maxSize": 7340032 - // }, - // { - // "type": "text", - // "name": "numero", - // "title": "numero", - // "valueName": "numero", - // "inputType": "number" - // }, - // { - // "type": "text", - // "name": "datetime", - // "title": "datetime", - // "valueName": "datetime", - // "inputType": "datetime-local" - // }, - // { - // "type": "boolean", - // "name": "reconhece", - // "title": "reconhece", - // "valueName": "reconhece" - // }, - // { - // "type": "text", - // "name": "intervalo", - // "title": "Intervalo", - // "valueName": "intervalo", - // "inputType": "range" - // } - // ] - // } - // ], - // "showQuestionNumbers": "off" - // } return questions; }; From f47fc5db6c5ea319f6b81d1e3246e32e66686a46 Mon Sep 17 00:00:00 2001 From: RenzoPrats Date: Thu, 23 May 2024 15:54:27 -0300 Subject: [PATCH 08/27] lint fixed --- src/utils/form/extractKoboFields.ts | 36 ++++++++++++++++++----------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/src/utils/form/extractKoboFields.ts b/src/utils/form/extractKoboFields.ts index fdd137414..809112638 100644 --- a/src/utils/form/extractKoboFields.ts +++ b/src/utils/form/extractKoboFields.ts @@ -18,7 +18,7 @@ const AVAILABLE_TYPES = [ 'begin_rank', 'rank__level', 'range', - 'image' + 'image', ]; /** @@ -77,11 +77,15 @@ export const extractKoboFields = (survey: any, title: string, choices: any) => { title: question.label[0], valueName: question.$autoname.toLowerCase(), isRequired: question.required, - choices: - choices - .filter(choice => question.select_from_list_name === choice.list_name) - .map(choice => ({ value: choice.$autovalue, text: choice.label[0] })), - showSelectAllItem: true + choices: choices + .filter( + (choice) => question.select_from_list_name === choice.list_name + ) + .map((choice) => ({ + value: choice.$autovalue, + text: choice.label[0], + })), + showSelectAllItem: true, }; questions.pages[0].elements.push(newQuestion); break; @@ -119,10 +123,12 @@ export const extractKoboFields = (survey: any, title: string, choices: any) => { title: question.label[0], valueName: question.$autoname.toLowerCase(), isRequired: question.required, - choices: - choices - .filter(choice => scoreChoiceId === choice.list_name) - .map(choice => ({ value: choice.$autovalue, text: choice.label[0] })), + choices: choices + .filter((choice) => scoreChoiceId === choice.list_name) + .map((choice) => ({ + value: choice.$autovalue, + text: choice.label[0], + })), }; questions.pages[0].elements.push(newQuestion); break; @@ -138,10 +144,12 @@ export const extractKoboFields = (survey: any, title: string, choices: any) => { title: question.label[0], valueName: question.$autoname.toLowerCase(), isRequired: question.required, - choices: - choices - .filter(choice => rankChoiceId === choice.list_name) - .map(choice => ({ value: choice.$autovalue, text: choice.label[0] })), + choices: choices + .filter((choice) => rankChoiceId === choice.list_name) + .map((choice) => ({ + value: choice.$autovalue, + text: choice.label[0], + })), }; questions.pages[0].elements.push(newQuestion); break; From f4cf4897dc097cc516d4c81bdd427ddd6f9fdc0e Mon Sep 17 00:00:00 2001 From: Estela Silva Date: Wed, 19 Jun 2024 16:04:08 +0200 Subject: [PATCH 09/27] added fields to form and resource, added version to form, added specific catch error when creating form from kobo, added to questions the description and to some the default value, fixed range question step --- src/i18n/en.json | 1 + src/i18n/test.json | 1 + src/schema/mutation/addForm.mutation.ts | 51 ++++++++++++++++++++++--- src/utils/form/extractKoboFields.ts | 20 +++++++++- 4 files changed, 66 insertions(+), 7 deletions(-) diff --git a/src/i18n/en.json b/src/i18n/en.json index 734a74f19..c477ad01d 100644 --- a/src/i18n/en.json +++ b/src/i18n/en.json @@ -128,6 +128,7 @@ "form": { "add": { "errors": { + "koboForm": "An error occurred when trying to create a new form from a Kobo form", "resourceDuplicated": "An existing resource with that name already exists." } }, diff --git a/src/i18n/test.json b/src/i18n/test.json index bfa77f925..5ed0a9c18 100644 --- a/src/i18n/test.json +++ b/src/i18n/test.json @@ -128,6 +128,7 @@ "form": { "add": { "errors": { + "koboForm": "******", "resourceDuplicated": "******" } }, diff --git a/src/schema/mutation/addForm.mutation.ts b/src/schema/mutation/addForm.mutation.ts index e762b1739..a0f4da5b9 100644 --- a/src/schema/mutation/addForm.mutation.ts +++ b/src/schema/mutation/addForm.mutation.ts @@ -5,8 +5,19 @@ import { GraphQLError, } from 'graphql'; import { validateGraphQLTypeName } from '@utils/validators'; -import { extractKoboFields } from '@utils/form'; -import { Resource, Form, Role, ReferenceData, ApiConfiguration } from '@models'; +import { + extractFields, + extractKoboFields, + findDuplicateFields, +} from '@utils/form'; +import { + Resource, + Form, + Role, + ReferenceData, + ApiConfiguration, + Version, +} from '@models'; import { FormType } from '../types'; import { AppAbility } from '@security/defineUserAbility'; import { status } from '@const/enumTypes'; @@ -17,6 +28,7 @@ import { Context } from '@server/apollo/context'; import axios from 'axios'; import config from 'config'; import * as CryptoJS from 'crypto-js'; +import checkDefaultFields from '@utils/form/checkDefaultFields'; /** Arguments for the addForm mutation */ type AddFormArgs = { @@ -103,15 +115,32 @@ export default { const choices = response.data.content.choices; const title = response.data.name; - // get structure + // Get structure from the kobo form const structure = JSON.stringify( extractKoboFields(survey, title, choices) ); - // create resource + // Extract fields + const fields = []; + const structureObj = JSON.parse(structure); + for (const page of structureObj.pages) { + await extractFields(page, fields, true); + findDuplicateFields(fields); + } + // Check if default fields are used + checkDefaultFields(fields); + + // Create version with structure + const version = new Version({ + data: structure, + }); + await version.save(); + + // create resource and form const resource = new Resource({ name: args.name, permissions: defaultResourcePermissions, + fields, }); await resource.save(); const form = new Form({ @@ -121,11 +150,21 @@ export default { resource, core: true, permissions: defaultFormPermissions, - structure: structure, + structure, + fields, + versions: [version._id], }); await form.save(); return form; - } else if (!args.resource) { + } + } catch (err) { + logger.error(err.message, { stack: err.stack }); + throw new GraphQLError( + context.i18next.t('mutations.form.add.errors.koboForm:') + ); + } + try { + if (!args.resource) { // Check permission to create resource if (ability.cannot('create', 'Resource')) { throw new GraphQLError( diff --git a/src/utils/form/extractKoboFields.ts b/src/utils/form/extractKoboFields.ts index 809112638..7eb522e6f 100644 --- a/src/utils/form/extractKoboFields.ts +++ b/src/utils/form/extractKoboFields.ts @@ -55,6 +55,8 @@ export const extractKoboFields = (survey: any, title: string, choices: any) => { valueName: question.$autoname.toLowerCase(), isRequired: question.required, inputType: 'number', + ...(question.hint && { description: question.hint[0] }), + ...(question.default && { defaultValue: question.default }), }; questions.pages[0].elements.push(newQuestion); break; @@ -66,6 +68,7 @@ export const extractKoboFields = (survey: any, title: string, choices: any) => { title: question.label[0], valueName: question.$autoname.toLowerCase(), isRequired: question.required, + ...(question.hint && { description: question.hint[0] }), }; questions.pages[0].elements.push(newQuestion); break; @@ -86,6 +89,7 @@ export const extractKoboFields = (survey: any, title: string, choices: any) => { text: choice.label[0], })), showSelectAllItem: true, + ...(question.hint && { description: question.hint[0] }), }; questions.pages[0].elements.push(newQuestion); break; @@ -98,6 +102,7 @@ export const extractKoboFields = (survey: any, title: string, choices: any) => { isRequired: question.required, valueName: question.$autoname.toLowerCase(), inputType: 'date', + ...(question.hint && { description: question.hint[0] }), }; questions.pages[0].elements.push(newQuestion); break; @@ -108,6 +113,7 @@ export const extractKoboFields = (survey: any, title: string, choices: any) => { name: question.$autoname.toLowerCase(), title: question.label[0], valueName: question.$autoname.toLowerCase(), + ...(question.hint && { description: question.hint[0] }), }; questions.pages[0].elements.push(newQuestion); break; @@ -129,6 +135,7 @@ export const extractKoboFields = (survey: any, title: string, choices: any) => { value: choice.$autovalue, text: choice.label[0], })), + ...(question.hint && { description: question.hint[0] }), }; questions.pages[0].elements.push(newQuestion); break; @@ -150,6 +157,7 @@ export const extractKoboFields = (survey: any, title: string, choices: any) => { value: choice.$autovalue, text: choice.label[0], })), + ...(question.hint && { description: question.hint[0] }), }; questions.pages[0].elements.push(newQuestion); break; @@ -161,6 +169,8 @@ export const extractKoboFields = (survey: any, title: string, choices: any) => { title: question.label[0], isRequired: question.required, valueName: question.$autoname.toLowerCase(), + ...(question.hint && { description: question.hint[0] }), + ...(question.default && { defaultValue: question.default }), }; questions.pages[0].elements.push(newQuestion); break; @@ -173,6 +183,7 @@ export const extractKoboFields = (survey: any, title: string, choices: any) => { isRequired: question.required, valueName: question.$autoname.toLowerCase(), inputType: 'time', + ...(question.hint && { description: question.hint[0] }), }; questions.pages[0].elements.push(newQuestion); break; @@ -187,6 +198,7 @@ export const extractKoboFields = (survey: any, title: string, choices: any) => { valueName: question.$autoname.toLowerCase(), storeDataAsText: false, maxSize: 7340032, + ...(question.hint && { description: question.hint[0] }), }; questions.pages[0].elements.push(newQuestion); break; @@ -199,6 +211,8 @@ export const extractKoboFields = (survey: any, title: string, choices: any) => { isRequired: question.required, valueName: question.$autoname.toLowerCase(), inputType: 'number', + ...(question.hint && { description: question.hint[0] }), + ...(question.default && { defaultValue: question.default }), }; questions.pages[0].elements.push(newQuestion); break; @@ -211,6 +225,7 @@ export const extractKoboFields = (survey: any, title: string, choices: any) => { isRequired: question.required, valueName: question.$autoname.toLowerCase(), inputType: 'datetime-local', + ...(question.hint && { description: question.hint[0] }), }; questions.pages[0].elements.push(newQuestion); break; @@ -221,6 +236,7 @@ export const extractKoboFields = (survey: any, title: string, choices: any) => { name: question.$autoname.toLowerCase(), title: question.label[0], valueName: question.$autoname.toLowerCase(), + ...(question.hint && { description: question.hint[0] }), }; questions.pages[0].elements.push(newQuestion); break; @@ -232,7 +248,9 @@ export const extractKoboFields = (survey: any, title: string, choices: any) => { title: question.label[0], valueName: question.$autoname.toLowerCase(), inputType: 'range', - step: question.parameters.split('step=1')[1], + step: question.parameters.split('step=')[1], + ...(question.hint && { description: question.hint[0] }), + ...(question.default && { defaultValue: question.default }), }; questions.pages[0].elements.push(newQuestion); break; From fbac78650360a10ff3a80e71e472537892a58a81 Mon Sep 17 00:00:00 2001 From: Estela Silva Date: Wed, 19 Jun 2024 17:14:48 +0200 Subject: [PATCH 10/27] added questions types: audio, video, geoshape and select_one & added question property randomize --- src/utils/form/extractKoboFields.ts | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/src/utils/form/extractKoboFields.ts b/src/utils/form/extractKoboFields.ts index 7eb522e6f..4bc10e1b5 100644 --- a/src/utils/form/extractKoboFields.ts +++ b/src/utils/form/extractKoboFields.ts @@ -5,6 +5,7 @@ const AVAILABLE_TYPES = [ 'decimal', 'geopoint', 'select_multiple', + 'select_one', 'date', 'note', 'begin_score', @@ -19,6 +20,9 @@ const AVAILABLE_TYPES = [ 'rank__level', 'range', 'image', + 'audio', + 'video', + 'geoshape', ]; /** @@ -61,6 +65,7 @@ export const extractKoboFields = (survey: any, title: string, choices: any) => { questions.pages[0].elements.push(newQuestion); break; } + case 'geoshape': case 'geopoint': { const newQuestion = { type: 'geospatial', @@ -73,9 +78,14 @@ export const extractKoboFields = (survey: any, title: string, choices: any) => { questions.pages[0].elements.push(newQuestion); break; } + case 'select_one': case 'select_multiple': { const newQuestion = { - type: 'checkbox', + type: + question.type === 'select_multiple' ? 'checkbox' : 'radiogroup', + ...(question.type === 'select_multiple' && { + showSelectAllItem: true, + }), name: question.$autoname.toLowerCase(), title: question.label[0], valueName: question.$autoname.toLowerCase(), @@ -88,8 +98,11 @@ export const extractKoboFields = (survey: any, title: string, choices: any) => { value: choice.$autovalue, text: choice.label[0], })), - showSelectAllItem: true, ...(question.hint && { description: question.hint[0] }), + ...(question.parameters && + question.parameters.split('randomize=')[1]?.includes('true') && { + choicesOrder: 'random', + }), }; questions.pages[0].elements.push(newQuestion); break; @@ -188,6 +201,8 @@ export const extractKoboFields = (survey: any, title: string, choices: any) => { questions.pages[0].elements.push(newQuestion); break; } + case 'audio': + case 'video': case 'image': case 'file': { const newQuestion = { From 8b214721245a89521aee00507f6ddfc4b4f48b70 Mon Sep 17 00:00:00 2001 From: Estela Silva Date: Thu, 20 Jun 2024 15:32:39 +0200 Subject: [PATCH 11/27] created mapKoboSkipLogic method to map kobo expression to the questions property visibleIf --- src/utils/form/extractKoboFields.ts | 86 +++++++++++++++++++++++++++++ 1 file changed, 86 insertions(+) diff --git a/src/utils/form/extractKoboFields.ts b/src/utils/form/extractKoboFields.ts index 4bc10e1b5..b4bba190e 100644 --- a/src/utils/form/extractKoboFields.ts +++ b/src/utils/form/extractKoboFields.ts @@ -25,6 +25,50 @@ const AVAILABLE_TYPES = [ 'geoshape', ]; +/** + * Maps the "Skip Logic" expression of kobo questions (used to display the question only if the expression is true) + * to a condition that will work with the visibleIf property of the SurveyJS. + * + * The numeric operators (Greater than >, Less than <, Greater than or equal to >=, Less than or equal to <=) are used the in same way in Kobo and SurveyJS. + * + * @param koboExpression the initial kobo logic expression + * @returns the mapped logic expression that will work on the SurveyJS form + */ +const mapKoboSkipLogic = (koboExpression: string) => { + // Not contains + koboExpression = koboExpression.replace( + /not\(selected\(\$\{(\w+)\}, '(.*?)'\)\)/g, + "{$1} notcontains '$2'" + ); + // Empty + koboExpression = koboExpression.replace(/\$\{(\w+)\} = ''/g, '{$1} empty'); + // Equal to + koboExpression = koboExpression.replace( + /\$\{(\w+)\} = '(.*?)'/g, + "({$1} = '$2')" + ); + // Contains + koboExpression = koboExpression.replace( + /selected\(\$\{(\w+)\}, '(.*?)'\)/g, + "{$1} contains '$2'" + ); + // No empty + koboExpression = koboExpression.replace( + /\$\{(\w+)\} != ''/g, + '{$1} notempty' + ); + // Not equal to + koboExpression = koboExpression.replace( + /\$\{(\w+)\} != '(.*?)'/g, + "{$1} <> '$2'" + ); + // Date values + koboExpression = koboExpression.replace(/date\('(.*?)'\)/g, "'$1'"); + // Replace any remaining ${variable} to {variable} + koboExpression = koboExpression.replace(/\$\{(\w+)\}/g, '{$1}'); + return koboExpression; +}; + /** * Extract kobo form fields and convert to oort fields * @@ -61,6 +105,9 @@ export const extractKoboFields = (survey: any, title: string, choices: any) => { inputType: 'number', ...(question.hint && { description: question.hint[0] }), ...(question.default && { defaultValue: question.default }), + ...(question.relevant && { + visibleIf: mapKoboSkipLogic(question.relevant), + }), }; questions.pages[0].elements.push(newQuestion); break; @@ -74,6 +121,9 @@ export const extractKoboFields = (survey: any, title: string, choices: any) => { valueName: question.$autoname.toLowerCase(), isRequired: question.required, ...(question.hint && { description: question.hint[0] }), + ...(question.relevant && { + visibleIf: mapKoboSkipLogic(question.relevant), + }), }; questions.pages[0].elements.push(newQuestion); break; @@ -103,6 +153,9 @@ export const extractKoboFields = (survey: any, title: string, choices: any) => { question.parameters.split('randomize=')[1]?.includes('true') && { choicesOrder: 'random', }), + ...(question.relevant && { + visibleIf: mapKoboSkipLogic(question.relevant), + }), }; questions.pages[0].elements.push(newQuestion); break; @@ -116,6 +169,9 @@ export const extractKoboFields = (survey: any, title: string, choices: any) => { valueName: question.$autoname.toLowerCase(), inputType: 'date', ...(question.hint && { description: question.hint[0] }), + ...(question.relevant && { + visibleIf: mapKoboSkipLogic(question.relevant), + }), }; questions.pages[0].elements.push(newQuestion); break; @@ -127,6 +183,9 @@ export const extractKoboFields = (survey: any, title: string, choices: any) => { title: question.label[0], valueName: question.$autoname.toLowerCase(), ...(question.hint && { description: question.hint[0] }), + ...(question.relevant && { + visibleIf: mapKoboSkipLogic(question.relevant), + }), }; questions.pages[0].elements.push(newQuestion); break; @@ -149,6 +208,9 @@ export const extractKoboFields = (survey: any, title: string, choices: any) => { text: choice.label[0], })), ...(question.hint && { description: question.hint[0] }), + ...(question.relevant && { + visibleIf: mapKoboSkipLogic(question.relevant), + }), }; questions.pages[0].elements.push(newQuestion); break; @@ -171,6 +233,9 @@ export const extractKoboFields = (survey: any, title: string, choices: any) => { text: choice.label[0], })), ...(question.hint && { description: question.hint[0] }), + ...(question.relevant && { + visibleIf: mapKoboSkipLogic(question.relevant), + }), }; questions.pages[0].elements.push(newQuestion); break; @@ -184,6 +249,9 @@ export const extractKoboFields = (survey: any, title: string, choices: any) => { valueName: question.$autoname.toLowerCase(), ...(question.hint && { description: question.hint[0] }), ...(question.default && { defaultValue: question.default }), + ...(question.relevant && { + visibleIf: mapKoboSkipLogic(question.relevant), + }), }; questions.pages[0].elements.push(newQuestion); break; @@ -197,6 +265,9 @@ export const extractKoboFields = (survey: any, title: string, choices: any) => { valueName: question.$autoname.toLowerCase(), inputType: 'time', ...(question.hint && { description: question.hint[0] }), + ...(question.relevant && { + visibleIf: mapKoboSkipLogic(question.relevant), + }), }; questions.pages[0].elements.push(newQuestion); break; @@ -214,6 +285,9 @@ export const extractKoboFields = (survey: any, title: string, choices: any) => { storeDataAsText: false, maxSize: 7340032, ...(question.hint && { description: question.hint[0] }), + ...(question.relevant && { + visibleIf: mapKoboSkipLogic(question.relevant), + }), }; questions.pages[0].elements.push(newQuestion); break; @@ -228,6 +302,9 @@ export const extractKoboFields = (survey: any, title: string, choices: any) => { inputType: 'number', ...(question.hint && { description: question.hint[0] }), ...(question.default && { defaultValue: question.default }), + ...(question.relevant && { + visibleIf: mapKoboSkipLogic(question.relevant), + }), }; questions.pages[0].elements.push(newQuestion); break; @@ -241,6 +318,9 @@ export const extractKoboFields = (survey: any, title: string, choices: any) => { valueName: question.$autoname.toLowerCase(), inputType: 'datetime-local', ...(question.hint && { description: question.hint[0] }), + ...(question.relevant && { + visibleIf: mapKoboSkipLogic(question.relevant), + }), }; questions.pages[0].elements.push(newQuestion); break; @@ -252,6 +332,9 @@ export const extractKoboFields = (survey: any, title: string, choices: any) => { title: question.label[0], valueName: question.$autoname.toLowerCase(), ...(question.hint && { description: question.hint[0] }), + ...(question.relevant && { + visibleIf: mapKoboSkipLogic(question.relevant), + }), }; questions.pages[0].elements.push(newQuestion); break; @@ -266,6 +349,9 @@ export const extractKoboFields = (survey: any, title: string, choices: any) => { step: question.parameters.split('step=')[1], ...(question.hint && { description: question.hint[0] }), ...(question.default && { defaultValue: question.default }), + ...(question.relevant && { + visibleIf: mapKoboSkipLogic(question.relevant), + }), }; questions.pages[0].elements.push(newQuestion); break; From 849a4916c9cf13d84b75bc54f941e9efafbd655e Mon Sep 17 00:00:00 2001 From: Estela Silva Date: Thu, 20 Jun 2024 17:27:25 +0200 Subject: [PATCH 12/27] updated mapKoboExpression method & wip: map kobo constraints into surveyjs validators --- src/utils/form/extractKoboFields.ts | 70 ++++++++++++++++++++++------- 1 file changed, 53 insertions(+), 17 deletions(-) diff --git a/src/utils/form/extractKoboFields.ts b/src/utils/form/extractKoboFields.ts index b4bba190e..0aac2e22e 100644 --- a/src/utils/form/extractKoboFields.ts +++ b/src/utils/form/extractKoboFields.ts @@ -26,15 +26,21 @@ const AVAILABLE_TYPES = [ ]; /** - * Maps the "Skip Logic" expression of kobo questions (used to display the question only if the expression is true) - * to a condition that will work with the visibleIf property of the SurveyJS. + * Maps expressions from kobo questions to a expression format that will work on the SurveyJS. * * The numeric operators (Greater than >, Less than <, Greater than or equal to >=, Less than or equal to <=) are used the in same way in Kobo and SurveyJS. * * @param koboExpression the initial kobo logic expression + * @param questionName name of the question to replace in the expression * @returns the mapped logic expression that will work on the SurveyJS form */ -const mapKoboSkipLogic = (koboExpression: string) => { +const mapKoboExpression = (koboExpression: string, questionName?: string) => { + // Replace . with {questionName} + if (questionName) { + // Expressions in Kobo can have ' . ' to indicate that the expression is about the question in which it is defined. + // Example: a Validation Criteria can be ". > 5 ": the value of the question itself must be greater than 5 + koboExpression = koboExpression.replace(/\./g, `{${questionName}}`); + } // Not contains koboExpression = koboExpression.replace( /not\(selected\(\$\{(\w+)\}, '(.*?)'\)\)/g, @@ -91,6 +97,7 @@ export const extractKoboFields = (survey: any, title: string, choices: any) => { let scoreChoiceId = ''; let rankChoiceId = ''; + let constraintMessage = ''; survey.map((question: any) => { if (AVAILABLE_TYPES.includes(question.type)) { @@ -106,7 +113,7 @@ export const extractKoboFields = (survey: any, title: string, choices: any) => { ...(question.hint && { description: question.hint[0] }), ...(question.default && { defaultValue: question.default }), ...(question.relevant && { - visibleIf: mapKoboSkipLogic(question.relevant), + visibleIf: mapKoboExpression(question.relevant), }), }; questions.pages[0].elements.push(newQuestion); @@ -122,7 +129,7 @@ export const extractKoboFields = (survey: any, title: string, choices: any) => { isRequired: question.required, ...(question.hint && { description: question.hint[0] }), ...(question.relevant && { - visibleIf: mapKoboSkipLogic(question.relevant), + visibleIf: mapKoboExpression(question.relevant), }), }; questions.pages[0].elements.push(newQuestion); @@ -154,7 +161,7 @@ export const extractKoboFields = (survey: any, title: string, choices: any) => { choicesOrder: 'random', }), ...(question.relevant && { - visibleIf: mapKoboSkipLogic(question.relevant), + visibleIf: mapKoboExpression(question.relevant), }), }; questions.pages[0].elements.push(newQuestion); @@ -170,7 +177,7 @@ export const extractKoboFields = (survey: any, title: string, choices: any) => { inputType: 'date', ...(question.hint && { description: question.hint[0] }), ...(question.relevant && { - visibleIf: mapKoboSkipLogic(question.relevant), + visibleIf: mapKoboExpression(question.relevant), }), }; questions.pages[0].elements.push(newQuestion); @@ -184,7 +191,7 @@ export const extractKoboFields = (survey: any, title: string, choices: any) => { valueName: question.$autoname.toLowerCase(), ...(question.hint && { description: question.hint[0] }), ...(question.relevant && { - visibleIf: mapKoboSkipLogic(question.relevant), + visibleIf: mapKoboExpression(question.relevant), }), }; questions.pages[0].elements.push(newQuestion); @@ -209,14 +216,16 @@ export const extractKoboFields = (survey: any, title: string, choices: any) => { })), ...(question.hint && { description: question.hint[0] }), ...(question.relevant && { - visibleIf: mapKoboSkipLogic(question.relevant), + visibleIf: mapKoboExpression(question.relevant), }), + // This question does not have Validation Criteria settings }; questions.pages[0].elements.push(newQuestion); break; } case 'begin_rank': { rankChoiceId = question['kobo--rank-items']; + constraintMessage = question['kobo--rank-constraint-message']; break; } case 'rank__level': { @@ -234,8 +243,9 @@ export const extractKoboFields = (survey: any, title: string, choices: any) => { })), ...(question.hint && { description: question.hint[0] }), ...(question.relevant && { - visibleIf: mapKoboSkipLogic(question.relevant), + visibleIf: mapKoboExpression(question.relevant), }), + // This question does not have Validation Criteria settings }; questions.pages[0].elements.push(newQuestion); break; @@ -250,7 +260,20 @@ export const extractKoboFields = (survey: any, title: string, choices: any) => { ...(question.hint && { description: question.hint[0] }), ...(question.default && { defaultValue: question.default }), ...(question.relevant && { - visibleIf: mapKoboSkipLogic(question.relevant), + visibleIf: mapKoboExpression(question.relevant), + }), + ...(question.constraint && { + validators: [ + { + type: 'expression', + text: question.constraint_message ?? '', + expression: mapKoboExpression( + question.constraint, + question.$autoname.toLowerCase() + ), + }, + ], + validateOnValueChange: true, }), }; questions.pages[0].elements.push(newQuestion); @@ -266,7 +289,7 @@ export const extractKoboFields = (survey: any, title: string, choices: any) => { inputType: 'time', ...(question.hint && { description: question.hint[0] }), ...(question.relevant && { - visibleIf: mapKoboSkipLogic(question.relevant), + visibleIf: mapKoboExpression(question.relevant), }), }; questions.pages[0].elements.push(newQuestion); @@ -286,7 +309,7 @@ export const extractKoboFields = (survey: any, title: string, choices: any) => { maxSize: 7340032, ...(question.hint && { description: question.hint[0] }), ...(question.relevant && { - visibleIf: mapKoboSkipLogic(question.relevant), + visibleIf: mapKoboExpression(question.relevant), }), }; questions.pages[0].elements.push(newQuestion); @@ -303,7 +326,20 @@ export const extractKoboFields = (survey: any, title: string, choices: any) => { ...(question.hint && { description: question.hint[0] }), ...(question.default && { defaultValue: question.default }), ...(question.relevant && { - visibleIf: mapKoboSkipLogic(question.relevant), + visibleIf: mapKoboExpression(question.relevant), + }), + ...(question.constraint && { + validators: [ + { + type: 'expression', + text: question.constraint_message ?? '', + expression: mapKoboExpression( + question.constraint, + question.$autoname.toLowerCase() + ), + }, + ], + validateOnValueChange: true, }), }; questions.pages[0].elements.push(newQuestion); @@ -319,7 +355,7 @@ export const extractKoboFields = (survey: any, title: string, choices: any) => { inputType: 'datetime-local', ...(question.hint && { description: question.hint[0] }), ...(question.relevant && { - visibleIf: mapKoboSkipLogic(question.relevant), + visibleIf: mapKoboExpression(question.relevant), }), }; questions.pages[0].elements.push(newQuestion); @@ -333,7 +369,7 @@ export const extractKoboFields = (survey: any, title: string, choices: any) => { valueName: question.$autoname.toLowerCase(), ...(question.hint && { description: question.hint[0] }), ...(question.relevant && { - visibleIf: mapKoboSkipLogic(question.relevant), + visibleIf: mapKoboExpression(question.relevant), }), }; questions.pages[0].elements.push(newQuestion); @@ -350,7 +386,7 @@ export const extractKoboFields = (survey: any, title: string, choices: any) => { ...(question.hint && { description: question.hint[0] }), ...(question.default && { defaultValue: question.default }), ...(question.relevant && { - visibleIf: mapKoboSkipLogic(question.relevant), + visibleIf: mapKoboExpression(question.relevant), }), }; questions.pages[0].elements.push(newQuestion); From 394eae9bfd25d8dbb6ebebee667fbd543a20edc0 Mon Sep 17 00:00:00 2001 From: Estela Silva Date: Mon, 24 Jun 2024 12:00:17 -0300 Subject: [PATCH 13/27] added validators property to questions --- src/utils/form/extractKoboFields.ts | 68 +++++++++++++++++------------ 1 file changed, 39 insertions(+), 29 deletions(-) diff --git a/src/utils/form/extractKoboFields.ts b/src/utils/form/extractKoboFields.ts index 0aac2e22e..1a1cb307d 100644 --- a/src/utils/form/extractKoboFields.ts +++ b/src/utils/form/extractKoboFields.ts @@ -75,6 +75,32 @@ const mapKoboExpression = (koboExpression: string, questionName?: string) => { return koboExpression; }; +/** + * Extract from a Kobo question the constraint properties and return the validators property for an SurveyJS question + * + * @param question Kobo question object + * @returns validators property for an SurveyJS question + */ +const validators = (question: any) => { + return { + validators: [ + { + type: 'expression', + text: question.constraint_message + ? typeof question.constraint_message === 'string' + ? question.constraint_message + : question.constraint_message[0] + : '', + expression: mapKoboExpression( + question.constraint, + question.$autoname.toLowerCase() + ), + }, + ], + validateOnValueChange: true, + }; +}; + /** * Extract kobo form fields and convert to oort fields * @@ -97,7 +123,6 @@ export const extractKoboFields = (survey: any, title: string, choices: any) => { let scoreChoiceId = ''; let rankChoiceId = ''; - let constraintMessage = ''; survey.map((question: any) => { if (AVAILABLE_TYPES.includes(question.type)) { @@ -115,6 +140,7 @@ export const extractKoboFields = (survey: any, title: string, choices: any) => { ...(question.relevant && { visibleIf: mapKoboExpression(question.relevant), }), + ...(question.constraint && validators(question)), }; questions.pages[0].elements.push(newQuestion); break; @@ -131,6 +157,7 @@ export const extractKoboFields = (survey: any, title: string, choices: any) => { ...(question.relevant && { visibleIf: mapKoboExpression(question.relevant), }), + ...(question.constraint && validators(question)), }; questions.pages[0].elements.push(newQuestion); break; @@ -163,6 +190,7 @@ export const extractKoboFields = (survey: any, title: string, choices: any) => { ...(question.relevant && { visibleIf: mapKoboExpression(question.relevant), }), + ...(question.constraint && validators(question)), }; questions.pages[0].elements.push(newQuestion); break; @@ -179,6 +207,7 @@ export const extractKoboFields = (survey: any, title: string, choices: any) => { ...(question.relevant && { visibleIf: mapKoboExpression(question.relevant), }), + ...(question.constraint && validators(question)), }; questions.pages[0].elements.push(newQuestion); break; @@ -193,6 +222,7 @@ export const extractKoboFields = (survey: any, title: string, choices: any) => { ...(question.relevant && { visibleIf: mapKoboExpression(question.relevant), }), + ...(question.constraint && validators(question)), }; questions.pages[0].elements.push(newQuestion); break; @@ -218,14 +248,13 @@ export const extractKoboFields = (survey: any, title: string, choices: any) => { ...(question.relevant && { visibleIf: mapKoboExpression(question.relevant), }), - // This question does not have Validation Criteria settings + // This question does not have Validation Criteria settings (validators property) }; questions.pages[0].elements.push(newQuestion); break; } case 'begin_rank': { rankChoiceId = question['kobo--rank-items']; - constraintMessage = question['kobo--rank-constraint-message']; break; } case 'rank__level': { @@ -262,19 +291,7 @@ export const extractKoboFields = (survey: any, title: string, choices: any) => { ...(question.relevant && { visibleIf: mapKoboExpression(question.relevant), }), - ...(question.constraint && { - validators: [ - { - type: 'expression', - text: question.constraint_message ?? '', - expression: mapKoboExpression( - question.constraint, - question.$autoname.toLowerCase() - ), - }, - ], - validateOnValueChange: true, - }), + ...(question.constraint && validators(question)), }; questions.pages[0].elements.push(newQuestion); break; @@ -291,6 +308,7 @@ export const extractKoboFields = (survey: any, title: string, choices: any) => { ...(question.relevant && { visibleIf: mapKoboExpression(question.relevant), }), + ...(question.constraint && validators(question)), }; questions.pages[0].elements.push(newQuestion); break; @@ -311,6 +329,7 @@ export const extractKoboFields = (survey: any, title: string, choices: any) => { ...(question.relevant && { visibleIf: mapKoboExpression(question.relevant), }), + ...(question.constraint && validators(question)), }; questions.pages[0].elements.push(newQuestion); break; @@ -328,19 +347,7 @@ export const extractKoboFields = (survey: any, title: string, choices: any) => { ...(question.relevant && { visibleIf: mapKoboExpression(question.relevant), }), - ...(question.constraint && { - validators: [ - { - type: 'expression', - text: question.constraint_message ?? '', - expression: mapKoboExpression( - question.constraint, - question.$autoname.toLowerCase() - ), - }, - ], - validateOnValueChange: true, - }), + ...(question.constraint && validators(question)), }; questions.pages[0].elements.push(newQuestion); break; @@ -357,6 +364,7 @@ export const extractKoboFields = (survey: any, title: string, choices: any) => { ...(question.relevant && { visibleIf: mapKoboExpression(question.relevant), }), + ...(question.constraint && validators(question)), }; questions.pages[0].elements.push(newQuestion); break; @@ -371,6 +379,7 @@ export const extractKoboFields = (survey: any, title: string, choices: any) => { ...(question.relevant && { visibleIf: mapKoboExpression(question.relevant), }), + ...(question.constraint && validators(question)), }; questions.pages[0].elements.push(newQuestion); break; @@ -388,6 +397,7 @@ export const extractKoboFields = (survey: any, title: string, choices: any) => { ...(question.relevant && { visibleIf: mapKoboExpression(question.relevant), }), + ...(question.constraint && validators(question)), }; questions.pages[0].elements.push(newQuestion); break; From 684a0e1ab2dfdf5b58cdeeacba829fdc980fdd29 Mon Sep 17 00:00:00 2001 From: Estela Silva Date: Mon, 24 Jun 2024 14:08:39 -0300 Subject: [PATCH 14/27] added calculate questions, added mapping for iif function & refactored properties constructor --- src/utils/form/extractKoboFields.ts | 195 ++++++++-------------------- 1 file changed, 55 insertions(+), 140 deletions(-) diff --git a/src/utils/form/extractKoboFields.ts b/src/utils/form/extractKoboFields.ts index 1a1cb307d..3a33ef011 100644 --- a/src/utils/form/extractKoboFields.ts +++ b/src/utils/form/extractKoboFields.ts @@ -23,6 +23,7 @@ const AVAILABLE_TYPES = [ 'audio', 'video', 'geoshape', + 'calculate', ]; /** @@ -68,6 +69,8 @@ const mapKoboExpression = (koboExpression: string, questionName?: string) => { /\$\{(\w+)\} != '(.*?)'/g, "{$1} <> '$2'" ); + // Replace if with iif + koboExpression = koboExpression.replace(/if\(/g, 'iif('); // Date values koboExpression = koboExpression.replace(/date\('(.*?)'\)/g, "'$1'"); // Replace any remaining ${variable} to {variable} @@ -101,6 +104,31 @@ const validators = (question: any) => { }; }; +/** + * Extract from a Kobo question the common properties in a object for a SurveyJS question + * + * @param question Kobo question object + * @param type type of the questions + * @param title optional title + * @returns the common properties in a object for a SurveyJS question extracted from the Kobo question + */ +const commonProperties = (question: any, type: string, title?: string) => { + return { + type, + name: question.$autoname.toLowerCase(), + title: title ?? question.label[0], + valueName: question.$autoname.toLowerCase(), + isRequired: question.required, + ...(question.hint && { description: question.hint[0] }), + // TODO: make sure that the defaultValue works for all type of questions. Works for sure for the questions type: integer, text, decimal, range. + ...(question.default && { defaultValue: question.default }), + ...(question.relevant && { + visibleIf: mapKoboExpression(question.relevant), + }), + ...(question.constraint && validators(question)), + }; +}; + /** * Extract kobo form fields and convert to oort fields * @@ -129,18 +157,8 @@ export const extractKoboFields = (survey: any, title: string, choices: any) => { switch (question.type) { case 'decimal': { const newQuestion = { - type: 'text', - name: question.$autoname.toLowerCase(), - title: question.label[0], - valueName: question.$autoname.toLowerCase(), - isRequired: question.required, + ...commonProperties(question, 'text'), inputType: 'number', - ...(question.hint && { description: question.hint[0] }), - ...(question.default && { defaultValue: question.default }), - ...(question.relevant && { - visibleIf: mapKoboExpression(question.relevant), - }), - ...(question.constraint && validators(question)), }; questions.pages[0].elements.push(newQuestion); break; @@ -148,16 +166,7 @@ export const extractKoboFields = (survey: any, title: string, choices: any) => { case 'geoshape': case 'geopoint': { const newQuestion = { - type: 'geospatial', - name: question.$autoname.toLowerCase(), - title: question.label[0], - valueName: question.$autoname.toLowerCase(), - isRequired: question.required, - ...(question.hint && { description: question.hint[0] }), - ...(question.relevant && { - visibleIf: mapKoboExpression(question.relevant), - }), - ...(question.constraint && validators(question)), + ...commonProperties(question, 'geospatial'), }; questions.pages[0].elements.push(newQuestion); break; @@ -165,15 +174,13 @@ export const extractKoboFields = (survey: any, title: string, choices: any) => { case 'select_one': case 'select_multiple': { const newQuestion = { - type: - question.type === 'select_multiple' ? 'checkbox' : 'radiogroup', + ...commonProperties( + question, + question.type === 'select_multiple' ? 'checkbox' : 'radiogroup' + ), ...(question.type === 'select_multiple' && { showSelectAllItem: true, }), - name: question.$autoname.toLowerCase(), - title: question.label[0], - valueName: question.$autoname.toLowerCase(), - isRequired: question.required, choices: choices .filter( (choice) => question.select_from_list_name === choice.list_name @@ -182,47 +189,25 @@ export const extractKoboFields = (survey: any, title: string, choices: any) => { value: choice.$autovalue, text: choice.label[0], })), - ...(question.hint && { description: question.hint[0] }), ...(question.parameters && question.parameters.split('randomize=')[1]?.includes('true') && { choicesOrder: 'random', }), - ...(question.relevant && { - visibleIf: mapKoboExpression(question.relevant), - }), - ...(question.constraint && validators(question)), }; questions.pages[0].elements.push(newQuestion); break; } case 'date': { const newQuestion = { - type: 'text', - name: question.$autoname.toLowerCase(), - title: question.label[0], - isRequired: question.required, - valueName: question.$autoname.toLowerCase(), + ...commonProperties(question, 'text'), inputType: 'date', - ...(question.hint && { description: question.hint[0] }), - ...(question.relevant && { - visibleIf: mapKoboExpression(question.relevant), - }), - ...(question.constraint && validators(question)), }; questions.pages[0].elements.push(newQuestion); break; } case 'note': { const newQuestion = { - type: 'expression', - name: question.$autoname.toLowerCase(), - title: question.label[0], - valueName: question.$autoname.toLowerCase(), - ...(question.hint && { description: question.hint[0] }), - ...(question.relevant && { - visibleIf: mapKoboExpression(question.relevant), - }), - ...(question.constraint && validators(question)), + ...commonProperties(question, 'expression'), }; questions.pages[0].elements.push(newQuestion); break; @@ -233,21 +218,13 @@ export const extractKoboFields = (survey: any, title: string, choices: any) => { } case 'score__row': { const newQuestion = { - type: 'radiogroup', - name: question.$autoname.toLowerCase(), - title: question.label[0], - valueName: question.$autoname.toLowerCase(), - isRequired: question.required, + ...commonProperties(question, 'radiogroup'), choices: choices .filter((choice) => scoreChoiceId === choice.list_name) .map((choice) => ({ value: choice.$autovalue, text: choice.label[0], })), - ...(question.hint && { description: question.hint[0] }), - ...(question.relevant && { - visibleIf: mapKoboExpression(question.relevant), - }), // This question does not have Validation Criteria settings (validators property) }; questions.pages[0].elements.push(newQuestion); @@ -259,21 +236,13 @@ export const extractKoboFields = (survey: any, title: string, choices: any) => { } case 'rank__level': { const newQuestion = { - type: 'dropdown', - name: question.$autoname.toLowerCase(), - title: question.label[0], - valueName: question.$autoname.toLowerCase(), - isRequired: question.required, + ...commonProperties(question, 'dropdown'), choices: choices .filter((choice) => rankChoiceId === choice.list_name) .map((choice) => ({ value: choice.$autovalue, text: choice.label[0], })), - ...(question.hint && { description: question.hint[0] }), - ...(question.relevant && { - visibleIf: mapKoboExpression(question.relevant), - }), // This question does not have Validation Criteria settings }; questions.pages[0].elements.push(newQuestion); @@ -281,34 +250,15 @@ export const extractKoboFields = (survey: any, title: string, choices: any) => { } case 'text': { const newQuestion = { - type: 'text', - name: question.$autoname.toLowerCase(), - title: question.label[0], - isRequired: question.required, - valueName: question.$autoname.toLowerCase(), - ...(question.hint && { description: question.hint[0] }), - ...(question.default && { defaultValue: question.default }), - ...(question.relevant && { - visibleIf: mapKoboExpression(question.relevant), - }), - ...(question.constraint && validators(question)), + ...commonProperties(question, 'text'), }; questions.pages[0].elements.push(newQuestion); break; } case 'time': { const newQuestion = { - type: 'text', - name: question.$autoname.toLowerCase(), - title: question.label[0], - isRequired: question.required, - valueName: question.$autoname.toLowerCase(), + ...commonProperties(question, 'text'), inputType: 'time', - ...(question.hint && { description: question.hint[0] }), - ...(question.relevant && { - visibleIf: mapKoboExpression(question.relevant), - }), - ...(question.constraint && validators(question)), }; questions.pages[0].elements.push(newQuestion); break; @@ -318,86 +268,51 @@ export const extractKoboFields = (survey: any, title: string, choices: any) => { case 'image': case 'file': { const newQuestion = { - type: 'file', - name: question.$autoname.toLowerCase(), - title: question.label[0], - isRequired: question.required, - valueName: question.$autoname.toLowerCase(), + ...commonProperties(question, 'file'), storeDataAsText: false, maxSize: 7340032, - ...(question.hint && { description: question.hint[0] }), - ...(question.relevant && { - visibleIf: mapKoboExpression(question.relevant), - }), - ...(question.constraint && validators(question)), }; questions.pages[0].elements.push(newQuestion); break; } case 'integer': { const newQuestion = { - type: 'text', - name: question.$autoname.toLowerCase(), - title: question.label[0], - isRequired: question.required, - valueName: question.$autoname.toLowerCase(), + ...commonProperties(question, 'text'), inputType: 'number', - ...(question.hint && { description: question.hint[0] }), - ...(question.default && { defaultValue: question.default }), - ...(question.relevant && { - visibleIf: mapKoboExpression(question.relevant), - }), - ...(question.constraint && validators(question)), }; questions.pages[0].elements.push(newQuestion); break; } case 'datetime': { const newQuestion = { - type: 'text', - name: question.$autoname.toLowerCase(), - title: question.label[0], - isRequired: question.required, - valueName: question.$autoname.toLowerCase(), + ...commonProperties(question, 'text'), inputType: 'datetime-local', - ...(question.hint && { description: question.hint[0] }), - ...(question.relevant && { - visibleIf: mapKoboExpression(question.relevant), - }), - ...(question.constraint && validators(question)), }; questions.pages[0].elements.push(newQuestion); break; } case 'acknowledge': { const newQuestion = { - type: 'boolean', - name: question.$autoname.toLowerCase(), - title: question.label[0], - valueName: question.$autoname.toLowerCase(), - ...(question.hint && { description: question.hint[0] }), - ...(question.relevant && { - visibleIf: mapKoboExpression(question.relevant), - }), - ...(question.constraint && validators(question)), + ...commonProperties(question, 'boolean'), }; questions.pages[0].elements.push(newQuestion); break; } case 'range': { const newQuestion = { - type: 'text', - name: question.$autoname.toLowerCase(), - title: question.label[0], - valueName: question.$autoname.toLowerCase(), + ...commonProperties(question, 'text'), inputType: 'range', step: question.parameters.split('step=')[1], - ...(question.hint && { description: question.hint[0] }), - ...(question.default && { defaultValue: question.default }), - ...(question.relevant && { - visibleIf: mapKoboExpression(question.relevant), - }), - ...(question.constraint && validators(question)), + }; + questions.pages[0].elements.push(newQuestion); + break; + } + case 'calculate': { + const newQuestion = { + ...commonProperties(question, 'expression', question.calculation), + visible: false, // They are not displayed in the Kobo form, so make it invisible by default for the SurveyJS + expression: mapKoboExpression(question.calculation), + // This question does not have hint (description) }; questions.pages[0].elements.push(newQuestion); break; From d51aa8627872b1513ae50f0a6d0c5fbcbef73cbd Mon Sep 17 00:00:00 2001 From: Estela Silva Date: Mon, 24 Jun 2024 15:37:59 -0300 Subject: [PATCH 15/27] added mapping for endsWith, currentDate functions & fix questions name removing toLowerCase() --- src/utils/form/extractKoboFields.ts | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/utils/form/extractKoboFields.ts b/src/utils/form/extractKoboFields.ts index 3a33ef011..4e3005287 100644 --- a/src/utils/form/extractKoboFields.ts +++ b/src/utils/form/extractKoboFields.ts @@ -69,8 +69,12 @@ const mapKoboExpression = (koboExpression: string, questionName?: string) => { /\$\{(\w+)\} != '(.*?)'/g, "{$1} <> '$2'" ); + // Replace ends-with with endsWith + koboExpression = koboExpression.replace(/ends-with\(/g, 'endsWith('); // Replace if with iif koboExpression = koboExpression.replace(/if\(/g, 'iif('); + // Replace now() with currentDate() + koboExpression = koboExpression.replace(/now\(\)/g, 'currentDate()'); // Date values koboExpression = koboExpression.replace(/date\('(.*?)'\)/g, "'$1'"); // Replace any remaining ${variable} to {variable} @@ -94,10 +98,7 @@ const validators = (question: any) => { ? question.constraint_message : question.constraint_message[0] : '', - expression: mapKoboExpression( - question.constraint, - question.$autoname.toLowerCase() - ), + expression: mapKoboExpression(question.constraint, question.$autoname), }, ], validateOnValueChange: true, @@ -115,13 +116,12 @@ const validators = (question: any) => { const commonProperties = (question: any, type: string, title?: string) => { return { type, - name: question.$autoname.toLowerCase(), + name: question.$autoname, title: title ?? question.label[0], - valueName: question.$autoname.toLowerCase(), + valueName: question.$autoname, isRequired: question.required, ...(question.hint && { description: question.hint[0] }), - // TODO: make sure that the defaultValue works for all type of questions. Works for sure for the questions type: integer, text, decimal, range. - ...(question.default && { defaultValue: question.default }), + ...(question.default && { defaultValue: question.default }), // TODO: add mapKoboExpression because the value can be a expression or function ...(question.relevant && { visibleIf: mapKoboExpression(question.relevant), }), From a921e0a46341f11018c381b7ba3fa250e145bd46 Mon Sep 17 00:00:00 2001 From: Estela Silva Date: Tue, 25 Jun 2024 15:25:24 -0300 Subject: [PATCH 16/27] added mapping for grous and repeating groups to panels and dynamic panels & refactor variables names --- src/utils/form/extractKoboFields.ts | 184 +++++++++++++++++++++++----- 1 file changed, 152 insertions(+), 32 deletions(-) diff --git a/src/utils/form/extractKoboFields.ts b/src/utils/form/extractKoboFields.ts index 4e3005287..46e57eb8c 100644 --- a/src/utils/form/extractKoboFields.ts +++ b/src/utils/form/extractKoboFields.ts @@ -1,3 +1,28 @@ +/** + * Saves all groupElement questions equivalent to the panel question. + * Groups and panels can be nested. + */ +const groupElements = []; +/** + * Saves the repeatGroupElement question equivalent to the dynamic panel question. + * Groups and panels can be nested. + */ +const repeatGroupElements = []; + +/** + * SurveyJS survey structure + */ +const survey = { + title: '', + pages: [ + { + name: 'page1', + elements: [], + }, + ], + showQuestionNumbers: 'off', +}; + /** * available fields types in kobo that are compatible with oort */ @@ -24,6 +49,10 @@ const AVAILABLE_TYPES = [ 'video', 'geoshape', 'calculate', + 'begin_group', + 'end_group', + 'begin_repeat', + 'end_repeat', ]; /** @@ -129,30 +158,68 @@ const commonProperties = (question: any, type: string, title?: string) => { }; }; +/** + * Get all the groups names that are related to a kobo element. + * + * @param groupPath string with the group path of the kobo element (e.g. 'group_au9sz58/group_hl8uz79/ig1') + * @returns array with all the groups in the groupPath of the element. + */ +const getElementsGroups = (groupPath: string) => { + const groupDelimiter = 'group_'; + // Split all the paths in the group path + const allPaths = groupPath.split('/'); + // Filter parts that start with the groupDelimiter (that are group names) + return allPaths.filter((part: string) => part.startsWith(groupDelimiter)); +}; + +/** + * Adds a new question/element to the page elements, panel elements or to the dynamic panel template elements + * + * @param newElement element to add + * @param elementPath string with the element path of the kobo element (e.g. 'group_au9sz58/group_hl8uz79/ig1') + */ +const addToElements = (newElement: any, elementPath: string | null) => { + const groups = elementPath ? getElementsGroups(elementPath) : []; + // If element is not part of a group + if (!groups.length) { + survey.pages[0].elements.push(newElement); + } else { + // If element is part of a group, find out which one to add it to + const groupElement = groupElements.find( + (element: any) => element.name === groups[groups.length - 1] + ); + if (groupElement) { + groupElement.elements.push(newElement); + } else { + const repeatGroupElement = repeatGroupElements.find( + (element: any) => element.name === groups[groups.length - 1] + ); + if (repeatGroupElement) { + repeatGroupElement.templateElements.push(newElement); + } + } + } +}; + /** * Extract kobo form fields and convert to oort fields * - * @param survey survey structure - * @param title title - * @param choices choices + * @param koboSurvey Kobo survey structure + * @param title Kobo survey title + * @param choices Kobo choices data for the questions * @returns oort survey */ -export const extractKoboFields = (survey: any, title: string, choices: any) => { - const questions = { - title: title, - pages: [ - { - name: 'page1', - elements: [], - }, - ], - showQuestionNumbers: 'off', - }; - +export const extractKoboFields = ( + koboSurvey: any, + title: string, + choices: any +) => { + survey.title = title; + survey.pages[0].elements = []; let scoreChoiceId = ''; let rankChoiceId = ''; - survey.map((question: any) => { + koboSurvey.map((question: any) => { if (AVAILABLE_TYPES.includes(question.type)) { switch (question.type) { case 'decimal': { @@ -160,7 +227,7 @@ export const extractKoboFields = (survey: any, title: string, choices: any) => { ...commonProperties(question, 'text'), inputType: 'number', }; - questions.pages[0].elements.push(newQuestion); + addToElements(newQuestion, question.$xpath); break; } case 'geoshape': @@ -168,7 +235,7 @@ export const extractKoboFields = (survey: any, title: string, choices: any) => { const newQuestion = { ...commonProperties(question, 'geospatial'), }; - questions.pages[0].elements.push(newQuestion); + addToElements(newQuestion, question.$xpath); break; } case 'select_one': @@ -194,7 +261,7 @@ export const extractKoboFields = (survey: any, title: string, choices: any) => { choicesOrder: 'random', }), }; - questions.pages[0].elements.push(newQuestion); + addToElements(newQuestion, question.$xpath); break; } case 'date': { @@ -202,14 +269,14 @@ export const extractKoboFields = (survey: any, title: string, choices: any) => { ...commonProperties(question, 'text'), inputType: 'date', }; - questions.pages[0].elements.push(newQuestion); + addToElements(newQuestion, question.$xpath); break; } case 'note': { const newQuestion = { ...commonProperties(question, 'expression'), }; - questions.pages[0].elements.push(newQuestion); + addToElements(newQuestion, question.$xpath); break; } case 'begin_score': { @@ -227,7 +294,7 @@ export const extractKoboFields = (survey: any, title: string, choices: any) => { })), // This question does not have Validation Criteria settings (validators property) }; - questions.pages[0].elements.push(newQuestion); + addToElements(newQuestion, question.$xpath); break; } case 'begin_rank': { @@ -245,14 +312,14 @@ export const extractKoboFields = (survey: any, title: string, choices: any) => { })), // This question does not have Validation Criteria settings }; - questions.pages[0].elements.push(newQuestion); + addToElements(newQuestion, question.$xpath); break; } case 'text': { const newQuestion = { ...commonProperties(question, 'text'), }; - questions.pages[0].elements.push(newQuestion); + addToElements(newQuestion, question.$xpath); break; } case 'time': { @@ -260,7 +327,7 @@ export const extractKoboFields = (survey: any, title: string, choices: any) => { ...commonProperties(question, 'text'), inputType: 'time', }; - questions.pages[0].elements.push(newQuestion); + addToElements(newQuestion, question.$xpath); break; } case 'audio': @@ -272,7 +339,7 @@ export const extractKoboFields = (survey: any, title: string, choices: any) => { storeDataAsText: false, maxSize: 7340032, }; - questions.pages[0].elements.push(newQuestion); + addToElements(newQuestion, question.$xpath); break; } case 'integer': { @@ -280,7 +347,7 @@ export const extractKoboFields = (survey: any, title: string, choices: any) => { ...commonProperties(question, 'text'), inputType: 'number', }; - questions.pages[0].elements.push(newQuestion); + addToElements(newQuestion, question.$xpath); break; } case 'datetime': { @@ -288,14 +355,14 @@ export const extractKoboFields = (survey: any, title: string, choices: any) => { ...commonProperties(question, 'text'), inputType: 'datetime-local', }; - questions.pages[0].elements.push(newQuestion); + addToElements(newQuestion, question.$xpath); break; } case 'acknowledge': { const newQuestion = { ...commonProperties(question, 'boolean'), }; - questions.pages[0].elements.push(newQuestion); + addToElements(newQuestion, question.$xpath); break; } case 'range': { @@ -304,7 +371,7 @@ export const extractKoboFields = (survey: any, title: string, choices: any) => { inputType: 'range', step: question.parameters.split('step=')[1], }; - questions.pages[0].elements.push(newQuestion); + addToElements(newQuestion, question.$xpath); break; } case 'calculate': { @@ -314,11 +381,64 @@ export const extractKoboFields = (survey: any, title: string, choices: any) => { expression: mapKoboExpression(question.calculation), // This question does not have hint (description) }; - questions.pages[0].elements.push(newQuestion); + addToElements(newQuestion, question.$xpath); + break; + } + case 'begin_group': { + // Get all the groups names that are related to this group + const groups = getElementsGroups(question.$xpath); + // Remove the last group because it is this group + groups.pop(); + const newGroupElement = { + ...commonProperties(question, 'panel'), + state: 'expanded', + elements: [], + groupId: question.$kuid, + // If groups still has names, the last one is the directly parent group (i.e. this group is within another group) + parentGroup: groups.length ? groups[groups.length - 1] : null, + }; + groupElements.push(newGroupElement); + break; + } + case 'end_group': { + const groupIndex = groupElements.findIndex( + (element: any) => element.groupId === question.$kuid.substring(1) + ); + const groupElement = groupElements.splice(groupIndex, 1)[0]; + addToElements(groupElement, groupElement.parentGroup); + break; + } + case 'begin_repeat': { + // Get all the groups names that are related to this group + const groups = getElementsGroups(question.$xpath); + // Remove the last group because it is this group + groups.pop(); + const newRepeatGroupElement = { + ...commonProperties(question, 'paneldynamic'), + state: 'expanded', + confirmDelete: true, + panelCount: 1, + templateElements: [], + groupId: question.$kuid, + // If groups still has group names, the last one is the directly parent group (i.e. this group is within another group) + parentGroup: groups.length ? groups[groups.length - 1] : null, + }; + repeatGroupElements.push(newRepeatGroupElement); + break; + } + case 'end_repeat': { + const groupIndex = repeatGroupElements.findIndex( + (element: any) => element.groupId === question.$kuid.substring(1) + ); + const repeatGroupElement = repeatGroupElements.splice( + groupIndex, + 1 + )[0]; + addToElements(repeatGroupElement, repeatGroupElement.parentGroup); break; } } } }); - return questions; + return survey; }; From 9588eb6a943a88ec2ec9dbbdcdda95009d473eee Mon Sep 17 00:00:00 2001 From: Estela Silva Date: Wed, 26 Jun 2024 16:26:41 -0300 Subject: [PATCH 17/27] added mapping for the functions today, format-date-time, mod, div, count-selected, not --- src/utils/form/extractKoboFields.ts | 40 +++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/src/utils/form/extractKoboFields.ts b/src/utils/form/extractKoboFields.ts index 46e57eb8c..643186434 100644 --- a/src/utils/form/extractKoboFields.ts +++ b/src/utils/form/extractKoboFields.ts @@ -76,6 +76,26 @@ const mapKoboExpression = (koboExpression: string, questionName?: string) => { /not\(selected\(\$\{(\w+)\}, '(.*?)'\)\)/g, "{$1} notcontains '$2'" ); + // Replace not(...) with !(...) + koboExpression = koboExpression.replace(/not\(/g, '!('); + // Replace mod with % + koboExpression = koboExpression.replace( + /([^\/]*)\s+mod\s+([^\/]*)/g, + (match, before, after) => { + const transformedBefore = before.replace(/\$\{(\w+)\}/g, '{$1}'); + const transformedAfter = after.replace(/\$\{(\w+)\}/g, '{$1}'); + return `${transformedBefore} % ${transformedAfter}`; + } + ); + // Replace div with / + koboExpression = koboExpression.replace( + /([^\/]*)\s+div\s+([^\/]*)/g, + (match, before, after) => { + const transformedBefore = before.replace(/\$\{(\w+)\}/g, '{$1}'); + const transformedAfter = after.replace(/\$\{(\w+)\}/g, '{$1}'); + return `${transformedBefore} / ${transformedAfter}`; + } + ); // Empty koboExpression = koboExpression.replace(/\$\{(\w+)\} = ''/g, '{$1} empty'); // Equal to @@ -100,8 +120,28 @@ const mapKoboExpression = (koboExpression: string, questionName?: string) => { ); // Replace ends-with with endsWith koboExpression = koboExpression.replace(/ends-with\(/g, 'endsWith('); + // Replace count-selected with length + koboExpression = koboExpression.replace(/count-selected\(/g, 'length('); + // Replace of format-date-time with formatDateTime + koboExpression = koboExpression.replace( + /format-date-time\(/g, + 'formatDateTime(' + ); // Replace if with iif koboExpression = koboExpression.replace(/if\(/g, 'iif('); + // TODO: FIX not working with expressions like if(${number1} + ${number2} > 10,45,30) + today() + // For calculations with today() + or - days, add addDays() custom function to work on oort + koboExpression = koboExpression.replace( + /today\(\)\s*([\+\-])\s*(\w+)/g, + (match, operator, term) => { + const transformedTerm = term.replace(/\$\{(\w+)\}/g, '{$1}'); + if (operator === '+') { + return `addDays(today(), ${transformedTerm})`; + } else { + return `addDays(today(), -${transformedTerm})`; + } + } + ); // Replace now() with currentDate() koboExpression = koboExpression.replace(/now\(\)/g, 'currentDate()'); // Date values From bc2bd78e85713c8176c0f62b170a154dae0b8038 Mon Sep 17 00:00:00 2001 From: Estela Silva Date: Thu, 27 Jun 2024 15:55:56 -0300 Subject: [PATCH 18/27] added mapping for the functions indexed-repeat, position --- src/utils/form/extractKoboFields.ts | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/utils/form/extractKoboFields.ts b/src/utils/form/extractKoboFields.ts index 643186434..5db3af18c 100644 --- a/src/utils/form/extractKoboFields.ts +++ b/src/utils/form/extractKoboFields.ts @@ -120,6 +120,13 @@ const mapKoboExpression = (koboExpression: string, questionName?: string) => { ); // Replace ends-with with endsWith koboExpression = koboExpression.replace(/ends-with\(/g, 'endsWith('); + // Replace indexed-repeat(...) with indexedRepeat(...) and handle the first parameter to be the question/field name and not the question value + koboExpression = koboExpression.replace( + /indexed-repeat\(\$\{(\w+)\},\s*\$\{(\w+)\},\s*(.*?)\)/g, + 'indexedRepeat($1, {$2}, $3)' + ); + // Replace position(..) with {panelIndex} + koboExpression = koboExpression.replace(/position\(\.\.\)/g, '{panelIndex}'); // Replace count-selected with length koboExpression = koboExpression.replace(/count-selected\(/g, 'length('); // Replace of format-date-time with formatDateTime @@ -190,7 +197,9 @@ const commonProperties = (question: any, type: string, title?: string) => { valueName: question.$autoname, isRequired: question.required, ...(question.hint && { description: question.hint[0] }), - ...(question.default && { defaultValue: question.default }), // TODO: add mapKoboExpression because the value can be a expression or function + ...(question.default && { + defaultValue: mapKoboExpression(question.default), + }), ...(question.relevant && { visibleIf: mapKoboExpression(question.relevant), }), From 8bdf3db4994107efd2f338fce3da908eca7121b0 Mon Sep 17 00:00:00 2001 From: Estela Silva Date: Fri, 28 Jun 2024 17:07:58 -0300 Subject: [PATCH 19/27] added mapping for the functions sum, max, min, join --- src/utils/form/extractKoboFields.ts | 155 +---------------------- src/utils/form/kobo/commonProperties.ts | 54 ++++++++ src/utils/form/kobo/mapKoboExpression.ts | 125 ++++++++++++++++++ 3 files changed, 182 insertions(+), 152 deletions(-) create mode 100644 src/utils/form/kobo/commonProperties.ts create mode 100644 src/utils/form/kobo/mapKoboExpression.ts diff --git a/src/utils/form/extractKoboFields.ts b/src/utils/form/extractKoboFields.ts index 5db3af18c..7d839afe1 100644 --- a/src/utils/form/extractKoboFields.ts +++ b/src/utils/form/extractKoboFields.ts @@ -1,3 +1,6 @@ +import { mapKoboExpression } from './kobo/mapKoboExpression'; +import { commonProperties } from './kobo/commonProperties'; + /** * Saves all groupElement questions equivalent to the panel question. * Groups and panels can be nested. @@ -55,158 +58,6 @@ const AVAILABLE_TYPES = [ 'end_repeat', ]; -/** - * Maps expressions from kobo questions to a expression format that will work on the SurveyJS. - * - * The numeric operators (Greater than >, Less than <, Greater than or equal to >=, Less than or equal to <=) are used the in same way in Kobo and SurveyJS. - * - * @param koboExpression the initial kobo logic expression - * @param questionName name of the question to replace in the expression - * @returns the mapped logic expression that will work on the SurveyJS form - */ -const mapKoboExpression = (koboExpression: string, questionName?: string) => { - // Replace . with {questionName} - if (questionName) { - // Expressions in Kobo can have ' . ' to indicate that the expression is about the question in which it is defined. - // Example: a Validation Criteria can be ". > 5 ": the value of the question itself must be greater than 5 - koboExpression = koboExpression.replace(/\./g, `{${questionName}}`); - } - // Not contains - koboExpression = koboExpression.replace( - /not\(selected\(\$\{(\w+)\}, '(.*?)'\)\)/g, - "{$1} notcontains '$2'" - ); - // Replace not(...) with !(...) - koboExpression = koboExpression.replace(/not\(/g, '!('); - // Replace mod with % - koboExpression = koboExpression.replace( - /([^\/]*)\s+mod\s+([^\/]*)/g, - (match, before, after) => { - const transformedBefore = before.replace(/\$\{(\w+)\}/g, '{$1}'); - const transformedAfter = after.replace(/\$\{(\w+)\}/g, '{$1}'); - return `${transformedBefore} % ${transformedAfter}`; - } - ); - // Replace div with / - koboExpression = koboExpression.replace( - /([^\/]*)\s+div\s+([^\/]*)/g, - (match, before, after) => { - const transformedBefore = before.replace(/\$\{(\w+)\}/g, '{$1}'); - const transformedAfter = after.replace(/\$\{(\w+)\}/g, '{$1}'); - return `${transformedBefore} / ${transformedAfter}`; - } - ); - // Empty - koboExpression = koboExpression.replace(/\$\{(\w+)\} = ''/g, '{$1} empty'); - // Equal to - koboExpression = koboExpression.replace( - /\$\{(\w+)\} = '(.*?)'/g, - "({$1} = '$2')" - ); - // Contains - koboExpression = koboExpression.replace( - /selected\(\$\{(\w+)\}, '(.*?)'\)/g, - "{$1} contains '$2'" - ); - // No empty - koboExpression = koboExpression.replace( - /\$\{(\w+)\} != ''/g, - '{$1} notempty' - ); - // Not equal to - koboExpression = koboExpression.replace( - /\$\{(\w+)\} != '(.*?)'/g, - "{$1} <> '$2'" - ); - // Replace ends-with with endsWith - koboExpression = koboExpression.replace(/ends-with\(/g, 'endsWith('); - // Replace indexed-repeat(...) with indexedRepeat(...) and handle the first parameter to be the question/field name and not the question value - koboExpression = koboExpression.replace( - /indexed-repeat\(\$\{(\w+)\},\s*\$\{(\w+)\},\s*(.*?)\)/g, - 'indexedRepeat($1, {$2}, $3)' - ); - // Replace position(..) with {panelIndex} - koboExpression = koboExpression.replace(/position\(\.\.\)/g, '{panelIndex}'); - // Replace count-selected with length - koboExpression = koboExpression.replace(/count-selected\(/g, 'length('); - // Replace of format-date-time with formatDateTime - koboExpression = koboExpression.replace( - /format-date-time\(/g, - 'formatDateTime(' - ); - // Replace if with iif - koboExpression = koboExpression.replace(/if\(/g, 'iif('); - // TODO: FIX not working with expressions like if(${number1} + ${number2} > 10,45,30) + today() - // For calculations with today() + or - days, add addDays() custom function to work on oort - koboExpression = koboExpression.replace( - /today\(\)\s*([\+\-])\s*(\w+)/g, - (match, operator, term) => { - const transformedTerm = term.replace(/\$\{(\w+)\}/g, '{$1}'); - if (operator === '+') { - return `addDays(today(), ${transformedTerm})`; - } else { - return `addDays(today(), -${transformedTerm})`; - } - } - ); - // Replace now() with currentDate() - koboExpression = koboExpression.replace(/now\(\)/g, 'currentDate()'); - // Date values - koboExpression = koboExpression.replace(/date\('(.*?)'\)/g, "'$1'"); - // Replace any remaining ${variable} to {variable} - koboExpression = koboExpression.replace(/\$\{(\w+)\}/g, '{$1}'); - return koboExpression; -}; - -/** - * Extract from a Kobo question the constraint properties and return the validators property for an SurveyJS question - * - * @param question Kobo question object - * @returns validators property for an SurveyJS question - */ -const validators = (question: any) => { - return { - validators: [ - { - type: 'expression', - text: question.constraint_message - ? typeof question.constraint_message === 'string' - ? question.constraint_message - : question.constraint_message[0] - : '', - expression: mapKoboExpression(question.constraint, question.$autoname), - }, - ], - validateOnValueChange: true, - }; -}; - -/** - * Extract from a Kobo question the common properties in a object for a SurveyJS question - * - * @param question Kobo question object - * @param type type of the questions - * @param title optional title - * @returns the common properties in a object for a SurveyJS question extracted from the Kobo question - */ -const commonProperties = (question: any, type: string, title?: string) => { - return { - type, - name: question.$autoname, - title: title ?? question.label[0], - valueName: question.$autoname, - isRequired: question.required, - ...(question.hint && { description: question.hint[0] }), - ...(question.default && { - defaultValue: mapKoboExpression(question.default), - }), - ...(question.relevant && { - visibleIf: mapKoboExpression(question.relevant), - }), - ...(question.constraint && validators(question)), - }; -}; - /** * Get all the groups names that are related to a kobo element. * diff --git a/src/utils/form/kobo/commonProperties.ts b/src/utils/form/kobo/commonProperties.ts new file mode 100644 index 000000000..af77ddd98 --- /dev/null +++ b/src/utils/form/kobo/commonProperties.ts @@ -0,0 +1,54 @@ +import { mapKoboExpression } from './mapKoboExpression'; + +/** + * Extract from a Kobo question the constraint properties and return the validators property for an SurveyJS question + * + * @param question Kobo question object + * @returns validators property for an SurveyJS question + */ +const validators = (question: any) => { + return { + validators: [ + { + type: 'expression', + text: question.constraint_message + ? typeof question.constraint_message === 'string' + ? question.constraint_message + : question.constraint_message[0] + : '', + expression: mapKoboExpression(question.constraint, question.$autoname), + }, + ], + validateOnValueChange: true, + }; +}; + +/** + * Extract from a Kobo question the common properties in a object for a SurveyJS question + * + * @param question Kobo question object + * @param type type of the questions + * @param title optional title + * @returns the common properties in a object for a SurveyJS question extracted from the Kobo question + */ +export const commonProperties = ( + question: any, + type: string, + title?: string +) => { + return { + type, + name: question.$autoname, + title: title ?? question.label[0], + valueName: question.$autoname, + isRequired: question.required, + ...(question.hint && { description: question.hint[0] }), + ...(question.default && { + defaultValue: mapKoboExpression(question.default), + }), + ...(question.relevant && { + visibleIf: mapKoboExpression(question.relevant), + }), + ...(question.constraint && validators(question)), + }; +}; diff --git a/src/utils/form/kobo/mapKoboExpression.ts b/src/utils/form/kobo/mapKoboExpression.ts new file mode 100644 index 000000000..88b2fda9c --- /dev/null +++ b/src/utils/form/kobo/mapKoboExpression.ts @@ -0,0 +1,125 @@ +/** + * Maps expressions from kobo questions to a expression format that will work on the SurveyJS. + * + * The numeric operators (Greater than >, Less than <, Greater than or equal to >=, Less than or equal to <=) are used the in same way in Kobo and SurveyJS. + * + * @param koboExpression the initial kobo logic expression + * @param questionName name of the question to replace in the expression + * @returns the mapped logic expression that will work on the SurveyJS form + */ +export const mapKoboExpression = ( + koboExpression: string, + questionName?: string +) => { + // Replace . with {questionName} + if (questionName) { + // Expressions in Kobo can have ' . ' to indicate that the expression is about the question in which it is defined. + // Example: a Validation Criteria can be ". > 5 ": the value of the question itself must be greater than 5 + koboExpression = koboExpression.replace(/\./g, `{${questionName}}`); + } + // Not contains + koboExpression = koboExpression.replace( + /not\(selected\(\$\{(\w+)\}, '(.*?)'\)\)/g, + "{$1} notcontains '$2'" + ); + // Replace not(...) with !(...) + koboExpression = koboExpression.replace(/not\(/g, '!('); + // Replace mod with % + koboExpression = koboExpression.replace( + /([^\/]*)\s+mod\s+([^\/]*)/g, + (match, before, after) => { + const transformedBefore = before.replace(/\$\{(\w+)\}/g, '{$1}'); + const transformedAfter = after.replace(/\$\{(\w+)\}/g, '{$1}'); + return `${transformedBefore} % ${transformedAfter}`; + } + ); + // Replace div with / + koboExpression = koboExpression.replace( + /([^\/]*)\s+div\s+([^\/]*)/g, + (match, before, after) => { + const transformedBefore = before.replace(/\$\{(\w+)\}/g, '{$1}'); + const transformedAfter = after.replace(/\$\{(\w+)\}/g, '{$1}'); + return `${transformedBefore} / ${transformedAfter}`; + } + ); + // Empty + koboExpression = koboExpression.replace(/\$\{(\w+)\} = ''/g, '{$1} empty'); + // Equal to + koboExpression = koboExpression.replace( + /\$\{(\w+)\} = '(.*?)'/g, + "({$1} = '$2')" + ); + // Contains + koboExpression = koboExpression.replace( + /selected\(\$\{(\w+)\}, '(.*?)'\)/g, + "{$1} contains '$2'" + ); + // No empty + koboExpression = koboExpression.replace( + /\$\{(\w+)\} != ''/g, + '{$1} notempty' + ); + // Not equal to + koboExpression = koboExpression.replace( + /\$\{(\w+)\} != '(.*?)'/g, + "{$1} <> '$2'" + ); + // Replace ends-with with endsWith + koboExpression = koboExpression.replace(/ends-with\(/g, 'endsWith('); + // Replace indexed-repeat(...) with indexedRepeat(...) and handle the first parameter to be the question/field name and not the question value + koboExpression = koboExpression.replace( + /indexed-repeat\(\$\{(\w+)\},\s*\$\{(\w+)\},\s*(.*?)\)/g, + 'indexedRepeat($1, {$2}, $3)' + ); + // Replace sum(...) with sumElements(...) and handle the parameter (from question reference to question name only) + koboExpression = koboExpression.replace( + /sum\(\$\{(\w+)\}\)/g, + 'sumElements($1)' + ); + // Replace max(...) with maxElements(...) and handle the parameter (from question reference to question name only) + koboExpression = koboExpression.replace( + /max\(\$\{(\w+)\}\)/g, + 'maxElements($1)' + ); + // Replace min(...) with minElements(...) and handle the parameter (from question reference to question name only) + koboExpression = koboExpression.replace( + /min\(\$\{(\w+)\}\)/g, + 'minElements($1)' + ); + // Replace join(' ', ${...}) with join(' ', ...) (handle the parameter, replacing the question reference for a question name only) + koboExpression = koboExpression.replace( + /join\(\s*'([^']*)'\s*,\s*\$\{(\w+)\}\s*\)/g, + "join('$1', $2)" + ); + // Replace position(..) with {panelIndex} + koboExpression = koboExpression.replace(/position\(\.\.\)/g, '{panelIndex}'); + // Replace count-selected with length + koboExpression = koboExpression.replace(/count-selected\(/g, 'length('); + // Replace of format-date-time with formatDateTime + koboExpression = koboExpression.replace( + /format-date-time\(/g, + 'formatDateTime(' + ); + // Replace if with iif + koboExpression = koboExpression.replace(/if\(/g, 'iif('); + // TODO: FIX not working with expressions like if(${number1} + ${number2} > 10,45,30) + today() + // For calculations with today() + or - days, add addDays() custom function to work on oort + koboExpression = koboExpression.replace( + /today\(\)\s*([\+\-])\s*(\w+)/g, + (match, operator, term) => { + const transformedTerm = term.replace(/\$\{(\w+)\}/g, '{$1}'); + if (operator === '+') { + return `addDays(today(), ${transformedTerm})`; + } else { + return `addDays(today(), -${transformedTerm})`; + } + } + ); + // Replace now() with currentDate() + koboExpression = koboExpression.replace(/now\(\)/g, 'currentDate()'); + // Date values + koboExpression = koboExpression.replace(/date\('(.*?)'\)/g, "'$1'"); + // Replace any remaining ${variable} to {variable} + koboExpression = koboExpression.replace(/\$\{(\w+)\}/g, '{$1}'); + return koboExpression; +}; From 7d4793a6930a94bd6311a1ce7d87c6a90a74e3aa Mon Sep 17 00:00:00 2001 From: Estela Silva Date: Mon, 1 Jul 2024 15:29:58 -0300 Subject: [PATCH 20/27] added mapping choices visibleIf and fixes --- src/utils/form/extractKoboFields.ts | 9 +++++++++ src/utils/form/kobo/commonProperties.ts | 2 +- src/utils/form/kobo/mapKoboExpression.ts | 18 ++++++++++++------ 3 files changed, 22 insertions(+), 7 deletions(-) diff --git a/src/utils/form/extractKoboFields.ts b/src/utils/form/extractKoboFields.ts index 7d839afe1..0cb86ed30 100644 --- a/src/utils/form/extractKoboFields.ts +++ b/src/utils/form/extractKoboFields.ts @@ -155,6 +155,15 @@ export const extractKoboFields = ( .map((choice) => ({ value: choice.$autovalue, text: choice.label[0], + ...(question.choice_filter && + // If in the Kobo form the choice has the 'other' property, we will not add the visibleIf because of the 'or other=0' in the expression + !Object.prototype.hasOwnProperty.call(choice, 'other') && { + visibleIf: mapKoboExpression( + question.choice_filter, + null, + choice.$autovalue + ), + }), })), ...(question.parameters && question.parameters.split('randomize=')[1]?.includes('true') && { diff --git a/src/utils/form/kobo/commonProperties.ts b/src/utils/form/kobo/commonProperties.ts index af77ddd98..340022362 100644 --- a/src/utils/form/kobo/commonProperties.ts +++ b/src/utils/form/kobo/commonProperties.ts @@ -39,7 +39,7 @@ export const commonProperties = ( return { type, name: question.$autoname, - title: title ?? question.label[0], + title: title ?? question.label ? question.label[0] : question.$autoname, valueName: question.$autoname, isRequired: question.required, ...(question.hint && { description: question.hint[0] }), diff --git a/src/utils/form/kobo/mapKoboExpression.ts b/src/utils/form/kobo/mapKoboExpression.ts index 88b2fda9c..9fc06acfb 100644 --- a/src/utils/form/kobo/mapKoboExpression.ts +++ b/src/utils/form/kobo/mapKoboExpression.ts @@ -5,12 +5,23 @@ * * @param koboExpression the initial kobo logic expression * @param questionName name of the question to replace in the expression + * @param choiceValue value of the choice to replace in the expression * @returns the mapped logic expression that will work on the SurveyJS form */ export const mapKoboExpression = ( koboExpression: string, - questionName?: string + questionName?: string, + choiceValue?: string ) => { + // Replace 'name' with choiceValue in selected expressions (for choice_filter on select questions) + if (choiceValue) { + koboExpression = koboExpression.replace( + /selected\(\$\{(\w+)\}, name\)/g, + `selected(\$\{$1\}, ${choiceValue})` + ); + // If in the Kobo form the choice has a other property, we will remove the 'or other=0' from the choice visibleIf + koboExpression = koboExpression.replace(/or other=0/g, ''); + } // Replace . with {questionName} if (questionName) { // Expressions in Kobo can have ' . ' to indicate that the expression is about the question in which it is defined. @@ -49,11 +60,6 @@ export const mapKoboExpression = ( /\$\{(\w+)\} = '(.*?)'/g, "({$1} = '$2')" ); - // Contains - koboExpression = koboExpression.replace( - /selected\(\$\{(\w+)\}, '(.*?)'\)/g, - "{$1} contains '$2'" - ); // No empty koboExpression = koboExpression.replace( /\$\{(\w+)\} != ''/g, From 5864086abc16c0394a296c922bec0a8a928bf6a4 Mon Sep 17 00:00:00 2001 From: Estela Silva Date: Mon, 1 Jul 2024 17:14:56 -0300 Subject: [PATCH 21/27] correct elements order, and adapt group logic to when user edited group name --- src/utils/form/extractKoboFields.ts | 61 ++++++++++++++++--------- src/utils/form/kobo/commonProperties.ts | 5 +- 2 files changed, 44 insertions(+), 22 deletions(-) diff --git a/src/utils/form/extractKoboFields.ts b/src/utils/form/extractKoboFields.ts index 0cb86ed30..da47544ea 100644 --- a/src/utils/form/extractKoboFields.ts +++ b/src/utils/form/extractKoboFields.ts @@ -65,11 +65,16 @@ const AVAILABLE_TYPES = [ * @returns array with all the groups in the groupPath of the element. */ const getElementsGroups = (groupPath: string) => { - const groupDelimiter = 'group_'; // Split all the paths in the group path const allPaths = groupPath.split('/'); + // Remove the last element (is the name of the current question) + allPaths.pop(); + return allPaths; + + // NO LONGER USED BECAUSE IN THE INTEGRATION TESTS ALL THE GROUPS ARE RE-NAMED AND THE ID IS NOT THE SAME + // const groupDelimiter = 'group_'; // Filter parts that start with the groupDelimiter (that are group names) - return allPaths.filter((part: string) => part.startsWith(groupDelimiter)); + // return allPaths.filter((part: string) => part.startsWith(groupDelimiter)); }; /** @@ -119,12 +124,12 @@ export const extractKoboFields = ( let scoreChoiceId = ''; let rankChoiceId = ''; - koboSurvey.map((question: any) => { + koboSurvey.map((question: any, index: number) => { if (AVAILABLE_TYPES.includes(question.type)) { switch (question.type) { case 'decimal': { const newQuestion = { - ...commonProperties(question, 'text'), + ...commonProperties(index, question, 'text'), inputType: 'number', }; addToElements(newQuestion, question.$xpath); @@ -133,7 +138,7 @@ export const extractKoboFields = ( case 'geoshape': case 'geopoint': { const newQuestion = { - ...commonProperties(question, 'geospatial'), + ...commonProperties(index, question, 'geospatial'), }; addToElements(newQuestion, question.$xpath); break; @@ -142,6 +147,7 @@ export const extractKoboFields = ( case 'select_multiple': { const newQuestion = { ...commonProperties( + index, question, question.type === 'select_multiple' ? 'checkbox' : 'radiogroup' ), @@ -175,7 +181,7 @@ export const extractKoboFields = ( } case 'date': { const newQuestion = { - ...commonProperties(question, 'text'), + ...commonProperties(index, question, 'text'), inputType: 'date', }; addToElements(newQuestion, question.$xpath); @@ -183,7 +189,7 @@ export const extractKoboFields = ( } case 'note': { const newQuestion = { - ...commonProperties(question, 'expression'), + ...commonProperties(index, question, 'expression'), }; addToElements(newQuestion, question.$xpath); break; @@ -194,7 +200,7 @@ export const extractKoboFields = ( } case 'score__row': { const newQuestion = { - ...commonProperties(question, 'radiogroup'), + ...commonProperties(index, question, 'radiogroup'), choices: choices .filter((choice) => scoreChoiceId === choice.list_name) .map((choice) => ({ @@ -212,7 +218,7 @@ export const extractKoboFields = ( } case 'rank__level': { const newQuestion = { - ...commonProperties(question, 'dropdown'), + ...commonProperties(index, question, 'dropdown'), choices: choices .filter((choice) => rankChoiceId === choice.list_name) .map((choice) => ({ @@ -226,14 +232,14 @@ export const extractKoboFields = ( } case 'text': { const newQuestion = { - ...commonProperties(question, 'text'), + ...commonProperties(index, question, 'text'), }; addToElements(newQuestion, question.$xpath); break; } case 'time': { const newQuestion = { - ...commonProperties(question, 'text'), + ...commonProperties(index, question, 'text'), inputType: 'time', }; addToElements(newQuestion, question.$xpath); @@ -244,7 +250,7 @@ export const extractKoboFields = ( case 'image': case 'file': { const newQuestion = { - ...commonProperties(question, 'file'), + ...commonProperties(index, question, 'file'), storeDataAsText: false, maxSize: 7340032, }; @@ -253,7 +259,7 @@ export const extractKoboFields = ( } case 'integer': { const newQuestion = { - ...commonProperties(question, 'text'), + ...commonProperties(index, question, 'text'), inputType: 'number', }; addToElements(newQuestion, question.$xpath); @@ -261,7 +267,7 @@ export const extractKoboFields = ( } case 'datetime': { const newQuestion = { - ...commonProperties(question, 'text'), + ...commonProperties(index, question, 'text'), inputType: 'datetime-local', }; addToElements(newQuestion, question.$xpath); @@ -269,14 +275,14 @@ export const extractKoboFields = ( } case 'acknowledge': { const newQuestion = { - ...commonProperties(question, 'boolean'), + ...commonProperties(index, question, 'boolean'), }; addToElements(newQuestion, question.$xpath); break; } case 'range': { const newQuestion = { - ...commonProperties(question, 'text'), + ...commonProperties(index, question, 'text'), inputType: 'range', step: question.parameters.split('step=')[1], }; @@ -285,7 +291,12 @@ export const extractKoboFields = ( } case 'calculate': { const newQuestion = { - ...commonProperties(question, 'expression', question.calculation), + ...commonProperties( + index, + question, + 'expression', + question.calculation + ), visible: false, // They are not displayed in the Kobo form, so make it invisible by default for the SurveyJS expression: mapKoboExpression(question.calculation), // This question does not have hint (description) @@ -299,7 +310,7 @@ export const extractKoboFields = ( // Remove the last group because it is this group groups.pop(); const newGroupElement = { - ...commonProperties(question, 'panel'), + ...commonProperties(index, question, 'panel'), state: 'expanded', elements: [], groupId: question.$kuid, @@ -311,7 +322,9 @@ export const extractKoboFields = ( } case 'end_group': { const groupIndex = groupElements.findIndex( - (element: any) => element.groupId === question.$kuid.substring(1) + (element: any) => + element.groupId === question.$kuid.substring(1) || + question.name === element.valueName ); const groupElement = groupElements.splice(groupIndex, 1)[0]; addToElements(groupElement, groupElement.parentGroup); @@ -323,7 +336,7 @@ export const extractKoboFields = ( // Remove the last group because it is this group groups.pop(); const newRepeatGroupElement = { - ...commonProperties(question, 'paneldynamic'), + ...commonProperties(index, question, 'paneldynamic'), state: 'expanded', confirmDelete: true, panelCount: 1, @@ -337,7 +350,9 @@ export const extractKoboFields = ( } case 'end_repeat': { const groupIndex = repeatGroupElements.findIndex( - (element: any) => element.groupId === question.$kuid.substring(1) + (element: any) => + element.groupId === question.$kuid.substring(1) || + question.name === element.valueName ); const repeatGroupElement = repeatGroupElements.splice( groupIndex, @@ -349,5 +364,9 @@ export const extractKoboFields = ( } } }); + // Order elements (is necessary because of groups being added to the survey elements only after closed) + survey.pages[0].elements = survey.pages[0].elements.sort((a, b) => + a.index > b.index ? 1 : -1 + ); return survey; }; diff --git a/src/utils/form/kobo/commonProperties.ts b/src/utils/form/kobo/commonProperties.ts index 340022362..db2fc1004 100644 --- a/src/utils/form/kobo/commonProperties.ts +++ b/src/utils/form/kobo/commonProperties.ts @@ -26,20 +26,23 @@ const validators = (question: any) => { /** * Extract from a Kobo question the common properties in a object for a SurveyJS question * + * @param index index/order of the element inside the Kobo structure * @param question Kobo question object * @param type type of the questions * @param title optional title * @returns the common properties in a object for a SurveyJS question extracted from the Kobo question */ export const commonProperties = ( + index: number, question: any, type: string, title?: string ) => { return { + index, type, name: question.$autoname, - title: title ?? question.label ? question.label[0] : question.$autoname, + title: title ?? (question.label ? question.label[0] : question.$autoname), valueName: question.$autoname, isRequired: question.required, ...(question.hint && { description: question.hint[0] }), From c4eb46be3e7a22c0b27af5eb11cc2e1d5c6534c8 Mon Sep 17 00:00:00 2001 From: Estela Silva Date: Tue, 2 Jul 2024 17:51:28 -0300 Subject: [PATCH 22/27] fix: get group's parentGroup after getElementsGroups refactor --- src/utils/form/extractKoboFields.ts | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/src/utils/form/extractKoboFields.ts b/src/utils/form/extractKoboFields.ts index da47544ea..72751d452 100644 --- a/src/utils/form/extractKoboFields.ts +++ b/src/utils/form/extractKoboFields.ts @@ -82,22 +82,29 @@ const getElementsGroups = (groupPath: string) => { * * @param newElement element to add * @param elementPath string with the element path of the kobo element (e.g. 'group_au9sz58/group_hl8uz79/ig1') + * @param parentGroup if element is a group and have a parent group, we already know it's parent group */ -const addToElements = (newElement: any, elementPath: string | null) => { +const addToElements = ( + newElement: any, + elementPath: string | null, + parentGroup?: string +) => { const groups = elementPath ? getElementsGroups(elementPath) : []; // If element is not part of a group - if (!groups.length) { + if (!groups.length && !parentGroup) { survey.pages[0].elements.push(newElement); } else { // If element is part of a group, find out which one to add it to const groupElement = groupElements.find( - (element: any) => element.name === groups[groups.length - 1] + (element: any) => + element.name === (parentGroup ?? groups[groups.length - 1]) ); if (groupElement) { groupElement.elements.push(newElement); } else { const repeatGroupElement = repeatGroupElements.find( - (element: any) => element.name === groups[groups.length - 1] + (element: any) => + element.name === (parentGroup ?? groups[groups.length - 1]) ); if (repeatGroupElement) { repeatGroupElement.templateElements.push(newElement); @@ -307,8 +314,6 @@ export const extractKoboFields = ( case 'begin_group': { // Get all the groups names that are related to this group const groups = getElementsGroups(question.$xpath); - // Remove the last group because it is this group - groups.pop(); const newGroupElement = { ...commonProperties(index, question, 'panel'), state: 'expanded', @@ -327,14 +332,12 @@ export const extractKoboFields = ( question.name === element.valueName ); const groupElement = groupElements.splice(groupIndex, 1)[0]; - addToElements(groupElement, groupElement.parentGroup); + addToElements(groupElement, null, groupElement.parentGroup); break; } case 'begin_repeat': { // Get all the groups names that are related to this group const groups = getElementsGroups(question.$xpath); - // Remove the last group because it is this group - groups.pop(); const newRepeatGroupElement = { ...commonProperties(index, question, 'paneldynamic'), state: 'expanded', @@ -358,7 +361,11 @@ export const extractKoboFields = ( groupIndex, 1 )[0]; - addToElements(repeatGroupElement, repeatGroupElement.parentGroup); + addToElements( + repeatGroupElement, + null, + repeatGroupElement.parentGroup + ); break; } } From 481e645ebffcad959969444f3ec3cbf109510e58 Mon Sep 17 00:00:00 2001 From: Estela Silva Date: Mon, 8 Jul 2024 16:40:42 -0300 Subject: [PATCH 23/27] added kobo info to form model - to save kobo info on forms imported from kobotoolbox --- src/models/form.model.ts | 14 ++++++++++++++ src/schema/mutation/addForm.mutation.ts | 6 ++++++ src/schema/types/form.type.ts | 2 ++ src/schema/types/koboForm.type.ts | 25 +++++++++++++++++++++++++ 4 files changed, 47 insertions(+) create mode 100644 src/schema/types/koboForm.type.ts diff --git a/src/models/form.model.ts b/src/models/form.model.ts index 6105faa10..d9ed852d5 100644 --- a/src/models/form.model.ts +++ b/src/models/form.model.ts @@ -10,6 +10,7 @@ import { getGraphQLTypeName } from '@utils/validators'; import { deleteFolder } from '@utils/files/deleteFolder'; import { logger } from '@services/logger.service'; import { DEFAULT_IMPORT_FIELD } from './resource.model'; +import { ApiConfiguration } from './apiConfiguration.model'; /** Form documents interface declaration */ interface FormDocument extends Document { @@ -38,6 +39,11 @@ interface FormDocument extends Document { versions?: any[]; channel?: any; layouts?: any; + kobo?: { + id: string; + deployedVersionId: string; + apiConfiguration: ApiConfiguration; + }; } /** Interface of form */ @@ -68,6 +74,14 @@ const schema = new Schema
( type: String, enum: Object.values(status), }, + kobo: { + id: String, + deployedVersionId: String, + apiConfiguration: { + type: mongoose.Schema.Types.ObjectId, + ref: 'ApiConfiguration', + }, + }, permissions: { canSee: [ { diff --git a/src/schema/mutation/addForm.mutation.ts b/src/schema/mutation/addForm.mutation.ts index a0f4da5b9..3c29eface 100644 --- a/src/schema/mutation/addForm.mutation.ts +++ b/src/schema/mutation/addForm.mutation.ts @@ -114,6 +114,7 @@ export default { const survey = response.data.content.survey; const choices = response.data.content.choices; const title = response.data.name; + const deployedVersionId = response.data.deployed_version_id; // Get structure from the kobo form const structure = JSON.stringify( @@ -153,6 +154,11 @@ export default { structure, fields, versions: [version._id], + kobo: { + id: args.kobo, + deployedVersionId, + apiConfiguration: args.apiConfiguration, + }, }); await form.save(); return form; diff --git a/src/schema/types/form.type.ts b/src/schema/types/form.type.ts index 844aeb9b4..0fd9bc541 100644 --- a/src/schema/types/form.type.ts +++ b/src/schema/types/form.type.ts @@ -28,6 +28,7 @@ import extendAbilityForContent from '@security/extendAbilityForContent'; import { getMetaData } from '@utils/form/metadata.helper'; import { getAccessibleFields } from '@utils/form'; import { accessibleBy } from '@casl/mongoose'; +import { KoboFormType } from './koboForm.type'; /** Default page size */ const DEFAULT_FIRST = 10; @@ -49,6 +50,7 @@ export const FormType = new GraphQLObjectType({ structure: { type: GraphQLJSON }, status: { type: StatusEnumType }, allowUploadRecords: { type: GraphQLBoolean }, + kobo: { type: KoboFormType }, permissions: { type: AccessType, async resolve(parent, args, context) { diff --git a/src/schema/types/koboForm.type.ts b/src/schema/types/koboForm.type.ts new file mode 100644 index 000000000..ef09388b9 --- /dev/null +++ b/src/schema/types/koboForm.type.ts @@ -0,0 +1,25 @@ +import { GraphQLObjectType, GraphQLString } from 'graphql'; +import { ApiConfigurationType } from './apiConfiguration.type'; +import { AppAbility } from '@security/defineUserAbility'; +import { ApiConfiguration } from '@models'; +import { accessibleBy } from '@casl/mongoose'; + +/** GraphQL Kobo form type definition */ +export const KoboFormType = new GraphQLObjectType({ + name: 'Geospatial', + fields: () => ({ + id: { type: GraphQLString }, + deployedVersionId: { type: GraphQLString }, + apiConfiguration: { + type: ApiConfigurationType, + async resolve(parent, args, context) { + const ability: AppAbility = context.user.ability; + const apiConfig = await ApiConfiguration.findOne({ + _id: parent.apiConfiguration, + ...accessibleBy(ability, 'read').ApiConfiguration, + }); + return apiConfig; + }, + }, + }), +}); From 27074d0f9f14103160cb41e792b575d3d82285fe Mon Sep 17 00:00:00 2001 From: Estela Silva Date: Tue, 9 Jul 2024 14:36:24 -0300 Subject: [PATCH 24/27] added dataFromDeployedVersion to kobo properties and updated editForm mutation --- src/models/form.model.ts | 2 ++ src/schema/mutation/editForm.mutation.ts | 13 ++++++++++++- src/schema/types/koboForm.type.ts | 3 ++- 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/src/models/form.model.ts b/src/models/form.model.ts index d9ed852d5..39d628a95 100644 --- a/src/models/form.model.ts +++ b/src/models/form.model.ts @@ -42,6 +42,7 @@ interface FormDocument extends Document { kobo?: { id: string; deployedVersionId: string; + dataFromDeployedVersion: boolean; apiConfiguration: ApiConfiguration; }; } @@ -77,6 +78,7 @@ const schema = new Schema( kobo: { id: String, deployedVersionId: String, + dataFromDeployedVersion: { type: Boolean, default: false }, apiConfiguration: { type: mongoose.Schema.Types.ObjectId, ref: 'ApiConfiguration', diff --git a/src/schema/mutation/editForm.mutation.ts b/src/schema/mutation/editForm.mutation.ts index 48cb9aaf5..e8fd5489f 100644 --- a/src/schema/mutation/editForm.mutation.ts +++ b/src/schema/mutation/editForm.mutation.ts @@ -3,6 +3,7 @@ import { GraphQLID, GraphQLString, GraphQLError, + GraphQLBoolean, } from 'graphql'; import GraphQLJSON from 'graphql-type-json'; import { Form, Resource, Version, Channel, ReferenceData } from '@models'; @@ -22,7 +23,7 @@ import isEqual from 'lodash/isEqual'; import differenceWith from 'lodash/differenceWith'; import unionWith from 'lodash/unionWith'; import i18next from 'i18next'; -import { get, isArray } from 'lodash'; +import { get, isArray, isNil } from 'lodash'; import { logger } from '@services/logger.service'; import checkDefaultFields from '@utils/form/checkDefaultFields'; import { graphQLAuthCheck } from '@schema/shared'; @@ -74,6 +75,7 @@ type EditFormArgs = { status?: StatusType; name?: string; permissions?: any; + dataFromDeployedVersion?: boolean; }; /** @@ -88,6 +90,7 @@ export default { status: { type: StatusEnumType }, name: { type: GraphQLString }, permissions: { type: GraphQLJSON }, + dataFromDeployedVersion: { type: GraphQLBoolean }, }, async resolve(parent, args: EditFormArgs, context: Context) { graphQLAuthCheck(context); @@ -208,6 +211,14 @@ export default { } } + // Update kobo dataFromDeployedVersion + if (!isNil(args.dataFromDeployedVersion)) { + update.kobo = { + ...form.kobo, + dataFromDeployedVersion: args.dataFromDeployedVersion, + }; + } + // Update fields and structure, check that structure is different if (args.structure && !isEqual(form.structure, args.structure)) { update.structure = args.structure; diff --git a/src/schema/types/koboForm.type.ts b/src/schema/types/koboForm.type.ts index ef09388b9..e90d10398 100644 --- a/src/schema/types/koboForm.type.ts +++ b/src/schema/types/koboForm.type.ts @@ -1,4 +1,4 @@ -import { GraphQLObjectType, GraphQLString } from 'graphql'; +import { GraphQLBoolean, GraphQLObjectType, GraphQLString } from 'graphql'; import { ApiConfigurationType } from './apiConfiguration.type'; import { AppAbility } from '@security/defineUserAbility'; import { ApiConfiguration } from '@models'; @@ -10,6 +10,7 @@ export const KoboFormType = new GraphQLObjectType({ fields: () => ({ id: { type: GraphQLString }, deployedVersionId: { type: GraphQLString }, + dataFromDeployedVersion: { type: GraphQLBoolean }, apiConfiguration: { type: ApiConfigurationType, async resolve(parent, args, context) { From bbcc93999b9a5b2c58628a7dc5886e9b94d885a2 Mon Sep 17 00:00:00 2001 From: Estela Silva Date: Fri, 12 Jul 2024 14:38:49 -0300 Subject: [PATCH 25/27] wip: created addRecordsFromKobo mutation --- src/models/record.model.ts | 2 + .../mutation/addRecordsFromKobo.mutation.ts | 209 ++++++++++++++++++ src/schema/mutation/index.ts | 2 + src/schema/types/koboForm.type.ts | 2 +- src/schema/types/record.type.ts | 1 + 5 files changed, 215 insertions(+), 1 deletion(-) create mode 100644 src/schema/mutation/addRecordsFromKobo.mutation.ts diff --git a/src/models/record.model.ts b/src/models/record.model.ts index bd80c0e88..27abc0cae 100644 --- a/src/models/record.model.ts +++ b/src/models/record.model.ts @@ -26,6 +26,7 @@ export interface Record extends AccessibleFieldsDocument { modifiedAt: Date; archived: boolean; data: any; + koboId?: string; versions: any; permissions: { canSee?: any[]; @@ -59,6 +60,7 @@ const recordSchema = new Schema( type: mongoose.Schema.Types.Mixed, required: true, }, + koboId: String, lastUpdateForm: { type: mongoose.Schema.Types.ObjectId, ref: 'Form', diff --git a/src/schema/mutation/addRecordsFromKobo.mutation.ts b/src/schema/mutation/addRecordsFromKobo.mutation.ts new file mode 100644 index 000000000..b8120a6e4 --- /dev/null +++ b/src/schema/mutation/addRecordsFromKobo.mutation.ts @@ -0,0 +1,209 @@ +import { ApiConfiguration, Form, Record, Version } from '@models'; +import { + GraphQLBoolean, + GraphQLError, + GraphQLID, + GraphQLNonNull, +} from 'graphql'; +import mongoose from 'mongoose'; +import { Context } from '@server/apollo/context'; +import { graphQLAuthCheck } from '@schema/shared'; +import extendAbilityForRecords from '@security/extendAbilityForRecords'; +import { logger } from '@services/logger.service'; +import config from 'config'; +import * as CryptoJS from 'crypto-js'; +import axios from 'axios'; +import { getNextId, transformRecord } from '@utils/form'; +import { cloneDeep, isEqual } from 'lodash'; + +/** Arguments for the addRecordsFromKobo mutation */ +type AddRecordsFromKoboArgs = { + form: string | mongoose.Types.ObjectId; +}; + +/** + * From the kobo data submission, extract the form data. + * + * @param submission kobo data submission + * @param fields Oort form fields + * @param fieldsNames Array with the fields names + * @returns data object for the record + */ +const getData = (submission: any, fields: any, fieldsNames: string[]) => { + // Filter submission object, keeping only questions data + for (const key in submission) { + if (!fieldsNames.includes(key)) { + delete submission[key]; + } + } + return transformRecord(submission, fields); +}; + +/** + * For a form created from a Kobotoolbox form, import data submissions to create records. + * Throw an error if not logged or authorized, or if arguments are invalid. + */ +export default { + // type: new GraphQLList(RecordType), + type: GraphQLBoolean, + args: { + form: { type: new GraphQLNonNull(GraphQLID) }, + }, + async resolve(parent, args: AddRecordsFromKoboArgs, context: Context) { + graphQLAuthCheck(context); + try { + const user = context.user; + // Get the form + const form = await Form.findById(args.form); + if (!form || !form.kobo.id) { + throw new GraphQLError(context.i18next.t('common.errors.dataNotFound')); + } + // Check the ability with permissions for this form + const ability = await extendAbilityForRecords(user, form); + if (ability.cannot('create', 'Record')) { + throw new GraphQLError( + context.i18next.t('common.errors.permissionNotGranted') + ); + } + // Get Kobo data submissions + const apiConfiguration = await ApiConfiguration.findById( + form.kobo.apiConfiguration + ); + if (!apiConfiguration) { + throw new GraphQLError(context.i18next.t('common.errors.dataNotFound')); + } + const url = `https://kf.kobotoolbox.org/api/v2/assets/${form.kobo.id}/data.json`; + const settings = JSON.parse( + CryptoJS.AES.decrypt( + apiConfiguration.settings, + config.get('encryption.key') + ).toString(CryptoJS.enc.Utf8) + ); + const response = await axios.get(url, { + headers: { + Authorization: `${settings.tokenPrefix} ${settings.token}`, + }, + }); + let submissions = response.data.results; + // Only data submissions sent when the form was in the same version as when it was imported will be used to create records + if (form.kobo.dataFromDeployedVersion) { + submissions = submissions.filter( + (submission: any) => + submission.__version__ === form.kobo.deployedVersionId + ); + } + if (!submissions.length) { + // Nothing to synchronize + return false; + } else { + // Get existing records already rested from data synchronization with kobo + const oldRecords = await Record.find({ + form: args.form, + koboId: { $ne: null }, + }); + + const versionsToCreate: Version[] = []; + const recordsToCreate: Record[] = []; + const recordsToUpdate: Record[] = []; + const fieldsNames = form.fields.map((field: any) => field.name); + + for (const submission of submissions) { + const oldRecord = oldRecords.find((rec: Record) => + rec.koboId === submission._uuid || + Object.prototype.hasOwnProperty.call(submission, 'meta/rootUuid') + ? rec.koboId === submission['meta/rootUuid'] + : false + ); + if (oldRecord) { + const recordToUpdate = cloneDeep(oldRecord); + const data = getData(submission, form.fields, fieldsNames); + recordToUpdate.data = { ...recordToUpdate.data, ...data }; + if (!isEqual(oldRecord.data, recordToUpdate.data)) { + const version = new Version({ + createdAt: oldRecord.modifiedAt + ? oldRecord.modifiedAt + : oldRecord.createdAt, + data: oldRecord.data, + createdBy: context.user.id, + }); + recordToUpdate.versions.push(version); + recordToUpdate.markModified('versions'); + recordToUpdate.modifiedAt = new Date(); + versionsToCreate.push(version); + recordsToUpdate.push(recordToUpdate); + } + // Records already exists, check if data changed on Kobo and we need to updated it here + } else { + // Create record from submission data + const koboId = submission._uuid; + const { incrementalId, incID } = await getNextId( + String(form.resource ? form.resource : args.form) + ); + const data = getData(submission, form.fields, fieldsNames); + const record = new Record({ + incrementalId, + incID, + form: args.form, + data, + resource: form.resource ? form.resource : null, + koboId, + createdBy: { + user: user._id, + roles: user.roles.map((x) => x._id), + positionAttributes: user.positionAttributes.map((x) => { + return { + value: x.value, + category: x.category._id, + }; + }), + }, + lastUpdateForm: form.id, + _createdBy: { + user: { + _id: context.user._id, + name: context.user.name, + username: context.user.username, + }, + }, + _form: { + _id: form._id, + name: form.name, + }, + _lastUpdateForm: { + _id: form._id, + name: form.name, + }, + }); + recordsToCreate.push(record); + } + } + + const recordsToSave = [...recordsToCreate, ...recordsToUpdate]; + // If no new data, return + if (!recordsToSave.length) { + return false; + } else { + // Save all changes + try { + await Version.bulkSave(versionsToCreate); + await Record.bulkSave(recordsToSave); + return true; + } catch (err2) { + logger.error(err2.message, { stack: err2.stack }); + throw new GraphQLError( + context.i18next.t('common.errors.internalServerError') + ); + } + } + } + } catch (err) { + logger.error(err.message, { stack: err.stack }); + if (err instanceof GraphQLError) { + throw new GraphQLError(err.message); + } + throw new GraphQLError( + context.i18next.t('common.errors.internalServerError') + ); + } + }, +}; diff --git a/src/schema/mutation/index.ts b/src/schema/mutation/index.ts index 992e550e2..7f8820a37 100644 --- a/src/schema/mutation/index.ts +++ b/src/schema/mutation/index.ts @@ -89,6 +89,7 @@ import restorePage from './restorePage.mutation'; import addDraftRecord from './addDraftRecord.mutation'; import deleteDraftRecord from './deleteDraftRecord.mutation'; import editDraftRecord from './editDraftRecord.mutation'; +import addRecordsFromKobo from './addRecordsFromKobo.mutation'; /** GraphQL mutation definition */ const Mutation = new GraphQLObjectType({ @@ -184,6 +185,7 @@ const Mutation = new GraphQLObjectType({ addDraftRecord, deleteDraftRecord, editDraftRecord, + addRecordsFromKobo, }, }); diff --git a/src/schema/types/koboForm.type.ts b/src/schema/types/koboForm.type.ts index e90d10398..6c67d7317 100644 --- a/src/schema/types/koboForm.type.ts +++ b/src/schema/types/koboForm.type.ts @@ -6,7 +6,7 @@ import { accessibleBy } from '@casl/mongoose'; /** GraphQL Kobo form type definition */ export const KoboFormType = new GraphQLObjectType({ - name: 'Geospatial', + name: 'KoboForm', fields: () => ({ id: { type: GraphQLString }, deployedVersionId: { type: GraphQLString }, diff --git a/src/schema/types/record.type.ts b/src/schema/types/record.type.ts index 9aaa7fe8a..f8a25f82b 100644 --- a/src/schema/types/record.type.ts +++ b/src/schema/types/record.type.ts @@ -23,6 +23,7 @@ export const RecordType = new GraphQLObjectType({ createdAt: { type: GraphQLString }, modifiedAt: { type: GraphQLString }, archived: { type: GraphQLBoolean }, + koboId: { type: GraphQLString }, form: { type: FormType, async resolve(parent, args, context) { From 70ee57b635e3bc7ce98759a75dbf9d6d4118ba26 Mon Sep 17 00:00:00 2001 From: Estela Silva Date: Mon, 15 Jul 2024 13:49:05 -0300 Subject: [PATCH 26/27] fix: updating records on addRecordsFromKobo conflicts with ids + added channel verification --- .../mutation/addRecordsFromKobo.mutation.ts | 48 +++++++++++++++---- 1 file changed, 39 insertions(+), 9 deletions(-) diff --git a/src/schema/mutation/addRecordsFromKobo.mutation.ts b/src/schema/mutation/addRecordsFromKobo.mutation.ts index b8120a6e4..d12811b5b 100644 --- a/src/schema/mutation/addRecordsFromKobo.mutation.ts +++ b/src/schema/mutation/addRecordsFromKobo.mutation.ts @@ -1,4 +1,11 @@ -import { ApiConfiguration, Form, Record, Version } from '@models'; +import { + ApiConfiguration, + Channel, + Notification, + Form, + Record, + Version, +} from '@models'; import { GraphQLBoolean, GraphQLError, @@ -15,6 +22,7 @@ import * as CryptoJS from 'crypto-js'; import axios from 'axios'; import { getNextId, transformRecord } from '@utils/form'; import { cloneDeep, isEqual } from 'lodash'; +import pubsub from '../../server/pubsub'; /** Arguments for the addRecordsFromKobo mutation */ type AddRecordsFromKoboArgs = { @@ -81,7 +89,8 @@ export default { ); const response = await axios.get(url, { headers: { - Authorization: `${settings.tokenPrefix} ${settings.token}`, + // settings.tokenPrefix MUST be 'Token' + Authorization: `Token ${settings.token}`, }, }); let submissions = response.data.results; @@ -108,12 +117,15 @@ export default { const fieldsNames = form.fields.map((field: any) => field.name); for (const submission of submissions) { - const oldRecord = oldRecords.find((rec: Record) => - rec.koboId === submission._uuid || - Object.prototype.hasOwnProperty.call(submission, 'meta/rootUuid') - ? rec.koboId === submission['meta/rootUuid'] - : false - ); + const oldRecord = oldRecords.find((rec: Record) => { + const submissionId = Object.prototype.hasOwnProperty.call( + submission, + 'meta/rootUuid' + ) + ? submission['meta/rootUuid'] + : submission._uuid; + return rec.koboId === submissionId; + }); if (oldRecord) { const recordToUpdate = cloneDeep(oldRecord); const data = getData(submission, form.fields, fieldsNames); @@ -135,7 +147,12 @@ export default { // Records already exists, check if data changed on Kobo and we need to updated it here } else { // Create record from submission data - const koboId = submission._uuid; + const koboId = Object.prototype.hasOwnProperty.call( + submission, + 'meta/rootUuid' + ) + ? submission['meta/rootUuid'] + : submission._uuid; const { incrementalId, incID } = await getNextId( String(form.resource ? form.resource : args.form) ); @@ -187,6 +204,19 @@ export default { try { await Version.bulkSave(versionsToCreate); await Record.bulkSave(recordsToSave); + // Send notifications to channel + const channel = await Channel.findOne({ form: form._id }); + if (channel) { + const notification = new Notification({ + action: `Records created from Kobo synchronized data submissions - ${form.name}`, + content: '', + channel: channel.id, + seenBy: [], + }); + await notification.save(); + const publisher = await pubsub(); + publisher.publish(channel.id, { notification }); + } return true; } catch (err2) { logger.error(err2.message, { stack: err2.stack }); From 3b55947ff7bde9739013387f6dc643b69a38174f Mon Sep 17 00:00:00 2001 From: Estela Silva Date: Fri, 19 Jul 2024 15:27:50 -0300 Subject: [PATCH 27/27] remove comment --- src/schema/mutation/addRecordsFromKobo.mutation.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/schema/mutation/addRecordsFromKobo.mutation.ts b/src/schema/mutation/addRecordsFromKobo.mutation.ts index d12811b5b..4e19fe9b8 100644 --- a/src/schema/mutation/addRecordsFromKobo.mutation.ts +++ b/src/schema/mutation/addRecordsFromKobo.mutation.ts @@ -52,7 +52,6 @@ const getData = (submission: any, fields: any, fieldsNames: string[]) => { * Throw an error if not logged or authorized, or if arguments are invalid. */ export default { - // type: new GraphQLList(RecordType), type: GraphQLBoolean, args: { form: { type: new GraphQLNonNull(GraphQLID) },