diff --git a/src/i18n/en.json b/src/i18n/en.json index 734a74f19..c477ad01d 100644 --- a/src/i18n/en.json +++ b/src/i18n/en.json @@ -128,6 +128,7 @@ "form": { "add": { "errors": { + "koboForm": "An error occurred when trying to create a new form from a Kobo form", "resourceDuplicated": "An existing resource with that name already exists." } }, diff --git a/src/i18n/test.json b/src/i18n/test.json index bfa77f925..5ed0a9c18 100644 --- a/src/i18n/test.json +++ b/src/i18n/test.json @@ -128,6 +128,7 @@ "form": { "add": { "errors": { + "koboForm": "******", "resourceDuplicated": "******" } }, diff --git a/src/routes/proxy/index.ts b/src/routes/proxy/index.ts index 89c9dca3b..a44cf554d 100644 --- a/src/routes/proxy/index.ts +++ b/src/routes/proxy/index.ts @@ -66,7 +66,9 @@ const proxyAPIRequest = async ( url, method: req.method, headers: { - Authorization: `Bearer ${token}`, + // if user-to-service token may contain prefix so it's get directly in getToken + Authorization: + api.authType == 'user-to-service' ? token : `Bearer ${token}`, 'Content-Type': 'application/json', }, maxRedirects: 35, diff --git a/src/schema/mutation/addForm.mutation.ts b/src/schema/mutation/addForm.mutation.ts index ca4769ed6..a0f4da5b9 100644 --- a/src/schema/mutation/addForm.mutation.ts +++ b/src/schema/mutation/addForm.mutation.ts @@ -5,7 +5,19 @@ import { GraphQLError, } from 'graphql'; import { validateGraphQLTypeName } from '@utils/validators'; -import { Resource, Form, Role, ReferenceData } from '@models'; +import { + extractFields, + extractKoboFields, + findDuplicateFields, +} from '@utils/form'; +import { + Resource, + Form, + Role, + ReferenceData, + ApiConfiguration, + Version, +} from '@models'; import { FormType } from '../types'; import { AppAbility } from '@security/defineUserAbility'; import { status } from '@const/enumTypes'; @@ -13,12 +25,18 @@ import { logger } from '@services/logger.service'; import { graphQLAuthCheck } from '@schema/shared'; import { Types } from 'mongoose'; import { Context } from '@server/apollo/context'; +import axios from 'axios'; +import config from 'config'; +import * as CryptoJS from 'crypto-js'; +import checkDefaultFields from '@utils/form/checkDefaultFields'; /** Arguments for the addForm mutation */ type AddFormArgs = { name: string; resource?: string | Types.ObjectId; template?: string | Types.ObjectId; + apiConfiguration?: string | Types.ObjectId; + kobo?: string; }; /** @@ -31,6 +49,8 @@ export default { name: { type: new GraphQLNonNull(GraphQLString) }, resource: { type: GraphQLID }, template: { type: GraphQLID }, + apiConfiguration: { type: GraphQLID }, + kobo: { type: GraphQLString }, }, async resolve(parent, args: AddFormArgs, context: Context) { graphQLAuthCheck(context); @@ -72,6 +92,77 @@ export default { canUpdateRecords: [], canDeleteRecords: [], }; + try { + if (args.apiConfiguration) { + const apiConfiguration = await ApiConfiguration.findById( + args.apiConfiguration + ); + const url = + apiConfiguration.endpoint + `assets/${args.kobo}?format=json`; + const settings = JSON.parse( + CryptoJS.AES.decrypt( + apiConfiguration.settings, + config.get('encryption.key') + ).toString(CryptoJS.enc.Utf8) + ); + // get kobo form data + const response = await axios.get(url, { + headers: { + Authorization: `${settings.tokenPrefix} ${settings.token}`, + }, + }); + const survey = response.data.content.survey; + const choices = response.data.content.choices; + const title = response.data.name; + + // Get structure from the kobo form + const structure = JSON.stringify( + extractKoboFields(survey, title, choices) + ); + + // Extract fields + const fields = []; + const structureObj = JSON.parse(structure); + for (const page of structureObj.pages) { + await extractFields(page, fields, true); + findDuplicateFields(fields); + } + // Check if default fields are used + checkDefaultFields(fields); + + // Create version with structure + const version = new Version({ + data: structure, + }); + await version.save(); + + // create resource and form + const resource = new Resource({ + name: args.name, + permissions: defaultResourcePermissions, + fields, + }); + await resource.save(); + const form = new Form({ + name: args.name, + graphQLTypeName, + status: status.pending, + resource, + core: true, + permissions: defaultFormPermissions, + structure, + fields, + versions: [version._id], + }); + await form.save(); + return form; + } + } catch (err) { + logger.error(err.message, { stack: err.stack }); + throw new GraphQLError( + context.i18next.t('mutations.form.add.errors.koboForm:') + ); + } try { if (!args.resource) { // Check permission to create resource diff --git a/src/utils/form/extractKoboFields.ts b/src/utils/form/extractKoboFields.ts new file mode 100644 index 000000000..72751d452 --- /dev/null +++ b/src/utils/form/extractKoboFields.ts @@ -0,0 +1,379 @@ +import { mapKoboExpression } from './kobo/mapKoboExpression'; +import { commonProperties } from './kobo/commonProperties'; + +/** + * Saves all groupElement questions equivalent to the panel question. + * Groups and panels can be nested. + */ +const groupElements = []; +/** + * Saves the repeatGroupElement question equivalent to the dynamic panel question. + * Groups and panels can be nested. + */ +const repeatGroupElements = []; + +/** + * SurveyJS survey structure + */ +const survey = { + title: '', + pages: [ + { + name: 'page1', + elements: [], + }, + ], + showQuestionNumbers: 'off', +}; + +/** + * available fields types in kobo that are compatible with oort + */ +const AVAILABLE_TYPES = [ + 'decimal', + 'geopoint', + 'select_multiple', + 'select_one', + 'date', + 'note', + 'begin_score', + 'score__row', + 'text', + 'time', + 'file', + 'integer', + 'datetime', + 'acknowledge', + 'begin_rank', + 'rank__level', + 'range', + 'image', + 'audio', + 'video', + 'geoshape', + 'calculate', + 'begin_group', + 'end_group', + 'begin_repeat', + 'end_repeat', +]; + +/** + * Get all the groups names that are related to a kobo element. + * + * @param groupPath string with the group path of the kobo element (e.g. 'group_au9sz58/group_hl8uz79/ig1') + * @returns array with all the groups in the groupPath of the element. + */ +const getElementsGroups = (groupPath: string) => { + // Split all the paths in the group path + const allPaths = groupPath.split('/'); + // Remove the last element (is the name of the current question) + allPaths.pop(); + return allPaths; + + // NO LONGER USED BECAUSE IN THE INTEGRATION TESTS ALL THE GROUPS ARE RE-NAMED AND THE ID IS NOT THE SAME + // const groupDelimiter = 'group_'; + // Filter parts that start with the groupDelimiter (that are group names) + // return allPaths.filter((part: string) => part.startsWith(groupDelimiter)); +}; + +/** + * Adds a new question/element to the page elements, panel elements or to the dynamic panel template elements + * + * @param newElement element to add + * @param elementPath string with the element path of the kobo element (e.g. 'group_au9sz58/group_hl8uz79/ig1') + * @param parentGroup if element is a group and have a parent group, we already know it's parent group + */ +const addToElements = ( + newElement: any, + elementPath: string | null, + parentGroup?: string +) => { + const groups = elementPath ? getElementsGroups(elementPath) : []; + // If element is not part of a group + if (!groups.length && !parentGroup) { + survey.pages[0].elements.push(newElement); + } else { + // If element is part of a group, find out which one to add it to + const groupElement = groupElements.find( + (element: any) => + element.name === (parentGroup ?? groups[groups.length - 1]) + ); + if (groupElement) { + groupElement.elements.push(newElement); + } else { + const repeatGroupElement = repeatGroupElements.find( + (element: any) => + element.name === (parentGroup ?? groups[groups.length - 1]) + ); + if (repeatGroupElement) { + repeatGroupElement.templateElements.push(newElement); + } + } + } +}; + +/** + * Extract kobo form fields and convert to oort fields + * + * @param koboSurvey Kobo survey structure + * @param title Kobo survey title + * @param choices Kobo choices data for the questions + * @returns oort survey + */ +export const extractKoboFields = ( + koboSurvey: any, + title: string, + choices: any +) => { + survey.title = title; + survey.pages[0].elements = []; + let scoreChoiceId = ''; + let rankChoiceId = ''; + + koboSurvey.map((question: any, index: number) => { + if (AVAILABLE_TYPES.includes(question.type)) { + switch (question.type) { + case 'decimal': { + const newQuestion = { + ...commonProperties(index, question, 'text'), + inputType: 'number', + }; + addToElements(newQuestion, question.$xpath); + break; + } + case 'geoshape': + case 'geopoint': { + const newQuestion = { + ...commonProperties(index, question, 'geospatial'), + }; + addToElements(newQuestion, question.$xpath); + break; + } + case 'select_one': + case 'select_multiple': { + const newQuestion = { + ...commonProperties( + index, + question, + question.type === 'select_multiple' ? 'checkbox' : 'radiogroup' + ), + ...(question.type === 'select_multiple' && { + showSelectAllItem: true, + }), + choices: choices + .filter( + (choice) => question.select_from_list_name === choice.list_name + ) + .map((choice) => ({ + value: choice.$autovalue, + text: choice.label[0], + ...(question.choice_filter && + // If in the Kobo form the choice has the 'other' property, we will not add the visibleIf because of the 'or other=0' in the expression + !Object.prototype.hasOwnProperty.call(choice, 'other') && { + visibleIf: mapKoboExpression( + question.choice_filter, + null, + choice.$autovalue + ), + }), + })), + ...(question.parameters && + question.parameters.split('randomize=')[1]?.includes('true') && { + choicesOrder: 'random', + }), + }; + addToElements(newQuestion, question.$xpath); + break; + } + case 'date': { + const newQuestion = { + ...commonProperties(index, question, 'text'), + inputType: 'date', + }; + addToElements(newQuestion, question.$xpath); + break; + } + case 'note': { + const newQuestion = { + ...commonProperties(index, question, 'expression'), + }; + addToElements(newQuestion, question.$xpath); + break; + } + case 'begin_score': { + scoreChoiceId = question['kobo--score-choices']; + break; + } + case 'score__row': { + const newQuestion = { + ...commonProperties(index, question, 'radiogroup'), + choices: choices + .filter((choice) => scoreChoiceId === choice.list_name) + .map((choice) => ({ + value: choice.$autovalue, + text: choice.label[0], + })), + // This question does not have Validation Criteria settings (validators property) + }; + addToElements(newQuestion, question.$xpath); + break; + } + case 'begin_rank': { + rankChoiceId = question['kobo--rank-items']; + break; + } + case 'rank__level': { + const newQuestion = { + ...commonProperties(index, question, 'dropdown'), + choices: choices + .filter((choice) => rankChoiceId === choice.list_name) + .map((choice) => ({ + value: choice.$autovalue, + text: choice.label[0], + })), + // This question does not have Validation Criteria settings + }; + addToElements(newQuestion, question.$xpath); + break; + } + case 'text': { + const newQuestion = { + ...commonProperties(index, question, 'text'), + }; + addToElements(newQuestion, question.$xpath); + break; + } + case 'time': { + const newQuestion = { + ...commonProperties(index, question, 'text'), + inputType: 'time', + }; + addToElements(newQuestion, question.$xpath); + break; + } + case 'audio': + case 'video': + case 'image': + case 'file': { + const newQuestion = { + ...commonProperties(index, question, 'file'), + storeDataAsText: false, + maxSize: 7340032, + }; + addToElements(newQuestion, question.$xpath); + break; + } + case 'integer': { + const newQuestion = { + ...commonProperties(index, question, 'text'), + inputType: 'number', + }; + addToElements(newQuestion, question.$xpath); + break; + } + case 'datetime': { + const newQuestion = { + ...commonProperties(index, question, 'text'), + inputType: 'datetime-local', + }; + addToElements(newQuestion, question.$xpath); + break; + } + case 'acknowledge': { + const newQuestion = { + ...commonProperties(index, question, 'boolean'), + }; + addToElements(newQuestion, question.$xpath); + break; + } + case 'range': { + const newQuestion = { + ...commonProperties(index, question, 'text'), + inputType: 'range', + step: question.parameters.split('step=')[1], + }; + addToElements(newQuestion, question.$xpath); + break; + } + case 'calculate': { + const newQuestion = { + ...commonProperties( + index, + question, + 'expression', + question.calculation + ), + visible: false, // They are not displayed in the Kobo form, so make it invisible by default for the SurveyJS + expression: mapKoboExpression(question.calculation), + // This question does not have hint (description) + }; + addToElements(newQuestion, question.$xpath); + break; + } + case 'begin_group': { + // Get all the groups names that are related to this group + const groups = getElementsGroups(question.$xpath); + const newGroupElement = { + ...commonProperties(index, question, 'panel'), + state: 'expanded', + elements: [], + groupId: question.$kuid, + // If groups still has names, the last one is the directly parent group (i.e. this group is within another group) + parentGroup: groups.length ? groups[groups.length - 1] : null, + }; + groupElements.push(newGroupElement); + break; + } + case 'end_group': { + const groupIndex = groupElements.findIndex( + (element: any) => + element.groupId === question.$kuid.substring(1) || + question.name === element.valueName + ); + const groupElement = groupElements.splice(groupIndex, 1)[0]; + addToElements(groupElement, null, groupElement.parentGroup); + break; + } + case 'begin_repeat': { + // Get all the groups names that are related to this group + const groups = getElementsGroups(question.$xpath); + const newRepeatGroupElement = { + ...commonProperties(index, question, 'paneldynamic'), + state: 'expanded', + confirmDelete: true, + panelCount: 1, + templateElements: [], + groupId: question.$kuid, + // If groups still has group names, the last one is the directly parent group (i.e. this group is within another group) + parentGroup: groups.length ? groups[groups.length - 1] : null, + }; + repeatGroupElements.push(newRepeatGroupElement); + break; + } + case 'end_repeat': { + const groupIndex = repeatGroupElements.findIndex( + (element: any) => + element.groupId === question.$kuid.substring(1) || + question.name === element.valueName + ); + const repeatGroupElement = repeatGroupElements.splice( + groupIndex, + 1 + )[0]; + addToElements( + repeatGroupElement, + null, + repeatGroupElement.parentGroup + ); + break; + } + } + } + }); + // Order elements (is necessary because of groups being added to the survey elements only after closed) + survey.pages[0].elements = survey.pages[0].elements.sort((a, b) => + a.index > b.index ? 1 : -1 + ); + return survey; +}; diff --git a/src/utils/form/index.ts b/src/utils/form/index.ts index 5365f87e2..eeab45c23 100644 --- a/src/utils/form/index.ts +++ b/src/utils/form/index.ts @@ -4,6 +4,7 @@ export * from './addField'; export * from './replaceField'; export * from './findDuplicateFields'; export * from './extractFields'; +export * from './extractKoboFields'; export * from './getFieldType'; // === RECORDS === diff --git a/src/utils/form/kobo/commonProperties.ts b/src/utils/form/kobo/commonProperties.ts new file mode 100644 index 000000000..db2fc1004 --- /dev/null +++ b/src/utils/form/kobo/commonProperties.ts @@ -0,0 +1,57 @@ +import { mapKoboExpression } from './mapKoboExpression'; + +/** + * Extract from a Kobo question the constraint properties and return the validators property for an SurveyJS question + * + * @param question Kobo question object + * @returns validators property for an SurveyJS question + */ +const validators = (question: any) => { + return { + validators: [ + { + type: 'expression', + text: question.constraint_message + ? typeof question.constraint_message === 'string' + ? question.constraint_message + : question.constraint_message[0] + : '', + expression: mapKoboExpression(question.constraint, question.$autoname), + }, + ], + validateOnValueChange: true, + }; +}; + +/** + * Extract from a Kobo question the common properties in a object for a SurveyJS question + * + * @param index index/order of the element inside the Kobo structure + * @param question Kobo question object + * @param type type of the questions + * @param title optional title + * @returns the common properties in a object for a SurveyJS question extracted from the Kobo question + */ +export const commonProperties = ( + index: number, + question: any, + type: string, + title?: string +) => { + return { + index, + type, + name: question.$autoname, + title: title ?? (question.label ? question.label[0] : question.$autoname), + valueName: question.$autoname, + isRequired: question.required, + ...(question.hint && { description: question.hint[0] }), + ...(question.default && { + defaultValue: mapKoboExpression(question.default), + }), + ...(question.relevant && { + visibleIf: mapKoboExpression(question.relevant), + }), + ...(question.constraint && validators(question)), + }; +}; diff --git a/src/utils/form/kobo/mapKoboExpression.ts b/src/utils/form/kobo/mapKoboExpression.ts new file mode 100644 index 000000000..9fc06acfb --- /dev/null +++ b/src/utils/form/kobo/mapKoboExpression.ts @@ -0,0 +1,131 @@ +/** + * Maps expressions from kobo questions to a expression format that will work on the SurveyJS. + * + * The numeric operators (Greater than >, Less than <, Greater than or equal to >=, Less than or equal to <=) are used the in same way in Kobo and SurveyJS. + * + * @param koboExpression the initial kobo logic expression + * @param questionName name of the question to replace in the expression + * @param choiceValue value of the choice to replace in the expression + * @returns the mapped logic expression that will work on the SurveyJS form + */ +export const mapKoboExpression = ( + koboExpression: string, + questionName?: string, + choiceValue?: string +) => { + // Replace 'name' with choiceValue in selected expressions (for choice_filter on select questions) + if (choiceValue) { + koboExpression = koboExpression.replace( + /selected\(\$\{(\w+)\}, name\)/g, + `selected(\$\{$1\}, ${choiceValue})` + ); + // If in the Kobo form the choice has a other property, we will remove the 'or other=0' from the choice visibleIf + koboExpression = koboExpression.replace(/or other=0/g, ''); + } + // Replace . with {questionName} + if (questionName) { + // Expressions in Kobo can have ' . ' to indicate that the expression is about the question in which it is defined. + // Example: a Validation Criteria can be ". > 5 ": the value of the question itself must be greater than 5 + koboExpression = koboExpression.replace(/\./g, `{${questionName}}`); + } + // Not contains + koboExpression = koboExpression.replace( + /not\(selected\(\$\{(\w+)\}, '(.*?)'\)\)/g, + "{$1} notcontains '$2'" + ); + // Replace not(...) with !(...) + koboExpression = koboExpression.replace(/not\(/g, '!('); + // Replace mod with % + koboExpression = koboExpression.replace( + /([^\/]*)\s+mod\s+([^\/]*)/g, + (match, before, after) => { + const transformedBefore = before.replace(/\$\{(\w+)\}/g, '{$1}'); + const transformedAfter = after.replace(/\$\{(\w+)\}/g, '{$1}'); + return `${transformedBefore} % ${transformedAfter}`; + } + ); + // Replace div with / + koboExpression = koboExpression.replace( + /([^\/]*)\s+div\s+([^\/]*)/g, + (match, before, after) => { + const transformedBefore = before.replace(/\$\{(\w+)\}/g, '{$1}'); + const transformedAfter = after.replace(/\$\{(\w+)\}/g, '{$1}'); + return `${transformedBefore} / ${transformedAfter}`; + } + ); + // Empty + koboExpression = koboExpression.replace(/\$\{(\w+)\} = ''/g, '{$1} empty'); + // Equal to + koboExpression = koboExpression.replace( + /\$\{(\w+)\} = '(.*?)'/g, + "({$1} = '$2')" + ); + // No empty + koboExpression = koboExpression.replace( + /\$\{(\w+)\} != ''/g, + '{$1} notempty' + ); + // Not equal to + koboExpression = koboExpression.replace( + /\$\{(\w+)\} != '(.*?)'/g, + "{$1} <> '$2'" + ); + // Replace ends-with with endsWith + koboExpression = koboExpression.replace(/ends-with\(/g, 'endsWith('); + // Replace indexed-repeat(...) with indexedRepeat(...) and handle the first parameter to be the question/field name and not the question value + koboExpression = koboExpression.replace( + /indexed-repeat\(\$\{(\w+)\},\s*\$\{(\w+)\},\s*(.*?)\)/g, + 'indexedRepeat($1, {$2}, $3)' + ); + // Replace sum(...) with sumElements(...) and handle the parameter (from question reference to question name only) + koboExpression = koboExpression.replace( + /sum\(\$\{(\w+)\}\)/g, + 'sumElements($1)' + ); + // Replace max(...) with maxElements(...) and handle the parameter (from question reference to question name only) + koboExpression = koboExpression.replace( + /max\(\$\{(\w+)\}\)/g, + 'maxElements($1)' + ); + // Replace min(...) with minElements(...) and handle the parameter (from question reference to question name only) + koboExpression = koboExpression.replace( + /min\(\$\{(\w+)\}\)/g, + 'minElements($1)' + ); + // Replace join(' ', ${...}) with join(' ', ...) (handle the parameter, replacing the question reference for a question name only) + koboExpression = koboExpression.replace( + /join\(\s*'([^']*)'\s*,\s*\$\{(\w+)\}\s*\)/g, + "join('$1', $2)" + ); + // Replace position(..) with {panelIndex} + koboExpression = koboExpression.replace(/position\(\.\.\)/g, '{panelIndex}'); + // Replace count-selected with length + koboExpression = koboExpression.replace(/count-selected\(/g, 'length('); + // Replace of format-date-time with formatDateTime + koboExpression = koboExpression.replace( + /format-date-time\(/g, + 'formatDateTime(' + ); + // Replace if with iif + koboExpression = koboExpression.replace(/if\(/g, 'iif('); + // TODO: FIX not working with expressions like if(${number1} + ${number2} > 10,45,30) + today() + // For calculations with today() + or - days, add addDays() custom function to work on oort + koboExpression = koboExpression.replace( + /today\(\)\s*([\+\-])\s*(\w+)/g, + (match, operator, term) => { + const transformedTerm = term.replace(/\$\{(\w+)\}/g, '{$1}'); + if (operator === '+') { + return `addDays(today(), ${transformedTerm})`; + } else { + return `addDays(today(), -${transformedTerm})`; + } + } + ); + // Replace now() with currentDate() + koboExpression = koboExpression.replace(/now\(\)/g, 'currentDate()'); + // Date values + koboExpression = koboExpression.replace(/date\('(.*?)'\)/g, "'$1'"); + // Replace any remaining ${variable} to {variable} + koboExpression = koboExpression.replace(/\$\{(\w+)\}/g, '{$1}'); + return koboExpression; +}; diff --git a/src/utils/proxy/authManagement.ts b/src/utils/proxy/authManagement.ts index 386935ff6..432507985 100644 --- a/src/utils/proxy/authManagement.ts +++ b/src/utils/proxy/authManagement.ts @@ -99,7 +99,7 @@ export const getToken = async ( } if (apiConfiguration.authType === authType.userToService) { // Retrieve access token from settings, store it and return it - const settings: { token: string } = ping + const settings: { token: string; tokenPrefix: string } = ping ? apiConfiguration.settings : JSON.parse( CryptoJS.AES.decrypt( @@ -107,8 +107,12 @@ export const getToken = async ( config.get('encryption.key') ).toString(CryptoJS.enc.Utf8) ); - cache.set(tokenID, settings.token, 3570); - return settings.token; + const token = + (settings.tokenPrefix ? settings.tokenPrefix : 'Bearer') + + ' ' + + settings.token; + cache.set(tokenID, token, 3570); + return token; } if (apiConfiguration.authType === authType.authorizationCode) { // Making sure to return only string, as access token is typed string | string[]