diff --git a/src/models/referenceData.model.ts b/src/models/referenceData.model.ts index efd84dfd9..e5fc2a661 100644 --- a/src/models/referenceData.model.ts +++ b/src/models/referenceData.model.ts @@ -2,6 +2,7 @@ import { AccessibleRecordModel, accessibleRecordsPlugin } from '@casl/mongoose'; import mongoose, { Schema, Document } from 'mongoose'; import { getGraphQLTypeName } from '@utils/validators'; import { referenceDataType } from '@const/enumTypes'; +import { ApiConfiguration } from './apiConfiguration.model'; /** Reference data document interface. */ interface ReferenceDataDocument extends Document { @@ -10,7 +11,7 @@ interface ReferenceDataDocument extends Document { graphQLTypeName: string; modifiedAt: Date; type: string; - apiConfiguration: mongoose.Types.ObjectId; + apiConfiguration: mongoose.Types.ObjectId | ApiConfiguration; query: string; fields: { name: string; type: string; graphQLFieldName: string }[]; valueField: string; diff --git a/src/schema/mutation/editForm.mutation.ts b/src/schema/mutation/editForm.mutation.ts index f1e69f5c2..bf6ca0755 100644 --- a/src/schema/mutation/editForm.mutation.ts +++ b/src/schema/mutation/editForm.mutation.ts @@ -4,6 +4,15 @@ import { GraphQLString, GraphQLError, } from 'graphql'; +import { + cloneDeep, + get, + isArray, + set, + unset, + isEqual, + unionWith, +} from 'lodash'; import GraphQLJSON from 'graphql-type-json'; import { Form, @@ -12,6 +21,7 @@ import { Channel, ReferenceData, DEFAULT_INCREMENTAL_ID_SHAPE, + Record, } from '@models'; import { removeField, @@ -26,16 +36,14 @@ import { validateGraphQLTypeName } from '@utils/validators'; import mongoose from 'mongoose'; import { AppAbility } from '@security/defineUserAbility'; import { status, StatusEnumType, StatusType } from '@const/enumTypes'; -import isEqual from 'lodash/isEqual'; import differenceWith from 'lodash/differenceWith'; -import unionWith from 'lodash/unionWith'; import i18next from 'i18next'; -import { get, isArray } from 'lodash'; import { logger } from '@services/logger.service'; import checkDefaultFields from '@utils/form/checkDefaultFields'; import { preserveChildProperties } from '@utils/form/preserveChildProperties'; import { graphQLAuthCheck } from '@schema/shared'; import { Context } from '@server/apollo/context'; +import { getNewFieldValue } from '@utils/form/getNewFieldValue'; /** * List of keys of the structure's object which we want to inherit to the children forms when they are modified on the core form @@ -543,8 +551,130 @@ export default { }); await version.save(); update.$push = { versions: version._id }; - } + // Update old records + const { + updateOldRecords, + onConversionFail: conFail, + strategyForArrays: arrStrategy, + } = structure as { + updateOldRecords?: boolean; + onConversionFail?: 'skip' | 'archive' | 'ignore'; + strategyForArrays?: 'first' | 'last' | 'random'; + }; + + const onConversionFail = conFail ?? 'skip'; + const strategyForArrays = arrStrategy ?? 'first'; + + if (updateOldRecords) { + const now = new Date(); + const oldFields = form.fields; + const newFields = fields; + + // Update old records, 5000 records at a time + const RECORDS_PER_BATCH = 5000; + const numRecords = await Record.countDocuments({ + form: form.id, + archived: { + $ne: true, + }, + }); + + logger.info( + `Trying to update ${numRecords} records for form ${form.name}...` + ); + for (let i = 0; i < numRecords; i += RECORDS_PER_BATCH) { + logger.info( + `Updating records from ${i + 1} to ${ + i + RECORDS_PER_BATCH + 1 + }...` + ); + const versionsToCreate: Version[] = []; + const recordsToUpdate: Record[] = []; + const records = await Record.find({ + form: form.id, + archived: { + $ne: true, + }, + }) + .limit(RECORDS_PER_BATCH) + .skip(i); + + for (const record of records) { + const data = cloneDeep(record.data || {}); + + // We loop through the old fields and update the data + for (const oldField of oldFields) { + const newField = newFields.find( + (field) => field.name === oldField.name + ); + + // The field has been removed + if (!newField) { + unset(data, oldField.name); + continue; + } + + // The field has been modified + if (!isEqual(oldField, newField)) { + const value = get(data, oldField.name); + + try { + const newValue = await getNewFieldValue( + value, + newField, + context.dataSources, + { + strategyForArrays, + onConversionFail, + } + ); + // If no conversion error, we update the field + set(data, newField.name, newValue); + } catch (_) { + // Conversion failed + switch (onConversionFail) { + case 'ignore': + // We keep the old value + continue; + case 'skip': + // We remove the field value + unset(data, oldField.name); + continue; + case 'archive': + // We archive the record + record.archived = true; + break; + } + } + } + } + + // If record is being archived, we don't update anything + if (!record.archived) { + const newVersion = new Version({ + createdAt: now, + data: cloneDeep(record.data), + createdBy: user._id, + }); + versionsToCreate.push(newVersion); + record.versions.push(newVersion._id); + record.data = data; + } + + if (record.isModified()) { + record.modifiedAt = now; + recordsToUpdate.push(record); + } + } + + // Save all the versions + await Version.bulkSave(versionsToCreate); + // Save the updated records + await Record.bulkSave(recordsToUpdate); + } + } + } const resForm = await Form.findByIdAndUpdate(args.id, update, { new: true, }); diff --git a/src/utils/form/getFieldType.ts b/src/utils/form/getFieldType.ts index 49dbbb3a2..0c235cf42 100644 --- a/src/utils/form/getFieldType.ts +++ b/src/utils/form/getFieldType.ts @@ -18,6 +18,7 @@ export const getFieldType = async (question: { case 'text': return 'text'; case 'number': + case 'range': return 'numeric'; case 'color': return 'color'; @@ -29,6 +30,10 @@ export const getFieldType = async (question: { return 'datetime'; case 'time': return 'time'; + case 'week': + return 'week'; + case 'month': + return 'month'; case 'url': return 'url'; case 'tel': diff --git a/src/utils/form/getNewFieldValue.ts b/src/utils/form/getNewFieldValue.ts new file mode 100644 index 000000000..5d68fae66 --- /dev/null +++ b/src/utils/form/getNewFieldValue.ts @@ -0,0 +1,313 @@ +import { referenceDataType } from '@const/enumTypes'; +import { + ApiConfiguration, + Record as RecordModel, + ReferenceData, + Role, + User, +} from '@models'; +import { CustomAPI } from '@server/apollo/dataSources'; +import { cloneDeep, get, isNil } from 'lodash'; +import { Types } from 'mongoose'; + +type ValidatorSync = (value: T) => boolean; +type ValidatorAsync = (value: T) => Promise; +type Validator = ValidatorSync | ValidatorAsync; + +/** + * Get the choices for a field + * + * @param field The field + * @param dataSources The data sources + * @returns The choices for the field + */ +const getChoices = async ( + field: any, + dataSources: Record +) => { + let items: any[] = []; + const refDataID = field?.referenceData?.id; + + if (refDataID) { + const refData = await ReferenceData.findById(refDataID).populate({ + path: 'apiConfiguration', + model: 'ApiConfiguration', + }); + + if (refData?.type === referenceDataType.static) { + items = refData.data; + } else { + // If the reference data is dynamic, we call the API to get the data + const apiConfiguration = refData.apiConfiguration as ApiConfiguration; + const dataSource: CustomAPI = dataSources[apiConfiguration.name]; + if (dataSource) { + items = await dataSource.getReferenceDataItems( + refData, + apiConfiguration + ); + } + } + + const choiceLabels = field.referenceData.displayField ?? refData.valueField; + + // If the reference data is static, we just map the data to the choices format + return items.map((x) => ({ + text: String(get(x, choiceLabels, x)), + value: String(get(x, refData.valueField, x)), + })); + } + + return get(field, 'choices', []); +}; + +/** + * Get the new value for a field, formatted according to the new field definition + * + * @param value The value to be converted + * @param field New field definition + * @param dataSources The data sources + * @param options Options for the conversion + * @param options.strategyForArrays Strategy to use when the value is an array + * @param options.onConversionFail What to do when the value does not match the new field type + * @returns The new value, formatted according to the new field definition + */ +export const getNewFieldValue = ( + value: unknown, + field: any, + dataSources: Record, + options: { + strategyForArrays: 'first' | 'last' | 'random'; + onConversionFail: 'skip' | 'ignore' | 'archive'; + } +) => { + // If the value is nullish, return null + if (isNil(value)) { + return null; + } + + const type = field.type; + + const getValue = async (opt: { + useArray?: boolean; + keepPreviousValue?: boolean; + validator?: Validator; + converter?: (value: unknown) => T; + }): Promise => { + const { useArray, keepPreviousValue, validator, converter } = opt; + let val = value; + // Nullish values are valid + if (isNil(val)) { + return null; + } + + // Update the value if it is an array + if (!useArray && Array.isArray(val)) { + const array = val as any[]; + const strategy = options.strategyForArrays; + + switch (strategy) { + case 'first': + val = array[0]; + break; + case 'last': + val = array[array.length - 1]; + break; + case 'random': + val = array[Math.floor(Math.random() * array.length)]; + } + } else if (useArray && !Array.isArray(val)) { + val = [val]; + } + + const previousValue = cloneDeep(val); + // Convert the value if a converter is provided + if (converter) { + val = converter(val); + } + + // Validate the value if a validator is provided + if (validator && !(await validator(val as T))) { + throw new Error('The value does not match the new field type'); + } + + return keepPreviousValue ? previousValue : (val as T); + }; + + switch (type) { + case 'color': + return getValue({ + // Check if it is a valid hex color + validator: (color) => /^#[0-9A-F]{6}$/i.test(color), + converter: (val) => val.toString(), + }); + case 'date': + return getValue({ + validator: (date) => { + const isValid = /^\d{4}-\d{2}-\d{2}$/.test(date); + + if (!isValid) { + return false; + } + + const [year, month, day] = date.split('-').map(Number); + return year > 0 && month > 0 && month < 13 && day > 0 && day < 32; + }, + converter: (val) => val.toString(), + }); + case 'month': + return getValue({ + validator: (date) => { + const isValid = /^\d{4}-\d{2}$/.test(date); + + if (!isValid) { + return false; + } + + const [year, month] = date.split('-').map(Number); + return year > 0 && month > 0 && month < 13; + }, + converter: (val) => val.toString(), + }); + case 'week': + return getValue({ + validator: (date) => /^\d{4}-W\d{2}$/.test(date), + converter: (val) => val.toString(), + }); + case 'time': + case 'datetime-local': + return getValue({ + validator: (date) => !isNaN(date.getTime()), + converter: (val) => new Date(val.toString()), + keepPreviousValue: true, + }); + case 'numeric': + return getValue({ + validator: (num: number) => !isNaN(num), + converter: (val) => Number(val), + }); + case 'email': + return getValue({ + validator: (email: string) => /^[^\s@]+@[^\s@]+\.[^\s@]+$/.test(email), + converter: (val) => val.toString(), + }); + case 'radiogroup': + case 'dropdown': + return getValue({ + validator: async (val: string) => { + const choices = await getChoices(field, dataSources); + return choices.map((c) => `${c.value}`).includes(`${val}`); + }, + converter: (val) => val.toString(), + }); + case 'checkbox': + case 'tagbox': + return getValue({ + useArray: true, + validator: async (val) => { + const choices = await getChoices(field, dataSources); + // If we are skipping invalid values, we remove them from the array + if (options.onConversionFail === 'skip') { + const newVal = val.filter((v) => + choices.map((c) => `${c.value}`).includes(v) + ); + val.splice(0, val.length); + val.push(...newVal); + } else if (options.onConversionFail === 'archive') { + // If not all values are valid, we throw an error + if ( + val.some((v) => !choices.map((c) => `${c.value}`).includes(v)) + ) { + throw new Error('Some values do not match the new field type'); + } + } + + return true; + }, + converter: (val: any[]) => val.map(String), + }); + case 'boolean': + return getValue({ + converter: (val) => (Array.isArray(val) ? val.length > 0 : !!val), + }); + case 'text': + return getValue({ + converter: (val) => val.toString(), + }); + case 'owner': + return getValue({ + useArray: true, + converter: (val: any[]) => val.map(String), + validator: async (val) => { + const distinctRoles = new Set(val); + const rolesCount = await Role.countDocuments({ + _id: { + $in: [...distinctRoles], + }, + application: { + $in: field.applications ?? [], + }, + }); + + return rolesCount === distinctRoles.size; + }, + }); + + case 'resource': + return getValue({ + validator: async (val) => { + const recordExists = await RecordModel.exists({ + _id: new Types.ObjectId(val), + resource: new Types.ObjectId(field.resource), + }); + + return !!recordExists; + }, + converter: (val) => val.toString(), + }); + case 'resources': + return getValue({ + useArray: true, + converter: (val: any[]) => val.map(String), + validator: async (val) => { + const records = await RecordModel.find({ + _id: { + $in: val.map((v) => new Types.ObjectId(v)), + }, + resource: new Types.ObjectId(field.resource), + }).select('_id'); + + // If we are skipping invalid values, we remove them from the array + if (options.onConversionFail === 'skip') { + const newVal = records.map((r) => r._id.toString()); + val.splice(0, val.length); + val.push(...newVal); + } else if (options.onConversionFail === 'archive') { + // If not all values are valid, we throw an error + if (records.length !== val.length) { + throw new Error('Some values do not match the new field type'); + } + } + + return true; + }, + }); + case 'users': + return getValue({ + useArray: true, + converter: (val: any[]) => val.map(String), + validator: async (val) => { + const distinctUsers = new Set(val); + const usersCount = await User.countDocuments({ + _id: { + $in: [...distinctUsers], + }, + }); + + return usersCount === distinctUsers.size; + }, + }); + default: + // file, multipletext, paneldynamic, matrices... + throw new Error('Conversion not implemented for this field type'); + } +};