diff --git a/cdk/hrm-search-index-stack.ts b/cdk/hrm-search-index-stack.ts new file mode 100644 index 000000000..210182d22 --- /dev/null +++ b/cdk/hrm-search-index-stack.ts @@ -0,0 +1,154 @@ +/** + * Copyright (C) 2021-2023 Technology Matters + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see https://www.gnu.org/licenses/. + */ + +import * as cdk from 'aws-cdk-lib'; +import * as lambdaNode from 'aws-cdk-lib/aws-lambda-nodejs'; +import { SqsEventSource } from 'aws-cdk-lib/aws-lambda-event-sources'; +import * as sqs from 'aws-cdk-lib/aws-sqs'; +import * as s3 from 'aws-cdk-lib/aws-s3'; + +export default class ContactRetrieveStack extends cdk.Stack { + constructor({ + scope, + id, + params = { + skipLambda: false, + }, + props, + }: { + scope: cdk.Construct; + id: string; + params: { + completeQueue: sqs.Queue; + docsBucket: s3.Bucket; + skipLambda?: boolean; + }; + props?: cdk.StackProps; + }) { + super(scope, id, props); + + const queue = new cdk.aws_sqs.Queue(this, id, { + deadLetterQueue: { maxReceiveCount: 1, queue: params.completeQueue }, + }); + + new cdk.CfnOutput(this, `queueUrl`, { + value: queue.queueUrl, + description: `The url of the ${id} queue`, + }); + + new cdk.aws_ssm.StringParameter(this, `queue-url-consumer`, { + parameterName: `/local/us-east-1/sqs/jobs/hrm-search-index/queue-url-consumer`, + stringValue: queue.queueUrl, + }); + + // duplicated for test env + new cdk.aws_ssm.StringParameter(this, `queue-url-consumer-test`, { + parameterName: `/local/us-east-1/sqs/jobs/hrm-search-index/queue-url-consumer-test`, + stringValue: queue.queueUrl, + }); + + if (params.skipLambda) return; + + /* + Here, there be dragons. + + To use the queue urls from inside of a lambda, we have to replace + 'localhost' with 'localstack' so that container to container dns + lookups resolve correctly on the docker network. + + This is WAY more complicated than it should be and took way too long + to figure out. But here goes: + + CDK passes around tokens that are used inside of the final CloudFormation + template that is generated and deployed, not the actual string values for + things like queue urls that aren't known until the deployment is partially + complete. + + Tokens are unresolvable until they are applied in the produced CF template. + So, you can't just do normal string replace operations. You have to use + native cloudformation functions to manipulate the string. + + BUUUT. There is no "replace" function in cloudformation. So you have + to use split/join to do a janky replace. + + Also... for some reason we can't use "Fn::split" inside of a "Fn::Join" + function directly. We have to use the select to iterate over items in the + split or else we just get a string of "Fn::split" as our url. I have no idea + why, but i discovered this working pattern by trial and error mixed with reviewing + generate cloudformation templates from the CDK that used the join/split replace + pattern. + + This is pretty fragile since we can't arbitrarily split/join if there are + multiple instances of the needle in the haystack. But it works for this + simple case. + (rbd 08/10/22) + */ + const splitCompleteQueueUrl = cdk.Fn.split( + 'localhost', + params.completeQueue.queueUrl, + ); + const completedQueueUrl = cdk.Fn.join('localstack', [ + cdk.Fn.select(0, splitCompleteQueueUrl), + cdk.Fn.select(1, splitCompleteQueueUrl), + ]); + + const fn = new lambdaNode.NodejsFunction(this, 'search-index-consumer', { + // TODO: change this back to 18 once it isn't broken upstream + runtime: cdk.aws_lambda.Runtime.NODEJS_16_X, + memorySize: 512, + timeout: cdk.Duration.seconds(10), + handler: 'handler', + entry: `./hrm-domain/lambdas/search-index-consumer/index.ts`, + environment: { + NODE_OPTIONS: '--enable-source-maps', + S3_ENDPOINT: 'http://localstack:4566', + S3_FORCE_PATH_STYLE: 'true', + S3_REGION: 'us-east-1', + SSM_ENDPOINT: 'http://localstack:4566', + SQS_ENDPOINT: 'http://localstack:4566', + NODE_ENV: 'local', + completed_sqs_queue_url: completedQueueUrl, + }, + bundling: { sourceMap: true }, + deadLetterQueueEnabled: true, + deadLetterQueue: params.completeQueue, + }); + + fn.addEventSource( + new SqsEventSource(queue, { batchSize: 10, reportBatchItemFailures: true }), + ); + + fn.addToRolePolicy( + new cdk.aws_iam.PolicyStatement({ + actions: ['ssm:GetParametersByPath'], + resources: [`arn:aws:ssm:${this.region}:*:parameter/local/*`], + }), + ); + + fn.addToRolePolicy( + new cdk.aws_iam.PolicyStatement({ + actions: [ + 's3:PutObject', + 's3:PutObjectAcl', + 's3:GetObject', + 's3:GetObjectAcl', + 's3:DeleteObject', + ], + resources: [params.docsBucket.bucketArn], + }), + ); + } +} diff --git a/cdk/init-stack.ts b/cdk/init-stack.ts index 76bb6c7da..ce49d9934 100644 --- a/cdk/init-stack.ts +++ b/cdk/init-stack.ts @@ -22,6 +22,7 @@ import * as dotenv from 'dotenv'; import ContactCompleteStack from './contact-complete-stack'; import ContactCoreStack from './contact-core-stack'; import ContactRetrieveStack from './contact-retrieve-stack'; +import HrmSearchIndexStack from './hrm-search-index-stack'; import HrmMicoservicesStack from './hrm-micoroservices-stack'; import LocalCoreStack from './local-core-stack'; import ResourcesCoreStack from './resources-core-stack'; @@ -100,6 +101,18 @@ async function main() { }, }); + new HrmSearchIndexStack({ + scope: app, + id: 'hrm-search-index', + params: { + completeQueue: contactComplete.completeQueue, + docsBucket: localCore.docsBucket, + }, + props: { + env: { region: app.node.tryGetContext('region') }, + }, + }); + new ResourcesCoreStack({ scope: app, id: 'resources-core', diff --git a/hrm-domain/hrm-core/case/caseDataAccess.ts b/hrm-domain/hrm-core/case/caseDataAccess.ts index 376d70aa3..0ae995971 100644 --- a/hrm-domain/hrm-core/case/caseDataAccess.ts +++ b/hrm-domain/hrm-core/case/caseDataAccess.ts @@ -29,10 +29,11 @@ import { Contact } from '../contact/contactDataAccess'; import { DateFilter, OrderByDirectionType } from '../sql'; import { TKConditionsSets } from '../permissions/rulesMap'; import { TwilioUser } from '@tech-matters/twilio-worker-auth'; -import { TwilioUserIdentifier } from '@tech-matters/types'; +import { AccountSID, TwilioUserIdentifier } from '@tech-matters/types'; import { PrecalculatedCasePermissionConditions, CaseRecordCommon, + CaseService, } from '@tech-matters/hrm-types'; import { CaseSectionRecord } from './caseSection/types'; import { pick } from 'lodash'; @@ -249,8 +250,8 @@ export const searchByProfileId = generalizedSearchQueryFunction<{ }), ); -export const deleteById = async (id, accountSid) => { - return db.oneOrNone(DELETE_BY_ID, [accountSid, id]); +export const deleteById = async (id: CaseService['id'], accountSid: AccountSID) => { + return db.oneOrNone(DELETE_BY_ID, [accountSid, id]); }; export const updateStatus = async ( diff --git a/hrm-domain/hrm-core/case/caseRoutesV0.ts b/hrm-domain/hrm-core/case/caseRoutesV0.ts index b3aca4ace..b7751f610 100644 --- a/hrm-domain/hrm-core/case/caseRoutesV0.ts +++ b/hrm-domain/hrm-core/case/caseRoutesV0.ts @@ -15,7 +15,6 @@ */ import createError from 'http-errors'; -import * as casesDb from './caseDataAccess'; import * as caseApi from './caseService'; import { publicEndpoint, SafeRouter } from '../permissions'; import { @@ -93,7 +92,7 @@ casesRouter.expressRouter.use('/:caseId/sections', caseSectionRoutesV0); casesRouter.delete('/:id', publicEndpoint, async (req, res) => { const { hrmAccountId } = req; const { id } = req.params; - const deleted = await casesDb.deleteById(id, hrmAccountId); + const deleted = await caseApi.deleteCaseById({ accountSid: hrmAccountId, caseId: id }); if (!deleted) { throw createError(404); } diff --git a/hrm-domain/hrm-core/case/caseSection/caseSectionService.ts b/hrm-domain/hrm-core/case/caseSection/caseSectionService.ts index 520ccabc8..25c00d4b3 100644 --- a/hrm-domain/hrm-core/case/caseSection/caseSectionService.ts +++ b/hrm-domain/hrm-core/case/caseSection/caseSectionService.ts @@ -36,6 +36,7 @@ import { TwilioUser } from '@tech-matters/twilio-worker-auth'; import { RulesFile, TKConditionsSets } from '../../permissions/rulesMap'; import { ListConfiguration } from '../caseDataAccess'; import { HrmAccountId } from '@tech-matters/types'; +import { indexCaseInSearchIndex } from '../caseService'; const sectionRecordToSection = ( sectionRecord: CaseSectionRecord | undefined, @@ -65,7 +66,13 @@ export const createCaseSection = async ( createdAt: nowISO, accountSid, }; - return sectionRecordToSection(await create()(record)); + + const created = await create()(record); + + // trigger index operation but don't await for it + indexCaseInSearchIndex({ accountSid, caseId: parseInt(caseId, 10) }); + + return sectionRecordToSection(created); }; export const replaceCaseSection = async ( @@ -83,15 +90,19 @@ export const replaceCaseSection = async ( updatedBy: workerSid, updatedAt: nowISO, }; - return sectionRecordToSection( - await updateById()( - accountSid, - Number.parseInt(caseId), - sectionType, - sectionId, - record, - ), + + const updated = await updateById()( + accountSid, + Number.parseInt(caseId), + sectionType, + sectionId, + record, ); + + // trigger index operation but don't await for it + indexCaseInSearchIndex({ accountSid, caseId: parseInt(caseId, 10) }); + + return sectionRecordToSection(updated); }; export const getCaseSection = async ( @@ -155,13 +166,16 @@ export const deleteCaseSection = async ( sectionId: string, { user }: { user: TwilioUser }, ): Promise => { - return sectionRecordToSection( - await deleteById()( - accountSid, - Number.parseInt(caseId), - sectionType, - sectionId, - user.workerSid, - ), + const deleted = await deleteById()( + accountSid, + Number.parseInt(caseId), + sectionType, + sectionId, + user.workerSid, ); + + // trigger index operation but don't await for it + indexCaseInSearchIndex({ accountSid, caseId: parseInt(caseId, 10) }); + + return sectionRecordToSection(deleted); }; diff --git a/hrm-domain/hrm-core/case/caseService.ts b/hrm-domain/hrm-core/case/caseService.ts index 7dce4a355..3353f759b 100644 --- a/hrm-domain/hrm-core/case/caseService.ts +++ b/hrm-domain/hrm-core/case/caseService.ts @@ -32,6 +32,7 @@ import { updateStatus, CaseRecordUpdate, updateCaseInfo, + deleteById, } from './caseDataAccess'; import { randomUUID } from 'crypto'; import { InitializedCan } from '../permissions/initializeCanForRules'; @@ -48,6 +49,9 @@ import { import { RulesFile, TKConditionsSets } from '../permissions/rulesMap'; import { CaseSectionRecord } from './caseSection/types'; import { pick } from 'lodash'; +import type { IndexMessage } from '@tech-matters/hrm-search-config'; +import { publishCaseToSearchIndex } from '../jobs/search/publishToSearchIndex'; +import { enablePublishHrmSearchIndex } from '../featureFlags'; export { WELL_KNOWN_CASE_SECTION_NAMES, CaseService, CaseInfoSection }; @@ -267,6 +271,57 @@ const mapEssentialData = }; }; +// TODO: use the factored out version once that's merged +const maxPermissions: { + user: TwilioUser; + can: () => boolean; +} = { + can: () => true, + user: { + accountSid: 'ACxxx', + workerSid: 'WKxxx', + roles: ['supervisor'], + isSupervisor: true, + }, +}; + +const doCaseInSearchIndexOP = + (operation: IndexMessage['operation']) => + async ({ + accountSid, + caseId, + caseRecord, + }: { + accountSid: CaseService['accountSid']; + caseId: CaseService['id']; + caseRecord?: CaseRecord; + }) => { + try { + if (!enablePublishHrmSearchIndex) { + return; + } + + const caseObj = + caseRecord || (await getById(caseId, accountSid, maxPermissions.user, [])); + + if (caseObj) { + await publishCaseToSearchIndex({ + accountSid, + case: caseRecordToCase(caseObj), + operation, + }); + } + } catch (err) { + console.error( + `Error trying to index case: accountSid ${accountSid} caseId ${caseId}`, + err, + ); + } + }; + +export const indexCaseInSearchIndex = doCaseInSearchIndexOP('index'); +const removeCaseInSearchIndex = doCaseInSearchIndexOP('remove'); + export const createCase = async ( body: Partial, accountSid: CaseService['accountSid'], @@ -289,6 +344,9 @@ export const createCase = async ( ); const created = await create(record); + // trigger index operation but don't await for it + indexCaseInSearchIndex({ accountSid, caseId: created.id }); + // A new case is always initialized with empty connected contacts. No need to apply mapContactTransformations here return caseRecordToCase(created); }; @@ -315,6 +373,9 @@ export const updateCaseStatus = async ( const withTransformedContacts = mapContactTransformations({ can, user })(updated); + // trigger index operation but don't await for it + indexCaseInSearchIndex({ accountSid, caseId: updated.id }); + return caseRecordToCase(withTransformedContacts); }; @@ -327,6 +388,9 @@ export const updateCaseOverview = async ( const validOverview = pick(overview, CASE_OVERVIEW_PROPERTIES); const updated = await updateCaseInfo(accountSid, id, validOverview, workerSid); + // trigger index operation but don't await for it + indexCaseInSearchIndex({ accountSid, caseId: updated.id }); + return caseRecordToCase(updated); }; @@ -459,3 +523,18 @@ export const getCasesByProfileId = async ( }); } }; + +export const deleteCaseById = async ({ + accountSid, + caseId, +}: { + accountSid: HrmAccountId; + caseId: number; +}) => { + const deleted = await deleteById(caseId, accountSid); + + // trigger remove operation but don't await for it + removeCaseInSearchIndex({ accountSid, caseId: deleted?.id, caseRecord: deleted }); + + return deleted; +}; diff --git a/hrm-domain/hrm-core/contact-job/contact-job-complete.ts b/hrm-domain/hrm-core/contact-job/contact-job-complete.ts index 85a32fc49..6b9490840 100644 --- a/hrm-domain/hrm-core/contact-job/contact-job-complete.ts +++ b/hrm-domain/hrm-core/contact-job/contact-job-complete.ts @@ -20,6 +20,7 @@ import { completeContactJob, getContactJobById, } from './contact-job-data-access'; +import { updateConversationMediaData } from '../contact/contactService'; import { ContactJobAttemptResult, ContactJobType } from '@tech-matters/types'; import { ContactJobCompleteProcessorError, @@ -41,7 +42,6 @@ import type { import { ConversationMedia, getConversationMediaById, - updateConversationMediaData, } from '../conversation-media/conversation-media'; export const processCompletedRetrieveContactTranscript = async ( @@ -59,7 +59,7 @@ export const processCompletedRetrieveContactTranscript = async ( location: completedJob.attemptPayload, }; - return updateConversationMediaData( + return updateConversationMediaData(completedJob.contactId)( completedJob.accountSid, completedJob.conversationMediaId, storeTypeSpecificData, diff --git a/hrm-domain/hrm-core/contact/contactService.ts b/hrm-domain/hrm-core/contact/contactService.ts index 655631639..c5b0f56d2 100644 --- a/hrm-domain/hrm-core/contact/contactService.ts +++ b/hrm-domain/hrm-core/contact/contactService.ts @@ -51,7 +51,10 @@ import type { TwilioUser } from '@tech-matters/twilio-worker-auth'; import { createReferral } from '../referral/referral-model'; import { createContactJob } from '../contact-job/contact-job'; import { isChatChannel } from '@tech-matters/hrm-types'; -import { enableCreateContactJobsFlag } from '../featureFlags'; +import { + enableCreateContactJobsFlag, + enablePublishHrmSearchIndex, +} from '../featureFlags'; import { db } from '../connection-pool'; import { ConversationMedia, @@ -59,6 +62,7 @@ import { isS3StoredTranscript, isS3StoredTranscriptPending, NewConversationMedia, + updateConversationMediaSpecificData, } from '../conversation-media/conversation-media'; import { Profile, getOrCreateProfileWithIdentifier } from '../profile/profileService'; import { deleteContactReferrals } from '../referral/referral-data-access'; @@ -69,6 +73,8 @@ import { } from '../sql'; import { systemUser } from '@tech-matters/twilio-worker-auth'; import { RulesFile, TKConditionsSets } from '../permissions/rulesMap'; +import type { IndexMessage } from '@tech-matters/hrm-search-config'; +import { publishContactToSearchIndex } from '../jobs/search/publishToSearchIndex'; // Re export as is: export { Contact } from './contactDataAccess'; @@ -172,6 +178,36 @@ const initProfile = async ( }); }; +const doContactInSearchIndexOP = + (operation: IndexMessage['operation']) => + async ({ + accountSid, + contactId, + }: { + accountSid: Contact['accountSid']; + contactId: Contact['id']; + }) => { + try { + if (!enablePublishHrmSearchIndex) { + return; + } + + const contact = await getById(accountSid, contactId); + + if (contact) { + await publishContactToSearchIndex({ accountSid, contact, operation }); + } + } catch (err) { + console.error( + `Error trying to index contact: accountSid ${accountSid} contactId ${contactId}`, + err, + ); + } + }; + +const indexContactInSearchIndex = doContactInSearchIndexOP('index'); +const removeContactInSearchIndex = doContactInSearchIndexOP('remove'); + // Creates a contact with all its related records within a single transaction export const createContact = async ( accountSid: HrmAccountId, @@ -220,6 +256,8 @@ export const createContact = async ( return newOk({ data: applyTransformations(contact) }); }); if (isOk(result)) { + // trigger index operation but don't await for it + indexContactInSearchIndex({ accountSid, contactId: result.data.id }); return result.data; } // This operation can fail with a unique constraint violation if a contact with the same ID is being created concurrently @@ -243,6 +281,7 @@ export const createContact = async ( return result.unwrap(); } } + return result.unwrap(); }; @@ -287,6 +326,9 @@ export const patchContact = async ( const applyTransformations = bindApplyTransformations(can, user); + // trigger index operation but don't await for it + indexContactInSearchIndex({ accountSid, contactId: parseInt(contactId, 10) }); + return applyTransformations(updated); }); @@ -296,6 +338,11 @@ export const connectContactToCase = async ( caseId: string, { can, user }: { can: InitializedCan; user: TwilioUser }, ): Promise => { + if (caseId === null) { + // trigger remove operation, awaiting for it, since we'll lost the information of which is the "old case" otherwise + await removeContactInSearchIndex({ accountSid, contactId: parseInt(contactId, 10) }); + } + const updated: Contact | undefined = await connectToCase()( accountSid, contactId, @@ -307,6 +354,10 @@ export const connectContactToCase = async ( } const applyTransformations = bindApplyTransformations(can, user); + + // trigger index operation but don't await for it + indexContactInSearchIndex({ accountSid, contactId: parseInt(contactId, 10) }); + return applyTransformations(updated); }; @@ -351,6 +402,10 @@ export const addConversationMediaToContact = async ( ...contact, conversationMedia: [...contact.conversationMedia, ...createdConversationMedia], }; + + // trigger index operation but don't await for it + indexContactInSearchIndex({ accountSid, contactId: parseInt(contactIdString, 10) }); + return applyTransformations(updated); }); }; @@ -425,3 +480,25 @@ export const getContactsByProfileId = async ( }); } }; + +/** + * wrapper around updateSpecificData that also triggers a re-index operation when the conversation media gets updated (e.g. when transcript is exported) + */ +export const updateConversationMediaData = + (contactId: Contact['id']) => + async ( + ...[accountSid, id, storeTypeSpecificData]: Parameters< + typeof updateConversationMediaSpecificData + > + ): ReturnType => { + const result = await updateConversationMediaSpecificData( + accountSid, + id, + storeTypeSpecificData, + ); + + // trigger index operation but don't await for it + indexContactInSearchIndex({ accountSid, contactId }); + + return result; + }; diff --git a/hrm-domain/hrm-core/contact/sql/contact-get-sql.ts b/hrm-domain/hrm-core/contact/sql/contact-get-sql.ts index a421285ba..5cc3468c8 100644 --- a/hrm-domain/hrm-core/contact/sql/contact-get-sql.ts +++ b/hrm-domain/hrm-core/contact/sql/contact-get-sql.ts @@ -21,6 +21,9 @@ import { selectCoalesceReferralsByContactId } from '../../referral/sql/referral- const ID_WHERE_CLAUSE = `WHERE c."accountSid" = $ AND c."id" = $`; const TASKID_WHERE_CLAUSE = `WHERE c."accountSid" = $ AND c."taskId" = $`; +/** + * Note: this query is also used to index Contact records in ES. If the JOINs are ever removed from this query, make sure that the JOINs are preserved for the ES dedicated one + */ export const selectContactsWithRelations = (table: string) => ` SELECT c.*, reports."csamReports", joinedReferrals."referrals", media."conversationMedia" FROM "${table}" c diff --git a/hrm-domain/hrm-core/conversation-media/conversation-media-data-access.ts b/hrm-domain/hrm-core/conversation-media/conversation-media-data-access.ts index c02a7eeee..0fc7b2d7b 100644 --- a/hrm-domain/hrm-core/conversation-media/conversation-media-data-access.ts +++ b/hrm-domain/hrm-core/conversation-media/conversation-media-data-access.ts @@ -116,6 +116,9 @@ export const getByContactId = async ( }), ); +/** + * NOTE: This function should not be used, but via the wrapper exposed from contact service. This is because otherwise, no contact re-index will be triggered. + */ export const updateSpecificData = async ( accountSid: HrmAccountId, id: ConversationMedia['id'], diff --git a/hrm-domain/hrm-core/conversation-media/conversation-media.ts b/hrm-domain/hrm-core/conversation-media/conversation-media.ts index 73c683519..a403b520e 100644 --- a/hrm-domain/hrm-core/conversation-media/conversation-media.ts +++ b/hrm-domain/hrm-core/conversation-media/conversation-media.ts @@ -26,5 +26,5 @@ export { create as createConversationMedia, getById as getConversationMediaById, getByContactId as getConversationMediaByContactId, - updateSpecificData as updateConversationMediaData, + updateSpecificData as updateConversationMediaSpecificData, } from './conversation-media-data-access'; diff --git a/hrm-domain/hrm-core/featureFlags.ts b/hrm-domain/hrm-core/featureFlags.ts index 7a1dd0ed3..61116fdf1 100644 --- a/hrm-domain/hrm-core/featureFlags.ts +++ b/hrm-domain/hrm-core/featureFlags.ts @@ -24,3 +24,6 @@ export const enableCleanupJobs = /^true$/i.test(process.env.ENABLE_CLEANUP_JOBS) export const enableProfileFlagsCleanup = /^true$/i.test( process.env.ENABLE_PROFILE_FLAGS_CLEANUP, ); +export const enablePublishHrmSearchIndex = /^true$/i.test( + process.env.ENABLE_PUBLISH_HRM_SEARCH_INDEX, +); diff --git a/hrm-domain/hrm-core/jobs/search/publishToSearchIndex.ts b/hrm-domain/hrm-core/jobs/search/publishToSearchIndex.ts new file mode 100644 index 000000000..04a8af6c7 --- /dev/null +++ b/hrm-domain/hrm-core/jobs/search/publishToSearchIndex.ts @@ -0,0 +1,80 @@ +/** + * Copyright (C) 2021-2023 Technology Matters + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see https://www.gnu.org/licenses/. + */ + +import { sendSqsMessage } from '@tech-matters/sqs-client'; +import { getSsmParameter } from '../../config/ssmCache'; +import { IndexMessage } from '@tech-matters/hrm-search-config'; +import { CaseService, Contact } from '@tech-matters/hrm-types'; +import { AccountSID } from '@tech-matters/types'; + +const PENDING_INDEX_QUEUE_SSM_PATH = `/${process.env.NODE_ENV}/${ + process.env.AWS_REGION ?? process.env.AWS_DEFAULT_REGION +}/sqs/jobs/hrm-search-index/queue-url-consumer`; + +const publishToSearchIndex = async ({ + message, + messageGroupId, +}: { + message: IndexMessage; + messageGroupId: string; +}) => { + try { + console.log( + '>>>> publishToSearchIndex invoked with message: ', + JSON.stringify(message), + ); + const queueUrl = await getSsmParameter(PENDING_INDEX_QUEUE_SSM_PATH); + + return await sendSqsMessage({ + queueUrl, + message: JSON.stringify(message), + messageGroupId, + }); + } catch (err) { + console.error( + `Error trying to send message to SQS queue ${PENDING_INDEX_QUEUE_SSM_PATH}`, + err, + ); + } +}; + +export const publishContactToSearchIndex = async ({ + accountSid, + contact, + operation, +}: { + accountSid: AccountSID; + contact: Contact; + operation: IndexMessage['operation']; +}) => + publishToSearchIndex({ + message: { accountSid, type: 'contact', contact, operation }, + messageGroupId: `${accountSid}-contact-${contact.id}`, + }); + +export const publishCaseToSearchIndex = async ({ + accountSid, + case: caseObj, + operation, +}: { + accountSid: AccountSID; + case: CaseService; + operation: IndexMessage['operation']; +}) => + publishToSearchIndex({ + message: { accountSid, type: 'case', case: caseObj, operation }, + messageGroupId: `${accountSid}-case-${caseObj.id}`, + }); diff --git a/hrm-domain/hrm-core/package.json b/hrm-domain/hrm-core/package.json index efe0e6757..055b26c0a 100644 --- a/hrm-domain/hrm-core/package.json +++ b/hrm-domain/hrm-core/package.json @@ -23,6 +23,7 @@ }, "homepage": "https://github.com/tech-matters/hrm#readme", "dependencies": { + "@tech-matters/hrm-search-config": "^1.0.0", "@tech-matters/hrm-types": "^1.0.0", "@tech-matters/http": "^1.0.0", "@tech-matters/resources-service": "^1.0.0", diff --git a/hrm-domain/hrm-core/setTestEnvVars.js b/hrm-domain/hrm-core/setTestEnvVars.js index 451d6a681..a2d53b13e 100644 --- a/hrm-domain/hrm-core/setTestEnvVars.js +++ b/hrm-domain/hrm-core/setTestEnvVars.js @@ -45,6 +45,7 @@ process.env.STATIC_KEY_ACCOUNT_SID = 'BBC'; process.env.INCLUDE_ERROR_IN_RESPONSE = true; +process.env.ENABLE_PUBLISH_HRM_SEARCH_INDEX = true; process.env.ENABLE_CREATE_CONTACT_JOBS = true; process.env.ENABLE_PROCESS_CONTACT_JOBS = true; process.env.ENABLE_CLEANUP_JOBS = true; diff --git a/hrm-domain/hrm-core/unit-tests/case/caseService.test.ts b/hrm-domain/hrm-core/unit-tests/case/caseService.test.ts index 519766a9b..b05d91851 100644 --- a/hrm-domain/hrm-core/unit-tests/case/caseService.test.ts +++ b/hrm-domain/hrm-core/unit-tests/case/caseService.test.ts @@ -24,6 +24,11 @@ import { workerSid, accountSid } from '../mocks'; import { newTwilioUser } from '@tech-matters/twilio-worker-auth'; import { rulesMap } from '../../permissions'; import { RulesFile } from '../../permissions/rulesMap'; +import * as publishToSearchIndex from '../../jobs/search/publishToSearchIndex'; + +const publishToSearchIndexSpy = jest + .spyOn(publishToSearchIndex, 'publishCaseToSearchIndex') + .mockImplementation(async () => Promise.resolve('Ok') as any); jest.mock('../../case/caseDataAccess'); const baselineCreatedDate = new Date(2013, 6, 13).toISOString(); @@ -54,6 +59,8 @@ test('create case', async () => { accountSid, }; const createSpy = jest.spyOn(caseDb, 'create').mockResolvedValue(createdCaseRecord); + // const getByIdSpy = + jest.spyOn(caseDb, 'getById').mockResolvedValueOnce(createdCaseRecord); const createdCase = await caseApi.createCase(caseToBeCreated, accountSid, workerSid); // any worker & account specified on the object should be overwritten with the ones from the user @@ -68,6 +75,9 @@ test('create case', async () => { userOwnsContact: false, }, }); + + await new Promise(process.nextTick); + expect(publishToSearchIndexSpy).toHaveBeenCalled(); }); describe('searchCases', () => { diff --git a/hrm-domain/hrm-core/unit-tests/contact/contactService.test.ts b/hrm-domain/hrm-core/unit-tests/contact/contactService.test.ts index f0d83bd95..588c51f25 100644 --- a/hrm-domain/hrm-core/unit-tests/contact/contactService.test.ts +++ b/hrm-domain/hrm-core/unit-tests/contact/contactService.test.ts @@ -34,6 +34,17 @@ import { ALWAYS_CAN, OPEN_CONTACT_ACTION_CONDITIONS } from '../mocks'; import '@tech-matters/testing/expectToParseAsDate'; import { openPermissions } from '../../permissions/json-permissions'; import { RulesFile, TKConditionsSets } from '../../permissions/rulesMap'; +import * as publishToSearchIndex from '../../jobs/search/publishToSearchIndex'; + +const flushPromises = async () => { + await new Promise(process.nextTick); + await new Promise(process.nextTick); + await new Promise(process.nextTick); +}; + +const publishToSearchIndexSpy = jest + .spyOn(publishToSearchIndex, 'publishContactToSearchIndex') + .mockImplementation(async () => Promise.resolve('Ok') as any); const accountSid = 'AC-accountSid'; const workerSid = 'WK-WORKER_SID'; @@ -112,21 +123,26 @@ describe('createContact', () => { }; const spyOnContact = ({ - contactMockReturn, + mocks, }: { - contactMockReturn?: ReturnType; + mocks?: { + contactMockReturn: ReturnType; + getContactMock: contactDb.Contact; + }; } = {}) => { - const createContactMock = jest.fn( - contactMockReturn || - (() => Promise.resolve({ contact: mockContact, isNewRecord: true })), - ); - jest.spyOn(contactDb, 'create').mockReturnValue(createContactMock); - - return createContactMock; + const createContactMock = mocks + ? jest.fn(mocks.contactMockReturn) + : jest.fn(() => Promise.resolve({ contact: mockContact, isNewRecord: true })); + const createSpy = jest.spyOn(contactDb, 'create').mockReturnValue(createContactMock); + const getByIdSpy = mocks + ? jest.spyOn(contactDb, 'getById').mockResolvedValueOnce(mocks.getContactMock) + : jest.spyOn(contactDb, 'getById').mockResolvedValueOnce(mockContact); + + return { createContactMock, createSpy, getByIdSpy }; }; test("Passes payload down to data layer with user workerSid used for 'createdBy'", async () => { - const createContactMock = spyOnContact(); + const { createContactMock } = spyOnContact(); const returnValue = await createContact( parameterAccountSid, 'WK-contact-creator', @@ -140,11 +156,13 @@ describe('createContact', () => { identifierId: 1, }); + await flushPromises(); + expect(publishToSearchIndexSpy).toHaveBeenCalled(); expect(returnValue).toStrictEqual(mockContact); }); test("If no identifier record exists for 'number', call createIdentifierAndProfile", async () => { - const createContactMock = spyOnContact(); + const { createContactMock } = spyOnContact(); getIdentifierWithProfilesSpy.mockImplementationOnce(() => async () => null); @@ -168,11 +186,13 @@ describe('createContact', () => { identifierId: 2, }); + await flushPromises(); + expect(publishToSearchIndexSpy).toHaveBeenCalled(); expect(returnValue).toStrictEqual(mockContact); }); test('Missing values are converted to empty strings for several fields', async () => { - const createContactMock = spyOnContact(); + const { createContactMock } = spyOnContact(); const minimalPayload = omit( sampleCreateContactPayload, @@ -202,11 +222,13 @@ describe('createContact', () => { identifierId: undefined, }); + await flushPromises(); + expect(publishToSearchIndexSpy).toHaveBeenCalled(); expect(returnValue).toStrictEqual(mockContact); }); test('Missing timeOfContact value is substituted with current date', async () => { - const createContactMock = spyOnContact(); + const { createContactMock } = spyOnContact(); const payload = omit(sampleCreateContactPayload, 'timeOfContact'); const returnValue = await createContact( @@ -223,11 +245,13 @@ describe('createContact', () => { identifierId: 1, }); + await flushPromises(); + expect(publishToSearchIndexSpy).toHaveBeenCalled(); expect(returnValue).toStrictEqual(mockContact); }); test('queue will be empty if not present', async () => { - const createContactMock = spyOnContact(); + const { createContactMock } = spyOnContact(); const payload = omit(sampleCreateContactPayload, 'queueName'); const legacyPayload = omit(sampleCreateContactPayload, 'queueName'); @@ -245,6 +269,8 @@ describe('createContact', () => { identifierId: 1, }); + await flushPromises(); + expect(publishToSearchIndexSpy).toHaveBeenCalled(); expect(returnValue).toStrictEqual(mockContact); }); }); @@ -252,6 +278,7 @@ describe('createContact', () => { describe('connectContactToCase', () => { test('Returns contact produced by data access layer', async () => { const connectSpy = jest.fn(); + jest.spyOn(contactDb, 'getById').mockResolvedValueOnce(mockContact); connectSpy.mockResolvedValue(mockContact); jest.spyOn(contactDb, 'connectToCase').mockImplementation(() => connectSpy); const result = await connectContactToCase(accountSid, '1234', '4321', ALWAYS_CAN); @@ -261,6 +288,9 @@ describe('connectContactToCase', () => { '4321', ALWAYS_CAN.user.workerSid, ); + + await flushPromises(); + expect(publishToSearchIndexSpy).toHaveBeenCalled(); expect(result).toStrictEqual(mockContact); }); @@ -271,6 +301,7 @@ describe('connectContactToCase', () => { expect( connectContactToCase(accountSid, '1234', '4321', ALWAYS_CAN), ).rejects.toThrow(); + expect(publishToSearchIndexSpy).not.toHaveBeenCalled(); }); }); @@ -296,6 +327,7 @@ describe('patchContact', () => { test('Passes callerInformation, childInformation, caseInformation & categories to data layer as separate properties', async () => { const patchSpy = jest.fn(); jest.spyOn(contactDb, 'patch').mockReturnValue(patchSpy); + jest.spyOn(contactDb, 'getById').mockResolvedValueOnce(mockContact); patchSpy.mockResolvedValue(mockContact); const result = await patchContact( accountSid, @@ -305,6 +337,9 @@ describe('patchContact', () => { samplePatch, ALWAYS_CAN, ); + + await flushPromises(); + expect(publishToSearchIndexSpy).toHaveBeenCalled(); expect(result).toStrictEqual(mockContact); expect(patchSpy).toHaveBeenCalledWith(accountSid, '1234', true, { updatedBy: contactPatcherSid, @@ -328,6 +363,8 @@ describe('patchContact', () => { const patchSpy = jest.fn(); jest.spyOn(contactDb, 'patch').mockReturnValue(patchSpy); patchSpy.mockResolvedValue(undefined); + + expect(publishToSearchIndexSpy).not.toHaveBeenCalled(); expect( patchContact(accountSid, contactPatcherSid, true, '1234', samplePatch, ALWAYS_CAN), ).rejects.toThrow(); diff --git a/hrm-domain/hrm-service/Dockerfile b/hrm-domain/hrm-service/Dockerfile index 1690f12f7..b587c8cc6 100644 --- a/hrm-domain/hrm-service/Dockerfile +++ b/hrm-domain/hrm-service/Dockerfile @@ -35,6 +35,7 @@ RUN apk add --no-cache rsync \ && npx tsc -b tsconfig.build.json \ && cp -r hrm-domain/hrm-service/* /home/node/ \ && mkdir -p /home/node/hrm-domain/ \ + && cp -r hrm-domain/packages /home/node/hrm-domain/packages \ && cp -r hrm-domain/hrm-core /home/node/hrm-domain/hrm-core \ && cp -r hrm-domain/scheduled-tasks /home/node/hrm-domain/scheduled-tasks \ && cp -r packages /home/node/ \ diff --git a/hrm-domain/hrm-service/service-tests/case.test.ts b/hrm-domain/hrm-service/service-tests/case.test.ts index a67c9fb1b..2cc1201d5 100644 --- a/hrm-domain/hrm-service/service-tests/case.test.ts +++ b/hrm-domain/hrm-service/service-tests/case.test.ts @@ -29,7 +29,11 @@ import { CaseService } from '@tech-matters/hrm-core/case/caseService'; import * as caseDb from '@tech-matters/hrm-core/case/caseDataAccess'; import { convertCaseInfoToExpectedInfo } from './case/caseValidation'; -import { mockingProxy, mockSuccessfulTwilioAuthentication } from '@tech-matters/testing'; +import { + mockingProxy, + mockSuccessfulTwilioAuthentication, + newSQSmock, +} from '@tech-matters/testing'; import * as mocks from './mocks'; import { ruleFileActionOverride } from './permissions-overrides'; import { headers, getRequest, getServer, setRules, useOpenRules } from './server'; @@ -44,18 +48,35 @@ const request = getRequest(server); const { case1, case2, accountSid, workerSid } = mocks; -afterAll(done => { - mockingProxy.stop().finally(() => { - server.close(done); - }); -}); +let hrmIndexSQSMock: ReturnType; beforeAll(async () => { await mockingProxy.start(); + const mockttp = await mockingProxy.mockttpServer(); + hrmIndexSQSMock = newSQSmock({ + mockttp, + pathPattern: + /\/(test|local|development)\/xx-fake-1\/sqs\/jobs\/hrm-search-index\/queue-url-consumer/, + }); +}); + +afterAll(async () => { + await hrmIndexSQSMock.teardownSQSMock(); + await mockingProxy.stop(); + server.close(); }); beforeEach(async () => { await mockSuccessfulTwilioAuthentication(workerSid); + await hrmIndexSQSMock.createSQSMockQueue({ + queueName: 'test-hrm-search-index-consumer-pending', + }); +}); + +afterEach(async () => { + // await hrmIndexSQSMock.drestoySQSMockQueue({ + // queueUrl: hrmIndexSQSMock.getMockSQSQueueUrl(), + // }); }); // eslint-disable-next-line @typescript-eslint/no-shadow @@ -102,7 +123,7 @@ describe('/cases route', () => { expect(response.status).toBe(401); expect(response.body.error).toBe('Authorization failed'); }); - test('should return 200', async () => { + test.only('should return 200', async () => { const response = await request.post(route).set(headers).send(case1); expect(response.status).toBe(200); diff --git a/hrm-domain/hrm-service/service-tests/contact-job/contactJobCleanup.test.ts b/hrm-domain/hrm-service/service-tests/contact-job/contactJobCleanup.test.ts index 07d4346ed..cca2b5846 100644 --- a/hrm-domain/hrm-service/service-tests/contact-job/contactJobCleanup.test.ts +++ b/hrm-domain/hrm-service/service-tests/contact-job/contactJobCleanup.test.ts @@ -24,12 +24,10 @@ import { mockSuccessfulTwilioAuthentication, } from '@tech-matters/testing'; import { createContactJob } from '@tech-matters/hrm-core/contact-job/contact-job-data-access'; -import { - isS3StoredTranscriptPending, - updateConversationMediaData, -} from '@tech-matters/hrm-core/conversation-media/conversation-media'; +import { isS3StoredTranscriptPending } from '@tech-matters/hrm-core/conversation-media/conversation-media'; import { S3ContactMediaType } from '@tech-matters/hrm-core/conversation-media/conversation-media'; import { getById as getContactById } from '@tech-matters/hrm-core/contact/contactDataAccess'; +import { updateConversationMediaData } from '@tech-matters/hrm-core/contact/contactService'; import * as cleanupContactJobsApi from '@tech-matters/contact-job-cleanup'; import { completeContactJob, @@ -206,7 +204,7 @@ describe('cleanupContactJobs', () => { job = await completeContactJob({ id: job.id, completionPayload }); job = await backDateJob(job.id); - await updateConversationMediaData( + await updateConversationMediaData(contact.id)( accountSid, job.additionalPayload.conversationMediaId, completionPayload, diff --git a/hrm-domain/hrm-service/service-tests/contact-job/jobTypes/retrieveTranscript.test.ts b/hrm-domain/hrm-service/service-tests/contact-job/jobTypes/retrieveTranscript.test.ts index 69d0af6b8..021944669 100644 --- a/hrm-domain/hrm-service/service-tests/contact-job/jobTypes/retrieveTranscript.test.ts +++ b/hrm-domain/hrm-service/service-tests/contact-job/jobTypes/retrieveTranscript.test.ts @@ -515,7 +515,7 @@ describe('complete retrieve-transcript job type', () => { // ); const updateConversationMediaSpy = jest.spyOn( - conversationMediaApi, + contactApi, 'updateConversationMediaData', ); diff --git a/hrm-domain/hrm-service/setTestEnvVars.js b/hrm-domain/hrm-service/setTestEnvVars.js index 55e7ce4d5..b8321c32f 100644 --- a/hrm-domain/hrm-service/setTestEnvVars.js +++ b/hrm-domain/hrm-service/setTestEnvVars.js @@ -43,9 +43,11 @@ process.env.STATIC_KEY_ACCOUNT_SID = 'BBC'; process.env.INCLUDE_ERROR_IN_RESPONSE = true; +process.env.ENABLE_PUBLISH_HRM_SEARCH_INDEX = true; process.env.ENABLE_CREATE_CONTACT_JOBS = true; process.env.ENABLE_PROCESS_CONTACT_JOBS = true; process.env.ENABLE_CLEANUP_JOBS = true; process.env.AWS_REGION = 'xx-fake-1'; process.env.AWS_ACCESS_KEY_ID = 'mock-access-key'; process.env.AWS_SECRET_ACCESS_KEY = 'mock-secret-key'; +process.env.LOCAL_SQS_PORT = '3010'; diff --git a/hrm-domain/lambdas/search-index-consumer/messagesToPayloads.ts b/hrm-domain/lambdas/search-index-consumer/messagesToPayloads.ts index 0fc19ae9c..8434cd95c 100644 --- a/hrm-domain/lambdas/search-index-consumer/messagesToPayloads.ts +++ b/hrm-domain/lambdas/search-index-consumer/messagesToPayloads.ts @@ -36,7 +36,7 @@ export type PayloadWithMeta = { documentId: number; payload: IndexPayload; messageId: string; - indexHandler: 'indexDocument' | 'updateDocument' | 'updateScript'; + indexHandler: 'indexDocument' | 'updateDocument' | 'updateScript' | 'deleteDocument'; }; export type PayloadsByIndex = { [indexType: string]: PayloadWithMeta[]; @@ -111,49 +111,82 @@ const generatePayloadFromContact = ( ps: PayloadsByIndex, m: ContactIndexingInputData, ): PayloadsByIndex => { - return { - ...ps, - // add an upsert job to HRM_CONTACTS_INDEX_TYPE index - [HRM_CONTACTS_INDEX_TYPE]: [ - ...(ps[HRM_CONTACTS_INDEX_TYPE] ?? []), - { - ...m, - documentId: m.message.contact.id, - payload: { ...m.message, transcript: m.transcript }, - indexHandler: 'updateDocument', - }, - ], - // if associated to a case, add an upsert with script job to HRM_CASES_INDEX_TYPE index - [HRM_CASES_INDEX_TYPE]: m.message.contact.caseId - ? [ - ...(ps[HRM_CASES_INDEX_TYPE] ?? []), + switch (m.message.operation) { + // both operations are handled internally by the hrm-search-config package, so just cascade the cases + case 'index': + case 'remove': { + return { + ...ps, + // add an upsert job to HRM_CONTACTS_INDEX_TYPE index + [HRM_CONTACTS_INDEX_TYPE]: [ + ...(ps[HRM_CONTACTS_INDEX_TYPE] ?? []), { ...m, - documentId: parseInt(m.message.contact.caseId, 10), + documentId: m.message.contact.id, payload: { ...m.message, transcript: m.transcript }, - indexHandler: 'updateScript', + indexHandler: 'updateDocument', }, - ] - : ps[HRM_CASES_INDEX_TYPE] ?? [], - }; + ], + // if associated to a case, add an upsert with script job to HRM_CASES_INDEX_TYPE index + [HRM_CASES_INDEX_TYPE]: m.message.contact.caseId + ? [ + ...(ps[HRM_CASES_INDEX_TYPE] ?? []), + { + ...m, + documentId: parseInt(m.message.contact.caseId, 10), + payload: { ...m.message, transcript: m.transcript }, + indexHandler: 'updateScript', + }, + ] + : ps[HRM_CASES_INDEX_TYPE] ?? [], + }; + } + default: { + return assertExhaustive(m.message.operation); + } + } }; const generatePayloadFromCase = ( ps: PayloadsByIndex, m: CaseIndexingInputData, -): PayloadsByIndex => ({ - ...ps, - // add an upsert job to HRM_CASES_INDEX_TYPE index - [HRM_CASES_INDEX_TYPE]: [ - ...(ps[HRM_CASES_INDEX_TYPE] ?? []), - { - ...m, - documentId: m.message.case.id, - payload: { ...m.message }, - indexHandler: 'updateDocument', - }, - ], -}); +): PayloadsByIndex => { + switch (m.message.operation) { + case 'index': { + return { + ...ps, + // add an upsert job to HRM_CASES_INDEX_TYPE index + [HRM_CASES_INDEX_TYPE]: [ + ...(ps[HRM_CASES_INDEX_TYPE] ?? []), + { + ...m, + documentId: m.message.case.id, + payload: { ...m.message }, + indexHandler: 'updateDocument', + }, + ], + }; + } + case 'remove': { + return { + ...ps, + // add a delete job to HRM_CASES_INDEX_TYPE index + [HRM_CASES_INDEX_TYPE]: [ + ...(ps[HRM_CASES_INDEX_TYPE] ?? []), + { + ...m, + documentId: m.message.case.id, + payload: { ...m.message }, + indexHandler: 'deleteDocument', + }, + ], + }; + } + default: { + return assertExhaustive(m.message.operation); + } + } +}; const messagesToPayloadReducer = ( accum: PayloadsByIndex, diff --git a/hrm-domain/lambdas/search-index-consumer/payloadToIndex.ts b/hrm-domain/lambdas/search-index-consumer/payloadToIndex.ts index d3faca0ee..6575e5e1b 100644 --- a/hrm-domain/lambdas/search-index-consumer/payloadToIndex.ts +++ b/hrm-domain/lambdas/search-index-consumer/payloadToIndex.ts @@ -84,6 +84,18 @@ const handleIndexPayload = result: newOkFromData(result), }; } + case 'deleteDocument': { + const result = await client.deleteDocument({ + id: documentId.toString(), + }); + + return { + accountSid, + indexType, + messageId, + result: newOkFromData(result), + }; + } default: { return assertExhaustive(indexHandler); } diff --git a/hrm-domain/packages/hrm-search-config/convertToIndexDocument.ts b/hrm-domain/packages/hrm-search-config/convertToIndexDocument.ts index f1ae36ced..238aff6b6 100644 --- a/hrm-domain/packages/hrm-search-config/convertToIndexDocument.ts +++ b/hrm-domain/packages/hrm-search-config/convertToIndexDocument.ts @@ -23,13 +23,13 @@ import { import { CreateIndexConvertedDocument } from '@tech-matters/elasticsearch-client'; import { IndexPayload, IndexPayloadCase, IndexPayloadContact } from './payload'; -const filterEmpty = (doc: T): T => +const filterUndefined = (doc: T): T => Object.entries(doc).reduce((accum, [key, value]) => { - if (value) { - return { ...accum, [key]: value }; + if (value === undefined) { + return accum; } - return accum; + return { ...accum, [key]: value }; }, {} as T); export const convertContactToContactDocument = ({ @@ -71,7 +71,7 @@ export const convertContactToContactDocument = ({ // low_boost_global: '', // lowBoostGlobal.join(' '), }; - return filterEmpty(contactDocument); + return filterUndefined(contactDocument); }; const convertCaseToCaseDocument = ({ @@ -130,7 +130,7 @@ const convertCaseToCaseDocument = ({ // low_boost_global: '', // lowBoostGlobal.join(' '), }; - return filterEmpty(caseDocument); + return filterUndefined(caseDocument); }; const convertToContactIndexDocument = (payload: IndexPayload) => { diff --git a/package-lock.json b/package-lock.json index 4d68813c4..ef036281f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -66,6 +66,7 @@ "version": "1.0.0", "license": "AGPL", "dependencies": { + "@tech-matters/hrm-search-config": "^1.0.0", "@tech-matters/hrm-types": "^1.0.0", "@tech-matters/http": "^1.0.0", "@tech-matters/resources-service": "^1.0.0", @@ -17474,6 +17475,7 @@ "@tech-matters/hrm-core": { "version": "file:hrm-domain/hrm-core", "requires": { + "@tech-matters/hrm-search-config": "^1.0.0", "@tech-matters/hrm-types": "^1.0.0", "@tech-matters/http": "^1.0.0", "@tech-matters/resources-service": "^1.0.0", diff --git a/packages/elasticsearch-client/src/client.ts b/packages/elasticsearch-client/src/client.ts index 265d76502..385b7502f 100644 --- a/packages/elasticsearch-client/src/client.ts +++ b/packages/elasticsearch-client/src/client.ts @@ -35,6 +35,10 @@ import { search, SearchExtraParams } from './search'; import { suggest, SuggestExtraParams } from './suggest'; import { SearchConfiguration, IndexConfiguration } from './config'; import { IndicesRefreshResponse } from '@elastic/elasticsearch/lib/api/types'; +import deleteDocument, { + DeleteDocumentExtraParams, + DeleteDocumentResponse, +} from './deleteDocument'; // import { getMockClient } from './mockClient'; type AccountSidOrShortCodeRequired = @@ -108,6 +112,7 @@ export type IndexClient = { indexDocument: (args: IndexDocumentExtraParams) => Promise; updateDocument: (args: UpdateDocumentExtraParams) => Promise; updateScript: (args: UpdateScriptExtraParams) => Promise; + deleteDocument: (args: DeleteDocumentExtraParams) => Promise; refreshIndex: () => Promise; executeBulk: (args: ExecuteBulkExtraParams) => Promise; createIndex: (args: CreateIndexExtraParams) => Promise; @@ -160,6 +165,8 @@ const getClientOrMock = async ({ updateDocument({ ...passThroughConfig, ...args }), updateScript: (args: UpdateScriptExtraParams) => updateScript({ ...passThroughConfig, ...args }), + deleteDocument: (args: DeleteDocumentExtraParams) => + deleteDocument({ ...passThroughConfig, ...args }), executeBulk: (args: ExecuteBulkExtraParams) => executeBulk({ ...passThroughConfig, ...args }), }; diff --git a/packages/elasticsearch-client/src/deleteDocument.ts b/packages/elasticsearch-client/src/deleteDocument.ts new file mode 100644 index 000000000..de54a52e8 --- /dev/null +++ b/packages/elasticsearch-client/src/deleteDocument.ts @@ -0,0 +1,37 @@ +/** + * Copyright (C) 2021-2023 Technology Matters + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see https://www.gnu.org/licenses/. + */ +import { DeleteResponse } from '@elastic/elasticsearch/lib/api/types'; +import { PassThroughConfig } from './client'; + +export type DeleteDocumentExtraParams = { + id: string; +}; + +export type DeleteDocumentParams = PassThroughConfig & DeleteDocumentExtraParams; +export type DeleteDocumentResponse = DeleteResponse; + +export const deleteDocument = async ({ + client, + id, + index, +}: DeleteDocumentParams): Promise => { + return client.delete({ + index, + id, + }); +}; + +export default deleteDocument; diff --git a/packages/testing/index.ts b/packages/testing/index.ts index 63dd9c52f..b3a7494cd 100644 --- a/packages/testing/index.ts +++ b/packages/testing/index.ts @@ -20,3 +20,4 @@ export * from './mockSsm'; import { start, stop, mockttpServer } from './mocking-proxy'; export const mockingProxy = { start, stop, mockttpServer }; import './expectToParseAsDate'; +export { newSQSmock } from './mockSQS'; diff --git a/packages/testing/mockSQS.ts b/packages/testing/mockSQS.ts new file mode 100644 index 000000000..b23009085 --- /dev/null +++ b/packages/testing/mockSQS.ts @@ -0,0 +1,77 @@ +/** + * Copyright (C) 2021-2023 Technology Matters + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see https://www.gnu.org/licenses/. + */ + +import sqslite from 'sqslite'; +import { SQS } from 'aws-sdk'; +import { Mockttp } from 'mockttp'; +import { mockSsmParameters } from './mockSsm'; + +export const newSQSmock = ({ + mockttp, + pathPattern, +}: { + mockttp: Mockttp; + pathPattern: RegExp; +}) => { + const sqsService = sqslite({}); + const sqsClient = new SQS({ + endpoint: `http://localhost:${process.env.LOCAL_SQS_PORT}`, + }); + + let mockSQSQueueUrl: URL; + + const initializeSQSMock = async () => { + await sqsService.listen({ port: parseInt(process.env.LOCAL_SQS_PORT!) }); + await mockSsmParameters(mockttp, [ + { + pathPattern, + // /\/(test|local|development)\/xx-fake-1\/sqs\/jobs\/hrm-resources-search\/queue-url-index/, + valueGenerator: () => mockSQSQueueUrl.toString(), + }, + ]); + }; + + const teardownSQSMock = async () => { + await sqsService.close(); + }; + + const createSQSMockQueue = async ({ queueName }: { queueName: string }) => { + const { QueueUrl } = await sqsClient + .createQueue({ + QueueName: queueName, + }) + .promise(); + mockSQSQueueUrl = new URL(QueueUrl!); + }; + + const drestoySQSMockQueue = async ({ queueUrl }: { queueUrl: URL }) => { + await sqsClient + .deleteQueue({ + QueueUrl: queueUrl.toString(), + }) + .promise(); + }; + + const getMockSQSQueueUrl = () => mockSQSQueueUrl; + + return { + initializeSQSMock, + teardownSQSMock, + createSQSMockQueue, + drestoySQSMockQueue, + getMockSQSQueueUrl, + }; +};