From 7e0a00f9daf0449ba5f2f68f54d61dd616aaf524 Mon Sep 17 00:00:00 2001 From: Gianfranco Paoloni Date: Fri, 3 May 2024 18:45:58 -0300 Subject: [PATCH 01/55] refactor: move contacts and cases types to HrmTypes module --- hrm-domain/hrm-core/case/caseDataAccess.ts | 25 ++--- hrm-domain/hrm-core/case/caseSection/types.ts | 18 +--- hrm-domain/hrm-core/case/caseService.ts | 66 ++----------- .../hrm-core/contact/contactDataAccess.ts | 18 +--- hrm-domain/hrm-core/contact/contactJson.ts | 24 +---- .../hrm-core/contact/sql/contactInsertSql.ts | 21 +--- .../conversation-media-data-access.ts | 90 +++++------------- .../hrm-core/referral/referral-data-access.ts | 9 +- packages/types/HrmTypes/Case.ts | 95 +++++++++++++++++++ packages/types/HrmTypes/CaseSection.ts | 32 +++++++ packages/types/HrmTypes/Contact.ts | 74 +++++++++++++++ packages/types/HrmTypes/ConversationMedia.ts | 65 +++++++++++++ packages/types/HrmTypes/Referral.ts | 22 +++++ packages/types/HrmTypes/index.ts | 21 ++++ 14 files changed, 358 insertions(+), 222 deletions(-) create mode 100644 packages/types/HrmTypes/Case.ts create mode 100644 packages/types/HrmTypes/CaseSection.ts create mode 100644 packages/types/HrmTypes/Contact.ts create mode 100644 packages/types/HrmTypes/ConversationMedia.ts create mode 100644 packages/types/HrmTypes/Referral.ts create mode 100644 packages/types/HrmTypes/index.ts diff --git a/hrm-domain/hrm-core/case/caseDataAccess.ts b/hrm-domain/hrm-core/case/caseDataAccess.ts index 7e5a34b3e..f390c5cd2 100644 --- a/hrm-domain/hrm-core/case/caseDataAccess.ts +++ b/hrm-domain/hrm-core/case/caseDataAccess.ts @@ -29,29 +29,16 @@ import { Contact } from '../contact/contactDataAccess'; import { DateFilter, OrderByDirectionType } from '../sql'; import { TKConditionsSets } from '../permissions/rulesMap'; import { TwilioUser } from '@tech-matters/twilio-worker-auth'; -import { TwilioUserIdentifier, WorkerSID } from '@tech-matters/types'; +import { TwilioUserIdentifier } from '@tech-matters/types'; +import { + PrecalculatedCasePermissionConditions, + CaseRecordCommon, +} from '@tech-matters/types/HrmTypes'; import { CaseSectionRecord } from './caseSection/types'; import { pick } from 'lodash'; import { HrmAccountId } from '@tech-matters/types'; -export type PrecalculatedCasePermissionConditions = { - isCaseContactOwner: boolean; // Does the requesting user own any of the contacts currently connected to the case? -}; - -export type CaseRecordCommon = { - info: any; - helpline: string; - status: string; - twilioWorkerId: WorkerSID; - createdBy: TwilioUserIdentifier; - updatedBy: TwilioUserIdentifier; - accountSid: HrmAccountId; - createdAt: string; - updatedAt: string; - statusUpdatedAt?: string; - statusUpdatedBy?: string; - previousStatus?: string; -}; +export { PrecalculatedCasePermissionConditions, CaseRecordCommon }; // Exported for testing export const VALID_CASE_CREATE_FIELDS: (keyof CaseRecordCommon)[] = [ diff --git a/hrm-domain/hrm-core/case/caseSection/types.ts b/hrm-domain/hrm-core/case/caseSection/types.ts index 7c373821b..0a12dd3db 100644 --- a/hrm-domain/hrm-core/case/caseSection/types.ts +++ b/hrm-domain/hrm-core/case/caseSection/types.ts @@ -13,23 +13,9 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see https://www.gnu.org/licenses/. */ +import { CaseSectionRecord, CaseSection } from '@tech-matters/types/HrmTypes'; -import { HrmAccountId } from '@tech-matters/types'; - -export type CaseSectionRecord = { - caseId: number; - sectionType: string; - sectionId: string; - sectionTypeSpecificData: Record; - accountSid: HrmAccountId; - createdAt: string; - createdBy: string; - updatedAt?: string; - updatedBy?: string; - eventTimestamp: string; -}; - -export type CaseSection = Omit; +export { CaseSectionRecord, CaseSection }; export type CaseSectionUpdate = Omit< CaseSection, diff --git a/hrm-domain/hrm-core/case/caseService.ts b/hrm-domain/hrm-core/case/caseService.ts index dfe111f43..f5050542c 100644 --- a/hrm-domain/hrm-core/case/caseService.ts +++ b/hrm-domain/hrm-core/case/caseService.ts @@ -23,7 +23,6 @@ import { CaseListConfiguration, CaseListFilters, CaseRecord, - CaseRecordCommon, CaseSearchCriteria, SearchQueryFunction, create, @@ -35,77 +34,26 @@ import { updateCaseInfo, } from './caseDataAccess'; import { randomUUID } from 'crypto'; -import type { Contact } from '../contact/contactDataAccess'; import { InitializedCan } from '../permissions/initializeCanForRules'; import type { TwilioUser } from '@tech-matters/twilio-worker-auth'; import { bindApplyTransformations as bindApplyContactTransformations } from '../contact/contactService'; import type { Profile } from '../profile/profileDataAccess'; import type { PaginationQuery } from '../search'; import { HrmAccountId, TResult, newErr, newOk } from '@tech-matters/types'; +import { + WELL_KNOWN_CASE_SECTION_NAMES, + CaseService, + CaseInfoSection, +} from '@tech-matters/types/HrmTypes'; import { RulesFile, TKConditionsSets } from '../permissions/rulesMap'; import { CaseSectionRecord } from './caseSection/types'; import { pick } from 'lodash'; +export { WELL_KNOWN_CASE_SECTION_NAMES, CaseService, CaseInfoSection }; + const CASE_OVERVIEW_PROPERTIES = ['summary', 'followUpDate', 'childIsAtRisk'] as const; type CaseOverviewProperties = (typeof CASE_OVERVIEW_PROPERTIES)[number]; -type CaseSection = Omit; - -type CaseInfoSection = { - id: string; - twilioWorkerId: string; - updatedAt?: string; - updatedBy?: string; -} & Record; - -const getSectionSpecificDataFromNotesOrReferrals = ( - caseSection: CaseInfoSection, -): Record => { - const { - id, - twilioWorkerId, - createdAt, - updatedBy, - updatedAt, - accountSid, - ...sectionSpecificData - } = caseSection; - return sectionSpecificData; -}; - -export const WELL_KNOWN_CASE_SECTION_NAMES = { - households: { getSectionSpecificData: s => s.household, sectionTypeName: 'household' }, - perpetrators: { - getSectionSpecificData: s => s.perpetrator, - sectionTypeName: 'perpetrator', - }, - incidents: { getSectionSpecificData: s => s.incident, sectionTypeName: 'incident' }, - counsellorNotes: { - getSectionSpecificData: getSectionSpecificDataFromNotesOrReferrals, - sectionTypeName: 'note', - }, - referrals: { - getSectionSpecificData: getSectionSpecificDataFromNotesOrReferrals, - sectionTypeName: 'referral', - }, - documents: { getSectionSpecificData: s => s.document, sectionTypeName: 'document' }, -} as const; - -type PrecalculatedPermissions = Record<'userOwnsContact', boolean>; - -type CaseSectionsMap = { - [k in (typeof WELL_KNOWN_CASE_SECTION_NAMES)[keyof typeof WELL_KNOWN_CASE_SECTION_NAMES]['sectionTypeName']]?: CaseSection[]; -}; - -export type CaseService = CaseRecordCommon & { - id: number; - childName?: string; - categories: Record; - precalculatedPermissions?: PrecalculatedPermissions; - connectedContacts?: Contact[]; - sections: CaseSectionsMap; -}; - type RecursivePartial = { [P in keyof T]?: RecursivePartial; }; diff --git a/hrm-domain/hrm-core/contact/contactDataAccess.ts b/hrm-domain/hrm-core/contact/contactDataAccess.ts index 98f42c38a..9898b7cb2 100644 --- a/hrm-domain/hrm-core/contact/contactDataAccess.ts +++ b/hrm-domain/hrm-core/contact/contactDataAccess.ts @@ -23,29 +23,17 @@ import { selectSingleContactByTaskId, } from './sql/contact-get-sql'; import { INSERT_CONTACT_SQL, NewContactRecord } from './sql/contactInsertSql'; -import { ContactRawJson, ReferralWithoutContactId } from './contactJson'; +import { ContactRawJson } from './contactJson'; import type { ITask } from 'pg-promise'; import { txIfNotInOne } from '../sql'; -import { ConversationMedia } from '../conversation-media/conversation-media'; import { TOUCH_CASE_SQL } from '../case/sql/caseUpdateSql'; import { TKConditionsSets } from '../permissions/rulesMap'; import { TwilioUser } from '@tech-matters/twilio-worker-auth'; import { TwilioUserIdentifier, HrmAccountId } from '@tech-matters/types'; -export type ExistingContactRecord = { - id: number; - accountSid: HrmAccountId; - createdAt: string; - finalizedAt?: string; - updatedAt?: string; - updatedBy?: TwilioUserIdentifier; -} & Partial; +import { ExistingContactRecord, Contact } from '@tech-matters/types/HrmTypes'; -export type Contact = ExistingContactRecord & { - csamReports: any[]; - referrals?: ReferralWithoutContactId[]; - conversationMedia?: ConversationMedia[]; -}; +export { ExistingContactRecord, Contact }; export type SearchParameters = { helpline?: string; diff --git a/hrm-domain/hrm-core/contact/contactJson.ts b/hrm-domain/hrm-core/contact/contactJson.ts index 38ac75c0c..98364ce1f 100644 --- a/hrm-domain/hrm-core/contact/contactJson.ts +++ b/hrm-domain/hrm-core/contact/contactJson.ts @@ -13,27 +13,5 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see https://www.gnu.org/licenses/. */ -import { Referral } from '../referral/referral-data-access'; -/** - * This and contained types are copied from Flex - */ -export type ContactRawJson = { - definitionVersion?: string; - callType: string; - childInformation: { - [key: string]: string | boolean; - }; - callerInformation?: { - [key: string]: string | boolean; - }; - categories: Record; - caseInformation: { - [key: string]: string | boolean; - }; - contactlessTask?: { [key: string]: string | boolean }; - referrals?: Referral[]; -}; - -// Represents a referral when part of a contact structure, so no contact ID -export type ReferralWithoutContactId = Omit; +export { ContactRawJson, ReferralWithoutContactId } from '@tech-matters/types/HrmTypes'; diff --git a/hrm-domain/hrm-core/contact/sql/contactInsertSql.ts b/hrm-domain/hrm-core/contact/sql/contactInsertSql.ts index 91c923e49..c9dac3357 100644 --- a/hrm-domain/hrm-core/contact/sql/contactInsertSql.ts +++ b/hrm-domain/hrm-core/contact/sql/contactInsertSql.ts @@ -14,27 +14,10 @@ * along with this program. If not, see https://www.gnu.org/licenses/. */ -import { ContactRawJson } from '../contactJson'; import { selectSingleContactByTaskId } from './contact-get-sql'; -import { TwilioUserIdentifier, WorkerSID } from '@tech-matters/types'; +import { NewContactRecord } from '@tech-matters/types/HrmTypes'; -export type NewContactRecord = { - rawJson: ContactRawJson; - queueName: string; - twilioWorkerId?: WorkerSID; - createdBy?: TwilioUserIdentifier; - helpline?: string; - number?: string; - channel?: string; - conversationDuration: number; - timeOfContact?: string; - taskId: string; - channelSid?: string; - serviceSid?: string; - caseId?: string; - profileId?: number; - identifierId?: number; -}; +export { NewContactRecord }; export const INSERT_CONTACT_SQL = ` WITH existing AS ( diff --git a/hrm-domain/hrm-core/conversation-media/conversation-media-data-access.ts b/hrm-domain/hrm-core/conversation-media/conversation-media-data-access.ts index 18b7d4c51..7dbd580a9 100644 --- a/hrm-domain/hrm-core/conversation-media/conversation-media-data-access.ts +++ b/hrm-domain/hrm-core/conversation-media/conversation-media-data-access.ts @@ -32,73 +32,33 @@ import { import { insertConversationMediaSql } from './sql/conversation-media-insert-sql'; import { updateSpecificDataByIdSql } from './sql/conversation-media-update-sql'; import { HrmAccountId } from '@tech-matters/types'; +import { + S3ContactMediaType, + S3StoredTranscript, + S3StoredRecording, + S3StoredConversationMedia, + ConversationMedia, + NewConversationMedia, + isTwilioStoredMedia, + isS3StoredTranscript, + isS3StoredTranscriptPending, + isS3StoredRecording, + isS3StoredConversationMedia, +} from '@tech-matters/types/HrmTypes'; -/** - * - */ -type ConversationMediaCommons = { - id: number; - contactId: number; - accountSid: HrmAccountId; - createdAt: Date; - updatedAt: Date; -}; - -export enum S3ContactMediaType { - RECORDING = 'recording', - TRANSCRIPT = 'transcript', -} - -type NewTwilioStoredMedia = { - storeType: 'twilio'; - storeTypeSpecificData: { reservationSid: string }; -}; -type TwilioStoredMedia = ConversationMediaCommons & NewTwilioStoredMedia; - -type NewS3StoredTranscript = { - storeType: 'S3'; - storeTypeSpecificData: { - type: S3ContactMediaType.TRANSCRIPT; - location?: { - bucket: string; - key: string; - }; - }; -}; - -type NewS3StoredRecording = { - storeType: 'S3'; - storeTypeSpecificData: { - type: S3ContactMediaType.RECORDING; - location?: { - bucket: string; - key: string; - }; - }; +export { + S3ContactMediaType, + S3StoredTranscript, + S3StoredRecording, + S3StoredConversationMedia, + ConversationMedia, + NewConversationMedia, + isTwilioStoredMedia, + isS3StoredTranscript, + isS3StoredTranscriptPending, + isS3StoredRecording, + isS3StoredConversationMedia, }; -export type S3StoredTranscript = ConversationMediaCommons & NewS3StoredTranscript; -export type S3StoredRecording = ConversationMediaCommons & NewS3StoredRecording; -export type S3StoredConversationMedia = S3StoredTranscript | S3StoredRecording; - -export type ConversationMedia = TwilioStoredMedia | S3StoredConversationMedia; - -export type NewConversationMedia = - | NewTwilioStoredMedia - | NewS3StoredTranscript - | NewS3StoredRecording; - -export const isTwilioStoredMedia = (m: ConversationMedia): m is TwilioStoredMedia => - m.storeType === 'twilio'; -export const isS3StoredTranscript = (m: ConversationMedia): m is S3StoredTranscript => - // eslint-disable-next-line @typescript-eslint/no-use-before-define - m.storeType === 'S3' && m.storeTypeSpecificData?.type === S3ContactMediaType.TRANSCRIPT; -export const isS3StoredTranscriptPending = (m: ConversationMedia) => - isS3StoredTranscript(m) && !m.storeTypeSpecificData?.location; -export const isS3StoredRecording = (m: ConversationMedia): m is S3StoredRecording => - m.storeType === 'S3' && m.storeTypeSpecificData?.type === S3ContactMediaType.RECORDING; -export const isS3StoredConversationMedia = ( - m: ConversationMedia, -): m is S3StoredConversationMedia => isS3StoredTranscript(m) || isS3StoredRecording(m); export const create = (task?) => diff --git a/hrm-domain/hrm-core/referral/referral-data-access.ts b/hrm-domain/hrm-core/referral/referral-data-access.ts index 66c92cede..36d5189ca 100644 --- a/hrm-domain/hrm-core/referral/referral-data-access.ts +++ b/hrm-domain/hrm-core/referral/referral-data-access.ts @@ -14,6 +14,8 @@ * along with this program. If not, see https://www.gnu.org/licenses/. */ +import { Referral } from '@tech-matters/types/HrmTypes'; + import { insertReferralSql } from './sql/referral-insert-sql'; import { DatabaseForeignKeyViolationError, @@ -49,12 +51,7 @@ export class OrphanedReferralError extends Error { } } -export type Referral = { - contactId: string; - resourceId: string; - referredAt: string; - resourceName?: string; -}; +export { Referral }; export const createReferralRecord = (task?) => diff --git a/packages/types/HrmTypes/Case.ts b/packages/types/HrmTypes/Case.ts new file mode 100644 index 000000000..24c0cdb67 --- /dev/null +++ b/packages/types/HrmTypes/Case.ts @@ -0,0 +1,95 @@ +/** + * Copyright (C) 2021-2023 Technology Matters + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see https://www.gnu.org/licenses/. + */ + +import { HrmAccountId, TwilioUserIdentifier, WorkerSID } from '..'; +import { CaseSectionRecord } from './CaseSection'; +import { Contact } from './Contact'; + +type CaseSection = Omit; + +export type PrecalculatedCasePermissionConditions = { + isCaseContactOwner: boolean; // Does the requesting user own any of the contacts currently connected to the case? +}; + +export type CaseRecordCommon = { + info: any; + helpline: string; + status: string; + twilioWorkerId: WorkerSID; + createdBy: TwilioUserIdentifier; + updatedBy: TwilioUserIdentifier; + accountSid: HrmAccountId; + createdAt: string; + updatedAt: string; + statusUpdatedAt?: string; + statusUpdatedBy?: string; + previousStatus?: string; +}; + +export type CaseInfoSection = { + id: string; + twilioWorkerId: string; + updatedAt?: string; + updatedBy?: string; +} & Record; + +const getSectionSpecificDataFromNotesOrReferrals = ( + caseSection: CaseInfoSection, +): Record => { + const { + id, + twilioWorkerId, + createdAt, + updatedBy, + updatedAt, + accountSid, + ...sectionSpecificData + } = caseSection; + return sectionSpecificData; +}; + +export const WELL_KNOWN_CASE_SECTION_NAMES = { + households: { getSectionSpecificData: s => s.household, sectionTypeName: 'household' }, + perpetrators: { + getSectionSpecificData: s => s.perpetrator, + sectionTypeName: 'perpetrator', + }, + incidents: { getSectionSpecificData: s => s.incident, sectionTypeName: 'incident' }, + counsellorNotes: { + getSectionSpecificData: getSectionSpecificDataFromNotesOrReferrals, + sectionTypeName: 'note', + }, + referrals: { + getSectionSpecificData: getSectionSpecificDataFromNotesOrReferrals, + sectionTypeName: 'referral', + }, + documents: { getSectionSpecificData: s => s.document, sectionTypeName: 'document' }, +} as const; + +type PrecalculatedPermissions = Record<'userOwnsContact', boolean>; + +type CaseSectionsMap = { + [k in (typeof WELL_KNOWN_CASE_SECTION_NAMES)[keyof typeof WELL_KNOWN_CASE_SECTION_NAMES]['sectionTypeName']]?: CaseSection[]; +}; + +export type CaseService = CaseRecordCommon & { + id: number; + childName?: string; + categories: Record; + precalculatedPermissions?: PrecalculatedPermissions; + connectedContacts?: Contact[]; + sections: CaseSectionsMap; +}; diff --git a/packages/types/HrmTypes/CaseSection.ts b/packages/types/HrmTypes/CaseSection.ts new file mode 100644 index 000000000..fa9263253 --- /dev/null +++ b/packages/types/HrmTypes/CaseSection.ts @@ -0,0 +1,32 @@ +/** + * Copyright (C) 2021-2023 Technology Matters + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see https://www.gnu.org/licenses/. + */ + +import { HrmAccountId } from '..'; + +export type CaseSectionRecord = { + caseId: number; + sectionType: string; + sectionId: string; + sectionTypeSpecificData: Record; + accountSid: HrmAccountId; + createdAt: string; + createdBy: string; + updatedAt?: string; + updatedBy?: string; + eventTimestamp: string; +}; + +export type CaseSection = Omit; diff --git a/packages/types/HrmTypes/Contact.ts b/packages/types/HrmTypes/Contact.ts new file mode 100644 index 000000000..c02b9fa1f --- /dev/null +++ b/packages/types/HrmTypes/Contact.ts @@ -0,0 +1,74 @@ +/** + * Copyright (C) 2021-2023 Technology Matters + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see https://www.gnu.org/licenses/. + */ +import { HrmAccountId, TwilioUserIdentifier, WorkerSID } from '..'; +import { ConversationMedia } from './ConversationMedia'; +import { Referral } from './Referral'; + +/** + * This and contained types are copied from Flex + */ +export type ContactRawJson = { + definitionVersion?: string; + callType: string; + childInformation: { + [key: string]: string | boolean; + }; + callerInformation?: { + [key: string]: string | boolean; + }; + categories: Record; + caseInformation: { + [key: string]: string | boolean; + }; + contactlessTask?: { [key: string]: string | boolean }; + referrals?: Referral[]; +}; + +// Represents a referral when part of a contact structure, so no contact ID +export type ReferralWithoutContactId = Omit; + +export type NewContactRecord = { + rawJson: ContactRawJson; + queueName: string; + twilioWorkerId?: WorkerSID; + createdBy?: TwilioUserIdentifier; + helpline?: string; + number?: string; + channel?: string; + conversationDuration: number; + timeOfContact?: string; + taskId: string; + channelSid?: string; + serviceSid?: string; + caseId?: string; + profileId?: number; + identifierId?: number; +}; + +export type ExistingContactRecord = { + id: number; + accountSid: HrmAccountId; + createdAt: string; + finalizedAt?: string; + updatedAt?: string; + updatedBy?: TwilioUserIdentifier; +} & Partial; + +export type Contact = ExistingContactRecord & { + csamReports: any[]; + referrals?: ReferralWithoutContactId[]; + conversationMedia?: ConversationMedia[]; +}; diff --git a/packages/types/HrmTypes/ConversationMedia.ts b/packages/types/HrmTypes/ConversationMedia.ts new file mode 100644 index 000000000..17b678248 --- /dev/null +++ b/packages/types/HrmTypes/ConversationMedia.ts @@ -0,0 +1,65 @@ +import { HrmAccountId } from '../HrmAccountId'; + +type ConversationMediaCommons = { + id: number; + contactId: number; + accountSid: HrmAccountId; + createdAt: Date; + updatedAt: Date; +}; + +export enum S3ContactMediaType { + RECORDING = 'recording', + TRANSCRIPT = 'transcript', +} + +type NewTwilioStoredMedia = { + storeType: 'twilio'; + storeTypeSpecificData: { reservationSid: string }; +}; +type TwilioStoredMedia = ConversationMediaCommons & NewTwilioStoredMedia; + +type NewS3StoredTranscript = { + storeType: 'S3'; + storeTypeSpecificData: { + type: S3ContactMediaType.TRANSCRIPT; + location?: { + bucket: string; + key: string; + }; + }; +}; + +type NewS3StoredRecording = { + storeType: 'S3'; + storeTypeSpecificData: { + type: S3ContactMediaType.RECORDING; + location?: { + bucket: string; + key: string; + }; + }; +}; +export type S3StoredTranscript = ConversationMediaCommons & NewS3StoredTranscript; +export type S3StoredRecording = ConversationMediaCommons & NewS3StoredRecording; +export type S3StoredConversationMedia = S3StoredTranscript | S3StoredRecording; + +export type ConversationMedia = TwilioStoredMedia | S3StoredConversationMedia; + +export type NewConversationMedia = + | NewTwilioStoredMedia + | NewS3StoredTranscript + | NewS3StoredRecording; + +export const isTwilioStoredMedia = (m: ConversationMedia): m is TwilioStoredMedia => + m.storeType === 'twilio'; +export const isS3StoredTranscript = (m: ConversationMedia): m is S3StoredTranscript => + // eslint-disable-next-line @typescript-eslint/no-use-before-define + m.storeType === 'S3' && m.storeTypeSpecificData?.type === S3ContactMediaType.TRANSCRIPT; +export const isS3StoredTranscriptPending = (m: ConversationMedia) => + isS3StoredTranscript(m) && !m.storeTypeSpecificData?.location; +export const isS3StoredRecording = (m: ConversationMedia): m is S3StoredRecording => + m.storeType === 'S3' && m.storeTypeSpecificData?.type === S3ContactMediaType.RECORDING; +export const isS3StoredConversationMedia = ( + m: ConversationMedia, +): m is S3StoredConversationMedia => isS3StoredTranscript(m) || isS3StoredRecording(m); diff --git a/packages/types/HrmTypes/Referral.ts b/packages/types/HrmTypes/Referral.ts new file mode 100644 index 000000000..4b79eee96 --- /dev/null +++ b/packages/types/HrmTypes/Referral.ts @@ -0,0 +1,22 @@ +/** + * Copyright (C) 2021-2023 Technology Matters + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see https://www.gnu.org/licenses/. + */ + +export type Referral = { + contactId: string; + resourceId: string; + referredAt: string; + resourceName?: string; +}; diff --git a/packages/types/HrmTypes/index.ts b/packages/types/HrmTypes/index.ts new file mode 100644 index 000000000..8366a362a --- /dev/null +++ b/packages/types/HrmTypes/index.ts @@ -0,0 +1,21 @@ +/** + * Copyright (C) 2021-2023 Technology Matters + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see https://www.gnu.org/licenses/. + */ + +export * from './Contact'; +export * from './Referral'; +export * from './ConversationMedia'; +export * from './Case'; +export * from './CaseSection'; From 0c62562703a9b094851878d9e2aab240df31b8c4 Mon Sep 17 00:00:00 2001 From: Gianfranco Paoloni Date: Fri, 3 May 2024 18:52:54 -0300 Subject: [PATCH 02/55] chore: license header --- packages/types/HrmTypes/ConversationMedia.ts | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/packages/types/HrmTypes/ConversationMedia.ts b/packages/types/HrmTypes/ConversationMedia.ts index 17b678248..a519cf178 100644 --- a/packages/types/HrmTypes/ConversationMedia.ts +++ b/packages/types/HrmTypes/ConversationMedia.ts @@ -1,3 +1,19 @@ +/** + * Copyright (C) 2021-2023 Technology Matters + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see https://www.gnu.org/licenses/. + */ + import { HrmAccountId } from '../HrmAccountId'; type ConversationMediaCommons = { From 03d3a216c176a264ec7498805e944a0dc183c14c Mon Sep 17 00:00:00 2001 From: Gianfranco Paoloni Date: Mon, 6 May 2024 15:36:14 -0300 Subject: [PATCH 03/55] chore: moved hrm types to own package hrm-types --- hrm-domain/hrm-core/case/caseDataAccess.ts | 2 +- hrm-domain/hrm-core/case/caseSection/types.ts | 2 +- hrm-domain/hrm-core/case/caseService.ts | 2 +- .../hrm-core/contact/contactDataAccess.ts | 2 +- hrm-domain/hrm-core/contact/contactJson.ts | 2 +- .../hrm-core/contact/sql/contactInsertSql.ts | 2 +- .../conversation-media-data-access.ts | 2 +- hrm-domain/hrm-core/package.json | 1 + .../hrm-core/referral/referral-data-access.ts | 2 +- hrm-domain/hrm-service/tsconfig.build.json | 1 + .../packages/hrm-search-config/package.json | 1 + .../packages/hrm-types}/Case.ts | 2 +- .../packages/hrm-types}/CaseSection.ts | 2 +- .../packages/hrm-types}/Contact.ts | 2 +- .../packages/hrm-types}/ConversationMedia.ts | 2 +- .../packages/hrm-types}/Referral.ts | 0 .../packages/hrm-types}/index.ts | 0 hrm-domain/packages/hrm-types/package.json | 15 +++++++++++ hrm-domain/packages/hrm-types/tsconfig.json | 8 ++++++ package-lock.json | 27 +++++++++++++++++++ tsconfig.json | 1 + 21 files changed, 66 insertions(+), 12 deletions(-) rename {packages/types/HrmTypes => hrm-domain/packages/hrm-types}/Case.ts (97%) rename {packages/types/HrmTypes => hrm-domain/packages/hrm-types}/CaseSection.ts (95%) rename {packages/types/HrmTypes => hrm-domain/packages/hrm-types}/Contact.ts (96%) rename {packages/types/HrmTypes => hrm-domain/packages/hrm-types}/ConversationMedia.ts (98%) rename {packages/types/HrmTypes => hrm-domain/packages/hrm-types}/Referral.ts (100%) rename {packages/types/HrmTypes => hrm-domain/packages/hrm-types}/index.ts (100%) create mode 100644 hrm-domain/packages/hrm-types/package.json create mode 100644 hrm-domain/packages/hrm-types/tsconfig.json diff --git a/hrm-domain/hrm-core/case/caseDataAccess.ts b/hrm-domain/hrm-core/case/caseDataAccess.ts index f390c5cd2..376d70aa3 100644 --- a/hrm-domain/hrm-core/case/caseDataAccess.ts +++ b/hrm-domain/hrm-core/case/caseDataAccess.ts @@ -33,7 +33,7 @@ import { TwilioUserIdentifier } from '@tech-matters/types'; import { PrecalculatedCasePermissionConditions, CaseRecordCommon, -} from '@tech-matters/types/HrmTypes'; +} from '@tech-matters/hrm-types'; import { CaseSectionRecord } from './caseSection/types'; import { pick } from 'lodash'; import { HrmAccountId } from '@tech-matters/types'; diff --git a/hrm-domain/hrm-core/case/caseSection/types.ts b/hrm-domain/hrm-core/case/caseSection/types.ts index 0a12dd3db..471f0ae6b 100644 --- a/hrm-domain/hrm-core/case/caseSection/types.ts +++ b/hrm-domain/hrm-core/case/caseSection/types.ts @@ -13,7 +13,7 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see https://www.gnu.org/licenses/. */ -import { CaseSectionRecord, CaseSection } from '@tech-matters/types/HrmTypes'; +import { CaseSectionRecord, CaseSection } from '@tech-matters/hrm-types'; export { CaseSectionRecord, CaseSection }; diff --git a/hrm-domain/hrm-core/case/caseService.ts b/hrm-domain/hrm-core/case/caseService.ts index f5050542c..7dce4a355 100644 --- a/hrm-domain/hrm-core/case/caseService.ts +++ b/hrm-domain/hrm-core/case/caseService.ts @@ -44,7 +44,7 @@ import { WELL_KNOWN_CASE_SECTION_NAMES, CaseService, CaseInfoSection, -} from '@tech-matters/types/HrmTypes'; +} from '@tech-matters/hrm-types'; import { RulesFile, TKConditionsSets } from '../permissions/rulesMap'; import { CaseSectionRecord } from './caseSection/types'; import { pick } from 'lodash'; diff --git a/hrm-domain/hrm-core/contact/contactDataAccess.ts b/hrm-domain/hrm-core/contact/contactDataAccess.ts index 9898b7cb2..f301c048a 100644 --- a/hrm-domain/hrm-core/contact/contactDataAccess.ts +++ b/hrm-domain/hrm-core/contact/contactDataAccess.ts @@ -31,7 +31,7 @@ import { TKConditionsSets } from '../permissions/rulesMap'; import { TwilioUser } from '@tech-matters/twilio-worker-auth'; import { TwilioUserIdentifier, HrmAccountId } from '@tech-matters/types'; -import { ExistingContactRecord, Contact } from '@tech-matters/types/HrmTypes'; +import { ExistingContactRecord, Contact } from '@tech-matters/hrm-types'; export { ExistingContactRecord, Contact }; diff --git a/hrm-domain/hrm-core/contact/contactJson.ts b/hrm-domain/hrm-core/contact/contactJson.ts index 98364ce1f..4cdf2d8c9 100644 --- a/hrm-domain/hrm-core/contact/contactJson.ts +++ b/hrm-domain/hrm-core/contact/contactJson.ts @@ -14,4 +14,4 @@ * along with this program. If not, see https://www.gnu.org/licenses/. */ -export { ContactRawJson, ReferralWithoutContactId } from '@tech-matters/types/HrmTypes'; +export { ContactRawJson, ReferralWithoutContactId } from '@tech-matters/hrm-types'; diff --git a/hrm-domain/hrm-core/contact/sql/contactInsertSql.ts b/hrm-domain/hrm-core/contact/sql/contactInsertSql.ts index c9dac3357..bb0cc49cf 100644 --- a/hrm-domain/hrm-core/contact/sql/contactInsertSql.ts +++ b/hrm-domain/hrm-core/contact/sql/contactInsertSql.ts @@ -15,7 +15,7 @@ */ import { selectSingleContactByTaskId } from './contact-get-sql'; -import { NewContactRecord } from '@tech-matters/types/HrmTypes'; +import { NewContactRecord } from '@tech-matters/hrm-types'; export { NewContactRecord }; diff --git a/hrm-domain/hrm-core/conversation-media/conversation-media-data-access.ts b/hrm-domain/hrm-core/conversation-media/conversation-media-data-access.ts index 7dbd580a9..c02a7eeee 100644 --- a/hrm-domain/hrm-core/conversation-media/conversation-media-data-access.ts +++ b/hrm-domain/hrm-core/conversation-media/conversation-media-data-access.ts @@ -44,7 +44,7 @@ import { isS3StoredTranscriptPending, isS3StoredRecording, isS3StoredConversationMedia, -} from '@tech-matters/types/HrmTypes'; +} from '@tech-matters/hrm-types'; export { S3ContactMediaType, diff --git a/hrm-domain/hrm-core/package.json b/hrm-domain/hrm-core/package.json index f70df87e5..efe0e6757 100644 --- a/hrm-domain/hrm-core/package.json +++ b/hrm-domain/hrm-core/package.json @@ -23,6 +23,7 @@ }, "homepage": "https://github.com/tech-matters/hrm#readme", "dependencies": { + "@tech-matters/hrm-types": "^1.0.0", "@tech-matters/http": "^1.0.0", "@tech-matters/resources-service": "^1.0.0", "@tech-matters/s3-client": "^1.0.0", diff --git a/hrm-domain/hrm-core/referral/referral-data-access.ts b/hrm-domain/hrm-core/referral/referral-data-access.ts index 36d5189ca..677f6d4ea 100644 --- a/hrm-domain/hrm-core/referral/referral-data-access.ts +++ b/hrm-domain/hrm-core/referral/referral-data-access.ts @@ -14,7 +14,7 @@ * along with this program. If not, see https://www.gnu.org/licenses/. */ -import { Referral } from '@tech-matters/types/HrmTypes'; +import { Referral } from '@tech-matters/hrm-types'; import { insertReferralSql } from './sql/referral-insert-sql'; import { diff --git a/hrm-domain/hrm-service/tsconfig.build.json b/hrm-domain/hrm-service/tsconfig.build.json index 6287a8075..7694eb726 100644 --- a/hrm-domain/hrm-service/tsconfig.build.json +++ b/hrm-domain/hrm-service/tsconfig.build.json @@ -15,6 +15,7 @@ { "path": "packages/service-discovery" }, { "path": "resources-domain/packages/resources-search-config" }, { "path": "resources-domain/resources-service" }, + { "path": "hrm-domain/packages/hrm-types" }, { "path": "hrm-domain/packages/hrm-search-config" }, { "path": "hrm-domain/hrm-core" }, { "path": "hrm-domain/scheduled-tasks/hrm-data-pull" }, diff --git a/hrm-domain/packages/hrm-search-config/package.json b/hrm-domain/packages/hrm-search-config/package.json index 00cedf47a..1e43204bd 100644 --- a/hrm-domain/packages/hrm-search-config/package.json +++ b/hrm-domain/packages/hrm-search-config/package.json @@ -7,6 +7,7 @@ "license": "AGPL", "dependencies": { "@tech-matters/elasticsearch-client": "^1.0.0", + "@tech-matters/hrm-types": "^1.0.0", "@tech-matters/types": "^1.0.0" }, "devDependencies": { diff --git a/packages/types/HrmTypes/Case.ts b/hrm-domain/packages/hrm-types/Case.ts similarity index 97% rename from packages/types/HrmTypes/Case.ts rename to hrm-domain/packages/hrm-types/Case.ts index 24c0cdb67..7d07b5524 100644 --- a/packages/types/HrmTypes/Case.ts +++ b/hrm-domain/packages/hrm-types/Case.ts @@ -14,7 +14,7 @@ * along with this program. If not, see https://www.gnu.org/licenses/. */ -import { HrmAccountId, TwilioUserIdentifier, WorkerSID } from '..'; +import { HrmAccountId, TwilioUserIdentifier, WorkerSID } from '@tech-matters/types'; import { CaseSectionRecord } from './CaseSection'; import { Contact } from './Contact'; diff --git a/packages/types/HrmTypes/CaseSection.ts b/hrm-domain/packages/hrm-types/CaseSection.ts similarity index 95% rename from packages/types/HrmTypes/CaseSection.ts rename to hrm-domain/packages/hrm-types/CaseSection.ts index fa9263253..76bffac48 100644 --- a/packages/types/HrmTypes/CaseSection.ts +++ b/hrm-domain/packages/hrm-types/CaseSection.ts @@ -14,7 +14,7 @@ * along with this program. If not, see https://www.gnu.org/licenses/. */ -import { HrmAccountId } from '..'; +import { HrmAccountId } from '@tech-matters/types'; export type CaseSectionRecord = { caseId: number; diff --git a/packages/types/HrmTypes/Contact.ts b/hrm-domain/packages/hrm-types/Contact.ts similarity index 96% rename from packages/types/HrmTypes/Contact.ts rename to hrm-domain/packages/hrm-types/Contact.ts index c02b9fa1f..c1bf79121 100644 --- a/packages/types/HrmTypes/Contact.ts +++ b/hrm-domain/packages/hrm-types/Contact.ts @@ -13,7 +13,7 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see https://www.gnu.org/licenses/. */ -import { HrmAccountId, TwilioUserIdentifier, WorkerSID } from '..'; +import { HrmAccountId, TwilioUserIdentifier, WorkerSID } from '@tech-matters/types'; import { ConversationMedia } from './ConversationMedia'; import { Referral } from './Referral'; diff --git a/packages/types/HrmTypes/ConversationMedia.ts b/hrm-domain/packages/hrm-types/ConversationMedia.ts similarity index 98% rename from packages/types/HrmTypes/ConversationMedia.ts rename to hrm-domain/packages/hrm-types/ConversationMedia.ts index a519cf178..75e3bb94f 100644 --- a/packages/types/HrmTypes/ConversationMedia.ts +++ b/hrm-domain/packages/hrm-types/ConversationMedia.ts @@ -14,7 +14,7 @@ * along with this program. If not, see https://www.gnu.org/licenses/. */ -import { HrmAccountId } from '../HrmAccountId'; +import { HrmAccountId } from '@tech-matters/types'; type ConversationMediaCommons = { id: number; diff --git a/packages/types/HrmTypes/Referral.ts b/hrm-domain/packages/hrm-types/Referral.ts similarity index 100% rename from packages/types/HrmTypes/Referral.ts rename to hrm-domain/packages/hrm-types/Referral.ts diff --git a/packages/types/HrmTypes/index.ts b/hrm-domain/packages/hrm-types/index.ts similarity index 100% rename from packages/types/HrmTypes/index.ts rename to hrm-domain/packages/hrm-types/index.ts diff --git a/hrm-domain/packages/hrm-types/package.json b/hrm-domain/packages/hrm-types/package.json new file mode 100644 index 000000000..d141809d7 --- /dev/null +++ b/hrm-domain/packages/hrm-types/package.json @@ -0,0 +1,15 @@ +{ + "name": "@tech-matters/hrm-types", + "version": "1.0.0", + "description": "", + "main": "dist/index.js", + "author": "", + "license": "AGPL", + "dependencies": { + "@tech-matters/types": "^1.0.0" + }, + "devDependencies": { + "@tsconfig/node16": "^1.0.3" + }, + "scripts": {} +} diff --git a/hrm-domain/packages/hrm-types/tsconfig.json b/hrm-domain/packages/hrm-types/tsconfig.json new file mode 100644 index 000000000..1e9446c0c --- /dev/null +++ b/hrm-domain/packages/hrm-types/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../../tsconfig.json", + "include": ["*.ts"], + "compilerOptions": { + "outDir": "./dist" + } +} + diff --git a/package-lock.json b/package-lock.json index a0e32ac3b..983d627fa 100644 --- a/package-lock.json +++ b/package-lock.json @@ -66,6 +66,7 @@ "version": "1.0.0", "license": "AGPL", "dependencies": { + "@tech-matters/hrm-types": "^1.0.0", "@tech-matters/http": "^1.0.0", "@tech-matters/resources-service": "^1.0.0", "@tech-matters/s3-client": "^1.0.0", @@ -252,6 +253,7 @@ } }, "hrm-domain/lambdas/search-index-consumer": { + "name": "@tech-matters/hrm-search-index-consumer", "version": "1.0.0", "license": "AGPL", "dependencies": { @@ -272,6 +274,7 @@ "license": "AGPL", "dependencies": { "@tech-matters/elasticsearch-client": "^1.0.0", + "@tech-matters/hrm-types": "^1.0.0", "@tech-matters/types": "^1.0.0" }, "devDependencies": { @@ -279,6 +282,17 @@ "@tsconfig/node16": "^1.0.3" } }, + "hrm-domain/packages/hrm-types": { + "name": "@tech-matters/hrm-types", + "version": "1.0.0", + "license": "AGPL", + "dependencies": { + "@tech-matters/types": "^1.0.0" + }, + "devDependencies": { + "@tsconfig/node16": "^1.0.3" + } + }, "hrm-domain/scheduled-tasks/case-status-transition": { "name": "@tech-matters/case-status-transition", "version": "1.0.0", @@ -3904,6 +3918,10 @@ "resolved": "hrm-domain/hrm-service", "link": true }, + "node_modules/@tech-matters/hrm-types": { + "resolved": "hrm-domain/packages/hrm-types", + "link": true + }, "node_modules/@tech-matters/http": { "resolved": "packages/http", "link": true @@ -17452,6 +17470,7 @@ "@tech-matters/hrm-core": { "version": "file:hrm-domain/hrm-core", "requires": { + "@tech-matters/hrm-types": "^1.0.0", "@tech-matters/http": "^1.0.0", "@tech-matters/resources-service": "^1.0.0", "@tech-matters/s3-client": "^1.0.0", @@ -17595,6 +17614,7 @@ "requires": { "@elastic/elasticsearch": "^8.13.1", "@tech-matters/elasticsearch-client": "^1.0.0", + "@tech-matters/hrm-types": "^1.0.0", "@tech-matters/types": "^1.0.0", "@tsconfig/node16": "^1.0.3" } @@ -17670,6 +17690,13 @@ } } }, + "@tech-matters/hrm-types": { + "version": "file:hrm-domain/packages/hrm-types", + "requires": { + "@tech-matters/types": "^1.0.0", + "@tsconfig/node16": "^1.0.3" + } + }, "@tech-matters/http": { "version": "file:packages/http", "requires": { diff --git a/tsconfig.json b/tsconfig.json index 621b49d05..804066797 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -17,6 +17,7 @@ { "path": "packages/service-discovery" }, { "path": "lambdas/packages/hrm-authentication" }, { "path": "lambdas/job-complete" }, + { "path": "hrm-domain/packages/hrm-types" }, { "path": "hrm-domain/packages/hrm-search-config" }, { "path": "hrm-domain/lambdas/contact-complete" }, { "path": "hrm-domain/lambdas/contact-retrieve-transcript" }, From 02e996fde5c94eee336fbafb781273cebad5f343 Mon Sep 17 00:00:00 2001 From: Gianfranco Paoloni Date: Mon, 6 May 2024 16:48:50 -0300 Subject: [PATCH 04/55] chore: pull hrm-types from correct package --- .../hrm-search-config/convertToIndexDocument.ts | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/hrm-domain/packages/hrm-search-config/convertToIndexDocument.ts b/hrm-domain/packages/hrm-search-config/convertToIndexDocument.ts index 51799b58c..d735cf3fd 100644 --- a/hrm-domain/packages/hrm-search-config/convertToIndexDocument.ts +++ b/hrm-domain/packages/hrm-search-config/convertToIndexDocument.ts @@ -24,10 +24,7 @@ */ import { assertExhaustive } from '@tech-matters/types'; -import type { Contact } from '../../hrm-core/contact/contactService'; -import type { CaseService } from '../../hrm-core/case/caseService'; -// import type { Contact } from '@tech-matters/hrm-core/contact/contactService'; -// import type { CaseService } from '@tech-matters/hrm-core/case/caseService'; +import type { CaseService, Contact } from '@tech-matters/hrm-types'; import type { ContactDocument, CaseDocument, @@ -35,19 +32,24 @@ import type { } from './hrmIndexDocumentMappings'; import { CreateIndexConvertedDocument } from '@tech-matters/elasticsearch-client'; -type IndexPayloadContact = { +export type IndexContactMessage = { type: 'contact'; contact: Contact; - transcript?: string; }; -type IndexPayloadCase = { +export type IndexCaseMessage = { type: 'case'; case: Omit & { sections: NonNullable; }; }; +export type IndexPayloadContact = IndexContactMessage & { + transcript: NonNullable; +}; + +export type IndexPayloadCase = IndexCaseMessage; + export type IndexPayload = IndexPayloadContact | IndexPayloadCase; const convertToContactDocument = ({ From 9e44933e2d63f1d75404a73ee24588988826965e Mon Sep 17 00:00:00 2001 From: Gianfranco Paoloni Date: Mon, 6 May 2024 16:49:20 -0300 Subject: [PATCH 05/55] chore: added dependencies to search-index-consumer --- hrm-domain/lambdas/search-index-consumer/package.json | 2 ++ hrm-domain/lambdas/search-index-consumer/tsconfig.build.json | 3 ++- package-lock.json | 4 ++++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/hrm-domain/lambdas/search-index-consumer/package.json b/hrm-domain/lambdas/search-index-consumer/package.json index de21af940..32608150a 100644 --- a/hrm-domain/lambdas/search-index-consumer/package.json +++ b/hrm-domain/lambdas/search-index-consumer/package.json @@ -7,6 +7,8 @@ "license": "AGPL", "dependencies": { "@tech-matters/elasticsearch-client": "^1.0.0", + "@tech-matters/hrm-search-config": "^1.0.0", + "@tech-matters/hrm-types": "^1.0.0", "@tech-matters/job-errors": "^1.0.0", "@tech-matters/sqs-client": "^1.0.0", "@tech-matters/ssm-cache": "^1.0.0", diff --git a/hrm-domain/lambdas/search-index-consumer/tsconfig.build.json b/hrm-domain/lambdas/search-index-consumer/tsconfig.build.json index 8254c0f0c..a8f8b404b 100644 --- a/hrm-domain/lambdas/search-index-consumer/tsconfig.build.json +++ b/hrm-domain/lambdas/search-index-consumer/tsconfig.build.json @@ -7,7 +7,8 @@ { "path": "packages/job-errors" }, { "path": "packages/ssm-cache" }, { "path": "packages/elasticsearch-client" }, - // { "path": "hrm-domain/packages/hrm-search-config" }, + { "path": "hrm-domain/packages/hrm-types" }, + { "path": "hrm-domain/packages/hrm-search-config" }, { "path": "hrm-domain/lambdas/search-index-consumer" } ] } diff --git a/package-lock.json b/package-lock.json index 983d627fa..19c4b7e32 100644 --- a/package-lock.json +++ b/package-lock.json @@ -258,6 +258,8 @@ "license": "AGPL", "dependencies": { "@tech-matters/elasticsearch-client": "^1.0.0", + "@tech-matters/hrm-search-config": "^1.0.0", + "@tech-matters/hrm-types": "^1.0.0", "@tech-matters/job-errors": "^1.0.0", "@tech-matters/sqs-client": "^1.0.0", "@tech-matters/ssm-cache": "^1.0.0", @@ -17623,6 +17625,8 @@ "version": "file:hrm-domain/lambdas/search-index-consumer", "requires": { "@tech-matters/elasticsearch-client": "^1.0.0", + "@tech-matters/hrm-search-config": "^1.0.0", + "@tech-matters/hrm-types": "^1.0.0", "@tech-matters/job-errors": "^1.0.0", "@tech-matters/sqs-client": "^1.0.0", "@tech-matters/ssm-cache": "^1.0.0", From b3a140c808588ad01fda5129a6bd5e260d43cddf Mon Sep 17 00:00:00 2001 From: Gianfranco Paoloni Date: Tue, 7 May 2024 17:43:41 -0300 Subject: [PATCH 06/55] chore: move types around to ease lambda implementation --- .../convertToIndexDocument.ts | 46 +++++++++++++++---- .../packages/hrm-search-config/index.ts | 6 ++- package-lock.json | 2 + packages/job-errors/index.ts | 7 +++ 4 files changed, 50 insertions(+), 11 deletions(-) diff --git a/hrm-domain/packages/hrm-search-config/convertToIndexDocument.ts b/hrm-domain/packages/hrm-search-config/convertToIndexDocument.ts index d735cf3fd..42d9cdbca 100644 --- a/hrm-domain/packages/hrm-search-config/convertToIndexDocument.ts +++ b/hrm-domain/packages/hrm-search-config/convertToIndexDocument.ts @@ -23,7 +23,7 @@ * see: https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-query-string-query.html */ -import { assertExhaustive } from '@tech-matters/types'; +import { assertExhaustive, AccountSID } from '@tech-matters/types'; import type { CaseService, Contact } from '@tech-matters/hrm-types'; import type { ContactDocument, @@ -32,27 +32,54 @@ import type { } from './hrmIndexDocumentMappings'; import { CreateIndexConvertedDocument } from '@tech-matters/elasticsearch-client'; -export type IndexContactMessage = { +type IndexContactMessage = { type: 'contact'; contact: Contact; }; -export type IndexCaseMessage = { +type IndexCaseMessage = { type: 'case'; case: Omit & { sections: NonNullable; }; }; -export type IndexPayloadContact = IndexContactMessage & { +export type IndexMessage = { accountSid: AccountSID } & ( + | IndexContactMessage + | IndexCaseMessage +); + +const getContactDocumentId = ({ contact, type }: IndexContactMessage) => + `${type}_${contact.id}`; + +const getCaseDocumentId = ({ case: caseObj, type }: IndexCaseMessage) => + `${type}_${caseObj.id}`; + +export const getDocumentId = (m: IndexMessage) => { + const { type } = m; + switch (type) { + case 'contact': { + return getContactDocumentId(m); + } + case 'case': { + return getCaseDocumentId(m); + } + default: { + return assertExhaustive(type); + } + } +}; + +type IndexPayloadContact = IndexContactMessage & { transcript: NonNullable; }; -export type IndexPayloadCase = IndexCaseMessage; +type IndexPayloadCase = IndexCaseMessage; export type IndexPayload = IndexPayloadContact | IndexPayloadCase; const convertToContactDocument = ({ + type, contact, transcript, }: IndexPayloadContact): CreateIndexConvertedDocument => { @@ -72,8 +99,7 @@ const convertToContactDocument = ({ twilioWorkerId, rawJson, } = contact; - const type = 'contact' as const; - const compundId = `${type}_${id}`; + const compundId = getContactDocumentId({ type, contact }); return { type, @@ -99,6 +125,7 @@ const convertToContactDocument = ({ }; const convertToCaseDocument = ({ + type, case: caseObj, }: IndexPayloadCase): CreateIndexConvertedDocument => { const { @@ -117,8 +144,7 @@ const convertToCaseDocument = ({ sections, info, } = caseObj; - const type = 'case' as const; - const compundId = `${type}_${id}`; + const compundId = getCaseDocumentId({ type, case: caseObj }); const mappedSections: CaseDocument['sections'] = Object.entries(sections).flatMap( ([sectionType, sectionsArray]) => @@ -172,7 +198,7 @@ export const convertToIndexDocument = ( return convertToCaseDocument(payload); } default: { - assertExhaustive(type); + return assertExhaustive(type); } } }; diff --git a/hrm-domain/packages/hrm-search-config/index.ts b/hrm-domain/packages/hrm-search-config/index.ts index aaa59c323..b8a829737 100644 --- a/hrm-domain/packages/hrm-search-config/index.ts +++ b/hrm-domain/packages/hrm-search-config/index.ts @@ -21,7 +21,11 @@ import type { SearchConfiguration, } from '@tech-matters/elasticsearch-client'; -export { HRM_CASES_CONTACTS_INDEX_TYPE } from './hrmIndexDocumentMappings'; +export { + HRM_CASES_CONTACTS_INDEX_TYPE, + CasesContactsDocument, +} from './hrmIndexDocumentMappings'; +export { IndexMessage, IndexPayload, getDocumentId } from './convertToIndexDocument'; export const hrmSearchConfiguration: SearchConfiguration = { searchFieldBoosts: { diff --git a/package-lock.json b/package-lock.json index 19c4b7e32..1d3847b6b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -261,6 +261,7 @@ "@tech-matters/hrm-search-config": "^1.0.0", "@tech-matters/hrm-types": "^1.0.0", "@tech-matters/job-errors": "^1.0.0", + "@tech-matters/s3-client": "^1.0.0", "@tech-matters/sqs-client": "^1.0.0", "@tech-matters/ssm-cache": "^1.0.0", "@tech-matters/types": "^1.0.0" @@ -17628,6 +17629,7 @@ "@tech-matters/hrm-search-config": "^1.0.0", "@tech-matters/hrm-types": "^1.0.0", "@tech-matters/job-errors": "^1.0.0", + "@tech-matters/s3-client": "^1.0.0", "@tech-matters/sqs-client": "^1.0.0", "@tech-matters/ssm-cache": "^1.0.0", "@tech-matters/types": "^1.0.0", diff --git a/packages/job-errors/index.ts b/packages/job-errors/index.ts index fbba9a98e..826a1604d 100644 --- a/packages/job-errors/index.ts +++ b/packages/job-errors/index.ts @@ -21,6 +21,13 @@ export class ContactJobProcessorError extends Error { } } +export class HrmIndexProcessorError extends Error { + constructor(message: string) { + super(message); + this.name = 'HrmIndexProcessorError'; + } +} + export class ResourceImportProcessorError extends Error { constructor(message: string) { super(message); From e3d37cac3f5a988c5edcf907516709507e0ff96d Mon Sep 17 00:00:00 2001 From: Gianfranco Paoloni Date: Tue, 7 May 2024 17:44:20 -0300 Subject: [PATCH 07/55] chore: implement search-index-consumer indexing --- .../lambdas/search-index-consumer/index.ts | 247 +++++++++++++++--- .../search-index-consumer/package.json | 1 + 2 files changed, 217 insertions(+), 31 deletions(-) diff --git a/hrm-domain/lambdas/search-index-consumer/index.ts b/hrm-domain/lambdas/search-index-consumer/index.ts index ee9423c78..c635ce720 100644 --- a/hrm-domain/lambdas/search-index-consumer/index.ts +++ b/hrm-domain/lambdas/search-index-consumer/index.ts @@ -14,47 +14,232 @@ * along with this program. If not, see https://www.gnu.org/licenses/. */ import type { SQSBatchResponse, SQSEvent, SQSRecord } from 'aws-lambda'; +import { getClient } from '@tech-matters/elasticsearch-client'; +// import { GetSignedUrlMethods, GET_SIGNED_URL_METHODS } from '@tech-matters/s3-client'; +import { HrmIndexProcessorError } from '@tech-matters/job-errors'; +import { + IndexMessage, + HRM_CASES_CONTACTS_INDEX_TYPE, + hrmIndexConfiguration, + IndexPayload, + getDocumentId, +} from '@tech-matters/hrm-search-config'; +import { + AccountSID, + assertExhaustive, + isErr, + newErr, + newOkFromData, +} from '@tech-matters/types'; -export const handler = async (event: SQSEvent): Promise => { - const response: SQSBatchResponse = { batchItemFailures: [] }; - console.debug('Received event:', JSON.stringify(event, null, 2)); - // We need to keep track of the documentId to messageId mapping so we can - // return the correct messageId in the batchItemFailures array on error. - const documentIdToMessageId: Record = {}; +export type MessagesByAccountSid = Record< + AccountSID, + { message: IndexMessage; documentId: string; messageId: string }[] +>; +type PayloadWithMeta = { + payload: IndexPayload; + documentId: string; + messageId: string; +}; +type PayloadsByIndex = { + [indexType: string]: PayloadWithMeta[]; +}; +export type PayloadsByAccountSid = Record; - // Passthrough function to add the documentId to the messageId mapping. - const addDocumentIdToMessageId = (documentId: string, messageId: string) => { - documentIdToMessageId[documentId] = messageId; +type MessagesByDocumentId = { + [documentId: string]: { + documentId: string; + messageId: string; + message: IndexMessage; }; +}; - // Passthrough function to add the documentId to the batchItemFailures array. - // const addDocumentIdToFailures = (documentId: string) => - // response.batchItemFailures.push({ - // itemIdentifier: documentIdToMessageId[documentId], - // }); +const reduceByDocumentId = ( + accum: MessagesByDocumentId, + curr: SQSRecord, +): MessagesByDocumentId => { + const { messageId, body } = curr; + const message = JSON.parse(body) as IndexMessage; - try { - // Map the messages and add the documentId to messageId mapping. - // const messages = mapMessages(event.Records, addDocumentIdToMessageId); - const messages = event.Records.map((record: SQSRecord) => { - const { messageId, body } = record; - const message = JSON.parse(body); - addDocumentIdToMessageId(message.document.id, messageId); + const documentId = getDocumentId(message); - return message; - }); - console.debug('Mapped messages:', JSON.stringify(messages, null, 2)); + return { ...accum, [documentId]: { documentId, messageId, message } }; +}; - console.debug(`Successfully indexed documents`); - } catch (err) { - console.error(new Error('Failed to process search index request'), err); +const groupMessagesByAccountSid = ( + accum: MessagesByAccountSid, + curr: { + documentId: string; + messageId: string; + message: IndexMessage; + }, +): MessagesByAccountSid => { + const { message } = curr; + const { accountSid } = message; - response.batchItemFailures = event.Records.map(record => { + if (!accum[accountSid]) { + return { ...accum, [accountSid]: [curr] }; + } + + return { ...accum, [accountSid]: [...accum[accountSid], curr] }; +}; + +const messagesToPayloadsByIndex = ( + accum: PayloadsByIndex, + currM: { + documentId: string; + messageId: string; + message: IndexMessage; + }, +): PayloadsByIndex => { + const { message } = currM; + + const { type } = message; + + switch (type) { + case 'contact': { + // TODO: Pull the transcripts from S3 (if any) return { - itemIdentifier: record.messageId, + ...accum, + [HRM_CASES_CONTACTS_INDEX_TYPE]: [ + ...(accum[HRM_CASES_CONTACTS_INDEX_TYPE] ?? []), + { ...currM, payload: { ...message, transcript: '' } }, + ], }; - }); + } + case 'case': { + return { + ...accum, + [HRM_CASES_CONTACTS_INDEX_TYPE]: [ + ...(accum[HRM_CASES_CONTACTS_INDEX_TYPE] ?? []), + { ...currM, payload: { ...message } }, + ], + }; + } + default: { + return assertExhaustive(type); + } } +}; + +const indexDocumentsByIndex = + (accountSid: string) => + async ([indexType, payloads]: [string, PayloadWithMeta[]]) => { + // get the client for the accountSid-indexType pair + const client = (await getClient({ accountSid, indexType })).indexClient( + hrmIndexConfiguration, + ); + + const indexed = await Promise.all( + payloads.map(({ documentId, messageId, payload }) => + client + .indexDocument({ + id: documentId, + document: payload, + autocreate: true, + }) + .then(result => ({ + accountSid, + indexType, + documentId, + messageId, + result: newOkFromData(result), + })) + .catch(err => { + console.error( + new HrmIndexProcessorError('Failed to process search index request'), + err, + ); + + return { + accountSid, + indexType, + documentId, + messageId, + result: newErr({ + error: 'ErrorFailedToInex', + message: err instanceof Error ? err.message : String(err), + }), + }; + }), + ), + ); + + return indexed; + }; - return response; +const indexDocumentsByAccount = async ([accountSid, payloadsByIndex]: [ + string, + PayloadsByIndex, +]) => { + const resultsByIndex = await Promise.all( + Object.entries(payloadsByIndex).map(indexDocumentsByIndex(accountSid)), + ); + + return resultsByIndex; +}; + +export const handler = async (event: SQSEvent): Promise => { + console.debug('Received event:', JSON.stringify(event, null, 2)); + + try { + // link each composite "documentId" to it's corresponding "messageId" + const documentIdToMessage = event.Records.reduce(reduceByDocumentId, {}); + + // group the messages by accountSid + const messagesByAccoundSid = Object.values( + documentIdToMessage, + ).reduce(groupMessagesByAccountSid, {}); + + // generate corresponding IndexPayload for each IndexMessage and group them by target indexType + const documentsByAccountSid: PayloadsByAccountSid = Object.fromEntries( + Object.entries(messagesByAccoundSid).map(([accountSid, messages]) => { + const payloads = messages.reduce(messagesToPayloadsByIndex, {}); + + return [accountSid, payloads] as const; + }), + ); + + console.debug('Mapped messages:', JSON.stringify(documentsByAccountSid, null, 2)); + + const resultsByAccount = await Promise.all( + Object.entries(documentsByAccountSid).map(indexDocumentsByAccount), + ); + + console.debug(`Successfully indexed documents`); + + const documentsWithErrors = resultsByAccount + .flat(2) + .filter(({ result }) => isErr(result)); + + if (documentsWithErrors.length) { + console.debug( + 'Errors indexing documents', + JSON.stringify(documentsWithErrors, null, 2), + ); + } + + const response: SQSBatchResponse = { + batchItemFailures: documentsWithErrors.map(({ messageId }) => ({ + itemIdentifier: messageId, + })), + }; + + return response; + } catch (err) { + console.error( + new HrmIndexProcessorError('Failed to process search index request'), + err, + ); + + const response: SQSBatchResponse = { + batchItemFailures: event.Records.map(record => { + return { + itemIdentifier: record.messageId, + }; + }), + }; + + return response; + } }; diff --git a/hrm-domain/lambdas/search-index-consumer/package.json b/hrm-domain/lambdas/search-index-consumer/package.json index 32608150a..77873183a 100644 --- a/hrm-domain/lambdas/search-index-consumer/package.json +++ b/hrm-domain/lambdas/search-index-consumer/package.json @@ -10,6 +10,7 @@ "@tech-matters/hrm-search-config": "^1.0.0", "@tech-matters/hrm-types": "^1.0.0", "@tech-matters/job-errors": "^1.0.0", + "@tech-matters/s3-client": "^1.0.0", "@tech-matters/sqs-client": "^1.0.0", "@tech-matters/ssm-cache": "^1.0.0", "@tech-matters/types": "^1.0.0" From d5ec170e72e75fdbdaae05dce68e84e5e80efc79 Mon Sep 17 00:00:00 2001 From: Gianfranco Paoloni Date: Tue, 7 May 2024 18:36:39 -0300 Subject: [PATCH 08/55] chore: fix TS complaints --- .../convertToIndexDocument.ts | 30 +++++++++---------- hrm-domain/packages/hrm-types/Case.ts | 17 ++++++++--- 2 files changed, 28 insertions(+), 19 deletions(-) diff --git a/hrm-domain/packages/hrm-search-config/convertToIndexDocument.ts b/hrm-domain/packages/hrm-search-config/convertToIndexDocument.ts index 42d9cdbca..5cc1673f1 100644 --- a/hrm-domain/packages/hrm-search-config/convertToIndexDocument.ts +++ b/hrm-domain/packages/hrm-search-config/convertToIndexDocument.ts @@ -107,17 +107,17 @@ const convertToContactDocument = ({ id, compundId, createdAt, - updatedAt, - createdBy, - updatedBy, + updatedAt: updatedAt ?? '', + createdBy: createdBy ?? '', + updatedBy: updatedBy ?? '', finalized: Boolean(finalizedAt), - helpline, - channel, - number, - timeOfContact, + helpline: helpline ?? '', + channel: channel ?? '', + number: number ?? '', + timeOfContact: timeOfContact ?? '', transcript, - twilioWorkerId, - content: typeof rawJson === 'object' ? JSON.stringify(rawJson) : rawJson, + twilioWorkerId: twilioWorkerId ?? '', + content: JSON.stringify(rawJson) ?? '', join_field: { name: 'contact', ...(caseId && { parent: `case_${caseId}` }) }, high_boost_global: '', // highBoostGlobal.join(' '), low_boost_global: '', // lowBoostGlobal.join(' '), @@ -151,9 +151,9 @@ const convertToCaseDocument = ({ sectionsArray.map(section => ({ accountSid: accountSid as string, createdAt: section.createdAt, - updatedAt: section.updatedAt, createdBy: section.createdBy, - updatedBy: section.updatedBy, + updatedAt: section.updatedAt ?? '', + updatedBy: section.updatedBy ?? '', sectionId: section.sectionId, sectionType, content: @@ -174,11 +174,11 @@ const convertToCaseDocument = ({ updatedBy, helpline, twilioWorkerId, - previousStatus, status, - statusUpdatedAt, - statusUpdatedBy, - content: typeof info === 'object' ? JSON.stringify(info) : info, + previousStatus: previousStatus ?? '', + statusUpdatedAt: statusUpdatedAt ?? '', + statusUpdatedBy: statusUpdatedBy ?? '', + content: JSON.stringify(info) ?? '', sections: mappedSections, join_field: { name: 'case' }, high_boost_global: '', // highBoostGlobal.join(' '), diff --git a/hrm-domain/packages/hrm-types/Case.ts b/hrm-domain/packages/hrm-types/Case.ts index 7d07b5524..03b8ed5c2 100644 --- a/hrm-domain/packages/hrm-types/Case.ts +++ b/hrm-domain/packages/hrm-types/Case.ts @@ -62,12 +62,18 @@ const getSectionSpecificDataFromNotesOrReferrals = ( }; export const WELL_KNOWN_CASE_SECTION_NAMES = { - households: { getSectionSpecificData: s => s.household, sectionTypeName: 'household' }, + households: { + getSectionSpecificData: (s: any) => s.household, + sectionTypeName: 'household', + }, perpetrators: { - getSectionSpecificData: s => s.perpetrator, + getSectionSpecificData: (s: any) => s.perpetrator, sectionTypeName: 'perpetrator', }, - incidents: { getSectionSpecificData: s => s.incident, sectionTypeName: 'incident' }, + incidents: { + getSectionSpecificData: (s: any) => s.incident, + sectionTypeName: 'incident', + }, counsellorNotes: { getSectionSpecificData: getSectionSpecificDataFromNotesOrReferrals, sectionTypeName: 'note', @@ -76,7 +82,10 @@ export const WELL_KNOWN_CASE_SECTION_NAMES = { getSectionSpecificData: getSectionSpecificDataFromNotesOrReferrals, sectionTypeName: 'referral', }, - documents: { getSectionSpecificData: s => s.document, sectionTypeName: 'document' }, + documents: { + getSectionSpecificData: (s: any) => s.document, + sectionTypeName: 'document', + }, } as const; type PrecalculatedPermissions = Record<'userOwnsContact', boolean>; From 43456e22a0b1eef661316079a6d8299a132b58fc Mon Sep 17 00:00:00 2001 From: Gianfranco Paoloni Date: Wed, 8 May 2024 17:42:25 -0300 Subject: [PATCH 09/55] fix: add emtpy parent if missing in contact document --- .../lambdas/search-index-consumer/index.ts | 14 ++++++++++++-- .../hrm-search-config/convertToIndexDocument.ts | 17 +++++++++++++++-- .../elasticsearch-client/src/indexDocument.ts | 3 +++ 3 files changed, 30 insertions(+), 4 deletions(-) diff --git a/hrm-domain/lambdas/search-index-consumer/index.ts b/hrm-domain/lambdas/search-index-consumer/index.ts index c635ce720..9ca426264 100644 --- a/hrm-domain/lambdas/search-index-consumer/index.ts +++ b/hrm-domain/lambdas/search-index-consumer/index.ts @@ -31,6 +31,7 @@ import { newErr, newOkFromData, } from '@tech-matters/types'; +import { getContactParentId } from '@tech-matters/hrm-search-config/convertToIndexDocument'; export type MessagesByAccountSid = Record< AccountSID, @@ -40,6 +41,7 @@ type PayloadWithMeta = { payload: IndexPayload; documentId: string; messageId: string; + routing?: string; }; type PayloadsByIndex = { [indexType: string]: PayloadWithMeta[]; @@ -103,7 +105,14 @@ const messagesToPayloadsByIndex = ( ...accum, [HRM_CASES_CONTACTS_INDEX_TYPE]: [ ...(accum[HRM_CASES_CONTACTS_INDEX_TYPE] ?? []), - { ...currM, payload: { ...message, transcript: '' } }, + { + ...currM, + payload: { ...message, transcript: '' }, + routing: getContactParentId( + HRM_CASES_CONTACTS_INDEX_TYPE, + message.contact.caseId, + ), + }, ], }; } @@ -131,12 +140,13 @@ const indexDocumentsByIndex = ); const indexed = await Promise.all( - payloads.map(({ documentId, messageId, payload }) => + payloads.map(({ documentId, messageId, payload, routing }) => client .indexDocument({ id: documentId, document: payload, autocreate: true, + routing, }) .then(result => ({ accountSid, diff --git a/hrm-domain/packages/hrm-search-config/convertToIndexDocument.ts b/hrm-domain/packages/hrm-search-config/convertToIndexDocument.ts index 5cc1673f1..9b3486c45 100644 --- a/hrm-domain/packages/hrm-search-config/convertToIndexDocument.ts +++ b/hrm-domain/packages/hrm-search-config/convertToIndexDocument.ts @@ -25,10 +25,11 @@ import { assertExhaustive, AccountSID } from '@tech-matters/types'; import type { CaseService, Contact } from '@tech-matters/hrm-types'; -import type { +import { ContactDocument, CaseDocument, CasesContactsDocument, + HRM_CASES_CONTACTS_INDEX_TYPE, } from './hrmIndexDocumentMappings'; import { CreateIndexConvertedDocument } from '@tech-matters/elasticsearch-client'; @@ -55,6 +56,15 @@ const getContactDocumentId = ({ contact, type }: IndexContactMessage) => const getCaseDocumentId = ({ case: caseObj, type }: IndexCaseMessage) => `${type}_${caseObj.id}`; +export const getContactParentId = ( + indexType: typeof HRM_CASES_CONTACTS_INDEX_TYPE, + parentId?: string | number, +) => { + if (indexType === HRM_CASES_CONTACTS_INDEX_TYPE) { + return parentId ? `case_${parentId}` : ''; + } +}; + export const getDocumentId = (m: IndexMessage) => { const { type } = m; switch (type) { @@ -118,7 +128,10 @@ const convertToContactDocument = ({ transcript, twilioWorkerId: twilioWorkerId ?? '', content: JSON.stringify(rawJson) ?? '', - join_field: { name: 'contact', ...(caseId && { parent: `case_${caseId}` }) }, + join_field: { + name: 'contact', + parent: getContactParentId(HRM_CASES_CONTACTS_INDEX_TYPE, caseId), + }, high_boost_global: '', // highBoostGlobal.join(' '), low_boost_global: '', // lowBoostGlobal.join(' '), }; diff --git a/packages/elasticsearch-client/src/indexDocument.ts b/packages/elasticsearch-client/src/indexDocument.ts index a1811688a..8b098f070 100644 --- a/packages/elasticsearch-client/src/indexDocument.ts +++ b/packages/elasticsearch-client/src/indexDocument.ts @@ -21,6 +21,7 @@ export type IndexDocumentExtraParams = { id: string; document: T; autocreate?: boolean; + routing?: string; }; export type IndexDocumentParams = PassThroughConfig & IndexDocumentExtraParams; @@ -33,6 +34,7 @@ export const indexDocument = async ({ index, indexConfig, autocreate = false, + routing = undefined, }: IndexDocumentParams): Promise => { if (autocreate) { // const exists = await client.indices.exists({ index }); @@ -46,6 +48,7 @@ export const indexDocument = async ({ index, id, document: convertedDocument, + ...(routing && { routing }), }); }; From 83c177748addc0d5b1a9ea4fb09edad326ee29ca Mon Sep 17 00:00:00 2001 From: Gianfranco Paoloni Date: Wed, 8 May 2024 18:03:21 -0300 Subject: [PATCH 10/55] fix: wrong import --- hrm-domain/lambdas/search-index-consumer/index.ts | 2 +- hrm-domain/packages/hrm-search-config/index.ts | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/hrm-domain/lambdas/search-index-consumer/index.ts b/hrm-domain/lambdas/search-index-consumer/index.ts index 9ca426264..ec93a2c06 100644 --- a/hrm-domain/lambdas/search-index-consumer/index.ts +++ b/hrm-domain/lambdas/search-index-consumer/index.ts @@ -23,6 +23,7 @@ import { hrmIndexConfiguration, IndexPayload, getDocumentId, + getContactParentId, } from '@tech-matters/hrm-search-config'; import { AccountSID, @@ -31,7 +32,6 @@ import { newErr, newOkFromData, } from '@tech-matters/types'; -import { getContactParentId } from '@tech-matters/hrm-search-config/convertToIndexDocument'; export type MessagesByAccountSid = Record< AccountSID, diff --git a/hrm-domain/packages/hrm-search-config/index.ts b/hrm-domain/packages/hrm-search-config/index.ts index b8a829737..41eb6c363 100644 --- a/hrm-domain/packages/hrm-search-config/index.ts +++ b/hrm-domain/packages/hrm-search-config/index.ts @@ -25,7 +25,12 @@ export { HRM_CASES_CONTACTS_INDEX_TYPE, CasesContactsDocument, } from './hrmIndexDocumentMappings'; -export { IndexMessage, IndexPayload, getDocumentId } from './convertToIndexDocument'; +export { + IndexMessage, + IndexPayload, + getDocumentId, + getContactParentId, +} from './convertToIndexDocument'; export const hrmSearchConfiguration: SearchConfiguration = { searchFieldBoosts: { From 941756786c99523434165d32d650e7069a722dc2 Mon Sep 17 00:00:00 2001 From: Gianfranco Paoloni Date: Fri, 10 May 2024 13:47:22 -0300 Subject: [PATCH 11/55] chore: added logic to update documents via elasticsearch client --- packages/elasticsearch-client/src/client.ts | 13 +++ .../elasticsearch-client/src/config/index.ts | 4 +- .../src/config/indexConfiguration.ts | 14 ++- .../elasticsearch-client/src/indexDocument.ts | 5 +- .../src/updateDocument.ts | 88 +++++++++++++++++++ 5 files changed, 116 insertions(+), 8 deletions(-) create mode 100644 packages/elasticsearch-client/src/updateDocument.ts diff --git a/packages/elasticsearch-client/src/client.ts b/packages/elasticsearch-client/src/client.ts index 91963f7fd..0286d781d 100644 --- a/packages/elasticsearch-client/src/client.ts +++ b/packages/elasticsearch-client/src/client.ts @@ -23,6 +23,13 @@ import { IndexDocumentExtraParams, IndexDocumentResponse, } from './indexDocument'; +import { + updateDocument, + UpdateDocumentExtraParams, + UpdateDocumentResponse, + updateScript, + UpdateScriptExtraParams, +} from './updateDocument'; import getAccountSid from './getAccountSid'; import { search, SearchExtraParams } from './search'; import { suggest, SuggestExtraParams } from './suggest'; @@ -92,6 +99,8 @@ const getEsConfig = async ({ */ export type IndexClient = { indexDocument: (args: IndexDocumentExtraParams) => Promise; + updateDocument: (args: UpdateDocumentExtraParams) => Promise; + updateScript: (args: UpdateScriptExtraParams) => Promise; refreshIndex: () => Promise; executeBulk: (args: ExecuteBulkExtraParams) => Promise; createIndex: (args: CreateIndexExtraParams) => Promise; @@ -133,6 +142,10 @@ const getClientOrMock = async ({ config, index, indexType }: GetClientOrMockArgs deleteIndex: () => deleteIndex(passThroughConfig), indexDocument: (args: IndexDocumentExtraParams) => indexDocument({ ...passThroughConfig, ...args }), + updateDocument: (args: UpdateDocumentExtraParams) => + updateDocument({ ...passThroughConfig, ...args }), + updateScript: (args: UpdateScriptExtraParams) => + updateScript({ ...passThroughConfig, ...args }), executeBulk: (args: ExecuteBulkExtraParams) => executeBulk({ ...passThroughConfig, ...args }), }; diff --git a/packages/elasticsearch-client/src/config/index.ts b/packages/elasticsearch-client/src/config/index.ts index ba0319010..421aa4f10 100644 --- a/packages/elasticsearch-client/src/config/index.ts +++ b/packages/elasticsearch-client/src/config/index.ts @@ -17,6 +17,6 @@ export * from './indexConfiguration'; export * from './searchConfiguration'; export type CreateIndexConvertedDocument = { - high_boost_global: string; - low_boost_global: string; + high_boost_global?: string; + low_boost_global?: string; } & TDoc; diff --git a/packages/elasticsearch-client/src/config/indexConfiguration.ts b/packages/elasticsearch-client/src/config/indexConfiguration.ts index f6a5bfb6f..b52a3fae6 100644 --- a/packages/elasticsearch-client/src/config/indexConfiguration.ts +++ b/packages/elasticsearch-client/src/config/indexConfiguration.ts @@ -14,10 +14,20 @@ * along with this program. If not, see https://www.gnu.org/licenses/. */ -import type { IndicesCreateRequest } from '@elastic/elasticsearch/lib/api/types'; +import type { IndicesCreateRequest, Script } from '@elastic/elasticsearch/lib/api/types'; import { CreateIndexConvertedDocument } from './index'; export type IndexConfiguration = { getCreateIndexParams: (indexName: string) => IndicesCreateRequest; - convertToIndexDocument: (sourceEntity: T) => CreateIndexConvertedDocument; + convertToIndexDocument: ( + sourceEntity: T, + indexName: string, + ) => CreateIndexConvertedDocument; + convertToScriptUpdate?: ( + sourceEntity: T, + indexName: string, + ) => { + documentUpdate: CreateIndexConvertedDocument; + scriptUpdate: Script; + }; }; diff --git a/packages/elasticsearch-client/src/indexDocument.ts b/packages/elasticsearch-client/src/indexDocument.ts index 8b098f070..12c7b5e54 100644 --- a/packages/elasticsearch-client/src/indexDocument.ts +++ b/packages/elasticsearch-client/src/indexDocument.ts @@ -21,7 +21,6 @@ export type IndexDocumentExtraParams = { id: string; document: T; autocreate?: boolean; - routing?: string; }; export type IndexDocumentParams = PassThroughConfig & IndexDocumentExtraParams; @@ -34,7 +33,6 @@ export const indexDocument = async ({ index, indexConfig, autocreate = false, - routing = undefined, }: IndexDocumentParams): Promise => { if (autocreate) { // const exists = await client.indices.exists({ index }); @@ -42,13 +40,12 @@ export const indexDocument = async ({ await createIndex({ client, index, indexConfig }); } - const convertedDocument = indexConfig.convertToIndexDocument(document); + const convertedDocument = indexConfig.convertToIndexDocument(document, index); return client.index({ index, id, document: convertedDocument, - ...(routing && { routing }), }); }; diff --git a/packages/elasticsearch-client/src/updateDocument.ts b/packages/elasticsearch-client/src/updateDocument.ts new file mode 100644 index 000000000..7dba5b051 --- /dev/null +++ b/packages/elasticsearch-client/src/updateDocument.ts @@ -0,0 +1,88 @@ +/** + * Copyright (C) 2021-2023 Technology Matters + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see https://www.gnu.org/licenses/. + */ +import { UpdateResponse } from '@elastic/elasticsearch/lib/api/types'; +import { PassThroughConfig } from './client'; +import createIndex from './createIndex'; + +type UpdateParams = { id: string; document: T; autocreate?: boolean }; + +export type UpdateDocumentExtraParams = UpdateParams & { + docAsUpsert?: boolean; +}; + +export type UpdateDocumentParams = PassThroughConfig & UpdateDocumentExtraParams; +export type UpdateDocumentResponse = UpdateResponse; + +export const updateDocument = async ({ + client, + document, + id, + index, + indexConfig, + docAsUpsert = false, + autocreate = false, +}: UpdateDocumentParams): Promise => { + if (docAsUpsert && autocreate) { + // const exists = await client.indices.exists({ index }); + // NOTE: above check is already performed in createIndex + await createIndex({ client, index, indexConfig }); + } + + const documentUpdate = indexConfig.convertToIndexDocument(document, index); + + return client.update({ + index, + id, + doc: documentUpdate, + doc_as_upsert: docAsUpsert, + }); +}; + +export type UpdateScriptExtraParams = UpdateParams & { scriptedUpsert?: boolean }; +export type UpdateScriptParams = PassThroughConfig & UpdateScriptExtraParams; + +export const updateScript = async ({ + client, + document, + id, + index, + indexConfig, + scriptedUpsert = false, + autocreate = false, +}: UpdateScriptParams): Promise => { + if (!indexConfig.convertToScriptUpdate) { + throw new Error(`updateScript error: convertToScriptDocument not provided`); + } + + if (scriptedUpsert && autocreate) { + // const exists = await client.indices.exists({ index }); + // NOTE: above check is already performed in createIndex + await createIndex({ client, index, indexConfig }); + } + + const { documentUpdate, scriptUpdate } = indexConfig.convertToScriptUpdate( + document, + index, + ); + + return client.update({ + index, + id, + script: scriptUpdate, + upsert: documentUpdate, + scripted_upsert: scriptedUpsert, + }); +}; From 2bc2d8e3e110348f2eac683754878fbb9da735a1 Mon Sep 17 00:00:00 2001 From: Gianfranco Paoloni Date: Fri, 10 May 2024 14:25:31 -0300 Subject: [PATCH 12/55] chore: update hrm-search-config package to generate case update scripts based off contacts --- .../convertToIndexDocument.ts | 234 +++++++++++------- .../hrm-search-config/getCreateIndexParams.ts | 37 ++- .../hrmIndexDocumentMappings/index.ts | 64 ++--- .../{mappingCasesContacts.ts => mappings.ts} | 25 +- .../packages/hrm-search-config/index.ts | 18 +- 5 files changed, 224 insertions(+), 154 deletions(-) rename hrm-domain/packages/hrm-search-config/hrmIndexDocumentMappings/{mappingCasesContacts.ts => mappings.ts} (85%) diff --git a/hrm-domain/packages/hrm-search-config/convertToIndexDocument.ts b/hrm-domain/packages/hrm-search-config/convertToIndexDocument.ts index 9b3486c45..d9bcb6b4f 100644 --- a/hrm-domain/packages/hrm-search-config/convertToIndexDocument.ts +++ b/hrm-domain/packages/hrm-search-config/convertToIndexDocument.ts @@ -23,26 +23,32 @@ * see: https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-query-string-query.html */ -import { assertExhaustive, AccountSID } from '@tech-matters/types'; +import type { Script } from '@elastic/elasticsearch/lib/api/types'; import type { CaseService, Contact } from '@tech-matters/hrm-types'; +import { assertExhaustive, AccountSID } from '@tech-matters/types'; import { ContactDocument, CaseDocument, - CasesContactsDocument, - HRM_CASES_CONTACTS_INDEX_TYPE, + HRM_CONTACTS_INDEX_TYPE, + HRM_CASES_INDEX_TYPE, } from './hrmIndexDocumentMappings'; import { CreateIndexConvertedDocument } from '@tech-matters/elasticsearch-client'; +type IndexOperation = 'index' | 'remove'; + type IndexContactMessage = { type: 'contact'; - contact: Contact; + operation: IndexOperation; + contact: Pick & Partial; }; type IndexCaseMessage = { type: 'case'; - case: Omit & { - sections: NonNullable; - }; + operation: IndexOperation; + case: Pick & + Partial> & { + sections: NonNullable; + }; }; export type IndexMessage = { accountSid: AccountSID } & ( @@ -50,36 +56,6 @@ export type IndexMessage = { accountSid: AccountSID } & ( | IndexCaseMessage ); -const getContactDocumentId = ({ contact, type }: IndexContactMessage) => - `${type}_${contact.id}`; - -const getCaseDocumentId = ({ case: caseObj, type }: IndexCaseMessage) => - `${type}_${caseObj.id}`; - -export const getContactParentId = ( - indexType: typeof HRM_CASES_CONTACTS_INDEX_TYPE, - parentId?: string | number, -) => { - if (indexType === HRM_CASES_CONTACTS_INDEX_TYPE) { - return parentId ? `case_${parentId}` : ''; - } -}; - -export const getDocumentId = (m: IndexMessage) => { - const { type } = m; - switch (type) { - case 'contact': { - return getContactDocumentId(m); - } - case 'case': { - return getCaseDocumentId(m); - } - default: { - return assertExhaustive(type); - } - } -}; - type IndexPayloadContact = IndexContactMessage & { transcript: NonNullable; }; @@ -88,8 +64,16 @@ type IndexPayloadCase = IndexCaseMessage; export type IndexPayload = IndexPayloadContact | IndexPayloadCase; -const convertToContactDocument = ({ - type, +const filterEmpty = (doc: T): T => + Object.entries(doc).reduce((accum, [key, value]) => { + if (value) { + return { ...accum, [key]: value }; + } + + return accum; + }, {} as T); + +const convertContactToContactDocument = ({ contact, transcript, }: IndexPayloadContact): CreateIndexConvertedDocument => { @@ -102,43 +86,36 @@ const convertToContactDocument = ({ updatedBy, finalizedAt, helpline, - caseId, number, channel, timeOfContact, twilioWorkerId, rawJson, } = contact; - const compundId = getContactDocumentId({ type, contact }); - return { - type, + const contactDocument: ContactDocument = { accountSid, id, - compundId, createdAt, - updatedAt: updatedAt ?? '', - createdBy: createdBy ?? '', - updatedBy: updatedBy ?? '', + updatedAt: updatedAt, + createdBy: createdBy, + updatedBy: updatedBy, finalized: Boolean(finalizedAt), - helpline: helpline ?? '', - channel: channel ?? '', - number: number ?? '', - timeOfContact: timeOfContact ?? '', + helpline: helpline, + channel: channel, + number: number, + timeOfContact: timeOfContact, transcript, - twilioWorkerId: twilioWorkerId ?? '', - content: JSON.stringify(rawJson) ?? '', - join_field: { - name: 'contact', - parent: getContactParentId(HRM_CASES_CONTACTS_INDEX_TYPE, caseId), - }, - high_boost_global: '', // highBoostGlobal.join(' '), - low_boost_global: '', // lowBoostGlobal.join(' '), + twilioWorkerId: twilioWorkerId, + content: JSON.stringify(rawJson), + // high_boost_global: '', // highBoostGlobal.join(' '), + // low_boost_global: '', // lowBoostGlobal.join(' '), }; + + return filterEmpty(contactDocument); }; -const convertToCaseDocument = ({ - type, +const convertCaseToCaseDocument = ({ case: caseObj, }: IndexPayloadCase): CreateIndexConvertedDocument => { const { @@ -157,16 +134,14 @@ const convertToCaseDocument = ({ sections, info, } = caseObj; - const compundId = getCaseDocumentId({ type, case: caseObj }); - const mappedSections: CaseDocument['sections'] = Object.entries(sections).flatMap( ([sectionType, sectionsArray]) => sectionsArray.map(section => ({ accountSid: accountSid as string, createdAt: section.createdAt, createdBy: section.createdBy, - updatedAt: section.updatedAt ?? '', - updatedBy: section.updatedBy ?? '', + updatedAt: section.updatedAt, + updatedBy: section.updatedBy, sectionId: section.sectionId, sectionType, content: @@ -176,11 +151,9 @@ const convertToCaseDocument = ({ })), ); - return { - type, + const caseDocument: CaseDocument = { accountSid, id, - compundId, createdAt, updatedAt, createdBy, @@ -188,30 +161,125 @@ const convertToCaseDocument = ({ helpline, twilioWorkerId, status, - previousStatus: previousStatus ?? '', - statusUpdatedAt: statusUpdatedAt ?? '', - statusUpdatedBy: statusUpdatedBy ?? '', - content: JSON.stringify(info) ?? '', + previousStatus: previousStatus, + statusUpdatedAt: statusUpdatedAt, + statusUpdatedBy: statusUpdatedBy, + content: JSON.stringify(info), sections: mappedSections, - join_field: { name: 'case' }, - high_boost_global: '', // highBoostGlobal.join(' '), - low_boost_global: '', // lowBoostGlobal.join(' '), + contacts: null, + // high_boost_global: '', // highBoostGlobal.join(' '), + // low_boost_global: '', // lowBoostGlobal.join(' '), }; + + return filterEmpty(caseDocument); +}; + +const convertToContactIndexDocument = (payload: IndexPayload) => { + if (payload.type === 'contact') { + return convertContactToContactDocument(payload); + } + + throw new Error( + `convertToContactIndexDocument not implemented for type ${payload.type} and operation ${payload.operation}`, + ); +}; + +const convertToCaseIndexDocument = (payload: IndexPayload) => { + if (payload.type === 'case') { + return convertCaseToCaseDocument(payload); + } + + throw new Error( + `convertToCaseIndexDocument not implemented for type ${payload.type} and operation ${payload.operation}`, + ); }; export const convertToIndexDocument = ( payload: IndexPayload, -): CreateIndexConvertedDocument => { - const { type } = payload; - switch (type) { - case 'contact': { - return convertToContactDocument(payload); + indexName: string, +): CreateIndexConvertedDocument => { + if (indexName.endsWith(HRM_CONTACTS_INDEX_TYPE)) { + return convertToContactIndexDocument(payload); + } + + if (indexName.endsWith(HRM_CASES_INDEX_TYPE)) { + return convertToCaseIndexDocument(payload); + } + + throw new Error(`convertToIndexDocument not implemented for index ${indexName}`); +}; + +const convertContactToCaseScriptUpdate = ( + payload: IndexPayloadContact, +): { + documentUpdate: CreateIndexConvertedDocument; + scriptUpdate: Script; +} => { + const { operation } = payload; + const { accountSid, caseId } = payload.contact; + + switch (operation) { + case 'index': { + const contactDocument = convertContactToContactDocument(payload); + + const documentUpdate: CreateIndexConvertedDocument = { + id: parseInt(caseId, 10), + accountSid, + contacts: [contactDocument], + }; + + const scriptUpdate: Script = { + source: + 'def replaceContact(Map newContact, List contacts) { contacts.removeIf(contact -> contact.id == newContact.id); contacts.add(newContact); } replaceContact(params.newContact, ctx._source.contacts);', + params: { + newContact: contactDocument, + }, + }; + + return { documentUpdate, scriptUpdate }; } - case 'case': { - return convertToCaseDocument(payload); + case 'remove': { + const scriptUpdate: Script = { + source: + 'def removeContact(int contactId, List contacts) { contacts.removeIf(contact -> contact.id == contactId); } removeContact(params.contactId, ctx._source.contacts);', + params: { + contactId: payload.contact.id, + }, + }; + + return { documentUpdate: undefined, scriptUpdate }; } default: { - return assertExhaustive(type); + return assertExhaustive(operation); } } }; + +const convertToCaseScriptUpdate = ( + payload: IndexPayload, +): { + documentUpdate: CreateIndexConvertedDocument; + scriptUpdate: Script; +} => { + if (payload.type === 'contact') { + return convertContactToCaseScriptUpdate(payload); + } + + throw new Error( + `convertToCaseScriptDocument not implemented for type ${payload.type} and operation ${payload.operation}`, + ); +}; + +export const convertToScriptUpdate = ( + payload: IndexPayload, + indexName: string, +): { + documentUpdate: CreateIndexConvertedDocument; + scriptUpdate: Script; +} => { + if (indexName.endsWith(HRM_CASES_INDEX_TYPE)) { + return convertToCaseScriptUpdate(payload); + } + + throw new Error(`convertToScriptDocument not implemented for index ${indexName}`); +}; diff --git a/hrm-domain/packages/hrm-search-config/getCreateIndexParams.ts b/hrm-domain/packages/hrm-search-config/getCreateIndexParams.ts index cd20c81d8..5850c6cd1 100644 --- a/hrm-domain/packages/hrm-search-config/getCreateIndexParams.ts +++ b/hrm-domain/packages/hrm-search-config/getCreateIndexParams.ts @@ -26,11 +26,13 @@ import type { IndicesCreateRequest } from '@elastic/elasticsearch/lib/api/types'; import { - mappingCasesContacts, - HRM_CASES_CONTACTS_INDEX_TYPE, + HRM_CASES_INDEX_TYPE, + HRM_CONTACTS_INDEX_TYPE, + caseMapping, + contactMapping, } from './hrmIndexDocumentMappings'; -const getCreateHrmCasesContactsIndexParams = (index: string): IndicesCreateRequest => { +const getCreateHrmContactsIndexParams = (index: string): IndicesCreateRequest => { return { index, // settings: { @@ -43,7 +45,26 @@ const getCreateHrmCasesContactsIndexParams = (index: string): IndicesCreateReque low_boost_global: { type: 'text', }, - ...mappingCasesContacts, + ...contactMapping, + }, + }, + }; +}; + +const getCreateHrmCaseIndexParams = (index: string): IndicesCreateRequest => { + return { + index, + // settings: { + // }, + mappings: { + properties: { + high_boost_global: { + type: 'text', + }, + low_boost_global: { + type: 'text', + }, + ...caseMapping, }, }, }; @@ -54,8 +75,12 @@ const getCreateHrmCasesContactsIndexParams = (index: string): IndicesCreateReque * @param index */ export const getCreateIndexParams = (index: string): IndicesCreateRequest => { - if (index.endsWith(HRM_CASES_CONTACTS_INDEX_TYPE)) { - return getCreateHrmCasesContactsIndexParams(index); + if (index.endsWith(HRM_CONTACTS_INDEX_TYPE)) { + return getCreateHrmContactsIndexParams(index); + } + + if (index.endsWith(HRM_CASES_INDEX_TYPE)) { + return getCreateHrmCaseIndexParams(index); } throw new Error(`getCreateIndexParams not implemented for index ${index}`); diff --git a/hrm-domain/packages/hrm-search-config/hrmIndexDocumentMappings/index.ts b/hrm-domain/packages/hrm-search-config/hrmIndexDocumentMappings/index.ts index 7f4d638a2..52962c84b 100644 --- a/hrm-domain/packages/hrm-search-config/hrmIndexDocumentMappings/index.ts +++ b/hrm-domain/packages/hrm-search-config/hrmIndexDocumentMappings/index.ts @@ -15,46 +15,36 @@ */ import type { MappingProperty } from '@elastic/elasticsearch/lib/api/types'; -import { caseMapping, contactMapping } from './mappingCasesContacts'; +import { caseMapping, contactMapping } from './mappings'; -export { mapping as mappingCasesContacts } from './mappingCasesContacts'; +export { caseMapping, contactMapping } from './mappings'; -type MappingToDocument>> = { - [k in keyof T]: k extends string - ? T[k]['type'] extends 'keyword' - ? string - : T[k]['type'] extends 'text' - ? string - : T[k]['type'] extends 'integer' - ? number - : T[k]['type'] extends 'boolean' - ? boolean - : T[k]['type'] extends 'date' - ? string - : T[k]['type'] extends 'join' - ? { name: string; parent?: string } - : T[k]['type'] extends 'nested' - ? T[k] extends { - properties: Record; - } - ? MappingToDocument[] - : never - : never // forbid non-used types to force proper implementation - : never; -}; - -export type ContactDocument = MappingToDocument & - NonNullable<{ - type: 'contact'; - join_field: NonNullable<{ name: 'contact'; parent?: string }>; +export type MappingToDocument>> = + Partial<{ + [k in keyof T]: k extends string + ? T[k]['type'] extends 'keyword' + ? string | null + : T[k]['type'] extends 'text' + ? string | null + : T[k]['type'] extends 'integer' + ? number | null + : T[k]['type'] extends 'boolean' + ? boolean | null + : T[k]['type'] extends 'date' + ? string | null + : T[k]['type'] extends 'nested' + ? T[k] extends { + properties: Record; + } + ? MappingToDocument[] + : never + : never // forbid non-used types to force proper implementation + : never; }>; -export type CaseDocument = MappingToDocument & - NonNullable<{ - type: 'case'; - join_field: NonNullable<{ name: 'case' }>; - }>; +export type ContactDocument = MappingToDocument; -export type CasesContactsDocument = ContactDocument | CaseDocument; +export type CaseDocument = MappingToDocument; -export const HRM_CASES_CONTACTS_INDEX_TYPE = 'hrm-cases-contacts'; +export const HRM_CONTACTS_INDEX_TYPE = 'hrm-contacts'; +export const HRM_CASES_INDEX_TYPE = 'hrm-cases'; diff --git a/hrm-domain/packages/hrm-search-config/hrmIndexDocumentMappings/mappingCasesContacts.ts b/hrm-domain/packages/hrm-search-config/hrmIndexDocumentMappings/mappings.ts similarity index 85% rename from hrm-domain/packages/hrm-search-config/hrmIndexDocumentMappings/mappingCasesContacts.ts rename to hrm-domain/packages/hrm-search-config/hrmIndexDocumentMappings/mappings.ts index ef810fde3..fd4dd1006 100644 --- a/hrm-domain/packages/hrm-search-config/hrmIndexDocumentMappings/mappingCasesContacts.ts +++ b/hrm-domain/packages/hrm-search-config/hrmIndexDocumentMappings/mappings.ts @@ -14,8 +14,6 @@ * along with this program. If not, see https://www.gnu.org/licenses/. */ -import type { MappingProperty } from '@elastic/elasticsearch/lib/api/types'; - // Properties present in root and nested documents const commonProperties = { accountSid: { @@ -40,15 +38,9 @@ const commonProperties = { // Properties shared by both types of documents, cases and contacts const rootProperties = { - type: { - type: 'keyword', - }, id: { type: 'integer', }, - compundId: { - type: 'keyword', - }, twilioWorkerId: { type: 'keyword', }, @@ -56,12 +48,6 @@ const rootProperties = { type: 'keyword', }, ...commonProperties, - join_field: { - type: 'join', - relations: { - case: 'contact', - }, - }, } as const; // Properties specific to contacts @@ -111,9 +97,10 @@ export const caseMapping = { ...commonProperties, }, }, -} as const; - -export const mapping: Record = { - ...contactMapping, - ...caseMapping, + contacts: { + type: 'nested', + properties: { + ...contactMapping, + }, + }, } as const; diff --git a/hrm-domain/packages/hrm-search-config/index.ts b/hrm-domain/packages/hrm-search-config/index.ts index 41eb6c363..c4c76df56 100644 --- a/hrm-domain/packages/hrm-search-config/index.ts +++ b/hrm-domain/packages/hrm-search-config/index.ts @@ -14,7 +14,11 @@ * along with this program. If not, see https://www.gnu.org/licenses/. */ -import { IndexPayload, convertToIndexDocument } from './convertToIndexDocument'; +import { + IndexPayload, + convertToIndexDocument, + convertToScriptUpdate, +} from './convertToIndexDocument'; import { getCreateIndexParams } from './getCreateIndexParams'; import type { IndexConfiguration, @@ -22,15 +26,10 @@ import type { } from '@tech-matters/elasticsearch-client'; export { - HRM_CASES_CONTACTS_INDEX_TYPE, - CasesContactsDocument, + HRM_CASES_INDEX_TYPE, + HRM_CONTACTS_INDEX_TYPE, } from './hrmIndexDocumentMappings'; -export { - IndexMessage, - IndexPayload, - getDocumentId, - getContactParentId, -} from './convertToIndexDocument'; +export { IndexMessage, IndexPayload } from './convertToIndexDocument'; export const hrmSearchConfiguration: SearchConfiguration = { searchFieldBoosts: { @@ -47,5 +46,6 @@ export const hrmSearchConfiguration: SearchConfiguration = { export const hrmIndexConfiguration: IndexConfiguration = { convertToIndexDocument, + convertToScriptUpdate, getCreateIndexParams, }; From 340bdd24715825cf9e50fee1af73c8bf746ef655 Mon Sep 17 00:00:00 2001 From: Gianfranco Paoloni Date: Fri, 10 May 2024 14:26:02 -0300 Subject: [PATCH 13/55] chore: search-index-consumer rework --- .../lambdas/search-index-consumer/index.ts | 223 ++++++++++-------- 1 file changed, 128 insertions(+), 95 deletions(-) diff --git a/hrm-domain/lambdas/search-index-consumer/index.ts b/hrm-domain/lambdas/search-index-consumer/index.ts index ec93a2c06..d6b405a5a 100644 --- a/hrm-domain/lambdas/search-index-consumer/index.ts +++ b/hrm-domain/lambdas/search-index-consumer/index.ts @@ -14,16 +14,15 @@ * along with this program. If not, see https://www.gnu.org/licenses/. */ import type { SQSBatchResponse, SQSEvent, SQSRecord } from 'aws-lambda'; -import { getClient } from '@tech-matters/elasticsearch-client'; +import { getClient, IndexClient } from '@tech-matters/elasticsearch-client'; // import { GetSignedUrlMethods, GET_SIGNED_URL_METHODS } from '@tech-matters/s3-client'; import { HrmIndexProcessorError } from '@tech-matters/job-errors'; import { - IndexMessage, - HRM_CASES_CONTACTS_INDEX_TYPE, + HRM_CASES_INDEX_TYPE, + HRM_CONTACTS_INDEX_TYPE, hrmIndexConfiguration, + IndexMessage, IndexPayload, - getDocumentId, - getContactParentId, } from '@tech-matters/hrm-search-config'; import { AccountSID, @@ -33,66 +32,38 @@ import { newOkFromData, } from '@tech-matters/types'; -export type MessagesByAccountSid = Record< - AccountSID, - { message: IndexMessage; documentId: string; messageId: string }[] ->; +type MessageWithMeta = { message: IndexMessage; messageId: string }; +type MessagesByAccountSid = Record; type PayloadWithMeta = { + documentId: number; payload: IndexPayload; - documentId: string; messageId: string; - routing?: string; + indexHandler: 'indexDocument' | 'updateDocument' | 'updateScript'; }; type PayloadsByIndex = { [indexType: string]: PayloadWithMeta[]; }; -export type PayloadsByAccountSid = Record; +type PayloadsByAccountSid = Record; -type MessagesByDocumentId = { - [documentId: string]: { - documentId: string; - messageId: string; - message: IndexMessage; - }; -}; - -const reduceByDocumentId = ( - accum: MessagesByDocumentId, +const groupMessagesByAccountSid = ( + accum: MessagesByAccountSid, curr: SQSRecord, -): MessagesByDocumentId => { +): MessagesByAccountSid => { const { messageId, body } = curr; const message = JSON.parse(body) as IndexMessage; - const documentId = getDocumentId(message); - - return { ...accum, [documentId]: { documentId, messageId, message } }; -}; - -const groupMessagesByAccountSid = ( - accum: MessagesByAccountSid, - curr: { - documentId: string; - messageId: string; - message: IndexMessage; - }, -): MessagesByAccountSid => { - const { message } = curr; const { accountSid } = message; if (!accum[accountSid]) { - return { ...accum, [accountSid]: [curr] }; + return { ...accum, [accountSid]: [{ messageId, message }] }; } - return { ...accum, [accountSid]: [...accum[accountSid], curr] }; + return { ...accum, [accountSid]: [...accum[accountSid], { messageId, message }] }; }; const messagesToPayloadsByIndex = ( accum: PayloadsByIndex, - currM: { - documentId: string; - messageId: string; - message: IndexMessage; - }, + currM: MessageWithMeta, ): PayloadsByIndex => { const { message } = currM; @@ -103,25 +74,42 @@ const messagesToPayloadsByIndex = ( // TODO: Pull the transcripts from S3 (if any) return { ...accum, - [HRM_CASES_CONTACTS_INDEX_TYPE]: [ - ...(accum[HRM_CASES_CONTACTS_INDEX_TYPE] ?? []), + // add an upsert job to HRM_CONTACTS_INDEX_TYPE index + [HRM_CONTACTS_INDEX_TYPE]: [ + ...(accum[HRM_CONTACTS_INDEX_TYPE] ?? []), { ...currM, + documentId: message.contact.id, payload: { ...message, transcript: '' }, - routing: getContactParentId( - HRM_CASES_CONTACTS_INDEX_TYPE, - message.contact.caseId, - ), + indexHandler: 'updateDocument', }, ], + // if associated to a case, add an upsert with script job to HRM_CASES_INDEX_TYPE index + [HRM_CASES_INDEX_TYPE]: message.contact.caseId + ? [ + ...(accum[HRM_CASES_INDEX_TYPE] ?? []), + { + ...currM, + documentId: parseInt(message.contact.caseId, 10), + payload: { ...message, transcript: '' }, + indexHandler: 'updateScript', + }, + ] + : accum[HRM_CASES_INDEX_TYPE], }; } case 'case': { return { ...accum, - [HRM_CASES_CONTACTS_INDEX_TYPE]: [ - ...(accum[HRM_CASES_CONTACTS_INDEX_TYPE] ?? []), - { ...currM, payload: { ...message } }, + // add an upsert job to HRM_CASES_INDEX_TYPE index + [HRM_CASES_INDEX_TYPE]: [ + ...(accum[HRM_CASES_INDEX_TYPE] ?? []), + { + ...currM, + documentId: message.case.id, + payload: { ...message }, + indexHandler: 'updateDocument', + }, ], }; } @@ -131,50 +119,97 @@ const messagesToPayloadsByIndex = ( } }; -const indexDocumentsByIndex = - (accountSid: string) => - async ([indexType, payloads]: [string, PayloadWithMeta[]]) => { - // get the client for the accountSid-indexType pair - const client = (await getClient({ accountSid, indexType })).indexClient( - hrmIndexConfiguration, - ); +const handleIndexPayload = + ({ + accountSid, + client, + indexType, + }: { + accountSid: string; + client: IndexClient; + indexType: string; + }) => + async ({ documentId, indexHandler, messageId, payload }: PayloadWithMeta) => { + try { + switch (indexHandler) { + case 'indexDocument': { + const result = await client.indexDocument({ + id: documentId.toString(), + document: payload, + autocreate: true, + }); + + return { + accountSid, + indexType, + messageId, + result: newOkFromData(result), + }; + } + case 'updateDocument': { + const result = await client.updateDocument({ + id: documentId.toString(), + document: payload, + autocreate: true, + docAsUpsert: true, + }); - const indexed = await Promise.all( - payloads.map(({ documentId, messageId, payload, routing }) => - client - .indexDocument({ - id: documentId, + return { + accountSid, + indexType, + messageId, + result: newOkFromData(result), + }; + } + case 'updateScript': { + const result = await client.updateScript({ document: payload, + id: documentId.toString(), autocreate: true, - routing, - }) - .then(result => ({ + scriptedUpsert: true, + }); + + return { accountSid, indexType, - documentId, messageId, result: newOkFromData(result), - })) - .catch(err => { - console.error( - new HrmIndexProcessorError('Failed to process search index request'), - err, - ); - - return { - accountSid, - indexType, - documentId, - messageId, - result: newErr({ - error: 'ErrorFailedToInex', - message: err instanceof Error ? err.message : String(err), - }), - }; - }), - ), + }; + } + default: { + return assertExhaustive(indexHandler); + } + } + } catch (err) { + console.error( + new HrmIndexProcessorError('handleIndexPayload: Failed to process index request'), + err, + ); + + return { + accountSid, + indexType, + messageId, + result: newErr({ + error: 'ErrorFailedToInex', + message: err instanceof Error ? err.message : String(err), + }), + }; + } + }; + +const indexDocumentsByIndex = + (accountSid: string) => + async ([indexType, payloads]: [string, PayloadWithMeta[]]) => { + // get the client for the accountSid-indexType pair + const client = (await getClient({ accountSid, indexType })).indexClient( + hrmIndexConfiguration, ); + const mapper = handleIndexPayload({ client, accountSid, indexType }); + + const indexed = await Promise.all(payloads.map(mapper)); + return indexed; }; @@ -193,13 +228,11 @@ export const handler = async (event: SQSEvent): Promise => { console.debug('Received event:', JSON.stringify(event, null, 2)); try { - // link each composite "documentId" to it's corresponding "messageId" - const documentIdToMessage = event.Records.reduce(reduceByDocumentId, {}); - - // group the messages by accountSid - const messagesByAccoundSid = Object.values( - documentIdToMessage, - ).reduce(groupMessagesByAccountSid, {}); + // group the messages by accountSid while adding message meta + const messagesByAccoundSid = event.Records.reduce( + groupMessagesByAccountSid, + {}, + ); // generate corresponding IndexPayload for each IndexMessage and group them by target indexType const documentsByAccountSid: PayloadsByAccountSid = Object.fromEntries( From 7cf998ce812a4ca6a5244e9d8d0a1898538d4f6a Mon Sep 17 00:00:00 2001 From: Gianfranco Paoloni Date: Fri, 10 May 2024 14:33:28 -0300 Subject: [PATCH 14/55] fix: small TS errors --- .../packages/hrm-search-config/convertToIndexDocument.ts | 4 ++-- .../hrm-search-config/hrmIndexDocumentMappings/index.ts | 2 +- packages/elasticsearch-client/src/executeBulk.ts | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/hrm-domain/packages/hrm-search-config/convertToIndexDocument.ts b/hrm-domain/packages/hrm-search-config/convertToIndexDocument.ts index d9bcb6b4f..2226cf2e2 100644 --- a/hrm-domain/packages/hrm-search-config/convertToIndexDocument.ts +++ b/hrm-domain/packages/hrm-search-config/convertToIndexDocument.ts @@ -223,7 +223,7 @@ const convertContactToCaseScriptUpdate = ( const contactDocument = convertContactToContactDocument(payload); const documentUpdate: CreateIndexConvertedDocument = { - id: parseInt(caseId, 10), + id: parseInt(caseId!, 10), accountSid, contacts: [contactDocument], }; @@ -247,7 +247,7 @@ const convertContactToCaseScriptUpdate = ( }, }; - return { documentUpdate: undefined, scriptUpdate }; + return { documentUpdate: {}, scriptUpdate }; } default: { return assertExhaustive(operation); diff --git a/hrm-domain/packages/hrm-search-config/hrmIndexDocumentMappings/index.ts b/hrm-domain/packages/hrm-search-config/hrmIndexDocumentMappings/index.ts index 52962c84b..87ea5e883 100644 --- a/hrm-domain/packages/hrm-search-config/hrmIndexDocumentMappings/index.ts +++ b/hrm-domain/packages/hrm-search-config/hrmIndexDocumentMappings/index.ts @@ -36,7 +36,7 @@ export type MappingToDocument; } - ? MappingToDocument[] + ? MappingToDocument[] | null : never : never // forbid non-used types to force proper implementation : never; diff --git a/packages/elasticsearch-client/src/executeBulk.ts b/packages/elasticsearch-client/src/executeBulk.ts index 07c701766..3e3f2b579 100644 --- a/packages/elasticsearch-client/src/executeBulk.ts +++ b/packages/elasticsearch-client/src/executeBulk.ts @@ -51,7 +51,7 @@ export const executeBulk = async ({ } else { return [ { index: { _index: index, _id: documentItem.id } }, - indexConfig.convertToIndexDocument(documentItem.document), + indexConfig.convertToIndexDocument(documentItem.document, index), ]; } }, From f5b3bef688fae6729fd976f2662200d6dd38bfc7 Mon Sep 17 00:00:00 2001 From: Gianfranco Paoloni Date: Fri, 10 May 2024 18:00:03 -0300 Subject: [PATCH 15/55] debug --- hrm-domain/lambdas/search-index-consumer/index.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/hrm-domain/lambdas/search-index-consumer/index.ts b/hrm-domain/lambdas/search-index-consumer/index.ts index d6b405a5a..82c78c577 100644 --- a/hrm-domain/lambdas/search-index-consumer/index.ts +++ b/hrm-domain/lambdas/search-index-consumer/index.ts @@ -217,6 +217,8 @@ const indexDocumentsByAccount = async ([accountSid, payloadsByIndex]: [ string, PayloadsByIndex, ]) => { + console.log('>> indexDocumentsByAccount payloadsByIndex ', payloadsByIndex); + console.log('>> indexDocumentsByAccount: ', Object.entries(payloadsByIndex)); const resultsByIndex = await Promise.all( Object.entries(payloadsByIndex).map(indexDocumentsByIndex(accountSid)), ); From c0cabb15b28f904965e197d3368eda5fc77626e8 Mon Sep 17 00:00:00 2001 From: Gianfranco Paoloni Date: Fri, 10 May 2024 18:08:29 -0300 Subject: [PATCH 16/55] fix: contemplate case where case index payloads is undefined --- hrm-domain/lambdas/search-index-consumer/index.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/hrm-domain/lambdas/search-index-consumer/index.ts b/hrm-domain/lambdas/search-index-consumer/index.ts index 82c78c577..1ce9daa35 100644 --- a/hrm-domain/lambdas/search-index-consumer/index.ts +++ b/hrm-domain/lambdas/search-index-consumer/index.ts @@ -95,7 +95,7 @@ const messagesToPayloadsByIndex = ( indexHandler: 'updateScript', }, ] - : accum[HRM_CASES_INDEX_TYPE], + : accum[HRM_CASES_INDEX_TYPE] ?? [], }; } case 'case': { @@ -217,8 +217,6 @@ const indexDocumentsByAccount = async ([accountSid, payloadsByIndex]: [ string, PayloadsByIndex, ]) => { - console.log('>> indexDocumentsByAccount payloadsByIndex ', payloadsByIndex); - console.log('>> indexDocumentsByAccount: ', Object.entries(payloadsByIndex)); const resultsByIndex = await Promise.all( Object.entries(payloadsByIndex).map(indexDocumentsByIndex(accountSid)), ); From 8097a42de5a6192720440d0483080ff35efbecc6 Mon Sep 17 00:00:00 2001 From: Gianfranco Paoloni Date: Mon, 13 May 2024 16:26:11 -0300 Subject: [PATCH 17/55] chore: fix unit tests --- .../tests/unit/convertDocumentResources.test.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/resources-domain/packages/resources-search-config/tests/unit/convertDocumentResources.test.ts b/resources-domain/packages/resources-search-config/tests/unit/convertDocumentResources.test.ts index ba163ee5b..81c692684 100644 --- a/resources-domain/packages/resources-search-config/tests/unit/convertDocumentResources.test.ts +++ b/resources-domain/packages/resources-search-config/tests/unit/convertDocumentResources.test.ts @@ -15,7 +15,7 @@ */ import { FlatResource } from '@tech-matters/types'; -import { resourceIndexConfiguration } from '../../index'; +import { resourceIndexConfiguration, RESOURCE_INDEX_TYPE } from '../../index'; const BASELINE_DATE = new Date('2021-01-01T00:00:00.000Z'); @@ -53,7 +53,10 @@ describe('convertIndexDocument', () => { ], }; - const document = resourceIndexConfiguration.convertToIndexDocument(resource); + const document = resourceIndexConfiguration.convertToIndexDocument( + resource, + RESOURCE_INDEX_TYPE, + ); expect(document).toEqual({ id: '1234', From a10662dbb048f82272f0af3ac3ee7ba03ee8493b Mon Sep 17 00:00:00 2001 From: Gianfranco Paoloni Date: Mon, 13 May 2024 17:46:25 -0300 Subject: [PATCH 18/55] debug logs --- .../packages/hrm-search-config/convertToIndexDocument.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/hrm-domain/packages/hrm-search-config/convertToIndexDocument.ts b/hrm-domain/packages/hrm-search-config/convertToIndexDocument.ts index 2226cf2e2..ee3a3f684 100644 --- a/hrm-domain/packages/hrm-search-config/convertToIndexDocument.ts +++ b/hrm-domain/packages/hrm-search-config/convertToIndexDocument.ts @@ -236,6 +236,10 @@ const convertContactToCaseScriptUpdate = ( }, }; + console.log('>>>> documentUpdate', payload) + console.log('>>>> documentUpdate', documentUpdate) + console.log('>>>> scriptUpdate', scriptUpdate) + return { documentUpdate, scriptUpdate }; } case 'remove': { From 08e28c9c48da11717615ab9870b730ad30fb587e Mon Sep 17 00:00:00 2001 From: Gianfranco Paoloni Date: Mon, 13 May 2024 17:58:54 -0300 Subject: [PATCH 19/55] debug --- hrm-domain/lambdas/search-index-consumer/index.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/hrm-domain/lambdas/search-index-consumer/index.ts b/hrm-domain/lambdas/search-index-consumer/index.ts index 1ce9daa35..42fb0c4ca 100644 --- a/hrm-domain/lambdas/search-index-consumer/index.ts +++ b/hrm-domain/lambdas/search-index-consumer/index.ts @@ -184,6 +184,7 @@ const handleIndexPayload = console.error( new HrmIndexProcessorError('handleIndexPayload: Failed to process index request'), err, + { documentId, indexHandler, messageId, payload }, ); return { From c59db2a16189a840c3dd6f7364307b535ea825f4 Mon Sep 17 00:00:00 2001 From: Gianfranco Paoloni Date: Tue, 14 May 2024 12:50:30 -0300 Subject: [PATCH 20/55] chore: factor out convertToScriptUpdate logic --- .../convertToIndexDocument.ts | 115 +----------------- .../convertToScriptUpdate.ts | 109 +++++++++++++++++ .../packages/hrm-search-config/index.ts | 10 +- .../packages/hrm-search-config/payload.ts | 56 +++++++++ 4 files changed, 171 insertions(+), 119 deletions(-) create mode 100644 hrm-domain/packages/hrm-search-config/convertToScriptUpdate.ts create mode 100644 hrm-domain/packages/hrm-search-config/payload.ts diff --git a/hrm-domain/packages/hrm-search-config/convertToIndexDocument.ts b/hrm-domain/packages/hrm-search-config/convertToIndexDocument.ts index ee3a3f684..e3a48207a 100644 --- a/hrm-domain/packages/hrm-search-config/convertToIndexDocument.ts +++ b/hrm-domain/packages/hrm-search-config/convertToIndexDocument.ts @@ -23,9 +23,6 @@ * see: https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-query-string-query.html */ -import type { Script } from '@elastic/elasticsearch/lib/api/types'; -import type { CaseService, Contact } from '@tech-matters/hrm-types'; -import { assertExhaustive, AccountSID } from '@tech-matters/types'; import { ContactDocument, CaseDocument, @@ -33,36 +30,7 @@ import { HRM_CASES_INDEX_TYPE, } from './hrmIndexDocumentMappings'; import { CreateIndexConvertedDocument } from '@tech-matters/elasticsearch-client'; - -type IndexOperation = 'index' | 'remove'; - -type IndexContactMessage = { - type: 'contact'; - operation: IndexOperation; - contact: Pick & Partial; -}; - -type IndexCaseMessage = { - type: 'case'; - operation: IndexOperation; - case: Pick & - Partial> & { - sections: NonNullable; - }; -}; - -export type IndexMessage = { accountSid: AccountSID } & ( - | IndexContactMessage - | IndexCaseMessage -); - -type IndexPayloadContact = IndexContactMessage & { - transcript: NonNullable; -}; - -type IndexPayloadCase = IndexCaseMessage; - -export type IndexPayload = IndexPayloadContact | IndexPayloadCase; +import { IndexPayload, IndexPayloadCase, IndexPayloadContact } from './payload'; const filterEmpty = (doc: T): T => Object.entries(doc).reduce((accum, [key, value]) => { @@ -73,7 +41,7 @@ const filterEmpty = (doc: T): T => return accum; }, {} as T); -const convertContactToContactDocument = ({ +export const convertContactToContactDocument = ({ contact, transcript, }: IndexPayloadContact): CreateIndexConvertedDocument => { @@ -208,82 +176,3 @@ export const convertToIndexDocument = ( throw new Error(`convertToIndexDocument not implemented for index ${indexName}`); }; - -const convertContactToCaseScriptUpdate = ( - payload: IndexPayloadContact, -): { - documentUpdate: CreateIndexConvertedDocument; - scriptUpdate: Script; -} => { - const { operation } = payload; - const { accountSid, caseId } = payload.contact; - - switch (operation) { - case 'index': { - const contactDocument = convertContactToContactDocument(payload); - - const documentUpdate: CreateIndexConvertedDocument = { - id: parseInt(caseId!, 10), - accountSid, - contacts: [contactDocument], - }; - - const scriptUpdate: Script = { - source: - 'def replaceContact(Map newContact, List contacts) { contacts.removeIf(contact -> contact.id == newContact.id); contacts.add(newContact); } replaceContact(params.newContact, ctx._source.contacts);', - params: { - newContact: contactDocument, - }, - }; - - console.log('>>>> documentUpdate', payload) - console.log('>>>> documentUpdate', documentUpdate) - console.log('>>>> scriptUpdate', scriptUpdate) - - return { documentUpdate, scriptUpdate }; - } - case 'remove': { - const scriptUpdate: Script = { - source: - 'def removeContact(int contactId, List contacts) { contacts.removeIf(contact -> contact.id == contactId); } removeContact(params.contactId, ctx._source.contacts);', - params: { - contactId: payload.contact.id, - }, - }; - - return { documentUpdate: {}, scriptUpdate }; - } - default: { - return assertExhaustive(operation); - } - } -}; - -const convertToCaseScriptUpdate = ( - payload: IndexPayload, -): { - documentUpdate: CreateIndexConvertedDocument; - scriptUpdate: Script; -} => { - if (payload.type === 'contact') { - return convertContactToCaseScriptUpdate(payload); - } - - throw new Error( - `convertToCaseScriptDocument not implemented for type ${payload.type} and operation ${payload.operation}`, - ); -}; - -export const convertToScriptUpdate = ( - payload: IndexPayload, - indexName: string, -): { - documentUpdate: CreateIndexConvertedDocument; - scriptUpdate: Script; -} => { - if (indexName.endsWith(HRM_CASES_INDEX_TYPE)) { - return convertToCaseScriptUpdate(payload); - } - - throw new Error(`convertToScriptDocument not implemented for index ${indexName}`); -}; diff --git a/hrm-domain/packages/hrm-search-config/convertToScriptUpdate.ts b/hrm-domain/packages/hrm-search-config/convertToScriptUpdate.ts new file mode 100644 index 000000000..d918d3e57 --- /dev/null +++ b/hrm-domain/packages/hrm-search-config/convertToScriptUpdate.ts @@ -0,0 +1,109 @@ +/** + * Copyright (C) 2021-2023 Technology Matters + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see https://www.gnu.org/licenses/. + */ + +/** + * This is a very early example of a rudimentary configuration for a multi-language index in ES. + * + * There is a lot of room for improvement here to allow more robust use of the ES query string + * syntax, but this is a start that gets us close to the functionality we scoped out for cloudsearch. + * + * see: https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-query-string-query.html + */ +import type { Script } from '@elastic/elasticsearch/lib/api/types'; +import { assertExhaustive } from '@tech-matters/types'; +import { CreateIndexConvertedDocument } from '@tech-matters/elasticsearch-client'; +import { IndexPayload, IndexPayloadContact } from './payload'; +import { + CaseDocument, + ContactDocument, + HRM_CASES_INDEX_TYPE, +} from './hrmIndexDocumentMappings'; +import { convertContactToContactDocument } from './convertToIndexDocument'; + +const convertContactToCaseScriptUpdate = ( + payload: IndexPayloadContact, +): { + documentUpdate: CreateIndexConvertedDocument; + scriptUpdate: Script; +} => { + const { operation } = payload; + const { accountSid, caseId } = payload.contact; + + switch (operation) { + case 'index': { + const contactDocument = convertContactToContactDocument(payload); + + const documentUpdate: CreateIndexConvertedDocument = { + id: parseInt(caseId!, 10), + accountSid, + contacts: [contactDocument], + }; + + const scriptUpdate: Script = { + source: + 'def replaceContact(Map newContact, List contacts) { contacts.removeIf(contact -> contact.id == newContact.id); contacts.add(newContact); } replaceContact(params.newContact, ctx._source.contacts);', + params: { + newContact: contactDocument, + }, + }; + + return { documentUpdate, scriptUpdate }; + } + case 'remove': { + const scriptUpdate: Script = { + source: + 'def removeContact(int contactId, List contacts) { contacts.removeIf(contact -> contact.id == contactId); } removeContact(params.contactId, ctx._source.contacts);', + params: { + contactId: payload.contact.id, + }, + }; + + return { documentUpdate: {}, scriptUpdate }; + } + default: { + return assertExhaustive(operation); + } + } +}; + +const convertToCaseScriptUpdate = ( + payload: IndexPayload, +): { + documentUpdate: CreateIndexConvertedDocument; + scriptUpdate: Script; +} => { + if (payload.type === 'contact') { + return convertContactToCaseScriptUpdate(payload); + } + + throw new Error( + `convertToCaseScriptDocument not implemented for type ${payload.type} and operation ${payload.operation}`, + ); +}; + +export const convertToScriptUpdate = ( + payload: IndexPayload, + indexName: string, +): { + documentUpdate: CreateIndexConvertedDocument; + scriptUpdate: Script; +} => { + if (indexName.endsWith(HRM_CASES_INDEX_TYPE)) { + return convertToCaseScriptUpdate(payload); + } + + throw new Error(`convertToScriptDocument not implemented for index ${indexName}`); +}; diff --git a/hrm-domain/packages/hrm-search-config/index.ts b/hrm-domain/packages/hrm-search-config/index.ts index c4c76df56..10017f9dc 100644 --- a/hrm-domain/packages/hrm-search-config/index.ts +++ b/hrm-domain/packages/hrm-search-config/index.ts @@ -14,22 +14,20 @@ * along with this program. If not, see https://www.gnu.org/licenses/. */ -import { - IndexPayload, - convertToIndexDocument, - convertToScriptUpdate, -} from './convertToIndexDocument'; +import { convertToIndexDocument } from './convertToIndexDocument'; +import { convertToScriptUpdate } from './convertToScriptUpdate'; import { getCreateIndexParams } from './getCreateIndexParams'; import type { IndexConfiguration, SearchConfiguration, } from '@tech-matters/elasticsearch-client'; +import { IndexPayload } from './payload'; export { HRM_CASES_INDEX_TYPE, HRM_CONTACTS_INDEX_TYPE, } from './hrmIndexDocumentMappings'; -export { IndexMessage, IndexPayload } from './convertToIndexDocument'; +export { IndexMessage, IndexPayload } from './payload'; export const hrmSearchConfiguration: SearchConfiguration = { searchFieldBoosts: { diff --git a/hrm-domain/packages/hrm-search-config/payload.ts b/hrm-domain/packages/hrm-search-config/payload.ts new file mode 100644 index 000000000..2627cab9e --- /dev/null +++ b/hrm-domain/packages/hrm-search-config/payload.ts @@ -0,0 +1,56 @@ +/** + * Copyright (C) 2021-2023 Technology Matters + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see https://www.gnu.org/licenses/. + */ + +/** + * This is a very early example of a rudimentary configuration for a multi-language index in ES. + * + * There is a lot of room for improvement here to allow more robust use of the ES query string + * syntax, but this is a start that gets us close to the functionality we scoped out for cloudsearch. + * + * see: https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-query-string-query.html + */ +import type { CaseService, Contact } from '@tech-matters/hrm-types'; +import { AccountSID } from '@tech-matters/types'; + +type IndexOperation = 'index' | 'remove'; + +type IndexContactMessage = { + type: 'contact'; + operation: IndexOperation; + contact: Pick & Partial; +}; + +type IndexCaseMessage = { + type: 'case'; + operation: IndexOperation; + case: Pick & + Partial> & { + sections: NonNullable; + }; +}; + +export type IndexMessage = { accountSid: AccountSID } & ( + | IndexContactMessage + | IndexCaseMessage +); + +export type IndexPayloadContact = IndexContactMessage & { + transcript: NonNullable; +}; + +export type IndexPayloadCase = IndexCaseMessage; + +export type IndexPayload = IndexPayloadContact | IndexPayloadCase; From 4101c0c211612b42f75c738703ad41ab572b93a1 Mon Sep 17 00:00:00 2001 From: Gianfranco Paoloni Date: Tue, 14 May 2024 16:51:51 -0300 Subject: [PATCH 21/55] chore: refactor search-index-consumer to improve redability --- .../lambdas/search-index-consumer/index.ts | 238 +----------------- .../lambdas/search-index-consumer/messages.ts | 41 +++ .../messagesToPayloads.ts | 126 ++++++++++ .../search-index-consumer/payloadToIndex.ts | 138 ++++++++++ 4 files changed, 318 insertions(+), 225 deletions(-) create mode 100644 hrm-domain/lambdas/search-index-consumer/messages.ts create mode 100644 hrm-domain/lambdas/search-index-consumer/messagesToPayloads.ts create mode 100644 hrm-domain/lambdas/search-index-consumer/payloadToIndex.ts diff --git a/hrm-domain/lambdas/search-index-consumer/index.ts b/hrm-domain/lambdas/search-index-consumer/index.ts index 42fb0c4ca..443fc8c07 100644 --- a/hrm-domain/lambdas/search-index-consumer/index.ts +++ b/hrm-domain/lambdas/search-index-consumer/index.ts @@ -13,245 +13,32 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see https://www.gnu.org/licenses/. */ -import type { SQSBatchResponse, SQSEvent, SQSRecord } from 'aws-lambda'; -import { getClient, IndexClient } from '@tech-matters/elasticsearch-client'; +import type { SQSBatchResponse, SQSEvent } from 'aws-lambda'; // import { GetSignedUrlMethods, GET_SIGNED_URL_METHODS } from '@tech-matters/s3-client'; import { HrmIndexProcessorError } from '@tech-matters/job-errors'; -import { - HRM_CASES_INDEX_TYPE, - HRM_CONTACTS_INDEX_TYPE, - hrmIndexConfiguration, - IndexMessage, - IndexPayload, -} from '@tech-matters/hrm-search-config'; -import { - AccountSID, - assertExhaustive, - isErr, - newErr, - newOkFromData, -} from '@tech-matters/types'; - -type MessageWithMeta = { message: IndexMessage; messageId: string }; -type MessagesByAccountSid = Record; -type PayloadWithMeta = { - documentId: number; - payload: IndexPayload; - messageId: string; - indexHandler: 'indexDocument' | 'updateDocument' | 'updateScript'; -}; -type PayloadsByIndex = { - [indexType: string]: PayloadWithMeta[]; -}; -type PayloadsByAccountSid = Record; - -const groupMessagesByAccountSid = ( - accum: MessagesByAccountSid, - curr: SQSRecord, -): MessagesByAccountSid => { - const { messageId, body } = curr; - const message = JSON.parse(body) as IndexMessage; - - const { accountSid } = message; - - if (!accum[accountSid]) { - return { ...accum, [accountSid]: [{ messageId, message }] }; - } - - return { ...accum, [accountSid]: [...accum[accountSid], { messageId, message }] }; -}; - -const messagesToPayloadsByIndex = ( - accum: PayloadsByIndex, - currM: MessageWithMeta, -): PayloadsByIndex => { - const { message } = currM; - - const { type } = message; - - switch (type) { - case 'contact': { - // TODO: Pull the transcripts from S3 (if any) - return { - ...accum, - // add an upsert job to HRM_CONTACTS_INDEX_TYPE index - [HRM_CONTACTS_INDEX_TYPE]: [ - ...(accum[HRM_CONTACTS_INDEX_TYPE] ?? []), - { - ...currM, - documentId: message.contact.id, - payload: { ...message, transcript: '' }, - indexHandler: 'updateDocument', - }, - ], - // if associated to a case, add an upsert with script job to HRM_CASES_INDEX_TYPE index - [HRM_CASES_INDEX_TYPE]: message.contact.caseId - ? [ - ...(accum[HRM_CASES_INDEX_TYPE] ?? []), - { - ...currM, - documentId: parseInt(message.contact.caseId, 10), - payload: { ...message, transcript: '' }, - indexHandler: 'updateScript', - }, - ] - : accum[HRM_CASES_INDEX_TYPE] ?? [], - }; - } - case 'case': { - return { - ...accum, - // add an upsert job to HRM_CASES_INDEX_TYPE index - [HRM_CASES_INDEX_TYPE]: [ - ...(accum[HRM_CASES_INDEX_TYPE] ?? []), - { - ...currM, - documentId: message.case.id, - payload: { ...message }, - indexHandler: 'updateDocument', - }, - ], - }; - } - default: { - return assertExhaustive(type); - } - } -}; - -const handleIndexPayload = - ({ - accountSid, - client, - indexType, - }: { - accountSid: string; - client: IndexClient; - indexType: string; - }) => - async ({ documentId, indexHandler, messageId, payload }: PayloadWithMeta) => { - try { - switch (indexHandler) { - case 'indexDocument': { - const result = await client.indexDocument({ - id: documentId.toString(), - document: payload, - autocreate: true, - }); - - return { - accountSid, - indexType, - messageId, - result: newOkFromData(result), - }; - } - case 'updateDocument': { - const result = await client.updateDocument({ - id: documentId.toString(), - document: payload, - autocreate: true, - docAsUpsert: true, - }); - - return { - accountSid, - indexType, - messageId, - result: newOkFromData(result), - }; - } - case 'updateScript': { - const result = await client.updateScript({ - document: payload, - id: documentId.toString(), - autocreate: true, - scriptedUpsert: true, - }); - - return { - accountSid, - indexType, - messageId, - result: newOkFromData(result), - }; - } - default: { - return assertExhaustive(indexHandler); - } - } - } catch (err) { - console.error( - new HrmIndexProcessorError('handleIndexPayload: Failed to process index request'), - err, - { documentId, indexHandler, messageId, payload }, - ); - - return { - accountSid, - indexType, - messageId, - result: newErr({ - error: 'ErrorFailedToInex', - message: err instanceof Error ? err.message : String(err), - }), - }; - } - }; - -const indexDocumentsByIndex = - (accountSid: string) => - async ([indexType, payloads]: [string, PayloadWithMeta[]]) => { - // get the client for the accountSid-indexType pair - const client = (await getClient({ accountSid, indexType })).indexClient( - hrmIndexConfiguration, - ); - - const mapper = handleIndexPayload({ client, accountSid, indexType }); - - const indexed = await Promise.all(payloads.map(mapper)); - - return indexed; - }; - -const indexDocumentsByAccount = async ([accountSid, payloadsByIndex]: [ - string, - PayloadsByIndex, -]) => { - const resultsByIndex = await Promise.all( - Object.entries(payloadsByIndex).map(indexDocumentsByIndex(accountSid)), - ); - - return resultsByIndex; -}; +import { isErr } from '@tech-matters/types'; +import { groupMessagesByAccountSid } from './messages'; +import { messagesToPayloadsByAccountSid } from './messagesToPayloads'; +import { indexDocumentsByAccount } from './payloadToIndex'; export const handler = async (event: SQSEvent): Promise => { console.debug('Received event:', JSON.stringify(event, null, 2)); try { // group the messages by accountSid while adding message meta - const messagesByAccoundSid = event.Records.reduce( - groupMessagesByAccountSid, - {}, - ); + const messagesByAccoundSid = groupMessagesByAccountSid(event.Records); - // generate corresponding IndexPayload for each IndexMessage and group them by target indexType - const documentsByAccountSid: PayloadsByAccountSid = Object.fromEntries( - Object.entries(messagesByAccoundSid).map(([accountSid, messages]) => { - const payloads = messages.reduce(messagesToPayloadsByIndex, {}); + // generate corresponding IndexPayload for each IndexMessage and group them by target accountSid-indexType pair + const payloadsByAccountSid = messagesToPayloadsByAccountSid(messagesByAccoundSid); - return [accountSid, payloads] as const; - }), - ); + console.debug('Mapped messages:', JSON.stringify(payloadsByAccountSid, null, 2)); - console.debug('Mapped messages:', JSON.stringify(documentsByAccountSid, null, 2)); - - const resultsByAccount = await Promise.all( - Object.entries(documentsByAccountSid).map(indexDocumentsByAccount), - ); + // index all the payloads + const resultsByAccount = await indexDocumentsByAccount(payloadsByAccountSid); console.debug(`Successfully indexed documents`); + // filter the payloads that failed indexing const documentsWithErrors = resultsByAccount .flat(2) .filter(({ result }) => isErr(result)); @@ -263,6 +50,7 @@ export const handler = async (event: SQSEvent): Promise => { ); } + // send the failed payloads back to SQS so they are redrive to DLQ const response: SQSBatchResponse = { batchItemFailures: documentsWithErrors.map(({ messageId }) => ({ itemIdentifier: messageId, diff --git a/hrm-domain/lambdas/search-index-consumer/messages.ts b/hrm-domain/lambdas/search-index-consumer/messages.ts new file mode 100644 index 000000000..447fd01fd --- /dev/null +++ b/hrm-domain/lambdas/search-index-consumer/messages.ts @@ -0,0 +1,41 @@ +/** + * Copyright (C) 2021-2023 Technology Matters + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see https://www.gnu.org/licenses/. + */ + +import type { IndexMessage } from '@tech-matters/hrm-search-config'; +import type { AccountSID } from '@tech-matters/types'; +import type { SQSRecord } from 'aws-lambda'; + +export type MessageWithMeta = { message: IndexMessage; messageId: string }; +export type MessagesByAccountSid = Record; + +const groupMessagesReducer = ( + accum: MessagesByAccountSid, + curr: SQSRecord, +): MessagesByAccountSid => { + const { messageId, body } = curr; + const message = JSON.parse(body) as IndexMessage; + + const { accountSid } = message; + + if (!accum[accountSid]) { + return { ...accum, [accountSid]: [{ messageId, message }] }; + } + + return { ...accum, [accountSid]: [...accum[accountSid], { messageId, message }] }; +}; + +export const groupMessagesByAccountSid = (records: SQSRecord[]): MessagesByAccountSid => + records.reduce(groupMessagesReducer, {}); diff --git a/hrm-domain/lambdas/search-index-consumer/messagesToPayloads.ts b/hrm-domain/lambdas/search-index-consumer/messagesToPayloads.ts new file mode 100644 index 000000000..868c15dc0 --- /dev/null +++ b/hrm-domain/lambdas/search-index-consumer/messagesToPayloads.ts @@ -0,0 +1,126 @@ +/** + * Copyright (C) 2021-2023 Technology Matters + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see https://www.gnu.org/licenses/. + */ + +import { + HRM_CASES_INDEX_TYPE, + HRM_CONTACTS_INDEX_TYPE, + type IndexPayload, +} from '@tech-matters/hrm-search-config'; +import { assertExhaustive, type AccountSID } from '@tech-matters/types'; +import type { MessageWithMeta, MessagesByAccountSid } from './messages'; + +/** + * A payload is single object that should be indexed in a particular index. A single message might represent multiple payloads. + * The indexHandler represents the operation that should be used when indexing the given document: + * - indexDocument: Used when we don't care overriding the previous versions of the document. An example is when a document is created for the first time + * - updateDocument: Used when we want to preserve the existing document (if any), using the document object for the update. An example is update a case + * - updateScript: Used when we want to preserve the existing document (if any), using the generated "update script" for the update. An example is updating case.contacts list when a contact is indexed + */ +export type PayloadWithMeta = { + documentId: number; + payload: IndexPayload; + messageId: string; + indexHandler: 'indexDocument' | 'updateDocument' | 'updateScript'; +}; +export type PayloadsByIndex = { + [indexType: string]: PayloadWithMeta[]; +}; +export type PayloadsByAccountSid = Record; + +// TODO: Pull the transcripts from S3 (if any) +const generatePayloadFromContact = ( + ps: PayloadsByIndex, + m: MessageWithMeta & { message: { type: 'contact' } }, +): PayloadsByIndex => ({ + ...ps, + // add an upsert job to HRM_CONTACTS_INDEX_TYPE index + [HRM_CONTACTS_INDEX_TYPE]: [ + ...(ps[HRM_CONTACTS_INDEX_TYPE] ?? []), + { + ...m, + documentId: m.message.contact.id, + payload: { ...m.message, transcript: '' }, + indexHandler: 'updateDocument', + }, + ], + // if associated to a case, add an upsert with script job to HRM_CASES_INDEX_TYPE index + [HRM_CASES_INDEX_TYPE]: m.message.contact.caseId + ? [ + ...(ps[HRM_CASES_INDEX_TYPE] ?? []), + { + ...m, + documentId: parseInt(m.message.contact.caseId, 10), + payload: { ...m.message, transcript: '' }, + indexHandler: 'updateScript', + }, + ] + : ps[HRM_CASES_INDEX_TYPE] ?? [], +}); + +const generatePayloadFromCase = ( + ps: PayloadsByIndex, + m: MessageWithMeta & { message: { type: 'case' } }, +): PayloadsByIndex => ({ + ...ps, + // add an upsert job to HRM_CASES_INDEX_TYPE index + [HRM_CASES_INDEX_TYPE]: [ + ...(ps[HRM_CASES_INDEX_TYPE] ?? []), + { + ...m, + documentId: m.message.case.id, + payload: { ...m.message }, + indexHandler: 'updateDocument', + }, + ], +}); + +const messagesToPayloadReducer = ( + accum: PayloadsByIndex, + currM: MessageWithMeta, +): PayloadsByIndex => { + const { message, messageId } = currM; + + const { type } = message; + + switch (type) { + case 'contact': { + return generatePayloadFromContact(accum, { message, messageId }); + } + case 'case': { + return generatePayloadFromCase(accum, { message, messageId }); + } + default: { + return assertExhaustive(type); + } + } +}; + +const messagesToPayloadsByIndex = (messages: MessageWithMeta[]): PayloadsByIndex => + messages.reduce(messagesToPayloadReducer, {}); + +export const messagesToPayloadsByAccountSid = ( + messages: MessagesByAccountSid, +): PayloadsByAccountSid => { + const payloadsByAccountSidEntries = Object.entries(messages).map(([accountSid, ms]) => { + const payloads = messagesToPayloadsByIndex(ms); + + return [accountSid, payloads] as const; + }); + + const payloadsByAccountSid = Object.fromEntries(payloadsByAccountSidEntries); + + return payloadsByAccountSid; +}; diff --git a/hrm-domain/lambdas/search-index-consumer/payloadToIndex.ts b/hrm-domain/lambdas/search-index-consumer/payloadToIndex.ts new file mode 100644 index 000000000..0e6b96d29 --- /dev/null +++ b/hrm-domain/lambdas/search-index-consumer/payloadToIndex.ts @@ -0,0 +1,138 @@ +/** + * Copyright (C) 2021-2023 Technology Matters + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see https://www.gnu.org/licenses/. + */ + +import { type IndexClient, getClient } from '@tech-matters/elasticsearch-client'; +import { + type IndexPayload, + hrmIndexConfiguration, +} from '@tech-matters/hrm-search-config'; +import type { + PayloadWithMeta, + PayloadsByAccountSid, + PayloadsByIndex, +} from './messagesToPayloads'; +import { assertExhaustive, newErr, newOkFromData } from '@tech-matters/types'; +import { HrmIndexProcessorError } from '@tech-matters/job-errors'; + +const handleIndexPayload = + ({ + accountSid, + client, + indexType, + }: { + accountSid: string; + client: IndexClient; + indexType: string; + }) => + async ({ documentId, indexHandler, messageId, payload }: PayloadWithMeta) => { + try { + switch (indexHandler) { + case 'indexDocument': { + const result = await client.indexDocument({ + id: documentId.toString(), + document: payload, + autocreate: true, + }); + + return { + accountSid, + indexType, + messageId, + result: newOkFromData(result), + }; + } + case 'updateDocument': { + const result = await client.updateDocument({ + id: documentId.toString(), + document: payload, + autocreate: true, + docAsUpsert: true, + }); + + return { + accountSid, + indexType, + messageId, + result: newOkFromData(result), + }; + } + case 'updateScript': { + const result = await client.updateScript({ + document: payload, + id: documentId.toString(), + autocreate: true, + scriptedUpsert: true, + }); + + return { + accountSid, + indexType, + messageId, + result: newOkFromData(result), + }; + } + default: { + return assertExhaustive(indexHandler); + } + } + } catch (err) { + console.error( + new HrmIndexProcessorError('handleIndexPayload: Failed to process index request'), + err, + { documentId, indexHandler, messageId, payload }, + ); + + return { + accountSid, + indexType, + messageId, + result: newErr({ + error: 'ErrorFailedToInex', + message: err instanceof Error ? err.message : String(err), + }), + }; + } + }; + +const indexDocumentsByIndexMapper = + (accountSid: string) => + async ([indexType, payloads]: [string, PayloadWithMeta[]]) => { + // get the client for the accountSid-indexType pair + const client = (await getClient({ accountSid, indexType })).indexClient( + hrmIndexConfiguration, + ); + + const mapper = handleIndexPayload({ client, accountSid, indexType }); + + const indexed = await Promise.all(payloads.map(mapper)); + + return indexed; + }; + +const indexDocumentsByAccountMapper = async ([accountSid, payloadsByIndex]: [ + string, + PayloadsByIndex, +]) => { + const resultsByIndex = await Promise.all( + Object.entries(payloadsByIndex).map(indexDocumentsByIndexMapper(accountSid)), + ); + + return resultsByIndex; +}; + +export const indexDocumentsByAccount = async ( + payloadsByAccountSid: PayloadsByAccountSid, +) => Promise.all(Object.entries(payloadsByAccountSid).map(indexDocumentsByAccountMapper)); From 7df50bd907d6a617f23905f9d8db9edf54749b48 Mon Sep 17 00:00:00 2001 From: Gianfranco Paoloni Date: Tue, 14 May 2024 17:49:33 -0300 Subject: [PATCH 22/55] chore: lint --- .../convertToIndexDocument.ts | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/hrm-domain/packages/hrm-search-config/convertToIndexDocument.ts b/hrm-domain/packages/hrm-search-config/convertToIndexDocument.ts index e3a48207a..1ea4ce253 100644 --- a/hrm-domain/packages/hrm-search-config/convertToIndexDocument.ts +++ b/hrm-domain/packages/hrm-search-config/convertToIndexDocument.ts @@ -65,16 +65,16 @@ export const convertContactToContactDocument = ({ accountSid, id, createdAt, - updatedAt: updatedAt, - createdBy: createdBy, - updatedBy: updatedBy, + updatedAt, + createdBy, + updatedBy, finalized: Boolean(finalizedAt), - helpline: helpline, - channel: channel, - number: number, - timeOfContact: timeOfContact, + helpline, + channel, + number, + timeOfContact, transcript, - twilioWorkerId: twilioWorkerId, + twilioWorkerId, content: JSON.stringify(rawJson), // high_boost_global: '', // highBoostGlobal.join(' '), // low_boost_global: '', // lowBoostGlobal.join(' '), @@ -129,9 +129,9 @@ const convertCaseToCaseDocument = ({ helpline, twilioWorkerId, status, - previousStatus: previousStatus, - statusUpdatedAt: statusUpdatedAt, - statusUpdatedBy: statusUpdatedBy, + previousStatus, + statusUpdatedAt, + statusUpdatedBy, content: JSON.stringify(info), sections: mappedSections, contacts: null, From 06f0254b0c9e154bcb2bf1184d543a0d8699b1fe Mon Sep 17 00:00:00 2001 From: Gianfranco Paoloni Date: Tue, 14 May 2024 18:08:52 -0300 Subject: [PATCH 23/55] chore: removed comments --- .../packages/hrm-search-config/convertToIndexDocument.ts | 9 --------- .../packages/hrm-search-config/convertToScriptUpdate.ts | 9 --------- .../packages/hrm-search-config/getCreateIndexParams.ts | 9 --------- hrm-domain/packages/hrm-search-config/payload.ts | 8 -------- 4 files changed, 35 deletions(-) diff --git a/hrm-domain/packages/hrm-search-config/convertToIndexDocument.ts b/hrm-domain/packages/hrm-search-config/convertToIndexDocument.ts index 1ea4ce253..f1ae36ced 100644 --- a/hrm-domain/packages/hrm-search-config/convertToIndexDocument.ts +++ b/hrm-domain/packages/hrm-search-config/convertToIndexDocument.ts @@ -14,15 +14,6 @@ * along with this program. If not, see https://www.gnu.org/licenses/. */ -/** - * This is a very early example of a rudimentary configuration for a multi-language index in ES. - * - * There is a lot of room for improvement here to allow more robust use of the ES query string - * syntax, but this is a start that gets us close to the functionality we scoped out for cloudsearch. - * - * see: https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-query-string-query.html - */ - import { ContactDocument, CaseDocument, diff --git a/hrm-domain/packages/hrm-search-config/convertToScriptUpdate.ts b/hrm-domain/packages/hrm-search-config/convertToScriptUpdate.ts index d918d3e57..23ac05389 100644 --- a/hrm-domain/packages/hrm-search-config/convertToScriptUpdate.ts +++ b/hrm-domain/packages/hrm-search-config/convertToScriptUpdate.ts @@ -13,15 +13,6 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see https://www.gnu.org/licenses/. */ - -/** - * This is a very early example of a rudimentary configuration for a multi-language index in ES. - * - * There is a lot of room for improvement here to allow more robust use of the ES query string - * syntax, but this is a start that gets us close to the functionality we scoped out for cloudsearch. - * - * see: https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-query-string-query.html - */ import type { Script } from '@elastic/elasticsearch/lib/api/types'; import { assertExhaustive } from '@tech-matters/types'; import { CreateIndexConvertedDocument } from '@tech-matters/elasticsearch-client'; diff --git a/hrm-domain/packages/hrm-search-config/getCreateIndexParams.ts b/hrm-domain/packages/hrm-search-config/getCreateIndexParams.ts index 5850c6cd1..884ce917a 100644 --- a/hrm-domain/packages/hrm-search-config/getCreateIndexParams.ts +++ b/hrm-domain/packages/hrm-search-config/getCreateIndexParams.ts @@ -14,15 +14,6 @@ * along with this program. If not, see https://www.gnu.org/licenses/. */ -/** - * This is a very early example of a rudimentary configuration for a multi-language index in ES. - * - * There is a lot of room for improvement here to allow more robust use of the ES query string - * syntax, but this is a start that gets us close to the functionality we scoped out for cloudsearch. - * - * see: https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-query-string-query.html - */ - import type { IndicesCreateRequest } from '@elastic/elasticsearch/lib/api/types'; import { diff --git a/hrm-domain/packages/hrm-search-config/payload.ts b/hrm-domain/packages/hrm-search-config/payload.ts index 2627cab9e..379976a2e 100644 --- a/hrm-domain/packages/hrm-search-config/payload.ts +++ b/hrm-domain/packages/hrm-search-config/payload.ts @@ -14,14 +14,6 @@ * along with this program. If not, see https://www.gnu.org/licenses/. */ -/** - * This is a very early example of a rudimentary configuration for a multi-language index in ES. - * - * There is a lot of room for improvement here to allow more robust use of the ES query string - * syntax, but this is a start that gets us close to the functionality we scoped out for cloudsearch. - * - * see: https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-query-string-query.html - */ import type { CaseService, Contact } from '@tech-matters/hrm-types'; import { AccountSID } from '@tech-matters/types'; From b208991c1343d77c852bae211302d03258c50a49 Mon Sep 17 00:00:00 2001 From: Gianfranco Paoloni Date: Thu, 16 May 2024 16:00:38 -0300 Subject: [PATCH 24/55] chore: moved channelTypes constants to hrm-types module --- hrm-domain/hrm-core/contact/contactService.ts | 2 +- hrm-domain/hrm-service/package.json | 1 + .../contact-job/jobTypes/retrieveTranscript.test.ts | 2 +- .../service-tests/contact/contactConversationMedia.test.ts | 2 +- .../{hrm-core/contact => packages/hrm-types}/channelTypes.ts | 1 - hrm-domain/packages/hrm-types/index.ts | 1 + package-lock.json | 2 ++ 7 files changed, 7 insertions(+), 4 deletions(-) rename hrm-domain/{hrm-core/contact => packages/hrm-types}/channelTypes.ts (95%) diff --git a/hrm-domain/hrm-core/contact/contactService.ts b/hrm-domain/hrm-core/contact/contactService.ts index 7435ed9c0..655631639 100644 --- a/hrm-domain/hrm-core/contact/contactService.ts +++ b/hrm-domain/hrm-core/contact/contactService.ts @@ -50,7 +50,7 @@ import { actionsMaps } from '../permissions'; import type { TwilioUser } from '@tech-matters/twilio-worker-auth'; import { createReferral } from '../referral/referral-model'; import { createContactJob } from '../contact-job/contact-job'; -import { isChatChannel } from './channelTypes'; +import { isChatChannel } from '@tech-matters/hrm-types'; import { enableCreateContactJobsFlag } from '../featureFlags'; import { db } from '../connection-pool'; import { diff --git a/hrm-domain/hrm-service/package.json b/hrm-domain/hrm-service/package.json index cbbefe1bc..770d9ee79 100644 --- a/hrm-domain/hrm-service/package.json +++ b/hrm-domain/hrm-service/package.json @@ -50,6 +50,7 @@ "@tech-matters/contact-job-cleanup": "^1.0.0", "@tech-matters/hrm-core": "^1.0.0", "@tech-matters/hrm-data-pull": "^1.0.0", + "@tech-matters/hrm-types": "^1.0.0", "@tech-matters/http": "^1.0.0", "@tech-matters/profile-flags-cleanup": "^1.0.0", "@tech-matters/resources-service": "^1.0.0", diff --git a/hrm-domain/hrm-service/service-tests/contact-job/jobTypes/retrieveTranscript.test.ts b/hrm-domain/hrm-service/service-tests/contact-job/jobTypes/retrieveTranscript.test.ts index e330789ee..69d0af6b8 100644 --- a/hrm-domain/hrm-service/service-tests/contact-job/jobTypes/retrieveTranscript.test.ts +++ b/hrm-domain/hrm-service/service-tests/contact-job/jobTypes/retrieveTranscript.test.ts @@ -23,7 +23,7 @@ import * as contactJobApi from '@tech-matters/hrm-core/contact-job/contact-job-d import { db } from '@tech-matters/hrm-core/connection-pool'; import '../../case/caseValidation'; import * as conversationMediaApi from '@tech-matters/hrm-core/conversation-media/conversation-media'; -import { chatChannels } from '@tech-matters/hrm-core/contact/channelTypes'; +import { chatChannels } from '@tech-matters/hrm-types'; import { JOB_MAX_ATTEMPTS } from '@tech-matters/hrm-core/contact-job/contact-job-processor'; import { diff --git a/hrm-domain/hrm-service/service-tests/contact/contactConversationMedia.test.ts b/hrm-domain/hrm-service/service-tests/contact/contactConversationMedia.test.ts index 4f0d73c17..a6bf4a4ba 100644 --- a/hrm-domain/hrm-service/service-tests/contact/contactConversationMedia.test.ts +++ b/hrm-domain/hrm-service/service-tests/contact/contactConversationMedia.test.ts @@ -34,7 +34,7 @@ import { cleanupReferrals, } from './dbCleanup'; import each from 'jest-each'; -import { chatChannels } from '@tech-matters/hrm-core/contact/channelTypes'; +import { chatChannels } from '@tech-matters/hrm-types'; import { ContactJobType } from '@tech-matters/types/ContactJob'; import { ruleFileActionOverride } from '../permissions-overrides'; import { selectJobsByContactId } from './db-validations'; diff --git a/hrm-domain/hrm-core/contact/channelTypes.ts b/hrm-domain/packages/hrm-types/channelTypes.ts similarity index 95% rename from hrm-domain/hrm-core/contact/channelTypes.ts rename to hrm-domain/packages/hrm-types/channelTypes.ts index e940822df..8d97d64de 100644 --- a/hrm-domain/hrm-core/contact/channelTypes.ts +++ b/hrm-domain/packages/hrm-types/channelTypes.ts @@ -40,6 +40,5 @@ export const chatChannels = [ channelTypes.modica, ]; -// eslint-disable-next-line @typescript-eslint/no-unused-vars export const isVoiceChannel = (channel: string) => channel === channelTypes.voice; export const isChatChannel = (channel: string) => chatChannels.includes(channel as any); diff --git a/hrm-domain/packages/hrm-types/index.ts b/hrm-domain/packages/hrm-types/index.ts index 8366a362a..f2bd9fedd 100644 --- a/hrm-domain/packages/hrm-types/index.ts +++ b/hrm-domain/packages/hrm-types/index.ts @@ -19,3 +19,4 @@ export * from './Referral'; export * from './ConversationMedia'; export * from './Case'; export * from './CaseSection'; +export * from './channelTypes'; diff --git a/package-lock.json b/package-lock.json index 1d3847b6b..4d68813c4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -142,6 +142,7 @@ "@tech-matters/contact-job-cleanup": "^1.0.0", "@tech-matters/hrm-core": "^1.0.0", "@tech-matters/hrm-data-pull": "^1.0.0", + "@tech-matters/hrm-types": "^1.0.0", "@tech-matters/http": "^1.0.0", "@tech-matters/profile-flags-cleanup": "^1.0.0", "@tech-matters/resources-service": "^1.0.0", @@ -17644,6 +17645,7 @@ "@tech-matters/contact-job-cleanup": "^1.0.0", "@tech-matters/hrm-core": "^1.0.0", "@tech-matters/hrm-data-pull": "^1.0.0", + "@tech-matters/hrm-types": "^1.0.0", "@tech-matters/http": "^1.0.0", "@tech-matters/profile-flags-cleanup": "^1.0.0", "@tech-matters/resources-service": "^1.0.0", From acb12451a99d0f5fb51209f6863c849a2da07d91 Mon Sep 17 00:00:00 2001 From: Gianfranco Paoloni Date: Thu, 16 May 2024 18:39:08 -0300 Subject: [PATCH 25/55] chore fetch transcripts from S3 --- .../lambdas/search-index-consumer/index.ts | 4 +- .../messagesToPayloads.ts | 152 +++++++++++++----- .../packages/hrm-search-config/index.ts | 7 +- .../packages/hrm-search-config/payload.ts | 6 +- 4 files changed, 124 insertions(+), 45 deletions(-) diff --git a/hrm-domain/lambdas/search-index-consumer/index.ts b/hrm-domain/lambdas/search-index-consumer/index.ts index 443fc8c07..576343531 100644 --- a/hrm-domain/lambdas/search-index-consumer/index.ts +++ b/hrm-domain/lambdas/search-index-consumer/index.ts @@ -14,7 +14,6 @@ * along with this program. If not, see https://www.gnu.org/licenses/. */ import type { SQSBatchResponse, SQSEvent } from 'aws-lambda'; -// import { GetSignedUrlMethods, GET_SIGNED_URL_METHODS } from '@tech-matters/s3-client'; import { HrmIndexProcessorError } from '@tech-matters/job-errors'; import { isErr } from '@tech-matters/types'; import { groupMessagesByAccountSid } from './messages'; @@ -29,7 +28,8 @@ export const handler = async (event: SQSEvent): Promise => { const messagesByAccoundSid = groupMessagesByAccountSid(event.Records); // generate corresponding IndexPayload for each IndexMessage and group them by target accountSid-indexType pair - const payloadsByAccountSid = messagesToPayloadsByAccountSid(messagesByAccoundSid); + const payloadsByAccountSid = + await messagesToPayloadsByAccountSid(messagesByAccoundSid); console.debug('Mapped messages:', JSON.stringify(payloadsByAccountSid, null, 2)); diff --git a/hrm-domain/lambdas/search-index-consumer/messagesToPayloads.ts b/hrm-domain/lambdas/search-index-consumer/messagesToPayloads.ts index 868c15dc0..123063f44 100644 --- a/hrm-domain/lambdas/search-index-consumer/messagesToPayloads.ts +++ b/hrm-domain/lambdas/search-index-consumer/messagesToPayloads.ts @@ -13,13 +13,16 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see https://www.gnu.org/licenses/. */ - +import { getS3Object } from '@tech-matters/s3-client'; import { HRM_CASES_INDEX_TYPE, HRM_CONTACTS_INDEX_TYPE, type IndexPayload, + type IndexContactMessage, + type IndexCaseMessage, } from '@tech-matters/hrm-search-config'; import { assertExhaustive, type AccountSID } from '@tech-matters/types'; +import { isChatChannel, isS3StoredTranscript } from '@tech-matters/hrm-types'; import type { MessageWithMeta, MessagesByAccountSid } from './messages'; /** @@ -40,39 +43,100 @@ export type PayloadsByIndex = { }; export type PayloadsByAccountSid = Record; -// TODO: Pull the transcripts from S3 (if any) +type IntermidiateIndexContactMessage = MessageWithMeta & { + message: IndexContactMessage; +} & { + transcript: string | null; +}; + +const intermediateContactMessage = async ( + m: MessageWithMeta & { + message: IndexContactMessage; + }, +): Promise => { + let transcript: string | null = null; + + if (m.message.contact.channel && isChatChannel(m.message.contact.channel)) { + const transcriptEntry = + m.message.contact.conversationMedia?.find(isS3StoredTranscript); + if (transcriptEntry) { + const { location } = transcriptEntry.storeTypeSpecificData; + const { bucket, key } = location || {}; + if (bucket && key) { + transcript = await getS3Object({ bucket, key }); + } + } + } + + return { ...m, transcript }; +}; + +type IntermidiateIndexCaseMessage = MessageWithMeta & { + message: IndexCaseMessage; +}; +const intermediateCaseMessage = async ( + m: MessageWithMeta & { + message: IndexCaseMessage; + }, +): Promise => m; + +type IntermidiateIndexMessage = + | IntermidiateIndexContactMessage + | IntermidiateIndexCaseMessage; +const intermediateMessagesMapper = ( + m: MessageWithMeta, +): Promise => { + const { message, messageId } = m; + + const { type } = message; + + switch (type) { + case 'contact': { + return intermediateContactMessage({ message, messageId }); + } + case 'case': { + return intermediateCaseMessage({ message, messageId }); + } + default: { + return assertExhaustive(type); + } + } +}; + const generatePayloadFromContact = ( ps: PayloadsByIndex, - m: MessageWithMeta & { message: { type: 'contact' } }, -): PayloadsByIndex => ({ - ...ps, - // add an upsert job to HRM_CONTACTS_INDEX_TYPE index - [HRM_CONTACTS_INDEX_TYPE]: [ - ...(ps[HRM_CONTACTS_INDEX_TYPE] ?? []), - { - ...m, - documentId: m.message.contact.id, - payload: { ...m.message, transcript: '' }, - indexHandler: 'updateDocument', - }, - ], - // if associated to a case, add an upsert with script job to HRM_CASES_INDEX_TYPE index - [HRM_CASES_INDEX_TYPE]: m.message.contact.caseId - ? [ - ...(ps[HRM_CASES_INDEX_TYPE] ?? []), - { - ...m, - documentId: parseInt(m.message.contact.caseId, 10), - payload: { ...m.message, transcript: '' }, - indexHandler: 'updateScript', - }, - ] - : ps[HRM_CASES_INDEX_TYPE] ?? [], -}); + m: IntermidiateIndexContactMessage, +): PayloadsByIndex => { + return { + ...ps, + // add an upsert job to HRM_CONTACTS_INDEX_TYPE index + [HRM_CONTACTS_INDEX_TYPE]: [ + ...(ps[HRM_CONTACTS_INDEX_TYPE] ?? []), + { + ...m, + documentId: m.message.contact.id, + payload: { ...m.message, transcript: m.transcript }, + indexHandler: 'updateDocument', + }, + ], + // if associated to a case, add an upsert with script job to HRM_CASES_INDEX_TYPE index + [HRM_CASES_INDEX_TYPE]: m.message.contact.caseId + ? [ + ...(ps[HRM_CASES_INDEX_TYPE] ?? []), + { + ...m, + documentId: parseInt(m.message.contact.caseId, 10), + payload: { ...m.message, transcript: m.transcript }, + indexHandler: 'updateScript', + }, + ] + : ps[HRM_CASES_INDEX_TYPE] ?? [], + }; +}; const generatePayloadFromCase = ( ps: PayloadsByIndex, - m: MessageWithMeta & { message: { type: 'case' } }, + m: IntermidiateIndexCaseMessage, ): PayloadsByIndex => ({ ...ps, // add an upsert job to HRM_CASES_INDEX_TYPE index @@ -89,7 +153,7 @@ const generatePayloadFromCase = ( const messagesToPayloadReducer = ( accum: PayloadsByIndex, - currM: MessageWithMeta, + currM: IntermidiateIndexMessage, ): PayloadsByIndex => { const { message, messageId } = currM; @@ -97,7 +161,8 @@ const messagesToPayloadReducer = ( switch (type) { case 'contact': { - return generatePayloadFromContact(accum, { message, messageId }); + const { transcript } = currM as IntermidiateIndexContactMessage; + return generatePayloadFromContact(accum, { message, messageId, transcript }); } case 'case': { return generatePayloadFromCase(accum, { message, messageId }); @@ -108,17 +173,26 @@ const messagesToPayloadReducer = ( } }; -const messagesToPayloadsByIndex = (messages: MessageWithMeta[]): PayloadsByIndex => - messages.reduce(messagesToPayloadReducer, {}); +const messagesToPayloadsByIndex = async ( + messages: MessageWithMeta[], +): Promise => { + const intermidiateMessages = await Promise.all( + messages.map(intermediateMessagesMapper), + ); + + return intermidiateMessages.reduce(messagesToPayloadReducer, {}); +}; -export const messagesToPayloadsByAccountSid = ( +export const messagesToPayloadsByAccountSid = async ( messages: MessagesByAccountSid, -): PayloadsByAccountSid => { - const payloadsByAccountSidEntries = Object.entries(messages).map(([accountSid, ms]) => { - const payloads = messagesToPayloadsByIndex(ms); +): Promise => { + const payloadsByAccountSidEntries = await Promise.all( + Object.entries(messages).map(async ([accountSid, ms]) => { + const payloads = await messagesToPayloadsByIndex(ms); - return [accountSid, payloads] as const; - }); + return [accountSid, payloads] as const; + }), + ); const payloadsByAccountSid = Object.fromEntries(payloadsByAccountSidEntries); diff --git a/hrm-domain/packages/hrm-search-config/index.ts b/hrm-domain/packages/hrm-search-config/index.ts index 10017f9dc..352a2105e 100644 --- a/hrm-domain/packages/hrm-search-config/index.ts +++ b/hrm-domain/packages/hrm-search-config/index.ts @@ -27,7 +27,12 @@ export { HRM_CASES_INDEX_TYPE, HRM_CONTACTS_INDEX_TYPE, } from './hrmIndexDocumentMappings'; -export { IndexMessage, IndexPayload } from './payload'; +export { + IndexMessage, + IndexCaseMessage, + IndexContactMessage, + IndexPayload, +} from './payload'; export const hrmSearchConfiguration: SearchConfiguration = { searchFieldBoosts: { diff --git a/hrm-domain/packages/hrm-search-config/payload.ts b/hrm-domain/packages/hrm-search-config/payload.ts index 379976a2e..dbca12692 100644 --- a/hrm-domain/packages/hrm-search-config/payload.ts +++ b/hrm-domain/packages/hrm-search-config/payload.ts @@ -19,13 +19,13 @@ import { AccountSID } from '@tech-matters/types'; type IndexOperation = 'index' | 'remove'; -type IndexContactMessage = { +export type IndexContactMessage = { type: 'contact'; operation: IndexOperation; contact: Pick & Partial; }; -type IndexCaseMessage = { +export type IndexCaseMessage = { type: 'case'; operation: IndexOperation; case: Pick & @@ -40,7 +40,7 @@ export type IndexMessage = { accountSid: AccountSID } & ( ); export type IndexPayloadContact = IndexContactMessage & { - transcript: NonNullable; + transcript: string | null; }; export type IndexPayloadCase = IndexCaseMessage; From 0982e55599a38a77635cf489aa35df9daac72c75 Mon Sep 17 00:00:00 2001 From: Gianfranco Paoloni Date: Thu, 16 May 2024 18:41:53 -0300 Subject: [PATCH 26/55] debug --- .../lambdas/search-index-consumer/messagesToPayloads.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/hrm-domain/lambdas/search-index-consumer/messagesToPayloads.ts b/hrm-domain/lambdas/search-index-consumer/messagesToPayloads.ts index 123063f44..063ebe4a5 100644 --- a/hrm-domain/lambdas/search-index-consumer/messagesToPayloads.ts +++ b/hrm-domain/lambdas/search-index-consumer/messagesToPayloads.ts @@ -59,11 +59,15 @@ const intermediateContactMessage = async ( if (m.message.contact.channel && isChatChannel(m.message.contact.channel)) { const transcriptEntry = m.message.contact.conversationMedia?.find(isS3StoredTranscript); + + console.log('>>>>>>>> transcriptEntry', transcriptEntry); + if (transcriptEntry) { const { location } = transcriptEntry.storeTypeSpecificData; const { bucket, key } = location || {}; if (bucket && key) { transcript = await getS3Object({ bucket, key }); + console.log('>>>>>>>> transcript', transcript); } } } From 9ef239b0313542505ad0437f152e7bc75d475599 Mon Sep 17 00:00:00 2001 From: Gianfranco Paoloni Date: Thu, 16 May 2024 19:04:22 -0300 Subject: [PATCH 27/55] chore: add packages/s3-client to tsconfig.build --- hrm-domain/lambdas/search-index-consumer/tsconfig.build.json | 1 + 1 file changed, 1 insertion(+) diff --git a/hrm-domain/lambdas/search-index-consumer/tsconfig.build.json b/hrm-domain/lambdas/search-index-consumer/tsconfig.build.json index a8f8b404b..22ca951e5 100644 --- a/hrm-domain/lambdas/search-index-consumer/tsconfig.build.json +++ b/hrm-domain/lambdas/search-index-consumer/tsconfig.build.json @@ -5,6 +5,7 @@ "references": [ { "path": "packages/types" }, { "path": "packages/job-errors" }, + { "path": "packages/s3-client" }, { "path": "packages/ssm-cache" }, { "path": "packages/elasticsearch-client" }, { "path": "hrm-domain/packages/hrm-types" }, From c0a62d16489627aba794fad1489e540bc315f6aa Mon Sep 17 00:00:00 2001 From: Gianfranco Paoloni Date: Fri, 17 May 2024 13:32:39 -0300 Subject: [PATCH 28/55] chore: remove debug logs --- hrm-domain/lambdas/search-index-consumer/messagesToPayloads.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/hrm-domain/lambdas/search-index-consumer/messagesToPayloads.ts b/hrm-domain/lambdas/search-index-consumer/messagesToPayloads.ts index 063ebe4a5..521567975 100644 --- a/hrm-domain/lambdas/search-index-consumer/messagesToPayloads.ts +++ b/hrm-domain/lambdas/search-index-consumer/messagesToPayloads.ts @@ -60,14 +60,11 @@ const intermediateContactMessage = async ( const transcriptEntry = m.message.contact.conversationMedia?.find(isS3StoredTranscript); - console.log('>>>>>>>> transcriptEntry', transcriptEntry); - if (transcriptEntry) { const { location } = transcriptEntry.storeTypeSpecificData; const { bucket, key } = location || {}; if (bucket && key) { transcript = await getS3Object({ bucket, key }); - console.log('>>>>>>>> transcript', transcript); } } } From 838780423141202fa94af44b1a91934bd34e1fe2 Mon Sep 17 00:00:00 2001 From: Gianfranco Paoloni Date: Tue, 21 May 2024 12:35:18 -0300 Subject: [PATCH 29/55] chore: add support for parameterized ES config --- packages/elasticsearch-client/src/client.ts | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/packages/elasticsearch-client/src/client.ts b/packages/elasticsearch-client/src/client.ts index 0286d781d..265d76502 100644 --- a/packages/elasticsearch-client/src/client.ts +++ b/packages/elasticsearch-client/src/client.ts @@ -50,6 +50,7 @@ type AccountSidOrShortCodeRequired = export type GetClientArgs = { config?: ClientOptions; indexType: string; + ssmConfigParameter?: string; } & AccountSidOrShortCodeRequired; export type GetClientOrMockArgs = GetClientArgs & { @@ -69,9 +70,11 @@ const getConfigSsmParameterKey = (indexType: string) => const getEsConfig = async ({ config, indexType, + ssmConfigParameter, }: { config: ClientOptions | undefined; indexType: string; + ssmConfigParameter?: string; }) => { console.log('config', config); if (config) return config; @@ -91,6 +94,10 @@ const getEsConfig = async ({ }; } + if (ssmConfigParameter) { + return JSON.parse(await getSsmParameter(ssmConfigParameter)); + } + return JSON.parse(await getSsmParameter(getConfigSsmParameterKey(indexType))); }; @@ -107,14 +114,21 @@ export type IndexClient = { deleteIndex: () => Promise; }; -const getClientOrMock = async ({ config, index, indexType }: GetClientOrMockArgs) => { +const getClientOrMock = async ({ + config, + index, + indexType, + ssmConfigParameter, +}: GetClientOrMockArgs) => { // TODO: mock client for unit tests // if (authToken === 'mockAuthToken') { // const mock = (getMockClient({ config }) as unknown) as Twilio; // return mock; // } - const client = new EsClient(await getEsConfig({ config, indexType })); + const client = new EsClient( + await getEsConfig({ config, indexType, ssmConfigParameter }), + ); return { client, index, From d6e187b2875dba43132df41db1bbe7b73e3aa58a Mon Sep 17 00:00:00 2001 From: Gianfranco Paoloni Date: Tue, 21 May 2024 12:35:53 -0300 Subject: [PATCH 30/55] chore: provide ssm parameter name from env vars to ES config --- hrm-domain/lambdas/search-index-consumer/index.ts | 1 - .../search-index-consumer/payloadToIndex.ts | 14 +++++++++++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/hrm-domain/lambdas/search-index-consumer/index.ts b/hrm-domain/lambdas/search-index-consumer/index.ts index 443fc8c07..4beedcad6 100644 --- a/hrm-domain/lambdas/search-index-consumer/index.ts +++ b/hrm-domain/lambdas/search-index-consumer/index.ts @@ -14,7 +14,6 @@ * along with this program. If not, see https://www.gnu.org/licenses/. */ import type { SQSBatchResponse, SQSEvent } from 'aws-lambda'; -// import { GetSignedUrlMethods, GET_SIGNED_URL_METHODS } from '@tech-matters/s3-client'; import { HrmIndexProcessorError } from '@tech-matters/job-errors'; import { isErr } from '@tech-matters/types'; import { groupMessagesByAccountSid } from './messages'; diff --git a/hrm-domain/lambdas/search-index-consumer/payloadToIndex.ts b/hrm-domain/lambdas/search-index-consumer/payloadToIndex.ts index 0e6b96d29..b78928418 100644 --- a/hrm-domain/lambdas/search-index-consumer/payloadToIndex.ts +++ b/hrm-domain/lambdas/search-index-consumer/payloadToIndex.ts @@ -110,10 +110,18 @@ const handleIndexPayload = const indexDocumentsByIndexMapper = (accountSid: string) => async ([indexType, payloads]: [string, PayloadWithMeta[]]) => { + if (!process.env.ELASTICSEARCH_CONFIG_PARAMETER) { + throw new Error('ELASTICSEARCH_CONFIG_PARAMETER missing in environment variables'); + } + // get the client for the accountSid-indexType pair - const client = (await getClient({ accountSid, indexType })).indexClient( - hrmIndexConfiguration, - ); + const client = ( + await getClient({ + accountSid, + indexType, + ssmConfigParameter: process.env.ELASTICSEARCH_CONFIG_PARAMETER, + }) + ).indexClient(hrmIndexConfiguration); const mapper = handleIndexPayload({ client, accountSid, indexType }); From 7c51980923cfa77627d6be129c41f60de30f531c Mon Sep 17 00:00:00 2001 From: Gianfranco Paoloni Date: Tue, 21 May 2024 13:17:14 -0300 Subject: [PATCH 31/55] chore: rename env var for ES config --- hrm-domain/lambdas/search-index-consumer/payloadToIndex.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hrm-domain/lambdas/search-index-consumer/payloadToIndex.ts b/hrm-domain/lambdas/search-index-consumer/payloadToIndex.ts index b78928418..d3faca0ee 100644 --- a/hrm-domain/lambdas/search-index-consumer/payloadToIndex.ts +++ b/hrm-domain/lambdas/search-index-consumer/payloadToIndex.ts @@ -110,8 +110,8 @@ const handleIndexPayload = const indexDocumentsByIndexMapper = (accountSid: string) => async ([indexType, payloads]: [string, PayloadWithMeta[]]) => { - if (!process.env.ELASTICSEARCH_CONFIG_PARAMETER) { - throw new Error('ELASTICSEARCH_CONFIG_PARAMETER missing in environment variables'); + if (!process.env.SSM_PARAM_ELASTICSEARCH_CONFIG) { + throw new Error('SSM_PARAM_ELASTICSEARCH_CONFIG missing in environment variables'); } // get the client for the accountSid-indexType pair @@ -119,7 +119,7 @@ const indexDocumentsByIndexMapper = await getClient({ accountSid, indexType, - ssmConfigParameter: process.env.ELASTICSEARCH_CONFIG_PARAMETER, + ssmConfigParameter: process.env.SSM_PARAM_ELASTICSEARCH_CONFIG, }) ).indexClient(hrmIndexConfiguration); From 217892888d22b9e57f9f4660d03817867521dc56 Mon Sep 17 00:00:00 2001 From: Gianfranco Paoloni Date: Wed, 22 May 2024 14:57:32 -0300 Subject: [PATCH 32/55] chore: added hrm-search-config package to hrm-core --- hrm-domain/hrm-core/package.json | 1 + package-lock.json | 2 ++ 2 files changed, 3 insertions(+) diff --git a/hrm-domain/hrm-core/package.json b/hrm-domain/hrm-core/package.json index efe0e6757..055b26c0a 100644 --- a/hrm-domain/hrm-core/package.json +++ b/hrm-domain/hrm-core/package.json @@ -23,6 +23,7 @@ }, "homepage": "https://github.com/tech-matters/hrm#readme", "dependencies": { + "@tech-matters/hrm-search-config": "^1.0.0", "@tech-matters/hrm-types": "^1.0.0", "@tech-matters/http": "^1.0.0", "@tech-matters/resources-service": "^1.0.0", diff --git a/package-lock.json b/package-lock.json index 4d68813c4..ef036281f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -66,6 +66,7 @@ "version": "1.0.0", "license": "AGPL", "dependencies": { + "@tech-matters/hrm-search-config": "^1.0.0", "@tech-matters/hrm-types": "^1.0.0", "@tech-matters/http": "^1.0.0", "@tech-matters/resources-service": "^1.0.0", @@ -17474,6 +17475,7 @@ "@tech-matters/hrm-core": { "version": "file:hrm-domain/hrm-core", "requires": { + "@tech-matters/hrm-search-config": "^1.0.0", "@tech-matters/hrm-types": "^1.0.0", "@tech-matters/http": "^1.0.0", "@tech-matters/resources-service": "^1.0.0", From 0b8307ecb6511155333b89599968610bb9e7e52a Mon Sep 17 00:00:00 2001 From: Gianfranco Paoloni Date: Wed, 22 May 2024 14:59:18 -0300 Subject: [PATCH 33/55] chore: added wrapper to publish index-search messages to corresponding queue --- .../jobs/search/publishToSearchIndex.ts | 61 +++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 hrm-domain/hrm-core/jobs/search/publishToSearchIndex.ts diff --git a/hrm-domain/hrm-core/jobs/search/publishToSearchIndex.ts b/hrm-domain/hrm-core/jobs/search/publishToSearchIndex.ts new file mode 100644 index 000000000..c37d3e50f --- /dev/null +++ b/hrm-domain/hrm-core/jobs/search/publishToSearchIndex.ts @@ -0,0 +1,61 @@ +/** + * Copyright (C) 2021-2023 Technology Matters + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see https://www.gnu.org/licenses/. + */ + +import { sendSqsMessage } from '@tech-matters/sqs-client'; +import { getSsmParameter } from '../../config/ssmCache'; +import { IndexMessage } from '@tech-matters/hrm-search-config'; +import { CaseService, Contact } from '@tech-matters/hrm-types'; +import { AccountSID } from '@tech-matters/types'; + +const PENDING_INDEX_QUEUE_SSM_PATH = `/${process.env.NODE_ENV}/${ + process.env.AWS_REGION ?? process.env.AWS_DEFAULT_REGION +}/sqs/jobs/hrm-search-index/queue-url-consumer`; + +const publishToSearchIndex = async (message: IndexMessage) => { + try { + const queueUrl = await getSsmParameter(PENDING_INDEX_QUEUE_SSM_PATH); + + return await sendSqsMessage({ + queueUrl, + message: JSON.stringify(message), + }); + } catch (err) { + console.error( + `Error trying to send message to SQS queue ${PENDING_INDEX_QUEUE_SSM_PATH}`, + err, + ); + } +}; + +export const publishContactToSearchIndex = async ({ + accountSid, + contact, + operation, +}: { + accountSid: AccountSID; + contact: Contact; + operation: IndexMessage['operation']; +}) => publishToSearchIndex({ accountSid, type: 'contact', contact, operation }); + +export const publishCaseToSearchIndex = async ({ + accountSid, + case: caseObj, + operation, +}: { + accountSid: AccountSID; + case: CaseService; + operation: IndexMessage['operation']; +}) => publishToSearchIndex({ accountSid, type: 'case', case: caseObj, operation }); From 21e0087bfd9d182662477329d1b4f3b67155b105 Mon Sep 17 00:00:00 2001 From: mythilytm Date: Thu, 30 May 2024 14:16:00 -0400 Subject: [PATCH 34/55] feat: added reindexContacts service and admin endpoint --- .../hrm-core/contact/adminContactRoutesV0.ts | 45 +++++++++++ .../contact/contactsReindexService.ts | 80 +++++++++++++++++++ hrm-domain/hrm-core/routes.ts | 6 +- 3 files changed, 130 insertions(+), 1 deletion(-) create mode 100644 hrm-domain/hrm-core/contact/adminContactRoutesV0.ts create mode 100644 hrm-domain/hrm-core/contact/contactsReindexService.ts diff --git a/hrm-domain/hrm-core/contact/adminContactRoutesV0.ts b/hrm-domain/hrm-core/contact/adminContactRoutesV0.ts new file mode 100644 index 000000000..ec350d4a4 --- /dev/null +++ b/hrm-domain/hrm-core/contact/adminContactRoutesV0.ts @@ -0,0 +1,45 @@ +/** + * Copyright (C) 2021-2023 Technology Matters + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see https://www.gnu.org/licenses/. + */ + +// eslint-disable-next-line @typescript-eslint/no-unused-vars +import type { Request, Response, NextFunction } from 'express'; +import { isErr, mapHTTPError } from '@tech-matters/types'; +// import createError from 'http-errors'; +import { SafeRouter, publicEndpoint } from '../permissions'; +import { reindexContacts } from './contactsReindexService'; + +const adminContactsRouter = SafeRouter(); + +// admin POST endpoint to reindex contacts. req body has accountSid, dateFrom, dateTo +adminContactsRouter.post( + '/reindexContacts', + publicEndpoint, + async (req: Request, res: Response, next: NextFunction) => { + const { accountSid, dateFrom, dateTo } = req.body; + + const result = await reindexContacts(accountSid, dateFrom, dateTo); + + if (isErr(result)) { + return next( + mapHTTPError(result, { + InvalidParameterError: 400, + }), + ); + } + + res.json(result.data); + }, +); diff --git a/hrm-domain/hrm-core/contact/contactsReindexService.ts b/hrm-domain/hrm-core/contact/contactsReindexService.ts new file mode 100644 index 000000000..11113c74a --- /dev/null +++ b/hrm-domain/hrm-core/contact/contactsReindexService.ts @@ -0,0 +1,80 @@ +/** + * Copyright (C) 2021-2023 Technology Matters + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see https://www.gnu.org/licenses/. + */ + +import { HrmAccountId, newErr, newOkFromData } from '@tech-matters/types'; +import { TwilioUser } from '@tech-matters/twilio-worker-auth'; +import { rulesMap } from '@tech-matters/hrm-core/permissions/rulesMap'; +// import * as contactApi from '@tech-matters/hrm-core/contact/contactService'; +import { searchContacts } from './contactService'; +import { publishContactToSearchIndex } from '../jobs/search/publishToSearchIndex'; +import { autoPaginate } from '../../../hrm-domain/scheduled-tasks/hrm-data-pull/auto-paginate'; + +// TODO: refactor maxPermissions out of data-pull +// TODO: refactor autoPaginate out of data-pull + +const maxPermissions: { + user: TwilioUser; + can: () => boolean; + permissions: (typeof rulesMap)[keyof typeof rulesMap]; +} = { + can: () => true, + user: { + accountSid: 'ACxxx', + workerSid: 'WKxxx', + roles: ['supervisor'], + isSupervisor: true, + }, + permissions: rulesMap.open, +}; + +// fetch all the contacts updated between the date ranges(updatedAt) +// reindex the contacts in the search index with publishContactToSearchIndex +export const reindexContacts = async ( + accountSid: HrmAccountId, + dateFrom: string, + dateTo: string, +) => { + try { + const searchParameters = { + dateFrom, + dateTo, + }; + const contactsResponse = await autoPaginate(async limitAndOffset => { + const res = await searchContacts( + accountSid, + searchParameters, + limitAndOffset, + maxPermissions, + ); + return { records: res.contacts, count: res.count }; + }); + + const promises = contactsResponse.map(contact => { + return publishContactToSearchIndex({ + accountSid, + contact, + operation: 'index', + }); + }); + + await Promise.all(promises); + + return newOkFromData(promises); + } catch (error) { + console.error('Error reindexing contacts', error); + return newErr({ error, message: 'Error reindexing contacts' }); + } +}; diff --git a/hrm-domain/hrm-core/routes.ts b/hrm-domain/hrm-core/routes.ts index 60a4adf71..0334a03ab 100644 --- a/hrm-domain/hrm-core/routes.ts +++ b/hrm-domain/hrm-core/routes.ts @@ -24,6 +24,7 @@ import referrals from './referral/referral-routes-v0'; import permissions from './permissions/permissions-routes-v0'; import profiles from './profile/profileRoutesV0'; import adminProfiles from './profile/adminProfileRoutesV0'; +// import adminContacts from './contact/adminContactRoutesV0'; import { Permissions } from './permissions'; export const HRM_ROUTES: { @@ -49,7 +50,10 @@ export const apiV0 = (rules: Permissions) => { export const ADMIN_ROUTES: { path: string; routerFactory: () => Router; -}[] = [{ path: '/profiles', routerFactory: () => adminProfiles }]; +}[] = [ + { path: '/profiles', routerFactory: () => adminProfiles }, + // { path: '/contacts', routerFactory: () => adminContacts }, +]; export const adminApiV0 = () => { const router: IRouter = Router(); From 055fb3c5aea49322127da22a4d004ce18a7046a7 Mon Sep 17 00:00:00 2001 From: Gianfranco Paoloni Date: Thu, 30 May 2024 16:24:35 -0300 Subject: [PATCH 35/55] chore: factored out autoPagination & maxPermissins for reuse --- .../autoPaginate.ts} | 0 .../contact/contactsReindexService.ts | 26 ++-------------- hrm-domain/hrm-core/permissions/index.ts | 31 ++++++++++++++----- .../scheduled-tasks/hrm-data-pull/context.ts | 18 +---------- .../hrm-data-pull/pull-cases.ts | 2 +- .../hrm-data-pull/pull-contacts.ts | 2 +- .../hrm-data-pull/pull-profiles.ts | 2 +- 7 files changed, 29 insertions(+), 52 deletions(-) rename hrm-domain/{scheduled-tasks/hrm-data-pull/auto-paginate.ts => hrm-core/autoPaginate.ts} (100%) diff --git a/hrm-domain/scheduled-tasks/hrm-data-pull/auto-paginate.ts b/hrm-domain/hrm-core/autoPaginate.ts similarity index 100% rename from hrm-domain/scheduled-tasks/hrm-data-pull/auto-paginate.ts rename to hrm-domain/hrm-core/autoPaginate.ts diff --git a/hrm-domain/hrm-core/contact/contactsReindexService.ts b/hrm-domain/hrm-core/contact/contactsReindexService.ts index 11113c74a..b39c4317c 100644 --- a/hrm-domain/hrm-core/contact/contactsReindexService.ts +++ b/hrm-domain/hrm-core/contact/contactsReindexService.ts @@ -15,33 +15,11 @@ */ import { HrmAccountId, newErr, newOkFromData } from '@tech-matters/types'; -import { TwilioUser } from '@tech-matters/twilio-worker-auth'; -import { rulesMap } from '@tech-matters/hrm-core/permissions/rulesMap'; -// import * as contactApi from '@tech-matters/hrm-core/contact/contactService'; import { searchContacts } from './contactService'; import { publishContactToSearchIndex } from '../jobs/search/publishToSearchIndex'; -import { autoPaginate } from '../../../hrm-domain/scheduled-tasks/hrm-data-pull/auto-paginate'; +import { maxPermissions } from '../permissions'; +import { autoPaginate } from '../autoPaginate'; -// TODO: refactor maxPermissions out of data-pull -// TODO: refactor autoPaginate out of data-pull - -const maxPermissions: { - user: TwilioUser; - can: () => boolean; - permissions: (typeof rulesMap)[keyof typeof rulesMap]; -} = { - can: () => true, - user: { - accountSid: 'ACxxx', - workerSid: 'WKxxx', - roles: ['supervisor'], - isSupervisor: true, - }, - permissions: rulesMap.open, -}; - -// fetch all the contacts updated between the date ranges(updatedAt) -// reindex the contacts in the search index with publishContactToSearchIndex export const reindexContacts = async ( accountSid: HrmAccountId, dateFrom: string, diff --git a/hrm-domain/hrm-core/permissions/index.ts b/hrm-domain/hrm-core/permissions/index.ts index b529cc93d..fc5fadc44 100644 --- a/hrm-domain/hrm-core/permissions/index.ts +++ b/hrm-domain/hrm-core/permissions/index.ts @@ -15,16 +15,16 @@ */ import { SafeRouterRequest } from './safe-router'; +import { rulesMap } from './rulesMap'; +import { type InitializedCan, initializeCanForRules } from './initializeCanForRules'; +import { type RulesFile } from './rulesMap'; +import type { Request, Response, NextFunction } from 'express'; +import type { TwilioUser } from '@tech-matters/twilio-worker-auth'; +import type { AccountSID } from '@tech-matters/types'; export { SafeRouter, publicEndpoint } from './safe-router'; -export { rulesMap } from './rulesMap'; -export { Actions, actionsMaps } from './actions'; - -import { InitializedCan, initializeCanForRules } from './initializeCanForRules'; -import { RulesFile } from './rulesMap'; -import type { Request, Response, NextFunction } from 'express'; -import { TwilioUser } from '@tech-matters/twilio-worker-auth'; -import { AccountSID } from '@tech-matters/types'; +export { type Actions, actionsMaps } from './actions'; +export { rulesMap }; declare global { namespace Express { @@ -72,3 +72,18 @@ export const setupPermissions = export type RequestWithPermissions = SafeRouterRequest & { can: InitializedCan; }; + +export const maxPermissions: { + user: TwilioUser; + can: () => boolean; + permissions: (typeof rulesMap)[keyof typeof rulesMap]; +} = { + can: () => true, + user: { + accountSid: 'ACxxx', + workerSid: 'WKxxx', + roles: ['supervisor'], + isSupervisor: true, + }, + permissions: rulesMap.open, +}; diff --git a/hrm-domain/scheduled-tasks/hrm-data-pull/context.ts b/hrm-domain/scheduled-tasks/hrm-data-pull/context.ts index 81409168a..05f988f6b 100644 --- a/hrm-domain/scheduled-tasks/hrm-data-pull/context.ts +++ b/hrm-domain/scheduled-tasks/hrm-data-pull/context.ts @@ -15,9 +15,8 @@ */ import { getSsmParameter } from '@tech-matters/ssm-cache'; -import { rulesMap } from '@tech-matters/hrm-core/permissions/rulesMap'; import { HrmAccountId } from '@tech-matters/types'; -import { TwilioUser } from '@tech-matters/twilio-worker-auth'; +export { maxPermissions } from '@tech-matters/hrm-core/permissions/index'; // const sanitizeEnv = (env: string) => (env === 'local' ? 'development' : env); @@ -63,18 +62,3 @@ export const getContext = async (): Promise => { return context; }; - -export const maxPermissions: { - user: TwilioUser; - can: () => boolean; - permissions: (typeof rulesMap)[keyof typeof rulesMap]; -} = { - can: () => true, - user: { - accountSid: 'ACxxx', - workerSid: 'WKxxx', - roles: ['supervisor'], - isSupervisor: true, - }, - permissions: rulesMap.open, -}; diff --git a/hrm-domain/scheduled-tasks/hrm-data-pull/pull-cases.ts b/hrm-domain/scheduled-tasks/hrm-data-pull/pull-cases.ts index 48521a517..f2f5c6cdb 100644 --- a/hrm-domain/scheduled-tasks/hrm-data-pull/pull-cases.ts +++ b/hrm-domain/scheduled-tasks/hrm-data-pull/pull-cases.ts @@ -18,9 +18,9 @@ import format from 'date-fns/format'; import formatISO from 'date-fns/formatISO'; import { putS3Object } from '@tech-matters/s3-client'; import * as caseApi from '@tech-matters/hrm-core/case/caseService'; +import { autoPaginate } from '@tech-matters/hrm-core/autoPaginate'; import { getContext, maxPermissions } from './context'; -import { autoPaginate } from './auto-paginate'; import { parseISO } from 'date-fns'; const getSearchParams = (startDate: Date, endDate: Date) => ({ diff --git a/hrm-domain/scheduled-tasks/hrm-data-pull/pull-contacts.ts b/hrm-domain/scheduled-tasks/hrm-data-pull/pull-contacts.ts index 2843825e7..dbb716bb5 100644 --- a/hrm-domain/scheduled-tasks/hrm-data-pull/pull-contacts.ts +++ b/hrm-domain/scheduled-tasks/hrm-data-pull/pull-contacts.ts @@ -20,7 +20,7 @@ import { putS3Object } from '@tech-matters/s3-client'; import { getContext, maxPermissions } from './context'; import * as contactApi from '@tech-matters/hrm-core/contact/contactService'; -import { autoPaginate } from './auto-paginate'; +import { autoPaginate } from '@tech-matters/hrm-core/autoPaginate'; import { parseISO } from 'date-fns'; const getSearchParams = (startDate: Date, endDate: Date) => ({ diff --git a/hrm-domain/scheduled-tasks/hrm-data-pull/pull-profiles.ts b/hrm-domain/scheduled-tasks/hrm-data-pull/pull-profiles.ts index 900e04a33..8002270c4 100644 --- a/hrm-domain/scheduled-tasks/hrm-data-pull/pull-profiles.ts +++ b/hrm-domain/scheduled-tasks/hrm-data-pull/pull-profiles.ts @@ -23,9 +23,9 @@ import type { ProfileSection, ProfileWithRelationships, } from '@tech-matters/hrm-core/profile/profileDataAccess'; +import { autoPaginate } from '@tech-matters/hrm-core/autoPaginate'; import { getContext } from './context'; -import { autoPaginate } from './auto-paginate'; import { parseISO } from 'date-fns'; const getSearchParams = (startDate: Date, endDate: Date) => ({ From 871218622da55d71301807387cd6dcfac84b7c23 Mon Sep 17 00:00:00 2001 From: Gianfranco Paoloni Date: Thu, 30 May 2024 17:00:08 -0300 Subject: [PATCH 36/55] chore: introduce processInBatch, reusing logic from autoPaginate --- hrm-domain/hrm-core/autoPaginate.ts | 50 +++++++++++++------ .../contact/contactsReindexService.ts | 29 ++++++----- 2 files changed, 53 insertions(+), 26 deletions(-) diff --git a/hrm-domain/hrm-core/autoPaginate.ts b/hrm-domain/hrm-core/autoPaginate.ts index ac7347f26..b8bc870d5 100644 --- a/hrm-domain/hrm-core/autoPaginate.ts +++ b/hrm-domain/hrm-core/autoPaginate.ts @@ -29,22 +29,22 @@ type LimitAndOffset = { offset: number; }; -/** - * This function takes care of keep calling the search function - * until there's no more data to be fetched. It works by dynamically - * adjusting the 'offset' on each subsequent call. - * - * @param searchFunction function to perform search of cases or contacts with the provided limit & offset - * @returns cases[] or contacts[] - */ -export const autoPaginate = async ( - searchFunction: (limitAndOffset: LimitAndOffset) => Promise>, -): Promise => { - let items: T[] = []; +export type SearchFunction = ( + limitAndOffset: LimitAndOffset, +) => Promise>; + +export type AsyncProcessor = (result: SearchResult) => Promise + +export const processInBatch = async ( + searchFunction: SearchFunction, + asyncProcessor: AsyncProcessor, +): Promise => { let hasMoreItems = true; let offset = Number(defaultLimitAndOffset.offset); const limit = Number(defaultLimitAndOffset.limit); + let processed = 0; + while (hasMoreItems) { /** * Updates 'limitAndOffset' param @@ -53,14 +53,36 @@ export const autoPaginate = async ( const searchResult = await searchFunction({ limit, offset }); const { count, records } = searchResult; - items = [...items, ...records]; - hasMoreItems = items.length < count; + await asyncProcessor(searchResult); + + processed += records.length; + hasMoreItems = processed < count; if (hasMoreItems) { offset += limit; } } +}; + +/** + * This function takes care of keep calling the search function + * until there's no more data to be fetched. It works by dynamically + * adjusting the 'offset' on each subsequent call. + * + * @param searchFunction function to perform search of cases or contacts with the provided limit & offset + * @returns cases[] or contacts[] + */ +export const autoPaginate = async ( + searchFunction: SearchFunction, +): Promise => { + let items: T[] = []; + + const asyncProcessor = async (result: SearchResult) => { + items.push(...result.records); + }; + + await processInBatch(searchFunction, asyncProcessor); return items; }; diff --git a/hrm-domain/hrm-core/contact/contactsReindexService.ts b/hrm-domain/hrm-core/contact/contactsReindexService.ts index b39c4317c..f7f397076 100644 --- a/hrm-domain/hrm-core/contact/contactsReindexService.ts +++ b/hrm-domain/hrm-core/contact/contactsReindexService.ts @@ -15,10 +15,10 @@ */ import { HrmAccountId, newErr, newOkFromData } from '@tech-matters/types'; -import { searchContacts } from './contactService'; +import { Contact, searchContacts } from './contactService'; import { publishContactToSearchIndex } from '../jobs/search/publishToSearchIndex'; import { maxPermissions } from '../permissions'; -import { autoPaginate } from '../autoPaginate'; +import { AsyncProcessor, SearchFunction, processInBatch } from '../autoPaginate'; export const reindexContacts = async ( accountSid: HrmAccountId, @@ -30,7 +30,8 @@ export const reindexContacts = async ( dateFrom, dateTo, }; - const contactsResponse = await autoPaginate(async limitAndOffset => { + + const searchFunction: SearchFunction = async limitAndOffset => { const res = await searchContacts( accountSid, searchParameters, @@ -38,19 +39,23 @@ export const reindexContacts = async ( maxPermissions, ); return { records: res.contacts, count: res.count }; - }); + }; - const promises = contactsResponse.map(contact => { - return publishContactToSearchIndex({ - accountSid, - contact, - operation: 'index', + const asyncProcessor: AsyncProcessor = async contactsResult => { + const promises = contactsResult.records.map(contact => { + return publishContactToSearchIndex({ + accountSid, + contact, + operation: 'index', + }); }); - }); - await Promise.all(promises); + await Promise.all(promises); + }; + + await processInBatch(searchFunction, asyncProcessor); - return newOkFromData(promises); + return newOkFromData('Successfully indexed contacts'); } catch (error) { console.error('Error reindexing contacts', error); return newErr({ error, message: 'Error reindexing contacts' }); From 63bb44caa9baf6d99201835ef691e94a3c42b2cb Mon Sep 17 00:00:00 2001 From: Gianfranco Paoloni Date: Thu, 30 May 2024 17:04:55 -0300 Subject: [PATCH 37/55] chore: fix lint --- hrm-domain/hrm-core/autoPaginate.ts | 2 +- .../scheduled-tasks/hrm-data-pull/tests/unit/pull-cases.test.ts | 2 +- .../hrm-data-pull/tests/unit/pull-contacts.test.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/hrm-domain/hrm-core/autoPaginate.ts b/hrm-domain/hrm-core/autoPaginate.ts index b8bc870d5..456619b1d 100644 --- a/hrm-domain/hrm-core/autoPaginate.ts +++ b/hrm-domain/hrm-core/autoPaginate.ts @@ -33,7 +33,7 @@ export type SearchFunction = ( limitAndOffset: LimitAndOffset, ) => Promise>; -export type AsyncProcessor = (result: SearchResult) => Promise +export type AsyncProcessor = (result: SearchResult) => Promise; export const processInBatch = async ( searchFunction: SearchFunction, diff --git a/hrm-domain/scheduled-tasks/hrm-data-pull/tests/unit/pull-cases.test.ts b/hrm-domain/scheduled-tasks/hrm-data-pull/tests/unit/pull-cases.test.ts index 876b3ec10..12a6c9e9c 100644 --- a/hrm-domain/scheduled-tasks/hrm-data-pull/tests/unit/pull-cases.test.ts +++ b/hrm-domain/scheduled-tasks/hrm-data-pull/tests/unit/pull-cases.test.ts @@ -21,9 +21,9 @@ import addDays from 'date-fns/addDays'; import * as caseApi from '@tech-matters/hrm-core/case/caseService'; import * as context from '../../context'; -import { defaultLimitAndOffset } from '../../auto-paginate'; import { pullCases } from '../../pull-cases'; import { HrmAccountId } from '@tech-matters/types'; +import { defaultLimitAndOffset } from '@tech-matters/hrm-core/autoPaginate'; const { maxPermissions } = context; diff --git a/hrm-domain/scheduled-tasks/hrm-data-pull/tests/unit/pull-contacts.test.ts b/hrm-domain/scheduled-tasks/hrm-data-pull/tests/unit/pull-contacts.test.ts index 64c2b4170..efff35b95 100644 --- a/hrm-domain/scheduled-tasks/hrm-data-pull/tests/unit/pull-contacts.test.ts +++ b/hrm-domain/scheduled-tasks/hrm-data-pull/tests/unit/pull-contacts.test.ts @@ -21,9 +21,9 @@ import addDays from 'date-fns/addDays'; import * as contactApi from '@tech-matters/hrm-core/contact/contactService'; import * as context from '../../context'; -import { defaultLimitAndOffset } from '../../auto-paginate'; import { pullContacts } from '../../pull-contacts'; import { HrmAccountId } from '@tech-matters/types'; +import { defaultLimitAndOffset } from '@tech-matters/hrm-core/autoPaginate'; const { maxPermissions } = context; From 004511982e916eec6afe6d654cf4c1bc4c1e051a Mon Sep 17 00:00:00 2001 From: mythilytm Date: Fri, 31 May 2024 15:31:38 -0400 Subject: [PATCH 38/55] feat: wire admin endpoint for reindexing --- hrm-domain/hrm-core/contact/adminContactRoutesV0.ts | 2 ++ hrm-domain/hrm-core/contact/contactDataAccess.ts | 4 ---- hrm-domain/hrm-core/routes.ts | 4 ++-- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/hrm-domain/hrm-core/contact/adminContactRoutesV0.ts b/hrm-domain/hrm-core/contact/adminContactRoutesV0.ts index ec350d4a4..bbe020f6a 100644 --- a/hrm-domain/hrm-core/contact/adminContactRoutesV0.ts +++ b/hrm-domain/hrm-core/contact/adminContactRoutesV0.ts @@ -43,3 +43,5 @@ adminContactsRouter.post( res.json(result.data); }, ); + +export default adminContactsRouter.expressRouter; diff --git a/hrm-domain/hrm-core/contact/contactDataAccess.ts b/hrm-domain/hrm-core/contact/contactDataAccess.ts index 5956bfa00..67fc1492e 100644 --- a/hrm-domain/hrm-core/contact/contactDataAccess.ts +++ b/hrm-domain/hrm-core/contact/contactDataAccess.ts @@ -309,10 +309,6 @@ const generalizedSearchQueryFunction = ( sqlQueryGenerator(viewPermissions, user.isSupervisor), sqlQueryParamsBuilder(accountSid, user, searchParameters, limit, offset), ); - console.log('>>> data access', '3. generalizedSearchQueryFunction', { - searchParameters, - viewPermissions, - }); return { rows: searchResults, count: searchResults.length ? searchResults[0].totalCount : 0, diff --git a/hrm-domain/hrm-core/routes.ts b/hrm-domain/hrm-core/routes.ts index 0334a03ab..486eb3309 100644 --- a/hrm-domain/hrm-core/routes.ts +++ b/hrm-domain/hrm-core/routes.ts @@ -24,7 +24,7 @@ import referrals from './referral/referral-routes-v0'; import permissions from './permissions/permissions-routes-v0'; import profiles from './profile/profileRoutesV0'; import adminProfiles from './profile/adminProfileRoutesV0'; -// import adminContacts from './contact/adminContactRoutesV0'; +import adminContacts from './contact/adminContactRoutesV0'; import { Permissions } from './permissions'; export const HRM_ROUTES: { @@ -52,7 +52,7 @@ export const ADMIN_ROUTES: { routerFactory: () => Router; }[] = [ { path: '/profiles', routerFactory: () => adminProfiles }, - // { path: '/contacts', routerFactory: () => adminContacts }, + { path: '/contacts', routerFactory: () => adminContacts }, ]; export const adminApiV0 = () => { From b89e8d3000450c87f2ada221e4a6a77bd5a4c2f0 Mon Sep 17 00:00:00 2001 From: mythilytm Date: Mon, 3 Jun 2024 15:14:36 -0400 Subject: [PATCH 39/55] fix admin route to use account id from req url --- hrm-domain/hrm-core/contact/adminContactRoutesV0.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/hrm-domain/hrm-core/contact/adminContactRoutesV0.ts b/hrm-domain/hrm-core/contact/adminContactRoutesV0.ts index bbe020f6a..3aaaa762d 100644 --- a/hrm-domain/hrm-core/contact/adminContactRoutesV0.ts +++ b/hrm-domain/hrm-core/contact/adminContactRoutesV0.ts @@ -28,9 +28,10 @@ adminContactsRouter.post( '/reindexContacts', publicEndpoint, async (req: Request, res: Response, next: NextFunction) => { - const { accountSid, dateFrom, dateTo } = req.body; + const { hrmAccountId } = req; + const { dateFrom, dateTo } = req.body; - const result = await reindexContacts(accountSid, dateFrom, dateTo); + const result = await reindexContacts(hrmAccountId, dateFrom, dateTo); if (isErr(result)) { return next( From f5f0f9523c73343146ad37e349b72582982982a6 Mon Sep 17 00:00:00 2001 From: mythilytm Date: Mon, 3 Jun 2024 17:22:15 -0400 Subject: [PATCH 40/55] feat: build script for reindexing contacts and cases --- hrm-domain/hrm-service/scripts/README.md | 2 + .../scripts/admin-commands/reindex.ts | 25 +++ .../scripts/admin-commands/reindex/hrm.ts | 142 ++++++++++++++++++ 3 files changed, 169 insertions(+) create mode 100644 hrm-domain/hrm-service/scripts/admin-commands/reindex.ts create mode 100644 hrm-domain/hrm-service/scripts/admin-commands/reindex/hrm.ts diff --git a/hrm-domain/hrm-service/scripts/README.md b/hrm-domain/hrm-service/scripts/README.md index 699ca6b5e..8f99e0c86 100644 --- a/hrm-domain/hrm-service/scripts/README.md +++ b/hrm-domain/hrm-service/scripts/README.md @@ -22,6 +22,8 @@ admin-cli | ├── create: # Create a new profile flag | ├── edit: # Edit an existing profile flag | ├── delete: # Delete an existing profile flag +├── reindex +| ├── hrm: # Reindex contacts and cases based on date range ``` ### Usage diff --git a/hrm-domain/hrm-service/scripts/admin-commands/reindex.ts b/hrm-domain/hrm-service/scripts/admin-commands/reindex.ts new file mode 100644 index 000000000..5b52548a2 --- /dev/null +++ b/hrm-domain/hrm-service/scripts/admin-commands/reindex.ts @@ -0,0 +1,25 @@ +/** + * Copyright (C) 2021-2023 Technology Matters + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see https://www.gnu.org/licenses/. + */ + +export const command = 'reindex '; +export const desc = 'admin endpoints for reindexing contacts and cases'; + +export const builder = function (yargs) { + return yargs.commandDir('reindex', { + exclude: /^(index|_)/, // Exclude files starting with 'index' or '_' + extensions: ['ts'], + }); +}; diff --git a/hrm-domain/hrm-service/scripts/admin-commands/reindex/hrm.ts b/hrm-domain/hrm-service/scripts/admin-commands/reindex/hrm.ts new file mode 100644 index 000000000..79dddb605 --- /dev/null +++ b/hrm-domain/hrm-service/scripts/admin-commands/reindex/hrm.ts @@ -0,0 +1,142 @@ +/** + * Copyright (C) 2021-2023 Technology Matters + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see https://www.gnu.org/licenses/. + */ + +import { getHRMInternalEndpointAccess } from '@tech-matters/service-discovery'; +// eslint-disable-next-line import/no-extraneous-dependencies +import { fetch } from 'undici'; +import { getAdminV0URL, staticKeyPattern } from '../../hrmInternalConfig'; + +export const command = 'hrm'; +export const describe = 'Reindex contacts and cases based on date range'; + +export const builder = { + contacts: { + alias: 'contacts', + describe: 'reindex contacts', + type: 'boolean', + default: false, + }, + cases: { + alias: 'cases', + describe: 'reindex cases', + type: 'boolean', + default: false, + }, + e: { + alias: 'environment', + describe: 'environment (e.g. development, staging, production)', + demandOption: true, + type: 'string', + }, + r: { + alias: 'region', + describe: 'region (e.g. us-east-1)', + demandOption: true, + type: 'string', + }, + a: { + alias: 'accountSid', + describe: 'account SID', + demandOption: true, + type: 'string', + }, + f: { + alias: 'dateFrom', + describe: 'start date (e.g. 2024-01-01)', + demandOption: true, + type: 'string', + }, + t: { + alias: 'dateTo', + describe: 'end date (e.g. 2024-12-31)', + demandOption: true, + type: 'string', + }, +}; + +export const handler = async ({ + region, + environment, + accountSid, + dateFrom, + dateTo, + contacts, + cases, +}) => { + try { + const timestamp = new Date().getTime(); + const assumeRoleParams = { + RoleArn: 'arn:aws:iam::712893914485:role/tf-admin', + RoleSessionName: `hrm-admin-cli-${timestamp}`, + }; + + const { authKey, internalResourcesUrl } = await getHRMInternalEndpointAccess({ + region, + environment, + staticKeyPattern, + assumeRoleParams, + }); + + if (!contacts && !cases) { + console.log( + 'Please specify contacts and/or cases option to reindex in your command', + ); + return; + } + + if (contacts) { + const url = getAdminV0URL( + internalResourcesUrl, + accountSid, + '/contacts/reindexContacts', + ); + const response = await fetch(url, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Authorization: `Basic ${authKey}`, + }, + body: JSON.stringify({ dateFrom, dateTo }), + }); + + if (!response.ok) { + throw new Error(`Failed to submit request: ${response.statusText}`); + } + + console.log(`Reindexing contacts from ${dateFrom} to ${dateTo}...`); + } + + if (cases) { + const url = getAdminV0URL(internalResourcesUrl, accountSid, '/cases/reindexCases'); + const response = await fetch(url, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Authorization: `Basic ${authKey}`, + }, + body: JSON.stringify({ dateFrom, dateTo }), + }); + + if (!response.ok) { + throw new Error(`Failed to submit request: ${response.statusText}`); + } + + console.log(`Reindexing cases from ${dateFrom} to ${dateTo}...`); + } + } catch (err) { + console.error(err); + } +}; From 971d2d93bb797240e5766695563df40f6bd17309 Mon Sep 17 00:00:00 2001 From: mythilytm Date: Tue, 4 Jun 2024 17:22:53 -0400 Subject: [PATCH 41/55] change options for indexing --- hrm-domain/hrm-service/scripts/admin-commands/reindex.ts | 2 +- hrm-domain/hrm-service/scripts/admin-commands/reindex/hrm.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/hrm-domain/hrm-service/scripts/admin-commands/reindex.ts b/hrm-domain/hrm-service/scripts/admin-commands/reindex.ts index 5b52548a2..c6846eb2a 100644 --- a/hrm-domain/hrm-service/scripts/admin-commands/reindex.ts +++ b/hrm-domain/hrm-service/scripts/admin-commands/reindex.ts @@ -19,7 +19,7 @@ export const desc = 'admin endpoints for reindexing contacts and cases'; export const builder = function (yargs) { return yargs.commandDir('reindex', { - exclude: /^(index|_)/, // Exclude files starting with 'index' or '_' + exclude: /^(index|_)/, extensions: ['ts'], }); }; diff --git a/hrm-domain/hrm-service/scripts/admin-commands/reindex/hrm.ts b/hrm-domain/hrm-service/scripts/admin-commands/reindex/hrm.ts index 79dddb605..2ae699d8b 100644 --- a/hrm-domain/hrm-service/scripts/admin-commands/reindex/hrm.ts +++ b/hrm-domain/hrm-service/scripts/admin-commands/reindex/hrm.ts @@ -23,13 +23,13 @@ export const command = 'hrm'; export const describe = 'Reindex contacts and cases based on date range'; export const builder = { - contacts: { + co: { alias: 'contacts', describe: 'reindex contacts', type: 'boolean', default: false, }, - cases: { + ca: { alias: 'cases', describe: 'reindex cases', type: 'boolean', From 7da308061525653d25a2aea82798b4f8225c3686 Mon Sep 17 00:00:00 2001 From: mythilytm Date: Tue, 4 Jun 2024 19:35:34 -0400 Subject: [PATCH 42/55] sqs debug --- hrm-domain/hrm-core/jobs/search/publishToSearchIndex.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hrm-domain/hrm-core/jobs/search/publishToSearchIndex.ts b/hrm-domain/hrm-core/jobs/search/publishToSearchIndex.ts index c37d3e50f..b02f64331 100644 --- a/hrm-domain/hrm-core/jobs/search/publishToSearchIndex.ts +++ b/hrm-domain/hrm-core/jobs/search/publishToSearchIndex.ts @@ -27,7 +27,7 @@ const PENDING_INDEX_QUEUE_SSM_PATH = `/${process.env.NODE_ENV}/${ const publishToSearchIndex = async (message: IndexMessage) => { try { const queueUrl = await getSsmParameter(PENDING_INDEX_QUEUE_SSM_PATH); - + console.log('>> publishToSearchIndex queueUrl, message', { queueUrl, message }); return await sendSqsMessage({ queueUrl, message: JSON.stringify(message), From 27d1239512e5c0622303ca4691b11b9dd69759ac Mon Sep 17 00:00:00 2001 From: Gianfranco Date: Tue, 4 Jun 2024 21:16:29 -0300 Subject: [PATCH 43/55] chore: publish pending index-search jobs on write/update ops for contacts [CHI-2724] (#645) --- .../contact-job/contact-job-complete.ts | 4 +- hrm-domain/hrm-core/contact/contactService.ts | 61 +++++++++++++++++++ .../hrm-core/contact/sql/contact-get-sql.ts | 3 + .../conversation-media-data-access.ts | 3 + .../conversation-media/conversation-media.ts | 2 +- .../jobs/search/publishToSearchIndex.ts | 25 +++++++- .../unit-tests/contact/contactService.test.ts | 17 ++++++ hrm-domain/hrm-service/Dockerfile | 1 + .../contact-job/contactJobCleanup.test.ts | 8 +-- .../jobTypes/retrieveTranscript.test.ts | 2 +- .../convertToIndexDocument.ts | 12 ++-- 11 files changed, 120 insertions(+), 18 deletions(-) diff --git a/hrm-domain/hrm-core/contact-job/contact-job-complete.ts b/hrm-domain/hrm-core/contact-job/contact-job-complete.ts index 85a32fc49..6b9490840 100644 --- a/hrm-domain/hrm-core/contact-job/contact-job-complete.ts +++ b/hrm-domain/hrm-core/contact-job/contact-job-complete.ts @@ -20,6 +20,7 @@ import { completeContactJob, getContactJobById, } from './contact-job-data-access'; +import { updateConversationMediaData } from '../contact/contactService'; import { ContactJobAttemptResult, ContactJobType } from '@tech-matters/types'; import { ContactJobCompleteProcessorError, @@ -41,7 +42,6 @@ import type { import { ConversationMedia, getConversationMediaById, - updateConversationMediaData, } from '../conversation-media/conversation-media'; export const processCompletedRetrieveContactTranscript = async ( @@ -59,7 +59,7 @@ export const processCompletedRetrieveContactTranscript = async ( location: completedJob.attemptPayload, }; - return updateConversationMediaData( + return updateConversationMediaData(completedJob.contactId)( completedJob.accountSid, completedJob.conversationMediaId, storeTypeSpecificData, diff --git a/hrm-domain/hrm-core/contact/contactService.ts b/hrm-domain/hrm-core/contact/contactService.ts index 655631639..ca578afb4 100644 --- a/hrm-domain/hrm-core/contact/contactService.ts +++ b/hrm-domain/hrm-core/contact/contactService.ts @@ -59,6 +59,7 @@ import { isS3StoredTranscript, isS3StoredTranscriptPending, NewConversationMedia, + updateConversationMediaSpecificData, } from '../conversation-media/conversation-media'; import { Profile, getOrCreateProfileWithIdentifier } from '../profile/profileService'; import { deleteContactReferrals } from '../referral/referral-data-access'; @@ -69,6 +70,8 @@ import { } from '../sql'; import { systemUser } from '@tech-matters/twilio-worker-auth'; import { RulesFile, TKConditionsSets } from '../permissions/rulesMap'; +import type { IndexMessage } from '@tech-matters/hrm-search-config'; +import { publishContactToSearchIndex } from '../jobs/search/publishToSearchIndex'; // Re export as is: export { Contact } from './contactDataAccess'; @@ -172,6 +175,23 @@ const initProfile = async ( }); }; +const doContactInSearchIndexOP = + (operation: IndexMessage['operation']) => + async ({ + accountSid, + contactId, + }: { + accountSid: Contact['accountSid']; + contactId: Contact['id']; + }) => { + const contact = await getById(accountSid, contactId); + + await publishContactToSearchIndex({ accountSid, contact, operation }); + }; + +export const indexContactInSearchIndex = doContactInSearchIndexOP('index'); +const removeContactInSearchIndex = doContactInSearchIndexOP('remove'); + // Creates a contact with all its related records within a single transaction export const createContact = async ( accountSid: HrmAccountId, @@ -220,6 +240,8 @@ export const createContact = async ( return newOk({ data: applyTransformations(contact) }); }); if (isOk(result)) { + // trigger index operation but don't await for it + indexContactInSearchIndex({ accountSid, contactId: result.data.id }); return result.data; } // This operation can fail with a unique constraint violation if a contact with the same ID is being created concurrently @@ -243,6 +265,7 @@ export const createContact = async ( return result.unwrap(); } } + return result.unwrap(); }; @@ -287,6 +310,9 @@ export const patchContact = async ( const applyTransformations = bindApplyTransformations(can, user); + // trigger index operation but don't await for it + indexContactInSearchIndex({ accountSid, contactId: updated.id }); + return applyTransformations(updated); }); @@ -296,6 +322,11 @@ export const connectContactToCase = async ( caseId: string, { can, user }: { can: InitializedCan; user: TwilioUser }, ): Promise => { + if (caseId === null) { + // trigger remove operation, awaiting for it, since we'll lost the information of which is the "old case" otherwise + await removeContactInSearchIndex({ accountSid, contactId: parseInt(contactId, 10) }); + } + const updated: Contact | undefined = await connectToCase()( accountSid, contactId, @@ -307,6 +338,10 @@ export const connectContactToCase = async ( } const applyTransformations = bindApplyTransformations(can, user); + + // trigger index operation but don't await for it + indexContactInSearchIndex({ accountSid, contactId: updated.id }); + return applyTransformations(updated); }; @@ -351,6 +386,10 @@ export const addConversationMediaToContact = async ( ...contact, conversationMedia: [...contact.conversationMedia, ...createdConversationMedia], }; + + // trigger index operation but don't await for it + indexContactInSearchIndex({ accountSid, contactId: updated.id }); + return applyTransformations(updated); }); }; @@ -425,3 +464,25 @@ export const getContactsByProfileId = async ( }); } }; + +/** + * wrapper around updateSpecificData that also triggers a re-index operation when the conversation media gets updated (e.g. when transcript is exported) + */ +export const updateConversationMediaData = + (contactId: Contact['id']) => + async ( + ...[accountSid, id, storeTypeSpecificData]: Parameters< + typeof updateConversationMediaSpecificData + > + ): ReturnType => { + const result = await updateConversationMediaSpecificData( + accountSid, + id, + storeTypeSpecificData, + ); + + // trigger index operation but don't await for it + indexContactInSearchIndex({ accountSid, contactId }); + + return result; + }; diff --git a/hrm-domain/hrm-core/contact/sql/contact-get-sql.ts b/hrm-domain/hrm-core/contact/sql/contact-get-sql.ts index a421285ba..5cc3468c8 100644 --- a/hrm-domain/hrm-core/contact/sql/contact-get-sql.ts +++ b/hrm-domain/hrm-core/contact/sql/contact-get-sql.ts @@ -21,6 +21,9 @@ import { selectCoalesceReferralsByContactId } from '../../referral/sql/referral- const ID_WHERE_CLAUSE = `WHERE c."accountSid" = $ AND c."id" = $`; const TASKID_WHERE_CLAUSE = `WHERE c."accountSid" = $ AND c."taskId" = $`; +/** + * Note: this query is also used to index Contact records in ES. If the JOINs are ever removed from this query, make sure that the JOINs are preserved for the ES dedicated one + */ export const selectContactsWithRelations = (table: string) => ` SELECT c.*, reports."csamReports", joinedReferrals."referrals", media."conversationMedia" FROM "${table}" c diff --git a/hrm-domain/hrm-core/conversation-media/conversation-media-data-access.ts b/hrm-domain/hrm-core/conversation-media/conversation-media-data-access.ts index c02a7eeee..0fc7b2d7b 100644 --- a/hrm-domain/hrm-core/conversation-media/conversation-media-data-access.ts +++ b/hrm-domain/hrm-core/conversation-media/conversation-media-data-access.ts @@ -116,6 +116,9 @@ export const getByContactId = async ( }), ); +/** + * NOTE: This function should not be used, but via the wrapper exposed from contact service. This is because otherwise, no contact re-index will be triggered. + */ export const updateSpecificData = async ( accountSid: HrmAccountId, id: ConversationMedia['id'], diff --git a/hrm-domain/hrm-core/conversation-media/conversation-media.ts b/hrm-domain/hrm-core/conversation-media/conversation-media.ts index 73c683519..a403b520e 100644 --- a/hrm-domain/hrm-core/conversation-media/conversation-media.ts +++ b/hrm-domain/hrm-core/conversation-media/conversation-media.ts @@ -26,5 +26,5 @@ export { create as createConversationMedia, getById as getConversationMediaById, getByContactId as getConversationMediaByContactId, - updateSpecificData as updateConversationMediaData, + updateSpecificData as updateConversationMediaSpecificData, } from './conversation-media-data-access'; diff --git a/hrm-domain/hrm-core/jobs/search/publishToSearchIndex.ts b/hrm-domain/hrm-core/jobs/search/publishToSearchIndex.ts index c37d3e50f..04a8af6c7 100644 --- a/hrm-domain/hrm-core/jobs/search/publishToSearchIndex.ts +++ b/hrm-domain/hrm-core/jobs/search/publishToSearchIndex.ts @@ -24,13 +24,24 @@ const PENDING_INDEX_QUEUE_SSM_PATH = `/${process.env.NODE_ENV}/${ process.env.AWS_REGION ?? process.env.AWS_DEFAULT_REGION }/sqs/jobs/hrm-search-index/queue-url-consumer`; -const publishToSearchIndex = async (message: IndexMessage) => { +const publishToSearchIndex = async ({ + message, + messageGroupId, +}: { + message: IndexMessage; + messageGroupId: string; +}) => { try { + console.log( + '>>>> publishToSearchIndex invoked with message: ', + JSON.stringify(message), + ); const queueUrl = await getSsmParameter(PENDING_INDEX_QUEUE_SSM_PATH); return await sendSqsMessage({ queueUrl, message: JSON.stringify(message), + messageGroupId, }); } catch (err) { console.error( @@ -48,7 +59,11 @@ export const publishContactToSearchIndex = async ({ accountSid: AccountSID; contact: Contact; operation: IndexMessage['operation']; -}) => publishToSearchIndex({ accountSid, type: 'contact', contact, operation }); +}) => + publishToSearchIndex({ + message: { accountSid, type: 'contact', contact, operation }, + messageGroupId: `${accountSid}-contact-${contact.id}`, + }); export const publishCaseToSearchIndex = async ({ accountSid, @@ -58,4 +73,8 @@ export const publishCaseToSearchIndex = async ({ accountSid: AccountSID; case: CaseService; operation: IndexMessage['operation']; -}) => publishToSearchIndex({ accountSid, type: 'case', case: caseObj, operation }); +}) => + publishToSearchIndex({ + message: { accountSid, type: 'case', case: caseObj, operation }, + messageGroupId: `${accountSid}-case-${caseObj.id}`, + }); diff --git a/hrm-domain/hrm-core/unit-tests/contact/contactService.test.ts b/hrm-domain/hrm-core/unit-tests/contact/contactService.test.ts index f0d83bd95..7e94f06cc 100644 --- a/hrm-domain/hrm-core/unit-tests/contact/contactService.test.ts +++ b/hrm-domain/hrm-core/unit-tests/contact/contactService.test.ts @@ -34,6 +34,11 @@ import { ALWAYS_CAN, OPEN_CONTACT_ACTION_CONDITIONS } from '../mocks'; import '@tech-matters/testing/expectToParseAsDate'; import { openPermissions } from '../../permissions/json-permissions'; import { RulesFile, TKConditionsSets } from '../../permissions/rulesMap'; +import * as publishToSearchIndex from '../../jobs/search/publishToSearchIndex'; + +const publishToSearchIndexSpy = jest + .spyOn(publishToSearchIndex, 'publishContactToSearchIndex') + .mockImplementation(async () => Promise.resolve('Ok') as any); const accountSid = 'AC-accountSid'; const workerSid = 'WK-WORKER_SID'; @@ -140,6 +145,7 @@ describe('createContact', () => { identifierId: 1, }); + expect(publishToSearchIndexSpy).toHaveBeenCalled(); expect(returnValue).toStrictEqual(mockContact); }); @@ -168,6 +174,7 @@ describe('createContact', () => { identifierId: 2, }); + expect(publishToSearchIndexSpy).toHaveBeenCalled(); expect(returnValue).toStrictEqual(mockContact); }); @@ -202,6 +209,7 @@ describe('createContact', () => { identifierId: undefined, }); + expect(publishToSearchIndexSpy).toHaveBeenCalled(); expect(returnValue).toStrictEqual(mockContact); }); @@ -223,6 +231,7 @@ describe('createContact', () => { identifierId: 1, }); + expect(publishToSearchIndexSpy).toHaveBeenCalled(); expect(returnValue).toStrictEqual(mockContact); }); @@ -245,6 +254,7 @@ describe('createContact', () => { identifierId: 1, }); + expect(publishToSearchIndexSpy).toHaveBeenCalled(); expect(returnValue).toStrictEqual(mockContact); }); }); @@ -261,6 +271,8 @@ describe('connectContactToCase', () => { '4321', ALWAYS_CAN.user.workerSid, ); + + expect(publishToSearchIndexSpy).toHaveBeenCalled(); expect(result).toStrictEqual(mockContact); }); @@ -271,6 +283,7 @@ describe('connectContactToCase', () => { expect( connectContactToCase(accountSid, '1234', '4321', ALWAYS_CAN), ).rejects.toThrow(); + expect(publishToSearchIndexSpy).not.toHaveBeenCalled(); }); }); @@ -305,6 +318,8 @@ describe('patchContact', () => { samplePatch, ALWAYS_CAN, ); + + expect(publishToSearchIndexSpy).toHaveBeenCalled(); expect(result).toStrictEqual(mockContact); expect(patchSpy).toHaveBeenCalledWith(accountSid, '1234', true, { updatedBy: contactPatcherSid, @@ -328,6 +343,8 @@ describe('patchContact', () => { const patchSpy = jest.fn(); jest.spyOn(contactDb, 'patch').mockReturnValue(patchSpy); patchSpy.mockResolvedValue(undefined); + + expect(publishToSearchIndexSpy).not.toHaveBeenCalled(); expect( patchContact(accountSid, contactPatcherSid, true, '1234', samplePatch, ALWAYS_CAN), ).rejects.toThrow(); diff --git a/hrm-domain/hrm-service/Dockerfile b/hrm-domain/hrm-service/Dockerfile index 1690f12f7..b587c8cc6 100644 --- a/hrm-domain/hrm-service/Dockerfile +++ b/hrm-domain/hrm-service/Dockerfile @@ -35,6 +35,7 @@ RUN apk add --no-cache rsync \ && npx tsc -b tsconfig.build.json \ && cp -r hrm-domain/hrm-service/* /home/node/ \ && mkdir -p /home/node/hrm-domain/ \ + && cp -r hrm-domain/packages /home/node/hrm-domain/packages \ && cp -r hrm-domain/hrm-core /home/node/hrm-domain/hrm-core \ && cp -r hrm-domain/scheduled-tasks /home/node/hrm-domain/scheduled-tasks \ && cp -r packages /home/node/ \ diff --git a/hrm-domain/hrm-service/service-tests/contact-job/contactJobCleanup.test.ts b/hrm-domain/hrm-service/service-tests/contact-job/contactJobCleanup.test.ts index 07d4346ed..cca2b5846 100644 --- a/hrm-domain/hrm-service/service-tests/contact-job/contactJobCleanup.test.ts +++ b/hrm-domain/hrm-service/service-tests/contact-job/contactJobCleanup.test.ts @@ -24,12 +24,10 @@ import { mockSuccessfulTwilioAuthentication, } from '@tech-matters/testing'; import { createContactJob } from '@tech-matters/hrm-core/contact-job/contact-job-data-access'; -import { - isS3StoredTranscriptPending, - updateConversationMediaData, -} from '@tech-matters/hrm-core/conversation-media/conversation-media'; +import { isS3StoredTranscriptPending } from '@tech-matters/hrm-core/conversation-media/conversation-media'; import { S3ContactMediaType } from '@tech-matters/hrm-core/conversation-media/conversation-media'; import { getById as getContactById } from '@tech-matters/hrm-core/contact/contactDataAccess'; +import { updateConversationMediaData } from '@tech-matters/hrm-core/contact/contactService'; import * as cleanupContactJobsApi from '@tech-matters/contact-job-cleanup'; import { completeContactJob, @@ -206,7 +204,7 @@ describe('cleanupContactJobs', () => { job = await completeContactJob({ id: job.id, completionPayload }); job = await backDateJob(job.id); - await updateConversationMediaData( + await updateConversationMediaData(contact.id)( accountSid, job.additionalPayload.conversationMediaId, completionPayload, diff --git a/hrm-domain/hrm-service/service-tests/contact-job/jobTypes/retrieveTranscript.test.ts b/hrm-domain/hrm-service/service-tests/contact-job/jobTypes/retrieveTranscript.test.ts index 69d0af6b8..021944669 100644 --- a/hrm-domain/hrm-service/service-tests/contact-job/jobTypes/retrieveTranscript.test.ts +++ b/hrm-domain/hrm-service/service-tests/contact-job/jobTypes/retrieveTranscript.test.ts @@ -515,7 +515,7 @@ describe('complete retrieve-transcript job type', () => { // ); const updateConversationMediaSpy = jest.spyOn( - conversationMediaApi, + contactApi, 'updateConversationMediaData', ); diff --git a/hrm-domain/packages/hrm-search-config/convertToIndexDocument.ts b/hrm-domain/packages/hrm-search-config/convertToIndexDocument.ts index f1ae36ced..238aff6b6 100644 --- a/hrm-domain/packages/hrm-search-config/convertToIndexDocument.ts +++ b/hrm-domain/packages/hrm-search-config/convertToIndexDocument.ts @@ -23,13 +23,13 @@ import { import { CreateIndexConvertedDocument } from '@tech-matters/elasticsearch-client'; import { IndexPayload, IndexPayloadCase, IndexPayloadContact } from './payload'; -const filterEmpty = (doc: T): T => +const filterUndefined = (doc: T): T => Object.entries(doc).reduce((accum, [key, value]) => { - if (value) { - return { ...accum, [key]: value }; + if (value === undefined) { + return accum; } - return accum; + return { ...accum, [key]: value }; }, {} as T); export const convertContactToContactDocument = ({ @@ -71,7 +71,7 @@ export const convertContactToContactDocument = ({ // low_boost_global: '', // lowBoostGlobal.join(' '), }; - return filterEmpty(contactDocument); + return filterUndefined(contactDocument); }; const convertCaseToCaseDocument = ({ @@ -130,7 +130,7 @@ const convertCaseToCaseDocument = ({ // low_boost_global: '', // lowBoostGlobal.join(' '), }; - return filterEmpty(caseDocument); + return filterUndefined(caseDocument); }; const convertToContactIndexDocument = (payload: IndexPayload) => { From c2da2b5bd4c614b2b1a37caf1727d24134181d1b Mon Sep 17 00:00:00 2001 From: mythilytm Date: Wed, 5 Jun 2024 13:12:53 -0400 Subject: [PATCH 44/55] revert logs --- hrm-domain/hrm-core/jobs/search/publishToSearchIndex.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/hrm-domain/hrm-core/jobs/search/publishToSearchIndex.ts b/hrm-domain/hrm-core/jobs/search/publishToSearchIndex.ts index b02f64331..1d07145ff 100644 --- a/hrm-domain/hrm-core/jobs/search/publishToSearchIndex.ts +++ b/hrm-domain/hrm-core/jobs/search/publishToSearchIndex.ts @@ -27,7 +27,6 @@ const PENDING_INDEX_QUEUE_SSM_PATH = `/${process.env.NODE_ENV}/${ const publishToSearchIndex = async (message: IndexMessage) => { try { const queueUrl = await getSsmParameter(PENDING_INDEX_QUEUE_SSM_PATH); - console.log('>> publishToSearchIndex queueUrl, message', { queueUrl, message }); return await sendSqsMessage({ queueUrl, message: JSON.stringify(message), From 4dda995a19319a189403d2a7e5eb52e62dfddf7b Mon Sep 17 00:00:00 2001 From: Gianfranco Paoloni Date: Thu, 6 Jun 2024 18:29:21 -0300 Subject: [PATCH 45/55] chore: added admin cases routes, implemented reindex case endpoint --- hrm-domain/hrm-core/case/adminCaseRoutesV0.ts | 46 ++++++++++++ .../hrm-core/case/caseReindexService.ts | 70 +++++++++++++++++++ hrm-domain/hrm-core/routes.ts | 2 + 3 files changed, 118 insertions(+) create mode 100644 hrm-domain/hrm-core/case/adminCaseRoutesV0.ts create mode 100644 hrm-domain/hrm-core/case/caseReindexService.ts diff --git a/hrm-domain/hrm-core/case/adminCaseRoutesV0.ts b/hrm-domain/hrm-core/case/adminCaseRoutesV0.ts new file mode 100644 index 000000000..da6302bb3 --- /dev/null +++ b/hrm-domain/hrm-core/case/adminCaseRoutesV0.ts @@ -0,0 +1,46 @@ +/** + * Copyright (C) 2021-2023 Technology Matters + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see https://www.gnu.org/licenses/. + */ + +import type { Request, Response, NextFunction } from 'express'; +import { isErr, mapHTTPError } from '@tech-matters/types'; +import { SafeRouter, publicEndpoint } from '../permissions'; +import { reindexCases } from './caseReindexService'; + +const adminContactsRouter = SafeRouter(); + +// admin POST endpoint to reindex contacts. req body has accountSid, dateFrom, dateTo +adminContactsRouter.post( + '/reindex', + publicEndpoint, + async (req: Request, res: Response, next: NextFunction) => { + const { hrmAccountId } = req; + const { dateFrom, dateTo } = req.body; + + const result = await reindexCases(hrmAccountId, dateFrom, dateTo); + + if (isErr(result)) { + return next( + mapHTTPError(result, { + InvalidParameterError: 400, + }), + ); + } + + res.json(result.data); + }, +); + +export default adminContactsRouter.expressRouter; diff --git a/hrm-domain/hrm-core/case/caseReindexService.ts b/hrm-domain/hrm-core/case/caseReindexService.ts new file mode 100644 index 000000000..f388fd4e5 --- /dev/null +++ b/hrm-domain/hrm-core/case/caseReindexService.ts @@ -0,0 +1,70 @@ +/** + * Copyright (C) 2021-2023 Technology Matters + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see https://www.gnu.org/licenses/. + */ + +import { HrmAccountId, newErr, newOkFromData } from '@tech-matters/types'; +import { CaseService, searchCases } from './caseService'; +import { publishCaseToSearchIndex } from '../jobs/search/publishToSearchIndex'; +import { maxPermissions } from '../permissions'; +import { AsyncProcessor, SearchFunction, processInBatch } from '../autoPaginate'; +import formatISO from 'date-fns/formatISO'; + +export const reindexCases = async ( + accountSid: HrmAccountId, + dateFrom: string, + dateTo: string, +) => { + try { + const filters = { + updatedAt: { + from: formatISO(new Date(dateFrom)), + to: formatISO(new Date(dateTo)), + }, + }; + + const searchFunction: SearchFunction = async limitAndOffset => { + const res = await searchCases( + accountSid, + { + limit: limitAndOffset.limit.toString(), + offset: limitAndOffset.offset.toString(), + }, + {}, + { filters }, + maxPermissions, + ); + return { records: res.cases as CaseService[], count: res.count }; + }; + + const asyncProcessor: AsyncProcessor = async casesResult => { + const promises = casesResult.records.map(caseObj => { + return publishCaseToSearchIndex({ + accountSid, + case: caseObj, + operation: 'index', + }); + }); + + await Promise.all(promises); + }; + + await processInBatch(searchFunction, asyncProcessor); + + return newOkFromData('Successfully indexed contacts'); + } catch (error) { + console.error('Error reindexing contacts', error); + return newErr({ error, message: 'Error reindexing contacts' }); + } +}; diff --git a/hrm-domain/hrm-core/routes.ts b/hrm-domain/hrm-core/routes.ts index 486eb3309..ab8f93199 100644 --- a/hrm-domain/hrm-core/routes.ts +++ b/hrm-domain/hrm-core/routes.ts @@ -25,6 +25,7 @@ import permissions from './permissions/permissions-routes-v0'; import profiles from './profile/profileRoutesV0'; import adminProfiles from './profile/adminProfileRoutesV0'; import adminContacts from './contact/adminContactRoutesV0'; +import adminCases from './case/adminCaseRoutesV0'; import { Permissions } from './permissions'; export const HRM_ROUTES: { @@ -53,6 +54,7 @@ export const ADMIN_ROUTES: { }[] = [ { path: '/profiles', routerFactory: () => adminProfiles }, { path: '/contacts', routerFactory: () => adminContacts }, + { path: '/cases', routerFactory: () => adminCases }, ]; export const adminApiV0 = () => { From 1d6e2263d5081feaa7ca9a372f3f31d98d55fbc9 Mon Sep 17 00:00:00 2001 From: Gianfranco Paoloni Date: Thu, 6 Jun 2024 18:30:20 -0300 Subject: [PATCH 46/55] chore: simpify path for reindex endpoints --- hrm-domain/hrm-core/contact/adminContactRoutesV0.ts | 4 +--- .../hrm-service/scripts/admin-commands/reindex/hrm.ts | 8 ++------ 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/hrm-domain/hrm-core/contact/adminContactRoutesV0.ts b/hrm-domain/hrm-core/contact/adminContactRoutesV0.ts index 3aaaa762d..2c81cf9f1 100644 --- a/hrm-domain/hrm-core/contact/adminContactRoutesV0.ts +++ b/hrm-domain/hrm-core/contact/adminContactRoutesV0.ts @@ -14,10 +14,8 @@ * along with this program. If not, see https://www.gnu.org/licenses/. */ -// eslint-disable-next-line @typescript-eslint/no-unused-vars import type { Request, Response, NextFunction } from 'express'; import { isErr, mapHTTPError } from '@tech-matters/types'; -// import createError from 'http-errors'; import { SafeRouter, publicEndpoint } from '../permissions'; import { reindexContacts } from './contactsReindexService'; @@ -25,7 +23,7 @@ const adminContactsRouter = SafeRouter(); // admin POST endpoint to reindex contacts. req body has accountSid, dateFrom, dateTo adminContactsRouter.post( - '/reindexContacts', + '/reindex', publicEndpoint, async (req: Request, res: Response, next: NextFunction) => { const { hrmAccountId } = req; diff --git a/hrm-domain/hrm-service/scripts/admin-commands/reindex/hrm.ts b/hrm-domain/hrm-service/scripts/admin-commands/reindex/hrm.ts index 2ae699d8b..c021ba8c2 100644 --- a/hrm-domain/hrm-service/scripts/admin-commands/reindex/hrm.ts +++ b/hrm-domain/hrm-service/scripts/admin-commands/reindex/hrm.ts @@ -98,11 +98,7 @@ export const handler = async ({ } if (contacts) { - const url = getAdminV0URL( - internalResourcesUrl, - accountSid, - '/contacts/reindexContacts', - ); + const url = getAdminV0URL(internalResourcesUrl, accountSid, '/contacts/reindex'); const response = await fetch(url, { method: 'POST', headers: { @@ -120,7 +116,7 @@ export const handler = async ({ } if (cases) { - const url = getAdminV0URL(internalResourcesUrl, accountSid, '/cases/reindexCases'); + const url = getAdminV0URL(internalResourcesUrl, accountSid, '/cases/reindex'); const response = await fetch(url, { method: 'POST', headers: { From ee189e07f265fbb48a29cc4414f7a9c06f692590 Mon Sep 17 00:00:00 2001 From: mythily <102122005+mythilytm@users.noreply.github.com> Date: Fri, 7 Jun 2024 12:35:23 -0400 Subject: [PATCH 47/55] CHI-2760 Build script for reindexing contacts and cases (#652) * feat: build script for reindexing contacts and cases * change options for indexing * sqs debug * revert logs * modify to have more graceful error handling for user * modify to have more graceful error handling for user * modify to have more graceful error handling for user --------- Co-authored-by: Gianfranco Paoloni --- .../jobs/search/publishToSearchIndex.ts | 1 - hrm-domain/hrm-service/scripts/README.md | 2 + .../scripts/admin-commands/reindex.ts | 25 +++ .../scripts/admin-commands/reindex/hrm.ts | 146 ++++++++++++++++++ 4 files changed, 173 insertions(+), 1 deletion(-) create mode 100644 hrm-domain/hrm-service/scripts/admin-commands/reindex.ts create mode 100644 hrm-domain/hrm-service/scripts/admin-commands/reindex/hrm.ts diff --git a/hrm-domain/hrm-core/jobs/search/publishToSearchIndex.ts b/hrm-domain/hrm-core/jobs/search/publishToSearchIndex.ts index 04a8af6c7..af9975e10 100644 --- a/hrm-domain/hrm-core/jobs/search/publishToSearchIndex.ts +++ b/hrm-domain/hrm-core/jobs/search/publishToSearchIndex.ts @@ -37,7 +37,6 @@ const publishToSearchIndex = async ({ JSON.stringify(message), ); const queueUrl = await getSsmParameter(PENDING_INDEX_QUEUE_SSM_PATH); - return await sendSqsMessage({ queueUrl, message: JSON.stringify(message), diff --git a/hrm-domain/hrm-service/scripts/README.md b/hrm-domain/hrm-service/scripts/README.md index 699ca6b5e..8f99e0c86 100644 --- a/hrm-domain/hrm-service/scripts/README.md +++ b/hrm-domain/hrm-service/scripts/README.md @@ -22,6 +22,8 @@ admin-cli | ├── create: # Create a new profile flag | ├── edit: # Edit an existing profile flag | ├── delete: # Delete an existing profile flag +├── reindex +| ├── hrm: # Reindex contacts and cases based on date range ``` ### Usage diff --git a/hrm-domain/hrm-service/scripts/admin-commands/reindex.ts b/hrm-domain/hrm-service/scripts/admin-commands/reindex.ts new file mode 100644 index 000000000..c6846eb2a --- /dev/null +++ b/hrm-domain/hrm-service/scripts/admin-commands/reindex.ts @@ -0,0 +1,25 @@ +/** + * Copyright (C) 2021-2023 Technology Matters + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see https://www.gnu.org/licenses/. + */ + +export const command = 'reindex '; +export const desc = 'admin endpoints for reindexing contacts and cases'; + +export const builder = function (yargs) { + return yargs.commandDir('reindex', { + exclude: /^(index|_)/, + extensions: ['ts'], + }); +}; diff --git a/hrm-domain/hrm-service/scripts/admin-commands/reindex/hrm.ts b/hrm-domain/hrm-service/scripts/admin-commands/reindex/hrm.ts new file mode 100644 index 000000000..de41cf52a --- /dev/null +++ b/hrm-domain/hrm-service/scripts/admin-commands/reindex/hrm.ts @@ -0,0 +1,146 @@ +/** + * Copyright (C) 2021-2023 Technology Matters + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see https://www.gnu.org/licenses/. + */ + +import { getHRMInternalEndpointAccess } from '@tech-matters/service-discovery'; +// eslint-disable-next-line import/no-extraneous-dependencies +import { fetch } from 'undici'; +import { getAdminV0URL, staticKeyPattern } from '../../hrmInternalConfig'; + +export const command = 'hrm'; +export const describe = 'Reindex contacts and cases based on date range'; + +export const builder = { + co: { + alias: 'contacts', + describe: 'reindex contacts', + type: 'boolean', + default: false, + }, + ca: { + alias: 'cases', + describe: 'reindex cases', + type: 'boolean', + default: false, + }, + e: { + alias: 'environment', + describe: 'environment (e.g. development, staging, production)', + demandOption: true, + type: 'string', + }, + r: { + alias: 'region', + describe: 'region (e.g. us-east-1)', + demandOption: true, + type: 'string', + }, + a: { + alias: 'accountSid', + describe: 'account SID', + demandOption: true, + type: 'string', + }, + f: { + alias: 'dateFrom', + describe: 'start date (e.g. 2024-01-01)', + demandOption: true, + type: 'string', + }, + t: { + alias: 'dateTo', + describe: 'end date (e.g. 2024-12-31)', + demandOption: true, + type: 'string', + }, +}; + +export const handler = async ({ + region, + environment, + accountSid, + dateFrom, + dateTo, + contacts, + cases, +}) => { + try { + const timestamp = new Date().getTime(); + const assumeRoleParams = { + RoleArn: 'arn:aws:iam::712893914485:role/tf-admin', + RoleSessionName: `hrm-admin-cli-${timestamp}`, + }; + + const { authKey, internalResourcesUrl } = await getHRMInternalEndpointAccess({ + region, + environment, + staticKeyPattern, + assumeRoleParams, + }); + + if (!contacts && !cases) { + console.log( + 'Please specify contacts and/or cases option to reindex in your command', + ); + return; + } + + if (contacts) { + const url = getAdminV0URL( + internalResourcesUrl, + accountSid, + '/contacts/reindexContacts', + ); + const response = await fetch(url, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Authorization: `Basic ${authKey}`, + }, + body: JSON.stringify({ dateFrom, dateTo }), + }); + + if (!response.ok) { + console.error( + `Failed to submit request for reindexing contacts: ${response.statusText}`, + ); + } else { + console.log(`Reindexing contacts from ${dateFrom} to ${dateTo}...`); + } + } + + if (cases) { + const url = getAdminV0URL(internalResourcesUrl, accountSid, '/cases/reindexCases'); + const response = await fetch(url, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Authorization: `Basic ${authKey}`, + }, + body: JSON.stringify({ dateFrom, dateTo }), + }); + + if (!response.ok) { + console.error( + `Failed to submit request for reindexing cases: ${response.statusText}`, + ); + } else { + console.log(`Reindexing cases from ${dateFrom} to ${dateTo}...`); + } + } + } catch (err) { + console.error(err); + } +}; From 936c768afccba8d4c185ec87346b839f093ad111 Mon Sep 17 00:00:00 2001 From: Gianfranco Paoloni Date: Fri, 7 Jun 2024 17:03:18 -0300 Subject: [PATCH 48/55] chore: moved auto paginate logic into batch-processing module --- packages/batch-processing/autoPaginate.ts | 88 +++++++++++++++++++++++ packages/batch-processing/index.ts | 17 +++++ packages/batch-processing/package.json | 16 +++++ 3 files changed, 121 insertions(+) create mode 100644 packages/batch-processing/autoPaginate.ts create mode 100644 packages/batch-processing/index.ts create mode 100644 packages/batch-processing/package.json diff --git a/packages/batch-processing/autoPaginate.ts b/packages/batch-processing/autoPaginate.ts new file mode 100644 index 000000000..456619b1d --- /dev/null +++ b/packages/batch-processing/autoPaginate.ts @@ -0,0 +1,88 @@ +/** + * Copyright (C) 2021-2023 Technology Matters + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see https://www.gnu.org/licenses/. + */ + +export const defaultLimitAndOffset = { + limit: '1000', // The underlying query limits this value to 1000. + offset: '0', +}; + +type SearchResult = { + count: number; + records: T[]; +}; + +type LimitAndOffset = { + limit: number; + offset: number; +}; + +export type SearchFunction = ( + limitAndOffset: LimitAndOffset, +) => Promise>; + +export type AsyncProcessor = (result: SearchResult) => Promise; + +export const processInBatch = async ( + searchFunction: SearchFunction, + asyncProcessor: AsyncProcessor, +): Promise => { + let hasMoreItems = true; + let offset = Number(defaultLimitAndOffset.offset); + const limit = Number(defaultLimitAndOffset.limit); + + let processed = 0; + + while (hasMoreItems) { + /** + * Updates 'limitAndOffset' param + * Keep the other params intact + */ + const searchResult = await searchFunction({ limit, offset }); + + const { count, records } = searchResult; + + await asyncProcessor(searchResult); + + processed += records.length; + hasMoreItems = processed < count; + + if (hasMoreItems) { + offset += limit; + } + } +}; + +/** + * This function takes care of keep calling the search function + * until there's no more data to be fetched. It works by dynamically + * adjusting the 'offset' on each subsequent call. + * + * @param searchFunction function to perform search of cases or contacts with the provided limit & offset + * @returns cases[] or contacts[] + */ +export const autoPaginate = async ( + searchFunction: SearchFunction, +): Promise => { + let items: T[] = []; + + const asyncProcessor = async (result: SearchResult) => { + items.push(...result.records); + }; + + await processInBatch(searchFunction, asyncProcessor); + + return items; +}; diff --git a/packages/batch-processing/index.ts b/packages/batch-processing/index.ts new file mode 100644 index 000000000..20f74fd3f --- /dev/null +++ b/packages/batch-processing/index.ts @@ -0,0 +1,17 @@ +/** + * Copyright (C) 2021-2023 Technology Matters + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see https://www.gnu.org/licenses/. + */ + +export * from './autoPaginate'; diff --git a/packages/batch-processing/package.json b/packages/batch-processing/package.json new file mode 100644 index 000000000..8ae27fab1 --- /dev/null +++ b/packages/batch-processing/package.json @@ -0,0 +1,16 @@ +{ + "name": "@tech-matters/batch-processing", + "version": "1.0.0", + "description": "", + "main": "dist/index.js", + "types": "index.d.ts", + "scripts": { + }, + "author": "", + "license": "AGPL", + "dependencies": { + }, + "devDependencies": { + "@types/node": "^18.15.11" + } +} From a26bc2aa103b4dadbbd036c7425180ffc6081943 Mon Sep 17 00:00:00 2001 From: Gianfranco Paoloni Date: Fri, 7 Jun 2024 17:04:23 -0300 Subject: [PATCH 49/55] chore: add dependency and import from appropriate place --- hrm-domain/hrm-core/autoPaginate.ts | 88 ------------------- .../hrm-core/case/caseReindexService.ts | 6 +- .../contact/contactsReindexService.ts | 6 +- hrm-domain/hrm-core/package.json | 1 + hrm-domain/hrm-service/tsconfig.build.json | 1 + .../hrm-data-pull/package.json | 1 + .../hrm-data-pull/pull-cases.ts | 3 +- .../hrm-data-pull/pull-contacts.ts | 2 +- .../hrm-data-pull/pull-profiles.ts | 3 +- package-lock.json | 51 +++++++++++ 10 files changed, 67 insertions(+), 95 deletions(-) delete mode 100644 hrm-domain/hrm-core/autoPaginate.ts diff --git a/hrm-domain/hrm-core/autoPaginate.ts b/hrm-domain/hrm-core/autoPaginate.ts deleted file mode 100644 index 456619b1d..000000000 --- a/hrm-domain/hrm-core/autoPaginate.ts +++ /dev/null @@ -1,88 +0,0 @@ -/** - * Copyright (C) 2021-2023 Technology Matters - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see https://www.gnu.org/licenses/. - */ - -export const defaultLimitAndOffset = { - limit: '1000', // The underlying query limits this value to 1000. - offset: '0', -}; - -type SearchResult = { - count: number; - records: T[]; -}; - -type LimitAndOffset = { - limit: number; - offset: number; -}; - -export type SearchFunction = ( - limitAndOffset: LimitAndOffset, -) => Promise>; - -export type AsyncProcessor = (result: SearchResult) => Promise; - -export const processInBatch = async ( - searchFunction: SearchFunction, - asyncProcessor: AsyncProcessor, -): Promise => { - let hasMoreItems = true; - let offset = Number(defaultLimitAndOffset.offset); - const limit = Number(defaultLimitAndOffset.limit); - - let processed = 0; - - while (hasMoreItems) { - /** - * Updates 'limitAndOffset' param - * Keep the other params intact - */ - const searchResult = await searchFunction({ limit, offset }); - - const { count, records } = searchResult; - - await asyncProcessor(searchResult); - - processed += records.length; - hasMoreItems = processed < count; - - if (hasMoreItems) { - offset += limit; - } - } -}; - -/** - * This function takes care of keep calling the search function - * until there's no more data to be fetched. It works by dynamically - * adjusting the 'offset' on each subsequent call. - * - * @param searchFunction function to perform search of cases or contacts with the provided limit & offset - * @returns cases[] or contacts[] - */ -export const autoPaginate = async ( - searchFunction: SearchFunction, -): Promise => { - let items: T[] = []; - - const asyncProcessor = async (result: SearchResult) => { - items.push(...result.records); - }; - - await processInBatch(searchFunction, asyncProcessor); - - return items; -}; diff --git a/hrm-domain/hrm-core/case/caseReindexService.ts b/hrm-domain/hrm-core/case/caseReindexService.ts index f388fd4e5..66fd34ed6 100644 --- a/hrm-domain/hrm-core/case/caseReindexService.ts +++ b/hrm-domain/hrm-core/case/caseReindexService.ts @@ -18,7 +18,11 @@ import { HrmAccountId, newErr, newOkFromData } from '@tech-matters/types'; import { CaseService, searchCases } from './caseService'; import { publishCaseToSearchIndex } from '../jobs/search/publishToSearchIndex'; import { maxPermissions } from '../permissions'; -import { AsyncProcessor, SearchFunction, processInBatch } from '../autoPaginate'; +import { + AsyncProcessor, + SearchFunction, + processInBatch, +} from '@tech-matters/batch-processing'; import formatISO from 'date-fns/formatISO'; export const reindexCases = async ( diff --git a/hrm-domain/hrm-core/contact/contactsReindexService.ts b/hrm-domain/hrm-core/contact/contactsReindexService.ts index f7f397076..6e4346f52 100644 --- a/hrm-domain/hrm-core/contact/contactsReindexService.ts +++ b/hrm-domain/hrm-core/contact/contactsReindexService.ts @@ -18,7 +18,11 @@ import { HrmAccountId, newErr, newOkFromData } from '@tech-matters/types'; import { Contact, searchContacts } from './contactService'; import { publishContactToSearchIndex } from '../jobs/search/publishToSearchIndex'; import { maxPermissions } from '../permissions'; -import { AsyncProcessor, SearchFunction, processInBatch } from '../autoPaginate'; +import { + AsyncProcessor, + SearchFunction, + processInBatch, +} from '@tech-matters/batch-processing'; export const reindexContacts = async ( accountSid: HrmAccountId, diff --git a/hrm-domain/hrm-core/package.json b/hrm-domain/hrm-core/package.json index 055b26c0a..55747057d 100644 --- a/hrm-domain/hrm-core/package.json +++ b/hrm-domain/hrm-core/package.json @@ -23,6 +23,7 @@ }, "homepage": "https://github.com/tech-matters/hrm#readme", "dependencies": { + "@tech-matters/batch-processing": "^1.0.0", "@tech-matters/hrm-search-config": "^1.0.0", "@tech-matters/hrm-types": "^1.0.0", "@tech-matters/http": "^1.0.0", diff --git a/hrm-domain/hrm-service/tsconfig.build.json b/hrm-domain/hrm-service/tsconfig.build.json index 7694eb726..578cb0055 100644 --- a/hrm-domain/hrm-service/tsconfig.build.json +++ b/hrm-domain/hrm-service/tsconfig.build.json @@ -13,6 +13,7 @@ { "path": "packages/twilio-client" }, { "path": "packages/types" }, { "path": "packages/service-discovery" }, + { "path": "packages/batch-processing" }, { "path": "resources-domain/packages/resources-search-config" }, { "path": "resources-domain/resources-service" }, { "path": "hrm-domain/packages/hrm-types" }, diff --git a/hrm-domain/scheduled-tasks/hrm-data-pull/package.json b/hrm-domain/scheduled-tasks/hrm-data-pull/package.json index db9c6d72a..b591c5cf0 100644 --- a/hrm-domain/scheduled-tasks/hrm-data-pull/package.json +++ b/hrm-domain/scheduled-tasks/hrm-data-pull/package.json @@ -11,6 +11,7 @@ ".": "./dist/index.js" }, "dependencies" : { + "@tech-matters/batch-processing": "^1.0.0", "@tech-matters/types": "^1.0.0", "@tech-matters/twilio-worker-auth": "^1.0.0", "@tech-matters/s3-client": "^1.0.0", diff --git a/hrm-domain/scheduled-tasks/hrm-data-pull/pull-cases.ts b/hrm-domain/scheduled-tasks/hrm-data-pull/pull-cases.ts index f2f5c6cdb..3183045b4 100644 --- a/hrm-domain/scheduled-tasks/hrm-data-pull/pull-cases.ts +++ b/hrm-domain/scheduled-tasks/hrm-data-pull/pull-cases.ts @@ -18,8 +18,7 @@ import format from 'date-fns/format'; import formatISO from 'date-fns/formatISO'; import { putS3Object } from '@tech-matters/s3-client'; import * as caseApi from '@tech-matters/hrm-core/case/caseService'; -import { autoPaginate } from '@tech-matters/hrm-core/autoPaginate'; - +import { autoPaginate } from '@tech-matters/batch-processing'; import { getContext, maxPermissions } from './context'; import { parseISO } from 'date-fns'; diff --git a/hrm-domain/scheduled-tasks/hrm-data-pull/pull-contacts.ts b/hrm-domain/scheduled-tasks/hrm-data-pull/pull-contacts.ts index dbb716bb5..d9dcbe27a 100644 --- a/hrm-domain/scheduled-tasks/hrm-data-pull/pull-contacts.ts +++ b/hrm-domain/scheduled-tasks/hrm-data-pull/pull-contacts.ts @@ -20,7 +20,7 @@ import { putS3Object } from '@tech-matters/s3-client'; import { getContext, maxPermissions } from './context'; import * as contactApi from '@tech-matters/hrm-core/contact/contactService'; -import { autoPaginate } from '@tech-matters/hrm-core/autoPaginate'; +import { autoPaginate } from '@tech-matters/batch-processing'; import { parseISO } from 'date-fns'; const getSearchParams = (startDate: Date, endDate: Date) => ({ diff --git a/hrm-domain/scheduled-tasks/hrm-data-pull/pull-profiles.ts b/hrm-domain/scheduled-tasks/hrm-data-pull/pull-profiles.ts index 8002270c4..3dc1e3199 100644 --- a/hrm-domain/scheduled-tasks/hrm-data-pull/pull-profiles.ts +++ b/hrm-domain/scheduled-tasks/hrm-data-pull/pull-profiles.ts @@ -23,8 +23,7 @@ import type { ProfileSection, ProfileWithRelationships, } from '@tech-matters/hrm-core/profile/profileDataAccess'; -import { autoPaginate } from '@tech-matters/hrm-core/autoPaginate'; - +import { autoPaginate } from '@tech-matters/batch-processing'; import { getContext } from './context'; import { parseISO } from 'date-fns'; diff --git a/package-lock.json b/package-lock.json index ef036281f..26efabda7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -66,6 +66,7 @@ "version": "1.0.0", "license": "AGPL", "dependencies": { + "@tech-matters/batch-processing": "^1.0.0", "@tech-matters/hrm-search-config": "^1.0.0", "@tech-matters/hrm-types": "^1.0.0", "@tech-matters/http": "^1.0.0", @@ -3871,6 +3872,10 @@ "resolved": "lambdas/packages/alb-handler", "link": true }, + "node_modules/@tech-matters/batch-processing": { + "resolved": "packages/batch-processing", + "link": true + }, "node_modules/@tech-matters/case-status-transition": { "resolved": "hrm-domain/scheduled-tasks/case-status-transition", "link": true @@ -13935,6 +13940,12 @@ "node": ">=14.0" } }, + "node_modules/undici-types": { + "version": "5.26.5", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", + "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==", + "dev": true + }, "node_modules/universalify": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.1.2.tgz", @@ -14389,6 +14400,22 @@ "resolved": "https://registry.npmjs.org/zstd-codec/-/zstd-codec-0.1.4.tgz", "integrity": "sha512-KYnWoFWgGtWyQEKNnUcb3u8ZtKO8dn5d8u+oGpxPlopqsPyv60U8suDyfk7Z7UtAO6Sk5i1aVcAs9RbaB1n36A==" }, + "packages/batch-processing": { + "version": "1.0.0", + "license": "AGPL", + "devDependencies": { + "@types/node": "^18.15.11" + } + }, + "packages/batch-processing/node_modules/@types/node": { + "version": "18.19.34", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.34.tgz", + "integrity": "sha512-eXF4pfBNV5DAMKGbI02NnDtWrQ40hAN558/2vvS4gMpMIxaf6JmD7YjnZbq0Q9TDSSkKBamime8ewRoomHdt4g==", + "dev": true, + "dependencies": { + "undici-types": "~5.26.4" + } + }, "packages/elasticsearch-client": { "name": "@tech-matters/elasticsearch-client", "version": "1.0.0", @@ -17426,6 +17453,23 @@ "@types/aws-lambda": "^8.10.108" } }, + "@tech-matters/batch-processing": { + "version": "file:packages/batch-processing", + "requires": { + "@types/node": "^18.15.11" + }, + "dependencies": { + "@types/node": { + "version": "18.19.34", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.34.tgz", + "integrity": "sha512-eXF4pfBNV5DAMKGbI02NnDtWrQ40hAN558/2vvS4gMpMIxaf6JmD7YjnZbq0Q9TDSSkKBamime8ewRoomHdt4g==", + "dev": true, + "requires": { + "undici-types": "~5.26.4" + } + } + } + }, "@tech-matters/case-status-transition": { "version": "file:hrm-domain/scheduled-tasks/case-status-transition", "requires": { @@ -17475,6 +17519,7 @@ "@tech-matters/hrm-core": { "version": "file:hrm-domain/hrm-core", "requires": { + "@tech-matters/batch-processing": "^1.0.0", "@tech-matters/hrm-search-config": "^1.0.0", "@tech-matters/hrm-types": "^1.0.0", "@tech-matters/http": "^1.0.0", @@ -25398,6 +25443,12 @@ "@fastify/busboy": "^2.0.0" } }, + "undici-types": { + "version": "5.26.5", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", + "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==", + "dev": true + }, "universalify": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.1.2.tgz", From 5b71d575e28f2b961bd88b95d32b51c9ceaa51fa Mon Sep 17 00:00:00 2001 From: Gianfranco Paoloni Date: Fri, 7 Jun 2024 17:20:47 -0300 Subject: [PATCH 50/55] chore: added missing tsconfig files --- packages/batch-processing/tsconfig.eslint.json | 4 ++++ packages/batch-processing/tsconfig.json | 9 +++++++++ 2 files changed, 13 insertions(+) create mode 100644 packages/batch-processing/tsconfig.eslint.json create mode 100644 packages/batch-processing/tsconfig.json diff --git a/packages/batch-processing/tsconfig.eslint.json b/packages/batch-processing/tsconfig.eslint.json new file mode 100644 index 000000000..b81b3b862 --- /dev/null +++ b/packages/batch-processing/tsconfig.eslint.json @@ -0,0 +1,4 @@ +{ + "extends": "./tsconfig.json", + "include": ["**/*.ts"] +} diff --git a/packages/batch-processing/tsconfig.json b/packages/batch-processing/tsconfig.json new file mode 100644 index 000000000..ec4161db5 --- /dev/null +++ b/packages/batch-processing/tsconfig.json @@ -0,0 +1,9 @@ +{ + "extends": "../tsconfig.packages-base.json", + "include": ["src/**/*.ts"], + "compilerOptions": { + "outDir": "./dist", + "allowJs": true, + "skipLibCheck": true + } +} From 1955b1151fef3cf730fa8731620de15499196f20 Mon Sep 17 00:00:00 2001 From: Gianfranco Paoloni Date: Fri, 7 Jun 2024 17:24:20 -0300 Subject: [PATCH 51/55] chore: use correct src dir --- packages/batch-processing/{ => src}/autoPaginate.ts | 0 packages/batch-processing/{ => src}/index.ts | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename packages/batch-processing/{ => src}/autoPaginate.ts (100%) rename packages/batch-processing/{ => src}/index.ts (100%) diff --git a/packages/batch-processing/autoPaginate.ts b/packages/batch-processing/src/autoPaginate.ts similarity index 100% rename from packages/batch-processing/autoPaginate.ts rename to packages/batch-processing/src/autoPaginate.ts diff --git a/packages/batch-processing/index.ts b/packages/batch-processing/src/index.ts similarity index 100% rename from packages/batch-processing/index.ts rename to packages/batch-processing/src/index.ts From bb6673b1dbcc9516d64038c097b758b9201ddd4e Mon Sep 17 00:00:00 2001 From: Gianfranco Paoloni Date: Fri, 7 Jun 2024 17:30:57 -0300 Subject: [PATCH 52/55] chore: tidy package.json --- packages/batch-processing/package.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/batch-processing/package.json b/packages/batch-processing/package.json index 8ae27fab1..fa0c90269 100644 --- a/packages/batch-processing/package.json +++ b/packages/batch-processing/package.json @@ -5,11 +5,10 @@ "main": "dist/index.js", "types": "index.d.ts", "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" }, "author": "", "license": "AGPL", - "dependencies": { - }, "devDependencies": { "@types/node": "^18.15.11" } From 7c1a91aee44fa2059eaab6abfaccc2a8731a0734 Mon Sep 17 00:00:00 2001 From: Gianfranco Paoloni Date: Fri, 7 Jun 2024 18:27:06 -0300 Subject: [PATCH 53/55] chore: fixed build issues (I think) --- hrm-domain/scheduled-tasks/hrm-data-pull/package.json | 10 +++++----- package-lock.json | 3 +++ packages/batch-processing/package.json | 1 - packages/batch-processing/tsconfig.eslint.json | 4 ---- packages/batch-processing/tsconfig.json | 6 ++---- 5 files changed, 10 insertions(+), 14 deletions(-) delete mode 100644 packages/batch-processing/tsconfig.eslint.json diff --git a/hrm-domain/scheduled-tasks/hrm-data-pull/package.json b/hrm-domain/scheduled-tasks/hrm-data-pull/package.json index b591c5cf0..fdeff1a13 100644 --- a/hrm-domain/scheduled-tasks/hrm-data-pull/package.json +++ b/hrm-domain/scheduled-tasks/hrm-data-pull/package.json @@ -10,13 +10,13 @@ "exports": { ".": "./dist/index.js" }, - "dependencies" : { + "dependencies": { "@tech-matters/batch-processing": "^1.0.0", - "@tech-matters/types": "^1.0.0", - "@tech-matters/twilio-worker-auth": "^1.0.0", - "@tech-matters/s3-client": "^1.0.0", "@tech-matters/hrm-core": "^1.0.0", + "@tech-matters/s3-client": "^1.0.0", "@tech-matters/ssm-cache": "^1.0.0", + "@tech-matters/twilio-worker-auth": "^1.0.0", + "@tech-matters/types": "^1.0.0", "date-fns": "^2.28.0" } -} \ No newline at end of file +} diff --git a/package-lock.json b/package-lock.json index 26efabda7..61963842f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -326,6 +326,7 @@ "name": "@tech-matters/hrm-data-pull", "version": "1.0.0", "dependencies": { + "@tech-matters/batch-processing": "^1.0.0", "@tech-matters/hrm-core": "^1.0.0", "@tech-matters/s3-client": "^1.0.0", "@tech-matters/ssm-cache": "^1.0.0", @@ -14401,6 +14402,7 @@ "integrity": "sha512-KYnWoFWgGtWyQEKNnUcb3u8ZtKO8dn5d8u+oGpxPlopqsPyv60U8suDyfk7Z7UtAO6Sk5i1aVcAs9RbaB1n36A==" }, "packages/batch-processing": { + "name": "@tech-matters/batch-processing", "version": "1.0.0", "license": "AGPL", "devDependencies": { @@ -17588,6 +17590,7 @@ "@tech-matters/hrm-data-pull": { "version": "file:hrm-domain/scheduled-tasks/hrm-data-pull", "requires": { + "@tech-matters/batch-processing": "*", "@tech-matters/hrm-core": "^1.0.0", "@tech-matters/s3-client": "^1.0.0", "@tech-matters/ssm-cache": "^1.0.0", diff --git a/packages/batch-processing/package.json b/packages/batch-processing/package.json index fa0c90269..25063b94b 100644 --- a/packages/batch-processing/package.json +++ b/packages/batch-processing/package.json @@ -3,7 +3,6 @@ "version": "1.0.0", "description": "", "main": "dist/index.js", - "types": "index.d.ts", "scripts": { "test": "echo \"Error: no test specified\" && exit 1" }, diff --git a/packages/batch-processing/tsconfig.eslint.json b/packages/batch-processing/tsconfig.eslint.json deleted file mode 100644 index b81b3b862..000000000 --- a/packages/batch-processing/tsconfig.eslint.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "extends": "./tsconfig.json", - "include": ["**/*.ts"] -} diff --git a/packages/batch-processing/tsconfig.json b/packages/batch-processing/tsconfig.json index ec4161db5..43d19bed4 100644 --- a/packages/batch-processing/tsconfig.json +++ b/packages/batch-processing/tsconfig.json @@ -1,9 +1,7 @@ { "extends": "../tsconfig.packages-base.json", - "include": ["src/**/*.ts"], + "include": ["src/*.ts"], "compilerOptions": { "outDir": "./dist", - "allowJs": true, - "skipLibCheck": true } -} +} \ No newline at end of file From 8094150ea6cb1df6b3a546516e4e8901280058fa Mon Sep 17 00:00:00 2001 From: Gianfranco Paoloni Date: Fri, 7 Jun 2024 18:29:48 -0300 Subject: [PATCH 54/55] chore: removed unnecessary dependency --- package-lock.json | 44 ++------------------------ packages/batch-processing/package.json | 5 +-- 2 files changed, 4 insertions(+), 45 deletions(-) diff --git a/package-lock.json b/package-lock.json index 61963842f..6f6b4fae2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13941,12 +13941,6 @@ "node": ">=14.0" } }, - "node_modules/undici-types": { - "version": "5.26.5", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", - "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==", - "dev": true - }, "node_modules/universalify": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.1.2.tgz", @@ -14404,19 +14398,7 @@ "packages/batch-processing": { "name": "@tech-matters/batch-processing", "version": "1.0.0", - "license": "AGPL", - "devDependencies": { - "@types/node": "^18.15.11" - } - }, - "packages/batch-processing/node_modules/@types/node": { - "version": "18.19.34", - "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.34.tgz", - "integrity": "sha512-eXF4pfBNV5DAMKGbI02NnDtWrQ40hAN558/2vvS4gMpMIxaf6JmD7YjnZbq0Q9TDSSkKBamime8ewRoomHdt4g==", - "dev": true, - "dependencies": { - "undici-types": "~5.26.4" - } + "license": "AGPL" }, "packages/elasticsearch-client": { "name": "@tech-matters/elasticsearch-client", @@ -17456,21 +17438,7 @@ } }, "@tech-matters/batch-processing": { - "version": "file:packages/batch-processing", - "requires": { - "@types/node": "^18.15.11" - }, - "dependencies": { - "@types/node": { - "version": "18.19.34", - "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.34.tgz", - "integrity": "sha512-eXF4pfBNV5DAMKGbI02NnDtWrQ40hAN558/2vvS4gMpMIxaf6JmD7YjnZbq0Q9TDSSkKBamime8ewRoomHdt4g==", - "dev": true, - "requires": { - "undici-types": "~5.26.4" - } - } - } + "version": "file:packages/batch-processing" }, "@tech-matters/case-status-transition": { "version": "file:hrm-domain/scheduled-tasks/case-status-transition", @@ -17590,7 +17558,7 @@ "@tech-matters/hrm-data-pull": { "version": "file:hrm-domain/scheduled-tasks/hrm-data-pull", "requires": { - "@tech-matters/batch-processing": "*", + "@tech-matters/batch-processing": "^1.0.0", "@tech-matters/hrm-core": "^1.0.0", "@tech-matters/s3-client": "^1.0.0", "@tech-matters/ssm-cache": "^1.0.0", @@ -25446,12 +25414,6 @@ "@fastify/busboy": "^2.0.0" } }, - "undici-types": { - "version": "5.26.5", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", - "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==", - "dev": true - }, "universalify": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.1.2.tgz", diff --git a/packages/batch-processing/package.json b/packages/batch-processing/package.json index 25063b94b..7b092bf2b 100644 --- a/packages/batch-processing/package.json +++ b/packages/batch-processing/package.json @@ -7,8 +7,5 @@ "test": "echo \"Error: no test specified\" && exit 1" }, "author": "", - "license": "AGPL", - "devDependencies": { - "@types/node": "^18.15.11" - } + "license": "AGPL" } From f0a29d5938b399e2017795a953a9f4581690a05b Mon Sep 17 00:00:00 2001 From: Gianfranco Paoloni Date: Sat, 8 Jun 2024 16:33:27 -0300 Subject: [PATCH 55/55] chore: resolved deps (hopefully) --- hrm-domain/hrm-core/package.json | 2 +- hrm-domain/hrm-service/tsconfig.build.json | 4 ++-- hrm-domain/scheduled-tasks/hrm-data-pull/package.json | 10 +++++----- tsconfig.json | 1 + 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/hrm-domain/hrm-core/package.json b/hrm-domain/hrm-core/package.json index 55747057d..31adfbcdd 100644 --- a/hrm-domain/hrm-core/package.json +++ b/hrm-domain/hrm-core/package.json @@ -23,9 +23,9 @@ }, "homepage": "https://github.com/tech-matters/hrm#readme", "dependencies": { - "@tech-matters/batch-processing": "^1.0.0", "@tech-matters/hrm-search-config": "^1.0.0", "@tech-matters/hrm-types": "^1.0.0", + "@tech-matters/batch-processing": "^1.0.0", "@tech-matters/http": "^1.0.0", "@tech-matters/resources-service": "^1.0.0", "@tech-matters/s3-client": "^1.0.0", diff --git a/hrm-domain/hrm-service/tsconfig.build.json b/hrm-domain/hrm-service/tsconfig.build.json index 578cb0055..da06997ab 100644 --- a/hrm-domain/hrm-service/tsconfig.build.json +++ b/hrm-domain/hrm-service/tsconfig.build.json @@ -3,6 +3,8 @@ "extends": "./tsconfig.base.json", "files": [], "references": [ + { "path": "packages/types" }, + { "path": "packages/batch-processing" }, { "path": "packages/http" }, { "path": "packages/twilio-worker-auth" }, { "path": "packages/s3-client" }, @@ -11,9 +13,7 @@ { "path": "packages/sqs-client" }, { "path": "packages/elasticsearch-client" }, { "path": "packages/twilio-client" }, - { "path": "packages/types" }, { "path": "packages/service-discovery" }, - { "path": "packages/batch-processing" }, { "path": "resources-domain/packages/resources-search-config" }, { "path": "resources-domain/resources-service" }, { "path": "hrm-domain/packages/hrm-types" }, diff --git a/hrm-domain/scheduled-tasks/hrm-data-pull/package.json b/hrm-domain/scheduled-tasks/hrm-data-pull/package.json index fdeff1a13..a2ee1beaa 100644 --- a/hrm-domain/scheduled-tasks/hrm-data-pull/package.json +++ b/hrm-domain/scheduled-tasks/hrm-data-pull/package.json @@ -10,13 +10,13 @@ "exports": { ".": "./dist/index.js" }, - "dependencies": { + "dependencies" : { + "@tech-matters/types": "^1.0.0", "@tech-matters/batch-processing": "^1.0.0", - "@tech-matters/hrm-core": "^1.0.0", + "@tech-matters/twilio-worker-auth": "^1.0.0", "@tech-matters/s3-client": "^1.0.0", + "@tech-matters/hrm-core": "^1.0.0", "@tech-matters/ssm-cache": "^1.0.0", - "@tech-matters/twilio-worker-auth": "^1.0.0", - "@tech-matters/types": "^1.0.0", "date-fns": "^2.28.0" } -} +} \ No newline at end of file diff --git a/tsconfig.json b/tsconfig.json index 804066797..07afb338c 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -4,6 +4,7 @@ // The references must be in order. If package A is a dependency for package B, then A must be referenced before B. For more details see https://www.typescriptlang.org/docs/handbook/project-references.html "references": [ { "path": "packages/types" }, + { "path": "packages/batch-processing" }, { "path": "packages/testing" }, { "path": "packages/http" }, { "path": "packages/job-errors" },