From acc87c3f4a08b5426a1655b3e126d3b42e90cd19 Mon Sep 17 00:00:00 2001 From: Stephen Hand Date: Thu, 24 Aug 2023 15:28:11 +0100 Subject: [PATCH 1/2] Refactor HRM SQL to use a composable function for selecting a full contact --- .../20230824113300-contact-function.js | 93 +++++++++++++++ .../service-tests/contacts.test.ts | 4 +- .../hrm-service/src/case/sql/case-get-sql.ts | 19 +-- .../src/case/sql/case-search-sql.ts | 18 +-- .../src/contact-job/sql/contact-job-sql.ts | 18 +-- .../src/contact/contact-data-access.ts | 8 +- .../src/contact/sql/contact-get-sql.ts | 32 +----- .../src/contact/sql/contact-search-sql.ts | 23 +--- .../src/contact/sql/contact-update-sql.ts | 11 +- .../sql/conversation-media-get-sql.ts | 16 --- .../csam-report/sql/csam-report-get-sql.ts | 16 --- .../src/referral/sql/referral-get-sql.ts | 31 ----- .../unit-tests/case/case-data-access.test.ts | 8 +- .../scripts/khp/importLocationsCsv.ts | 108 ++++-------------- 14 files changed, 151 insertions(+), 254 deletions(-) create mode 100644 hrm-domain/hrm-service/migrations/20230824113300-contact-function.js delete mode 100644 hrm-domain/hrm-service/src/referral/sql/referral-get-sql.ts diff --git a/hrm-domain/hrm-service/migrations/20230824113300-contact-function.js b/hrm-domain/hrm-service/migrations/20230824113300-contact-function.js new file mode 100644 index 000000000..6152e8464 --- /dev/null +++ b/hrm-domain/hrm-service/migrations/20230824113300-contact-function.js @@ -0,0 +1,93 @@ +/** + * Copyright (C) 2021-2023 Technology Matters + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see https://www.gnu.org/licenses/. + */ +'use strict'; + +/** @type {import('sequelize-cli').Migration} */ +module.exports = { + up: async queryInterface => { + await queryInterface.sequelize + .query(`CREATE OR REPLACE FUNCTION "contactRelations"(character varying(255), bigint) +RETURNS TABLE ( + "csamReports" jsonb, + "referrals" jsonb, + "conversationMedia" jsonb +) AS $$ + SELECT reports."csamReports", joinedReferrals."referrals", media."conversationMedia" FROM ( + SELECT COALESCE(jsonb_agg(to_jsonb(r)), '[]') AS "csamReports" + FROM "CSAMReports" r + WHERE r."contactId" = $2 AND r."accountSid" = $1 AND r."acknowledged" = TRUE + ) reports + LEFT JOIN LATERAL ( + SELECT COALESCE(jsonb_agg(to_jsonb(referral)), '[]') AS "referrals" + FROM "Referrals" referral + WHERE referral."contactId" = $2 AND referral."accountSid" = $1 + ) joinedReferrals ON true + LEFT JOIN LATERAL ( + SELECT COALESCE(jsonb_agg(to_jsonb(cm)), '[]') AS "conversationMedia" + FROM "ConversationMedias" cm + WHERE cm."contactId" = $2 AND cm."accountSid" = $1 + ) media ON true +$$ +LANGUAGE SQL +STABLE; + `); + await queryInterface.sequelize + .query(`CREATE OR REPLACE FUNCTION "permittedFullContacts"(character varying(255), character varying(255)) +RETURNS TABLE ( + id integer, + "createdAt" timestamp with time zone, + "updatedAt" timestamp with time zone, + "rawJson" jsonb, + "queueName" character varying(255), + "twilioWorkerId" character varying(255), + helpline character varying(255), + "number" character varying(255), + channel character varying(255), + "conversationDuration" integer, + "caseId" integer, + "accountSid" character varying(255), + "timeOfContact" timestamp with time zone, + "taskId" character varying(255), + "createdBy" character varying(255), + "channelSid" character varying(255), + "serviceSid" character varying(255), + "updatedBy" text, + "csamReports" jsonb, + "referrals" jsonb, + "conversationMedia" jsonb +) AS $$ + SELECT c.*, relations."csamReports", relations."referrals", relations."conversationMedia" + FROM "Contacts" c + LEFT JOIN LATERAL "contactRelations"($1, c.id) relations ON true + WHERE c."accountSid" = $1 AND ($2 IS NULL OR c."twilioWorkerId" = $2) +$$ +LANGUAGE SQL +STABLE; + `); + console.log('Function "permittedFullContacts" created'); + }, + + down: async queryInterface => { + await queryInterface.sequelize.query( + `DROP FUNCTION IF EXISTS public."permittedFullContacts"(character varying(255), character varying(255))`, + ); + console.log('Function "permittedFullContacts" dropped'); + await queryInterface.sequelize.query( + `DROP FUNCTION IF EXISTS public."contactRelations"(character varying(255), bigint)`, + ); + console.log('Function "contactRelations" dropped'); + }, +}; diff --git a/hrm-domain/hrm-service/service-tests/contacts.test.ts b/hrm-domain/hrm-service/service-tests/contacts.test.ts index fa1db5108..19eb8a49e 100644 --- a/hrm-domain/hrm-service/service-tests/contacts.test.ts +++ b/hrm-domain/hrm-service/service-tests/contacts.test.ts @@ -54,7 +54,7 @@ import { mockingProxy, mockSuccessfulTwilioAuthentication } from '@tech-matters/ import * as contactJobDataAccess from '../src/contact-job/contact-job-data-access'; import { chatChannels } from '../src/contact/channelTypes'; import * as contactInsertSql from '../src/contact/sql/contact-insert-sql'; -import { selectSingleContactByTaskId } from '../src/contact/sql/contact-get-sql'; +import { SELECT_SINGLE_CONTACT_BY_TASKSID } from '../src/contact/sql/contact-get-sql'; import { ruleFileWithOneActionOverride } from './permissions-overrides'; import * as csamReportApi from '../src/csam-report/csam-report'; import * as referralDB from '../src/referral/referral-data-access'; @@ -125,7 +125,7 @@ const cleanupReferrals = () => // eslint-disable-next-line @typescript-eslint/no-shadow const getContactByTaskId = (taskId: string, accountSid: string) => - db.oneOrNone(selectSingleContactByTaskId('Contacts'), { accountSid, taskId }); + db.oneOrNone(SELECT_SINGLE_CONTACT_BY_TASKSID, { accountSid, taskId }); // eslint-disable-next-line @typescript-eslint/no-shadow const deleteContactById = (id: number, accountSid: string) => diff --git a/hrm-domain/hrm-service/src/case/sql/case-get-sql.ts b/hrm-domain/hrm-service/src/case/sql/case-get-sql.ts index 966a57ae2..cff6477ff 100644 --- a/hrm-domain/hrm-service/src/case/sql/case-get-sql.ts +++ b/hrm-domain/hrm-service/src/case/sql/case-get-sql.ts @@ -14,10 +14,6 @@ * along with this program. If not, see https://www.gnu.org/licenses/. */ -import { selectCoalesceConversationMediasByContactId } from '../../conversation-media/sql/conversation-media-get-sql'; -import { selectCoalesceCsamReportsByContactId } from '../../csam-report/sql/csam-report-get-sql'; -import { selectCoalesceReferralsByContactId } from '../../referral/sql/referral-get-sql'; - const ID_WHERE_CLAUSE = `WHERE "cases"."accountSid" = $ AND "cases"."id" = $`; export const selectSingleCaseByIdSql = (tableName: string) => `SELECT @@ -26,18 +22,9 @@ export const selectSingleCaseByIdSql = (tableName: string) => `SELECT contacts."connectedContacts" FROM "${tableName}" AS cases LEFT JOIN LATERAL ( - SELECT COALESCE(jsonb_agg(to_jsonb(c) || to_jsonb(joinedReports) || to_jsonb(joinedReferrals) || to_jsonb(joinedConversationMedia)), '[]') AS "connectedContacts" - FROM "Contacts" c - LEFT JOIN LATERAL ( - ${selectCoalesceCsamReportsByContactId('c')} - ) joinedReports ON true - LEFT JOIN LATERAL ( - ${selectCoalesceReferralsByContactId('c')} - ) joinedReferrals ON true - LEFT JOIN LATERAL ( - ${selectCoalesceConversationMediasByContactId('c')} - ) joinedConversationMedia ON true - WHERE c."caseId" = cases.id AND c."accountSid" = cases."accountSid" + SELECT COALESCE(jsonb_agg(DISTINCT contacts.*) FILTER (WHERE contacts."caseId" IS NOT NULL), '[]') AS "connectedContacts" + FROM "permittedFullContacts"(cases."accountSid", NULL) AS contacts + WHERE contacts."caseId" = cases.id ) contacts ON true LEFT JOIN LATERAL ( SELECT COALESCE(jsonb_agg(to_jsonb(cs) ORDER BY cs."createdAt"), '[]') AS "caseSections" diff --git a/hrm-domain/hrm-service/src/case/sql/case-search-sql.ts b/hrm-domain/hrm-service/src/case/sql/case-search-sql.ts index c364e2b98..eb4cecc6a 100644 --- a/hrm-domain/hrm-service/src/case/sql/case-search-sql.ts +++ b/hrm-domain/hrm-service/src/case/sql/case-search-sql.ts @@ -17,9 +17,6 @@ import { pgp } from '../../connection-pool'; import { SELECT_CASE_SECTIONS } from './case-sections-sql'; import { CaseListFilters, DateExistsCondition, DateFilter } from '../case-data-access'; -import { leftJoinCsamReportsOnFK } from '../../csam-report/sql/csam-report-get-sql'; -import { leftJoinReferralsOnFK } from '../../referral/sql/referral-get-sql'; -import { leftJoinConversationMediasOnFK } from '../../conversation-media/sql/conversation-media-get-sql'; export const OrderByDirection = { ascendingNullsLast: 'ASC NULLS LAST', @@ -65,19 +62,8 @@ const generateOrderByClause = (clauses: OrderByClauseItem[]): string => { }; const SELECT_CONTACTS = `SELECT COALESCE(jsonb_agg(DISTINCT contacts.*) FILTER (WHERE contacts."caseId" IS NOT NULL), '[]') AS "connectedContacts" -FROM ( - SELECT - c.*, - COALESCE(jsonb_agg(DISTINCT r.*) FILTER (WHERE r.id IS NOT NULL), '[]') AS "csamReports", - COALESCE(jsonb_agg(DISTINCT referral.*) FILTER (WHERE referral IS NOT NULL), '[]') AS "referrals", - COALESCE(jsonb_agg(DISTINCT cm.*) FILTER (WHERE cm IS NOT NULL), '[]') AS "conversationMedia" - FROM "Contacts" c - ${leftJoinCsamReportsOnFK('c')} - ${leftJoinReferralsOnFK('c')} - ${leftJoinConversationMediasOnFK('c')} - WHERE c."caseId" = "cases".id AND c."accountSid" = "cases"."accountSid" - GROUP BY c."accountSid", c.id -) AS contacts WHERE contacts."caseId" = cases.id AND contacts."accountSid" = cases."accountSid"`; +FROM "permittedFullContacts"(cases."accountSid", NULL) AS contacts +WHERE contacts."caseId" = cases.id AND contacts."accountSid" = cases."accountSid"`; const enum FilterableDateField { CREATED_AT = 'cases."createdAt"::TIMESTAMP WITH TIME ZONE', diff --git a/hrm-domain/hrm-service/src/contact-job/sql/contact-job-sql.ts b/hrm-domain/hrm-service/src/contact-job/sql/contact-job-sql.ts index 4d82fb510..39bd129ce 100644 --- a/hrm-domain/hrm-service/src/contact-job/sql/contact-job-sql.ts +++ b/hrm-domain/hrm-service/src/contact-job/sql/contact-job-sql.ts @@ -14,8 +14,6 @@ * along with this program. If not, see https://www.gnu.org/licenses/. */ -import { selectContactsWithRelations } from '../../contact/sql/contact-get-sql'; - export enum ContactJobCleanupStatus { NOT_READY = 'not_ready', PENDING = 'pending', @@ -49,11 +47,9 @@ WITH due AS ( AND "completed" IS NOT NULL AND "completed" < (current_timestamp - interval '$ day') ) - SELECT due.*, to_jsonb(contacts.*) AS "resource" - FROM due LEFT JOIN LATERAL ( - ${selectContactsWithRelations( - 'Contacts', - )} WHERE c."accountSid" = due."accountSid" AND c."id" = due."contactId") AS contacts ON true + SELECT due.*, to_jsonb(contacts.*) AS "resource"FROM due + LEFT JOIN public."permittedFullContacts"(due."accountSid", NULL) AS contacts + ON contacts."id" = due."contactId" `; export const PENDING_CLEANUP_JOB_ACCOUNT_SIDS_SQL = ` @@ -69,11 +65,9 @@ export const PULL_DUE_JOBS_SQL = ` UPDATE "ContactJobs" SET "lastAttempt" = CURRENT_TIMESTAMP, "numberOfAttempts" = "numberOfAttempts" + 1 WHERE "completed" IS NULL AND "numberOfAttempts" < $ AND ("lastAttempt" IS NULL OR "lastAttempt" <= $::TIMESTAMP WITH TIME ZONE) RETURNING * ) - SELECT due.*, to_jsonb(contacts.*) AS "resource" - FROM due LEFT JOIN LATERAL ( - ${selectContactsWithRelations( - 'Contacts', - )} WHERE c."accountSid" = due."accountSid" AND c."id" = due."contactId") AS contacts ON true + SELECT due.*, to_jsonb(contacts.*) AS "resource" FROM due + LEFT JOIN public."permittedFullContacts"(due."accountSid", NULL) AS contacts + ON contacts."id" = due."contactId" `; export const UPDATE_JOB_CLEANUP_ACTIVE_SQL = ` diff --git a/hrm-domain/hrm-service/src/contact/contact-data-access.ts b/hrm-domain/hrm-service/src/contact/contact-data-access.ts index b24256746..ed3216585 100644 --- a/hrm-domain/hrm-service/src/contact/contact-data-access.ts +++ b/hrm-domain/hrm-service/src/contact/contact-data-access.ts @@ -19,8 +19,8 @@ import { UPDATE_CASEID_BY_ID, UPDATE_RAWJSON_BY_ID } from './sql/contact-update- import { SELECT_CONTACT_SEARCH } from './sql/contact-search-sql'; import { endOfDay, parseISO, startOfDay } from 'date-fns'; import { - selectSingleContactByIdSql, - selectSingleContactByTaskId, + SELECT_SINGLE_CONTACT_BY_ID, + SELECT_SINGLE_CONTACT_BY_TASKSID, } from './sql/contact-get-sql'; import { insertContactSql, NewContactRecord } from './sql/contact-insert-sql'; import { PersonInformation, ReferralWithoutContactId } from './contact-json'; @@ -160,7 +160,7 @@ export const create = ) => { if (newContact.taskId) { const existingContact: Contact = await conn.oneOrNone( - selectSingleContactByTaskId('Contacts'), + SELECT_SINGLE_CONTACT_BY_TASKSID, { accountSid, taskId: newContact.taskId, @@ -226,7 +226,7 @@ export const connectToCase = async ( export const getById = async (accountSid: string, contactId: number): Promise => db.task(async connection => - connection.oneOrNone(selectSingleContactByIdSql('Contacts'), { + connection.oneOrNone(SELECT_SINGLE_CONTACT_BY_ID, { accountSid, contactId, }), diff --git a/hrm-domain/hrm-service/src/contact/sql/contact-get-sql.ts b/hrm-domain/hrm-service/src/contact/sql/contact-get-sql.ts index e7a2c196e..9a48c5c0e 100644 --- a/hrm-domain/hrm-service/src/contact/sql/contact-get-sql.ts +++ b/hrm-domain/hrm-service/src/contact/sql/contact-get-sql.ts @@ -14,32 +14,12 @@ * along with this program. If not, see https://www.gnu.org/licenses/. */ -import { selectCoalesceConversationMediasByContactId } from '../../conversation-media/sql/conversation-media-get-sql'; -import { selectCoalesceCsamReportsByContactId } from '../../csam-report/sql/csam-report-get-sql'; -import { selectCoalesceReferralsByContactId } from '../../referral/sql/referral-get-sql'; +export const SELECT_SINGLE_CONTACT_BY_ID = ` + SELECT c.* FROM "permittedFullContacts"($, NULL) c + WHERE c."id" = $`; -const ID_WHERE_CLAUSE = `WHERE c."accountSid" = $ AND c."id" = $`; -const TASKID_WHERE_CLAUSE = `WHERE c."accountSid" = $ AND c."taskId" = $`; - -export const selectContactsWithRelations = (table: string) => ` - SELECT c.*, reports."csamReports", joinedReferrals."referrals", media."conversationMedia" - FROM "${table}" c - LEFT JOIN LATERAL ( - ${selectCoalesceCsamReportsByContactId('c')} - ) reports ON true - LEFT JOIN LATERAL ( - ${selectCoalesceReferralsByContactId('c')} - ) joinedReferrals ON true - LEFT JOIN LATERAL ( - ${selectCoalesceConversationMediasByContactId('c')} - ) media ON true`; - -export const selectSingleContactByIdSql = (table: string) => ` - ${selectContactsWithRelations(table)} - ${ID_WHERE_CLAUSE}`; - -export const selectSingleContactByTaskId = (table: string) => ` - ${selectContactsWithRelations(table)} - ${TASKID_WHERE_CLAUSE} +export const SELECT_SINGLE_CONTACT_BY_TASKSID = ` + SELECT c.* FROM "permittedFullContacts"($, NULL) c + WHERE c."taskId" = $ -- only take the latest, this ORDER / LIMIT clause would be redundant ORDER BY c."createdAt" DESC LIMIT 1`; diff --git a/hrm-domain/hrm-service/src/contact/sql/contact-search-sql.ts b/hrm-domain/hrm-service/src/contact/sql/contact-search-sql.ts index d5394d8dd..738dd2453 100644 --- a/hrm-domain/hrm-service/src/contact/sql/contact-search-sql.ts +++ b/hrm-domain/hrm-service/src/contact/sql/contact-search-sql.ts @@ -14,26 +14,12 @@ * along with this program. If not, see https://www.gnu.org/licenses/. */ -import { selectCoalesceConversationMediasByContactId } from '../../conversation-media/sql/conversation-media-get-sql'; -import { selectCoalesceCsamReportsByContactId } from '../../csam-report/sql/csam-report-get-sql'; -import { selectCoalesceReferralsByContactId } from '../../referral/sql/referral-get-sql'; - export const SELECT_CONTACT_SEARCH = ` SELECT (count(*) OVER())::INTEGER AS "totalCount", - contacts.*, reports."csamReports", joinedReferrals."referrals", media."conversationMedia" - FROM "Contacts" contacts - LEFT JOIN LATERAL ( - ${selectCoalesceCsamReportsByContactId('contacts')} - ) reports ON true - LEFT JOIN LATERAL ( - ${selectCoalesceReferralsByContactId('contacts')} - ) joinedReferrals ON true - LEFT JOIN LATERAL ( - ${selectCoalesceConversationMediasByContactId('contacts')} - ) media ON true - WHERE contacts."accountSid" = $ - AND ($ IS NULL OR contacts."helpline" = $) + contacts.* + FROM "permittedFullContacts"($, $) contacts + WHERE ($ IS NULL OR contacts."helpline" = $) AND ( ($ IS NULL AND $ IS NULL) OR ( @@ -63,9 +49,6 @@ export const SELECT_CONTACT_SEARCH = ` ) ) ) - AND ( - $ IS NULL OR contacts."twilioWorkerId" = $ - ) AND ( $ IS NULL OR "number" ILIKE $ diff --git a/hrm-domain/hrm-service/src/contact/sql/contact-update-sql.ts b/hrm-domain/hrm-service/src/contact/sql/contact-update-sql.ts index aa84d1de8..d4a67b972 100644 --- a/hrm-domain/hrm-service/src/contact/sql/contact-update-sql.ts +++ b/hrm-domain/hrm-service/src/contact/sql/contact-update-sql.ts @@ -14,10 +14,13 @@ * along with this program. If not, see https://www.gnu.org/licenses/. */ -import { selectSingleContactByIdSql } from './contact-get-sql'; - const ID_WHERE_CLAUSE = `WHERE "accountSid" = $ AND "id"=$`; +const SELECT_FULL_CONTACT_WITH_UPDATED = ` + SELECT c.*, relations."csamReports", relations."referrals", relations."conversationMedia" + FROM "updated" c LEFT JOIN LATERAL "contactRelations"($, c.id) relations ON true + `; + export const UPDATE_RAWJSON_BY_ID = `WITH updated AS ( UPDATE "Contacts" SET "rawJson" = COALESCE("rawJson", '{}'::JSONB) @@ -53,7 +56,7 @@ SET "rawJson" = COALESCE("rawJson", '{}'::JSONB) ${ID_WHERE_CLAUSE} RETURNING * ) -${selectSingleContactByIdSql('updated')} +${SELECT_FULL_CONTACT_WITH_UPDATED} `; export const UPDATE_CASEID_BY_ID = `WITH updated AS ( @@ -63,5 +66,5 @@ SET ${ID_WHERE_CLAUSE} RETURNING * ) -${selectSingleContactByIdSql('updated')} +${SELECT_FULL_CONTACT_WITH_UPDATED} `; diff --git a/hrm-domain/hrm-service/src/conversation-media/sql/conversation-media-get-sql.ts b/hrm-domain/hrm-service/src/conversation-media/sql/conversation-media-get-sql.ts index 044cd26c1..76629710e 100644 --- a/hrm-domain/hrm-service/src/conversation-media/sql/conversation-media-get-sql.ts +++ b/hrm-domain/hrm-service/src/conversation-media/sql/conversation-media-get-sql.ts @@ -28,19 +28,3 @@ export const selectConversationMediaByContactIdSql = ` FROM "ConversationMedias" cm ${CONTACT_ID_WHERE_CLAUSE} `; - -// Queries used in other modules for JOINs - -const onFkFilteredClause = (contactAlias: string) => ` - cm."contactId" = "${contactAlias}".id AND cm."accountSid" = "${contactAlias}"."accountSid" -`; - -export const selectCoalesceConversationMediasByContactId = (contactAlias: string) => ` - SELECT COALESCE(jsonb_agg(to_jsonb(cm)), '[]') AS "conversationMedia" - FROM "ConversationMedias" cm - WHERE ${onFkFilteredClause(contactAlias)} -`; - -export const leftJoinConversationMediasOnFK = (contactAlias: string) => ` - LEFT JOIN "ConversationMedias" cm ON ${onFkFilteredClause(contactAlias)} -`; diff --git a/hrm-domain/hrm-service/src/csam-report/sql/csam-report-get-sql.ts b/hrm-domain/hrm-service/src/csam-report/sql/csam-report-get-sql.ts index 72f59b6fd..13672ef16 100644 --- a/hrm-domain/hrm-service/src/csam-report/sql/csam-report-get-sql.ts +++ b/hrm-domain/hrm-service/src/csam-report/sql/csam-report-get-sql.ts @@ -28,19 +28,3 @@ export const selectCsamReportsByContactIdSql = ` FROM "CSAMReports" r ${CONTACT_ID_WHERE_CLAUSE} `; - -// Queries used in other modules for JOINs - -const onFkFilteredClause = (contactAlias: string) => ` - r."contactId" = "${contactAlias}".id AND r."accountSid" = "${contactAlias}"."accountSid" AND r."acknowledged" = TRUE -`; - -export const selectCoalesceCsamReportsByContactId = (contactAlias: string) => ` - SELECT COALESCE(jsonb_agg(to_jsonb(r)), '[]') AS "csamReports" - FROM "CSAMReports" r - WHERE ${onFkFilteredClause(contactAlias)} -`; - -export const leftJoinCsamReportsOnFK = (contactAlias: string) => ` - LEFT JOIN "CSAMReports" r ON ${onFkFilteredClause(contactAlias)} -`; diff --git a/hrm-domain/hrm-service/src/referral/sql/referral-get-sql.ts b/hrm-domain/hrm-service/src/referral/sql/referral-get-sql.ts deleted file mode 100644 index 334698826..000000000 --- a/hrm-domain/hrm-service/src/referral/sql/referral-get-sql.ts +++ /dev/null @@ -1,31 +0,0 @@ -/** - * Copyright (C) 2021-2023 Technology Matters - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see https://www.gnu.org/licenses/. - */ - -// Queries used in other modules for JOINs - -const onFkFilteredClause = (contactAlias: string) => ` - referral."contactId" = "${contactAlias}".id AND referral."accountSid" = "${contactAlias}"."accountSid" -`; - -export const selectCoalesceReferralsByContactId = (contactAlias: string) => ` - SELECT COALESCE(jsonb_agg(to_jsonb(referral)), '[]') AS "referrals" - FROM "Referrals" referral - WHERE ${onFkFilteredClause(contactAlias)} -`; - -export const leftJoinReferralsOnFK = (contactAlias: string) => ` - LEFT JOIN "Referrals" referral ON ${onFkFilteredClause(contactAlias)} -`; diff --git a/hrm-domain/hrm-service/unit-tests/case/case-data-access.test.ts b/hrm-domain/hrm-service/unit-tests/case/case-data-access.test.ts index 4718a1ba8..376a0b293 100644 --- a/hrm-domain/hrm-service/unit-tests/case/case-data-access.test.ts +++ b/hrm-domain/hrm-service/unit-tests/case/case-data-access.test.ts @@ -62,7 +62,7 @@ describe('getById', () => { const result = await caseDb.getById(caseId, accountSid); expect(oneOrNoneSpy).toHaveBeenCalledWith( - expect.stringContaining('CSAMReports'), + expect.stringContaining('permittedFullContacts'), expect.objectContaining({ accountSid, caseId }), ); expect(result).not.toBeDefined(); @@ -316,8 +316,7 @@ describe('search', () => { }), ); const statementExecuted = getSqlStatement(anySpy); - expect(statementExecuted).toContain('Contacts'); - expect(statementExecuted).toContain('CSAMReports'); + expect(statementExecuted).toContain('permittedFullContacts'); expect(result.count).toEqual(1337); expect(result.cases).toStrictEqual(expectedResult); }, @@ -350,8 +349,7 @@ describe('update', () => { const updateSql = getSqlStatement(noneSpy, 1); const selectSql = getSqlStatement(oneOrNoneSpy); expect(selectSql).toContain('Cases'); - expect(selectSql).toContain('Contacts'); - expect(selectSql).toContain('CSAMReports'); + expect(selectSql).toContain('permittedFullContacts'); expectValuesInSql(updateSql, { info: caseUpdate.info, status: caseUpdate.status }); expect(oneOrNoneSpy).toHaveBeenCalledWith( expect.any(String), diff --git a/resources-domain/resources-service/scripts/khp/importLocationsCsv.ts b/resources-domain/resources-service/scripts/khp/importLocationsCsv.ts index bc1421017..5c85bd5ad 100644 --- a/resources-domain/resources-service/scripts/khp/importLocationsCsv.ts +++ b/resources-domain/resources-service/scripts/khp/importLocationsCsv.ts @@ -17,7 +17,6 @@ // eslint-disable-next-line import/no-extraneous-dependencies import { parse } from 'csv-parse'; import fs from 'fs'; -import { pgp } from '../../src/connection-pool'; const CANADIAN_PROVINCE_NAME_CODE_MAP = { Alberta: ['AB', 48], @@ -35,22 +34,6 @@ const CANADIAN_PROVINCE_NAME_CODE_MAP = { Yukon: ['YT', 60], } as const; -const CANADIAN_PROVINCE_CODE_FR_MAP = { - AB: 'Alberta', - BC: 'Colombie-Britannique', - NL: 'Terre-Neuve-et-Labrador', - PE: 'Île-du-Prince-Édouard', - NS: 'Nouvelle-Écosse', - NB: 'Nouveau-Brunswick', - ON: 'Ontario', - MB: 'Manitoba', - SK: 'Saskatchewan', - YT: 'Yukon', - NT: 'Territoires du Nord-Ouest', - NU: 'Nunavut', - QC: 'Québec', -} as const; - type FilterOption = { value: string; label: string; @@ -62,92 +45,45 @@ const main = async () => { process.exit(1); } const accountSid = process.argv[2]; - const targetFilePath = `./reference-data/khp_cities_with_codes_20230622_${accountSid}.sql`; - const targetJsonCitiesFilePath = `./reference-data/khp_cities_with_codes_20230622_${accountSid}.json`; + const targetFilePath = `./reference-data/khp_cities_20230822_${accountSid}.sql`; + const targetJsonCitiesFilePath = `./reference-data/khp_cities_20230822_${accountSid}.json`; const targetJsonProvincesFilePath = `./reference-data/khp_provinces_20230622_${accountSid}.json`; const sqlFile = fs.createWriteStream(targetFilePath); const provincesJson: FilterOption[] = []; const citiesJson: FilterOption[] = []; const csvLines = fs - .createReadStream('./reference-data/khp_cities_with_codes_20230622.csv') + .createReadStream('./reference-data/khp_cities_20230822.csv') .pipe(parse({ fromLine: 2 })); sqlFile.write(`--- PROVINCES ---\n\n`); - Object.entries(CANADIAN_PROVINCE_NAME_CODE_MAP).forEach( - ([name, [code, geographicCode]]) => { - sqlFile.write( - pgp.as.format( - ` -INSERT INTO resources."ResourceReferenceStringAttributeValues" ("accountSid", "list", "id", "value", "language", "info") VALUES ($, 'provinces', $, $, 'en', $) -ON CONFLICT DO NOTHING; -INSERT INTO resources."ResourceReferenceStringAttributeValues" ("accountSid", "list", "id", "value", "language", "info") VALUES ($, 'provinces', $, $, 'fr', $) -ON CONFLICT DO NOTHING; -UPDATE resources."ResourceReferenceStringAttributes" SET "referenceId" = $ WHERE "accountSid" = $ AND "list" = 'provinces' AND "referenceId" = $; -UPDATE resources."ResourceReferenceStringAttributes" SET "referenceId" = $ WHERE "accountSid" = $ AND "list" = 'provinces' AND "referenceId" = $; -DELETE FROM resources."ResourceReferenceStringAttributeValues" WHERE "accountSid" = $ AND "list" = 'provinces' AND "id" IN ($, $); -`, - { - accountSid, - id: `CA-${geographicCode}-en`, - idFr: `CA-${geographicCode}-fr`, - value: `CA/${code}`, - info: { name, geographicCode }, - infoFr: { name: CANADIAN_PROVINCE_CODE_FR_MAP[code] }, - oldId: `CA-${code}-en`, - oldIdFr: `CA-${code}-fr`, - }, - ), - ); - provincesJson.push({ - label: name, - value: `CA/${code}`, - }); - }, - ); + Object.entries(CANADIAN_PROVINCE_NAME_CODE_MAP).forEach(([name, [code]]) => { + provincesJson.push({ + label: name, + value: `CA/${code}`, + }); + }); sqlFile.write('\n\n--- CITIES ---\n\n'); for await (const line of csvLines) { - const [geographicCode, cityEn, cityFr, csdType, , province] = line as string[]; + const [, province, , cityEn] = line as string[]; const [provinceCode] = CANADIAN_PROVINCE_NAME_CODE_MAP[ province as keyof typeof CANADIAN_PROVINCE_NAME_CODE_MAP ]; - const sqlStatement = pgp.as.format( - ` -INSERT INTO resources."ResourceReferenceStringAttributeValues" ("accountSid", "list", "id", "value", "language", "info") VALUES ($, 'cities', $, $, 'en', $) -ON CONFLICT DO NOTHING; -INSERT INTO resources."ResourceReferenceStringAttributeValues" ("accountSid", "list", "id", "value", "language", "info") VALUES ($, 'cities', $, $, 'fr', $) -ON CONFLICT DO NOTHING; -UPDATE resources."ResourceReferenceStringAttributes" SET "referenceId" = $ WHERE "accountSid" = $ AND "list" = 'cities' AND "referenceId" = $; -UPDATE resources."ResourceReferenceStringAttributes" SET "referenceId" = $ WHERE "accountSid" = $ AND "list" = 'cities' AND "referenceId" = $; -DELETE FROM resources."ResourceReferenceStringAttributeValues" WHERE "accountSid" = $ AND "list" = 'provinces' AND "id" IN ($, $); -`, - { - accountSid: process.argv[2], - id: `CA-${geographicCode}-en`, - idFr: `CA-${geographicCode}-fr`, - value: `CA/${provinceCode}/${cityEn}`, - info: { name: cityEn, geographicCode, csdType }, - infoFr: { name: cityFr, geographicCode, csdType }, - oldId: `CA-${provinceCode}-${cityEn}-en`, - oldIdFr: `CA-${provinceCode}-${cityEn}-fr`, - }, - ); - sqlFile.write(sqlStatement); - if ( - csdType === 'Town' || - csdType === 'City' || - csdType === 'Ville' || - csdType === 'Municipalité' - ) { - citiesJson.push({ - label: cityEn, - value: `CA/${provinceCode}/${cityEn}`, - }); - } + citiesJson.push({ + label: cityEn, + value: `CA/${provinceCode}/${cityEn}`, + }); } sqlFile.end(); - fs.writeFileSync(targetJsonCitiesFilePath, JSON.stringify(citiesJson, null, 2)); + fs.writeFileSync( + targetJsonCitiesFilePath, + JSON.stringify( + Object.values(Object.fromEntries(citiesJson.map(c => [c.value, c]))), + null, + 2, + ), + ); fs.writeFileSync(targetJsonProvincesFilePath, JSON.stringify(provincesJson, null, 2)); }; From f7d86437d32684a13b5fe72f471ecad607e7c4b4 Mon Sep 17 00:00:00 2001 From: Stephen Hand Date: Thu, 24 Aug 2023 15:48:57 +0100 Subject: [PATCH 2/2] Oops, revert changes that shouldn't have been pushed --- .../scripts/khp/importLocationsCsv.ts | 108 ++++++++++++++---- 1 file changed, 86 insertions(+), 22 deletions(-) diff --git a/resources-domain/resources-service/scripts/khp/importLocationsCsv.ts b/resources-domain/resources-service/scripts/khp/importLocationsCsv.ts index 5c85bd5ad..bc1421017 100644 --- a/resources-domain/resources-service/scripts/khp/importLocationsCsv.ts +++ b/resources-domain/resources-service/scripts/khp/importLocationsCsv.ts @@ -17,6 +17,7 @@ // eslint-disable-next-line import/no-extraneous-dependencies import { parse } from 'csv-parse'; import fs from 'fs'; +import { pgp } from '../../src/connection-pool'; const CANADIAN_PROVINCE_NAME_CODE_MAP = { Alberta: ['AB', 48], @@ -34,6 +35,22 @@ const CANADIAN_PROVINCE_NAME_CODE_MAP = { Yukon: ['YT', 60], } as const; +const CANADIAN_PROVINCE_CODE_FR_MAP = { + AB: 'Alberta', + BC: 'Colombie-Britannique', + NL: 'Terre-Neuve-et-Labrador', + PE: 'Île-du-Prince-Édouard', + NS: 'Nouvelle-Écosse', + NB: 'Nouveau-Brunswick', + ON: 'Ontario', + MB: 'Manitoba', + SK: 'Saskatchewan', + YT: 'Yukon', + NT: 'Territoires du Nord-Ouest', + NU: 'Nunavut', + QC: 'Québec', +} as const; + type FilterOption = { value: string; label: string; @@ -45,45 +62,92 @@ const main = async () => { process.exit(1); } const accountSid = process.argv[2]; - const targetFilePath = `./reference-data/khp_cities_20230822_${accountSid}.sql`; - const targetJsonCitiesFilePath = `./reference-data/khp_cities_20230822_${accountSid}.json`; + const targetFilePath = `./reference-data/khp_cities_with_codes_20230622_${accountSid}.sql`; + const targetJsonCitiesFilePath = `./reference-data/khp_cities_with_codes_20230622_${accountSid}.json`; const targetJsonProvincesFilePath = `./reference-data/khp_provinces_20230622_${accountSid}.json`; const sqlFile = fs.createWriteStream(targetFilePath); const provincesJson: FilterOption[] = []; const citiesJson: FilterOption[] = []; const csvLines = fs - .createReadStream('./reference-data/khp_cities_20230822.csv') + .createReadStream('./reference-data/khp_cities_with_codes_20230622.csv') .pipe(parse({ fromLine: 2 })); sqlFile.write(`--- PROVINCES ---\n\n`); - Object.entries(CANADIAN_PROVINCE_NAME_CODE_MAP).forEach(([name, [code]]) => { - provincesJson.push({ - label: name, - value: `CA/${code}`, - }); - }); + Object.entries(CANADIAN_PROVINCE_NAME_CODE_MAP).forEach( + ([name, [code, geographicCode]]) => { + sqlFile.write( + pgp.as.format( + ` +INSERT INTO resources."ResourceReferenceStringAttributeValues" ("accountSid", "list", "id", "value", "language", "info") VALUES ($, 'provinces', $, $, 'en', $) +ON CONFLICT DO NOTHING; +INSERT INTO resources."ResourceReferenceStringAttributeValues" ("accountSid", "list", "id", "value", "language", "info") VALUES ($, 'provinces', $, $, 'fr', $) +ON CONFLICT DO NOTHING; +UPDATE resources."ResourceReferenceStringAttributes" SET "referenceId" = $ WHERE "accountSid" = $ AND "list" = 'provinces' AND "referenceId" = $; +UPDATE resources."ResourceReferenceStringAttributes" SET "referenceId" = $ WHERE "accountSid" = $ AND "list" = 'provinces' AND "referenceId" = $; +DELETE FROM resources."ResourceReferenceStringAttributeValues" WHERE "accountSid" = $ AND "list" = 'provinces' AND "id" IN ($, $); +`, + { + accountSid, + id: `CA-${geographicCode}-en`, + idFr: `CA-${geographicCode}-fr`, + value: `CA/${code}`, + info: { name, geographicCode }, + infoFr: { name: CANADIAN_PROVINCE_CODE_FR_MAP[code] }, + oldId: `CA-${code}-en`, + oldIdFr: `CA-${code}-fr`, + }, + ), + ); + provincesJson.push({ + label: name, + value: `CA/${code}`, + }); + }, + ); sqlFile.write('\n\n--- CITIES ---\n\n'); for await (const line of csvLines) { - const [, province, , cityEn] = line as string[]; + const [geographicCode, cityEn, cityFr, csdType, , province] = line as string[]; const [provinceCode] = CANADIAN_PROVINCE_NAME_CODE_MAP[ province as keyof typeof CANADIAN_PROVINCE_NAME_CODE_MAP ]; - citiesJson.push({ - label: cityEn, - value: `CA/${provinceCode}/${cityEn}`, - }); + const sqlStatement = pgp.as.format( + ` +INSERT INTO resources."ResourceReferenceStringAttributeValues" ("accountSid", "list", "id", "value", "language", "info") VALUES ($, 'cities', $, $, 'en', $) +ON CONFLICT DO NOTHING; +INSERT INTO resources."ResourceReferenceStringAttributeValues" ("accountSid", "list", "id", "value", "language", "info") VALUES ($, 'cities', $, $, 'fr', $) +ON CONFLICT DO NOTHING; +UPDATE resources."ResourceReferenceStringAttributes" SET "referenceId" = $ WHERE "accountSid" = $ AND "list" = 'cities' AND "referenceId" = $; +UPDATE resources."ResourceReferenceStringAttributes" SET "referenceId" = $ WHERE "accountSid" = $ AND "list" = 'cities' AND "referenceId" = $; +DELETE FROM resources."ResourceReferenceStringAttributeValues" WHERE "accountSid" = $ AND "list" = 'provinces' AND "id" IN ($, $); +`, + { + accountSid: process.argv[2], + id: `CA-${geographicCode}-en`, + idFr: `CA-${geographicCode}-fr`, + value: `CA/${provinceCode}/${cityEn}`, + info: { name: cityEn, geographicCode, csdType }, + infoFr: { name: cityFr, geographicCode, csdType }, + oldId: `CA-${provinceCode}-${cityEn}-en`, + oldIdFr: `CA-${provinceCode}-${cityEn}-fr`, + }, + ); + sqlFile.write(sqlStatement); + if ( + csdType === 'Town' || + csdType === 'City' || + csdType === 'Ville' || + csdType === 'Municipalité' + ) { + citiesJson.push({ + label: cityEn, + value: `CA/${provinceCode}/${cityEn}`, + }); + } } sqlFile.end(); - fs.writeFileSync( - targetJsonCitiesFilePath, - JSON.stringify( - Object.values(Object.fromEntries(citiesJson.map(c => [c.value, c]))), - null, - 2, - ), - ); + fs.writeFileSync(targetJsonCitiesFilePath, JSON.stringify(citiesJson, null, 2)); fs.writeFileSync(targetJsonProvincesFilePath, JSON.stringify(provincesJson, null, 2)); };