CHI-1855: Refactor HRM SQL to use a composable function for selecting a full contact

hrm-domain/hrm-service/migrations/20230824113300-contact-function.js

@@ -0,0 +1,93 @@
+/**
+ * Copyright (C) 2021-2023 Technology Matters
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see https://www.gnu.org/licenses/.
+ */
+'use strict';
+
+/** @type {import('sequelize-cli').Migration} */
+module.exports = {
+  up: async queryInterface => {
+    await queryInterface.sequelize
+      .query(`CREATE OR REPLACE FUNCTION "contactRelations"(character varying(255), bigint)
+RETURNS TABLE (
+  "csamReports" jsonb,
+  "referrals" jsonb,
+  "conversationMedia" jsonb
+) AS $$
+  SELECT reports."csamReports", joinedReferrals."referrals", media."conversationMedia" FROM (
+    SELECT COALESCE(jsonb_agg(to_jsonb(r)), '[]') AS "csamReports"
+    FROM "CSAMReports" r
+    WHERE r."contactId" = $2 AND r."accountSid" = $1 AND r."acknowledged" = TRUE
+  ) reports
+  LEFT JOIN LATERAL (
+    SELECT COALESCE(jsonb_agg(to_jsonb(referral)), '[]') AS "referrals"
+    FROM "Referrals" referral
+    WHERE referral."contactId" = $2 AND referral."accountSid" = $1
+  ) joinedReferrals ON true
+  LEFT JOIN LATERAL (
+    SELECT COALESCE(jsonb_agg(to_jsonb(cm)), '[]') AS "conversationMedia"
+    FROM "ConversationMedias" cm
+    WHERE cm."contactId" = $2 AND cm."accountSid" = $1
+  ) media ON true
+$$
+LANGUAGE SQL
+STABLE;
+    `);
+    await queryInterface.sequelize
+      .query(`CREATE OR REPLACE FUNCTION "permittedFullContacts"(character varying(255), character varying(255))
+RETURNS TABLE (
+  id integer,
+  "createdAt" timestamp with time zone,
+  "updatedAt" timestamp with time zone,
+  "rawJson" jsonb,
+  "queueName" character varying(255),
+  "twilioWorkerId" character varying(255),
+  helpline character varying(255),
+  "number" character varying(255),
+  channel character varying(255),
+  "conversationDuration" integer,
+  "caseId" integer,
+  "accountSid" character varying(255),
+  "timeOfContact" timestamp with time zone,
+  "taskId" character varying(255),
+  "createdBy" character varying(255),
+  "channelSid" character varying(255),
+  "serviceSid" character varying(255),
+  "updatedBy" text,
+  "csamReports" jsonb,
+  "referrals" jsonb,
+  "conversationMedia" jsonb
+) AS $$
+  SELECT c.*, relations."csamReports", relations."referrals", relations."conversationMedia"
+  FROM "Contacts" c
+  LEFT JOIN LATERAL "contactRelations"($1, c.id) relations ON true
+  WHERE c."accountSid" = $1 AND ($2 IS NULL OR c."twilioWorkerId" = $2)
+$$
+LANGUAGE SQL
+STABLE;
+    `);
+    console.log('Function "permittedFullContacts" created');
+  },
+
+  down: async queryInterface => {
+    await queryInterface.sequelize.query(
+      `DROP FUNCTION IF EXISTS public."permittedFullContacts"(character varying(255), character varying(255))`,
+    );
+    console.log('Function "permittedFullContacts" dropped');
+    await queryInterface.sequelize.query(
+      `DROP FUNCTION IF EXISTS public."contactRelations"(character varying(255), bigint)`,
+    );
+    console.log('Function "contactRelations" dropped');
+  },
+};

hrm-domain/hrm-service/service-tests/contacts.test.ts

@@ -54,7 +54,7 @@ import { mockingProxy, mockSuccessfulTwilioAuthentication } from '@tech-matters/
 import * as contactJobDataAccess from '../src/contact-job/contact-job-data-access';
 import { chatChannels } from '../src/contact/channelTypes';
 import * as contactInsertSql from '../src/contact/sql/contact-insert-sql';
-import { selectSingleContactByTaskId } from '../src/contact/sql/contact-get-sql';
+import { SELECT_SINGLE_CONTACT_BY_TASKSID } from '../src/contact/sql/contact-get-sql';
 import { ruleFileWithOneActionOverride } from './permissions-overrides';
 import * as csamReportApi from '../src/csam-report/csam-report';
 import * as referralDB from '../src/referral/referral-data-access';
@@ -125,7 +125,7 @@ const cleanupReferrals = () =>
 
 // eslint-disable-next-line @typescript-eslint/no-shadow
 const getContactByTaskId = (taskId: string, accountSid: string) =>
-  db.oneOrNone(selectSingleContactByTaskId('Contacts'), { accountSid, taskId });
+  db.oneOrNone(SELECT_SINGLE_CONTACT_BY_TASKSID, { accountSid, taskId });
 
 // eslint-disable-next-line @typescript-eslint/no-shadow
 const deleteContactById = (id: number, accountSid: string) =>

hrm-domain/hrm-service/src/case/sql/case-get-sql.ts

@@ -14,10 +14,6 @@
  * along with this program.  If not, see https://www.gnu.org/licenses/.
  */
 
-import { selectCoalesceConversationMediasByContactId } from '../../conversation-media/sql/conversation-media-get-sql';
-import { selectCoalesceCsamReportsByContactId } from '../../csam-report/sql/csam-report-get-sql';
-import { selectCoalesceReferralsByContactId } from '../../referral/sql/referral-get-sql';
-
 const ID_WHERE_CLAUSE = `WHERE "cases"."accountSid" = $<accountSid> AND "cases"."id" = $<caseId>`;
 
 export const selectSingleCaseByIdSql = (tableName: string) => `SELECT
@@ -26,18 +22,9 @@ export const selectSingleCaseByIdSql = (tableName: string) => `SELECT
       contacts."connectedContacts"
       FROM "${tableName}" AS cases
       LEFT JOIN LATERAL (
-        SELECT COALESCE(jsonb_agg(to_jsonb(c) || to_jsonb(joinedReports) || to_jsonb(joinedReferrals) || to_jsonb(joinedConversationMedia)), '[]') AS  "connectedContacts"
-        FROM "Contacts" c 
-        LEFT JOIN LATERAL (
-          ${selectCoalesceCsamReportsByContactId('c')}
-        ) joinedReports ON true
-        LEFT JOIN LATERAL (
-          ${selectCoalesceReferralsByContactId('c')}
-        ) joinedReferrals ON true
-        LEFT JOIN LATERAL (
-          ${selectCoalesceConversationMediasByContactId('c')}
-        ) joinedConversationMedia ON true
-        WHERE c."caseId" = cases.id AND c."accountSid" = cases."accountSid"
+        SELECT COALESCE(jsonb_agg(DISTINCT contacts.*) FILTER (WHERE contacts."caseId" IS NOT NULL), '[]') AS "connectedContacts"
+        FROM "permittedFullContacts"(cases."accountSid", NULL) AS contacts 
+        WHERE contacts."caseId" = cases.id
       ) contacts ON true
       LEFT JOIN LATERAL (
         SELECT COALESCE(jsonb_agg(to_jsonb(cs) ORDER BY cs."createdAt"), '[]') AS  "caseSections"

hrm-domain/hrm-service/src/case/sql/case-search-sql.ts

@@ -17,9 +17,6 @@
 import { pgp } from '../../connection-pool';
 import { SELECT_CASE_SECTIONS } from './case-sections-sql';
 import { CaseListFilters, DateExistsCondition, DateFilter } from '../case-data-access';
-import { leftJoinCsamReportsOnFK } from '../../csam-report/sql/csam-report-get-sql';
-import { leftJoinReferralsOnFK } from '../../referral/sql/referral-get-sql';
-import { leftJoinConversationMediasOnFK } from '../../conversation-media/sql/conversation-media-get-sql';
 
 export const OrderByDirection = {
   ascendingNullsLast: 'ASC NULLS LAST',
@@ -65,19 +62,8 @@ const generateOrderByClause = (clauses: OrderByClauseItem[]): string => {
 };
 
 const SELECT_CONTACTS = `SELECT COALESCE(jsonb_agg(DISTINCT contacts.*) FILTER (WHERE contacts."caseId" IS NOT NULL), '[]') AS "connectedContacts"
-FROM (
-  SELECT
-    c.*,
-    COALESCE(jsonb_agg(DISTINCT r.*) FILTER (WHERE r.id IS NOT NULL), '[]') AS "csamReports",
-    COALESCE(jsonb_agg(DISTINCT referral.*) FILTER (WHERE referral IS NOT NULL), '[]') AS "referrals",
-    COALESCE(jsonb_agg(DISTINCT cm.*) FILTER (WHERE cm IS NOT NULL), '[]') AS "conversationMedia"
-  FROM "Contacts" c 
-  ${leftJoinCsamReportsOnFK('c')}
-  ${leftJoinReferralsOnFK('c')}
-  ${leftJoinConversationMediasOnFK('c')}
-  WHERE c."caseId" = "cases".id AND c."accountSid" = "cases"."accountSid"
-  GROUP BY c."accountSid", c.id
-) AS contacts WHERE contacts."caseId" = cases.id AND contacts."accountSid" = cases."accountSid"`;
+FROM "permittedFullContacts"(cases."accountSid", NULL) AS contacts 
+WHERE contacts."caseId" = cases.id AND contacts."accountSid" = cases."accountSid"`;
 
 const enum FilterableDateField {
   CREATED_AT = 'cases."createdAt"::TIMESTAMP WITH TIME ZONE',

hrm-domain/hrm-service/src/contact-job/sql/contact-job-sql.ts

@@ -14,8 +14,6 @@
  * along with this program.  If not, see https://www.gnu.org/licenses/.
  */
 
-import { selectContactsWithRelations } from '../../contact/sql/contact-get-sql';
-
 export enum ContactJobCleanupStatus {
   NOT_READY = 'not_ready',
   PENDING = 'pending',
@@ -49,11 +47,9 @@ WITH due AS (
     AND "completed" IS NOT NULL
     AND "completed" < (current_timestamp - interval '$<cleanupRetentionDays> day')
   )
-  SELECT due.*, to_jsonb(contacts.*) AS "resource"
-  FROM due LEFT JOIN LATERAL (
-  ${selectContactsWithRelations(
-    'Contacts',
-  )} WHERE c."accountSid" = due."accountSid" AND c."id" = due."contactId") AS contacts ON true
+  SELECT due.*, to_jsonb(contacts.*) AS "resource"FROM due 
+  LEFT JOIN public."permittedFullContacts"(due."accountSid", NULL) AS contacts 
+  ON contacts."id" = due."contactId"
 `;
 
 export const PENDING_CLEANUP_JOB_ACCOUNT_SIDS_SQL = `
@@ -69,11 +65,9 @@ export const PULL_DUE_JOBS_SQL = `
     UPDATE "ContactJobs" SET "lastAttempt" = CURRENT_TIMESTAMP, "numberOfAttempts" = "numberOfAttempts" + 1
     WHERE "completed" IS NULL AND "numberOfAttempts" < $<jobMaxAttempts> AND ("lastAttempt" IS NULL OR "lastAttempt" <= $<lastAttemptedBefore>::TIMESTAMP WITH TIME ZONE) RETURNING *
   )
-  SELECT due.*, to_jsonb(contacts.*) AS "resource"
-  FROM due LEFT JOIN LATERAL (
-  ${selectContactsWithRelations(
-    'Contacts',
-  )} WHERE c."accountSid" = due."accountSid" AND c."id" = due."contactId") AS contacts ON true
+  SELECT due.*, to_jsonb(contacts.*) AS "resource" FROM due 
+  LEFT JOIN public."permittedFullContacts"(due."accountSid", NULL) AS contacts 
+  ON contacts."id" = due."contactId"
 `;
 
 export const UPDATE_JOB_CLEANUP_ACTIVE_SQL = `

hrm-domain/hrm-service/src/contact/contact-data-access.ts

@@ -19,8 +19,8 @@ import { UPDATE_CASEID_BY_ID, UPDATE_RAWJSON_BY_ID } from './sql/contact-update-
 import { SELECT_CONTACT_SEARCH } from './sql/contact-search-sql';
 import { endOfDay, parseISO, startOfDay } from 'date-fns';
 import {
-  selectSingleContactByIdSql,
-  selectSingleContactByTaskId,
+  SELECT_SINGLE_CONTACT_BY_ID,
+  SELECT_SINGLE_CONTACT_BY_TASKSID,
 } from './sql/contact-get-sql';
 import { insertContactSql, NewContactRecord } from './sql/contact-insert-sql';
 import { PersonInformation, ReferralWithoutContactId } from './contact-json';
@@ -160,7 +160,7 @@ export const create =
     ) => {
       if (newContact.taskId) {
         const existingContact: Contact = await conn.oneOrNone<Contact>(
-          selectSingleContactByTaskId('Contacts'),
+          SELECT_SINGLE_CONTACT_BY_TASKSID,
           {
             accountSid,
             taskId: newContact.taskId,
@@ -226,7 +226,7 @@ export const connectToCase = async (
 
 export const getById = async (accountSid: string, contactId: number): Promise<Contact> =>
   db.task(async connection =>
-    connection.oneOrNone<Contact>(selectSingleContactByIdSql('Contacts'), {
+    connection.oneOrNone<Contact>(SELECT_SINGLE_CONTACT_BY_ID, {
       accountSid,
       contactId,
     }),

hrm-domain/hrm-service/src/contact/sql/contact-get-sql.ts

@@ -14,32 +14,12 @@
  * along with this program.  If not, see https://www.gnu.org/licenses/.
  */
 
-import { selectCoalesceConversationMediasByContactId } from '../../conversation-media/sql/conversation-media-get-sql';
-import { selectCoalesceCsamReportsByContactId } from '../../csam-report/sql/csam-report-get-sql';
-import { selectCoalesceReferralsByContactId } from '../../referral/sql/referral-get-sql';
+export const SELECT_SINGLE_CONTACT_BY_ID = `
+      SELECT c.* FROM "permittedFullContacts"($<accountSid>, NULL) c
+      WHERE c."id" = $<contactId>`;
 
-const ID_WHERE_CLAUSE = `WHERE c."accountSid" = $<accountSid> AND c."id" = $<contactId>`;
-const TASKID_WHERE_CLAUSE = `WHERE c."accountSid" = $<accountSid> AND c."taskId" = $<taskId>`;
-
-export const selectContactsWithRelations = (table: string) => `
-        SELECT c.*, reports."csamReports", joinedReferrals."referrals", media."conversationMedia"
-        FROM "${table}" c 
-        LEFT JOIN LATERAL (
-          ${selectCoalesceCsamReportsByContactId('c')}
-        ) reports ON true
-        LEFT JOIN LATERAL (
-          ${selectCoalesceReferralsByContactId('c')}
-        ) joinedReferrals ON true
-        LEFT JOIN LATERAL (
-          ${selectCoalesceConversationMediasByContactId('c')}
-        ) media ON true`;
-
-export const selectSingleContactByIdSql = (table: string) => `
-      ${selectContactsWithRelations(table)}
-      ${ID_WHERE_CLAUSE}`;
-
-export const selectSingleContactByTaskId = (table: string) => ` 
-      ${selectContactsWithRelations(table)}
-      ${TASKID_WHERE_CLAUSE}
+export const SELECT_SINGLE_CONTACT_BY_TASKSID = `
+      SELECT c.* FROM "permittedFullContacts"($<accountSid>, NULL) c
+      WHERE c."taskId" = $<taskId>
       -- only take the latest, this ORDER / LIMIT clause would be redundant 
       ORDER BY c."createdAt" DESC LIMIT 1`;

hrm-domain/hrm-service/src/contact/sql/contact-search-sql.ts

@@ -14,26 +14,12 @@
  * along with this program.  If not, see https://www.gnu.org/licenses/.
  */
 
-import { selectCoalesceConversationMediasByContactId } from '../../conversation-media/sql/conversation-media-get-sql';
-import { selectCoalesceCsamReportsByContactId } from '../../csam-report/sql/csam-report-get-sql';
-import { selectCoalesceReferralsByContactId } from '../../referral/sql/referral-get-sql';
-
 export const SELECT_CONTACT_SEARCH = `
         SELECT 
         (count(*) OVER())::INTEGER AS "totalCount",
-        contacts.*, reports."csamReports", joinedReferrals."referrals", media."conversationMedia"
-        FROM "Contacts" contacts
-        LEFT JOIN LATERAL (
-          ${selectCoalesceCsamReportsByContactId('contacts')}
-        ) reports ON true
-        LEFT JOIN LATERAL (
-          ${selectCoalesceReferralsByContactId('contacts')}
-        ) joinedReferrals ON true
-        LEFT JOIN LATERAL (
-          ${selectCoalesceConversationMediasByContactId('contacts')}
-        ) media ON true
-        WHERE contacts."accountSid" = $<accountSid>
-        AND ($<helpline> IS NULL OR contacts."helpline" = $<helpline>)
+        contacts.*
+        FROM "permittedFullContacts"($<accountSid>, $<counselor>) contacts
+        WHERE ($<helpline> IS NULL OR contacts."helpline" = $<helpline>)
         AND (
           ($<lastNamePattern> IS NULL AND $<firstNamePattern> IS NULL)
           OR (
@@ -63,9 +49,6 @@ export const SELECT_CONTACT_SEARCH = `
             )
           )
         )
-        AND (
-          $<counselor> IS NULL OR contacts."twilioWorkerId" = $<counselor>
-        )
         AND (
           $<phoneNumberPattern> IS NULL
           OR "number" ILIKE $<phoneNumberPattern>

hrm-domain/hrm-service/src/contact/sql/contact-update-sql.ts

@@ -14,10 +14,13 @@
  * along with this program.  If not, see https://www.gnu.org/licenses/.
  */
 
-import { selectSingleContactByIdSql } from './contact-get-sql';
-
 const ID_WHERE_CLAUSE = `WHERE "accountSid" = $<accountSid> AND "id"=$<contactId>`;
 
+const SELECT_FULL_CONTACT_WITH_UPDATED = `
+      SELECT c.*, relations."csamReports", relations."referrals", relations."conversationMedia"
+      FROM "updated" c LEFT JOIN LATERAL "contactRelations"($<accountSid>, c.id) relations ON true
+      `;
+
 export const UPDATE_RAWJSON_BY_ID = `WITH updated AS (
 UPDATE "Contacts" 
 SET "rawJson" = COALESCE("rawJson", '{}'::JSONB) 
@@ -53,7 +56,7 @@ SET "rawJson" = COALESCE("rawJson", '{}'::JSONB)
 ${ID_WHERE_CLAUSE}
 RETURNING *
 )
-${selectSingleContactByIdSql('updated')}
+${SELECT_FULL_CONTACT_WITH_UPDATED}
 `;
 
 export const UPDATE_CASEID_BY_ID = `WITH updated AS (
@@ -63,5 +66,5 @@ SET
 ${ID_WHERE_CLAUSE}
 RETURNING *
 )
-${selectSingleContactByIdSql('updated')}
+${SELECT_FULL_CONTACT_WITH_UPDATED}
 `;

hrm-domain/hrm-service/src/conversation-media/sql/conversation-media-get-sql.ts

@@ -28,19 +28,3 @@ export const selectConversationMediaByContactIdSql = `
   FROM "ConversationMedias" cm
   ${CONTACT_ID_WHERE_CLAUSE}
 `;
-
-// Queries used in other modules for JOINs
-
-const onFkFilteredClause = (contactAlias: string) => `
-  cm."contactId" = "${contactAlias}".id AND cm."accountSid" = "${contactAlias}"."accountSid"
-`;
-
-export const selectCoalesceConversationMediasByContactId = (contactAlias: string) => `
-  SELECT COALESCE(jsonb_agg(to_jsonb(cm)), '[]') AS  "conversationMedia"
-  FROM "ConversationMedias" cm
-  WHERE ${onFkFilteredClause(contactAlias)}
-`;
-
-export const leftJoinConversationMediasOnFK = (contactAlias: string) => `
-  LEFT JOIN "ConversationMedias" cm ON ${onFkFilteredClause(contactAlias)}
-`;

hrm-domain/hrm-service/src/csam-report/sql/csam-report-get-sql.ts

@@ -28,19 +28,3 @@ export const selectCsamReportsByContactIdSql = `
   FROM "CSAMReports" r
   ${CONTACT_ID_WHERE_CLAUSE}
 `;
-
-// Queries used in other modules for JOINs
-
-const onFkFilteredClause = (contactAlias: string) => `
-  r."contactId" = "${contactAlias}".id AND r."accountSid" = "${contactAlias}"."accountSid" AND r."acknowledged" = TRUE
-`;
-
-export const selectCoalesceCsamReportsByContactId = (contactAlias: string) => `
-  SELECT COALESCE(jsonb_agg(to_jsonb(r)), '[]') AS  "csamReports"
-  FROM "CSAMReports" r
-  WHERE ${onFkFilteredClause(contactAlias)}
-`;
-
-export const leftJoinCsamReportsOnFK = (contactAlias: string) => `
-  LEFT JOIN "CSAMReports" r ON ${onFkFilteredClause(contactAlias)}
-`;