Separate ClickHouse client for Query/Metrics (falls back to default)

Author: matt-aitken

apps/webapp/app/env.server.ts

@@ -1199,6 +1199,10 @@ const EnvironmentSchema = z
     QUERY_FEATURE_ENABLED: z.string().default("1"),
 
     // Query page ClickHouse limits (for TSQL queries)
+    QUERY_CLICKHOUSE_URL: z
+      .string()
+      .optional()
+      .transform((v) => v ?? process.env.CLICKHOUSE_URL),
     QUERY_CLICKHOUSE_MAX_EXECUTION_TIME: z.coerce.number().int().default(10),
     QUERY_CLICKHOUSE_MAX_MEMORY_USAGE: z.coerce.number().int().default(1_073_741_824), // 1GB in bytes
     QUERY_CLICKHOUSE_MAX_AST_ELEMENTS: z.coerce.number().int().default(4_000_000),

apps/webapp/app/services/clickhouseInstance.server.ts

@@ -16,7 +16,8 @@ function initializeClickhouseClient() {
   const logsQuerySettings = {
     list: {
       max_memory_usage: env.CLICKHOUSE_LOGS_LIST_MAX_MEMORY_USAGE.toString(),
-      max_bytes_before_external_sort: env.CLICKHOUSE_LOGS_LIST_MAX_BYTES_BEFORE_EXTERNAL_SORT.toString(),
+      max_bytes_before_external_sort:
+        env.CLICKHOUSE_LOGS_LIST_MAX_BYTES_BEFORE_EXTERNAL_SORT.toString(),
       max_threads: env.CLICKHOUSE_LOGS_LIST_MAX_THREADS,
       ...(env.CLICKHOUSE_LOGS_LIST_MAX_ROWS_TO_READ && {
         max_rows_to_read: env.CLICKHOUSE_LOGS_LIST_MAX_ROWS_TO_READ.toString(),
@@ -51,3 +52,30 @@ function initializeClickhouseClient() {
 
   return clickhouse;
 }
+
+export const queryClickhouseClient = singleton(
+  "queryClickhouseClient",
+  initializeQueryClickhouseClient
+);
+
+function initializeQueryClickhouseClient() {
+  if (!env.QUERY_CLICKHOUSE_URL) {
+    throw new Error("QUERY_CLICKHOUSE_URL is not set");
+  }
+
+  const url = new URL(env.QUERY_CLICKHOUSE_URL);
+
+  return new ClickHouse({
+    url: url.toString(),
+    name: "query-clickhouse",
+    keepAlive: {
+      enabled: env.CLICKHOUSE_KEEP_ALIVE_ENABLED === "1",
+      idleSocketTtl: env.CLICKHOUSE_KEEP_ALIVE_IDLE_SOCKET_TTL_MS,
+    },
+    logLevel: env.CLICKHOUSE_LOG_LEVEL,
+    compression: {
+      request: true,
+    },
+    maxOpenConnections: env.CLICKHOUSE_MAX_OPEN_CONNECTIONS,
+  });
+}

apps/webapp/app/services/queryService.server.ts

@@ -11,7 +11,7 @@ import type { TableSchema, WhereClauseCondition } from "@internal/tsql";
 import { z } from "zod";
 import { prisma } from "~/db.server";
 import { env } from "~/env.server";
-import { clickhouseClient } from "./clickhouseInstance.server";
+import { queryClickhouseClient } from "./clickhouseInstance.server";
 import {
   queryConcurrencyLimiter,
   DEFAULT_ORG_CONCURRENCY_LIMIT,
@@ -230,7 +230,7 @@ export async function executeQuery<TOut extends z.ZodSchema>(
       environment: Object.fromEntries(environments.map((e) => [e.id, e.slug])),
     };
 
-    const result = await executeTSQL(clickhouseClient.reader, {
+    const result = await executeTSQL(queryClickhouseClient.reader, {
       ...baseOptions,
       schema: z.record(z.any()),
       tableSchema: querySchemas,