Move the ClickHouse settings to the dedicated client

Author: matt-aitken

apps/webapp/app/env.server.ts

@@ -1180,7 +1180,7 @@ const EnvironmentSchema = z
     CLICKHOUSE_LOG_LEVEL: z.enum(["log", "error", "warn", "info", "debug"]).default("info"),
     CLICKHOUSE_COMPRESSION_REQUEST: z.string().default("1"),
 
-    // Logs List Query Settings (for paginated log views)
+    // Logs Query Settings
     CLICKHOUSE_LOGS_LIST_MAX_MEMORY_USAGE: z.coerce.number().int().default(1_000_000_000),
     CLICKHOUSE_LOGS_LIST_MAX_BYTES_BEFORE_EXTERNAL_SORT: z.coerce
       .number()
@@ -1190,11 +1190,6 @@ const EnvironmentSchema = z
     CLICKHOUSE_LOGS_LIST_MAX_ROWS_TO_READ: z.coerce.number().int().default(10_000_000),
     CLICKHOUSE_LOGS_LIST_MAX_EXECUTION_TIME: z.coerce.number().int().default(120),
 
-    // Logs Detail Query Settings (for single log views)
-    CLICKHOUSE_LOGS_DETAIL_MAX_MEMORY_USAGE: z.coerce.number().int().default(64_000_000),
-    CLICKHOUSE_LOGS_DETAIL_MAX_THREADS: z.coerce.number().int().default(2),
-    CLICKHOUSE_LOGS_DETAIL_MAX_EXECUTION_TIME: z.coerce.number().int().default(60),
-
     // Query feature flag
     QUERY_FEATURE_ENABLED: z.string().default("1"),
 

apps/webapp/app/services/clickhouseInstance.server.ts

@@ -12,29 +12,6 @@ function initializeClickhouseClient() {
 
   console.log(`🗃️  Clickhouse service enabled to host ${url.host}`);
 
-  // Build logs query settings from environment variables
-  const logsQuerySettings = {
-    list: {
-      max_memory_usage: env.CLICKHOUSE_LOGS_LIST_MAX_MEMORY_USAGE.toString(),
-      max_bytes_before_external_sort:
-        env.CLICKHOUSE_LOGS_LIST_MAX_BYTES_BEFORE_EXTERNAL_SORT.toString(),
-      max_threads: env.CLICKHOUSE_LOGS_LIST_MAX_THREADS,
-      ...(env.CLICKHOUSE_LOGS_LIST_MAX_ROWS_TO_READ && {
-        max_rows_to_read: env.CLICKHOUSE_LOGS_LIST_MAX_ROWS_TO_READ.toString(),
-      }),
-      ...(env.CLICKHOUSE_LOGS_LIST_MAX_EXECUTION_TIME && {
-        max_execution_time: env.CLICKHOUSE_LOGS_LIST_MAX_EXECUTION_TIME,
-      }),
-    },
-    detail: {
-      max_memory_usage: env.CLICKHOUSE_LOGS_DETAIL_MAX_MEMORY_USAGE.toString(),
-      max_threads: env.CLICKHOUSE_LOGS_DETAIL_MAX_THREADS,
-      ...(env.CLICKHOUSE_LOGS_DETAIL_MAX_EXECUTION_TIME && {
-        max_execution_time: env.CLICKHOUSE_LOGS_DETAIL_MAX_EXECUTION_TIME,
-      }),
-    },
-  };
-
   const clickhouse = new ClickHouse({
     url: url.toString(),
     name: "clickhouse-instance",
@@ -47,7 +24,6 @@ function initializeClickhouseClient() {
       request: true,
     },
     maxOpenConnections: env.CLICKHOUSE_MAX_OPEN_CONNECTIONS,
-    logsQuerySettings,
   });
 
   return clickhouse;
@@ -68,29 +44,6 @@ function initializeLogsClickhouseClient() {
   // Remove secure param
   url.searchParams.delete("secure");
 
-  // Build logs query settings from environment variables
-  const logsQuerySettings = {
-    list: {
-      max_memory_usage: env.CLICKHOUSE_LOGS_LIST_MAX_MEMORY_USAGE.toString(),
-      max_bytes_before_external_sort:
-        env.CLICKHOUSE_LOGS_LIST_MAX_BYTES_BEFORE_EXTERNAL_SORT.toString(),
-      max_threads: env.CLICKHOUSE_LOGS_LIST_MAX_THREADS,
-      ...(env.CLICKHOUSE_LOGS_LIST_MAX_ROWS_TO_READ && {
-        max_rows_to_read: env.CLICKHOUSE_LOGS_LIST_MAX_ROWS_TO_READ.toString(),
-      }),
-      ...(env.CLICKHOUSE_LOGS_LIST_MAX_EXECUTION_TIME && {
-        max_execution_time: env.CLICKHOUSE_LOGS_LIST_MAX_EXECUTION_TIME,
-      }),
-    },
-    detail: {
-      max_memory_usage: env.CLICKHOUSE_LOGS_DETAIL_MAX_MEMORY_USAGE.toString(),
-      max_threads: env.CLICKHOUSE_LOGS_DETAIL_MAX_THREADS,
-      ...(env.CLICKHOUSE_LOGS_DETAIL_MAX_EXECUTION_TIME && {
-        max_execution_time: env.CLICKHOUSE_LOGS_DETAIL_MAX_EXECUTION_TIME,
-      }),
-    },
-  };
-
   return new ClickHouse({
     url: url.toString(),
     name: "logs-clickhouse",
@@ -103,7 +56,18 @@ function initializeLogsClickhouseClient() {
       request: true,
     },
     maxOpenConnections: env.CLICKHOUSE_MAX_OPEN_CONNECTIONS,
-    logsQuerySettings,
+    clickhouseSettings: {
+      max_memory_usage: env.CLICKHOUSE_LOGS_LIST_MAX_MEMORY_USAGE.toString(),
+      max_bytes_before_external_sort:
+        env.CLICKHOUSE_LOGS_LIST_MAX_BYTES_BEFORE_EXTERNAL_SORT.toString(),
+      max_threads: env.CLICKHOUSE_LOGS_LIST_MAX_THREADS,
+      ...(env.CLICKHOUSE_LOGS_LIST_MAX_ROWS_TO_READ && {
+        max_rows_to_read: env.CLICKHOUSE_LOGS_LIST_MAX_ROWS_TO_READ.toString(),
+      }),
+      ...(env.CLICKHOUSE_LOGS_LIST_MAX_EXECUTION_TIME && {
+        max_execution_time: env.CLICKHOUSE_LOGS_LIST_MAX_EXECUTION_TIME,
+      }),
+    },
   });
 }
 

internal-packages/clickhouse/src/index.ts

@@ -61,11 +61,6 @@ export type { OutputColumnMetadata } from "@internal/tsql";
 // Errors
 export { QueryError } from "./client/errors.js";
 
-export type LogsQuerySettings = {
-  list?: ClickHouseSettings;
-  detail?: ClickHouseSettings;
-};
-
 export type ClickhouseCommonConfig = {
   keepAlive?: {
     enabled?: boolean;
@@ -80,7 +75,6 @@ export type ClickhouseCommonConfig = {
     response?: boolean;
   };
   maxOpenConnections?: number;
-  logsQuerySettings?: LogsQuerySettings;
 };
 
 export type ClickHouseConfig =
@@ -104,11 +98,9 @@ export class ClickHouse {
   public readonly writer: ClickhouseWriter;
   private readonly logger: Logger;
   private _splitClients: boolean;
-  private readonly logsQuerySettings?: LogsQuerySettings;
 
   constructor(config: ClickHouseConfig) {
     this.logger = config.logger ?? new Logger("ClickHouse", config.logLevel ?? "debug");
-    this.logsQuerySettings = config.logsQuerySettings;
 
     if (config.url) {
       const url = new URL(config.url);
@@ -220,13 +212,13 @@ export class ClickHouse {
       traceSummaryQueryBuilder: getTraceSummaryQueryBuilderV2(this.reader),
       traceDetailedSummaryQueryBuilder: getTraceDetailedSummaryQueryBuilderV2(this.reader),
       spanDetailsQueryBuilder: getSpanDetailsQueryBuilderV2(this.reader),
-      logDetailQueryBuilder: getLogDetailQueryBuilderV2(this.reader, this.logsQuerySettings?.detail),
+      logDetailQueryBuilder: getLogDetailQueryBuilderV2(this.reader),
     };
   }
 
   get taskEventsSearch() {
     return {
-      logsListQueryBuilder: getLogsSearchListQueryBuilder(this.reader, this.logsQuerySettings?.list),
+      logsListQueryBuilder: getLogsSearchListQueryBuilder(this.reader),
     };
   }
 }

internal-packages/clickhouse/src/taskEvents.ts

@@ -256,10 +256,7 @@ export const LogsSearchListResult = z.object({
 
 export type LogsSearchListResult = z.output<typeof LogsSearchListResult>;
 
-export function getLogsSearchListQueryBuilder(
-  ch: ClickhouseReader,
-  settings?: ClickHouseSettings
-) {
+export function getLogsSearchListQueryBuilder(ch: ClickhouseReader) {
   return ch.queryBuilderFast<LogsSearchListResult>({
     name: "getLogsSearchList",
     table: "trigger_dev.task_events_search_v1",
@@ -280,7 +277,6 @@ export function getLogsSearchListQueryBuilder(
       "attributes_text",
       "triggered_timestamp",
     ],
-    settings,
   });
 }
 
@@ -304,7 +300,7 @@ export const LogDetailV2Result = z.object({
 
 export type LogDetailV2Result = z.output<typeof LogDetailV2Result>;
 
-export function getLogDetailQueryBuilderV2(ch: ClickhouseReader, settings?: ClickHouseSettings) {
+export function getLogDetailQueryBuilderV2(ch: ClickhouseReader) {
   return ch.queryBuilderFast<LogDetailV2Result>({
     name: "getLogDetail",
     table: "trigger_dev.task_events_v2",
@@ -324,6 +320,5 @@ export function getLogDetailQueryBuilderV2(ch: ClickhouseReader, settings?: Clic
       "duration",
       "attributes_text",
     ],
-    settings,
   });
 }