Use a dedicated logs ClickHouse client

Author: matt-aitken

apps/webapp/app/env.server.ts

@@ -1198,6 +1198,12 @@ const EnvironmentSchema = z
     // Query feature flag
     QUERY_FEATURE_ENABLED: z.string().default("1"),
 
+    // Logs page ClickHouse URL (for logs queries)
+    LOGS_CLICKHOUSE_URL: z
+      .string()
+      .optional()
+      .transform((v) => v ?? process.env.CLICKHOUSE_URL),
+
     // Query page ClickHouse limits (for TSQL queries)
     QUERY_CLICKHOUSE_URL: z
       .string()

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs/route.tsx

@@ -16,7 +16,7 @@ import { findEnvironmentBySlug } from "~/models/runtimeEnvironment.server";
 import { LogsListPresenter, LogEntry } from "~/presenters/v3/LogsListPresenter.server";
 import type { LogLevel } from "~/utils/logUtils";
 import { $replica, prisma } from "~/db.server";
-import { clickhouseClient } from "~/services/clickhouseInstance.server";
+import { logsClickhouseClient } from "~/services/clickhouseInstance.server";
 import { NavBar, PageTitle } from "~/components/primitives/PageHeader";
 import { PageBody, PageContainer } from "~/components/layout/AppLayout";
 import { Suspense, useCallback, useEffect, useMemo, useRef, useState, useTransition } from "react";
@@ -134,7 +134,7 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => {
   const plan = await getCurrentPlan(project.organizationId);
   const retentionLimitDays = plan?.v3Subscription?.plan?.limits.logRetentionDays.number ?? 30;
 
-  const presenter = new LogsListPresenter($replica, clickhouseClient);
+  const presenter = new LogsListPresenter($replica, logsClickhouseClient);
 
   const listPromise = presenter
     .call(project.organizationId, environment.id, {

apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs.$logId.tsx

@@ -1,7 +1,7 @@
 import { type LoaderFunctionArgs } from "@remix-run/server-runtime";
 import { typedjson } from "remix-typedjson";
 import { z } from "zod";
-import { clickhouseClient } from "~/services/clickhouseInstance.server";
+import { logsClickhouseClient } from "~/services/clickhouseInstance.server";
 import { requireUserId } from "~/services/session.server";
 import { LogDetailPresenter } from "~/presenters/v3/LogDetailPresenter.server";
 import { findProjectBySlug } from "~/models/project.server";
@@ -43,7 +43,7 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => {
 
   const [traceId, spanId, , startTime] = parts;
 
-  const presenter = new LogDetailPresenter($replica, clickhouseClient);
+  const presenter = new LogDetailPresenter($replica, logsClickhouseClient);
 
   let result;
   try {

apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs.ts

@@ -6,7 +6,7 @@ import { findProjectBySlug } from "~/models/project.server";
 import { findEnvironmentBySlug } from "~/models/runtimeEnvironment.server";
 import { LogsListPresenter, type LogLevel, LogsListOptionsSchema } from "~/presenters/v3/LogsListPresenter.server";
 import { $replica } from "~/db.server";
-import { clickhouseClient } from "~/services/clickhouseInstance.server";
+import { logsClickhouseClient } from "~/services/clickhouseInstance.server";
 import { getCurrentPlan } from "~/services/platform.v3.server";
 
 // Valid log levels for filtering
@@ -69,7 +69,7 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => {
     retentionLimitDays,
   }) as any; // Validated by LogsListOptionsSchema at runtime
 
-  const presenter = new LogsListPresenter($replica, clickhouseClient);
+  const presenter = new LogsListPresenter($replica, logsClickhouseClient);
   const result = await presenter.call(project.organizationId, environment.id, options);
 
   return json({

apps/webapp/app/services/clickhouseInstance.server.ts

@@ -53,6 +53,60 @@ function initializeClickhouseClient() {
   return clickhouse;
 }
 
+export const logsClickhouseClient = singleton(
+  "logsClickhouseClient",
+  initializeLogsClickhouseClient
+);
+
+function initializeLogsClickhouseClient() {
+  if (!env.LOGS_CLICKHOUSE_URL) {
+    throw new Error("LOGS_CLICKHOUSE_URL is not set");
+  }
+
+  const url = new URL(env.LOGS_CLICKHOUSE_URL);
+
+  // Remove secure param
+  url.searchParams.delete("secure");
+
+  // Build logs query settings from environment variables
+  const logsQuerySettings = {
+    list: {
+      max_memory_usage: env.CLICKHOUSE_LOGS_LIST_MAX_MEMORY_USAGE.toString(),
+      max_bytes_before_external_sort:
+        env.CLICKHOUSE_LOGS_LIST_MAX_BYTES_BEFORE_EXTERNAL_SORT.toString(),
+      max_threads: env.CLICKHOUSE_LOGS_LIST_MAX_THREADS,
+      ...(env.CLICKHOUSE_LOGS_LIST_MAX_ROWS_TO_READ && {
+        max_rows_to_read: env.CLICKHOUSE_LOGS_LIST_MAX_ROWS_TO_READ.toString(),
+      }),
+      ...(env.CLICKHOUSE_LOGS_LIST_MAX_EXECUTION_TIME && {
+        max_execution_time: env.CLICKHOUSE_LOGS_LIST_MAX_EXECUTION_TIME,
+      }),
+    },
+    detail: {
+      max_memory_usage: env.CLICKHOUSE_LOGS_DETAIL_MAX_MEMORY_USAGE.toString(),
+      max_threads: env.CLICKHOUSE_LOGS_DETAIL_MAX_THREADS,
+      ...(env.CLICKHOUSE_LOGS_DETAIL_MAX_EXECUTION_TIME && {
+        max_execution_time: env.CLICKHOUSE_LOGS_DETAIL_MAX_EXECUTION_TIME,
+      }),
+    },
+  };
+
+  return new ClickHouse({
+    url: url.toString(),
+    name: "logs-clickhouse",
+    keepAlive: {
+      enabled: env.CLICKHOUSE_KEEP_ALIVE_ENABLED === "1",
+      idleSocketTtl: env.CLICKHOUSE_KEEP_ALIVE_IDLE_SOCKET_TTL_MS,
+    },
+    logLevel: env.CLICKHOUSE_LOG_LEVEL,
+    compression: {
+      request: true,
+    },
+    maxOpenConnections: env.CLICKHOUSE_MAX_OPEN_CONNECTIONS,
+    logsQuerySettings,
+  });
+}
+
 export const queryClickhouseClient = singleton(
   "queryClickhouseClient",
   initializeQueryClickhouseClient