Changed pagination to use span_id, not trace_id, more granular

Improved cursor
Removed debug from logs

Author: mpcgrid

apps/webapp/app/presenters/v3/LogsListPresenter.server.ts

@@ -50,7 +50,6 @@ export type LogsListOptions = {
   retentionLimitDays?: number;
   // search
   search?: string;
-  includeDebugLogs?: boolean;
   // pagination
   direction?: Direction;
   cursor?: string;
@@ -69,7 +68,6 @@ export const LogsListOptionsSchema = z.object({
   defaultPeriod: z.string().optional(),
   retentionLimitDays: z.number().int().positive().optional(),
   search: z.string().max(1000).optional(),
-  includeDebugLogs: z.boolean().optional(),
   direction: z.enum(["forward", "backward"]).optional(),
   cursor: z.string().optional(),
   pageSize: z.number().int().positive().max(1000).optional(),
@@ -83,12 +81,14 @@ export type LogsListAppliedFilters = LogsList["filters"];
 
 // Cursor is a base64 encoded JSON of the pagination keys
 type LogCursor = {
+  organizationId: string;
   environmentId: string;
   triggeredTimestamp: string; // DateTime64(9) string
   spanId: string;
 };
 
 const LogCursorSchema = z.object({
+  organizationId: z.string(),
   environmentId: z.string(),
   triggeredTimestamp: z.string(),
   spanId: z.string(),
@@ -166,7 +166,6 @@ export class LogsListPresenter extends BasePresenter {
       to,
       cursor,
       pageSize = DEFAULT_PAGE_SIZE,
-      includeDebugLogs = true,
       defaultPeriod,
       retentionLimitDays,
     }: LogsListOptions
@@ -349,32 +348,22 @@ export class LogsListPresenter extends BasePresenter {
       }
     }
 
-    // Debug logs are available only to admins
-    // if (includeDebugLogs === false) {
-    //   queryBuilder.where("kind NOT IN {debugKinds: Array(String)}", {
-    //     debugKinds: ["DEBUG_EVENT"],
-    //   });
-    //
-    //   queryBuilder.where("NOT ((kind = 'LOG_INFO') AND (attributes_text = '{}'))");
-    // }
-
-    // SPAN, ANCESTOR_OVERRIDE, SPAN_EVENT kinds are already filtered out by the materialized view
-
     // Cursor pagination using explicit lexicographic comparison
-    // Must mirror the ORDER BY columns: (environment_id DESC, triggered_timestamp DESC, span_id DESC)
+    // Must mirror the ORDER BY columns: (organization_id DESC, environment_id DESC, triggered_timestamp DESC, span_id DESC)
     const decodedCursor = cursor ? decodeCursor(cursor) : null;
     if (decodedCursor) {
       queryBuilder.where(
-        `((environment_id = {cursorEnvId: String} AND triggered_timestamp < {cursorTriggeredTimestamp: String}) OR (environment_id = {cursorEnvId: String} AND triggered_timestamp = {cursorTriggeredTimestamp: String} AND span_id < {cursorSpanId: String}) OR (environment_id < {cursorEnvId: String}))`,
+        `((organization_id = {cursorOrgId: String} AND environment_id = {cursorEnvId: String} AND triggered_timestamp = {cursorTriggeredTimestamp: String} AND span_id < {cursorSpanId: String}) OR (organization_id = {cursorOrgId: String} AND environment_id = {cursorEnvId: String} AND triggered_timestamp < {cursorTriggeredTimestamp: String}) OR (organization_id = {cursorOrgId: String} AND environment_id < {cursorEnvId: String}) OR (organization_id < {cursorOrgId: String}))`,
         {
+          cursorOrgId: decodedCursor.organizationId,
           cursorEnvId: decodedCursor.environmentId,
           cursorTriggeredTimestamp: decodedCursor.triggeredTimestamp,
           cursorSpanId: decodedCursor.spanId,
         }
       );
     }
 
-    queryBuilder.orderBy("environment_id DESC, triggered_timestamp DESC, span_id DESC");
+    queryBuilder.orderBy("organization_id DESC, environment_id DESC, triggered_timestamp DESC, span_id DESC");
     // Limit + 1 to check if there are more results
     queryBuilder.limit(pageSize + 1);
 
@@ -393,6 +382,7 @@ export class LogsListPresenter extends BasePresenter {
     if (hasMore && logs.length > 0) {
       const lastLog = logs[logs.length - 1];
       nextCursor = encodeCursor({
+        organizationId,
         environmentId,
         triggeredTimestamp: lastLog.triggered_timestamp,
         spanId: lastLog.span_id,

apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs/route.tsx

@@ -36,7 +36,6 @@ import {
   ResizablePanel,
   ResizablePanelGroup,
 } from "~/components/primitives/Resizable";
-import { Switch } from "~/components/primitives/Switch";
 import { Button } from "~/components/primitives/Buttons";
 import { FEATURE_FLAG, validateFeatureFlagValue } from "~/v3/featureFlags.server";
 
@@ -95,7 +94,6 @@ async function hasLogsPageAccess(
 export const loader = async ({ request, params }: LoaderFunctionArgs) => {
   const user = await requireUser(request);
   const userId = user.id;
-  const isAdmin = user.admin || user.isImpersonating;
 
   const { projectParam, organizationSlug, envParam } = EnvironmentParamSchema.parse(params);
 
@@ -126,7 +124,6 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => {
   const runId = url.searchParams.get("runId") ?? undefined;
   const search = url.searchParams.get("search") ?? undefined;
   const levels = parseLevelsFromUrl(url);
-  const showDebug = url.searchParams.get("showDebug") === "true";
   const period = url.searchParams.get("period") ?? undefined;
   const fromStr = url.searchParams.get("from");
   const toStr = url.searchParams.get("to");
@@ -150,7 +147,6 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => {
       period,
       from,
       to,
-      includeDebugLogs: isAdmin && showDebug,
       defaultPeriod: "1h",
       retentionLimitDays
     })
@@ -163,15 +159,13 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => {
 
   return typeddefer({
     data: listPromise,
-    isAdmin,
-    showDebug,
     defaultPeriod: "1h",
     retentionLimitDays,
   });
 };
 
 export default function Page() {
-  const { data, isAdmin, showDebug, defaultPeriod, retentionLimitDays } =
+  const { data, defaultPeriod, retentionLimitDays } =
     useTypedLoaderData<typeof loader>();
 
   return (
@@ -199,8 +193,6 @@ export default function Page() {
             errorElement={
               <div className="grid h-full max-h-full grid-rows-[2.5rem_auto_1fr] overflow-hidden">
                 <FiltersBar
-                  isAdmin={isAdmin}
-                  showDebug={showDebug}
                   defaultPeriod={defaultPeriod}
                   retentionLimitDays={retentionLimitDays}
                 />
@@ -218,8 +210,6 @@ export default function Page() {
                 return (
                   <div className="grid h-full max-h-full grid-rows-[2.5rem_auto_1fr] overflow-hidden">
                     <FiltersBar
-                      isAdmin={isAdmin}
-                      showDebug={showDebug}
                       defaultPeriod={defaultPeriod}
                       retentionLimitDays={retentionLimitDays}
                     />
@@ -235,15 +225,11 @@ export default function Page() {
                 <div className="grid h-full max-h-full grid-rows-[2.5rem_1fr] overflow-hidden">
                   <FiltersBar
                     list={result}
-                    isAdmin={isAdmin}
-                    showDebug={showDebug}
                     defaultPeriod={defaultPeriod}
                     retentionLimitDays={retentionLimitDays}
                   />
                   <LogsList
                     list={result}
-                    isAdmin={isAdmin}
-                    showDebug={showDebug}
                     defaultPeriod={defaultPeriod}
                   />
                 </div>
@@ -258,14 +244,10 @@ export default function Page() {
 
 function FiltersBar({
   list,
-  isAdmin,
-  showDebug,
   defaultPeriod,
   retentionLimitDays,
 }: {
   list?: Exclude<Awaited<UseDataFunctionReturn<typeof loader>["data"]>, { error: string }>;
-  isAdmin: boolean;
-  showDebug: boolean;
   defaultPeriod?: string;
   retentionLimitDays: number;
 }) {
@@ -280,16 +262,6 @@ function FiltersBar({
     searchParams.has("from") ||
     searchParams.has("to");
 
-  const handleDebugToggle = useCallback((checked: boolean) => {
-    const url = new URL(window.location.href);
-    if (checked) {
-      url.searchParams.set("showDebug", "true");
-    } else {
-      url.searchParams.delete("showDebug");
-    }
-    window.location.href = url.toString();
-  }, []);
-
   return (
     <div className="flex items-start justify-between gap-x-2 border-b border-grid-bright p-2">
       <div className="flex flex-row flex-wrap items-center gap-1">
@@ -329,16 +301,6 @@ function FiltersBar({
           </>
         )}
       </div>
-      <div className="flex items-center gap-2">
-        {isAdmin && (
-          <Switch
-            variant="small"
-            label="Debug"
-            checked={showDebug}
-            onCheckedChange={handleDebugToggle}
-          />
-        )}
-      </div>
     </div>
   );
 }
@@ -347,8 +309,6 @@ function LogsList({
   list,
 }: {
   list: Exclude<Awaited<UseDataFunctionReturn<typeof loader>["data"]>, { error: string }>; //exclude error, it is handled
-  isAdmin: boolean;
-  showDebug: boolean;
   defaultPeriod?: string;
 }) {
   const navigation = useNavigation();

apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs.ts

@@ -21,7 +21,6 @@ function parseLevelsFromUrl(url: URL): LogLevel[] | undefined {
 export const loader = async ({ request, params }: LoaderFunctionArgs) => {
   const user = await requireUser(request);
   const userId = user.id;
-  const isAdmin = user?.admin || user?.isImpersonating;
 
   const { projectParam, organizationSlug, envParam } = EnvironmentParamSchema.parse(params);
 
@@ -46,7 +45,6 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => {
   const search = url.searchParams.get("search") ?? undefined;
   const cursor = url.searchParams.get("cursor") ?? undefined;
   const levels = parseLevelsFromUrl(url);
-  const showDebug = url.searchParams.get("showDebug") === "true";
   const period = url.searchParams.get("period") ?? undefined;
   const fromStr = url.searchParams.get("from");
   const toStr = url.searchParams.get("to");
@@ -67,7 +65,6 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => {
     from,
     to,
     levels,
-    includeDebugLogs: isAdmin && showDebug,
     defaultPeriod: "1h",
     retentionLimitDays,
   }) as any; // Validated by LogsListOptionsSchema at runtime

internal-packages/clickhouse/schema/016_add_task_events_search_v1.sql

@@ -24,7 +24,7 @@ CREATE TABLE IF NOT EXISTS trigger_dev.task_events_search_v1
 )
 ENGINE = MergeTree
 PARTITION BY toDate(triggered_timestamp)
-ORDER BY (organization_id, environment_id, triggered_timestamp)
+ORDER BY (organization_id, environment_id, triggered_timestamp, span_id)
 --Right now we have maximum retention of up to 30 days based on plan.
 --We put a logical limit for now, the 90 DAY TTL is just a backup
 --This might need to be updated for longer retention periods
@@ -55,6 +55,7 @@ WHERE
     kind != 'DEBUG_EVENT'
     AND status != 'PARTIAL'
     AND NOT (kind = 'SPAN_EVENT' AND attributes_text = '{}')
+    AND kind != 'ANCESTOR_OVERRIDE'
     AND message != 'trigger.dev/start';
 
 -- +goose Down