From aef53133d84195e263230fd0d5d34bbf3c0519dc Mon Sep 17 00:00:00 2001 From: matt Date: Mon, 9 Mar 2026 05:30:16 -0700 Subject: [PATCH 01/17] [Python] Feat: Hatchet OTel (#2881) * feat: first pass at auto otel impl * refactor: clean up a bit, naming, etc. * refactor: rm instance vars * fix: rm one more instance var * chore: notes to self * traces view * minor changes * trace view by task external id * go sdk instrumentation * e2e tests for Py SDK trace --------- Co-authored-by: Mohammed Nafees --- examples/go/opentelemetry/main.go | 162 ++++++ .../hatchet/trigger.py | 46 ++ .../hatchet/worker.py | 144 ++++++ frontend/app/package.json | 5 + frontend/app/pnpm-lock.yaml | 55 ++ .../src/components/v1/agent-prism/Avatar.tsx | 143 ++++++ .../src/components/v1/agent-prism/Badge.tsx | 93 ++++ .../components/v1/agent-prism/BrandLogo.tsx | 102 ++++ .../src/components/v1/agent-prism/Button.tsx | 121 +++++ .../agent-prism/CollapseAndExpandControls.tsx | 43 ++ .../v1/agent-prism/CollapsibleSection.tsx | 124 +++++ .../components/v1/agent-prism/CopyButton.tsx | 62 +++ .../agent-prism/DetailsView/DetailsView.tsx | 140 +++++ .../DetailsView/DetailsViewAttributesTab.tsx | 123 +++++ .../DetailsView/DetailsViewContentViewer.tsx | 50 ++ .../DetailsView/DetailsViewHeader.tsx | 97 ++++ .../DetailsView/DetailsViewHeaderActions.tsx | 21 + .../DetailsView/DetailsViewInputOutputTab.tsx | 123 +++++ .../DetailsView/DetailsViewJsonOutput.tsx | 29 ++ .../DetailsView/DetailsViewRawDataTab.tsx | 29 ++ .../components/v1/agent-prism/IconButton.tsx | 73 +++ .../components/v1/agent-prism/PriceBadge.tsx | 12 + .../components/v1/agent-prism/SearchInput.tsx | 16 + .../components/v1/agent-prism/SpanBadge.tsx | 50 ++ .../v1/agent-prism/SpanCard/SpanCard.tsx | 477 ++++++++++++++++++ .../agent-prism/SpanCard/SpanCardBadges.tsx | 14 + .../SpanCard/SpanCardConnector.tsx | 36 ++ .../agent-prism/SpanCard/SpanCardTimeline.tsx | 59 +++ .../agent-prism/SpanCard/SpanCardToggle.tsx | 38 ++ .../components/v1/agent-prism/SpanStatus.tsx | 79 +++ .../components/v1/agent-prism/TabSelector.tsx | 34 ++ .../src/components/v1/agent-prism/Tabs.tsx | 139 +++++ .../components/v1/agent-prism/TextInput.tsx | 143 ++++++ .../v1/agent-prism/TimestampBadge.tsx | 20 + .../components/v1/agent-prism/TokensBadge.tsx | 24 + .../v1/agent-prism/TraceList/TraceList.tsx | 84 +++ .../agent-prism/TraceList/TraceListItem.tsx | 83 +++ .../TraceList/TraceListItemHeader.tsx | 35 ++ .../agent-prism/TraceViewer/TraceViewer.tsx | 167 ++++++ .../TraceViewer/TraceViewerDesktopLayout.tsx | 97 ++++ .../TraceViewer/TraceViewerMobileLayout.tsx | 101 ++++ .../TraceViewer/TraceViewerPlaceholder.tsx | 5 + .../TraceViewerSearchAndControls.tsx | 30 ++ .../TraceViewerTreeViewContainer.tsx | 74 +++ .../components/v1/agent-prism/TreeView.tsx | 70 +++ .../src/components/v1/agent-prism/shared.ts | 169 +++++++ .../components/v1/agent-prism/theme/index.ts | 99 ++++ .../components/v1/agent-prism/theme/theme.css | 245 +++++++++ .../app/src/lib/api/generated/cloud/Api.ts | 18 + .../lib/api/generated/cloud/data-contracts.ts | 23 + frontend/app/src/main.tsx | 1 + .../step-run-detail/otel-span-adapter.ts | 77 +++ .../step-run-detail/step-run-detail.tsx | 14 + .../step-run-detail/task-run-trace.tsx | 83 +++ frontend/app/tailwind.config.js | 32 ++ pkg/repository/otelcol.go | 24 + pkg/worker/context.go | 18 + pkg/worker/middleware_test.go | 12 + sdks/go/client.go | 13 + sdks/go/examples/opentelemetry/main.go | 162 ++++++ sdks/go/opentelemetry/attributes.go | 45 ++ sdks/go/opentelemetry/exporter.go | 30 ++ sdks/go/opentelemetry/instrumentor.go | 139 +++++ sdks/go/opentelemetry/middleware.go | 59 +++ sdks/go/opentelemetry/span_processor.go | 41 ++ .../hatchet/trigger.py | 46 ++ .../hatchet/worker.py | 144 ++++++ sdks/python/hatchet_sdk/hatchet.py | 2 + .../hatchet_sdk/opentelemetry/instrumentor.py | 117 ++++- .../hatchet_sdk/worker/runner/runner.py | 12 +- sdks/python/tests/otel_traces/__init__.py | 0 .../tests/otel_traces/test_otel_traces.py | 289 +++++++++++ sdks/python/tests/otel_traces/worker.py | 139 +++++ 73 files changed, 5711 insertions(+), 14 deletions(-) create mode 100644 examples/go/opentelemetry/main.go create mode 100644 examples/python/opentelemetry_instrumentation/hatchet/trigger.py create mode 100644 examples/python/opentelemetry_instrumentation/hatchet/worker.py create mode 100644 frontend/app/src/components/v1/agent-prism/Avatar.tsx create mode 100644 frontend/app/src/components/v1/agent-prism/Badge.tsx create mode 100644 frontend/app/src/components/v1/agent-prism/BrandLogo.tsx create mode 100644 frontend/app/src/components/v1/agent-prism/Button.tsx create mode 100644 frontend/app/src/components/v1/agent-prism/CollapseAndExpandControls.tsx create mode 100644 frontend/app/src/components/v1/agent-prism/CollapsibleSection.tsx create mode 100644 frontend/app/src/components/v1/agent-prism/CopyButton.tsx create mode 100644 frontend/app/src/components/v1/agent-prism/DetailsView/DetailsView.tsx create mode 100644 frontend/app/src/components/v1/agent-prism/DetailsView/DetailsViewAttributesTab.tsx create mode 100644 frontend/app/src/components/v1/agent-prism/DetailsView/DetailsViewContentViewer.tsx create mode 100644 frontend/app/src/components/v1/agent-prism/DetailsView/DetailsViewHeader.tsx create mode 100644 frontend/app/src/components/v1/agent-prism/DetailsView/DetailsViewHeaderActions.tsx create mode 100644 frontend/app/src/components/v1/agent-prism/DetailsView/DetailsViewInputOutputTab.tsx create mode 100644 frontend/app/src/components/v1/agent-prism/DetailsView/DetailsViewJsonOutput.tsx create mode 100644 frontend/app/src/components/v1/agent-prism/DetailsView/DetailsViewRawDataTab.tsx create mode 100644 frontend/app/src/components/v1/agent-prism/IconButton.tsx create mode 100644 frontend/app/src/components/v1/agent-prism/PriceBadge.tsx create mode 100644 frontend/app/src/components/v1/agent-prism/SearchInput.tsx create mode 100644 frontend/app/src/components/v1/agent-prism/SpanBadge.tsx create mode 100644 frontend/app/src/components/v1/agent-prism/SpanCard/SpanCard.tsx create mode 100644 frontend/app/src/components/v1/agent-prism/SpanCard/SpanCardBadges.tsx create mode 100644 frontend/app/src/components/v1/agent-prism/SpanCard/SpanCardConnector.tsx create mode 100644 frontend/app/src/components/v1/agent-prism/SpanCard/SpanCardTimeline.tsx create mode 100644 frontend/app/src/components/v1/agent-prism/SpanCard/SpanCardToggle.tsx create mode 100644 frontend/app/src/components/v1/agent-prism/SpanStatus.tsx create mode 100644 frontend/app/src/components/v1/agent-prism/TabSelector.tsx create mode 100644 frontend/app/src/components/v1/agent-prism/Tabs.tsx create mode 100644 frontend/app/src/components/v1/agent-prism/TextInput.tsx create mode 100644 frontend/app/src/components/v1/agent-prism/TimestampBadge.tsx create mode 100644 frontend/app/src/components/v1/agent-prism/TokensBadge.tsx create mode 100644 frontend/app/src/components/v1/agent-prism/TraceList/TraceList.tsx create mode 100644 frontend/app/src/components/v1/agent-prism/TraceList/TraceListItem.tsx create mode 100644 frontend/app/src/components/v1/agent-prism/TraceList/TraceListItemHeader.tsx create mode 100644 frontend/app/src/components/v1/agent-prism/TraceViewer/TraceViewer.tsx create mode 100644 frontend/app/src/components/v1/agent-prism/TraceViewer/TraceViewerDesktopLayout.tsx create mode 100644 frontend/app/src/components/v1/agent-prism/TraceViewer/TraceViewerMobileLayout.tsx create mode 100644 frontend/app/src/components/v1/agent-prism/TraceViewer/TraceViewerPlaceholder.tsx create mode 100644 frontend/app/src/components/v1/agent-prism/TraceViewer/TraceViewerSearchAndControls.tsx create mode 100644 frontend/app/src/components/v1/agent-prism/TraceViewer/TraceViewerTreeViewContainer.tsx create mode 100644 frontend/app/src/components/v1/agent-prism/TreeView.tsx create mode 100644 frontend/app/src/components/v1/agent-prism/shared.ts create mode 100644 frontend/app/src/components/v1/agent-prism/theme/index.ts create mode 100644 frontend/app/src/components/v1/agent-prism/theme/theme.css create mode 100644 frontend/app/src/pages/main/v1/workflow-runs-v1/$run/v2components/step-run-detail/otel-span-adapter.ts create mode 100644 frontend/app/src/pages/main/v1/workflow-runs-v1/$run/v2components/step-run-detail/task-run-trace.tsx create mode 100644 sdks/go/examples/opentelemetry/main.go create mode 100644 sdks/go/opentelemetry/attributes.go create mode 100644 sdks/go/opentelemetry/exporter.go create mode 100644 sdks/go/opentelemetry/instrumentor.go create mode 100644 sdks/go/opentelemetry/middleware.go create mode 100644 sdks/go/opentelemetry/span_processor.go create mode 100644 sdks/python/examples/opentelemetry_instrumentation/hatchet/trigger.py create mode 100644 sdks/python/examples/opentelemetry_instrumentation/hatchet/worker.py create mode 100644 sdks/python/tests/otel_traces/__init__.py create mode 100644 sdks/python/tests/otel_traces/test_otel_traces.py create mode 100644 sdks/python/tests/otel_traces/worker.py diff --git a/examples/go/opentelemetry/main.go b/examples/go/opentelemetry/main.go new file mode 100644 index 0000000000..3b6fd8c75b --- /dev/null +++ b/examples/go/opentelemetry/main.go @@ -0,0 +1,162 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "log" + "math/rand/v2" //nolint:gosec // G404: example code, not security-sensitive + "time" + + "go.opentelemetry.io/otel" + + "github.com/hatchet-dev/hatchet/pkg/cmdutils" + hatchet "github.com/hatchet-dev/hatchet/sdks/go" + hatchetotel "github.com/hatchet-dev/hatchet/sdks/go/opentelemetry" +) + +type PipelineInput struct { + URL string `json:"url"` +} + +type FetchOutput struct { + Data string `json:"data"` +} + +type ValidateOutput struct { + Valid bool `json:"valid"` + FieldCount int `json:"field_count"` +} + +type ProcessOutput struct { + ProcessedData string `json:"processed_data"` + RecordCount int `json:"record_count"` +} + +type SaveOutput struct { + Location string `json:"location"` + RecordsSaved int `json:"records_saved"` +} + +func randMillis(base, jitter int) time.Duration { + return time.Duration(base+rand.IntN(jitter)) * time.Millisecond //nolint:gosec // G404 +} + +func main() { + client, err := hatchet.NewClient() + if err != nil { + log.Fatalf("failed to create client: %v", err) + } + + // Set up OpenTelemetry instrumentation. + // EnableHatchetCollector() auto-configures from the same env vars as the client + // (HATCHET_CLIENT_HOST_PORT, HATCHET_CLIENT_TOKEN, HATCHET_CLIENT_TLS_STRATEGY). + instrumentor, err := hatchetotel.NewInstrumentor( + hatchetotel.EnableHatchetCollector(), + ) + if err != nil { + log.Fatalf("failed to create instrumentor: %v", err) + } + + tracer := otel.Tracer("otel-example") + + // Create a multi-task workflow + workflow := client.NewWorkflow("otel-data-pipeline") + + fetchData := workflow.NewTask("fetch-data", func(ctx hatchet.Context, input PipelineInput) (*FetchOutput, error) { + _, span := tracer.Start(ctx.GetContext(), fmt.Sprintf("GET %s", input.URL)) + time.Sleep(randMillis(10, 20)) + span.End() + + _, parseSpan := tracer.Start(ctx.GetContext(), "json.parse") + time.Sleep(randMillis(5, 10)) + parseSpan.End() + + return &FetchOutput{ + Data: `{"users": [{"name": "Alice"}, {"name": "Bob"}]}`, + }, nil + }) + + validateData := workflow.NewTask("validate-data", func(ctx hatchet.Context, input PipelineInput) (*ValidateOutput, error) { + var parentOutput FetchOutput + if parentErr := ctx.ParentOutput(fetchData, &parentOutput); parentErr != nil { + return nil, parentErr + } + + _, span := tracer.Start(ctx.GetContext(), "schema.validate") + time.Sleep(randMillis(5, 10)) + + var parsed map[string]any + if unmarshalErr := json.Unmarshal([]byte(parentOutput.Data), &parsed); unmarshalErr != nil { + span.End() + return nil, fmt.Errorf("invalid JSON: %w", unmarshalErr) + } + span.End() + + return &ValidateOutput{ + Valid: true, + FieldCount: len(parsed), + }, nil + }, hatchet.WithParents(fetchData)) + + processData := workflow.NewTask("process-data", func(ctx hatchet.Context, input PipelineInput) (*ProcessOutput, error) { + var validateOutput ValidateOutput + if parentErr := ctx.ParentOutput(validateData, &validateOutput); parentErr != nil { + return nil, parentErr + } + + _, span := tracer.Start(ctx.GetContext(), "data.transform") + time.Sleep(randMillis(10, 15)) + span.End() + + _, enrichSpan := tracer.Start(ctx.GetContext(), "data.enrich") + time.Sleep(randMillis(5, 10)) + enrichSpan.End() + + return &ProcessOutput{ + ProcessedData: "transformed_and_enriched", + RecordCount: validateOutput.FieldCount, + }, nil + }, hatchet.WithParents(validateData)) + + workflow.NewTask("save-results", func(ctx hatchet.Context, input PipelineInput) (*SaveOutput, error) { + var processOutput ProcessOutput + if parentErr := ctx.ParentOutput(processData, &processOutput); parentErr != nil { + return nil, parentErr + } + + _, span := tracer.Start(ctx.GetContext(), "db.insert") + time.Sleep(randMillis(10, 20)) + span.End() + + return &SaveOutput{ + RecordsSaved: processOutput.RecordCount, + Location: "postgresql://localhost/pipeline_results", + }, nil + }, hatchet.WithParents(processData)) + + // Create worker and register the OTel middleware + worker, err := client.NewWorker("otel-worker", hatchet.WithWorkflows(workflow)) + if err != nil { + log.Fatalf("failed to create worker: %v", err) + } + + worker.Use(instrumentor.Middleware()) + + interruptCtx, cancel := cmdutils.NewInterruptContext() + defer cancel() + + fmt.Println("Starting worker with OpenTelemetry instrumentation...") + + go func() { + <-interruptCtx.Done() + // Flush remaining spans before exit + if shutdownErr := instrumentor.Shutdown(context.Background()); shutdownErr != nil { + log.Printf("failed to shutdown instrumentor: %v", shutdownErr) + } + }() + + if startErr := worker.StartBlocking(interruptCtx); startErr != nil { + log.Printf("worker error: %v", startErr) + } +} diff --git a/examples/python/opentelemetry_instrumentation/hatchet/trigger.py b/examples/python/opentelemetry_instrumentation/hatchet/trigger.py new file mode 100644 index 0000000000..3e4913d3bd --- /dev/null +++ b/examples/python/opentelemetry_instrumentation/hatchet/trigger.py @@ -0,0 +1,46 @@ +""" +Trigger the OTelDataPipeline workflow. + +Make sure the worker is already running: + poetry run python -m examples.opentelemetry_instrumentation.hatchet.worker + +Then run this: + poetry run python -m examples.opentelemetry_instrumentation.hatchet.trigger +""" + +from opentelemetry.sdk.resources import SERVICE_NAME, Resource +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import ConsoleSpanExporter, SimpleSpanProcessor + +from examples.opentelemetry_instrumentation.hatchet.worker import otel_workflow +from hatchet_sdk.clients.admin import TriggerWorkflowOptions +from hatchet_sdk.opentelemetry.instrumentor import HatchetInstrumentor + +# Use the same console exporter so you can see trigger-side spans too +resource = Resource(attributes={SERVICE_NAME: "hatchet-otel-pipeline-trigger"}) +provider = TracerProvider(resource=resource) +provider.add_span_processor(SimpleSpanProcessor(ConsoleSpanExporter())) + +HatchetInstrumentor( + tracer_provider=provider, + enable_hatchet_otel_collector=True, +).instrument() + +tracer = provider.get_tracer(__name__) + + +def main() -> None: + # The run_workflow call is auto-traced with a "hatchet.run_workflow" span. + # The traceparent is automatically injected into additional_metadata, + # so the worker-side spans become children of this trigger span. + with tracer.start_as_current_span("trigger_otel_data_pipeline"): + result = otel_workflow.run( + options=TriggerWorkflowOptions( + additional_metadata={"source": "otel-example", "pipeline": "data-ingest"}, + ), + ) + print(f"Workflow result: {result}") + + +if __name__ == "__main__": + main() diff --git a/examples/python/opentelemetry_instrumentation/hatchet/worker.py b/examples/python/opentelemetry_instrumentation/hatchet/worker.py new file mode 100644 index 0000000000..0b0d73b034 --- /dev/null +++ b/examples/python/opentelemetry_instrumentation/hatchet/worker.py @@ -0,0 +1,144 @@ +""" +HatchetInstrumentor example with rich traces. + +Run the worker: + poetry run python -m examples.opentelemetry_instrumentation.hatchet.worker + +Then trigger it from another terminal: + poetry run python -m examples.opentelemetry_instrumentation.hatchet.trigger +""" + +import time + +from opentelemetry.sdk.resources import SERVICE_NAME, Resource +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import ConsoleSpanExporter, SimpleSpanProcessor +from opentelemetry.trace import StatusCode + +from hatchet_sdk import Context, EmptyModel, Hatchet +from hatchet_sdk.opentelemetry.instrumentor import HatchetInstrumentor + +hatchet = Hatchet() + +otel_workflow = hatchet.workflow(name="OTelDataPipeline") + +# Module-level tracer — will be set in main() before the worker starts. +# Tasks use this to create custom child spans inside the auto-instrumented +# hatchet.start_step_run parent span. +_tracer = None + + +def _get_tracer(): + global _tracer + if _tracer is None: + from opentelemetry.trace import get_tracer + + _tracer = get_tracer(__name__) + return _tracer + + +@otel_workflow.task() +def fetch_data(input: EmptyModel, ctx: Context) -> dict[str, str]: + tracer = _get_tracer() + + with tracer.start_as_current_span( + "http.request", + attributes={"http.method": "GET", "http.url": "https://api.example.com/data"}, + ) as span: + time.sleep(0.05) + span.set_attribute("http.status_code", 200) + span.set_attribute("http.response_content_length", 4096) + + with tracer.start_as_current_span("json.parse") as span: + time.sleep(0.01) + span.set_attribute("json.record_count", 42) + + return {"records_fetched": "42"} + + +@otel_workflow.task() +def validate_data(input: EmptyModel, ctx: Context) -> dict[str, str]: + tracer = _get_tracer() + + with tracer.start_as_current_span("schema.validate") as span: + time.sleep(0.02) + span.set_attribute("validation.schema", "v2.1") + span.set_attribute("validation.records_checked", 42) + span.set_attribute("validation.errors", 2) + span.set_status(StatusCode.OK, "2 records failed validation") + + with tracer.start_as_current_span("data.clean") as span: + time.sleep(0.01) + span.set_attribute("clean.records_dropped", 2) + span.set_attribute("clean.records_remaining", 40) + + return {"valid_records": "40", "dropped": "2"} + + +@otel_workflow.task() +def process_data(input: EmptyModel, ctx: Context) -> dict[str, str]: + tracer = _get_tracer() + + with tracer.start_as_current_span("transform.pipeline") as pipeline_span: + pipeline_span.set_attribute("pipeline.stages", 3) + + with tracer.start_as_current_span("transform.normalize"): + time.sleep(0.015) + + with tracer.start_as_current_span("transform.enrich") as enrich_span: + time.sleep(0.02) + enrich_span.set_attribute("enrich.source", "geocoding-api") + + with tracer.start_as_current_span("transform.aggregate") as agg_span: + time.sleep(0.03) + agg_span.set_attribute("aggregate.groups", 8) + agg_span.set_attribute("aggregate.method", "sum") + + return {"processed_groups": "8"} + + +@otel_workflow.task() +def save_results(input: EmptyModel, ctx: Context) -> dict[str, str]: + tracer = _get_tracer() + + with tracer.start_as_current_span( + "db.query", + attributes={"db.system": "postgresql", "db.operation": "INSERT"}, + ) as span: + time.sleep(0.04) + span.set_attribute("db.rows_affected", 8) + + with tracer.start_as_current_span("cache.invalidate") as span: + time.sleep(0.005) + span.set_attribute("cache.keys_invalidated", 3) + + with tracer.start_as_current_span("notification.send") as span: + time.sleep(0.01) + span.set_attribute("notification.channel", "webhook") + span.set_attribute("notification.status", "delivered") + + return {"saved": "true"} + + +def main() -> None: + resource = Resource(attributes={SERVICE_NAME: "hatchet-otel-pipeline-example"}) + provider = TracerProvider(resource=resource) + provider.add_span_processor(SimpleSpanProcessor(ConsoleSpanExporter())) + + HatchetInstrumentor( + tracer_provider=provider, + enable_hatchet_otel_collector=True, + ).instrument() + + global _tracer + _tracer = provider.get_tracer(__name__) + + worker = hatchet.worker( + "otel-pipeline-worker", + workflows=[otel_workflow], + ) + worker.start() + + +if __name__ == "__main__": + main() diff --git a/frontend/app/package.json b/frontend/app/package.json index 28504341c2..ce38767a26 100644 --- a/frontend/app/package.json +++ b/frontend/app/package.json @@ -23,6 +23,8 @@ "docs:gen": "cd ../snippets && python3 generate.py" }, "dependencies": { + "@evilmartians/agent-prism-data": "^0.0.9", + "@evilmartians/agent-prism-types": "^0.0.9", "@heroicons/react": "^2.2.0", "@hookform/resolvers": "^3.10.0", "@lukemorales/query-key-factory": "^1.3.4", @@ -57,6 +59,7 @@ "axios": "^1.13.5", "better-emitter": "^4.0.1", "class-variance-authority": "^0.7.1", + "classnames": "^2.5.1", "clsx": "^2.1.1", "cmdk": "^0.2.1", "cronstrue": "^2.57.0", @@ -75,6 +78,8 @@ "react-dom": "^18.3.1", "react-hook-form": "^7.55.0", "react-icons": "^5.5.0", + "react-json-pretty": "^2.2.0", + "react-resizable-panels": "^2.1.9", "react-syntax-highlighter": "^15.6.1", "reactflow": "^11.11.4", "recharts": "^2.15.1", diff --git a/frontend/app/pnpm-lock.yaml b/frontend/app/pnpm-lock.yaml index d270e60d91..bd8eee8ce7 100644 --- a/frontend/app/pnpm-lock.yaml +++ b/frontend/app/pnpm-lock.yaml @@ -18,6 +18,12 @@ importers: .: dependencies: + '@evilmartians/agent-prism-data': + specifier: ^0.0.9 + version: 0.0.9 + '@evilmartians/agent-prism-types': + specifier: ^0.0.9 + version: 0.0.9 '@heroicons/react': specifier: ^2.2.0 version: 2.2.0(react@18.3.1) @@ -120,6 +126,9 @@ importers: class-variance-authority: specifier: ^0.7.1 version: 0.7.1 + classnames: + specifier: ^2.5.1 + version: 2.5.1 clsx: specifier: ^2.1.1 version: 2.1.1 @@ -174,6 +183,12 @@ importers: react-icons: specifier: ^5.5.0 version: 5.5.0(react@18.3.1) + react-json-pretty: + specifier: ^2.2.0 + version: 2.2.0(react-dom@18.3.1(react@18.3.1))(react@18.3.1) + react-resizable-panels: + specifier: ^2.1.9 + version: 2.1.9(react-dom@18.3.1(react@18.3.1))(react@18.3.1) react-syntax-highlighter: specifier: ^15.6.1 version: 15.6.1(react@18.3.1) @@ -572,6 +587,12 @@ packages: resolution: {integrity: sha512-d9zaMRSTIKDLhctzH12MtXvJKSSUhaHcjV+2Z+GK+EEY7XKpP5yR4x+N3TAcHTcu963nIr+TMcCb4DBCYX1z6Q==} engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} + '@evilmartians/agent-prism-data@0.0.9': + resolution: {integrity: sha512-CyFGIMfTfRRhr+6c/EB+6wT2irx0x6gAMK7cCnmvGJtq/Hs/EWXnxprMCMf1Ct9dErlcYXiF8Y5AZbQP82ICoA==} + + '@evilmartians/agent-prism-types@0.0.9': + resolution: {integrity: sha512-rQW62z5XGF31oMnmdzNiGwzKTYJigsxKoPWWH/ZeVAxFi0c4bi+f8TndqgAebGBdHWcn5JRaFDZ76ywlNu4c6A==} + '@floating-ui/core@1.7.3': resolution: {integrity: sha512-sGnvb5dmrJaKEZ+LDIpguvdX3bDlEllmv4/ClQ9awcmCZrlx5jQyyMWFM5kBI+EyNOCDDiKk8il0zeuX3Zlg/w==} @@ -2121,6 +2142,9 @@ packages: classcat@5.0.5: resolution: {integrity: sha512-JhZUT7JFcQy/EzW605k/ktHtncoo9vnyW/2GspNYwFlN1C/WmjuV/xtS04e9SOkL2sTdw0VAZ2UGCcQ9lR6p6w==} + classnames@2.5.1: + resolution: {integrity: sha512-saHYOzhIQs6wy2sVxTM6bUDsQO4F50V9RQ22qBpEdCW+I+/Wmke2HOl6lS6dTpdxVhb88/I6+Hs+438c3lfUow==} + clean-stack@2.2.0: resolution: {integrity: sha512-4diC9HaTE+KRAMWhDhrGOECgWZxoevMc5TlkObMqNSsVU62PYzXZ/SMTjzyGAFF1YusgxGcSWTEXBhp0CPwQ1A==} engines: {node: '>=6'} @@ -3663,6 +3687,12 @@ packages: react-is@18.3.1: resolution: {integrity: sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg==} + react-json-pretty@2.2.0: + resolution: {integrity: sha512-3UMzlAXkJ4R8S4vmkRKtvJHTewG4/rn1Q18n0zqdu/ipZbUPLVZD+QwC7uVcD/IAY3s8iNVHlgR2dMzIUS0n1A==} + peerDependencies: + react: '>=15.0' + react-dom: '>=15.0' + react-refresh@0.17.0: resolution: {integrity: sha512-z6F7K9bV85EfseRCp2bzrpyQ0Gkw1uLoCel9XBVWPg/TjRj94SkJzUTGfOa4bs7iJvBWtQG0Wq7wnI0syw3EBQ==} engines: {node: '>=0.10.0'} @@ -3697,6 +3727,12 @@ packages: '@types/react': optional: true + react-resizable-panels@2.1.9: + resolution: {integrity: sha512-z77+X08YDIrgAes4jl8xhnUu1LNIRp4+E7cv4xHmLOxxUPO/ML7PSrE813b90vj7xvQ1lcf7g2uA9GeMZonjhQ==} + peerDependencies: + react: ^16.14.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc + react-dom: ^16.14.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc + react-smooth@4.0.4: resolution: {integrity: sha512-gnGKTpYwqL0Iii09gHobNolvX4Kiq4PKx6eWBCYYix+8cdw+cGo3do906l1NBPKkSWx1DghC1dlWG9L2uGd61Q==} peerDependencies: @@ -4623,6 +4659,12 @@ snapshots: '@eslint/js@8.57.1': {} + '@evilmartians/agent-prism-data@0.0.9': + dependencies: + '@evilmartians/agent-prism-types': 0.0.9 + + '@evilmartians/agent-prism-types@0.0.9': {} + '@floating-ui/core@1.7.3': dependencies: '@floating-ui/utils': 0.2.10 @@ -6325,6 +6367,8 @@ snapshots: classcat@5.0.5: {} + classnames@2.5.1: {} + clean-stack@2.2.0: {} cli-cursor@3.1.0: @@ -7939,6 +7983,12 @@ snapshots: react-is@18.3.1: {} + react-json-pretty@2.2.0(react-dom@18.3.1(react@18.3.1))(react@18.3.1): + dependencies: + prop-types: 15.8.1 + react: 18.3.1 + react-dom: 18.3.1(react@18.3.1) + react-refresh@0.17.0: {} react-remove-scroll-bar@2.3.8(@types/react@18.3.23)(react@18.3.1): @@ -7971,6 +8021,11 @@ snapshots: optionalDependencies: '@types/react': 18.3.23 + react-resizable-panels@2.1.9(react-dom@18.3.1(react@18.3.1))(react@18.3.1): + dependencies: + react: 18.3.1 + react-dom: 18.3.1(react@18.3.1) + react-smooth@4.0.4(react-dom@18.3.1(react@18.3.1))(react@18.3.1): dependencies: fast-equals: 5.2.2 diff --git a/frontend/app/src/components/v1/agent-prism/Avatar.tsx b/frontend/app/src/components/v1/agent-prism/Avatar.tsx new file mode 100644 index 0000000000..3005f2dae2 --- /dev/null +++ b/frontend/app/src/components/v1/agent-prism/Avatar.tsx @@ -0,0 +1,143 @@ +import type { ComponentSize } from './shared'; +import { ROUNDED_CLASSES } from './shared'; +import type { TraceSpanCategory } from '@evilmartians/agent-prism-types'; +import cn from 'classnames'; +import { User } from 'lucide-react'; +import type { ComponentPropsWithRef, ReactElement } from 'react'; +import { useState } from 'react'; + +export type AvatarSize = Extract< + ComponentSize, + '4' | '6' | '8' | '9' | '10' | '11' | '12' | '16' +>; + +const sizeClasses: Record = { + '4': 'size-4 text-xs', + '6': 'size-6 text-xs', + '8': 'size-8 text-xs', + '9': 'size-9 text-sm', + '10': 'size-10 text-base', + '11': 'size-11 text-lg', + '12': 'size-12 text-xl', + '16': 'size-16 text-2xl', +}; + +const iconSizeClasses: Record = { + '4': 'size-3', + '6': 'size-4', + '8': 'size-6', + '9': 'size-7', + '10': 'size-8', + '11': 'size-9', + '12': 'size-10', + '16': 'size-12', +}; + +const bgColorClasses: Record = { + llm_call: 'bg-agentprism-avatar-llm', + tool_execution: 'bg-agentprism-avatar-tool', + agent_invocation: 'bg-agentprism-avatar-agent', + chain_operation: 'bg-agentprism-avatar-chain', + retrieval: 'bg-agentprism-avatar-retrieval', + embedding: 'bg-agentprism-avatar-embedding', + create_agent: 'bg-agentprism-avatar-create-agent', + span: 'bg-agentprism-avatar-span', + event: 'bg-agentprism-avatar-event', + guardrail: 'bg-agentprism-avatar-guardrail', + unknown: 'bg-agentprism-avatar-unknown', +}; + +export type AvatarProps = ComponentPropsWithRef<'div'> & { + /** + * The category of the span which avatar is associated with + */ + category: TraceSpanCategory; + /** + * The image source for the avatar + */ + src?: string; + /** + * The alt text for the avatar + */ + alt?: string; + /** + * The size of the avatar + * @default "md" + */ + size?: AvatarSize; + /** + * The border radius of the avatar + * @default "full" + */ + rounded?: 'none' | 'sm' | 'md' | 'lg' | 'full'; + /** + * Custom letter to display (will use first letter of alt if not provided) + */ + letter?: string; + /** + * Optional className for additional styling + */ + className?: string; +}; + +export const Avatar = ({ + category, + src, + alt = 'Avatar', + size = '10', + rounded = 'full', + letter, + children, + className = '', + ...rest +}: AvatarProps): ReactElement => { + const [error, setError] = useState(false); + + const displayLetter = letter ? letter.charAt(0) : alt.charAt(0).toUpperCase(); + + return ( +
+ {children ? ( + children + ) : error ? ( + + ) : ( + <> + {src ? ( + {alt} setError(true)} + /> + ) : ( +
+ {displayLetter} +
+ )} + + )} +
+ ); +}; diff --git a/frontend/app/src/components/v1/agent-prism/Badge.tsx b/frontend/app/src/components/v1/agent-prism/Badge.tsx new file mode 100644 index 0000000000..6840456ac6 --- /dev/null +++ b/frontend/app/src/components/v1/agent-prism/Badge.tsx @@ -0,0 +1,93 @@ +import type { ComponentSize } from './shared'; +import cn from 'classnames'; +import type { ComponentPropsWithRef, ReactElement, ReactNode } from 'react'; + +type BadgeSize = Extract; + +const sizeClasses: Record = { + '4': 'px-1 gap-1 h-4', + '5': 'px-1.5 gap-1 h-5', + '6': 'px-2 gap-1.5 h-6', + '7': 'px-2.5 gap-2 h-7', +}; + +const textSizes: Record = { + '4': 'text-xs leading-3', + '5': 'text-xs', + '6': 'text-sm', + '7': 'text-sm', +}; + +export type BadgeProps = ComponentPropsWithRef<'span'> & { + /** + * The content of the badge + */ + label: ReactNode; + + /** + * The size of the badge + * @default "md" + */ + size?: BadgeSize; + + /** + * Optional icon to display at the start of the badge + */ + iconStart?: ReactElement; + + /** + * Optional icon to display at the end of the badge + */ + iconEnd?: ReactElement; + + /** + * Optional className for additional styling + */ + className?: string; + + /** + * Whether to render the badge without any default styles + * @default false + */ + unstyled?: boolean; +}; + +/** + * An unstyled badge component that displays a label with an optional icon + */ +export const Badge = ({ + label, + size = '4', + iconStart, + iconEnd, + className = '', + unstyled = false, + ...rest +}: BadgeProps): ReactElement => { + return ( + + {iconStart && {iconStart}} + + + {label} + + + {iconEnd && {iconEnd}} + + ); +}; diff --git a/frontend/app/src/components/v1/agent-prism/BrandLogo.tsx b/frontend/app/src/components/v1/agent-prism/BrandLogo.tsx new file mode 100644 index 0000000000..70ff567590 --- /dev/null +++ b/frontend/app/src/components/v1/agent-prism/BrandLogo.tsx @@ -0,0 +1,102 @@ +import { type FC } from 'react'; + +const OpenAILogo: FC<{ className?: string }> = ({ className }) => ( + + + +); + +const AnthropicLogo: FC<{ className?: string }> = ({ className }) => ( + + + +); + +const GoogleLogo: FC<{ className?: string }> = ({ className }) => ( + + + + + + +); + +const MetaLogo: FC<{ className?: string }> = ({ className }) => ( + + + +); + +const MistralLogo: FC<{ className?: string }> = ({ className }) => ( + + + + + + + + + + + + + + + + + + + + + + + +); + +const PerplexityLogo: FC<{ className?: string }> = ({ className }) => ( + + + +); + +// Logo registry +const LOGO_REGISTRY = { + openai: OpenAILogo, + anthropic: AnthropicLogo, + google: GoogleLogo, + meta: MetaLogo, + mistral: MistralLogo, + perplexity: PerplexityLogo, +} as const; + +type BrandType = keyof typeof LOGO_REGISTRY; + +type BrandLogoProps = { + brand: BrandType | string; + className?: string; + fallback?: React.ReactNode; +}; + +export const BrandLogo: FC = ({ + brand, + className = 'size-4', + fallback = null, +}) => { + const Logo = LOGO_REGISTRY[brand as BrandType]; + + if (!Logo) return <>{fallback}; + + return ; +}; diff --git a/frontend/app/src/components/v1/agent-prism/Button.tsx b/frontend/app/src/components/v1/agent-prism/Button.tsx new file mode 100644 index 0000000000..68f5a71c67 --- /dev/null +++ b/frontend/app/src/components/v1/agent-prism/Button.tsx @@ -0,0 +1,121 @@ +import type { ComponentSize } from './shared'; +import { ROUNDED_CLASSES } from './shared'; +import cn from 'classnames'; +import type { ComponentPropsWithRef, ReactElement } from 'react'; + +type ButtonSize = Extract< + ComponentSize, + '6' | '7' | '8' | '9' | '10' | '11' | '12' | '16' +>; + +type ButtonVariant = + | 'brand' + | 'primary' + | 'outlined' + | 'secondary' + | 'ghost' + | 'destructive' + | 'success'; + +const BASE_CLASSES = + 'inline-flex items-center justify-center font-medium transition-all duration-200'; + +const sizeClasses = { + '6': 'h-6 px-2 gap-1 text-xs', + '7': 'h-7 px-2 gap-1 text-xs', + '8': 'h-8 px-2 gap-1 text-xs', + '9': 'h-9 px-2.5 gap-2 text-sm', + '10': 'h-10 px-4 gap-2 text-sm', + '11': 'h-11 px-5 gap-3 text-base', + '12': 'h-12 px-5 gap-2.5 text-base', + '16': 'h-16 px-7 gap-3 text-lg', +}; + +const variantClasses: Record = { + brand: 'text-agentprism-brand-foreground bg-agentprism-brand', + primary: 'text-agentprism-primary-foreground bg-agentprism-primary', + outlined: + 'border border bg-transparent text-agentprism-foreground border-agentprism-foreground', + secondary: 'bg-agentprism-secondary text-agentprism-secondary-foreground', + ghost: 'bg-transparent text-agentprism-foreground', + destructive: 'bg-agentprism-error text-agentprism-primary-foreground', + success: 'bg-agentprism-success text-agentprism-primary-foreground', +}; + +export type ButtonProps = ComponentPropsWithRef<'button'> & { + /** + * The size of the button + * @default "6" + */ + size?: ButtonSize; + + /** + * The border radius of the button + * @default "md" + */ + rounded?: 'none' | 'sm' | 'md' | 'lg' | 'full'; + + /** + * The visual variant of the button + * @default "primary" + */ + variant?: ButtonVariant; + + /** + * Makes the button full width + * @default false + */ + fullWidth?: boolean; + + /** + * Optional icon to display at the start of the button + */ + iconStart?: ReactElement; + + /** + * Optional icon to display at the end of the button + */ + iconEnd?: ReactElement; +}; + +export const Button = ({ + children, + size = '6', + rounded = 'md', + variant = 'primary', + fullWidth = false, + disabled = false, + iconStart, + iconEnd, + type = 'button', + onClick, + className = '', + ...rest +}: ButtonProps) => { + const widthClass = fullWidth ? 'w-full' : ''; + const stateClasses = disabled + ? 'cursor-not-allowed opacity-50' + : 'hover:opacity-70'; + + return ( + + ); +}; diff --git a/frontend/app/src/components/v1/agent-prism/CollapseAndExpandControls.tsx b/frontend/app/src/components/v1/agent-prism/CollapseAndExpandControls.tsx new file mode 100644 index 0000000000..25a53ce3ef --- /dev/null +++ b/frontend/app/src/components/v1/agent-prism/CollapseAndExpandControls.tsx @@ -0,0 +1,43 @@ +import { IconButton } from './IconButton'; +import { ChevronsUpDown, ChevronsDownUp } from 'lucide-react'; +import type { ComponentPropsWithRef } from 'react'; + +export type SpanCardExpandAllButtonProps = ComponentPropsWithRef<'button'> & { + onExpandAll: () => void; +}; + +export type SpanCardCollapseAllButtonProps = ComponentPropsWithRef<'button'> & { + onCollapseAll: () => void; +}; + +export const ExpandAllButton = ({ + onExpandAll, + ...rest +}: SpanCardExpandAllButtonProps) => { + return ( + + + + ); +}; + +export const CollapseAllButton = ({ + onCollapseAll, + ...rest +}: SpanCardCollapseAllButtonProps) => { + return ( + + + + ); +}; diff --git a/frontend/app/src/components/v1/agent-prism/CollapsibleSection.tsx b/frontend/app/src/components/v1/agent-prism/CollapsibleSection.tsx new file mode 100644 index 0000000000..5371de1976 --- /dev/null +++ b/frontend/app/src/components/v1/agent-prism/CollapsibleSection.tsx @@ -0,0 +1,124 @@ +import * as Collapsible from '@radix-ui/react-collapsible'; +import cn from 'classnames'; +import { ChevronDown } from 'lucide-react'; +import * as React from 'react'; + +export interface CollapsibleSectionProps { + /** + * The title text displayed in the trigger button + */ + title: string; + + /** + * The content to display on the right side of the title + */ + rightContent?: React.ReactNode; + + /** + * The content to display when the section is expanded + */ + children: React.ReactNode; + + /** + * Whether the section starts in an open state + * @default false + */ + defaultOpen?: boolean; + + /** + * Optional className for the root container + */ + className?: string; + + /** + * Optional className for the trigger button + */ + triggerClassName?: string; + + /** + * Optional className for the content area + */ + contentClassName?: string; + + /** + * Optional callback fired when the section is expanded or collapsed + */ + onOpenChange?: (open: boolean) => void; +} + +export const CollapsibleSection: React.FC = ({ + title, + rightContent, + children, + defaultOpen = false, + className = '', + triggerClassName = '', + contentClassName = '', + onOpenChange, +}) => { + const [open, setOpen] = React.useState(defaultOpen); + + const handleOpenChange = React.useCallback( + (open: boolean): void => { + setOpen(open); + onOpenChange?.(open); + }, + [onOpenChange], + ); + + const handleKeyDown = React.useCallback( + (e: React.KeyboardEvent): void => { + if (e.key === 'Enter' || e.key === ' ') { + e.preventDefault(); + handleOpenChange(!open); + } + }, + [handleOpenChange, open], + ); + + return ( + + +
+
+ + + {title} + +
+ +
{rightContent}
+
+
+ + + {children} + +
+ ); +}; diff --git a/frontend/app/src/components/v1/agent-prism/CopyButton.tsx b/frontend/app/src/components/v1/agent-prism/CopyButton.tsx new file mode 100644 index 0000000000..069af38ed3 --- /dev/null +++ b/frontend/app/src/components/v1/agent-prism/CopyButton.tsx @@ -0,0 +1,62 @@ +import { IconButton } from './IconButton'; +import { Check, Copy, X } from 'lucide-react'; +import { useState } from 'react'; + +type CopyButtonProps = { + label: string; + content: string; +}; + +type CopyState = 'idle' | 'success' | 'error'; + +export const CopyButton = ({ label, content }: CopyButtonProps) => { + const [copyState, setCopyState] = useState('idle'); + + const onClick = async () => { + try { + if (!navigator.clipboard) { + throw new Error('Clipboard API not supported'); + } + + await navigator.clipboard.writeText(content); + setCopyState('success'); + setTimeout(() => setCopyState('idle'), 2000); + } catch { + setCopyState('error'); + setTimeout(() => setCopyState('idle'), 2000); + } + }; + + const getIcon = () => { + switch (copyState) { + case 'success': + return ; + case 'error': + return ; + default: + return ; + } + }; + + const getAriaLabel = () => { + switch (copyState) { + case 'success': + return `${label} Copied`; + case 'error': + return `Failed to copy ${label}`; + default: + return `Copy ${label}`; + } + }; + + return ( + + {getIcon()} + + ); +}; diff --git a/frontend/app/src/components/v1/agent-prism/DetailsView/DetailsView.tsx b/frontend/app/src/components/v1/agent-prism/DetailsView/DetailsView.tsx new file mode 100644 index 0000000000..9308b19f19 --- /dev/null +++ b/frontend/app/src/components/v1/agent-prism/DetailsView/DetailsView.tsx @@ -0,0 +1,140 @@ +import type { AvatarProps } from '../Avatar'; +import { TabSelector } from '../TabSelector'; +import type { TabItem } from '../Tabs'; +import { DetailsViewAttributesTab } from './DetailsViewAttributesTab'; +import { DetailsViewHeader } from './DetailsViewHeader'; +import { DetailsViewInputOutputTab } from './DetailsViewInputOutputTab'; +import { DetailsViewRawDataTab } from './DetailsViewRawDataTab'; +import type { TraceSpan } from '@evilmartians/agent-prism-types'; +import cn from 'classnames'; +import { SquareTerminal, Tags, ArrowRightLeft } from 'lucide-react'; +import type { ReactElement, ReactNode } from 'react'; +import { useState } from 'react'; + +type DetailsViewTab = 'input-output' | 'attributes' | 'raw'; + +export interface DetailsViewProps { + /** + * The span data to display in the details view + */ + data: TraceSpan; + + /** + * Optional avatar configuration for the header + */ + avatar?: AvatarProps; + + /** + * The initially selected tab + */ + defaultTab?: DetailsViewTab; + + /** + * Optional className for the root container + */ + className?: string; + + /** + * Configuration for the copy button functionality + */ + copyButton?: { + isEnabled?: boolean; + onCopy?: (data: TraceSpan) => void; + }; + + /** + * Custom header actions to render + * Can be a ReactNode or a render function that receives the data + */ + headerActions?: ReactNode | ((data: TraceSpan) => ReactNode); + + /** + * Optional custom header component to replace the default + */ + customHeader?: ReactNode | ((props: { data: TraceSpan }) => ReactNode); + + /** + * Callback fired when the active tab changes + */ + onTabChange?: (tabValue: DetailsViewTab) => void; +} + +const TAB_ITEMS: TabItem[] = [ + { + value: 'input-output', + label: 'In/Out', + icon: , + }, + { + value: 'attributes', + label: 'Attributes', + icon: , + }, + { + value: 'raw', + label: 'RAW', + icon: , + }, +]; + +export const DetailsView = ({ + data, + avatar, + defaultTab = 'input-output', + className, + copyButton, + headerActions, + customHeader, + onTabChange, +}: DetailsViewProps): ReactElement => { + const [tab, setTab] = useState(defaultTab); + + const handleTabChange = (tabValue: DetailsViewTab) => { + setTab(tabValue); + onTabChange?.(tabValue); + }; + + const resolvedHeaderActions = + typeof headerActions === 'function' ? headerActions(data) : headerActions; + + const headerContent = customHeader ? ( + typeof customHeader === 'function' ? ( + customHeader({ data }) + ) : ( + customHeader + ) + ) : ( + + ); + + return ( +
+
{headerContent}
+
+ +
+ +
+ {tab === 'input-output' && } + {tab === 'attributes' && } + {tab === 'raw' && } +
+
+ ); +}; diff --git a/frontend/app/src/components/v1/agent-prism/DetailsView/DetailsViewAttributesTab.tsx b/frontend/app/src/components/v1/agent-prism/DetailsView/DetailsViewAttributesTab.tsx new file mode 100644 index 0000000000..46731ceef8 --- /dev/null +++ b/frontend/app/src/components/v1/agent-prism/DetailsView/DetailsViewAttributesTab.tsx @@ -0,0 +1,123 @@ +import { CollapsibleSection } from '../CollapsibleSection'; +import { TabSelector } from '../TabSelector'; +import type { TabItem } from '../Tabs'; +import { + DetailsViewContentViewer, + type DetailsViewContentViewMode, +} from './DetailsViewContentViewer'; +import { type TraceSpan } from '@evilmartians/agent-prism-types'; +import { type ReactElement, useState } from 'react'; + +interface AttributesTabProps { + data: TraceSpan; +} + +const TAB_ITEMS: TabItem[] = [ + { value: 'json', label: 'JSON' }, + { value: 'plain', label: 'Plain' }, +]; + +export const DetailsViewAttributesTab = ({ + data, +}: AttributesTabProps): ReactElement => { + if (!data.attributes || data.attributes.length === 0) { + return ( +
+

+ No attributes available for this span. +

+
+ ); + } + + return ( +
+ {data.attributes.map((attribute, index) => { + const stringValue = attribute.value.stringValue; + const simpleValue = + stringValue || + attribute.value.intValue?.toString() || + attribute.value.boolValue?.toString() || + 'N/A'; + + let parsedJson: string | null = null; + if (typeof stringValue === 'string') { + try { + parsedJson = JSON.parse(stringValue); + } catch { + parsedJson = null; + } + } + + const isComplex = parsedJson !== null; + + if (isComplex && parsedJson && stringValue) { + return ( + + ); + } + + return ( +
+
+ {attribute.key} +
+
+ {simpleValue} +
+
+ ); + })} +
+ ); +}; + +interface AttributeSectionProps { + attributeKey: string; + content: string; + parsedContent: string; + id: string; +} + +const AttributeSection = ({ + attributeKey, + content, + parsedContent, + id, +}: AttributeSectionProps): ReactElement => { + const [tab, setTab] = useState('json'); + + return ( + + items={TAB_ITEMS} + defaultValue="json" + value={tab} + onValueChange={setTab} + theme="pill" + onClick={(event) => event.stopPropagation()} + /> + } + > + + + ); +}; diff --git a/frontend/app/src/components/v1/agent-prism/DetailsView/DetailsViewContentViewer.tsx b/frontend/app/src/components/v1/agent-prism/DetailsView/DetailsViewContentViewer.tsx new file mode 100644 index 0000000000..75d5b74856 --- /dev/null +++ b/frontend/app/src/components/v1/agent-prism/DetailsView/DetailsViewContentViewer.tsx @@ -0,0 +1,50 @@ +import { CopyButton } from '../CopyButton'; +import { DetailsViewJsonOutput } from './DetailsViewJsonOutput'; +import { type ReactElement } from 'react'; + +export type DetailsViewContentViewMode = 'json' | 'plain'; + +export interface DetailsViewContentViewerProps { + content: string; + parsedContent: string | null; + mode: DetailsViewContentViewMode; + label: string; + id: string; + className?: string; +} + +export const DetailsViewContentViewer = ({ + content, + parsedContent, + mode, + label, + id, + className = '', +}: DetailsViewContentViewerProps): ReactElement => { + if (!content) { + return ( +

+ No data available +

+ ); + } + + return ( +
+
+ +
+ {mode === 'json' && parsedContent ? ( + + ) : ( +
+
+            {content}
+          
+
+ )} +
+ ); +}; diff --git a/frontend/app/src/components/v1/agent-prism/DetailsView/DetailsViewHeader.tsx b/frontend/app/src/components/v1/agent-prism/DetailsView/DetailsViewHeader.tsx new file mode 100644 index 0000000000..bcae84e108 --- /dev/null +++ b/frontend/app/src/components/v1/agent-prism/DetailsView/DetailsViewHeader.tsx @@ -0,0 +1,97 @@ +import type { AvatarProps } from '../Avatar'; +import { Avatar } from '../Avatar'; +import { IconButton } from '../IconButton'; +import { PriceBadge } from '../PriceBadge'; +import { SpanBadge } from '../SpanBadge'; +import { SpanStatus } from '../SpanStatus'; +import { TimestampBadge } from '../TimestampBadge'; +import { TokensBadge } from '../TokensBadge'; +import { getDurationMs, formatDuration } from '@evilmartians/agent-prism-data'; +import type { TraceSpan } from '@evilmartians/agent-prism-types'; +import { Check, Copy } from 'lucide-react'; +import type { ReactNode } from 'react'; +import { useState } from 'react'; + +export interface DetailsViewHeaderProps { + data: TraceSpan; + avatar?: AvatarProps; + copyButton?: { + isEnabled?: boolean; + onCopy?: (data: TraceSpan) => void; + }; + /** + * Custom actions to render in the header + */ + actions?: ReactNode; + /** + * Optional className for the header container + */ + className?: string; +} + +export const DetailsViewHeader = ({ + data, + avatar, + copyButton, + actions, + className, +}: DetailsViewHeaderProps) => { + const [hasCopied, setHasCopied] = useState(false); + const durationMs = getDurationMs(data); + + const handleCopy = () => { + if (copyButton?.onCopy) { + copyButton.onCopy(data); + setHasCopied(true); + setTimeout(() => setHasCopied(false), 2000); + } + }; + + return ( +
+ {avatar && } + + + {data.title} + + +
+ +
+ + {copyButton && ( + + {hasCopied ? ( + + ) : ( + + )} + + )} + + + + {typeof data.tokensCount === 'number' && ( + + )} + + {typeof data.cost === 'number' && } + + + LATENCY: {formatDuration(durationMs)} + + + {typeof data.startTime === 'number' && ( + + )} + + {actions} +
+ ); +}; diff --git a/frontend/app/src/components/v1/agent-prism/DetailsView/DetailsViewHeaderActions.tsx b/frontend/app/src/components/v1/agent-prism/DetailsView/DetailsViewHeaderActions.tsx new file mode 100644 index 0000000000..ffe8387f75 --- /dev/null +++ b/frontend/app/src/components/v1/agent-prism/DetailsView/DetailsViewHeaderActions.tsx @@ -0,0 +1,21 @@ +import type { ReactNode } from 'react'; + +export interface DetailsViewHeaderActionsProps { + /** + * Custom actions to render in the header + */ + children?: ReactNode; + /** + * Optional className for the actions container + */ + className?: string; +} + +export const DetailsViewHeaderActions = ({ + children, + className = 'flex flex-wrap items-center gap-2', +}: DetailsViewHeaderActionsProps) => { + if (!children) return null; + + return
{children}
; +}; diff --git a/frontend/app/src/components/v1/agent-prism/DetailsView/DetailsViewInputOutputTab.tsx b/frontend/app/src/components/v1/agent-prism/DetailsView/DetailsViewInputOutputTab.tsx new file mode 100644 index 0000000000..3e772b07d3 --- /dev/null +++ b/frontend/app/src/components/v1/agent-prism/DetailsView/DetailsViewInputOutputTab.tsx @@ -0,0 +1,123 @@ +import { CollapsibleSection } from '../CollapsibleSection'; +import { TabSelector } from '../TabSelector'; +import type { TabItem } from '../Tabs'; +import { + DetailsViewContentViewer, + type DetailsViewContentViewMode, +} from './DetailsViewContentViewer'; +import type { TraceSpan } from '@evilmartians/agent-prism-types'; +import type { ReactElement } from 'react'; +import { useState, useEffect } from 'react'; + +interface DetailsViewInputOutputTabProps { + data: TraceSpan; +} + +type IOSection = 'Input' | 'Output'; + +export const DetailsViewInputOutputTab = ({ + data, +}: DetailsViewInputOutputTabProps): ReactElement => { + const hasInput = Boolean(data.input); + const hasOutput = Boolean(data.output); + + if (!hasInput && !hasOutput) { + return ( +
+

+ No input or output data available for this span +

+
+ ); + } + + let parsedInput: string | null = null; + let parsedOutput: string | null = null; + + if (typeof data.input === 'string') { + try { + parsedInput = JSON.parse(data.input); + } catch { + parsedInput = null; + } + } + + if (typeof data.output === 'string') { + try { + parsedOutput = JSON.parse(data.output); + } catch { + parsedOutput = null; + } + } + + return ( +
+ {typeof data.input === 'string' && ( + + )} + {typeof data.output === 'string' && ( + + )} +
+ ); +}; + +interface IOSectionProps { + section: IOSection; + content: string; + parsedContent: string | null; +} + +const IOSection = ({ + section, + content, + parsedContent, +}: IOSectionProps): ReactElement => { + const [tab, setTab] = useState( + parsedContent ? 'json' : 'plain', + ); + + useEffect(() => { + if (tab === 'json' && !parsedContent) { + setTab('plain'); + } + }, [tab, parsedContent]); + + const tabItems: TabItem[] = [ + { value: 'json', label: 'JSON', disabled: !parsedContent }, + { value: 'plain', label: 'Plain' }, + ]; + + return ( + + items={tabItems} + defaultValue={parsedContent ? 'json' : 'plain'} + value={tab} + onValueChange={setTab} + theme="pill" + onClick={(event) => event.stopPropagation()} + /> + } + > + + + ); +}; diff --git a/frontend/app/src/components/v1/agent-prism/DetailsView/DetailsViewJsonOutput.tsx b/frontend/app/src/components/v1/agent-prism/DetailsView/DetailsViewJsonOutput.tsx new file mode 100644 index 0000000000..1087d35ee6 --- /dev/null +++ b/frontend/app/src/components/v1/agent-prism/DetailsView/DetailsViewJsonOutput.tsx @@ -0,0 +1,29 @@ +import { agentPrismPrefix } from '../theme'; +import { type FC } from 'react'; +import JSONPretty from 'react-json-pretty'; +import colors from 'tailwindcss/colors'; + +export interface JsonViewerProps { + content: string; + id: string; + className?: string; +} + +export const DetailsViewJsonOutput: FC = ({ + content, + id, + className = '', +}) => { + return ( + + ); +}; diff --git a/frontend/app/src/components/v1/agent-prism/DetailsView/DetailsViewRawDataTab.tsx b/frontend/app/src/components/v1/agent-prism/DetailsView/DetailsViewRawDataTab.tsx new file mode 100644 index 0000000000..0384459873 --- /dev/null +++ b/frontend/app/src/components/v1/agent-prism/DetailsView/DetailsViewRawDataTab.tsx @@ -0,0 +1,29 @@ +import { CopyButton } from '../CopyButton'; +import { DetailsViewJsonOutput } from './DetailsViewJsonOutput'; +import type { TraceSpan } from '@evilmartians/agent-prism-types'; +import { type ReactElement } from 'react'; + +interface RawDataTabProps { + data: TraceSpan; +} + +export const DetailsViewRawDataTab = ({ + data, +}: RawDataTabProps): ReactElement => ( +
+
+
+
+ +
+
+ +
+ +
+
+
+); diff --git a/frontend/app/src/components/v1/agent-prism/IconButton.tsx b/frontend/app/src/components/v1/agent-prism/IconButton.tsx new file mode 100644 index 0000000000..036eaa9a9f --- /dev/null +++ b/frontend/app/src/components/v1/agent-prism/IconButton.tsx @@ -0,0 +1,73 @@ +import type { ComponentSize } from './shared'; +import cn from 'classnames'; +import type { ComponentPropsWithRef } from 'react'; + +type IconButtonSize = Extract< + ComponentSize, + '6' | '7' | '8' | '9' | '10' | '11' | '12' | '16' +>; +type IconButtonVariant = 'default' | 'ghost'; + +export type IconButtonProps = ComponentPropsWithRef<'button'> & { + /** + * The size of the icon button + */ + size?: IconButtonSize; + + /** + * The visual variant of the icon button + */ + variant?: IconButtonVariant; + + /** + * Accessible label for screen readers + * Required for accessibility compliance + */ + 'aria-label': string; +}; + +const sizeClasses: Record = { + '6': 'h-6 min-h-6', + '7': 'h-7 min-h-7', + '8': 'h-8 min-h-8', + '9': 'h-9 min-h-9', + '10': 'h-10 min-h-10', + '11': 'h-11 min-h-11', + '12': 'h-12 min-h-12', + '16': 'h-16 min-h-16', +}; + +const variantClasses: Record = { + default: 'border border-agentprism-border bg-transparent', + ghost: 'bg-transparent', +}; + +// TODO: Remake to call Icon component directly instead of passing children +export const IconButton = ({ + children, + className, + size = '6', + variant = 'default', + type = 'button', + 'aria-label': ariaLabel, + ...rest +}: IconButtonProps) => { + return ( + + ); +}; diff --git a/frontend/app/src/components/v1/agent-prism/PriceBadge.tsx b/frontend/app/src/components/v1/agent-prism/PriceBadge.tsx new file mode 100644 index 0000000000..d4390e21da --- /dev/null +++ b/frontend/app/src/components/v1/agent-prism/PriceBadge.tsx @@ -0,0 +1,12 @@ +import type { BadgeProps } from './Badge'; +import { Badge } from './Badge'; +import type { ComponentPropsWithRef } from 'react'; + +export type PriceBadgeProps = ComponentPropsWithRef<'span'> & { + cost: number; + size?: BadgeProps['size']; +}; + +export const PriceBadge = ({ cost, size, ...rest }: PriceBadgeProps) => { + return ; +}; diff --git a/frontend/app/src/components/v1/agent-prism/SearchInput.tsx b/frontend/app/src/components/v1/agent-prism/SearchInput.tsx new file mode 100644 index 0000000000..f5cbd5b0d5 --- /dev/null +++ b/frontend/app/src/components/v1/agent-prism/SearchInput.tsx @@ -0,0 +1,16 @@ +import { TextInput, type TextInputProps } from './TextInput'; +import { Search } from 'lucide-react'; + +/** + * A simple wrapper around the TextInput component. + * It adds a search icon and a placeholder. + */ +export const SearchInput = ({ ...props }: TextInputProps) => { + return ( + } + placeholder="Filter..." + {...props} + /> + ); +}; diff --git a/frontend/app/src/components/v1/agent-prism/SpanBadge.tsx b/frontend/app/src/components/v1/agent-prism/SpanBadge.tsx new file mode 100644 index 0000000000..01d15c8185 --- /dev/null +++ b/frontend/app/src/components/v1/agent-prism/SpanBadge.tsx @@ -0,0 +1,50 @@ +import { Badge, type BadgeProps } from './Badge'; +import { getSpanCategoryIcon, getSpanCategoryLabel } from './shared'; +import type { TraceSpanCategory } from '@evilmartians/agent-prism-types'; +import cn from 'classnames'; + +export interface SpanBadgeProps + extends Omit { + category: TraceSpanCategory; +} + +const badgeClasses: Record = { + llm_call: 'bg-agentprism-badge-llm text-agentprism-badge-llm-foreground', + tool_execution: + 'bg-agentprism-badge-tool text-agentprism-badge-tool-foreground', + chain_operation: + 'bg-agentprism-badge-chain text-agentprism-badge-chain-foreground', + retrieval: + 'bg-agentprism-badge-retrieval text-agentprism-badge-retrieval-foreground', + embedding: + 'bg-agentprism-badge-embedding text-agentprism-badge-embedding-foreground', + guardrail: + 'bg-agentprism-badge-guardrail text-agentprism-badge-guardrail-foreground', + agent_invocation: + 'bg-agentprism-badge-agent text-agentprism-badge-agent-foreground', + create_agent: + 'bg-agentprism-badge-create-agent text-agentprism-badge-create-agent-foreground', + span: 'bg-agentprism-badge-span text-agentprism-badge-span-foreground', + event: 'bg-agentprism-badge-event text-agentprism-badge-event-foreground', + unknown: + 'bg-agentprism-badge-unknown text-agentprism-badge-unknown-foreground', +}; + +export const SpanBadge = ({ + category, + className, + ...props +}: SpanBadgeProps) => { + const Icon = getSpanCategoryIcon(category); + const label = getSpanCategoryLabel(category); + + return ( + } + {...props} + label={label} + unstyled + /> + ); +}; diff --git a/frontend/app/src/components/v1/agent-prism/SpanCard/SpanCard.tsx b/frontend/app/src/components/v1/agent-prism/SpanCard/SpanCard.tsx new file mode 100644 index 0000000000..82bee082dd --- /dev/null +++ b/frontend/app/src/components/v1/agent-prism/SpanCard/SpanCard.tsx @@ -0,0 +1,477 @@ +import type { AvatarProps } from '../Avatar'; +import { Avatar } from '../Avatar'; +import { BrandLogo } from '../BrandLogo'; +import { SpanStatus } from '../SpanStatus'; +import { SpanCardBadges } from './SpanCardBadges'; +import type { SpanCardConnectorType } from './SpanCardConnector'; +import { SpanCardConnector } from './SpanCardConnector'; +import { SpanCardTimeline } from './SpanCardTimeline'; +import { SpanCardToggle } from './SpanCardToggle'; +import { + formatDuration, + getTimelineData, +} from '@evilmartians/agent-prism-data'; +import type { TraceSpan } from '@evilmartians/agent-prism-types'; +import * as Collapsible from '@radix-ui/react-collapsible'; +import cn from 'classnames'; +import type { FC, KeyboardEvent, MouseEvent } from 'react'; +import { useCallback } from 'react'; + +const LAYOUT_CONSTANTS = { + CONNECTOR_WIDTH: 20, + CONTENT_BASE_WIDTH: 320, +} as const; + +type ExpandButtonPlacement = 'inside' | 'outside'; + +export type SpanCardViewOptions = { + withStatus?: boolean; + expandButton?: ExpandButtonPlacement; +}; + +const DEFAULT_VIEW_OPTIONS: Required = { + withStatus: true, + expandButton: 'inside', +}; + +interface SpanCardProps { + data: TraceSpan; + level?: number; + selectedSpan?: TraceSpan; + avatar?: AvatarProps; + onSpanSelect?: (span: TraceSpan) => void; + minStart: number; + maxEnd: number; + isLastChild: boolean; + prevLevelConnectors?: SpanCardConnectorType[]; + expandedSpansIds: string[]; + onExpandSpansIdsChange: (ids: string[]) => void; + viewOptions?: SpanCardViewOptions; +} + +interface SpanCardState { + isExpanded: boolean; + hasChildren: boolean; + isSelected: boolean; +} + +const getContentWidth = ({ + level, + hasExpandButton, + contentPadding, + expandButton, +}: { + level: number; + hasExpandButton: boolean; + contentPadding: number; + expandButton: ExpandButtonPlacement; +}) => { + let width = + LAYOUT_CONSTANTS.CONTENT_BASE_WIDTH - + level * LAYOUT_CONSTANTS.CONNECTOR_WIDTH; + + if (hasExpandButton && expandButton === 'inside') { + width -= LAYOUT_CONSTANTS.CONNECTOR_WIDTH; + } + + if (expandButton === 'outside' && level === 0) { + width -= LAYOUT_CONSTANTS.CONNECTOR_WIDTH; + } + + return width - contentPadding; +}; + +const getGridTemplateColumns = ({ + connectorsColumnWidth, + expandButton, +}: { + connectorsColumnWidth: number; + expandButton: ExpandButtonPlacement; +}) => { + if (expandButton === 'inside') { + return `${connectorsColumnWidth}px 1fr`; + } + + return `${connectorsColumnWidth}px 1fr ${LAYOUT_CONSTANTS.CONNECTOR_WIDTH}px`; +}; + +const getContentPadding = ({ + level, + hasExpandButton, +}: { + level: number; + hasExpandButton: boolean; +}) => { + if (level === 0) return 0; + + if (hasExpandButton) return 4; + + return 8; +}; + +const getConnectorsLayout = ({ + level, + hasExpandButton, + isLastChild, + prevConnectors, + expandButton, +}: { + hasExpandButton: boolean; + isLastChild: boolean; + level: number; + prevConnectors: SpanCardConnectorType[]; + expandButton: ExpandButtonPlacement; +}): { + connectors: SpanCardConnectorType[]; + connectorsColumnWidth: number; +} => { + const connectors: SpanCardConnectorType[] = []; + + if (level === 0) { + return { + connectors: expandButton === 'inside' ? [] : ['vertical'], + connectorsColumnWidth: 20, + }; + } + + for (let i = 0; i < level - 1; i++) { + connectors.push('vertical'); + } + + if (!isLastChild) { + connectors.push('t-right'); + } + + if (isLastChild) { + connectors.push('corner-top-right'); + } + + let connectorsColumnWidth = + connectors.length * LAYOUT_CONSTANTS.CONNECTOR_WIDTH; + + if (hasExpandButton) { + connectorsColumnWidth += LAYOUT_CONSTANTS.CONNECTOR_WIDTH; + } + + for (let i = 0; i < prevConnectors.length; i++) { + if ( + prevConnectors[i] === 'empty' || + prevConnectors[i] === 'corner-top-right' + ) { + connectors[i] = 'empty'; + } + } + + return { + connectors, + connectorsColumnWidth, + }; +}; + +const useSpanCardEventHandlers = ( + data: TraceSpan, + onSpanSelect?: (span: TraceSpan) => void, +) => { + const handleCardClick = useCallback((): void => { + onSpanSelect?.(data); + }, [data, onSpanSelect]); + + const handleKeyDown = useCallback( + (e: KeyboardEvent): void => { + if (e.key === 'Enter' || e.key === ' ') { + e.preventDefault(); + handleCardClick(); + } + }, + [handleCardClick], + ); + + const handleToggleClick = useCallback( + (e: MouseEvent | KeyboardEvent): void => { + e.stopPropagation(); + }, + [], + ); + + return { + handleCardClick, + handleKeyDown, + handleToggleClick, + }; +}; + +const SpanCardChildren: FC<{ + data: TraceSpan; + level: number; + selectedSpan?: TraceSpan; + onSpanSelect?: (span: TraceSpan) => void; + minStart: number; + maxEnd: number; + prevLevelConnectors: SpanCardConnectorType[]; + expandedSpansIds: string[]; + onExpandSpansIdsChange: (ids: string[]) => void; + viewOptions?: SpanCardViewOptions; +}> = ({ + data, + level, + selectedSpan, + onSpanSelect, + minStart, + maxEnd, + prevLevelConnectors, + expandedSpansIds, + onExpandSpansIdsChange, + viewOptions = DEFAULT_VIEW_OPTIONS, +}) => { + if (!data.children?.length) return null; + + return ( +
+ +
    + {data.children.map((child, idx) => { + const brand = child.metadata?.brand as { type: string } | undefined; + + return ( + , + size: '4', + rounded: 'sm', + category: child.type, + } + : undefined + } + /> + ); + })} +
+
+
+ ); +}; + +export const SpanCard: FC = ({ + data, + level = 0, + selectedSpan, + onSpanSelect, + viewOptions = DEFAULT_VIEW_OPTIONS, + avatar, + minStart, + maxEnd, + isLastChild, + prevLevelConnectors = [], + expandedSpansIds, + onExpandSpansIdsChange, +}) => { + const isExpanded = expandedSpansIds.includes(data.id); + + const withStatus = viewOptions.withStatus ?? DEFAULT_VIEW_OPTIONS.withStatus; + const expandButton = + viewOptions.expandButton || DEFAULT_VIEW_OPTIONS.expandButton; + + const handleToggleClick = useCallback( + (expanded: boolean) => { + const alreadyExpanded = expandedSpansIds.includes(data.id); + + if (alreadyExpanded && !expanded) { + onExpandSpansIdsChange(expandedSpansIds.filter((id) => id !== data.id)); + } + + if (!alreadyExpanded && expanded) { + onExpandSpansIdsChange([...expandedSpansIds, data.id]); + } + }, + [expandedSpansIds, data.id, onExpandSpansIdsChange], + ); + + const state: SpanCardState = { + isExpanded, + hasChildren: Boolean(data.children?.length), + isSelected: selectedSpan?.id === data.id, + }; + + const eventHandlers = useSpanCardEventHandlers(data, onSpanSelect); + + const { durationMs } = getTimelineData({ + spanCard: data, + minStart, + maxEnd, + }); + + const hasExpandButtonAsFirstChild = + expandButton === 'inside' && state.hasChildren; + + const contentPadding = getContentPadding({ + level, + hasExpandButton: hasExpandButtonAsFirstChild, + }); + + const contentWidth = getContentWidth({ + level, + hasExpandButton: hasExpandButtonAsFirstChild, + contentPadding, + expandButton, + }); + + const { connectors, connectorsColumnWidth } = getConnectorsLayout({ + level, + hasExpandButton: hasExpandButtonAsFirstChild, + isLastChild, + prevConnectors: prevLevelConnectors, + expandButton, + }); + + const gridTemplateColumns = getGridTemplateColumns({ + connectorsColumnWidth, + expandButton, + }); + + return ( +
  • + +
    +
    + {connectors.map((connector, idx) => ( + + ))} + + {hasExpandButtonAsFirstChild && ( +
    + + + {state.isExpanded && } +
    + )} +
    +
    +
    + {avatar && } + +

    + {data.title} +

    + + +
    + +
    + {expandButton === 'outside' && withStatus && ( +
    + +
    + )} + + + +
    + + {formatDuration(durationMs)} + + + {expandButton === 'inside' && withStatus && ( +
    + +
    + )} +
    +
    +
    + + {expandButton === 'outside' && + (state.hasChildren ? ( + + ) : ( +
    + ))} +
    + + + +
  • + ); +}; diff --git a/frontend/app/src/components/v1/agent-prism/SpanCard/SpanCardBadges.tsx b/frontend/app/src/components/v1/agent-prism/SpanCard/SpanCardBadges.tsx new file mode 100644 index 0000000000..02495c57a9 --- /dev/null +++ b/frontend/app/src/components/v1/agent-prism/SpanCard/SpanCardBadges.tsx @@ -0,0 +1,14 @@ +import { SpanBadge } from '../SpanBadge'; +import type { TraceSpan } from '@evilmartians/agent-prism-types'; + +interface SpanCardBagdesProps { + data: TraceSpan; +} + +export const SpanCardBadges = ({ data }: SpanCardBagdesProps) => { + return ( +
    + +
    + ); +}; diff --git a/frontend/app/src/components/v1/agent-prism/SpanCard/SpanCardConnector.tsx b/frontend/app/src/components/v1/agent-prism/SpanCard/SpanCardConnector.tsx new file mode 100644 index 0000000000..211eb29983 --- /dev/null +++ b/frontend/app/src/components/v1/agent-prism/SpanCard/SpanCardConnector.tsx @@ -0,0 +1,36 @@ +export type SpanCardConnectorType = + | 'horizontal' + | 'vertical' + | 't-right' + | 'corner-top-right' + | 'empty'; + +interface SpanCardConnectorProps { + type: SpanCardConnectorType; +} + +export const SpanCardConnector = ({ type }: SpanCardConnectorProps) => { + if (type === 'empty') return
    ; + + return ( +
    + {(type === 'vertical' || type === 't-right') && ( +
    + )} + + {type === 't-right' && ( +
    + )} + + {type === 'corner-top-right' && ( + <> +
    + +
    + +
    + + )} +
    + ); +}; diff --git a/frontend/app/src/components/v1/agent-prism/SpanCard/SpanCardTimeline.tsx b/frontend/app/src/components/v1/agent-prism/SpanCard/SpanCardTimeline.tsx new file mode 100644 index 0000000000..b803072603 --- /dev/null +++ b/frontend/app/src/components/v1/agent-prism/SpanCard/SpanCardTimeline.tsx @@ -0,0 +1,59 @@ +import { getTimelineData } from '@evilmartians/agent-prism-data'; +import type { + TraceSpan, + TraceSpanCategory, +} from '@evilmartians/agent-prism-types'; +import cn from 'classnames'; + +interface SpanCardTimelineProps { + spanCard: TraceSpan; + minStart: number; + maxEnd: number; + className?: string; +} + +const timelineBgColors: Record = { + llm_call: 'bg-agentprism-timeline-llm', + agent_invocation: 'bg-agentprism-timeline-agent', + tool_execution: 'bg-agentprism-timeline-tool', + chain_operation: 'bg-agentprism-timeline-chain', + retrieval: 'bg-agentprism-timeline-retrieval', + embedding: 'bg-agentprism-timeline-embedding', + guardrail: 'bg-agentprism-timeline-guardrail', + create_agent: 'bg-agentprism-timeline-create-agent', + span: 'bg-agentprism-timeline-span', + event: 'bg-agentprism-timeline-event', + unknown: 'bg-agentprism-timeline-unknown', +}; + +export const SpanCardTimeline = ({ + spanCard, + minStart, + maxEnd, + className, +}: SpanCardTimelineProps) => { + const { startPercent, widthPercent } = getTimelineData({ + spanCard, + minStart, + maxEnd, + }); + + return ( + + + + + + ); +}; diff --git a/frontend/app/src/components/v1/agent-prism/SpanCard/SpanCardToggle.tsx b/frontend/app/src/components/v1/agent-prism/SpanCard/SpanCardToggle.tsx new file mode 100644 index 0000000000..9333fe52bf --- /dev/null +++ b/frontend/app/src/components/v1/agent-prism/SpanCard/SpanCardToggle.tsx @@ -0,0 +1,38 @@ +import * as Collapsible from '@radix-ui/react-collapsible'; +import { ChevronDown, ChevronRight } from 'lucide-react'; +import type { KeyboardEvent, MouseEvent } from 'react'; + +interface SpanCardToggleProps { + isExpanded: boolean; + title: string; + onToggleClick: (e: MouseEvent | KeyboardEvent) => void; +} + +export const SpanCardToggle = ({ + isExpanded, + title, + onToggleClick, +}: SpanCardToggleProps) => ( + + + +); diff --git a/frontend/app/src/components/v1/agent-prism/SpanStatus.tsx b/frontend/app/src/components/v1/agent-prism/SpanStatus.tsx new file mode 100644 index 0000000000..895b00d96c --- /dev/null +++ b/frontend/app/src/components/v1/agent-prism/SpanStatus.tsx @@ -0,0 +1,79 @@ +import type { TraceSpanStatus } from '@evilmartians/agent-prism-types'; +import cn from 'classnames'; +import { Check, Ellipsis, Info, TriangleAlert } from 'lucide-react'; +import type { ComponentPropsWithRef } from 'react'; + +type StatusVariant = 'dot' | 'badge'; + +export type StatusProps = ComponentPropsWithRef<'div'> & { + status: TraceSpanStatus; + variant?: StatusVariant; +}; + +const STATUS_COLORS_DOT: Record = { + success: 'bg-agentprism-success', + error: 'bg-agentprism-error', + pending: 'bg-agentprism-pending', + warning: 'bg-agentprism-warning', +}; + +const STATUS_COLORS_BADGE: Record = { + success: + 'bg-agentprism-success-muted text-agentprism-success-muted-foreground', + error: 'bg-agentprism-error-muted text-agentprism-error-muted-foreground', + pending: + 'bg-agentprism-pending-muted text-agentprism-pending-muted-foreground', + warning: + 'bg-agentprism-warning-muted text-agentprism-warning-muted-foreground', +}; + +export const SpanStatus = ({ + status, + variant = 'dot', + ...rest +}: StatusProps) => { + const title = `Status: ${status}`; + + return ( +
    + {variant === 'dot' ? ( + + ) : ( + + )} +
    + ); +}; + +interface StatusWithTitleProps extends StatusProps { + title: string; +} + +const SpanStatusDot = ({ status, title }: StatusWithTitleProps) => { + return ( + + ); +}; + +const SpanStatusBadge = ({ status, title }: StatusWithTitleProps) => { + return ( + + {status === 'success' && } + {status === 'error' && } + {status === 'warning' && } + {status === 'pending' && } + + ); +}; diff --git a/frontend/app/src/components/v1/agent-prism/TabSelector.tsx b/frontend/app/src/components/v1/agent-prism/TabSelector.tsx new file mode 100644 index 0000000000..054c18a5c8 --- /dev/null +++ b/frontend/app/src/components/v1/agent-prism/TabSelector.tsx @@ -0,0 +1,34 @@ +import { type TabItem, Tabs } from './Tabs'; +import { type ReactElement } from 'react'; + +export interface TabSelectorProps { + items: TabItem[]; + value: T; + onValueChange: (value: T) => void; + defaultValue?: T; + theme?: 'underline' | 'pill'; + className?: string; + onClick?: (event: React.MouseEvent) => void; +} + +export const TabSelector = ({ + items, + value, + onValueChange, + defaultValue, + theme = 'underline', + className, + onClick, +}: TabSelectorProps): ReactElement => { + return ( + + items={items} + value={value} + onValueChange={onValueChange} + defaultValue={defaultValue} + theme={theme} + className={className} + onClick={onClick} + /> + ); +}; diff --git a/frontend/app/src/components/v1/agent-prism/Tabs.tsx b/frontend/app/src/components/v1/agent-prism/Tabs.tsx new file mode 100644 index 0000000000..182a088bf8 --- /dev/null +++ b/frontend/app/src/components/v1/agent-prism/Tabs.tsx @@ -0,0 +1,139 @@ +import * as RadixTabs from '@radix-ui/react-tabs'; +import cn from 'classnames'; +import type { ComponentPropsWithRef } from 'react'; +import * as React from 'react'; + +export interface TabItem { + value: T; + label: string; + icon?: React.ReactNode; + disabled?: boolean; +} + +export type TabTheme = 'underline' | 'pill'; + +const BASE_TRIGGER = + 'text-sm font-medium transition-colors duration-200 disabled:opacity-50 disabled:cursor-not-allowed'; + +const THEMES = { + underline: { + list: 'h-9 flex border-b border-agentprism-border', + trigger: `w-full justify-center px-3 ${BASE_TRIGGER} + text-agentprism-secondary-foreground data-[state=active]:text-agentprism-foreground + border-b-2 border-transparent data-[state=active]:border-agentprism-border-inverse + -mb-[2px] + data-[state=inactive]:[&:not(:disabled)]:hover:border-agentprism-border-inverse/20 + data-[state=inactive]:[&:not(:disabled)]:hover:text-agentprism-muted-foreground`, + }, + pill: { + list: 'h-9 inline-flex gap-1 p-1 bg-agentprism-secondary rounded-lg', + trigger: `px-3 ${BASE_TRIGGER} rounded-md + text-agentprism-muted-foreground data-[state=active]:text-agentprism-foreground + data-[state=inactive]:[&:not(:disabled)]:hover:bg-agentprism-background/50 data-[state=active]:bg-agentprism-background data-[state=active]:shadow-sm + dark:data-[state=active]:shadow-none`, + }, +} as const; + +export type TabsProps = Omit< + ComponentPropsWithRef<'div'>, + 'dir' +> & { + /** + * Array of tab items to display + */ + items: TabItem[]; + + /** + * The initially selected tab value (uncontrolled) + */ + defaultValue?: T; + + /** + * The currently selected tab value (controlled) + */ + value?: T; + + /** + * Callback fired when the selected tab changes + */ + onValueChange?: (value: T) => void; + + /** + * Visual theme variant for the tabs + * @default "underline" + */ + theme?: TabTheme; + + /** + * Optional className for the root container + */ + className?: string; + + /** + * Optional className for the tabs list container + */ + tabsListClassName?: string; + + /** + * Optional className for individual tab triggers + */ + triggerClassName?: string; + + /** + * The direction of the content of the tabs + */ + dir?: 'ltr' | 'rtl'; +}; + +export const Tabs = ({ + items, + defaultValue, + value, + onValueChange, + theme = 'underline', + className = '', + tabsListClassName = '', + triggerClassName = '', + dir, + ...rest +}: TabsProps) => { + const defaultTab = defaultValue || items[0]?.value; + + const currentTheme = THEMES[theme]; + + return ( + void} + dir={dir} + {...rest} + > + + {items.map((item: TabItem) => ( + + {item.icon && ( + + {item.icon} + + )} + {item.label} + + ))} + + + ); +}; diff --git a/frontend/app/src/components/v1/agent-prism/TextInput.tsx b/frontend/app/src/components/v1/agent-prism/TextInput.tsx new file mode 100644 index 0000000000..5366cf88bc --- /dev/null +++ b/frontend/app/src/components/v1/agent-prism/TextInput.tsx @@ -0,0 +1,143 @@ +import cn from 'classnames'; +import { X } from 'lucide-react'; +import { + useRef, + type ChangeEvent, + type ComponentPropsWithRef, + type ReactNode, + type RefObject, +} from 'react'; + +export type TextInputProps = ComponentPropsWithRef<'input'> & { + /** + * Callback fired when the input value changes + */ + onValueChange?: (value: string) => void; + + /** + * Icon to display at the start of the input + */ + startIcon?: ReactNode; + + /** + * Callback fired when the clear button is clicked. If this callback is provided, + * the clear button will be shown. + */ + onClear?: () => void; + + /** + * Ref to the input element + */ + ref?: RefObject; + + /** + * Optional className for the input element + */ + inputClassName?: string; + + /** + * Unique identifier for the input (required) + */ + id: string; + + /** + * Label text for the input + */ + label?: string; + + /** + * Whether to visually hide the label while keeping it for screen readers + * @default false + */ + hideLabel?: boolean; +}; + +const iconBaseClassName = + 'absolute top-1/2 -translate-y-1/2 flex items-center justify-center text-agentprism-muted-foreground'; + +export const TextInput = ({ + className, + onChange, + onValueChange, + startIcon, + onClear, + ref, + inputClassName, + label, + hideLabel = false, + id, + ...rest +}: TextInputProps) => { + const inputRef = useRef(null); + + const handleChange = (e: ChangeEvent) => { + onChange?.(e); + onValueChange?.(e.target.value); + }; + + const handleClear = () => { + onClear?.(); + + if (ref) { + ref.current?.focus(); + return; + } + + inputRef.current?.focus(); + }; + + return ( +
    + {label && ( + + )} +
    + + {startIcon && ( +
    + {startIcon} +
    + )} + {onClear && rest.value && ( + + )} +
    +
    + ); +}; diff --git a/frontend/app/src/components/v1/agent-prism/TimestampBadge.tsx b/frontend/app/src/components/v1/agent-prism/TimestampBadge.tsx new file mode 100644 index 0000000000..91847947d1 --- /dev/null +++ b/frontend/app/src/components/v1/agent-prism/TimestampBadge.tsx @@ -0,0 +1,20 @@ +import type { BadgeProps } from './Badge'; +import { Badge } from './Badge'; +import type { ComponentPropsWithRef } from 'react'; + +export type TimestampBadgeProps = ComponentPropsWithRef<'span'> & { + timestamp: number; + size?: BadgeProps['size']; +}; + +export const TimestampBadge = ({ + timestamp, + size, + ...rest +}: TimestampBadgeProps) => { + return ; +}; + +function formatTimestamp(timestamp: number): string { + return new Date(timestamp).toLocaleString(); +} diff --git a/frontend/app/src/components/v1/agent-prism/TokensBadge.tsx b/frontend/app/src/components/v1/agent-prism/TokensBadge.tsx new file mode 100644 index 0000000000..8bbe81c275 --- /dev/null +++ b/frontend/app/src/components/v1/agent-prism/TokensBadge.tsx @@ -0,0 +1,24 @@ +import type { BadgeProps } from './Badge'; +import { Badge } from './Badge'; +import { Coins } from 'lucide-react'; +import type { ComponentPropsWithRef } from 'react'; + +export type TokensBadgeProps = ComponentPropsWithRef<'span'> & { + tokensCount: number; + size?: BadgeProps['size']; +}; + +export const TokensBadge = ({ + tokensCount, + size, + ...rest +}: TokensBadgeProps) => { + return ( + } + size={size} + {...rest} + label={tokensCount} + /> + ); +}; diff --git a/frontend/app/src/components/v1/agent-prism/TraceList/TraceList.tsx b/frontend/app/src/components/v1/agent-prism/TraceList/TraceList.tsx new file mode 100644 index 0000000000..b803cd87ea --- /dev/null +++ b/frontend/app/src/components/v1/agent-prism/TraceList/TraceList.tsx @@ -0,0 +1,84 @@ +import type { BadgeProps } from '../Badge'; +import { Badge } from '../Badge'; +import { IconButton } from '../IconButton'; +import { TraceListItem } from './TraceListItem'; +import type { TraceRecord } from '@evilmartians/agent-prism-types'; +import cn from 'classnames'; +import { ArrowLeft } from 'lucide-react'; + +type TraceRecordWithBadges = TraceRecord & { + badges?: Array; +}; + +type TraceListProps = { + traces: TraceRecordWithBadges[]; + expanded: boolean; + onExpandStateChange: (expanded: boolean) => void; + className?: string; + onTraceSelect?: (trace: TraceRecord) => void; + selectedTrace?: TraceRecord; +}; + +export const TraceList = ({ + traces, + expanded, + onExpandStateChange, + className, + onTraceSelect, + selectedTrace, +}: TraceListProps) => { + return ( +
    +
    +
    +

    Traces

    + + +
    + + onExpandStateChange(!expanded)} + > + + +
    + + {expanded && ( +
      +
      + {traces.map((trace) => ( +
    • + onTraceSelect?.(trace)} + isSelected={selectedTrace?.id === trace.id} + badges={trace.badges} + /> +
    • + ))} +
      +
    + )} +
    + ); +}; diff --git a/frontend/app/src/components/v1/agent-prism/TraceList/TraceListItem.tsx b/frontend/app/src/components/v1/agent-prism/TraceList/TraceListItem.tsx new file mode 100644 index 0000000000..ffa37f93ee --- /dev/null +++ b/frontend/app/src/components/v1/agent-prism/TraceList/TraceListItem.tsx @@ -0,0 +1,83 @@ +import type { AvatarProps } from '../Avatar'; +import type { BadgeProps } from '../Badge'; +import { Badge } from '../Badge'; +import { PriceBadge } from '../PriceBadge'; +import { TimestampBadge } from '../TimestampBadge'; +import { TokensBadge } from '../TokensBadge'; +import { TraceListItemHeader } from './TraceListItemHeader'; +import type { TraceRecord } from '@evilmartians/agent-prism-types'; +import cn from 'classnames'; +import type { KeyboardEvent } from 'react'; +import { useCallback } from 'react'; + +interface TraceListItemProps { + trace: TraceRecord; + badges?: Array; + avatar?: AvatarProps; + onClick?: () => void; + isSelected?: boolean; + showDescription?: boolean; +} + +export const TraceListItem = ({ + trace, + avatar, + onClick, + badges, + isSelected, + showDescription = true, +}: TraceListItemProps) => { + const handleKeyDown = useCallback( + (e: KeyboardEvent): void => { + if (e.key === 'Enter' || e.key === ' ') { + e.preventDefault(); + onClick?.(); + } + }, + [onClick], + ); + + const { name, agentDescription, totalCost, totalTokens, startTime } = trace; + + return ( +
    + + +
    + {showDescription && ( + + {agentDescription} + + )} + + {typeof totalCost === 'number' && } + + {typeof totalTokens === 'number' && ( + + )} + + {badges?.map((badge, index) => ( + + ))} + + {typeof startTime === 'number' && ( + + )} +
    +
    + ); +}; diff --git a/frontend/app/src/components/v1/agent-prism/TraceList/TraceListItemHeader.tsx b/frontend/app/src/components/v1/agent-prism/TraceList/TraceListItemHeader.tsx new file mode 100644 index 0000000000..c1f710e651 --- /dev/null +++ b/frontend/app/src/components/v1/agent-prism/TraceList/TraceListItemHeader.tsx @@ -0,0 +1,35 @@ +import type { AvatarProps } from '../Avatar'; +import { Avatar } from '../Avatar'; +import { Badge } from '../Badge'; +import type { TraceRecord } from '@evilmartians/agent-prism-types'; + +interface TraceListItemHeaderProps { + trace: TraceRecord; + avatar?: AvatarProps; +} + +export const TraceListItemHeader = ({ + trace, + avatar, +}: TraceListItemHeaderProps) => { + return ( +
    +
    + {avatar && } + +

    + {trace.name} +

    +
    + +
    + +
    +
    + ); +}; diff --git a/frontend/app/src/components/v1/agent-prism/TraceViewer/TraceViewer.tsx b/frontend/app/src/components/v1/agent-prism/TraceViewer/TraceViewer.tsx new file mode 100644 index 0000000000..3a912f2559 --- /dev/null +++ b/frontend/app/src/components/v1/agent-prism/TraceViewer/TraceViewer.tsx @@ -0,0 +1,167 @@ +import { type BadgeProps } from '../Badge'; +import { type SpanCardViewOptions } from '../SpanCard/SpanCard'; +import { useIsMobile, useIsMounted } from '../shared'; +import { TraceViewerDesktopLayout } from './TraceViewerDesktopLayout'; +import { TraceViewerMobileLayout } from './TraceViewerMobileLayout'; +import { + filterSpansRecursively, + flattenSpans, +} from '@evilmartians/agent-prism-data'; +import type { TraceRecord, TraceSpan } from '@evilmartians/agent-prism-types'; +import { useCallback, useEffect, useMemo, useState } from 'react'; + +export interface TraceViewerData { + traceRecord: TraceRecord; + badges?: Array; + spans: TraceSpan[]; + spanCardViewOptions?: SpanCardViewOptions; +} + +export interface TraceViewerProps { + data: Array; + spanCardViewOptions?: SpanCardViewOptions; +} + +export const TraceViewer = ({ + data, + spanCardViewOptions, +}: TraceViewerProps) => { + const isMobile = useIsMobile(); + const isMounted = useIsMounted(); + + const [selectedSpan, setSelectedSpan] = useState(); + const [searchValue, setSearchValue] = useState(''); + const [traceListExpanded, setTraceListExpanded] = useState(true); + + const [selectedTrace, setSelectedTrace] = useState< + TraceRecordWithDisplayData | undefined + >( + data[0] + ? { + ...data[0].traceRecord, + badges: data[0].badges, + spanCardViewOptions: data[0].spanCardViewOptions, + } + : undefined, + ); + const [selectedTraceSpans, setSelectedTraceSpans] = useState( + data[0]?.spans || [], + ); + + const traceRecords: TraceRecordWithDisplayData[] = useMemo(() => { + return data.map((item) => ({ + ...item.traceRecord, + badges: item.badges, + spanCardViewOptions: item.spanCardViewOptions, + })); + }, [data]); + + const filteredSpans = useMemo(() => { + if (!searchValue.trim()) { + return selectedTraceSpans; + } + return filterSpansRecursively(selectedTraceSpans, searchValue); + }, [selectedTraceSpans, searchValue]); + + const allIds = useMemo(() => { + return flattenSpans(selectedTraceSpans).map((span) => span.id); + }, [selectedTraceSpans]); + + const [expandedSpansIds, setExpandedSpansIds] = useState(allIds); + + useEffect(() => { + setExpandedSpansIds(allIds); + }, [allIds]); + + useEffect(() => { + if (!isMounted || isMobile) return; + + if (selectedTraceSpans.length > 0 && !selectedSpan) { + setSelectedSpan(selectedTraceSpans[0]); + } + }, [selectedTraceSpans, isMobile, isMounted, selectedSpan]); + + const handleExpandAll = useCallback(() => { + setExpandedSpansIds(allIds); + }, [allIds]); + + const handleCollapseAll = useCallback(() => { + setExpandedSpansIds([]); + }, []); + + const handleTraceSelect = useCallback( + (trace: TraceRecord) => { + setSelectedSpan(undefined); + setExpandedSpansIds([]); + setSelectedTrace(trace); + setSelectedTraceSpans( + data.find((item) => item.traceRecord.id === trace.id)?.spans ?? [], + ); + }, + [data], + ); + + const handleClearTraceSelection = useCallback(() => { + setSelectedTrace(undefined); + setSelectedTraceSpans([]); + setSelectedSpan(undefined); + setExpandedSpansIds([]); + }, []); + + const props: TraceViewerLayoutProps = { + traceRecords, + traceListExpanded, + setTraceListExpanded, + selectedTrace, + selectedTraceId: selectedTrace?.id, + selectedSpan, + setSelectedSpan, + searchValue, + setSearchValue, + filteredSpans, + expandedSpansIds, + setExpandedSpansIds, + handleExpandAll, + handleCollapseAll, + handleTraceSelect, + spanCardViewOptions: + spanCardViewOptions || selectedTrace?.spanCardViewOptions, + onClearTraceSelection: handleClearTraceSelection, + }; + + return ( +
    +
    + +
    +
    + +
    +
    + ); +}; + +export interface TraceRecordWithDisplayData extends TraceRecord { + spanCardViewOptions?: SpanCardViewOptions; + badges?: BadgeProps[]; +} + +export interface TraceViewerLayoutProps { + traceRecords: TraceRecordWithDisplayData[]; + traceListExpanded: boolean; + setTraceListExpanded: (expanded: boolean) => void; + selectedTrace: TraceRecordWithDisplayData | undefined; + selectedTraceId?: string; + selectedSpan: TraceSpan | undefined; + setSelectedSpan: (span: TraceSpan | undefined) => void; + searchValue: string; + setSearchValue: (value: string) => void; + filteredSpans: TraceSpan[]; + expandedSpansIds: string[]; + setExpandedSpansIds: (ids: string[]) => void; + handleExpandAll: () => void; + handleCollapseAll: () => void; + handleTraceSelect: (trace: TraceRecord) => void; + spanCardViewOptions?: SpanCardViewOptions; + onClearTraceSelection: () => void; +} diff --git a/frontend/app/src/components/v1/agent-prism/TraceViewer/TraceViewerDesktopLayout.tsx b/frontend/app/src/components/v1/agent-prism/TraceViewer/TraceViewerDesktopLayout.tsx new file mode 100644 index 0000000000..7cdeb60043 --- /dev/null +++ b/frontend/app/src/components/v1/agent-prism/TraceViewer/TraceViewerDesktopLayout.tsx @@ -0,0 +1,97 @@ +import { DetailsView } from '../DetailsView/DetailsView'; +import { TraceList } from '../TraceList/TraceList'; +import { type TraceViewerLayoutProps } from './TraceViewer'; +import { TraceViewerPlaceholder } from './TraceViewerPlaceholder'; +import { TraceViewerTreeViewContainer } from './TraceViewerTreeViewContainer'; +import { Panel, PanelGroup, PanelResizeHandle } from 'react-resizable-panels'; + +export const TraceViewerDesktopLayout = ({ + traceRecords, + traceListExpanded, + setTraceListExpanded, + selectedTrace, + selectedTraceId, + selectedSpan, + setSelectedSpan, + searchValue, + setSearchValue, + filteredSpans, + expandedSpansIds, + setExpandedSpansIds, + handleExpandAll, + handleCollapseAll, + handleTraceSelect, + spanCardViewOptions, +}: TraceViewerLayoutProps) => { + const actualSelectedTrace = + traceRecords.find((t) => t.id === selectedTraceId) || selectedTrace; + + return ( + + + + + + + + {selectedTrace ? ( + + + + ) : ( + + + + )} + + + + + {selectedSpan ? ( + + ) : ( + + )} + + + ); +}; diff --git a/frontend/app/src/components/v1/agent-prism/TraceViewer/TraceViewerMobileLayout.tsx b/frontend/app/src/components/v1/agent-prism/TraceViewer/TraceViewerMobileLayout.tsx new file mode 100644 index 0000000000..3e7fff99f9 --- /dev/null +++ b/frontend/app/src/components/v1/agent-prism/TraceViewer/TraceViewerMobileLayout.tsx @@ -0,0 +1,101 @@ +import { Button } from '../Button'; +import { DetailsView } from '../DetailsView/DetailsView'; +import { TraceList } from '../TraceList/TraceList'; +import { type TraceViewerLayoutProps } from '../TraceViewer/TraceViewer'; +import { TraceViewerTreeViewContainer } from './TraceViewerTreeViewContainer'; +import { ArrowLeft } from 'lucide-react'; + +export const TraceViewerMobileLayout = ({ + traceRecords, + traceListExpanded, + setTraceListExpanded, + selectedTrace, + selectedTraceId, + selectedSpan, + setSelectedSpan, + searchValue, + setSearchValue, + filteredSpans, + expandedSpansIds, + setExpandedSpansIds, + handleExpandAll, + handleCollapseAll, + handleTraceSelect, + spanCardViewOptions, + onClearTraceSelection, +}: TraceViewerLayoutProps) => { + if ( + selectedTrace && + selectedTraceId && + filteredSpans.length > 0 && + selectedSpan + ) { + return ( +
    + + +
    + ); + } + + if ( + selectedTrace && + selectedTraceId && + filteredSpans.length > 0 && + !selectedSpan + ) { + return ( +
    +
    + +
    + + +
    + ); + } + + return ( +
    + t.id === selectedTraceId)} + /> +
    + ); +}; diff --git a/frontend/app/src/components/v1/agent-prism/TraceViewer/TraceViewerPlaceholder.tsx b/frontend/app/src/components/v1/agent-prism/TraceViewer/TraceViewerPlaceholder.tsx new file mode 100644 index 0000000000..cd602d7c32 --- /dev/null +++ b/frontend/app/src/components/v1/agent-prism/TraceViewer/TraceViewerPlaceholder.tsx @@ -0,0 +1,5 @@ +export const TraceViewerPlaceholder = ({ title }: { title: string }) => ( +

    + {title} +

    +); diff --git a/frontend/app/src/components/v1/agent-prism/TraceViewer/TraceViewerSearchAndControls.tsx b/frontend/app/src/components/v1/agent-prism/TraceViewer/TraceViewerSearchAndControls.tsx new file mode 100644 index 0000000000..156099e25e --- /dev/null +++ b/frontend/app/src/components/v1/agent-prism/TraceViewer/TraceViewerSearchAndControls.tsx @@ -0,0 +1,30 @@ +import { + CollapseAllButton, + ExpandAllButton, +} from '../CollapseAndExpandControls'; +import { SearchInput } from '../SearchInput'; + +export const TraceViewerSearchAndControls = ({ + searchValue, + setSearchValue, + handleExpandAll, + handleCollapseAll, +}: { + searchValue: string; + setSearchValue: (value: string) => void; + handleExpandAll: () => void; + handleCollapseAll: () => void; +}) => ( +
    + setSearchValue(e.target.value)} + placeholder="Search spans" + /> +
    + + +
    +
    +); diff --git a/frontend/app/src/components/v1/agent-prism/TraceViewer/TraceViewerTreeViewContainer.tsx b/frontend/app/src/components/v1/agent-prism/TraceViewer/TraceViewerTreeViewContainer.tsx new file mode 100644 index 0000000000..04b22dec08 --- /dev/null +++ b/frontend/app/src/components/v1/agent-prism/TraceViewer/TraceViewerTreeViewContainer.tsx @@ -0,0 +1,74 @@ +import { Badge } from '../Badge'; +import type { SpanCardViewOptions } from '../SpanCard/SpanCard'; +import { TraceListItemHeader } from '../TraceList/TraceListItemHeader'; +import { TreeView } from '../TreeView'; +import { type TraceRecordWithDisplayData } from './TraceViewer'; +import { TraceViewerSearchAndControls } from './TraceViewerSearchAndControls'; +import type { TraceSpan } from '@evilmartians/agent-prism-types'; + +export const TraceViewerTreeViewContainer = ({ + searchValue, + setSearchValue, + handleExpandAll, + handleCollapseAll, + filteredSpans, + selectedSpan, + setSelectedSpan, + expandedSpansIds, + setExpandedSpansIds, + spanCardViewOptions, + selectedTrace, + showHeader = true, +}: { + searchValue: string; + setSearchValue: (value: string) => void; + handleExpandAll: () => void; + handleCollapseAll: () => void; + filteredSpans: TraceSpan[]; + selectedSpan: TraceSpan | undefined; + setSelectedSpan: (span: TraceSpan | undefined) => void; + expandedSpansIds: string[]; + setExpandedSpansIds: (ids: string[]) => void; + spanCardViewOptions?: SpanCardViewOptions; + selectedTrace?: TraceRecordWithDisplayData; + showHeader?: boolean; +}) => ( + <> + {showHeader && selectedTrace && ( +
    + + +
    + {selectedTrace.badges?.map((badge, index) => ( + + ))} +
    +
    + )} + +
    + +
    + {filteredSpans.length === 0 ? ( +
    + No spans found +
    + ) : ( + + )} +
    +
    + +); diff --git a/frontend/app/src/components/v1/agent-prism/TreeView.tsx b/frontend/app/src/components/v1/agent-prism/TreeView.tsx new file mode 100644 index 0000000000..e6d97532b5 --- /dev/null +++ b/frontend/app/src/components/v1/agent-prism/TreeView.tsx @@ -0,0 +1,70 @@ +import { BrandLogo } from './BrandLogo'; +import type { SpanCardViewOptions } from './SpanCard/SpanCard'; +import { SpanCard } from './SpanCard/SpanCard'; +import { flattenSpans, findTimeRange } from '@evilmartians/agent-prism-data'; +import type { TraceSpan } from '@evilmartians/agent-prism-types'; +import cn from 'classnames'; +import { type FC } from 'react'; + +interface TreeViewProps { + spans: TraceSpan[]; + className?: string; + selectedSpan?: TraceSpan; + onSpanSelect?: (span: TraceSpan) => void; + expandedSpansIds: string[]; + onExpandSpansIdsChange: (ids: string[]) => void; + spanCardViewOptions?: SpanCardViewOptions; +} + +export const TreeView: FC = ({ + spans, + onSpanSelect, + className = '', + selectedSpan, + expandedSpansIds, + onExpandSpansIdsChange, + spanCardViewOptions, +}) => { + const allCards = flattenSpans(spans); + const { minStart, maxEnd } = findTimeRange(allCards); + + return ( +
    +
      + {spans.map((span, idx) => { + const brand = span.metadata?.brand as { type: string } | undefined; + + return ( + , + size: '4', + rounded: 'sm', + category: span.type, + } + : undefined + } + /> + ); + })} +
    +
    + ); +}; diff --git a/frontend/app/src/components/v1/agent-prism/shared.ts b/frontend/app/src/components/v1/agent-prism/shared.ts new file mode 100644 index 0000000000..534e8bec8b --- /dev/null +++ b/frontend/app/src/components/v1/agent-prism/shared.ts @@ -0,0 +1,169 @@ +import type { TraceSpanCategory } from '@evilmartians/agent-prism-types'; +import type { LucideIcon } from 'lucide-react'; +import { + Zap, + Wrench, + Bot, + Link, + Search, + BarChart2, + Plus, + HelpCircle, + MoveHorizontal, + CircleDot, + ShieldCheck, +} from 'lucide-react'; +import { useEffect, useState } from 'react'; + +// TYPES + +export type ColorVariant = + | 'purple' + | 'indigo' + | 'orange' + | 'teal' + | 'cyan' + | 'sky' + | 'yellow' + | 'emerald' + | 'red' + | 'gray'; + +export type ComponentSize = + | '4' + | '5' + | '6' + | '7' + | '8' + | '9' + | '10' + | '11' + | '12' + | '16'; + +// CONSTANTS + +export const ROUNDED_CLASSES = { + none: 'rounded-none', + sm: 'rounded-sm', + md: 'rounded-md', + lg: 'rounded-lg', + full: 'rounded-full', +}; + +/** + * Shared configuration for span categories containing label, theme, and icon + */ +export const SPAN_CATEGORY_CONFIG: Record< + TraceSpanCategory, + { + label: string; + theme: ColorVariant; + icon: LucideIcon; + } +> = { + llm_call: { + label: 'LLM', + theme: 'purple', + icon: Zap, + }, + tool_execution: { + label: 'TOOL', + theme: 'orange', + icon: Wrench, + }, + agent_invocation: { + label: 'AGENT INVOCATION', + theme: 'indigo', + icon: Bot, + }, + chain_operation: { + label: 'CHAIN', + theme: 'teal', + icon: Link, + }, + retrieval: { + label: 'RETRIEVAL', + theme: 'cyan', + icon: Search, + }, + embedding: { + label: 'EMBEDDING', + theme: 'emerald', + icon: BarChart2, + }, + create_agent: { + label: 'CREATE AGENT', + theme: 'sky', + icon: Plus, + }, + span: { + label: 'SPAN', + theme: 'cyan', + icon: MoveHorizontal, + }, + event: { + label: 'EVENT', + theme: 'emerald', + icon: CircleDot, + }, + guardrail: { + label: 'GUARDRAIL', + theme: 'red', + icon: ShieldCheck, + }, + unknown: { + label: 'UNKNOWN', + theme: 'gray', + icon: HelpCircle, + }, +}; + +// UTILS + +export function getSpanCategoryTheme( + category: TraceSpanCategory, +): ColorVariant { + return SPAN_CATEGORY_CONFIG[category].theme; +} + +export function getSpanCategoryLabel(category: TraceSpanCategory): string { + return SPAN_CATEGORY_CONFIG[category].label; +} + +export function getSpanCategoryIcon(category: TraceSpanCategory): LucideIcon { + return SPAN_CATEGORY_CONFIG[category].icon; +} + +export const useIsMobile = () => { + const isMounted = useIsMounted(); + + const [isMobile, setIsMobile] = useState(false); + + useEffect(() => { + // TODO: replace with something more beautiful and correct (tailwind screens?) + const mediaQuery = window.matchMedia('(max-width: 1023px)'); + + const handleChange = (e: MediaQueryListEvent | MediaQueryList) => { + setIsMobile(e.matches); + }; + + handleChange(mediaQuery); + + mediaQuery.addEventListener('change', handleChange); + + return () => mediaQuery.removeEventListener('change', handleChange); + }, []); + + return isMounted ? isMobile : false; +}; + +export const useIsMounted = () => { + const [isMounted, setIsMounted] = useState(false); + + useEffect(() => { + setIsMounted(true); + }, []); + + return isMounted; +}; diff --git a/frontend/app/src/components/v1/agent-prism/theme/index.ts b/frontend/app/src/components/v1/agent-prism/theme/index.ts new file mode 100644 index 0000000000..b0687c31ed --- /dev/null +++ b/frontend/app/src/components/v1/agent-prism/theme/index.ts @@ -0,0 +1,99 @@ +export const agentPrismPrefix = 'agentprism'; + +export const AGENT_PRISM_TOKENS = [ + 'background', + 'foreground', + 'primary', + 'primary-foreground', + 'secondary', + 'secondary-foreground', + 'muted', + 'muted-foreground', + 'accent', + 'accent-foreground', + 'brand', + 'brand-foreground', + 'brand-secondary', + 'brand-secondary-foreground', + 'border', + 'border-subtle', + 'border-strong', + 'border-inverse', + 'success', + 'success-muted', + 'success-muted-foreground', + 'error', + 'error-muted', + 'error-muted-foreground', + 'warning', + 'warning-muted', + 'warning-muted-foreground', + 'pending', + 'pending-muted', + 'pending-muted-foreground', + 'code-string', + 'code-number', + 'code-key', + 'code-base', + 'badge-default', + 'badge-default-foreground', + 'avatar-llm', + 'badge-llm', + 'badge-llm-foreground', + 'timeline-llm', + 'avatar-agent', + 'badge-agent', + 'badge-agent-foreground', + 'timeline-agent', + 'avatar-tool', + 'badge-tool', + 'badge-tool-foreground', + 'timeline-tool', + 'avatar-chain', + 'badge-chain', + 'badge-chain-foreground', + 'timeline-chain', + 'avatar-retrieval', + 'badge-retrieval', + 'badge-retrieval-foreground', + 'timeline-retrieval', + 'avatar-embedding', + 'badge-embedding', + 'badge-embedding-foreground', + 'timeline-embedding', + 'avatar-guardrail', + 'badge-guardrail', + 'badge-guardrail-foreground', + 'timeline-guardrail', + 'avatar-create-agent', + 'badge-create-agent', + 'badge-create-agent-foreground', + 'timeline-create-agent', + 'avatar-span', + 'badge-span', + 'badge-span-foreground', + 'timeline-span', + 'avatar-event', + 'badge-event', + 'badge-event-foreground', + 'timeline-event', + 'avatar-unknown', + 'badge-unknown', + 'badge-unknown-foreground', + 'timeline-unknown', +] as const; + +export type AgentPrismToken = (typeof AGENT_PRISM_TOKENS)[number]; + +export type AgentPrismColors = Record; + +export const agentPrismTailwindColors = Object.fromEntries( + AGENT_PRISM_TOKENS.map((tokenName) => [ + `agentprism-${tokenName}`, + token(tokenName), + ]), +) as AgentPrismColors; + +function token(name: string) { + return `oklch(var(--${agentPrismPrefix}-${name}) / )`; +} diff --git a/frontend/app/src/components/v1/agent-prism/theme/theme.css b/frontend/app/src/components/v1/agent-prism/theme/theme.css new file mode 100644 index 0000000000..c44c3ea3ce --- /dev/null +++ b/frontend/app/src/components/v1/agent-prism/theme/theme.css @@ -0,0 +1,245 @@ +:root { + @media (prefers-color-scheme: light) { + /* General purpose colors */ + --agentprism-background: 1 0 0; /* white */ + --agentprism-foreground: 21% 0.034 264.665; /* gray.900 */ + --agentprism-primary: 21% 0.034 264.665; /* gray.900 */ + --agentprism-primary-foreground: 98.5% 0.002 247.839; /* gray.50 */ + --agentprism-secondary: 96.7% 0.003 264.542; /* gray.100 */ + --agentprism-secondary-foreground: 55.1% 0.027 264.364; /* gray.500 */ + --agentprism-muted: 98.5% 0.002 247.839; /* gray.50 */ + --agentprism-muted-foreground: 44.6% 0.03 256.802; /* gray.600 */ + --agentprism-accent: 96.7% 0.003 264.542; /* gray.100 */ + --agentprism-accent-foreground: 96.7% 0.003 264.542; /* gray.100 */ + + /* Brand colors */ + --agentprism-brand: 60.6% 0.25 292.717; /* violet.500 */ + --agentprism-brand-foreground: 1 0 0; /* white */ + --agentprism-brand-secondary: 70.5% 0.213 47.604; /* orange.500 */ + --agentprism-brand-secondary-foreground: 1 0 0; /* white */ + + /* Borders */ + --agentprism-border: 92.8% 0.006 264.531; /* gray.200 */ + --agentprism-border-subtle: 96.7% 0.003 264.542; /* gray.100 */ + --agentprism-border-strong: 87.2% 0.01 258.338; /* gray.300 */ + --agentprism-border-inverse: 21% 0.034 264.665; /* gray.900 */ + + /* Success status color */ + --agentprism-success: 69.6% 0.17 162.48; /* emerald.500 */ + --agentprism-success-muted: 97.9% 0.021 166.113; /* emerald.50 */ + --agentprism-success-muted-foreground: 50.8% 0.118 165.612; /* emerald.700 */ + + /* Error status color */ + --agentprism-error: 63.7% 0.237 25.331; /* red.500 */ + --agentprism-error-muted: 97.1% 0.013 17.38; /* red.50 */ + --agentprism-error-muted-foreground: 57.7% 0.245 27.325; /* red.600 */ + + /* Warning status color */ + --agentprism-warning: 79.5% 0.184 86.047; /* yellow.500 */ + --agentprism-warning-muted: 98.7% 0.026 102.212; /* yellow.50 */ + --agentprism-warning-muted-foreground: 55.4% 0.135 66.442; /* yellow.700 */ + + /* Pending status color */ + --agentprism-pending: 60.6% 0.25 292.717; /* violet.500 */ + --agentprism-pending-muted: 94.3% 0.029 294.588; /* violet.100 */ + --agentprism-pending-muted-foreground: 54.1% 0.281 293.009; /* violet.600 */ + + /* Code syntax highlighting */ + --agentprism-code-string: 57.7% 0.245 27.325; /* red.600 */ + --agentprism-code-number: 57.7% 0.245 27.325; /* red.600 */ + --agentprism-code-key: 54.6% 0.245 262.881; /* blue.600 */ + --agentprism-code-base: 55.1% 0.027 264.364; /* gray.500 */ + + /* Generic badge colors */ + --agentprism-badge-default: 96.7% 0.003 264.542; /* gray.100 */ + --agentprism-badge-default-foreground: 44.6% 0.03 256.802; /* gray.600 */ + + /* Trace colors (llm) */ + --agentprism-avatar-llm: 62.7% 0.265 303.9; /* purple.500 */ + --agentprism-badge-llm: 97.7% 0.014 308.299; /* purple.50 */ + --agentprism-badge-llm-foreground: 62.7% 0.265 303.9; /* purple.500 */ + --agentprism-timeline-llm: 71.4% 0.203 305.504; /* purple.400 */ + + /* Trace colors (agent) */ + --agentprism-avatar-agent: 58.5% 0.233 277.117; /* indigo.500 */ + --agentprism-badge-agent: 96.2% 0.018 272.314; /* indigo.50 */ + --agentprism-badge-agent-foreground: 58.5% 0.233 277.117; /* indigo.500 */ + --agentprism-timeline-agent: 67.3% 0.182 276.935; /* indigo.400 */ + + /* Trace colors (tool) */ + --agentprism-avatar-tool: 70.5% 0.213 47.604; /* orange.500 */ + --agentprism-badge-tool: 98% 0.016 73.684; /* orange.50 */ + --agentprism-badge-tool-foreground: 70.5% 0.213 47.604; /* orange.500 */ + --agentprism-timeline-tool: 75% 0.183 55.934; /* orange.400 */ + + /* Trace colors (chain) */ + --agentprism-avatar-chain: 70.4% 0.14 182.503; /* teal.500 */ + --agentprism-badge-chain: 98.4% 0.014 180.72; /* teal.50 */ + --agentprism-badge-chain-foreground: 70.4% 0.14 182.503; /* teal.500 */ + --agentprism-timeline-chain: 77.7% 0.152 181.912; /* teal.400 */ + + /* Trace colors (retrieval) */ + --agentprism-avatar-retrieval: 71.5% 0.143 215.221; /* cyan.500 */ + --agentprism-badge-retrieval: 98.4% 0.019 200.873; /* cyan.50 */ + --agentprism-badge-retrieval-foreground: 71.5% 0.143 215.221; /* cyan.500 */ + --agentprism-timeline-retrieval: 78.9% 0.154 211.53; /* cyan.400 */ + + /* Trace colors (embedding) */ + --agentprism-avatar-embedding: 69.6% 0.17 162.48; /* emerald.500 */ + --agentprism-badge-embedding: 97.9% 0.021 166.113; /* emerald.50 */ + --agentprism-badge-embedding-foreground: 69.6% 0.17 162.48; /* emerald.500 */ + --agentprism-timeline-embedding: 76.5% 0.177 163.223; /* emerald.400 */ + + /* Trace colors (guardrail) */ + --agentprism-avatar-guardrail: 63.7% 0.237 25.331; /* red.500 */ + --agentprism-badge-guardrail: 97.1% 0.013 17.38; /* red.50 */ + --agentprism-badge-guardrail-foreground: 63.7% 0.237 25.331; /* red.500 */ + --agentprism-timeline-guardrail: 70.4% 0.191 22.216; /* red.400 */ + + /* Trace colors (create agent) */ + --agentprism-avatar-create-agent: 68.5% 0.169 237.323; /* sky.500 */ + --agentprism-badge-create-agent: 97.7% 0.013 236.62; /* sky.50 */ + --agentprism-badge-create-agent-foreground: 68.5% 0.169 237.323; /* sky.500 */ + --agentprism-timeline-create-agent: 74.6% 0.16 232.661; /* sky.400 */ + + /* Trace colors (span) */ + --agentprism-avatar-span: 71.5% 0.143 215.221; /* cyan.500 */ + --agentprism-badge-span: 98.4% 0.019 200.873; /* cyan.50 */ + --agentprism-badge-span-foreground: 71.5% 0.143 215.221; /* cyan.500 */ + --agentprism-timeline-span: 78.9% 0.154 211.53; /* cyan.400 */ + + /* Trace colors (event) */ + --agentprism-avatar-event: 69.6% 0.17 162.48; /* emerald.500 */ + --agentprism-badge-event: 97.9% 0.021 166.113; /* emerald.50 */ + --agentprism-badge-event-foreground: 69.6% 0.17 162.48; /* emerald.500 */ + --agentprism-timeline-event: 76.5% 0.177 163.223; /* emerald.400 */ + + /* Trace colors (unknown) */ + --agentprism-avatar-unknown: 55.1% 0.027 264.364; /* gray.500 */ + --agentprism-badge-unknown: 98.5% 0.002 247.839; /* gray.50 */ + --agentprism-badge-unknown-foreground: 55.1% 0.027 264.364; /* gray.500 */ + --agentprism-timeline-unknown: 70.7% 0.022 261.325; /* gray.400 */ + } + + @media (prefers-color-scheme: dark) { + /* General purpose colors */ + --agentprism-background: 13% 0.028 261.692; /* gray.950 */ + --agentprism-foreground: 96.7% 0.003 264.542; /* gray.100 */ + --agentprism-primary: 96.7% 0.003 264.542; /* gray.100 */ + --agentprism-primary-foreground: 13% 0.028 261.692; /* gray.950 */ + --agentprism-secondary: 27.8% 0.033 256.848; /* gray.800 */ + --agentprism-secondary-foreground: 55.1% 0.027 264.364; /* gray.500 */ + --agentprism-muted: 21% 0.034 264.665; /* gray.900 */ + --agentprism-muted-foreground: 70.7% 0.022 261.325; /* gray.400 */ + --agentprism-accent: 96.7% 0.003 264.542; /* gray.100 */ + --agentprism-accent-foreground: 21% 0.034 264.665; /* gray.900 */ + + /* Brand colors */ + --agentprism-brand: 60.6% 0.25 292.717; /* violet.500 */ + --agentprism-brand-foreground: 1 0 0; /* white */ + --agentprism-brand-secondary: 70.5% 0.213 47.604; /* orange.500 */ + --agentprism-brand-secondary-foreground: 1 0 0; /* white */ + + /* Borders */ + --agentprism-border: 44.6% 0.03 256.802; /* gray.600 */ + --agentprism-border-subtle: 37.3% 0.034 259.733; /* gray.700 */ + --agentprism-border-strong: 55.1% 0.027 264.364; /* gray.500 */ + --agentprism-border-inverse: 92.8% 0.006 264.531; /* gray.200 */ + + /* Success status color */ + --agentprism-success: 72.3% 0.219 149.579; /* green.500 */ + --agentprism-success-muted: 26.6% 0.065 152.934; /* green.950 */ + --agentprism-success-muted-foreground: 87.1% 0.15 154.449; /* green.300 */ + + /* Error status color */ + --agentprism-error: 63.7% 0.237 25.331; /* red.500 */ + --agentprism-error-muted: 25.8% 0.092 26.042; /* red.950 */ + --agentprism-error-muted-foreground: 80.8% 0.114 19.571; /* red.300 */ + + /* Warning status color */ + --agentprism-warning: 79.5% 0.184 86.047; /* yellow.500 */ + --agentprism-warning-muted: 28.6% 0.066 53.813; /* yellow.950 */ + --agentprism-warning-muted-foreground: 90.5% 0.182 98.111; /* yellow.300 */ + + /* Pending status color */ + --agentprism-pending: 60.6% 0.25 292.717; /* violet.500 */ + --agentprism-pending-muted: 28.3% 0.141 291.089; /* violet.950 */ + --agentprism-pending-muted-foreground: 70.2% 0.183 293.541; /* violet.400 */ + + /* Code syntax highlighting */ + --agentprism-code-string: 70.4% 0.191 22.216; /* red.400 */ + --agentprism-code-number: 70.4% 0.191 22.216; /* red.400 */ + --agentprism-code-key: 80.9% 0.105 251.813; /* blue.300 */ + --agentprism-code-base: 70.7% 0.022 261.325; /* gray.400 */ + + /* Generic badge colors */ + --agentprism-badge-default: 21% 0.034 264.665; /* gray.900 */ + --agentprism-badge-default-foreground: 70.7% 0.022 261.325; /* gray.400 */ + + /* Trace colors (llm) */ + --agentprism-avatar-llm: 82.7% 0.119 306.383; /* purple.300 */ + --agentprism-badge-llm: 29.1% 0.149 302.717; /* purple.950 */ + --agentprism-badge-llm-foreground: 82.7% 0.119 306.383; /* purple.300 */ + --agentprism-timeline-llm: 71.4% 0.203 305.504; /* purple.400 */ + + /* Trace colors (agent) */ + --agentprism-avatar-agent: 78.5% 0.115 274.713; /* indigo.300 */ + --agentprism-badge-agent: 25.7% 0.09 281.288; /* indigo.950 */ + --agentprism-badge-agent-foreground: 78.5% 0.115 274.713; /* indigo.300 */ + --agentprism-timeline-agent: 67.3% 0.182 276.935; /* indigo.400 */ + + /* Trace colors (tool) */ + --agentprism-avatar-tool: 83.7% 0.128 66.29; /* orange.300 */ + --agentprism-badge-tool: 26.6% 0.079 36.259; /* orange.950 */ + --agentprism-badge-tool-foreground: 83.7% 0.128 66.29; /* orange.300 */ + --agentprism-timeline-tool: 75% 0.183 55.934; /* orange.400 */ + + /* Trace colors (chain) */ + --agentprism-avatar-chain: 85.5% 0.138 181.071; /* teal.300 */ + --agentprism-badge-chain: 27.7% 0.046 192.524; /* teal.950 */ + --agentprism-badge-chain-foreground: 85.5% 0.138 181.071; /* teal.300 */ + --agentprism-timeline-chain: 77.7% 0.152 181.912; /* teal.400 */ + + /* Trace colors (retrieval) */ + --agentprism-avatar-retrieval: 86.5% 0.127 207.078; /* cyan.300 */ + --agentprism-badge-retrieval: 30.2% 0.056 229.695; /* cyan.950 */ + --agentprism-badge-retrieval-foreground: 86.5% 0.127 207.078; /* cyan.300 */ + --agentprism-timeline-retrieval: 78.9% 0.154 211.53; /* cyan.400 */ + + /* Trace colors (embedding) */ + --agentprism-avatar-embedding: 84.5% 0.143 164.978; /* emerald.300 */ + --agentprism-badge-embedding: 26.2% 0.051 172.552; /* emerald.950 */ + --agentprism-badge-embedding-foreground: 84.5% 0.143 164.978; /* emerald.300 */ + --agentprism-timeline-embedding: 76.5% 0.177 163.223; /* emerald.400 */ + + /* Trace colors (guardrail) */ + --agentprism-avatar-guardrail: 80.8% 0.114 19.571; /* red.300 */ + --agentprism-badge-guardrail: 25.8% 0.092 26.042; /* red.950 */ + --agentprism-badge-guardrail-foreground: 80.8% 0.114 19.571; /* red.300 */ + --agentprism-timeline-guardrail: 70.4% 0.191 22.216; /* red.400 */ + + /* Trace colors (create agent) */ + --agentprism-avatar-create-agent: 82.8% 0.111 230.318; /* sky.300 */ + --agentprism-badge-create-agent: 29.3% 0.066 243.157; /* sky.950 */ + --agentprism-badge-create-agent-foreground: 82.8% 0.111 230.318; /* sky.300 */ + --agentprism-timeline-create-agent: 74.6% 0.16 232.661; /* sky.400 */ + + /* Trace colors (span) */ + --agentprism-avatar-span: 86.5% 0.127 207.078; /* cyan.300 */ + --agentprism-badge-span: 30.2% 0.056 229.695; /* cyan.950 */ + --agentprism-badge-span-foreground: 86.5% 0.127 207.078; /* cyan.300 */ + --agentprism-timeline-span: 78.9% 0.154 211.53; /* cyan.400 */ + + /* Trace colors (event) */ + --agentprism-avatar-event: 84.5% 0.143 164.978; /* emerald.300 */ + --agentprism-badge-event: 26.2% 0.051 172.552; /* emerald.950 */ + --agentprism-badge-event-foreground: 84.5% 0.143 164.978; /* emerald.300 */ + --agentprism-timeline-event: 76.5% 0.177 163.223; /* emerald.400 */ + + /* Trace colors (unknown) */ + --agentprism-avatar-unknown: 87.2% 0.01 258.338; /* gray.300 */ + --agentprism-badge-unknown: 13% 0.028 261.692; /* gray.950 */ + --agentprism-badge-unknown-foreground: 87.2% 0.01 258.338; /* gray.300 */ + --agentprism-timeline-unknown: 70.7% 0.022 261.325; /* gray.400 */ + } +} diff --git a/frontend/app/src/lib/api/generated/cloud/Api.ts b/frontend/app/src/lib/api/generated/cloud/Api.ts index 0e4ce7b191..3db0b7c591 100644 --- a/frontend/app/src/lib/api/generated/cloud/Api.ts +++ b/frontend/app/src/lib/api/generated/cloud/Api.ts @@ -45,6 +45,7 @@ import { OrganizationForUserList, OrganizationInviteList, OrganizationTenant, + OtelSpanList, RejectOrganizationInviteRequest, RemoveOrganizationMembersRequest, RuntimeConfigActionsResponse, @@ -739,6 +740,23 @@ export class Api< type: ContentType.Json, ...params, }); + /** + * @description Lists OTel spans for a task + * + * @tags Observability + * @name OtelTracesList + * @summary List OTel Traces + * @request GET:/api/v1/cloud/tasks/{task}/traces + * @secure + */ + otelTracesList = (task: string, params: RequestParams = {}) => + this.request({ + path: `/api/v1/cloud/tasks/${task}/traces`, + method: "GET", + secure: true, + format: "json", + ...params, + }); /** * @description Receive a webhook message from Autumn * diff --git a/frontend/app/src/lib/api/generated/cloud/data-contracts.ts b/frontend/app/src/lib/api/generated/cloud/data-contracts.ts index e2765deb36..70dfa35d89 100644 --- a/frontend/app/src/lib/api/generated/cloud/data-contracts.ts +++ b/frontend/app/src/lib/api/generated/cloud/data-contracts.ts @@ -478,6 +478,29 @@ export interface LogLineList { pagination?: PaginationResponse; } +export interface OtelSpan { + trace_id: string; + span_id: string; + parent_span_id?: string; + span_name: string; + span_kind: string; + service_name: string; + status_code: string; + status_message?: string; + /** @format int64 */ + duration: number; + /** @format date-time */ + created_at: string; + resource_attributes?: Record; + span_attributes?: Record; + scope_name?: string; + scope_version?: string; +} + +export interface OtelSpanList { + rows?: OtelSpan[]; +} + export type Matrix = SampleStream[]; export interface SampleStream { diff --git a/frontend/app/src/main.tsx b/frontend/app/src/main.tsx index 280a85eaf5..98c71f4d85 100644 --- a/frontend/app/src/main.tsx +++ b/frontend/app/src/main.tsx @@ -1,3 +1,4 @@ +import './components/v1/agent-prism/theme/theme.css'; import './index.css'; import queryClient from './query-client.tsx'; import Router from './router.tsx'; diff --git a/frontend/app/src/pages/main/v1/workflow-runs-v1/$run/v2components/step-run-detail/otel-span-adapter.ts b/frontend/app/src/pages/main/v1/workflow-runs-v1/$run/v2components/step-run-detail/otel-span-adapter.ts new file mode 100644 index 0000000000..d29c526325 --- /dev/null +++ b/frontend/app/src/pages/main/v1/workflow-runs-v1/$run/v2components/step-run-detail/otel-span-adapter.ts @@ -0,0 +1,77 @@ +import type { OtelSpan } from '@/lib/api/generated/cloud/data-contracts'; +import type { + OpenTelemetrySpan, + OpenTelemetrySpanKind, + OpenTelemetryStatusCode, + TraceSpanAttribute, +} from '@evilmartians/agent-prism-types'; + +const SPAN_KIND_MAP: Record = { + INTERNAL: 'SPAN_KIND_INTERNAL', + SERVER: 'SPAN_KIND_SERVER', + CLIENT: 'SPAN_KIND_CLIENT', + PRODUCER: 'SPAN_KIND_PRODUCER', + CONSUMER: 'SPAN_KIND_CONSUMER', +}; + +const STATUS_CODE_MAP: Record = { + OK: 'STATUS_CODE_OK', + ERROR: 'STATUS_CODE_ERROR', + UNSET: 'STATUS_CODE_UNSET', +}; + +function recordToAttributes( + record: Record | undefined, +): TraceSpanAttribute[] { + if (!record) return []; + return Object.entries(record).map(([key, value]) => ({ + key, + value: { stringValue: value }, + })); +} + +/** + * Converts our API's flat OtelSpan format to the OTLP-style OpenTelemetrySpan + * expected by @evilmartians/agent-prism-data adapters. + */ +export function convertOtelSpanToOpenTelemetrySpan( + span: OtelSpan, +): OpenTelemetrySpan { + const startNanos = BigInt(new Date(span.created_at).getTime()) * 1_000_000n; + const durationNanos = BigInt(span.duration); + const endNanos = startNanos + durationNanos; + + return { + traceId: span.trace_id, + spanId: span.span_id, + parentSpanId: span.parent_span_id || undefined, + name: span.span_name, + kind: SPAN_KIND_MAP[span.span_kind] || 'SPAN_KIND_INTERNAL', + startTimeUnixNano: startNanos.toString(), + endTimeUnixNano: endNanos.toString(), + attributes: [ + ...recordToAttributes(span.span_attributes), + ...recordToAttributes(span.resource_attributes), + { key: 'service.name', value: { stringValue: span.service_name } }, + ], + status: { + code: STATUS_CODE_MAP[span.status_code] || 'STATUS_CODE_UNSET', + message: span.status_message, + }, + flags: 0, + }; +} + +export function convertOtelSpans(spans: OtelSpan[]): OpenTelemetrySpan[] { + const converted = spans.map(convertOtelSpanToOpenTelemetrySpan); + + // Promote orphaned spans to root spans: if a span's parentSpanId + // references a span not in this set, clear it so the tree builder + // treats it as a root instead of silently dropping it. + const spanIdSet = new Set(converted.map((s) => s.spanId)); + return converted.map((s) => + s.parentSpanId && !spanIdSet.has(s.parentSpanId) + ? { ...s, parentSpanId: undefined } + : s, + ); +} diff --git a/frontend/app/src/pages/main/v1/workflow-runs-v1/$run/v2components/step-run-detail/step-run-detail.tsx b/frontend/app/src/pages/main/v1/workflow-runs-v1/$run/v2components/step-run-detail/step-run-detail.tsx index b3f566abc0..3af311cf64 100644 --- a/frontend/app/src/pages/main/v1/workflow-runs-v1/$run/v2components/step-run-detail/step-run-detail.tsx +++ b/frontend/app/src/pages/main/v1/workflow-runs-v1/$run/v2components/step-run-detail/step-run-detail.tsx @@ -8,6 +8,7 @@ import { StepRunEvents } from '../step-run-events-for-workflow-run'; import { Waterfall } from '../waterfall'; import { V1StepRunOutput } from './step-run-output'; import { TaskRunLogs } from './task-run-logs'; +import { TaskRunTrace } from './task-run-trace'; import RelativeDate from '@/components/v1/molecules/relative-date'; import { CopyWorkflowConfigButton } from '@/components/v1/shared/copy-workflow-config'; import { Button } from '@/components/v1/ui/button'; @@ -22,6 +23,7 @@ import { import { useSidePanel } from '@/hooks/use-side-panel'; import { V1TaskStatus, V1TaskSummary, queries } from '@/lib/api'; import { emptyGolangUUID, formatDuration } from '@/lib/utils'; +import useCloud from '@/pages/auth/hooks/use-cloud'; import { TaskRunActionButton } from '@/pages/main/v1/task-runs-v1/actions'; import { WorkflowDefinitionLink } from '@/pages/main/workflow-runs/$run/v2components/workflow-definition'; import { appRoutes } from '@/router'; @@ -35,6 +37,7 @@ export enum TabOption { ChildWorkflowRuns = 'child-workflow-runs', Input = 'input', Logs = 'logs', + Trace = 'trace', Waterfall = 'waterfall', AdditionalMetadata = 'additional-metadata', Activity = 'activity', @@ -106,6 +109,7 @@ export const TaskRunDetail = ({ showViewTaskRunButton, }: TaskRunDetailProps) => { const { open } = useSidePanel(); + const { isCloudEnabled } = useCloud(); const [logsResetKey, setLogsResetKey] = useState(0); const handleTaskRunExpand = useCallback( (taskRunId: string) => { @@ -248,6 +252,11 @@ export const TaskRunDetail = ({ Logs + {isCloudEnabled && ( + + Trace + + )} + {isCloudEnabled && ( + + + + )} { + const res = await cloudApi.otelTracesList(taskExternalId); + return res.data; + }, + }); + + const traceSpans = useMemo(() => { + const rows = tracesQuery.data?.rows; + if (!rows || rows.length === 0) return []; + + const otlpSpans = convertOtelSpans(rows); + return openTelemetrySpanAdapter.convertRawSpansToSpanTree(otlpSpans); + }, [tracesQuery.data]); + + const allIds = useMemo( + () => flattenSpans(traceSpans).map((s) => s.id), + [traceSpans], + ); + + const [expandedSpansIds, setExpandedSpansIds] = useState([]); + + // Auto-expand all spans when data loads + useEffect(() => { + if (allIds.length > 0) { + setExpandedSpansIds(allIds); + } + }, [allIds]); + + if (tracesQuery.isLoading) { + return ; + } + + if (tracesQuery.isError) { + return ( +
    + Failed to load traces. +
    + ); + } + + if (traceSpans.length === 0) { + return ( +
    + No trace found for this task run. To collect traces, use the{' '} + + HatchetInstrumentor + {' '} + in your SDK. +
    + ); + } + + return ( +
    +
    +
    +

    + OpenTelemetry Traces +

    +
    +
    +
    + +
    +
    + ); +} diff --git a/frontend/app/tailwind.config.js b/frontend/app/tailwind.config.js index fa3db6977b..93f70cd30a 100644 --- a/frontend/app/tailwind.config.js +++ b/frontend/app/tailwind.config.js @@ -1,3 +1,34 @@ +// Generate agent-prism Tailwind color mappings from CSS custom properties +const agentPrismTokens = [ + "background", "foreground", "primary", "primary-foreground", "secondary", + "secondary-foreground", "muted", "muted-foreground", "accent", "accent-foreground", + "brand", "brand-foreground", "brand-secondary", "brand-secondary-foreground", + "border", "border-subtle", "border-strong", "border-inverse", + "success", "success-muted", "success-muted-foreground", + "error", "error-muted", "error-muted-foreground", + "warning", "warning-muted", "warning-muted-foreground", + "pending", "pending-muted", "pending-muted-foreground", + "code-string", "code-number", "code-key", "code-base", + "badge-default", "badge-default-foreground", + "avatar-llm", "badge-llm", "badge-llm-foreground", "timeline-llm", + "avatar-agent", "badge-agent", "badge-agent-foreground", "timeline-agent", + "avatar-tool", "badge-tool", "badge-tool-foreground", "timeline-tool", + "avatar-chain", "badge-chain", "badge-chain-foreground", "timeline-chain", + "avatar-retrieval", "badge-retrieval", "badge-retrieval-foreground", "timeline-retrieval", + "avatar-embedding", "badge-embedding", "badge-embedding-foreground", "timeline-embedding", + "avatar-guardrail", "badge-guardrail", "badge-guardrail-foreground", "timeline-guardrail", + "avatar-create-agent", "badge-create-agent", "badge-create-agent-foreground", "timeline-create-agent", + "avatar-span", "badge-span", "badge-span-foreground", "timeline-span", + "avatar-event", "badge-event", "badge-event-foreground", "timeline-event", + "avatar-unknown", "badge-unknown", "badge-unknown-foreground", "timeline-unknown", +]; +const agentPrismColors = Object.fromEntries( + agentPrismTokens.map((name) => [ + `agentprism-${name}`, + `oklch(var(--agentprism-${name}) / )`, + ]) +); + /** @type {import('tailwindcss').Config} */ module.exports = { darkMode: ["class"], @@ -49,6 +80,7 @@ module.exports = { }, extend: { colors: { + ...agentPrismColors, "purple": { "50": "hsl(252, 82%, 95%)", "100": "hsl(252, 82%, 90%)", diff --git a/pkg/repository/otelcol.go b/pkg/repository/otelcol.go index 1f6c761425..001c06b2cb 100644 --- a/pkg/repository/otelcol.go +++ b/pkg/repository/otelcol.go @@ -2,6 +2,7 @@ package repository import ( "context" + "time" "github.com/google/uuid" ) @@ -31,8 +32,26 @@ type CreateSpansOpts struct { Spans []*SpanData } +type OtelSpanRow struct { + CreatedAt time.Time + SpanAttributes map[string]string + ResourceAttributes map[string]string + SpanName string + SpanKind string + ServiceName string + StatusCode string + StatusMessage string + TraceID string + ParentSpanID string + SpanID string + ScopeName string + ScopeVersion string + Duration uint64 +} + type OTelCollectorRepository interface { CreateSpans(ctx context.Context, tenantId uuid.UUID, opts *CreateSpansOpts) error + ListSpansByTaskExternalID(ctx context.Context, tenantId, taskExternalID uuid.UUID) ([]*OtelSpanRow, error) } type otelCollectorRepositoryImpl struct { @@ -49,3 +68,8 @@ func (o *otelCollectorRepositoryImpl) CreateSpans(ctx context.Context, tenantId // intentional no-op, intended to be overridden return nil } + +func (o *otelCollectorRepositoryImpl) ListSpansByTaskExternalID(ctx context.Context, tenantId, taskExternalID uuid.UUID) ([]*OtelSpanRow, error) { + // intentional no-op, intended to be overridden + return nil, nil +} diff --git a/pkg/worker/context.go b/pkg/worker/context.go index 3c9eec5925..931da48b05 100644 --- a/pkg/worker/context.go +++ b/pkg/worker/context.go @@ -91,6 +91,12 @@ type HatchetContext interface { WasSkipped(parent create.NamedTask) bool + TenantId() string + + WorkerId() string + + ActionId() string + client() client.Client action() *client.Action @@ -208,6 +214,18 @@ func (h *hatchetContext) action() *client.Action { return h.a } +func (h *hatchetContext) TenantId() string { + return h.a.TenantId +} + +func (h *hatchetContext) WorkerId() string { + return h.a.WorkerId +} + +func (h *hatchetContext) ActionId() string { + return h.a.ActionId +} + func (h *hatchetContext) Worker() HatchetWorkerContext { return h.w } diff --git a/pkg/worker/middleware_test.go b/pkg/worker/middleware_test.go index 88ff3066ce..0dfb6eb6fa 100644 --- a/pkg/worker/middleware_test.go +++ b/pkg/worker/middleware_test.go @@ -146,6 +146,18 @@ func (c *testHatchetContext) FilterPayload() map[string]interface{} { panic("not implemented") } +func (c *testHatchetContext) ActionId() string { + panic("not implemented") +} + +func (c *testHatchetContext) TenantId() string { + panic("not implemented") +} + +func (c *testHatchetContext) WorkerId() string { + panic("not implemented") +} + func TestAddMiddleware(t *testing.T) { m := middlewares{} middlewareFunc := func(ctx HatchetContext, next func(HatchetContext) error) error { diff --git a/sdks/go/client.go b/sdks/go/client.go index 10827c00ac..bfa72543b6 100644 --- a/sdks/go/client.go +++ b/sdks/go/client.go @@ -238,6 +238,19 @@ func resolveWorkerSlotConfig( return slotConfig } +// Use registers middleware functions on the worker. +// Middleware functions are called in order for each step run execution. +// +//nolint:staticcheck // SA1019: worker.MiddlewareFunc is deprecated but still used internally +func (w *Worker) Use(mws ...worker.MiddlewareFunc) { + if w.worker != nil { + w.worker.Use(mws...) //nolint:staticcheck // SA1019 + } + if w.legacyDurable != nil { + w.legacyDurable.Use(mws...) //nolint:staticcheck // SA1019 + } +} + // Starts the worker instance and returns a cleanup function. func (w *Worker) Start() (func() error, error) { var workers []*worker.Worker diff --git a/sdks/go/examples/opentelemetry/main.go b/sdks/go/examples/opentelemetry/main.go new file mode 100644 index 0000000000..3b6fd8c75b --- /dev/null +++ b/sdks/go/examples/opentelemetry/main.go @@ -0,0 +1,162 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "log" + "math/rand/v2" //nolint:gosec // G404: example code, not security-sensitive + "time" + + "go.opentelemetry.io/otel" + + "github.com/hatchet-dev/hatchet/pkg/cmdutils" + hatchet "github.com/hatchet-dev/hatchet/sdks/go" + hatchetotel "github.com/hatchet-dev/hatchet/sdks/go/opentelemetry" +) + +type PipelineInput struct { + URL string `json:"url"` +} + +type FetchOutput struct { + Data string `json:"data"` +} + +type ValidateOutput struct { + Valid bool `json:"valid"` + FieldCount int `json:"field_count"` +} + +type ProcessOutput struct { + ProcessedData string `json:"processed_data"` + RecordCount int `json:"record_count"` +} + +type SaveOutput struct { + Location string `json:"location"` + RecordsSaved int `json:"records_saved"` +} + +func randMillis(base, jitter int) time.Duration { + return time.Duration(base+rand.IntN(jitter)) * time.Millisecond //nolint:gosec // G404 +} + +func main() { + client, err := hatchet.NewClient() + if err != nil { + log.Fatalf("failed to create client: %v", err) + } + + // Set up OpenTelemetry instrumentation. + // EnableHatchetCollector() auto-configures from the same env vars as the client + // (HATCHET_CLIENT_HOST_PORT, HATCHET_CLIENT_TOKEN, HATCHET_CLIENT_TLS_STRATEGY). + instrumentor, err := hatchetotel.NewInstrumentor( + hatchetotel.EnableHatchetCollector(), + ) + if err != nil { + log.Fatalf("failed to create instrumentor: %v", err) + } + + tracer := otel.Tracer("otel-example") + + // Create a multi-task workflow + workflow := client.NewWorkflow("otel-data-pipeline") + + fetchData := workflow.NewTask("fetch-data", func(ctx hatchet.Context, input PipelineInput) (*FetchOutput, error) { + _, span := tracer.Start(ctx.GetContext(), fmt.Sprintf("GET %s", input.URL)) + time.Sleep(randMillis(10, 20)) + span.End() + + _, parseSpan := tracer.Start(ctx.GetContext(), "json.parse") + time.Sleep(randMillis(5, 10)) + parseSpan.End() + + return &FetchOutput{ + Data: `{"users": [{"name": "Alice"}, {"name": "Bob"}]}`, + }, nil + }) + + validateData := workflow.NewTask("validate-data", func(ctx hatchet.Context, input PipelineInput) (*ValidateOutput, error) { + var parentOutput FetchOutput + if parentErr := ctx.ParentOutput(fetchData, &parentOutput); parentErr != nil { + return nil, parentErr + } + + _, span := tracer.Start(ctx.GetContext(), "schema.validate") + time.Sleep(randMillis(5, 10)) + + var parsed map[string]any + if unmarshalErr := json.Unmarshal([]byte(parentOutput.Data), &parsed); unmarshalErr != nil { + span.End() + return nil, fmt.Errorf("invalid JSON: %w", unmarshalErr) + } + span.End() + + return &ValidateOutput{ + Valid: true, + FieldCount: len(parsed), + }, nil + }, hatchet.WithParents(fetchData)) + + processData := workflow.NewTask("process-data", func(ctx hatchet.Context, input PipelineInput) (*ProcessOutput, error) { + var validateOutput ValidateOutput + if parentErr := ctx.ParentOutput(validateData, &validateOutput); parentErr != nil { + return nil, parentErr + } + + _, span := tracer.Start(ctx.GetContext(), "data.transform") + time.Sleep(randMillis(10, 15)) + span.End() + + _, enrichSpan := tracer.Start(ctx.GetContext(), "data.enrich") + time.Sleep(randMillis(5, 10)) + enrichSpan.End() + + return &ProcessOutput{ + ProcessedData: "transformed_and_enriched", + RecordCount: validateOutput.FieldCount, + }, nil + }, hatchet.WithParents(validateData)) + + workflow.NewTask("save-results", func(ctx hatchet.Context, input PipelineInput) (*SaveOutput, error) { + var processOutput ProcessOutput + if parentErr := ctx.ParentOutput(processData, &processOutput); parentErr != nil { + return nil, parentErr + } + + _, span := tracer.Start(ctx.GetContext(), "db.insert") + time.Sleep(randMillis(10, 20)) + span.End() + + return &SaveOutput{ + RecordsSaved: processOutput.RecordCount, + Location: "postgresql://localhost/pipeline_results", + }, nil + }, hatchet.WithParents(processData)) + + // Create worker and register the OTel middleware + worker, err := client.NewWorker("otel-worker", hatchet.WithWorkflows(workflow)) + if err != nil { + log.Fatalf("failed to create worker: %v", err) + } + + worker.Use(instrumentor.Middleware()) + + interruptCtx, cancel := cmdutils.NewInterruptContext() + defer cancel() + + fmt.Println("Starting worker with OpenTelemetry instrumentation...") + + go func() { + <-interruptCtx.Done() + // Flush remaining spans before exit + if shutdownErr := instrumentor.Shutdown(context.Background()); shutdownErr != nil { + log.Printf("failed to shutdown instrumentor: %v", shutdownErr) + } + }() + + if startErr := worker.StartBlocking(interruptCtx); startErr != nil { + log.Printf("worker error: %v", startErr) + } +} diff --git a/sdks/go/opentelemetry/attributes.go b/sdks/go/opentelemetry/attributes.go new file mode 100644 index 0000000000..7518a0d5f4 --- /dev/null +++ b/sdks/go/opentelemetry/attributes.go @@ -0,0 +1,45 @@ +package opentelemetry + +import ( + "context" + + "go.opentelemetry.io/otel/attribute" + + "github.com/hatchet-dev/hatchet/pkg/worker" +) + +type hatchetAttrsKeyType struct{} + +var hatchetAttrsKey = hatchetAttrsKeyType{} + +// hatchetAttributes builds the set of hatchet.* span attributes from a HatchetContext. +func hatchetAttributes(ctx worker.HatchetContext) []attribute.KeyValue { + attrs := []attribute.KeyValue{ + attribute.String("hatchet.tenant_id", ctx.TenantId()), + attribute.String("hatchet.worker_id", ctx.WorkerId()), + attribute.String("hatchet.workflow_run_id", ctx.WorkflowRunId()), + attribute.String("hatchet.step_run_id", ctx.StepRunId()), + attribute.String("hatchet.step_id", ctx.StepId()), + attribute.String("hatchet.action_id", ctx.ActionId()), + attribute.String("hatchet.step_name", ctx.StepName()), + attribute.Int("hatchet.retry_count", ctx.RetryCount()), + } + + if wfID := ctx.WorkflowId(); wfID != nil { + attrs = append(attrs, attribute.String("hatchet.workflow_id", *wfID)) + } + + return attrs +} + +// withHatchetAttributes stores hatchet attributes in the context so the +// SpanProcessor can inject them into child spans. +func withHatchetAttributes(ctx context.Context, attrs []attribute.KeyValue) context.Context { + return context.WithValue(ctx, hatchetAttrsKey, attrs) +} + +// getHatchetAttributes retrieves hatchet attributes from the context. +func getHatchetAttributes(ctx context.Context) []attribute.KeyValue { + attrs, _ := ctx.Value(hatchetAttrsKey).([]attribute.KeyValue) + return attrs +} diff --git a/sdks/go/opentelemetry/exporter.go b/sdks/go/opentelemetry/exporter.go new file mode 100644 index 0000000000..477d7c3011 --- /dev/null +++ b/sdks/go/opentelemetry/exporter.go @@ -0,0 +1,30 @@ +package opentelemetry + +import ( + "context" + "crypto/tls" + + "google.golang.org/grpc/credentials" + + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc" + sdktrace "go.opentelemetry.io/otel/sdk/trace" +) + +// newHatchetExporter creates a gRPC OTLP trace exporter that sends spans +// to the Hatchet engine's collector endpoint. +func newHatchetExporter(endpoint, token string, insecureConn bool) (sdktrace.SpanExporter, error) { + opts := []otlptracegrpc.Option{ + otlptracegrpc.WithEndpoint(endpoint), + otlptracegrpc.WithHeaders(map[string]string{ + "authorization": "Bearer " + token, + }), + } + + if insecureConn { + opts = append(opts, otlptracegrpc.WithInsecure()) + } else { + opts = append(opts, otlptracegrpc.WithTLSCredentials(credentials.NewTLS(&tls.Config{MinVersion: tls.VersionTLS12}))) //nolint:gosec // G402 + } + + return otlptracegrpc.New(context.Background(), opts...) +} diff --git a/sdks/go/opentelemetry/instrumentor.go b/sdks/go/opentelemetry/instrumentor.go new file mode 100644 index 0000000000..51e3537cd1 --- /dev/null +++ b/sdks/go/opentelemetry/instrumentor.go @@ -0,0 +1,139 @@ +// Package opentelemetry provides OpenTelemetry instrumentation for the Hatchet Go SDK. +// +// It automatically creates spans for each step run, propagates hatchet.* attributes +// to all child spans, supports W3C traceparent propagation, and optionally sends +// traces to the Hatchet engine's OTLP collector. +// +// Basic usage: +// +// instrumentor := opentelemetry.NewInstrumentor() +// worker.Use(instrumentor.Middleware()) +// +// Sending traces to Hatchet (auto-configured from environment): +// +// instrumentor := opentelemetry.NewInstrumentor( +// opentelemetry.EnableHatchetCollector(), +// ) +// worker.Use(instrumentor.Middleware()) +package opentelemetry + +import ( + "context" + "fmt" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/sdk/resource" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + semconv "go.opentelemetry.io/otel/semconv/v1.26.0" + "go.opentelemetry.io/otel/trace" + + "github.com/hatchet-dev/hatchet/pkg/client/loader" + "github.com/hatchet-dev/hatchet/pkg/worker" +) + +// Instrumentor sets up OpenTelemetry tracing for Hatchet workers. +type Instrumentor struct { + tracerProvider trace.TracerProvider + tracer trace.Tracer + opts instrumentorOptions +} + +type instrumentorOptions struct { + tracerProvider *sdktrace.TracerProvider + + enableCollector bool +} + +// InstrumentorOption configures the Instrumentor. +type InstrumentorOption func(*instrumentorOptions) + +// WithTracerProvider sets a custom TracerProvider. If not set, a new one is created. +// The provider must be an SDK TracerProvider to support adding span processors. +func WithTracerProvider(tp *sdktrace.TracerProvider) InstrumentorOption { + return func(o *instrumentorOptions) { + o.tracerProvider = tp + } +} + +// EnableHatchetCollector enables sending traces to the Hatchet engine's OTLP collector. +// Connection settings (endpoint, token, TLS) are automatically loaded from the same +// environment variables used by the Hatchet client (HATCHET_CLIENT_HOST_PORT, +// HATCHET_CLIENT_TOKEN, HATCHET_CLIENT_TLS_STRATEGY). +func EnableHatchetCollector() InstrumentorOption { + return func(o *instrumentorOptions) { + o.enableCollector = true + } +} + +// NewInstrumentor creates a new HatchetInstrumentor. +func NewInstrumentor(opts ...InstrumentorOption) (*Instrumentor, error) { + o := &instrumentorOptions{} + for _, opt := range opts { + opt(o) + } + + // Set up TracerProvider + tp := o.tracerProvider + if tp == nil { + tp = sdktrace.NewTracerProvider( + sdktrace.WithResource(resource.NewWithAttributes( + semconv.SchemaURL, + semconv.ServiceNameKey.String("hatchet-worker"), + )), + ) + } + + // Add Hatchet collector exporter if enabled + if o.enableCollector { + cfgFile, err := loader.LoadClientConfigFile() + if err != nil { + return nil, fmt.Errorf("failed to load client config for OTel collector: %w", err) + } + + clientCfg, err := loader.GetClientConfigFromConfigFile(nil, cfgFile) + if err != nil { + return nil, fmt.Errorf("failed to resolve client config for OTel collector: %w", err) + } + + insecure := clientCfg.TLSConfig == nil + + exporter, err := newHatchetExporter(clientCfg.GRPCBroadcastAddress, clientCfg.Token, insecure) + if err != nil { + return nil, fmt.Errorf("failed to create Hatchet OTLP exporter: %w", err) + } + bsp := sdktrace.NewBatchSpanProcessor(exporter) + tp.RegisterSpanProcessor(NewHatchetAttributeSpanProcessor(bsp)) + } + + // Set as global provider + otel.SetTracerProvider(tp) + + tracer := tp.Tracer("github.com/hatchet-dev/hatchet/sdks/go/opentelemetry") + + return &Instrumentor{ + tracerProvider: tp, + tracer: tracer, + opts: *o, + }, nil +} + +// Middleware returns the OTel middleware that should be registered on the worker. +// +//nolint:staticcheck // SA1019: worker.MiddlewareFunc is deprecated but still used internally +func (i *Instrumentor) Middleware() worker.MiddlewareFunc { + return NewMiddleware(i.tracer) +} + +// TracerProvider returns the TracerProvider used by the instrumentor. +func (i *Instrumentor) TracerProvider() trace.TracerProvider { + return i.tracerProvider +} + +// Shutdown flushes any remaining spans and shuts down the TracerProvider. +// Call this before your application exits to ensure all spans are exported. +func (i *Instrumentor) Shutdown(ctx context.Context) error { + if tp, ok := i.tracerProvider.(*sdktrace.TracerProvider); ok { + return tp.Shutdown(ctx) + } + return nil +} diff --git a/sdks/go/opentelemetry/middleware.go b/sdks/go/opentelemetry/middleware.go new file mode 100644 index 0000000000..e8c8473364 --- /dev/null +++ b/sdks/go/opentelemetry/middleware.go @@ -0,0 +1,59 @@ +package opentelemetry + +import ( + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/propagation" + "go.opentelemetry.io/otel/trace" + + "github.com/hatchet-dev/hatchet/pkg/worker" +) + +// NewMiddleware creates a Hatchet middleware that wraps each step run execution +// with an OpenTelemetry span. It: +// - Extracts W3C traceparent from AdditionalMetadata for distributed trace propagation +// - Creates a "hatchet.start_step_run" span with hatchet.* attributes +// - Stores attributes in context so HatchetAttributeSpanProcessor can inject +// them into all child spans +// +//nolint:staticcheck // SA1019: worker.MiddlewareFunc is deprecated but still used internally +func NewMiddleware(tracer trace.Tracer) worker.MiddlewareFunc { + propagator := propagation.TraceContext{} + + return func(ctx worker.HatchetContext, next func(worker.HatchetContext) error) error { + // Build hatchet attributes from context + attrs := hatchetAttributes(ctx) + + // Extract traceparent from additional metadata if present + parentCtx := ctx.GetContext() + if meta := ctx.AdditionalMetadata(); meta != nil { + if tp, ok := meta["traceparent"]; ok && tp != "" { + carrier := propagation.MapCarrier(map[string]string{ + "traceparent": tp, + }) + parentCtx = propagator.Extract(parentCtx, carrier) + } + } + + // Store hatchet attributes in context for the SpanProcessor + parentCtx = withHatchetAttributes(parentCtx, attrs) + + // Start span + spanCtx, span := tracer.Start(parentCtx, "hatchet.start_step_run", + trace.WithSpanKind(trace.SpanKindConsumer), + trace.WithAttributes(attrs...), + ) + defer span.End() + + // Update the HatchetContext with the OTel-enriched context + ctx.SetContext(spanCtx) + + // Execute the next middleware/action + err := next(ctx) + if err != nil { + span.SetStatus(codes.Error, err.Error()) + span.RecordError(err) + } + + return err + } +} diff --git a/sdks/go/opentelemetry/span_processor.go b/sdks/go/opentelemetry/span_processor.go new file mode 100644 index 0000000000..6fe72169d5 --- /dev/null +++ b/sdks/go/opentelemetry/span_processor.go @@ -0,0 +1,41 @@ +package opentelemetry + +import ( + "context" + + sdktrace "go.opentelemetry.io/otel/sdk/trace" +) + +// HatchetAttributeSpanProcessor wraps an inner SpanProcessor and injects +// hatchet.* attributes into every span created within a step run context. +// This ensures child spans are queryable by the same attributes (e.g. +// hatchet.step_run_id) as the parent span. +type HatchetAttributeSpanProcessor struct { + inner sdktrace.SpanProcessor +} + +// NewHatchetAttributeSpanProcessor creates a new HatchetAttributeSpanProcessor +// that wraps the given inner processor. +func NewHatchetAttributeSpanProcessor(inner sdktrace.SpanProcessor) *HatchetAttributeSpanProcessor { + return &HatchetAttributeSpanProcessor{inner: inner} +} + +func (p *HatchetAttributeSpanProcessor) OnStart(ctx context.Context, span sdktrace.ReadWriteSpan) { + attrs := getHatchetAttributes(ctx) + if len(attrs) > 0 { + span.SetAttributes(attrs...) + } + p.inner.OnStart(ctx, span) +} + +func (p *HatchetAttributeSpanProcessor) OnEnd(span sdktrace.ReadOnlySpan) { + p.inner.OnEnd(span) +} + +func (p *HatchetAttributeSpanProcessor) Shutdown(ctx context.Context) error { + return p.inner.Shutdown(ctx) +} + +func (p *HatchetAttributeSpanProcessor) ForceFlush(ctx context.Context) error { + return p.inner.ForceFlush(ctx) +} diff --git a/sdks/python/examples/opentelemetry_instrumentation/hatchet/trigger.py b/sdks/python/examples/opentelemetry_instrumentation/hatchet/trigger.py new file mode 100644 index 0000000000..3e4913d3bd --- /dev/null +++ b/sdks/python/examples/opentelemetry_instrumentation/hatchet/trigger.py @@ -0,0 +1,46 @@ +""" +Trigger the OTelDataPipeline workflow. + +Make sure the worker is already running: + poetry run python -m examples.opentelemetry_instrumentation.hatchet.worker + +Then run this: + poetry run python -m examples.opentelemetry_instrumentation.hatchet.trigger +""" + +from opentelemetry.sdk.resources import SERVICE_NAME, Resource +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import ConsoleSpanExporter, SimpleSpanProcessor + +from examples.opentelemetry_instrumentation.hatchet.worker import otel_workflow +from hatchet_sdk.clients.admin import TriggerWorkflowOptions +from hatchet_sdk.opentelemetry.instrumentor import HatchetInstrumentor + +# Use the same console exporter so you can see trigger-side spans too +resource = Resource(attributes={SERVICE_NAME: "hatchet-otel-pipeline-trigger"}) +provider = TracerProvider(resource=resource) +provider.add_span_processor(SimpleSpanProcessor(ConsoleSpanExporter())) + +HatchetInstrumentor( + tracer_provider=provider, + enable_hatchet_otel_collector=True, +).instrument() + +tracer = provider.get_tracer(__name__) + + +def main() -> None: + # The run_workflow call is auto-traced with a "hatchet.run_workflow" span. + # The traceparent is automatically injected into additional_metadata, + # so the worker-side spans become children of this trigger span. + with tracer.start_as_current_span("trigger_otel_data_pipeline"): + result = otel_workflow.run( + options=TriggerWorkflowOptions( + additional_metadata={"source": "otel-example", "pipeline": "data-ingest"}, + ), + ) + print(f"Workflow result: {result}") + + +if __name__ == "__main__": + main() diff --git a/sdks/python/examples/opentelemetry_instrumentation/hatchet/worker.py b/sdks/python/examples/opentelemetry_instrumentation/hatchet/worker.py new file mode 100644 index 0000000000..0b0d73b034 --- /dev/null +++ b/sdks/python/examples/opentelemetry_instrumentation/hatchet/worker.py @@ -0,0 +1,144 @@ +""" +HatchetInstrumentor example with rich traces. + +Run the worker: + poetry run python -m examples.opentelemetry_instrumentation.hatchet.worker + +Then trigger it from another terminal: + poetry run python -m examples.opentelemetry_instrumentation.hatchet.trigger +""" + +import time + +from opentelemetry.sdk.resources import SERVICE_NAME, Resource +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import ConsoleSpanExporter, SimpleSpanProcessor +from opentelemetry.trace import StatusCode + +from hatchet_sdk import Context, EmptyModel, Hatchet +from hatchet_sdk.opentelemetry.instrumentor import HatchetInstrumentor + +hatchet = Hatchet() + +otel_workflow = hatchet.workflow(name="OTelDataPipeline") + +# Module-level tracer — will be set in main() before the worker starts. +# Tasks use this to create custom child spans inside the auto-instrumented +# hatchet.start_step_run parent span. +_tracer = None + + +def _get_tracer(): + global _tracer + if _tracer is None: + from opentelemetry.trace import get_tracer + + _tracer = get_tracer(__name__) + return _tracer + + +@otel_workflow.task() +def fetch_data(input: EmptyModel, ctx: Context) -> dict[str, str]: + tracer = _get_tracer() + + with tracer.start_as_current_span( + "http.request", + attributes={"http.method": "GET", "http.url": "https://api.example.com/data"}, + ) as span: + time.sleep(0.05) + span.set_attribute("http.status_code", 200) + span.set_attribute("http.response_content_length", 4096) + + with tracer.start_as_current_span("json.parse") as span: + time.sleep(0.01) + span.set_attribute("json.record_count", 42) + + return {"records_fetched": "42"} + + +@otel_workflow.task() +def validate_data(input: EmptyModel, ctx: Context) -> dict[str, str]: + tracer = _get_tracer() + + with tracer.start_as_current_span("schema.validate") as span: + time.sleep(0.02) + span.set_attribute("validation.schema", "v2.1") + span.set_attribute("validation.records_checked", 42) + span.set_attribute("validation.errors", 2) + span.set_status(StatusCode.OK, "2 records failed validation") + + with tracer.start_as_current_span("data.clean") as span: + time.sleep(0.01) + span.set_attribute("clean.records_dropped", 2) + span.set_attribute("clean.records_remaining", 40) + + return {"valid_records": "40", "dropped": "2"} + + +@otel_workflow.task() +def process_data(input: EmptyModel, ctx: Context) -> dict[str, str]: + tracer = _get_tracer() + + with tracer.start_as_current_span("transform.pipeline") as pipeline_span: + pipeline_span.set_attribute("pipeline.stages", 3) + + with tracer.start_as_current_span("transform.normalize"): + time.sleep(0.015) + + with tracer.start_as_current_span("transform.enrich") as enrich_span: + time.sleep(0.02) + enrich_span.set_attribute("enrich.source", "geocoding-api") + + with tracer.start_as_current_span("transform.aggregate") as agg_span: + time.sleep(0.03) + agg_span.set_attribute("aggregate.groups", 8) + agg_span.set_attribute("aggregate.method", "sum") + + return {"processed_groups": "8"} + + +@otel_workflow.task() +def save_results(input: EmptyModel, ctx: Context) -> dict[str, str]: + tracer = _get_tracer() + + with tracer.start_as_current_span( + "db.query", + attributes={"db.system": "postgresql", "db.operation": "INSERT"}, + ) as span: + time.sleep(0.04) + span.set_attribute("db.rows_affected", 8) + + with tracer.start_as_current_span("cache.invalidate") as span: + time.sleep(0.005) + span.set_attribute("cache.keys_invalidated", 3) + + with tracer.start_as_current_span("notification.send") as span: + time.sleep(0.01) + span.set_attribute("notification.channel", "webhook") + span.set_attribute("notification.status", "delivered") + + return {"saved": "true"} + + +def main() -> None: + resource = Resource(attributes={SERVICE_NAME: "hatchet-otel-pipeline-example"}) + provider = TracerProvider(resource=resource) + provider.add_span_processor(SimpleSpanProcessor(ConsoleSpanExporter())) + + HatchetInstrumentor( + tracer_provider=provider, + enable_hatchet_otel_collector=True, + ).instrument() + + global _tracer + _tracer = provider.get_tracer(__name__) + + worker = hatchet.worker( + "otel-pipeline-worker", + workflows=[otel_workflow], + ) + worker.start() + + +if __name__ == "__main__": + main() diff --git a/sdks/python/hatchet_sdk/hatchet.py b/sdks/python/hatchet_sdk/hatchet.py index eba5a15286..ea20446aa4 100644 --- a/sdks/python/hatchet_sdk/hatchet.py +++ b/sdks/python/hatchet_sdk/hatchet.py @@ -58,7 +58,9 @@ class Hatchet: def __init__( self, + # todo[next major version]: add debug to the client config instead of being a top-level parameter debug: bool = False, + # todo[next major version]: remove this parameter client: Client | None = None, config: ClientConfig | None = None, ): diff --git a/sdks/python/hatchet_sdk/opentelemetry/instrumentor.py b/sdks/python/hatchet_sdk/opentelemetry/instrumentor.py index 75a9237132..302d0f583d 100644 --- a/sdks/python/hatchet_sdk/opentelemetry/instrumentor.py +++ b/sdks/python/hatchet_sdk/opentelemetry/instrumentor.py @@ -1,3 +1,4 @@ +import contextvars import json from collections.abc import Callable, Collection, Coroutine from importlib.metadata import version @@ -8,11 +9,15 @@ try: from opentelemetry.context import Context + from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter from opentelemetry.instrumentation.instrumentor import ( # type: ignore[attr-defined] BaseInstrumentor, ) from opentelemetry.instrumentation.utils import unwrap from opentelemetry.metrics import MeterProvider, NoOpMeterProvider, get_meter + from opentelemetry.sdk.trace import ReadableSpan, Span + from opentelemetry.sdk.trace import TracerProvider as SDKTracerProvider + from opentelemetry.sdk.trace.export import BatchSpanProcessor, SpanExporter from opentelemetry.trace import ( NoOpTracerProvider, SpanKind, @@ -30,6 +35,28 @@ "To use the HatchetInstrumentor, you must install Hatchet's `otel` extra using (e.g.) `pip install hatchet-sdk[otel]`" ) from e +# ContextVar that holds the hatchet.* attributes from the active +# hatchet.start_step_run span so they can be injected into child spans. +_hatchet_span_attributes: contextvars.ContextVar[dict[str, str | int] | None] = ( + contextvars.ContextVar("_hatchet_span_attributes", default=None) +) + + +class _HatchetAttributeSpanProcessor(BatchSpanProcessor): + """SpanProcessor that injects hatchet.* attributes into every span + created within a step run context, so that child spans are queryable + by the same attributes (e.g. hatchet.step_run_id) as the parent.""" + + def __init__(self, span_exporter: SpanExporter) -> None: + super().__init__(span_exporter) + + def on_start(self, span: Span, parent_context: Context | None = None) -> None: + attrs = _hatchet_span_attributes.get() + if attrs and span.is_recording(): + for key, value in attrs.items(): + span.set_attribute(key, value) + super().on_start(span, parent_context) + import inspect from datetime import datetime @@ -185,11 +212,36 @@ class HatchetInstrumentor(BaseInstrumentor): # type: ignore[misc] tracing and metrics collection. :param tracer_provider: TracerProvider | None: The OpenTelemetry TracerProvider to use. - If not provided, the global tracer provider will be used. + If not provided and `enable_hatchet_otel_collector` is True, a new SDKTracerProvider + will be created. Otherwise, the global tracer provider will be used. :param meter_provider: MeterProvider | None: The OpenTelemetry MeterProvider to use. If not provided, a no-op meter provider will be used. :param config: ClientConfig | None: The configuration for the Hatchet client. If not provided, a default configuration will be used. + :param enable_hatchet_otel_collector: bool: If True, adds an OTLP exporter to send traces to the + Hatchet engine. Uses the same connection settings (host, TLS, token) as the Hatchet + client. This can be combined with your own tracer_provider to send traces to multiple + destinations (e.g., both Hatchet and Jaeger/Datadog). Default is False. + + Example usage:: + + # Send traces only to Hatchet + instrumentor = HatchetInstrumentor( + enable_hatchet_otel_collector=True, + ) + + # Send traces to both Hatchet and your own collector + from opentelemetry.sdk.trace import TracerProvider + from opentelemetry.sdk.trace.export import BatchSpanProcessor + from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter + + provider = TracerProvider() + provider.add_span_processor(BatchSpanProcessor(OTLPSpanExporter(endpoint="your-collector:4317"))) + + instrumentor = HatchetInstrumentor( + tracer_provider=provider, + enable_hatchet_otel_collector=True, # Also sends to Hatchet + ) """ def __init__( @@ -197,14 +249,47 @@ def __init__( tracer_provider: TracerProvider | None = None, meter_provider: MeterProvider | None = None, config: ClientConfig | None = None, + enable_hatchet_otel_collector: bool = False, ): self.config = config or ClientConfig() - self.tracer_provider = tracer_provider or get_tracer_provider() + if tracer_provider is not None: + self.tracer_provider = tracer_provider + elif enable_hatchet_otel_collector: + self.tracer_provider = SDKTracerProvider() + else: + self.tracer_provider = get_tracer_provider() + + if enable_hatchet_otel_collector: + self._add_hatchet_exporter() + self.meter_provider = meter_provider or NoOpMeterProvider() super().__init__() + def _add_hatchet_exporter(self) -> None: + if not isinstance(self.tracer_provider, SDKTracerProvider): + logger.warning( + "enable_hatchet_otel_collector requires an opentelemetry.sdk.trace.TracerProvider. " + "The provided tracer_provider does not support adding span processors. " + "Traces will not be forwarded to Hatchet." + ) + return + + endpoint = self.config.host_port + insecure = self.config.tls_config.strategy == "none" + headers = (("authorization", f"Bearer {self.config.token}"),) + + span_exporter = OTLPSpanExporter( + endpoint=endpoint, + headers=headers, + insecure=insecure, + ) + + self.tracer_provider.add_span_processor( + _HatchetAttributeSpanProcessor(span_exporter) + ) + def instrumentation_dependencies(self) -> Collection[str]: return () @@ -310,18 +395,24 @@ async def _wrap_handle_start_step_run( if self.config.otel.include_task_name_in_start_step_run_span_name: span_name += f".{action.action_id}" - with self._tracer.start_as_current_span( - span_name, - attributes=action.get_otel_attributes(self.config), - context=traceparent, - kind=SpanKind.CONSUMER, - ) as span: - result = await wrapped(*args, **kwargs) - - if isinstance(result, Exception): - span.set_status(StatusCode.ERROR, str(result)) + hatchet_attrs = action.get_otel_attributes(self.config) + token = _hatchet_span_attributes.set(hatchet_attrs) - return result + try: + with self._tracer.start_as_current_span( + span_name, + attributes=hatchet_attrs, + context=traceparent, + kind=SpanKind.CONSUMER, + ) as span: + result = await wrapped(*args, **kwargs) + + if isinstance(result, Exception): + span.set_status(StatusCode.ERROR, str(result)) + + return result + finally: + _hatchet_span_attributes.reset(token) ## IMPORTANT: Keep these types in sync with the wrapped method's signature async def _wrap_handle_cancel_action( diff --git a/sdks/python/hatchet_sdk/worker/runner/runner.py b/sdks/python/hatchet_sdk/worker/runner/runner.py index 7813a48b8d..3bcf096dd8 100644 --- a/sdks/python/hatchet_sdk/worker/runner/runner.py +++ b/sdks/python/hatchet_sdk/worker/runner/runner.py @@ -1,4 +1,5 @@ import asyncio +import contextvars import ctypes import functools from collections.abc import Callable @@ -312,8 +313,17 @@ async def async_wrapped_action_func( dependencies, ) + # Copy the full contextvars snapshot (including OpenTelemetry span + # context) so that child spans created in the thread inherit the + # correct trace_id and parent_span_id from the active + # hatchet.start_step_run span. + ctx_snapshot = contextvars.copy_context() loop = asyncio.get_event_loop() - return await loop.run_in_executor(self.thread_pool, pfunc) + return await loop.run_in_executor( + self.thread_pool, + ctx_snapshot.run, + pfunc, + ) finally: self.cleanup_run_id(action.key) diff --git a/sdks/python/tests/otel_traces/__init__.py b/sdks/python/tests/otel_traces/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/sdks/python/tests/otel_traces/test_otel_traces.py b/sdks/python/tests/otel_traces/test_otel_traces.py new file mode 100644 index 0000000000..666b685795 --- /dev/null +++ b/sdks/python/tests/otel_traces/test_otel_traces.py @@ -0,0 +1,289 @@ +import asyncio +import os +import shutil +import subprocess +import time +from subprocess import Popen +from typing import Any +from uuid import uuid4 + +import pytest +import requests + +from hatchet_sdk import Hatchet, TriggerWorkflowOptions +from hatchet_sdk.clients.rest.models.v1_task_status import V1TaskStatus +from tests.otel_traces.worker import otel_long_task, otel_simple_task + +SPANS_PORT = 8020 +WORKER_HEALTHCHECK_PORT = 8010 + + +def _get_spans() -> list[dict[str, Any]]: + """Fetch captured spans from the worker's span HTTP endpoint.""" + resp = requests.get(f"http://localhost:{SPANS_PORT}/spans", timeout=5) + resp.raise_for_status() + return resp.json() + + +def _clear_spans() -> None: + """Clear captured spans on the worker.""" + resp = requests.delete(f"http://localhost:{SPANS_PORT}/spans", timeout=5) + resp.raise_for_status() + + +@pytest.mark.parametrize( + "on_demand_worker", + [ + ( + [ + "poetry", + "run", + "python", + "tests/otel_traces/worker.py", + "--spans-port", + str(SPANS_PORT), + ], + WORKER_HEALTHCHECK_PORT, + ) + ], + indirect=True, +) +@pytest.mark.asyncio(loop_scope="session") +async def test_otel_spans_created_on_task_run( + hatchet: Hatchet, on_demand_worker: Popen[Any] +) -> None: + """Verify that running a task produces correct OTel spans with hatchet.* attributes.""" + _clear_spans() + + test_run_id = str(uuid4()) + + await otel_simple_task.aio_run( + options=TriggerWorkflowOptions( + additional_metadata={"test_run_id": test_run_id}, + ), + ) + + # Give the span processor a moment to flush + await asyncio.sleep(1) + + spans = _get_spans() + + # Find the hatchet.start_step_run span + step_run_spans = [s for s in spans if s["name"] == "hatchet.start_step_run"] + assert len(step_run_spans) >= 1, ( + f"Expected at least one hatchet.start_step_run span, got {len(step_run_spans)}. " + f"All spans: {[s['name'] for s in spans]}" + ) + + step_span = step_run_spans[-1] # Use the most recent one + + # Verify hatchet attributes exist + attrs = step_span["attributes"] + assert "hatchet.step_run_id" in attrs, f"Missing hatchet.step_run_id in {attrs}" + assert "hatchet.workflow_run_id" in attrs, ( + f"Missing hatchet.workflow_run_id in {attrs}" + ) + assert "hatchet.tenant_id" in attrs, f"Missing hatchet.tenant_id in {attrs}" + + # Verify span kind is CONSUMER (value=4 in OTel Python SDK) + assert step_span["kind"] == 4, f"Expected CONSUMER (4), got {step_span['kind']}" + + # Find the custom child span + child_spans = [s for s in spans if s["name"] == "custom.child.span"] + assert len(child_spans) >= 1, ( + f"Expected at least one custom.child.span, got {len(child_spans)}. " + f"All spans: {[s['name'] for s in spans]}" + ) + + child_span = child_spans[-1] + + # Child span should share the same trace_id as the step run span + assert child_span["trace_id"] == step_span["trace_id"], ( + f"Child trace_id {child_span['trace_id']} != step run trace_id {step_span['trace_id']}" + ) + + # Child span should have hatchet.* attributes injected by _HatchetAttributeSpanProcessor + child_attrs = child_span["attributes"] + assert "hatchet.step_run_id" in child_attrs, ( + f"Child span missing hatchet.step_run_id (attribute propagation failed). Attrs: {child_attrs}" + ) + assert child_attrs["hatchet.step_run_id"] == attrs["hatchet.step_run_id"], ( + "Child span hatchet.step_run_id doesn't match parent" + ) + + # Verify the custom attribute is present + assert child_attrs.get("test.marker") == "hello", ( + f"Missing test.marker attribute on child span. Attrs: {child_attrs}" + ) + + +def _find_engine_pids() -> list[int]: + """Find PIDs of running hatchet-engine processes.""" + try: + result = subprocess.run( + ["pgrep", "-f", "hatchet-engine"], + capture_output=True, + text=True, + ) + if result.returncode == 0: + return [int(pid) for pid in result.stdout.strip().split("\n") if pid] + except Exception: + pass + return [] + + +def _kill_engine() -> bool: + """Kill hatchet-engine processes. Returns True if any were found and killed.""" + pids = _find_engine_pids() + if not pids: + return False + subprocess.run(["pkill", "-f", "hatchet-engine"], capture_output=True) + return True + + +def _restart_engine() -> bool: + """Restart the hatchet-engine process. Returns True if successful. + + Set HATCHET_ENGINE_RESTART_CMD to a shell command that starts the engine, + e.g. 'cd /path/to/repo && go run ./cmd/hatchet-engine --config ./generated/' + or 'docker compose -f /path/to/docker-compose.yml up -d hatchet-engine'. + + Falls back to 'go run ./cmd/hatchet-engine --config ./generated/' from the repo root. + """ + custom_cmd = os.environ.get("HATCHET_ENGINE_RESTART_CMD") + if custom_cmd: + subprocess.Popen(custom_cmd, shell=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) # noqa: S602 + return True + + # Auto-detect repo root by walking up from this file + # tests/otel_traces/test_otel_traces.py -> tests/ -> sdks/python/ -> sdks/ -> repo root + sdk_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + repo_root = os.path.dirname(os.path.dirname(sdk_dir)) + + config_dir = os.path.join(repo_root, "generated") + if not os.path.isdir(config_dir): + return False + + go_bin = shutil.which("go") + if not go_bin: + return False + + subprocess.Popen( + [go_bin, "run", "./cmd/hatchet-engine", "--config", "./generated/"], + cwd=repo_root, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + return True + + +@pytest.mark.parametrize( + "on_demand_worker", + [ + ( + [ + "poetry", + "run", + "python", + "tests/otel_traces/worker.py", + "--spans-port", + str(SPANS_PORT), + ], + WORKER_HEALTHCHECK_PORT, + ) + ], + indirect=True, +) +@pytest.mark.asyncio(loop_scope="session") +async def test_otel_traces_after_engine_restart( + hatchet: Hatchet, on_demand_worker: Popen[Any] +) -> None: + """Verify traces work after the engine is killed and restarted mid-run. + + The worker should reconnect, the task should be retried, and the retried + run should produce valid OTel spans. + """ + # Skip if no engine process is found (not running in the expected environment) + engine_pids = _find_engine_pids() + if not engine_pids: + pytest.skip("No hatchet-engine process found — skipping engine restart test") + + _clear_spans() + + test_run_id = str(uuid4()) + + # Trigger a long-running task without waiting + run_ref = await otel_long_task.aio_run_no_wait( + options=TriggerWorkflowOptions( + additional_metadata={"test_run_id": test_run_id}, + ), + ) + + # Wait for the task to start (poll for hatchet.start_step_run span) + for _ in range(10): + await asyncio.sleep(1) + spans = _get_spans() + step_run_spans = [s for s in spans if s["name"] == "hatchet.start_step_run"] + if step_run_spans: + break + else: + pytest.fail("Task did not start within 10 seconds (no hatchet.start_step_run span)") + + # Kill the engine + killed = _kill_engine() + assert killed, "Failed to kill hatchet-engine process" + + # Wait for disconnect to be detected + await asyncio.sleep(3) + + # Restart the engine + restarted = _restart_engine() + if not restarted: + pytest.skip( + "Could not restart hatchet-engine (no generated/ config dir found). " + "Set HATCHET_ENGINE_RESTART_CMD env var to a shell command that starts the engine." + ) + + # Wait for engine to come back up and worker to reconnect + await asyncio.sleep(15) + + # Poll for the workflow run to reach a terminal state + terminal_statuses = {V1TaskStatus.COMPLETED, V1TaskStatus.FAILED, V1TaskStatus.CANCELLED} + final_status = None + + for _ in range(30): + runs = await otel_long_task.aio_list_runs( + additional_metadata={"test_run_id": test_run_id}, + ) + if runs: + run = runs[0] + if run.status in terminal_statuses: + final_status = run.status + break + await asyncio.sleep(2) + + # Verify the worker subprocess is still alive (didn't crash) + assert on_demand_worker.poll() is None, "Worker process crashed during engine restart" + + # Query final spans + spans = _get_spans() + step_run_spans = [s for s in spans if s["name"] == "hatchet.start_step_run"] + + # At least one step run span should exist (could be 2+ if retried) + assert len(step_run_spans) >= 1, ( + f"Expected at least one hatchet.start_step_run span after engine restart. " + f"All spans: {[s['name'] for s in spans]}" + ) + + # All step run spans should have valid hatchet.* attributes + for span in step_run_spans: + attrs = span["attributes"] + assert "hatchet.step_run_id" in attrs, ( + f"Step run span missing hatchet.step_run_id after restart. Attrs: {attrs}" + ) + assert "hatchet.workflow_run_id" in attrs, ( + f"Step run span missing hatchet.workflow_run_id after restart. Attrs: {attrs}" + ) + assert "hatchet.tenant_id" in attrs, ( + f"Step run span missing hatchet.tenant_id after restart. Attrs: {attrs}" + ) diff --git a/sdks/python/tests/otel_traces/worker.py b/sdks/python/tests/otel_traces/worker.py new file mode 100644 index 0000000000..03c4bbd132 --- /dev/null +++ b/sdks/python/tests/otel_traces/worker.py @@ -0,0 +1,139 @@ +""" +OTel test worker — runs with HatchetInstrumentor and exposes captured spans via HTTP. + +Usage: + poetry run python tests/otel_traces/worker.py [--spans-port 8020] +""" + +import argparse +import asyncio +import json +import time +from datetime import timedelta +from http.server import BaseHTTPRequestHandler, HTTPServer +from threading import Thread +from typing import Any, cast + +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from opentelemetry.trace import get_tracer, set_tracer_provider + +from hatchet_sdk import Context, EmptyModel, Hatchet +from hatchet_sdk.opentelemetry.instrumentor import HatchetInstrumentor + + +# -- OTel setup ---------------------------------------------------------------- + +span_exporter = InMemorySpanExporter() +provider = TracerProvider() +provider.add_span_processor(SimpleSpanProcessor(span_exporter)) +set_tracer_provider(provider) + +HatchetInstrumentor( + tracer_provider=provider, + enable_hatchet_otel_collector=True, +).instrument() + +# -- Hatchet workflows --------------------------------------------------------- + +hatchet = Hatchet(debug=True) + + +@hatchet.task() +def otel_simple_task(input: EmptyModel, ctx: Context) -> dict[str, str]: + """Simple task that creates a custom child span.""" + tracer = get_tracer("otel-test") + with tracer.start_as_current_span("custom.child.span") as span: + span.set_attribute("test.marker", "hello") + time.sleep(0.01) + return {"status": "ok"} + + +@hatchet.task(execution_timeout=timedelta(seconds=30), retries=2) +async def otel_long_task(input: EmptyModel, ctx: Context) -> dict[str, str]: + """Longer task for engine disconnect testing.""" + for _ in range(20): + await asyncio.sleep(0.5) + return {"status": "completed"} + + +# -- Span HTTP server ---------------------------------------------------------- + + +def _serialize_spans() -> list[dict[str, Any]]: + spans = span_exporter.get_finished_spans() + result = [] + for s in spans: + attrs = {} + for k, v in s.attributes.items(): + attrs[k] = str(v) if not isinstance(v, (str, int, float, bool)) else v + + result.append( + { + "name": s.name, + "trace_id": format(s.context.trace_id, "032x"), + "span_id": format(s.context.span_id, "016x"), + "parent_span_id": ( + format(s.parent.span_id, "016x") if s.parent else None + ), + "attributes": attrs, + "kind": s.kind.value if s.kind else None, + "status_code": s.status.status_code.name if s.status else None, + } + ) + return result + + +class SpanHandler(BaseHTTPRequestHandler): + def do_GET(self) -> None: + if self.path == "/spans": + body = json.dumps(_serialize_spans()).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.end_headers() + self.wfile.write(body) + else: + self.send_response(404) + self.end_headers() + + def do_DELETE(self) -> None: + if self.path == "/spans": + span_exporter.clear() + self.send_response(200) + self.end_headers() + else: + self.send_response(404) + self.end_headers() + + def log_message(self, format: str, *args: Any) -> None: + pass # suppress request logs + + +def _start_span_server(port: int) -> None: + server = HTTPServer(("0.0.0.0", port), SpanHandler) + server.serve_forever() + + +# -- Main ---------------------------------------------------------------------- + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--spans-port", type=int, default=8020) + args = parser.parse_args() + + spans_port = cast(int, args.spans_port) + + # Start span server in background thread + Thread(target=_start_span_server, args=(spans_port,), daemon=True).start() + + worker = hatchet.worker( + "otel-e2e-test-worker", + workflows=[otel_simple_task, otel_long_task], + ) + worker.start() + + +if __name__ == "__main__": + main() From 05f3b0f8e4ad5113112a0a599eee48078a05e92e Mon Sep 17 00:00:00 2001 From: Mohammed Nafees Date: Mon, 9 Mar 2026 13:57:24 +0100 Subject: [PATCH 02/17] fix CI --- .../components/v1/agent-prism/BrandLogo.tsx | 4 +- .../DetailsView/DetailsViewHeaderActions.tsx | 4 +- .../DetailsView/DetailsViewInputOutputTab.tsx | 6 +- .../DetailsView/DetailsViewJsonOutput.tsx | 3 +- .../v1/agent-prism/SpanCard/SpanCard.tsx | 12 +- .../SpanCard/SpanCardConnector.tsx | 4 +- .../agent-prism/TraceViewer/TraceViewer.tsx | 4 +- .../lib/api/generated/cloud/data-contracts.ts | 4 - .../step-run-detail/otel-span-adapter.ts | 4 +- .../step-run-detail/task-run-trace.tsx | 4 +- .../tests/otel_traces/test_otel_traces.py | 212 ++++-------------- sdks/python/tests/otel_traces/worker.py | 30 +-- 12 files changed, 94 insertions(+), 197 deletions(-) diff --git a/frontend/app/src/components/v1/agent-prism/BrandLogo.tsx b/frontend/app/src/components/v1/agent-prism/BrandLogo.tsx index 70ff567590..413892b76e 100644 --- a/frontend/app/src/components/v1/agent-prism/BrandLogo.tsx +++ b/frontend/app/src/components/v1/agent-prism/BrandLogo.tsx @@ -96,7 +96,9 @@ export const BrandLogo: FC = ({ }) => { const Logo = LOGO_REGISTRY[brand as BrandType]; - if (!Logo) return <>{fallback}; + if (!Logo) { + return <>{fallback}; + } return ; }; diff --git a/frontend/app/src/components/v1/agent-prism/DetailsView/DetailsViewHeaderActions.tsx b/frontend/app/src/components/v1/agent-prism/DetailsView/DetailsViewHeaderActions.tsx index ffe8387f75..f4308ac9b8 100644 --- a/frontend/app/src/components/v1/agent-prism/DetailsView/DetailsViewHeaderActions.tsx +++ b/frontend/app/src/components/v1/agent-prism/DetailsView/DetailsViewHeaderActions.tsx @@ -15,7 +15,9 @@ export const DetailsViewHeaderActions = ({ children, className = 'flex flex-wrap items-center gap-2', }: DetailsViewHeaderActionsProps) => { - if (!children) return null; + if (!children) { + return null; + } return
    {children}
    ; }; diff --git a/frontend/app/src/components/v1/agent-prism/DetailsView/DetailsViewInputOutputTab.tsx b/frontend/app/src/components/v1/agent-prism/DetailsView/DetailsViewInputOutputTab.tsx index 3e772b07d3..cc2e1fccb4 100644 --- a/frontend/app/src/components/v1/agent-prism/DetailsView/DetailsViewInputOutputTab.tsx +++ b/frontend/app/src/components/v1/agent-prism/DetailsView/DetailsViewInputOutputTab.tsx @@ -53,14 +53,14 @@ export const DetailsViewInputOutputTab = ({ return (
    {typeof data.input === 'string' && ( - )} {typeof data.output === 'string' && ( - = ({ }) => { return ( { - if (level === 0) return 0; + if (level === 0) { + return 0; + } - if (hasExpandButton) return 4; + if (hasExpandButton) { + return 4; + } return 8; }; @@ -223,7 +227,9 @@ const SpanCardChildren: FC<{ onExpandSpansIdsChange, viewOptions = DEFAULT_VIEW_OPTIONS, }) => { - if (!data.children?.length) return null; + if (!data.children?.length) { + return null; + } return (
    diff --git a/frontend/app/src/components/v1/agent-prism/SpanCard/SpanCardConnector.tsx b/frontend/app/src/components/v1/agent-prism/SpanCard/SpanCardConnector.tsx index 211eb29983..ade076f212 100644 --- a/frontend/app/src/components/v1/agent-prism/SpanCard/SpanCardConnector.tsx +++ b/frontend/app/src/components/v1/agent-prism/SpanCard/SpanCardConnector.tsx @@ -10,7 +10,9 @@ interface SpanCardConnectorProps { } export const SpanCardConnector = ({ type }: SpanCardConnectorProps) => { - if (type === 'empty') return
    ; + if (type === 'empty') { + return
    ; + } return (
    diff --git a/frontend/app/src/components/v1/agent-prism/TraceViewer/TraceViewer.tsx b/frontend/app/src/components/v1/agent-prism/TraceViewer/TraceViewer.tsx index 3a912f2559..8072a39aa7 100644 --- a/frontend/app/src/components/v1/agent-prism/TraceViewer/TraceViewer.tsx +++ b/frontend/app/src/components/v1/agent-prism/TraceViewer/TraceViewer.tsx @@ -74,7 +74,9 @@ export const TraceViewer = ({ }, [allIds]); useEffect(() => { - if (!isMounted || isMobile) return; + if (!isMounted || isMobile) { + return; + } if (selectedTraceSpans.length > 0 && !selectedSpan) { setSelectedSpan(selectedTraceSpans[0]); diff --git a/frontend/app/src/lib/api/generated/cloud/data-contracts.ts b/frontend/app/src/lib/api/generated/cloud/data-contracts.ts index 70dfa35d89..980e967a3e 100644 --- a/frontend/app/src/lib/api/generated/cloud/data-contracts.ts +++ b/frontend/app/src/lib/api/generated/cloud/data-contracts.ts @@ -39,7 +39,6 @@ export enum TemplateOptions { } export enum AutoscalingTargetKind { - PORTER = "PORTER", FLY = "FLY", } @@ -146,7 +145,6 @@ export interface ManagedWorker { metadata: APIResourceMeta; name: string; buildConfig?: ManagedWorkerBuildConfig; - isIac: boolean; directSecrets: ManagedWorkerSecret[]; globalSecrets: ManagedWorkerSecret[]; runtimeConfigs?: ManagedWorkerRuntimeConfig[]; @@ -256,7 +254,6 @@ export interface CreateManagedWorkerRequest { name: string; buildConfig: CreateManagedWorkerBuildConfigRequest; secrets?: CreateManagedWorkerSecretRequest; - isIac: boolean; runtimeConfig?: CreateManagedWorkerRuntimeConfigRequest; } @@ -264,7 +261,6 @@ export interface UpdateManagedWorkerRequest { name?: string; buildConfig?: CreateManagedWorkerBuildConfigRequest; secrets?: UpdateManagedWorkerSecretRequest; - isIac?: boolean; runtimeConfig?: CreateManagedWorkerRuntimeConfigRequest; } diff --git a/frontend/app/src/pages/main/v1/workflow-runs-v1/$run/v2components/step-run-detail/otel-span-adapter.ts b/frontend/app/src/pages/main/v1/workflow-runs-v1/$run/v2components/step-run-detail/otel-span-adapter.ts index d29c526325..7d4ac3f646 100644 --- a/frontend/app/src/pages/main/v1/workflow-runs-v1/$run/v2components/step-run-detail/otel-span-adapter.ts +++ b/frontend/app/src/pages/main/v1/workflow-runs-v1/$run/v2components/step-run-detail/otel-span-adapter.ts @@ -23,7 +23,9 @@ const STATUS_CODE_MAP: Record = { function recordToAttributes( record: Record | undefined, ): TraceSpanAttribute[] { - if (!record) return []; + if (!record) { + return []; + } return Object.entries(record).map(([key, value]) => ({ key, value: { stringValue: value }, diff --git a/frontend/app/src/pages/main/v1/workflow-runs-v1/$run/v2components/step-run-detail/task-run-trace.tsx b/frontend/app/src/pages/main/v1/workflow-runs-v1/$run/v2components/step-run-detail/task-run-trace.tsx index 09136f138d..da5895c70a 100644 --- a/frontend/app/src/pages/main/v1/workflow-runs-v1/$run/v2components/step-run-detail/task-run-trace.tsx +++ b/frontend/app/src/pages/main/v1/workflow-runs-v1/$run/v2components/step-run-detail/task-run-trace.tsx @@ -18,7 +18,9 @@ export function TaskRunTrace({ taskExternalId }: { taskExternalId: string }) { const traceSpans = useMemo(() => { const rows = tracesQuery.data?.rows; - if (!rows || rows.length === 0) return []; + if (!rows || rows.length === 0) { + return []; + } const otlpSpans = convertOtelSpans(rows); return openTelemetrySpanAdapter.convertRawSpansToSpanTree(otlpSpans); diff --git a/sdks/python/tests/otel_traces/test_otel_traces.py b/sdks/python/tests/otel_traces/test_otel_traces.py index 666b685795..3e24cd2661 100644 --- a/sdks/python/tests/otel_traces/test_otel_traces.py +++ b/sdks/python/tests/otel_traces/test_otel_traces.py @@ -1,8 +1,4 @@ import asyncio -import os -import shutil -import subprocess -import time from subprocess import Popen from typing import Any from uuid import uuid4 @@ -12,17 +8,32 @@ from hatchet_sdk import Hatchet, TriggerWorkflowOptions from hatchet_sdk.clients.rest.models.v1_task_status import V1TaskStatus -from tests.otel_traces.worker import otel_long_task, otel_simple_task +from tests.otel_traces.worker import otel_retry_task, otel_simple_task SPANS_PORT = 8020 WORKER_HEALTHCHECK_PORT = 8010 +ON_DEMAND_WORKER_PARAMS = [ + ( + [ + "poetry", + "run", + "python", + "tests/otel_traces/worker.py", + "--spans-port", + str(SPANS_PORT), + ], + WORKER_HEALTHCHECK_PORT, + ) +] + def _get_spans() -> list[dict[str, Any]]: """Fetch captured spans from the worker's span HTTP endpoint.""" resp = requests.get(f"http://localhost:{SPANS_PORT}/spans", timeout=5) resp.raise_for_status() - return resp.json() + result: list[dict[str, Any]] = resp.json() + return result def _clear_spans() -> None: @@ -31,23 +42,7 @@ def _clear_spans() -> None: resp.raise_for_status() -@pytest.mark.parametrize( - "on_demand_worker", - [ - ( - [ - "poetry", - "run", - "python", - "tests/otel_traces/worker.py", - "--spans-port", - str(SPANS_PORT), - ], - WORKER_HEALTHCHECK_PORT, - ) - ], - indirect=True, -) +@pytest.mark.parametrize("on_demand_worker", ON_DEMAND_WORKER_PARAMS, indirect=True) @pytest.mark.asyncio(loop_scope="session") async def test_otel_spans_created_on_task_run( hatchet: Hatchet, on_demand_worker: Popen[Any] @@ -117,173 +112,62 @@ async def test_otel_spans_created_on_task_run( ) -def _find_engine_pids() -> list[int]: - """Find PIDs of running hatchet-engine processes.""" - try: - result = subprocess.run( - ["pgrep", "-f", "hatchet-engine"], - capture_output=True, - text=True, - ) - if result.returncode == 0: - return [int(pid) for pid in result.stdout.strip().split("\n") if pid] - except Exception: - pass - return [] - - -def _kill_engine() -> bool: - """Kill hatchet-engine processes. Returns True if any were found and killed.""" - pids = _find_engine_pids() - if not pids: - return False - subprocess.run(["pkill", "-f", "hatchet-engine"], capture_output=True) - return True - - -def _restart_engine() -> bool: - """Restart the hatchet-engine process. Returns True if successful. - - Set HATCHET_ENGINE_RESTART_CMD to a shell command that starts the engine, - e.g. 'cd /path/to/repo && go run ./cmd/hatchet-engine --config ./generated/' - or 'docker compose -f /path/to/docker-compose.yml up -d hatchet-engine'. - - Falls back to 'go run ./cmd/hatchet-engine --config ./generated/' from the repo root. - """ - custom_cmd = os.environ.get("HATCHET_ENGINE_RESTART_CMD") - if custom_cmd: - subprocess.Popen(custom_cmd, shell=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) # noqa: S602 - return True - - # Auto-detect repo root by walking up from this file - # tests/otel_traces/test_otel_traces.py -> tests/ -> sdks/python/ -> sdks/ -> repo root - sdk_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - repo_root = os.path.dirname(os.path.dirname(sdk_dir)) - - config_dir = os.path.join(repo_root, "generated") - if not os.path.isdir(config_dir): - return False - - go_bin = shutil.which("go") - if not go_bin: - return False - - subprocess.Popen( - [go_bin, "run", "./cmd/hatchet-engine", "--config", "./generated/"], - cwd=repo_root, - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - ) - return True - - -@pytest.mark.parametrize( - "on_demand_worker", - [ - ( - [ - "poetry", - "run", - "python", - "tests/otel_traces/worker.py", - "--spans-port", - str(SPANS_PORT), - ], - WORKER_HEALTHCHECK_PORT, - ) - ], - indirect=True, -) +@pytest.mark.parametrize("on_demand_worker", ON_DEMAND_WORKER_PARAMS, indirect=True) @pytest.mark.asyncio(loop_scope="session") -async def test_otel_traces_after_engine_restart( +async def test_otel_traces_on_retry( hatchet: Hatchet, on_demand_worker: Popen[Any] ) -> None: - """Verify traces work after the engine is killed and restarted mid-run. + """Verify that traces are produced for both the failed attempt and the retry. - The worker should reconnect, the task should be retried, and the retried - run should produce valid OTel spans. + Uses a task that fails on the first attempt (raises an exception) and + succeeds on the second attempt (retries=1). Both attempts should produce + valid hatchet.start_step_run spans with correct attributes, and the retry + count attribute should differ between them. """ - # Skip if no engine process is found (not running in the expected environment) - engine_pids = _find_engine_pids() - if not engine_pids: - pytest.skip("No hatchet-engine process found — skipping engine restart test") - _clear_spans() test_run_id = str(uuid4()) - # Trigger a long-running task without waiting - run_ref = await otel_long_task.aio_run_no_wait( + await otel_retry_task.aio_run( options=TriggerWorkflowOptions( additional_metadata={"test_run_id": test_run_id}, ), ) - # Wait for the task to start (poll for hatchet.start_step_run span) - for _ in range(10): - await asyncio.sleep(1) - spans = _get_spans() - step_run_spans = [s for s in spans if s["name"] == "hatchet.start_step_run"] - if step_run_spans: - break - else: - pytest.fail("Task did not start within 10 seconds (no hatchet.start_step_run span)") - - # Kill the engine - killed = _kill_engine() - assert killed, "Failed to kill hatchet-engine process" - - # Wait for disconnect to be detected - await asyncio.sleep(3) - - # Restart the engine - restarted = _restart_engine() - if not restarted: - pytest.skip( - "Could not restart hatchet-engine (no generated/ config dir found). " - "Set HATCHET_ENGINE_RESTART_CMD env var to a shell command that starts the engine." - ) - - # Wait for engine to come back up and worker to reconnect - await asyncio.sleep(15) - - # Poll for the workflow run to reach a terminal state - terminal_statuses = {V1TaskStatus.COMPLETED, V1TaskStatus.FAILED, V1TaskStatus.CANCELLED} - final_status = None - - for _ in range(30): - runs = await otel_long_task.aio_list_runs( - additional_metadata={"test_run_id": test_run_id}, - ) - if runs: - run = runs[0] - if run.status in terminal_statuses: - final_status = run.status - break - await asyncio.sleep(2) - - # Verify the worker subprocess is still alive (didn't crash) - assert on_demand_worker.poll() is None, "Worker process crashed during engine restart" + # Give the span processor a moment to flush + await asyncio.sleep(1) - # Query final spans spans = _get_spans() + + # Both the failed first attempt and the successful retry should have spans step_run_spans = [s for s in spans if s["name"] == "hatchet.start_step_run"] + assert len(step_run_spans) >= 2, ( + f"Expected at least 2 hatchet.start_step_run spans (initial + retry), " + f"got {len(step_run_spans)}. All spans: {[s['name'] for s in spans]}" + ) - # At least one step run span should exist (could be 2+ if retried) - assert len(step_run_spans) >= 1, ( - f"Expected at least one hatchet.start_step_run span after engine restart. " - f"All spans: {[s['name'] for s in spans]}" + # The first attempt should have errored + error_spans = [s for s in step_run_spans if s["status_code"] == "ERROR"] + assert len(error_spans) >= 1, ( + f"Expected at least one ERROR span from the failed first attempt. " + f"Statuses: {[s['status_code'] for s in step_run_spans]}" ) # All step run spans should have valid hatchet.* attributes for span in step_run_spans: attrs = span["attributes"] assert "hatchet.step_run_id" in attrs, ( - f"Step run span missing hatchet.step_run_id after restart. Attrs: {attrs}" + f"Step run span missing hatchet.step_run_id. Attrs: {attrs}" ) assert "hatchet.workflow_run_id" in attrs, ( - f"Step run span missing hatchet.workflow_run_id after restart. Attrs: {attrs}" + f"Step run span missing hatchet.workflow_run_id. Attrs: {attrs}" ) assert "hatchet.tenant_id" in attrs, ( - f"Step run span missing hatchet.tenant_id after restart. Attrs: {attrs}" + f"Step run span missing hatchet.tenant_id. Attrs: {attrs}" ) + + # Verify retry count differs between attempts + retry_counts = [s["attributes"].get("hatchet.retry_count") for s in step_run_spans] + assert len(set(retry_counts)) >= 2, ( + f"Expected different retry_count values across attempts, got {retry_counts}" + ) diff --git a/sdks/python/tests/otel_traces/worker.py b/sdks/python/tests/otel_traces/worker.py index 03c4bbd132..a1dbea5e82 100644 --- a/sdks/python/tests/otel_traces/worker.py +++ b/sdks/python/tests/otel_traces/worker.py @@ -6,10 +6,8 @@ """ import argparse -import asyncio import json import time -from datetime import timedelta from http.server import BaseHTTPRequestHandler, HTTPServer from threading import Thread from typing import Any, cast @@ -50,12 +48,13 @@ def otel_simple_task(input: EmptyModel, ctx: Context) -> dict[str, str]: return {"status": "ok"} -@hatchet.task(execution_timeout=timedelta(seconds=30), retries=2) -async def otel_long_task(input: EmptyModel, ctx: Context) -> dict[str, str]: - """Longer task for engine disconnect testing.""" - for _ in range(20): - await asyncio.sleep(0.5) - return {"status": "completed"} +@hatchet.task(retries=1) +def otel_retry_task(input: EmptyModel, ctx: Context) -> dict[str, str]: + """Task that fails on first attempt and succeeds on retry.""" + retry_count = ctx.retry_count + if retry_count == 0: + raise RuntimeError("intentional failure on first attempt") + return {"status": "ok", "retry_count": str(retry_count)} # -- Span HTTP server ---------------------------------------------------------- @@ -63,11 +62,12 @@ async def otel_long_task(input: EmptyModel, ctx: Context) -> dict[str, str]: def _serialize_spans() -> list[dict[str, Any]]: spans = span_exporter.get_finished_spans() - result = [] + result: list[dict[str, Any]] = [] for s in spans: - attrs = {} - for k, v in s.attributes.items(): - attrs[k] = str(v) if not isinstance(v, (str, int, float, bool)) else v + attrs: dict[str, Any] = {} + if s.attributes is not None: + for k, v in s.attributes.items(): + attrs[k] = str(v) if not isinstance(v, (str, int, float, bool)) else v result.append( { @@ -78,8 +78,8 @@ def _serialize_spans() -> list[dict[str, Any]]: format(s.parent.span_id, "016x") if s.parent else None ), "attributes": attrs, - "kind": s.kind.value if s.kind else None, - "status_code": s.status.status_code.name if s.status else None, + "kind": s.kind.value, + "status_code": s.status.status_code.name, } ) return result @@ -130,7 +130,7 @@ def main() -> None: worker = hatchet.worker( "otel-e2e-test-worker", - workflows=[otel_simple_task, otel_long_task], + workflows=[otel_simple_task, otel_retry_task], ) worker.start() From 139a8d10c7d0ab06bbaf97818831a90a7c1c8080 Mon Sep 17 00:00:00 2001 From: Mohammed Nafees Date: Mon, 9 Mar 2026 14:00:06 +0100 Subject: [PATCH 03/17] fix black lint --- .../tests/otel_traces/test_otel_traces.py | 54 +++++++++---------- sdks/python/tests/otel_traces/worker.py | 1 - 2 files changed, 27 insertions(+), 28 deletions(-) diff --git a/sdks/python/tests/otel_traces/test_otel_traces.py b/sdks/python/tests/otel_traces/test_otel_traces.py index 3e24cd2661..1423fd3dbe 100644 --- a/sdks/python/tests/otel_traces/test_otel_traces.py +++ b/sdks/python/tests/otel_traces/test_otel_traces.py @@ -75,9 +75,9 @@ async def test_otel_spans_created_on_task_run( # Verify hatchet attributes exist attrs = step_span["attributes"] assert "hatchet.step_run_id" in attrs, f"Missing hatchet.step_run_id in {attrs}" - assert "hatchet.workflow_run_id" in attrs, ( - f"Missing hatchet.workflow_run_id in {attrs}" - ) + assert ( + "hatchet.workflow_run_id" in attrs + ), f"Missing hatchet.workflow_run_id in {attrs}" assert "hatchet.tenant_id" in attrs, f"Missing hatchet.tenant_id in {attrs}" # Verify span kind is CONSUMER (value=4 in OTel Python SDK) @@ -93,23 +93,23 @@ async def test_otel_spans_created_on_task_run( child_span = child_spans[-1] # Child span should share the same trace_id as the step run span - assert child_span["trace_id"] == step_span["trace_id"], ( - f"Child trace_id {child_span['trace_id']} != step run trace_id {step_span['trace_id']}" - ) + assert ( + child_span["trace_id"] == step_span["trace_id"] + ), f"Child trace_id {child_span['trace_id']} != step run trace_id {step_span['trace_id']}" # Child span should have hatchet.* attributes injected by _HatchetAttributeSpanProcessor child_attrs = child_span["attributes"] - assert "hatchet.step_run_id" in child_attrs, ( - f"Child span missing hatchet.step_run_id (attribute propagation failed). Attrs: {child_attrs}" - ) - assert child_attrs["hatchet.step_run_id"] == attrs["hatchet.step_run_id"], ( - "Child span hatchet.step_run_id doesn't match parent" - ) + assert ( + "hatchet.step_run_id" in child_attrs + ), f"Child span missing hatchet.step_run_id (attribute propagation failed). Attrs: {child_attrs}" + assert ( + child_attrs["hatchet.step_run_id"] == attrs["hatchet.step_run_id"] + ), "Child span hatchet.step_run_id doesn't match parent" # Verify the custom attribute is present - assert child_attrs.get("test.marker") == "hello", ( - f"Missing test.marker attribute on child span. Attrs: {child_attrs}" - ) + assert ( + child_attrs.get("test.marker") == "hello" + ), f"Missing test.marker attribute on child span. Attrs: {child_attrs}" @pytest.mark.parametrize("on_demand_worker", ON_DEMAND_WORKER_PARAMS, indirect=True) @@ -156,18 +156,18 @@ async def test_otel_traces_on_retry( # All step run spans should have valid hatchet.* attributes for span in step_run_spans: attrs = span["attributes"] - assert "hatchet.step_run_id" in attrs, ( - f"Step run span missing hatchet.step_run_id. Attrs: {attrs}" - ) - assert "hatchet.workflow_run_id" in attrs, ( - f"Step run span missing hatchet.workflow_run_id. Attrs: {attrs}" - ) - assert "hatchet.tenant_id" in attrs, ( - f"Step run span missing hatchet.tenant_id. Attrs: {attrs}" - ) + assert ( + "hatchet.step_run_id" in attrs + ), f"Step run span missing hatchet.step_run_id. Attrs: {attrs}" + assert ( + "hatchet.workflow_run_id" in attrs + ), f"Step run span missing hatchet.workflow_run_id. Attrs: {attrs}" + assert ( + "hatchet.tenant_id" in attrs + ), f"Step run span missing hatchet.tenant_id. Attrs: {attrs}" # Verify retry count differs between attempts retry_counts = [s["attributes"].get("hatchet.retry_count") for s in step_run_spans] - assert len(set(retry_counts)) >= 2, ( - f"Expected different retry_count values across attempts, got {retry_counts}" - ) + assert ( + len(set(retry_counts)) >= 2 + ), f"Expected different retry_count values across attempts, got {retry_counts}" diff --git a/sdks/python/tests/otel_traces/worker.py b/sdks/python/tests/otel_traces/worker.py index a1dbea5e82..e881efa445 100644 --- a/sdks/python/tests/otel_traces/worker.py +++ b/sdks/python/tests/otel_traces/worker.py @@ -20,7 +20,6 @@ from hatchet_sdk import Context, EmptyModel, Hatchet from hatchet_sdk.opentelemetry.instrumentor import HatchetInstrumentor - # -- OTel setup ---------------------------------------------------------------- span_exporter = InMemorySpanExporter() From 8c145bfebc438b51b9afc10c2ac824f67fa72ae4 Mon Sep 17 00:00:00 2001 From: Mohammed Nafees Date: Mon, 9 Mar 2026 17:45:28 +0100 Subject: [PATCH 04/17] fix example --- sdks/python/examples/run_details/test_run_detail_getter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdks/python/examples/run_details/test_run_detail_getter.py b/sdks/python/examples/run_details/test_run_detail_getter.py index cb598d1da2..f26f69d742 100644 --- a/sdks/python/examples/run_details/test_run_detail_getter.py +++ b/sdks/python/examples/run_details/test_run_detail_getter.py @@ -23,7 +23,7 @@ async def test_run(hatchet: Hatchet) -> None: assert details.status == RunStatus.RUNNING assert details.input == mock_input.model_dump() - assert details.additional_metadata == meta + assert meta.items() <= details.additional_metadata.items() assert len(details.task_runs) == 4 assert all( r.status in [V1TaskStatus.RUNNING, V1TaskStatus.QUEUED] @@ -42,7 +42,7 @@ async def test_run(hatchet: Hatchet) -> None: assert details.status == RunStatus.FAILED assert details.input == mock_input.model_dump() - assert details.additional_metadata == meta + assert meta.items() <= details.additional_metadata.items() assert len(details.task_runs) == 6 assert details.task_runs["step1"].status == V1TaskStatus.COMPLETED From c147b91fba344309a015274807fd6a26f999f504 Mon Sep 17 00:00:00 2001 From: Mohammed Nafees Date: Mon, 9 Mar 2026 18:37:45 +0100 Subject: [PATCH 05/17] fix lint --- .../opentelemetry_instrumentation/hatchet/trigger.py | 5 ++++- sdks/python/examples/run_details/test_run_detail_getter.py | 2 ++ sdks/python/hatchet_sdk/opentelemetry/instrumentor.py | 1 + 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/sdks/python/examples/opentelemetry_instrumentation/hatchet/trigger.py b/sdks/python/examples/opentelemetry_instrumentation/hatchet/trigger.py index 3e4913d3bd..b1282b3bd1 100644 --- a/sdks/python/examples/opentelemetry_instrumentation/hatchet/trigger.py +++ b/sdks/python/examples/opentelemetry_instrumentation/hatchet/trigger.py @@ -36,7 +36,10 @@ def main() -> None: with tracer.start_as_current_span("trigger_otel_data_pipeline"): result = otel_workflow.run( options=TriggerWorkflowOptions( - additional_metadata={"source": "otel-example", "pipeline": "data-ingest"}, + additional_metadata={ + "source": "otel-example", + "pipeline": "data-ingest", + }, ), ) print(f"Workflow result: {result}") diff --git a/sdks/python/examples/run_details/test_run_detail_getter.py b/sdks/python/examples/run_details/test_run_detail_getter.py index f26f69d742..655afd753a 100644 --- a/sdks/python/examples/run_details/test_run_detail_getter.py +++ b/sdks/python/examples/run_details/test_run_detail_getter.py @@ -23,6 +23,7 @@ async def test_run(hatchet: Hatchet) -> None: assert details.status == RunStatus.RUNNING assert details.input == mock_input.model_dump() + assert details.additional_metadata is not None assert meta.items() <= details.additional_metadata.items() assert len(details.task_runs) == 4 assert all( @@ -42,6 +43,7 @@ async def test_run(hatchet: Hatchet) -> None: assert details.status == RunStatus.FAILED assert details.input == mock_input.model_dump() + assert details.additional_metadata is not None assert meta.items() <= details.additional_metadata.items() assert len(details.task_runs) == 6 diff --git a/sdks/python/hatchet_sdk/opentelemetry/instrumentor.py b/sdks/python/hatchet_sdk/opentelemetry/instrumentor.py index 302d0f583d..f0581679bb 100644 --- a/sdks/python/hatchet_sdk/opentelemetry/instrumentor.py +++ b/sdks/python/hatchet_sdk/opentelemetry/instrumentor.py @@ -57,6 +57,7 @@ def on_start(self, span: Span, parent_context: Context | None = None) -> None: span.set_attribute(key, value) super().on_start(span, parent_context) + import inspect from datetime import datetime From c49e1d80ca973294e07d737ec18ca104f1bed49e Mon Sep 17 00:00:00 2001 From: Mohammed Nafees Date: Mon, 9 Mar 2026 18:49:00 +0100 Subject: [PATCH 06/17] inject traceparent in Go SDK --- pkg/worker/context.go | 25 +++++++++++++++++++++++++ sdks/go/client.go | 3 +++ sdks/go/workflow.go | 26 ++++++++++++++++++++++++++ 3 files changed, 54 insertions(+) diff --git a/pkg/worker/context.go b/pkg/worker/context.go index 931da48b05..367bbbdc1c 100644 --- a/pkg/worker/context.go +++ b/pkg/worker/context.go @@ -9,6 +9,7 @@ import ( "time" "github.com/rs/zerolog" + "go.opentelemetry.io/otel/propagation" "google.golang.org/protobuf/types/known/timestamppb" v1 "github.com/hatchet-dev/hatchet/internal/services/shared/proto/v1" @@ -499,6 +500,27 @@ func (h *hatchetContext) saveOrLoadListener() (*client.WorkflowRunsListener, err return h.client().Subscribe().SubscribeToWorkflowRunEvents(h) } +// injectTraceparent serializes the current span's W3C traceparent from ctx +// into the AdditionalMetadata map so child workflows inherit the trace. +func injectTraceparent(ctx context.Context, meta *map[string]string) *map[string]string { + propagator := propagation.TraceContext{} + carrier := propagation.MapCarrier{} + propagator.Inject(ctx, carrier) + + tp, ok := carrier["traceparent"] + if !ok || tp == "" { + return meta + } + + if meta == nil { + m := map[string]string{"traceparent": tp} + return &m + } + + (*meta)["traceparent"] = tp + return meta +} + // Deprecated: SpawnWorkflow is an internal method used by the new Go SDK. // Use the new Go SDK at github.com/hatchet-dev/hatchet/sdks/go instead of using this directly. Migration guide: https://docs.hatchet.run/home/migration-guide-go func (h *hatchetContext) SpawnWorkflow(workflowName string, input any, opts *SpawnWorkflowOpts) (*client.Workflow, error) { @@ -506,6 +528,8 @@ func (h *hatchetContext) SpawnWorkflow(workflowName string, input any, opts *Spa opts = &SpawnWorkflowOpts{} } + opts.AdditionalMetadata = injectTraceparent(h.GetContext(), opts.AdditionalMetadata) + var desiredWorker *string if opts.Sticky != nil { @@ -572,6 +596,7 @@ func (h *hatchetContext) SpawnWorkflows(childWorkflows []*SpawnWorkflowsOpts) ([ listener, err := h.saveOrLoadListener() for i, c := range childWorkflows { + c.AdditionalMetadata = injectTraceparent(h.GetContext(), c.AdditionalMetadata) var desiredWorker *string diff --git a/sdks/go/client.go b/sdks/go/client.go index bfa72543b6..f0a752118a 100644 --- a/sdks/go/client.go +++ b/sdks/go/client.go @@ -633,6 +633,9 @@ func (c *Client) RunNoWait(ctx context.Context, workflowName string, input any, } } + // Inject traceparent for cross-workflow trace propagation + additionalMetadata = injectTraceparentToMap(ctx, additionalMetadata) + var v0Opts []v0Client.RunOptFunc if additionalMetadata != nil { v0Opts = append(v0Opts, v0Client.WithRunMetadata(*additionalMetadata)) diff --git a/sdks/go/workflow.go b/sdks/go/workflow.go index 03a8ffe023..eab6bd4e92 100644 --- a/sdks/go/workflow.go +++ b/sdks/go/workflow.go @@ -9,6 +9,8 @@ import ( "sync" "time" + "go.opentelemetry.io/otel/propagation" + v1 "github.com/hatchet-dev/hatchet/internal/services/shared/proto/v1" v0Client "github.com/hatchet-dev/hatchet/pkg/client" "github.com/hatchet-dev/hatchet/pkg/client/create" @@ -19,6 +21,27 @@ import ( "github.com/hatchet-dev/hatchet/sdks/go/internal" ) +// injectTraceparentToMap serializes the current span's W3C traceparent from ctx +// into the metadata map so child workflows inherit the trace. +func injectTraceparentToMap(ctx context.Context, meta *map[string]string) *map[string]string { + propagator := propagation.TraceContext{} + carrier := propagation.MapCarrier{} + propagator.Inject(ctx, carrier) + + tp, ok := carrier["traceparent"] + if !ok || tp == "" { + return meta + } + + if meta == nil { + m := map[string]string{"traceparent": tp} + return &m + } + + (*meta)["traceparent"] = tp + return meta +} + type RunPriority = features.RunPriority type DesiredWorkerLabel = types.DesiredWorkerLabel @@ -604,6 +627,9 @@ func (w *Workflow) RunNoWait(ctx context.Context, input any, opts ...RunOptFunc) priority = &[]int32{int32(*runOpts.Priority)}[0] } + // Inject traceparent for cross-workflow trace propagation + runOpts.AdditionalMetadata = injectTraceparentToMap(ctx, runOpts.AdditionalMetadata) + var v0Opts []v0Client.RunOptFunc if runOpts.AdditionalMetadata != nil { From 374342e96b418e6e792c837b13a80a6aeaafb946 Mon Sep 17 00:00:00 2001 From: Mohammed Nafees Date: Mon, 9 Mar 2026 19:31:47 +0100 Subject: [PATCH 07/17] ctx propagation --- examples/go/opentelemetry-propagation/main.go | 119 ++++++++++++++++++ .../hatchet/trigger.py | 5 +- .../hatchet/worker.py | 2 +- .../step-run-detail/otel-span-adapter.ts | 55 +++++++- .../step-run-detail/task-run-trace.tsx | 4 +- sdks/go/client.go | 32 ++++- .../opentelemetry-propagation/main.go | 119 ++++++++++++++++++ sdks/go/opentelemetry/middleware.go | 6 +- sdks/go/workflow.go | 32 ++++- .../hatchet/worker.py | 2 +- .../hatchet_sdk/opentelemetry/instrumentor.py | 22 ++-- .../hatchet_sdk/worker/runner/runner.py | 2 +- .../tests/otel_traces/test_otel_traces.py | 12 +- 13 files changed, 383 insertions(+), 29 deletions(-) create mode 100644 examples/go/opentelemetry-propagation/main.go create mode 100644 sdks/go/examples/opentelemetry-propagation/main.go diff --git a/examples/go/opentelemetry-propagation/main.go b/examples/go/opentelemetry-propagation/main.go new file mode 100644 index 0000000000..f162de1ddb --- /dev/null +++ b/examples/go/opentelemetry-propagation/main.go @@ -0,0 +1,119 @@ +package main + +import ( + "context" + "fmt" + "log" + "time" + + "go.opentelemetry.io/otel" + + "github.com/hatchet-dev/hatchet/pkg/cmdutils" + hatchet "github.com/hatchet-dev/hatchet/sdks/go" + hatchetotel "github.com/hatchet-dev/hatchet/sdks/go/opentelemetry" +) + +// This example demonstrates cross-workflow trace propagation. +// A parent task spawns a child task via .Run(), and both tasks' spans +// appear under the same trace in the UI — even if they run on different workers. + +type ParentInput struct { + Name string `json:"name"` +} + +type ParentOutput struct { + ChildResult string `json:"child_result"` +} + +type ChildInput struct { + Greeting string `json:"greeting"` +} + +type ChildOutput struct { + Message string `json:"message"` +} + +func main() { + client, err := hatchet.NewClient() + if err != nil { + log.Fatalf("failed to create client: %v", err) + } + + instrumentor, err := hatchetotel.NewInstrumentor( + hatchetotel.EnableHatchetCollector(), + ) + if err != nil { + log.Fatalf("failed to create instrumentor: %v", err) + } + + tracer := otel.Tracer("otel-propagation-example") + + // Child task — a standalone task that will be spawned by the parent. + childTask := client.NewStandaloneTask( + "otel-child-task", + func(ctx hatchet.Context, input ChildInput) (ChildOutput, error) { + _, span := tracer.Start(ctx.GetContext(), "child.process") + time.Sleep(50 * time.Millisecond) + span.End() + + return ChildOutput{ + Message: fmt.Sprintf("Hello from child: %s", input.Greeting), + }, nil + }, + ) + + // Parent task — spawns the child task via .Run(), which propagates the traceparent. + parentTask := client.NewStandaloneTask( + "otel-parent-task", + func(ctx hatchet.Context, input ParentInput) (ParentOutput, error) { + _, span := tracer.Start(ctx.GetContext(), "parent.prepare") + time.Sleep(30 * time.Millisecond) + span.End() + + // This .Run() call automatically injects traceparent into AdditionalMetadata, + // so the child task's spans will appear under the same trace. + result, err := childTask.Run(ctx, ChildInput{ + Greeting: fmt.Sprintf("greetings from %s", input.Name), + }) + if err != nil { + return ParentOutput{}, fmt.Errorf("child task failed: %w", err) + } + + var childOutput ChildOutput + if err := result.Into(&childOutput); err != nil { + return ParentOutput{}, fmt.Errorf("failed to parse child output: %w", err) + } + + return ParentOutput{ + ChildResult: childOutput.Message, + }, nil + }, + ) + + worker, err := client.NewWorker( + "otel-propagation-worker", + hatchet.WithWorkflows(parentTask, childTask), + ) + if err != nil { + log.Fatalf("failed to create worker: %v", err) + } + + worker.Use(instrumentor.Middleware()) + + interruptCtx, cancel := cmdutils.NewInterruptContext() + defer cancel() + + fmt.Println("Starting worker with OTel trace propagation...") + fmt.Println("Trigger the parent task to see linked parent → child traces in the UI.") + + go func() { + <-interruptCtx.Done() + if shutdownErr := instrumentor.Shutdown(context.Background()); shutdownErr != nil { + log.Printf("failed to shutdown instrumentor: %v", shutdownErr) + } + }() + + if startErr := worker.StartBlocking(interruptCtx); startErr != nil { + log.Printf("worker error: %v", startErr) + } +} diff --git a/examples/python/opentelemetry_instrumentation/hatchet/trigger.py b/examples/python/opentelemetry_instrumentation/hatchet/trigger.py index 3e4913d3bd..b1282b3bd1 100644 --- a/examples/python/opentelemetry_instrumentation/hatchet/trigger.py +++ b/examples/python/opentelemetry_instrumentation/hatchet/trigger.py @@ -36,7 +36,10 @@ def main() -> None: with tracer.start_as_current_span("trigger_otel_data_pipeline"): result = otel_workflow.run( options=TriggerWorkflowOptions( - additional_metadata={"source": "otel-example", "pipeline": "data-ingest"}, + additional_metadata={ + "source": "otel-example", + "pipeline": "data-ingest", + }, ), ) print(f"Workflow result: {result}") diff --git a/examples/python/opentelemetry_instrumentation/hatchet/worker.py b/examples/python/opentelemetry_instrumentation/hatchet/worker.py index 0b0d73b034..42846868f3 100644 --- a/examples/python/opentelemetry_instrumentation/hatchet/worker.py +++ b/examples/python/opentelemetry_instrumentation/hatchet/worker.py @@ -24,7 +24,7 @@ # Module-level tracer — will be set in main() before the worker starts. # Tasks use this to create custom child spans inside the auto-instrumented -# hatchet.start_step_run parent span. +# hatchet task run parent span. _tracer = None diff --git a/frontend/app/src/pages/main/v1/workflow-runs-v1/$run/v2components/step-run-detail/otel-span-adapter.ts b/frontend/app/src/pages/main/v1/workflow-runs-v1/$run/v2components/step-run-detail/otel-span-adapter.ts index 7d4ac3f646..e6e6bc3c29 100644 --- a/frontend/app/src/pages/main/v1/workflow-runs-v1/$run/v2components/step-run-detail/otel-span-adapter.ts +++ b/frontend/app/src/pages/main/v1/workflow-runs-v1/$run/v2components/step-run-detail/otel-span-adapter.ts @@ -64,8 +64,59 @@ export function convertOtelSpanToOpenTelemetrySpan( }; } -export function convertOtelSpans(spans: OtelSpan[]): OpenTelemetrySpan[] { - const converted = spans.map(convertOtelSpanToOpenTelemetrySpan); +/** + * Filters spans to only include those belonging to a specific task and its + * descendants. Finds the task's root span via hatchet.step_run_id attribute, + * then collects all descendant spans by following parent_span_id chains. + */ +function filterSpansForTask( + spans: OtelSpan[], + taskExternalId: string, +): OtelSpan[] { + // Find the root span for this task (the "hatchet task run" span) + const taskRootSpan = spans.find( + (s) => s.span_attributes?.['hatchet.step_run_id'] === taskExternalId, + ); + + if (!taskRootSpan) { + // Fallback: if no root span found, return all spans + return spans; + } + + // Build a parent->children index + const childrenByParent = new Map(); + for (const s of spans) { + const pid = s.parent_span_id; + if (pid) { + const children = childrenByParent.get(pid) || []; + children.push(s); + childrenByParent.set(pid, children); + } + } + + // BFS from the task root span to collect all descendants + const result: OtelSpan[] = [taskRootSpan]; + const queue = [taskRootSpan.span_id]; + while (queue.length > 0) { + const parentId = queue.shift()!; + const children = childrenByParent.get(parentId) || []; + for (const child of children) { + result.push(child); + queue.push(child.span_id); + } + } + + return result; +} + +export function convertOtelSpans( + spans: OtelSpan[], + taskExternalId?: string, +): OpenTelemetrySpan[] { + const filtered = taskExternalId + ? filterSpansForTask(spans, taskExternalId) + : spans; + const converted = filtered.map(convertOtelSpanToOpenTelemetrySpan); // Promote orphaned spans to root spans: if a span's parentSpanId // references a span not in this set, clear it so the tree builder diff --git a/frontend/app/src/pages/main/v1/workflow-runs-v1/$run/v2components/step-run-detail/task-run-trace.tsx b/frontend/app/src/pages/main/v1/workflow-runs-v1/$run/v2components/step-run-detail/task-run-trace.tsx index da5895c70a..0e27c4a6ed 100644 --- a/frontend/app/src/pages/main/v1/workflow-runs-v1/$run/v2components/step-run-detail/task-run-trace.tsx +++ b/frontend/app/src/pages/main/v1/workflow-runs-v1/$run/v2components/step-run-detail/task-run-trace.tsx @@ -22,9 +22,9 @@ export function TaskRunTrace({ taskExternalId }: { taskExternalId: string }) { return []; } - const otlpSpans = convertOtelSpans(rows); + const otlpSpans = convertOtelSpans(rows, taskExternalId); return openTelemetrySpanAdapter.convertRawSpansToSpanTree(otlpSpans); - }, [tracesQuery.data]); + }, [tracesQuery.data, taskExternalId]); const allIds = useMemo( () => flattenSpans(traceSpans).map((s) => s.id), diff --git a/sdks/go/client.go b/sdks/go/client.go index f0a752118a..57b51c03b0 100644 --- a/sdks/go/client.go +++ b/sdks/go/client.go @@ -7,6 +7,10 @@ import ( "fmt" "sync" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/trace" "golang.org/x/sync/errgroup" v1 "github.com/hatchet-dev/hatchet/internal/services/shared/proto/v1" @@ -615,6 +619,27 @@ func (c *Client) Run(ctx context.Context, workflowName string, input any, opts . // RunNoWait executes a workflow with the provided input without waiting for completion. // Returns a workflow run reference that can be used to track the run status. func (c *Client) RunNoWait(ctx context.Context, workflowName string, input any, opts ...RunOptFunc) (*WorkflowRunRef, error) { + // Start OTel span using the underlying context (not the HatchetContext itself) + // so that the HatchetContext type assertion still works downstream. + tracer := otel.Tracer("github.com/hatchet-dev/hatchet/sdks/go") + otelCtx := ctx + if hCtx, ok := ctx.(Context); ok { + otelCtx = hCtx.GetContext() + } + otelCtx, span := tracer.Start(otelCtx, fmt.Sprintf("hatchet trigger task %s", workflowName), + trace.WithSpanKind(trace.SpanKindClient), + trace.WithAttributes(attribute.String("hatchet.task_name", workflowName)), + ) + defer span.End() + + // Update the HatchetContext's inner context with the OTel span context, + // or use the OTel context directly for plain context.Context callers. + if hCtx, ok := ctx.(Context); ok { + hCtx.SetContext(otelCtx) + } else { + ctx = otelCtx + } + runOpts := &runOpts{} for _, opt := range opts { opt(runOpts) @@ -634,7 +659,7 @@ func (c *Client) RunNoWait(ctx context.Context, workflowName string, input any, } // Inject traceparent for cross-workflow trace propagation - additionalMetadata = injectTraceparentToMap(ctx, additionalMetadata) + additionalMetadata = injectTraceparentToMap(otelCtx, additionalMetadata) var v0Opts []v0Client.RunOptFunc if additionalMetadata != nil { @@ -660,9 +685,14 @@ func (c *Client) RunNoWait(ctx context.Context, workflowName string, input any, } if err != nil { + span.SetStatus(codes.Error, err.Error()) + span.RecordError(err) return nil, err } + span.SetAttributes(attribute.String("hatchet.workflow_run_id", v0Workflow.RunId())) + span.SetStatus(codes.Ok, "") + return &WorkflowRunRef{RunId: v0Workflow.RunId(), v0Workflow: v0Workflow}, nil } diff --git a/sdks/go/examples/opentelemetry-propagation/main.go b/sdks/go/examples/opentelemetry-propagation/main.go new file mode 100644 index 0000000000..f162de1ddb --- /dev/null +++ b/sdks/go/examples/opentelemetry-propagation/main.go @@ -0,0 +1,119 @@ +package main + +import ( + "context" + "fmt" + "log" + "time" + + "go.opentelemetry.io/otel" + + "github.com/hatchet-dev/hatchet/pkg/cmdutils" + hatchet "github.com/hatchet-dev/hatchet/sdks/go" + hatchetotel "github.com/hatchet-dev/hatchet/sdks/go/opentelemetry" +) + +// This example demonstrates cross-workflow trace propagation. +// A parent task spawns a child task via .Run(), and both tasks' spans +// appear under the same trace in the UI — even if they run on different workers. + +type ParentInput struct { + Name string `json:"name"` +} + +type ParentOutput struct { + ChildResult string `json:"child_result"` +} + +type ChildInput struct { + Greeting string `json:"greeting"` +} + +type ChildOutput struct { + Message string `json:"message"` +} + +func main() { + client, err := hatchet.NewClient() + if err != nil { + log.Fatalf("failed to create client: %v", err) + } + + instrumentor, err := hatchetotel.NewInstrumentor( + hatchetotel.EnableHatchetCollector(), + ) + if err != nil { + log.Fatalf("failed to create instrumentor: %v", err) + } + + tracer := otel.Tracer("otel-propagation-example") + + // Child task — a standalone task that will be spawned by the parent. + childTask := client.NewStandaloneTask( + "otel-child-task", + func(ctx hatchet.Context, input ChildInput) (ChildOutput, error) { + _, span := tracer.Start(ctx.GetContext(), "child.process") + time.Sleep(50 * time.Millisecond) + span.End() + + return ChildOutput{ + Message: fmt.Sprintf("Hello from child: %s", input.Greeting), + }, nil + }, + ) + + // Parent task — spawns the child task via .Run(), which propagates the traceparent. + parentTask := client.NewStandaloneTask( + "otel-parent-task", + func(ctx hatchet.Context, input ParentInput) (ParentOutput, error) { + _, span := tracer.Start(ctx.GetContext(), "parent.prepare") + time.Sleep(30 * time.Millisecond) + span.End() + + // This .Run() call automatically injects traceparent into AdditionalMetadata, + // so the child task's spans will appear under the same trace. + result, err := childTask.Run(ctx, ChildInput{ + Greeting: fmt.Sprintf("greetings from %s", input.Name), + }) + if err != nil { + return ParentOutput{}, fmt.Errorf("child task failed: %w", err) + } + + var childOutput ChildOutput + if err := result.Into(&childOutput); err != nil { + return ParentOutput{}, fmt.Errorf("failed to parse child output: %w", err) + } + + return ParentOutput{ + ChildResult: childOutput.Message, + }, nil + }, + ) + + worker, err := client.NewWorker( + "otel-propagation-worker", + hatchet.WithWorkflows(parentTask, childTask), + ) + if err != nil { + log.Fatalf("failed to create worker: %v", err) + } + + worker.Use(instrumentor.Middleware()) + + interruptCtx, cancel := cmdutils.NewInterruptContext() + defer cancel() + + fmt.Println("Starting worker with OTel trace propagation...") + fmt.Println("Trigger the parent task to see linked parent → child traces in the UI.") + + go func() { + <-interruptCtx.Done() + if shutdownErr := instrumentor.Shutdown(context.Background()); shutdownErr != nil { + log.Printf("failed to shutdown instrumentor: %v", shutdownErr) + } + }() + + if startErr := worker.StartBlocking(interruptCtx); startErr != nil { + log.Printf("worker error: %v", startErr) + } +} diff --git a/sdks/go/opentelemetry/middleware.go b/sdks/go/opentelemetry/middleware.go index e8c8473364..8da529b41d 100644 --- a/sdks/go/opentelemetry/middleware.go +++ b/sdks/go/opentelemetry/middleware.go @@ -11,7 +11,7 @@ import ( // NewMiddleware creates a Hatchet middleware that wraps each step run execution // with an OpenTelemetry span. It: // - Extracts W3C traceparent from AdditionalMetadata for distributed trace propagation -// - Creates a "hatchet.start_step_run" span with hatchet.* attributes +// - Creates a "hatchet task run" span with hatchet.* attributes // - Stores attributes in context so HatchetAttributeSpanProcessor can inject // them into all child spans // @@ -38,7 +38,7 @@ func NewMiddleware(tracer trace.Tracer) worker.MiddlewareFunc { parentCtx = withHatchetAttributes(parentCtx, attrs) // Start span - spanCtx, span := tracer.Start(parentCtx, "hatchet.start_step_run", + spanCtx, span := tracer.Start(parentCtx, "hatchet task run", trace.WithSpanKind(trace.SpanKindConsumer), trace.WithAttributes(attrs...), ) @@ -52,6 +52,8 @@ func NewMiddleware(tracer trace.Tracer) worker.MiddlewareFunc { if err != nil { span.SetStatus(codes.Error, err.Error()) span.RecordError(err) + } else { + span.SetStatus(codes.Ok, "") } return err diff --git a/sdks/go/workflow.go b/sdks/go/workflow.go index eab6bd4e92..1603771d0c 100644 --- a/sdks/go/workflow.go +++ b/sdks/go/workflow.go @@ -9,7 +9,11 @@ import ( "sync" "time" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" "go.opentelemetry.io/otel/propagation" + "go.opentelemetry.io/otel/trace" v1 "github.com/hatchet-dev/hatchet/internal/services/shared/proto/v1" v0Client "github.com/hatchet-dev/hatchet/pkg/client" @@ -617,6 +621,27 @@ func (w *Workflow) Run(ctx context.Context, input any, opts ...RunOptFunc) (*Wor // RunNoWait executes the workflow with the provided input without waiting for completion. // Returns a workflow run reference that can be used to track the run status. func (w *Workflow) RunNoWait(ctx context.Context, input any, opts ...RunOptFunc) (*WorkflowRunRef, error) { + // Start OTel span using the underlying context (not the HatchetContext itself) + // so that the HatchetContext type assertion still works downstream. + tracer := otel.Tracer("github.com/hatchet-dev/hatchet/sdks/go") + otelCtx := ctx + if hCtx, ok := ctx.(Context); ok { + otelCtx = hCtx.GetContext() + } + otelCtx, span := tracer.Start(otelCtx, fmt.Sprintf("hatchet trigger task %s", w.declaration.Name()), + trace.WithSpanKind(trace.SpanKindClient), + trace.WithAttributes(attribute.String("hatchet.task_name", w.declaration.Name())), + ) + defer span.End() + + // Update the HatchetContext's inner context with the OTel span context, + // or use the OTel context directly for plain context.Context callers. + if hCtx, ok := ctx.(Context); ok { + hCtx.SetContext(otelCtx) + } else { + ctx = otelCtx + } + runOpts := &runOpts{} for _, opt := range opts { opt(runOpts) @@ -628,7 +653,7 @@ func (w *Workflow) RunNoWait(ctx context.Context, input any, opts ...RunOptFunc) } // Inject traceparent for cross-workflow trace propagation - runOpts.AdditionalMetadata = injectTraceparentToMap(ctx, runOpts.AdditionalMetadata) + runOpts.AdditionalMetadata = injectTraceparentToMap(otelCtx, runOpts.AdditionalMetadata) var v0Opts []v0Client.RunOptFunc @@ -661,9 +686,14 @@ func (w *Workflow) RunNoWait(ctx context.Context, input any, opts ...RunOptFunc) } if err != nil { + span.SetStatus(codes.Error, err.Error()) + span.RecordError(err) return nil, err } + span.SetAttributes(attribute.String("hatchet.workflow_run_id", v0Workflow.RunId())) + span.SetStatus(codes.Ok, "") + return &WorkflowRunRef{RunId: v0Workflow.RunId(), v0Workflow: v0Workflow}, nil } diff --git a/sdks/python/examples/opentelemetry_instrumentation/hatchet/worker.py b/sdks/python/examples/opentelemetry_instrumentation/hatchet/worker.py index 0b0d73b034..42846868f3 100644 --- a/sdks/python/examples/opentelemetry_instrumentation/hatchet/worker.py +++ b/sdks/python/examples/opentelemetry_instrumentation/hatchet/worker.py @@ -24,7 +24,7 @@ # Module-level tracer — will be set in main() before the worker starts. # Tasks use this to create custom child spans inside the auto-instrumented -# hatchet.start_step_run parent span. +# hatchet task run parent span. _tracer = None diff --git a/sdks/python/hatchet_sdk/opentelemetry/instrumentor.py b/sdks/python/hatchet_sdk/opentelemetry/instrumentor.py index f0581679bb..73cbb14288 100644 --- a/sdks/python/hatchet_sdk/opentelemetry/instrumentor.py +++ b/sdks/python/hatchet_sdk/opentelemetry/instrumentor.py @@ -36,7 +36,7 @@ ) from e # ContextVar that holds the hatchet.* attributes from the active -# hatchet.start_step_run span so they can be injected into child spans. +# hatchet task run span so they can be injected into child spans. _hatchet_span_attributes: contextvars.ContextVar[dict[str, str | int] | None] = ( contextvars.ContextVar("_hatchet_span_attributes", default=None) ) @@ -391,10 +391,10 @@ async def _wrap_handle_start_step_run( action = cast(Action, params[0]) traceparent = _parse_carrier_from_metadata(action.additional_metadata) - span_name = "hatchet.start_step_run" + span_name = "hatchet task run" if self.config.otel.include_task_name_in_start_step_run_span_name: - span_name += f".{action.action_id}" + span_name += f" {action.action_id}" hatchet_attrs = action.get_otel_attributes(self.config) token = _hatchet_span_attributes.set(hatchet_attrs) @@ -426,7 +426,7 @@ async def _wrap_handle_cancel_action( action = args[0] with self._tracer.start_as_current_span( - "hatchet.cancel_step_run", + "hatchet cancel task run", attributes={ "hatchet.step_run_id": action.step_run_id, }, @@ -467,7 +467,7 @@ def _wrap_push_event( } with self._tracer.start_as_current_span( - "hatchet.push_event", + "hatchet push event", attributes={ f"hatchet.{k.value}": v for k, v in attributes.items() @@ -509,7 +509,7 @@ def _wrap_bulk_push_event( unique_event_keys = {event.key for event in bulk_events} with self._tracer.start_as_current_span( - "hatchet.bulk_push_event", + "hatchet push events", attributes={ "hatchet.num_events": num_bulk_events, "hatchet.unique_event_keys": json.dumps(unique_event_keys, default=str), @@ -569,7 +569,7 @@ def _wrap_run_workflow( } with self._tracer.start_as_current_span( - "hatchet.run_workflow", + "hatchet trigger task", attributes={ f"hatchet.{k.value}": v for k, v in attributes.items() @@ -627,7 +627,7 @@ async def _wrap_async_run_workflow( } with self._tracer.start_as_current_span( - "hatchet.run_workflow", + "hatchet trigger task", attributes={ f"hatchet.{k.value}": v for k, v in attributes.items() @@ -709,7 +709,7 @@ def _wrap_schedule_workflow( } with self._tracer.start_as_current_span( - "hatchet.schedule_workflow", + "hatchet schedule task", attributes={ f"hatchet.{k.value}": v for k, v in attributes.items() @@ -749,7 +749,7 @@ def _wrap_run_workflows( } with self._tracer.start_as_current_span( - "hatchet.run_workflows", + "hatchet trigger tasks", attributes={ "hatchet.num_workflows": num_workflows, "hatchet.unique_workflow_names": json.dumps( @@ -792,7 +792,7 @@ async def _wrap_async_run_workflows( } with self._tracer.start_as_current_span( - "hatchet.run_workflows", + "hatchet trigger tasks", attributes={ "hatchet.num_workflows": num_workflows, "hatchet.unique_workflow_names": json.dumps( diff --git a/sdks/python/hatchet_sdk/worker/runner/runner.py b/sdks/python/hatchet_sdk/worker/runner/runner.py index 3bcf096dd8..739c99f736 100644 --- a/sdks/python/hatchet_sdk/worker/runner/runner.py +++ b/sdks/python/hatchet_sdk/worker/runner/runner.py @@ -316,7 +316,7 @@ async def async_wrapped_action_func( # Copy the full contextvars snapshot (including OpenTelemetry span # context) so that child spans created in the thread inherit the # correct trace_id and parent_span_id from the active - # hatchet.start_step_run span. + # hatchet task run span. ctx_snapshot = contextvars.copy_context() loop = asyncio.get_event_loop() return await loop.run_in_executor( diff --git a/sdks/python/tests/otel_traces/test_otel_traces.py b/sdks/python/tests/otel_traces/test_otel_traces.py index 1423fd3dbe..e75baae8a7 100644 --- a/sdks/python/tests/otel_traces/test_otel_traces.py +++ b/sdks/python/tests/otel_traces/test_otel_traces.py @@ -63,10 +63,10 @@ async def test_otel_spans_created_on_task_run( spans = _get_spans() - # Find the hatchet.start_step_run span - step_run_spans = [s for s in spans if s["name"] == "hatchet.start_step_run"] + # Find the hatchet task run span + step_run_spans = [s for s in spans if s["name"] == "hatchet task run"] assert len(step_run_spans) >= 1, ( - f"Expected at least one hatchet.start_step_run span, got {len(step_run_spans)}. " + f"Expected at least one hatchet task run span, got {len(step_run_spans)}. " f"All spans: {[s['name'] for s in spans]}" ) @@ -121,7 +121,7 @@ async def test_otel_traces_on_retry( Uses a task that fails on the first attempt (raises an exception) and succeeds on the second attempt (retries=1). Both attempts should produce - valid hatchet.start_step_run spans with correct attributes, and the retry + valid hatchet task run spans with correct attributes, and the retry count attribute should differ between them. """ _clear_spans() @@ -140,9 +140,9 @@ async def test_otel_traces_on_retry( spans = _get_spans() # Both the failed first attempt and the successful retry should have spans - step_run_spans = [s for s in spans if s["name"] == "hatchet.start_step_run"] + step_run_spans = [s for s in spans if s["name"] == "hatchet task run"] assert len(step_run_spans) >= 2, ( - f"Expected at least 2 hatchet.start_step_run spans (initial + retry), " + f"Expected at least 2 hatchet task run spans (initial + retry), " f"got {len(step_run_spans)}. All spans: {[s['name'] for s in spans]}" ) From 1cffd634247383e44352e0d5a6ab6db5c32955ba Mon Sep 17 00:00:00 2001 From: Mohammed Nafees Date: Mon, 9 Mar 2026 21:24:31 +0100 Subject: [PATCH 08/17] Feat: Opentelemetry for TS SDK (#2828) (#3218) * add: otel as optional dep on ts packages * feat: opentelemetry instrumentor for TS sdk, with example * fix: lint * revert: debug print * remove: trailing space * fix: ts otel patch file path, throw handlesteprun error upstream, ts otel examples * fix: lint * feat: add schedule_workflow instrumentor, add otel conig loader tests * add: more robust wrap unwrap for patched modules * fix: lint, update version * refactor: ts otel config type assertion * revert: rebase issues * fix: lint * fix: update worker patch for ts otel with InternalWorker * fix: lint * refactor: parsejson on otel * fix: pnpm-lock * fix: lint * docs: add otel instrumented method warnings Co-authored-by: Jishnu --- .../opentelemetry_instrumentation/client.ts | 7 + .../opentelemetry_instrumentation/tracer.ts | 70 ++ .../opentelemetry_instrumentation/triggers.ts | 162 ++++ .../opentelemetry_instrumentation/worker.ts | 82 ++ sdks/typescript/CHANGELOG.md | 8 +- sdks/typescript/package.json | 4 +- sdks/typescript/pnpm-lock.yaml | 76 ++ .../src/clients/event/event-client.ts | 8 + .../clients/hatchet-client/client-config.ts | 17 + .../hatchet-client/fixtures/.hatchet.yaml | 4 + .../src/legacy/legacy-client.test.ts | 16 + sdks/typescript/src/opentelemetry/index.ts | 3 + .../src/opentelemetry/instrumentor.ts | 698 ++++++++++++++++++ sdks/typescript/src/opentelemetry/types.ts | 6 + .../util/config-loader/config-loader.test.ts | 11 + .../src/util/config-loader/config-loader.ts | 23 +- .../util/config-loader/fixtures/.hatchet.yaml | 4 + sdks/typescript/src/util/opentelemetry.ts | 59 ++ sdks/typescript/src/v1/client/admin.ts | 6 + .../src/v1/client/features/schedules.ts | 3 + .../src/v1/client/worker/worker-internal.ts | 22 +- 21 files changed, 1281 insertions(+), 8 deletions(-) create mode 100644 examples/typescript/opentelemetry_instrumentation/client.ts create mode 100644 examples/typescript/opentelemetry_instrumentation/tracer.ts create mode 100644 examples/typescript/opentelemetry_instrumentation/triggers.ts create mode 100644 examples/typescript/opentelemetry_instrumentation/worker.ts create mode 100644 sdks/typescript/src/opentelemetry/index.ts create mode 100644 sdks/typescript/src/opentelemetry/instrumentor.ts create mode 100644 sdks/typescript/src/opentelemetry/types.ts create mode 100644 sdks/typescript/src/util/opentelemetry.ts diff --git a/examples/typescript/opentelemetry_instrumentation/client.ts b/examples/typescript/opentelemetry_instrumentation/client.ts new file mode 100644 index 0000000000..91e61a2eec --- /dev/null +++ b/examples/typescript/opentelemetry_instrumentation/client.ts @@ -0,0 +1,7 @@ +import './tracer'; + +import Hatchet from '@hatchet-dev/typescript-sdk'; + +export const hatchet = Hatchet.init({ + log_level: 'DEBUG', +}); diff --git a/examples/typescript/opentelemetry_instrumentation/tracer.ts b/examples/typescript/opentelemetry_instrumentation/tracer.ts new file mode 100644 index 0000000000..2ef7267ebc --- /dev/null +++ b/examples/typescript/opentelemetry_instrumentation/tracer.ts @@ -0,0 +1,70 @@ +const { NodeTracerProvider } = require('@opentelemetry/sdk-trace-node'); +const { OTLPTraceExporter } = require('@opentelemetry/exporter-trace-otlp-http'); +const { resourceFromAttributes } = require('@opentelemetry/resources'); +const { SEMRESATTRS_SERVICE_NAME } = require('@opentelemetry/semantic-conventions'); +const { BatchSpanProcessor } = require('@opentelemetry/sdk-trace-base'); +const { registerInstrumentations } = require('@opentelemetry/instrumentation'); +const { trace } = require('@opentelemetry/api'); + +import type { TracerProvider, Tracer } from '@opentelemetry/api'; +import { HatchetInstrumentor } from '@hatchet-dev/typescript-sdk/opentelemetry'; + +const isCI = process.env.CI === 'true'; + +let traceProvider: TracerProvider; + +if (isCI) { + traceProvider = trace.getTracerProvider(); + registerInstrumentations({ + tracerProvider: traceProvider, + instrumentations: [new HatchetInstrumentor()], + }); +} else { + const resource = resourceFromAttributes({ + [SEMRESATTRS_SERVICE_NAME]: + process.env.HATCHET_CLIENT_OTEL_SERVICE_NAME || 'hatchet-typescript-example', + }); + + // Parse headers from environment variable in format "key=value" + const headersEnv = process.env.HATCHET_CLIENT_OTEL_EXPORTER_OTLP_HEADERS; + const headers: Record | undefined = headersEnv + ? { [headersEnv.split('=')[0]]: headersEnv.split('=')[1] } + : undefined; + + const exporter = new OTLPTraceExporter({ + url: + process.env.HATCHET_CLIENT_OTEL_EXPORTER_OTLP_ENDPOINT || 'http://localhost:4318/v1/traces', + headers, + }); + + const provider = new NodeTracerProvider({ + resource, + spanProcessors: [new BatchSpanProcessor(exporter)], + }); + + provider.register(); + + traceProvider = provider; + + + // NOTE: Instrumentation has to be registered before the instrumented libraries are imported + registerInstrumentations({ + tracerProvider: traceProvider, + instrumentations: [ + new HatchetInstrumentor({ + // Optional: exclude sensitive attributes from spans + // excludedAttributes: ['payload', 'additional_metadata'], + + // Optional: include task name in span names for better filtering + includeTaskNameInSpanName: true, + }), + ], + }); +} + + +function getTracer(name: string): Tracer { + return trace.getTracer(name); +} + +export { traceProvider, getTracer }; diff --git a/examples/typescript/opentelemetry_instrumentation/triggers.ts b/examples/typescript/opentelemetry_instrumentation/triggers.ts new file mode 100644 index 0000000000..39a779fb74 --- /dev/null +++ b/examples/typescript/opentelemetry_instrumentation/triggers.ts @@ -0,0 +1,162 @@ +import { getTracer } from './tracer'; + +import { SpanStatusCode, type Span } from '@opentelemetry/api'; +import { hatchet } from './client'; +import { otelWorkflow } from './worker'; + +const tracer = getTracer('opentelemetry-triggers'); + +const ADDITIONAL_METADATA = { source: 'otel-example', version: '1.0' }; + +async function pushEvent() { + console.log('\n--- Push Event ---'); + + return tracer.startActiveSpan('push_event', async (span: Span) => { + try { + await hatchet.events.push( + 'otel:event', + { message: 'Hello from instrumented trigger!' }, + { additionalMetadata: ADDITIONAL_METADATA } + ); + console.log('Event pushed successfully'); + } finally { + span.end(); + } + }); +} + +async function bulkPushEvents() { + console.log('\n--- Bulk Push Events ---'); + + return tracer.startActiveSpan('bulk_push_event', async (span: Span) => { + try { + await hatchet.events.bulkPush('otel:event', [ + { + payload: { message: 'Bulk event 1' }, + additionalMetadata: ADDITIONAL_METADATA, + }, + { + payload: { message: 'Bulk event 2' }, + additionalMetadata: ADDITIONAL_METADATA, + }, + { + payload: { message: 'Bulk event 3' }, + additionalMetadata: ADDITIONAL_METADATA, + }, + ]); + console.log('Bulk events pushed successfully'); + } finally { + span.end(); + } + }); +} + +async function runWorkflow() { + console.log('\n--- Run Workflow ---'); + + return tracer.startActiveSpan('run_workflow', async (span: Span) => { + try { + const workflowRun = await hatchet.admin.runWorkflow(otelWorkflow.name, {}, { + additionalMetadata: ADDITIONAL_METADATA, + }); + const runId = await workflowRun.getWorkflowRunId(); + console.log(`Started workflow run: ${runId}`); + + const result = await workflowRun.output; + span.setStatus({ code: SpanStatusCode.OK, message: 'Workflow completed' }); + console.log(`Workflow completed with result:`, result); + } catch (error: any) { + const errorMessage = Array.isArray(error) ? error.join(', ') : error?.message || String(error); + console.error('Workflow failed:', errorMessage); + span.recordException(error); + span.setStatus({ code: SpanStatusCode.ERROR, message: errorMessage }); + } finally { + span.end(); + } + }); +} + +async function runWorkflows() { + console.log('\n--- Run Workflows (Bulk) ---'); + + return tracer.startActiveSpan('run_workflows', async (span: Span) => { + try { + const refs = await hatchet.admin.runWorkflows([ + { + workflowName: otelWorkflow.name, + input: {}, + options: { additionalMetadata: ADDITIONAL_METADATA }, + }, + { + workflowName: otelWorkflow.name, + input: {}, + options: { additionalMetadata: ADDITIONAL_METADATA }, + }, + ]); + console.log(`Started ${refs.length} workflow runs`); + + const results = await Promise.all(refs.map((ref: { result: () => Promise }) => ref.result())); + span.setStatus({ code: SpanStatusCode.OK, message: 'Workflows completed' }); + console.log(`Workflows completed with results:`, results); + } catch (error: any) { + const errorMessage = Array.isArray(error) ? error.join(', ') : error?.message || String(error); + console.error('Workflows failed:', errorMessage); + span.recordException(error); + span.setStatus({ code: SpanStatusCode.ERROR, message: errorMessage }); + } finally { + span.end(); + } + }); +} + +async function scheduleWorkflow() { + console.log('\n--- Schedule Workflow ---'); + + return tracer.startActiveSpan('schedule_workflow', async (span: Span) => { + try { + // Schedule workflow to run 10 seconds from now + const triggerAt = new Date(Date.now() + 10 * 1000); + + const scheduledRun = await hatchet.schedules.create(otelWorkflow.name, { + triggerAt, + input: { message: 'Hello from scheduled workflow!' }, + additionalMetadata: ADDITIONAL_METADATA, + }); + + console.log(`Scheduled workflow run: ${scheduledRun.metadata.id}`); + console.log(`Will trigger at: ${triggerAt.toISOString()}`); + + span.setStatus({ code: SpanStatusCode.OK, message: 'Workflow scheduled' }); + } catch (error: any) { + const errorMessage = Array.isArray(error) ? error.join(', ') : error?.message || String(error); + console.error('Schedule workflow failed:', errorMessage); + span.recordException(error); + span.setStatus({ code: SpanStatusCode.ERROR, message: errorMessage }); + } finally { + span.end(); + } + }); +} + + + +async function main() { + console.log('OpenTelemetry Triggers Example'); + console.log('==============================\n'); + + await pushEvent(); + await bulkPushEvents(); + await runWorkflow(); + await runWorkflows(); + await scheduleWorkflow(); + + console.log('\n--- Waiting for spans to be exported... ---'); + await new Promise((resolve) => setTimeout(resolve, 5000)); + console.log('Done!'); + + process.exit(0); +} + +if (require.main === module) { + main().catch(console.error); +} diff --git a/examples/typescript/opentelemetry_instrumentation/worker.ts b/examples/typescript/opentelemetry_instrumentation/worker.ts new file mode 100644 index 0000000000..3b49e05d92 --- /dev/null +++ b/examples/typescript/opentelemetry_instrumentation/worker.ts @@ -0,0 +1,82 @@ +import { hatchet } from './client'; +import { getTracer } from './tracer'; +import { SpanStatusCode } from '@opentelemetry/api'; + +const tracer = getTracer('opentelemetry-worker'); + +export const otelWorkflow = hatchet.workflow({ + name: 'otelworkflowtypescript', +}); + +otelWorkflow.task({ + name: 'step-with-custom-spans', + fn: async () => { + return tracer.startActiveSpan('custom-business-logic', async (span) => { + try { + console.log('Executing step with custom tracing...'); + await new Promise((resolve) => setTimeout(resolve, 100)); + + return { result: 'success' }; + } finally { + span.end(); + } + }); + }, +}); + +/** + * Task demonstrating that span hierarchy is preserved even when errors occur. + */ +otelWorkflow.task({ + name: 'step-with-error', + fn: async () => { + return tracer.startActiveSpan('custom-span-with-error', async (span) => { + try { + throw new Error('Intentional error for demonstration'); + } catch (error: any) { + span.recordException(error); + span.setStatus({ code: SpanStatusCode.ERROR, message: error?.message }); + throw error; + } finally { + span.end(); + } + }); + }, +}); + +/** + * Task that is automatically instrumented without any manual span creation. + */ +otelWorkflow.task({ + name: 'auto-instrumented-step', + fn: async () => { + console.log('This step is automatically traced without manual span code'); + return { automatically: 'instrumented' }; + }, +}); + +/** + * Task demonstrating error handling in auto-instrumented steps. + */ +otelWorkflow.task({ + name: 'auto-instrumented-step-with-error', + fn: async () => { + throw new Error('Auto-instrumented step error'); + }, +}); + +async function main() { + console.log('Starting OpenTelemetry instrumented worker...'); + console.log('Instrumentation is automatic via module patching.'); + + const worker = await hatchet.worker('otel-example-worker-ts', { + slots: 1, + workflows: [otelWorkflow], + }); + + await worker.start(); +} + +if (require.main === module) { + main().catch(console.error); +} diff --git a/sdks/typescript/CHANGELOG.md b/sdks/typescript/CHANGELOG.md index 2d64a71ee2..6b4a839459 100644 --- a/sdks/typescript/CHANGELOG.md +++ b/sdks/typescript/CHANGELOG.md @@ -1,10 +1,16 @@ # Changelog - All notable changes to Hatchet's TypeScript SDK will be documented in this changelog. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [1.16.0] - 2026-03-09 + +### Added + +- OpenTelemetry instrumentation via `HatchetInstrumentor` with automatic tracing for workflow runs, event pushes, and step executions +- OpenTelemetry example demonstrating automatic and custom span instrumentation (`examples/opentelemetry_instrumentation`) + ## [1.15.2] - 2026-03-06 ### Fixed diff --git a/sdks/typescript/package.json b/sdks/typescript/package.json index 358209ee3b..e1cd20adcc 100644 --- a/sdks/typescript/package.json +++ b/sdks/typescript/package.json @@ -1,6 +1,6 @@ { "name": "@hatchet-dev/typescript-sdk", - "version": "1.15.2", + "version": "1.16.0", "description": "Background task orchestration & visibility for developers", "types": "dist/index.d.ts", "files": [ @@ -85,6 +85,8 @@ "zod-to-json-schema": "^3.24.1" }, "optionalDependencies": { + "@opentelemetry/api": "^1.9.0", + "@opentelemetry/instrumentation": "^0.208.0", "prom-client": "^15.1.3" }, "packageManager": "pnpm@10.16.1", diff --git a/sdks/typescript/pnpm-lock.yaml b/sdks/typescript/pnpm-lock.yaml index 2f513c6c79..6d0af29366 100644 --- a/sdks/typescript/pnpm-lock.yaml +++ b/sdks/typescript/pnpm-lock.yaml @@ -131,6 +131,12 @@ importers: specifier: ^8.56.1 version: 8.56.1(eslint@10.0.3)(typescript@5.9.2) optionalDependencies: + '@opentelemetry/api': + specifier: ^1.9.0 + version: 1.9.0 + '@opentelemetry/instrumentation': + specifier: ^0.208.0 + version: 0.208.0(@opentelemetry/api@1.9.0) prom-client: specifier: ^15.1.3 version: 15.1.3 @@ -510,10 +516,20 @@ packages: resolution: {integrity: sha512-nn5ozdjYQpUCZlWGuxcJY/KpxkWQs4DcbMCmKojjyrYDEAGy4Ce19NN4v5MduafTwJlbKc99UA8YhSVqq9yPZA==} engines: {node: '>=12.4.0'} + '@opentelemetry/api-logs@0.208.0': + resolution: {integrity: sha512-CjruKY9V6NMssL/T1kAFgzosF1v9o6oeN+aX5JB/C/xPNtmgIJqcXHG7fA82Ou1zCpWGl4lROQUKwUNE1pMCyg==} + engines: {node: '>=8.0.0'} + '@opentelemetry/api@1.9.0': resolution: {integrity: sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==} engines: {node: '>=8.0.0'} + '@opentelemetry/instrumentation@0.208.0': + resolution: {integrity: sha512-Eju0L4qWcQS+oXxi6pgh7zvE2byogAkcsVv0OjHF/97iOz1N/aKE6etSGowYkie+YA1uo6DNwdSxaaNnLvcRlA==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + '@pkgr/core@0.2.9': resolution: {integrity: sha512-QNqXyfVS2wm9hweSYD2O7F0G06uurj9kZ96TRQE5Y9hU7+tgdZwIkbAKc5Ocy1HxEY2kuDQa6cQ1WRs/O5LFKA==} engines: {node: ^12.20.0 || ^14.18.0 || >=16.0.0} @@ -823,6 +839,11 @@ packages: abort-controller-x@0.4.3: resolution: {integrity: sha512-VtUwTNU8fpMwvWGn4xE93ywbogTYsuT+AUxAXOeelbXuQVIwNmC5YLeho9sH4vZ4ITW8414TTAOG1nW6uIVHCA==} + acorn-import-attributes@1.9.5: + resolution: {integrity: sha512-n02Vykv5uA3eHGM/Z2dQrcD56kL8TyDb2p1+0P83PClMnC/nc+anbQRhIOWnSq4Ke/KvDPrY3C9hDtC/A3eHnQ==} + peerDependencies: + acorn: ^8 + acorn-jsx@5.3.2: resolution: {integrity: sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==} peerDependencies: @@ -1046,6 +1067,9 @@ packages: cjs-module-lexer@1.4.3: resolution: {integrity: sha512-9z8TZaGM1pfswYeXrUpzPrkx8UnWYdhJclsiYMm6x/w5+nN+8Tf/LnAgfLGQCm59qAOxU8WwHEq2vNwF6i4j+Q==} + cjs-module-lexer@2.2.0: + resolution: {integrity: sha512-4bHTS2YuzUvtoLjdy+98ykbNB5jS0+07EvFNXerqZQJ89F7DI6ET7OQo/HJuW6K0aVsKA9hj9/RVb2kQVOrPDQ==} + cliui@8.0.1: resolution: {integrity: sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==} engines: {node: '>=12'} @@ -1625,6 +1649,9 @@ packages: resolution: {integrity: sha512-Hs59xBNfUIunMFgWAbGX5cq6893IbWg4KnrjbYwX3tx0ztorVgTDA6B2sxf8ejHJ4wz8BqGUMYlnzNBer5NvGg==} engines: {node: '>= 4'} + import-in-the-middle@2.0.4: + resolution: {integrity: sha512-Al0kMpa0BqfvDnxjxGlab9vdQ0vTDs82TBKrD59X9jReUoPAzSGBb6vGDzMUMFBGyyDF03RpLT4oxGn6BpASzQ==} + import-local@3.2.0: resolution: {integrity: sha512-2SPlun1JUPWoM6t3F0dw0FkCF/jWY8kttcY4f599GLTSjh2OCuuhdTkJQsEcZzBqbXZGKMK2OqW1oZsjtf/gQA==} engines: {node: '>=8'} @@ -2063,6 +2090,9 @@ packages: resolution: {integrity: sha512-KZxYo1BUkWD2TVFLr0MQoM8vUUigWD3LlD83a/75BqC+4qE0Hb1Vo5v1FgcfaNXvfXzr+5EhQ6ing/CaBijTlw==} engines: {node: '>= 18'} + module-details-from-path@1.0.4: + resolution: {integrity: sha512-EGWKgxALGMgzvxYF1UyGTy0HXX/2vHLkw6+NvDKW2jypWbHpjQuj4UMcqQWXHERJhVGKikolT06G3bcKe4fi7w==} + ms@2.1.3: resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} @@ -2316,6 +2346,10 @@ packages: resolution: {integrity: sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==} engines: {node: '>=0.10.0'} + require-in-the-middle@8.0.1: + resolution: {integrity: sha512-QT7FVMXfWOYFbeRBF6nu+I6tr2Tf3u0q8RIEjNob/heKY/nh7drD/k7eeMFmSQgnTtCzLDcCu/XEnpW2wk4xCQ==} + engines: {node: '>=9.3.0 || >=8.10.0 <9.0.0'} + resolve-cwd@3.0.0: resolution: {integrity: sha512-OrZaX2Mb+rJCpH/6CpSqt9xFVpN++x01XnN2ie9g6P5/3xelLAkXWVADpdz1IHD/KFfEXyE6V0U01OQ3UO2rEg==} engines: {node: '>=8'} @@ -3321,9 +3355,24 @@ snapshots: '@nolyfill/is-core-module@1.0.39': {} + '@opentelemetry/api-logs@0.208.0': + dependencies: + '@opentelemetry/api': 1.9.0 + optional: true + '@opentelemetry/api@1.9.0': optional: true + '@opentelemetry/instrumentation@0.208.0(@opentelemetry/api@1.9.0)': + dependencies: + '@opentelemetry/api': 1.9.0 + '@opentelemetry/api-logs': 0.208.0 + import-in-the-middle: 2.0.4 + require-in-the-middle: 8.0.1 + transitivePeerDependencies: + - supports-color + optional: true + '@pkgr/core@0.2.9': {} '@protobufjs/aspromise@1.1.2': {} @@ -3620,6 +3669,11 @@ snapshots: abort-controller-x@0.4.3: {} + acorn-import-attributes@1.9.5(acorn@8.15.0): + dependencies: + acorn: 8.15.0 + optional: true + acorn-jsx@5.3.2(acorn@8.16.0): dependencies: acorn: 8.16.0 @@ -3887,6 +3941,9 @@ snapshots: cjs-module-lexer@1.4.3: {} + cjs-module-lexer@2.2.0: + optional: true + cliui@8.0.1: dependencies: string-width: 4.2.3 @@ -4549,6 +4606,14 @@ snapshots: ignore@7.0.5: {} + import-in-the-middle@2.0.4: + dependencies: + acorn: 8.15.0 + acorn-import-attributes: 1.9.5(acorn@8.15.0) + cjs-module-lexer: 2.2.0 + module-details-from-path: 1.0.4 + optional: true + import-local@3.2.0: dependencies: pkg-dir: 4.2.0 @@ -5186,6 +5251,9 @@ snapshots: dependencies: minipass: 7.1.3 + module-details-from-path@1.0.4: + optional: true + ms@2.1.3: {} nanoid@3.3.11: {} @@ -5452,6 +5520,14 @@ snapshots: require-directory@2.1.1: {} + require-in-the-middle@8.0.1: + dependencies: + debug: 4.4.1 + module-details-from-path: 1.0.4 + transitivePeerDependencies: + - supports-color + optional: true + resolve-cwd@3.0.0: dependencies: resolve-from: 5.0.0 diff --git a/sdks/typescript/src/clients/event/event-client.ts b/sdks/typescript/src/clients/event/event-client.ts index 4c2fcf6f89..0b8c55cae8 100644 --- a/sdks/typescript/src/clients/event/event-client.ts +++ b/sdks/typescript/src/clients/event/event-client.ts @@ -55,6 +55,10 @@ export class EventClient { this.tenantId = config.tenant_id; } + /** + * @important This method is instrumented by HatchetInstrumentor._patchPushEvent. + * Keep the signature in sync with the instrumentor wrapper. + */ push(type: string, input: T, options: PushEventOptions = {}) { const namespacedType = applyNamespace(type, this.config.namespace); @@ -78,6 +82,10 @@ export class EventClient { } } + /** + * @important This method is instrumented by HatchetInstrumentor._patchBulkPushEvent. + * Keep the signature in sync with the instrumentor wrapper. + */ bulkPush(type: string, inputs: EventWithMetadata[], options: PushEventOptions = {}) { const namespacedType = applyNamespace(type, this.config.namespace); diff --git a/sdks/typescript/src/clients/hatchet-client/client-config.ts b/sdks/typescript/src/clients/hatchet-client/client-config.ts index 0739e21bc2..f657747481 100644 --- a/sdks/typescript/src/clients/hatchet-client/client-config.ts +++ b/sdks/typescript/src/clients/hatchet-client/client-config.ts @@ -16,6 +16,22 @@ const HealthcheckConfigSchema = z.object({ port: z.number().optional().default(8001), }); +export const OpenTelemetryConfigSchema = z.object({ + /** + * List of attribute keys to exclude from spans. + * Useful for filtering sensitive or verbose data like payloads. + */ + excludedAttributes: z.array(z.string()).optional().default([]), + + /** + * If true, includes the task name in the span name for start_step_run spans. + * e.g., "hatchet.start_step_run.my_task" instead of "hatchet.start_step_run" + */ + includeTaskNameInSpanName: z.boolean().optional().default(false), +}); + +export type OpenTelemetryConfig = z.infer; + const TaskMiddlewareSchema = z .object({ before: z.any().optional(), @@ -34,6 +50,7 @@ export const ClientConfigSchema = z.object({ log_level: z.enum(['OFF', 'DEBUG', 'INFO', 'WARN', 'ERROR']).optional(), tenant_id: z.string(), namespace: z.string().optional(), + otel: OpenTelemetryConfigSchema.optional(), middleware: TaskMiddlewareSchema, cancellation_grace_period: DurationMsSchema.optional().default(1000), cancellation_warning_threshold: DurationMsSchema.optional().default(300), diff --git a/sdks/typescript/src/clients/hatchet-client/fixtures/.hatchet.yaml b/sdks/typescript/src/clients/hatchet-client/fixtures/.hatchet.yaml index 62bc0f7cf0..2d33294f12 100644 --- a/sdks/typescript/src/clients/hatchet-client/fixtures/.hatchet.yaml +++ b/sdks/typescript/src/clients/hatchet-client/fixtures/.hatchet.yaml @@ -9,3 +9,7 @@ tls_config: healthcheck: enabled: true port: 8005 +otel: + excludedAttributes: + - additional_metadata + includeTaskNameInSpanName: true diff --git a/sdks/typescript/src/legacy/legacy-client.test.ts b/sdks/typescript/src/legacy/legacy-client.test.ts index 4af8e618bb..ab12112449 100644 --- a/sdks/typescript/src/legacy/legacy-client.test.ts +++ b/sdks/typescript/src/legacy/legacy-client.test.ts @@ -29,6 +29,10 @@ describe('Client', () => { enabled: true, port: 8002, }, + otel: { + excludedAttributes: ['tenant_id', 'workflow_id'], + includeTaskNameInSpanName: true, + }, }, { credentials: ChannelCredentials.createInsecure(), @@ -55,6 +59,10 @@ describe('Client', () => { enabled: true, port: 8002, }, + otel: { + excludedAttributes: ['tenant_id', 'workflow_id'], + includeTaskNameInSpanName: true, + }, }) ); }); @@ -89,6 +97,10 @@ describe('Client', () => { enabled: false, port: 8003, }, + otel: { + excludedAttributes: ['tenant_id'], + includeTaskNameInSpanName: false, + }, }, { config_path: './fixtures/.hatchet.yaml', @@ -116,6 +128,10 @@ describe('Client', () => { enabled: false, port: 8003, }, + otel: { + excludedAttributes: ['tenant_id'], + includeTaskNameInSpanName: false, + }, }) ); }); diff --git a/sdks/typescript/src/opentelemetry/index.ts b/sdks/typescript/src/opentelemetry/index.ts new file mode 100644 index 0000000000..b55f9030fb --- /dev/null +++ b/sdks/typescript/src/opentelemetry/index.ts @@ -0,0 +1,3 @@ +export { HatchetInstrumentor } from './instrumentor'; +export type { OpenTelemetryConfig } from './types'; +export { OTelAttribute, OTelAttributeType } from '../util/opentelemetry'; diff --git a/sdks/typescript/src/opentelemetry/instrumentor.ts b/sdks/typescript/src/opentelemetry/instrumentor.ts new file mode 100644 index 0000000000..7032974c3e --- /dev/null +++ b/sdks/typescript/src/opentelemetry/instrumentor.ts @@ -0,0 +1,698 @@ +/** + * Hatchet OpenTelemetry Instrumentor + * + * This module provides automatic instrumentation for Hatchet SDK operations + * including workflow runs, event pushes, and step executions. + * + * The instrumentor follows the OpenTelemetry instrumentation pattern, + * patching module prototypes to automatically instrument all instances. + */ + +import type { Context as OtelContext, Span, Attributes } from '@opentelemetry/api'; + +import type { InstrumentationConfig } from '@opentelemetry/instrumentation'; + +import { HATCHET_VERSION } from '@hatchet/version'; +import { Action } from '@hatchet/clients/dispatcher/action-listener'; +import type { + EventClient, + PushEventOptions, + EventWithMetadata, +} from '@hatchet/clients/event/event-client'; +import type { AdminClient } from '@hatchet/v1/client/admin'; +import type { InternalWorker } from '@hatchet/v1/client/worker/worker-internal'; +import { OTelAttribute, type ActionOTelAttributeValue } from '../util/opentelemetry'; +import { parseJSON } from '../util/parse'; +import { OpenTelemetryConfig, DEFAULT_CONFIG } from './types'; +import { ScheduledWorkflows } from '../clients/rest/generated/data-contracts'; +import { ScheduleClient, CreateScheduledRunInput } from '../v1/client/features/schedules'; + +try { + require.resolve('@opentelemetry/api'); + require.resolve('@opentelemetry/instrumentation'); +} catch { + throw new Error( + 'To use HatchetInstrumentor, you must install OpenTelemetry packages: npm install @opentelemetry/api @opentelemetry/instrumentation' + ); +} + +/* eslint-disable @typescript-eslint/no-require-imports */ +const otelApi = require('@opentelemetry/api') as typeof import('@opentelemetry/api'); +const otelInstrumentation = + require('@opentelemetry/instrumentation') as typeof import('@opentelemetry/instrumentation'); +/* eslint-enable @typescript-eslint/no-require-imports */ + +const { context, propagation, SpanKind, SpanStatusCode, diag } = otelApi; + +const { + InstrumentationBase, + InstrumentationNodeModuleDefinition, + InstrumentationNodeModuleFile, + isWrapped, +} = otelInstrumentation; + +type HatchetInstrumentationConfig = OpenTelemetryConfig & InstrumentationConfig; +type Carrier = Record; + +const INSTRUMENTOR_NAME = '@hatchet-dev/typescript-sdk'; +// FIXME: refactor version check to use the new pattern introduced in #2954 +const SUPPORTED_VERSIONS = ['>=1.16.0']; + +function extractContext(carrier: Carrier | undefined | null): OtelContext { + return propagation.extract(context.active(), carrier ?? {}); +} + +function injectContext(carrier: Carrier): void { + propagation.inject(context.active(), carrier); +} + +function getActionOtelAttributes( + action: Action, + excludedAttributes: string[] = [], + workerId?: string +): Attributes { + const attributes = { + [OTelAttribute.TENANT_ID]: action.tenantId, + [OTelAttribute.WORKER_ID]: workerId, + [OTelAttribute.WORKFLOW_RUN_ID]: action.workflowRunId, + [OTelAttribute.STEP_ID]: action.stepId, + [OTelAttribute.STEP_RUN_ID]: action.stepRunId, + [OTelAttribute.RETRY_COUNT]: action.retryCount, + [OTelAttribute.PARENT_WORKFLOW_RUN_ID]: action.parentWorkflowRunId, + [OTelAttribute.CHILD_WORKFLOW_INDEX]: action.childWorkflowIndex, + [OTelAttribute.CHILD_WORKFLOW_KEY]: action.childWorkflowKey, + [OTelAttribute.ACTION_PAYLOAD]: action.actionPayload, + [OTelAttribute.WORKFLOW_NAME]: action.jobName, + [OTelAttribute.ACTION_NAME]: action.actionId, + [OTelAttribute.WORKFLOW_ID]: action.workflowId, + [OTelAttribute.WORKFLOW_VERSION_ID]: action.workflowVersionId, + } satisfies Record; + + const filtered: Attributes = {}; + for (const [key, value] of Object.entries(attributes)) { + if (!excludedAttributes.includes(key) && value !== undefined && value !== '') { + filtered[key] = value; + } + } + + return filtered; +} + +function filterAttributes( + attributes: Record, + excludedAttributes: string[] = [] +): Attributes { + const filtered: Attributes = {}; + for (const [key, value] of Object.entries(attributes)) { + if ( + !excludedAttributes.includes(key) && + value !== undefined && + value !== null && + value !== '' && + value !== '{}' && + value !== '[]' + ) { + filtered[`hatchet.${key}`] = + typeof value === 'object' ? JSON.stringify(value) : (value as string | number | boolean); + } + } + return filtered; +} + +/** + * HatchetInstrumentor provides OpenTelemetry instrumentation for Hatchet SDK v1. + * + * It automatically instruments: + * - Workflow runs (runWorkflow, runWorkflows) + * - Scheduled workflow runs (schedules.create) + * - Event pushes (push, bulkPush) + * - Step executions (handleStartStepRun, handleCancelStepRun) + * + * Traceparent context is automatically propagated through metadata. + * + * The instrumentor uses the global tracer/meter providers by default. + * Use `setTracerProvider()` and `setMeterProvider()` to configure custom providers. + */ +export class HatchetInstrumentor extends InstrumentationBase { + constructor(config: Partial = {}) { + super(INSTRUMENTOR_NAME, HATCHET_VERSION, { ...DEFAULT_CONFIG, ...config }); + } + + override setConfig(config: Partial = {}): void { + super.setConfig({ ...DEFAULT_CONFIG, ...config }); + } + + private readonly _getConfig = (): HatchetInstrumentationConfig => this.getConfig(); + + protected init(): InstanceType[] { + const eventClientModuleFile = new InstrumentationNodeModuleFile( + '@hatchet-dev/typescript-sdk/clients/event/event-client.js', + SUPPORTED_VERSIONS, + this.patchEventClient.bind(this), + this.unpatchEventClient.bind(this) + ); + + const adminClientModuleFile = new InstrumentationNodeModuleFile( + '@hatchet-dev/typescript-sdk/v1/client/admin.js', + SUPPORTED_VERSIONS, + this.patchAdminClient.bind(this), + this.unpatchAdminClient.bind(this) + ); + + const scheduleClientModuleFile = new InstrumentationNodeModuleFile( + '@hatchet-dev/typescript-sdk/v1/client/features/schedules.js', + SUPPORTED_VERSIONS, + this.patchScheduleClient.bind(this), + this.unpatchScheduleClient.bind(this) + ); + + const workerModuleFile = new InstrumentationNodeModuleFile( + '@hatchet-dev/typescript-sdk/v1/client/worker/worker-internal.js', + SUPPORTED_VERSIONS, + this.patchWorker.bind(this), + this.unpatchWorker.bind(this) + ); + + const moduleDefinition = new InstrumentationNodeModuleDefinition( + INSTRUMENTOR_NAME, + SUPPORTED_VERSIONS, + undefined, + undefined, + [eventClientModuleFile, adminClientModuleFile, workerModuleFile, scheduleClientModuleFile] + ); + + return [moduleDefinition]; + } + + private patchEventClient(moduleExports: unknown, _moduleVersion?: string): unknown { + const exports = moduleExports as { EventClient?: { prototype: EventClient } }; + if (!exports?.EventClient?.prototype) { + diag.debug('hatchet instrumentation: EventClient not found in module exports'); + return moduleExports; + } + this._patchPushEvent(exports.EventClient.prototype); + this._patchBulkPushEvent(exports.EventClient.prototype); + + return moduleExports; + } + + private unpatchEventClient(moduleExports: unknown, _moduleVersion?: string): unknown { + const exports = moduleExports as { EventClient?: { prototype: EventClient } }; + if (!exports?.EventClient?.prototype) { + return moduleExports; + } + + if (isWrapped(exports.EventClient.prototype.push)) { + this._unwrap(exports.EventClient.prototype, 'push'); + } + if (isWrapped(exports.EventClient.prototype.bulkPush)) { + this._unwrap(exports.EventClient.prototype, 'bulkPush'); + } + + return moduleExports; + } + + private _patchPushEvent(prototype: EventClient): void { + if (isWrapped(prototype.push)) { + this._unwrap(prototype, 'push'); + } + const { tracer, _getConfig: getConfig } = this; + + this._wrap(prototype, 'push', (original: EventClient['push']) => { + return function wrappedPush( + this: EventClient, + type: string, + input: T, + options: PushEventOptions = {} + ) { + const attributes = filterAttributes( + { + [OTelAttribute.EVENT_KEY]: type, + [OTelAttribute.ACTION_PAYLOAD]: JSON.stringify(input), + [OTelAttribute.ADDITIONAL_METADATA]: options.additionalMetadata + ? JSON.stringify(options.additionalMetadata) + : undefined, + [OTelAttribute.PRIORITY]: options.priority, + [OTelAttribute.FILTER_SCOPE]: options.scope, + }, + getConfig().excludedAttributes + ); + + return tracer.startActiveSpan( + 'hatchet.push_event', + { + kind: SpanKind.PRODUCER, + attributes, + }, + (span: Span) => { + const enhancedMetadata: Carrier = { ...(options.additionalMetadata ?? {}) }; + injectContext(enhancedMetadata); + + const enhancedOptions: PushEventOptions = { + ...options, + additionalMetadata: enhancedMetadata, + }; + + const result = original.call(this, type, input, enhancedOptions); + + return result.finally(() => { + span.end(); + }); + } + ); + }; + }); + } + + private _patchBulkPushEvent(prototype: EventClient): void { + if (isWrapped(prototype.bulkPush)) { + this._unwrap(prototype, 'bulkPush'); + } + const { tracer, _getConfig: getConfig } = this; + + this._wrap(prototype, 'bulkPush', (original: EventClient['bulkPush']) => { + return function wrappedBulkPush( + this: EventClient, + type: string, + inputs: EventWithMetadata[], + options: PushEventOptions = {} + ) { + const attributes = filterAttributes( + { + [OTelAttribute.EVENT_KEY]: type, + [OTelAttribute.ACTION_PAYLOAD]: JSON.stringify(inputs), + [OTelAttribute.ADDITIONAL_METADATA]: options.additionalMetadata + ? JSON.stringify(options.additionalMetadata) + : undefined, + [OTelAttribute.PRIORITY]: options.priority, + }, + getConfig().excludedAttributes + ); + + return tracer.startActiveSpan( + 'hatchet.bulk_push_event', + { + kind: SpanKind.PRODUCER, + attributes, + }, + (span: Span) => { + const enhancedInputs = inputs.map((input) => { + const enhancedMetadata: Carrier = { + ...((input.additionalMetadata as Carrier) ?? {}), + }; + injectContext(enhancedMetadata); + return { + ...input, + additionalMetadata: enhancedMetadata, + }; + }); + + const result = original.call(this, type, enhancedInputs, options); + + return result.finally(() => { + span.end(); + }); + } + ); + }; + }); + } + + private patchAdminClient(moduleExports: unknown, _moduleVersion?: string): unknown { + const exports = moduleExports as { AdminClient?: { prototype: AdminClient } }; + if (!exports?.AdminClient?.prototype) { + diag.debug('hatchet instrumentation: AdminClient not found in module exports'); + return moduleExports; + } + + this._patchRunWorkflow(exports.AdminClient.prototype); + this._patchRunWorkflows(exports.AdminClient.prototype); + + return moduleExports; + } + + private unpatchAdminClient(moduleExports: unknown, _moduleVersion?: string): unknown { + const exports = moduleExports as { AdminClient?: { prototype: AdminClient } }; + if (!exports?.AdminClient?.prototype) { + return moduleExports; + } + + if (isWrapped(exports.AdminClient.prototype.runWorkflow)) { + this._unwrap(exports.AdminClient.prototype, 'runWorkflow'); + } + if (isWrapped(exports.AdminClient.prototype.runWorkflows)) { + this._unwrap(exports.AdminClient.prototype, 'runWorkflows'); + } + + return moduleExports; + } + + private _patchRunWorkflow(prototype: AdminClient): void { + if (isWrapped(prototype.runWorkflow)) { + this._unwrap(prototype, 'runWorkflow'); + } + const { tracer, _getConfig: getConfig } = this; + + this._wrap(prototype, 'runWorkflow', (original: AdminClient['runWorkflow']) => { + return async function wrappedRunWorkflow( + this: AdminClient, + workflowName: string, + input: unknown, + options?: { + parentId?: string; + parentStepRunId?: string; + childIndex?: number; + childKey?: string; + additionalMetadata?: Record; + desiredWorkerId?: string; + priority?: number; + } + ) { + const attributes = filterAttributes( + { + [OTelAttribute.WORKFLOW_NAME]: workflowName, + [OTelAttribute.ACTION_PAYLOAD]: JSON.stringify(input), + [OTelAttribute.PARENT_ID]: options?.parentId, + [OTelAttribute.PARENT_STEP_RUN_ID]: options?.parentStepRunId, + [OTelAttribute.CHILD_INDEX]: options?.childIndex, + [OTelAttribute.CHILD_KEY]: options?.childKey, + [OTelAttribute.ADDITIONAL_METADATA]: options?.additionalMetadata + ? JSON.stringify(options.additionalMetadata) + : undefined, + [OTelAttribute.PRIORITY]: options?.priority, + [OTelAttribute.DESIRED_WORKER_ID]: options?.desiredWorkerId, + }, + getConfig().excludedAttributes + ); + + return tracer.startActiveSpan( + 'hatchet.run_workflow', + { + kind: SpanKind.PRODUCER, + attributes, + }, + (span: Span) => { + const enhancedMetadata: Carrier = { ...(options?.additionalMetadata ?? {}) }; + injectContext(enhancedMetadata); + + const enhancedOptions = { + ...options, + additionalMetadata: enhancedMetadata, + }; + + return original + .call(this, workflowName, input, enhancedOptions) + .catch((error: Error) => { + span.recordException(error); + span.setStatus({ code: SpanStatusCode.ERROR, message: error?.message }); + throw error; + }) + .finally(() => { + span.end(); + }); + } + ); + } as AdminClient['runWorkflow']; + }); + } + + private _patchRunWorkflows(prototype: AdminClient): void { + if (isWrapped(prototype.runWorkflows)) { + this._unwrap(prototype, 'runWorkflows'); + } + const { tracer, _getConfig: getConfig } = this; + + this._wrap(prototype, 'runWorkflows', (original: AdminClient['runWorkflows']) => { + return async function wrappedRunWorkflows( + this: AdminClient, + workflowRuns: Array<{ + workflowName: string; + input: unknown; + options?: { + parentId?: string; + parentStepRunId?: string; + childIndex?: number; + childKey?: string; + additionalMetadata?: Record; + desiredWorkerId?: string; + priority?: number; + }; + }>, + batchSize?: number + ) { + const attributes = filterAttributes( + { + [OTelAttribute.WORKFLOW_NAME]: JSON.stringify(workflowRuns.map((r) => r.workflowName)), + [OTelAttribute.ACTION_PAYLOAD]: JSON.stringify(workflowRuns), + }, + getConfig().excludedAttributes + ); + + return tracer.startActiveSpan( + 'hatchet.run_workflows', + { + kind: SpanKind.PRODUCER, + attributes, + }, + (span: Span) => { + const enhancedWorkflowRuns = workflowRuns.map((run) => { + const enhancedMetadata: Carrier = { ...(run.options?.additionalMetadata ?? {}) }; + injectContext(enhancedMetadata); + return { + ...run, + options: { + ...run.options, + additionalMetadata: enhancedMetadata, + }, + }; + }); + + return original + .call(this, enhancedWorkflowRuns, batchSize) + .catch((error: Error) => { + span.recordException(error); + span.setStatus({ code: SpanStatusCode.ERROR, message: error?.message }); + throw error; + }) + .finally(() => { + span.end(); + }); + } + ); + } as AdminClient['runWorkflows']; + }); + } + + private patchWorker(moduleExports: unknown, _moduleVersion?: string): unknown { + const exports = moduleExports as { InternalWorker?: { prototype: InternalWorker } }; + if (!exports?.InternalWorker?.prototype) { + diag.debug('hatchet instrumentation: InternalWorker not found in module exports'); + return moduleExports; + } + + this._patchHandleStartStepRun(exports.InternalWorker.prototype); + this._patchHandleCancelStepRun(exports.InternalWorker.prototype); + + return moduleExports; + } + + private unpatchWorker(moduleExports: unknown, _moduleVersion?: string): unknown { + const exports = moduleExports as { InternalWorker?: { prototype: InternalWorker } }; + if (!exports?.InternalWorker?.prototype) { + return moduleExports; + } + + if (isWrapped(exports.InternalWorker.prototype.handleStartStepRun)) { + this._unwrap(exports.InternalWorker.prototype, 'handleStartStepRun'); + } + if (isWrapped(exports.InternalWorker.prototype.handleCancelStepRun)) { + this._unwrap(exports.InternalWorker.prototype, 'handleCancelStepRun'); + } + + return moduleExports; + } + + // IMPORTANT: Keep this wrapper's signature in sync with InternalWorker.handleStartStepRun + private _patchHandleStartStepRun(prototype: InternalWorker): void { + if (isWrapped(prototype.handleStartStepRun)) { + this._unwrap(prototype, 'handleStartStepRun'); + } + const { tracer, _getConfig: getConfig } = this; + + this._wrap( + prototype, + 'handleStartStepRun', + (original: InternalWorker['handleStartStepRun']) => { + return async function wrappedHandleStartStepRun( + this: InternalWorker, + action: Action + ): Promise { + const additionalMetadata = action.additionalMetadata + ? parseJSON(action.additionalMetadata) + : undefined; + const parentContext = extractContext(additionalMetadata); + const attributes = getActionOtelAttributes( + action, + getConfig().excludedAttributes, + this.workerId + ); + + let spanName = 'hatchet.start_step_run'; + if (getConfig().includeTaskNameInSpanName) { + spanName += `.${action.actionId}`; + } + + return tracer.startActiveSpan( + spanName, + { + kind: SpanKind.CONSUMER, + attributes, + }, + parentContext, + (span: Span) => { + return original + .call(this, action) + .then((taskError: Error | undefined) => { + if (taskError instanceof Error) { + span.recordException(taskError); + span.setStatus({ code: SpanStatusCode.ERROR, message: taskError.message }); + } + return taskError; + }) + .finally(() => { + span.end(); + }); + } + ); + }; + } + ); + } + + private _patchHandleCancelStepRun(prototype: InternalWorker): void { + if (isWrapped(prototype.handleCancelStepRun)) { + this._unwrap(prototype, 'handleCancelStepRun'); + } + const { tracer } = this; + + this._wrap( + prototype, + 'handleCancelStepRun', + (original: InternalWorker['handleCancelStepRun']) => { + return async function wrappedHandleCancelStepRun( + this: InternalWorker, + action: Action + ): Promise { + const attributes: Attributes = { + [`hatchet.${OTelAttribute.STEP_RUN_ID}`]: action.stepRunId, + }; + + return tracer.startActiveSpan( + 'hatchet.cancel_step_run', + { + kind: SpanKind.CONSUMER, + attributes, + }, + (span: Span) => { + const result = original.call(this, action); + + return result.finally(() => { + span.end(); + }); + } + ); + }; + } + ); + } + + private patchScheduleClient(moduleExports: unknown, _moduleVersion?: string): unknown { + const exports = moduleExports as { ScheduleClient?: { prototype: ScheduleClient } }; + if (!exports?.ScheduleClient?.prototype) { + diag.debug('hatchet instrumentation: ScheduleClient not found in module exports'); + return moduleExports; + } + + this._patchScheduleCreate(exports.ScheduleClient.prototype); + + return moduleExports; + } + + private unpatchScheduleClient(moduleExports: unknown, _moduleVersion?: string): unknown { + const exports = moduleExports as { ScheduleClient?: { prototype: ScheduleClient } }; + if (!exports?.ScheduleClient?.prototype) { + return moduleExports; + } + + if (isWrapped(exports.ScheduleClient.prototype.create)) { + this._unwrap(exports.ScheduleClient.prototype, 'create'); + } + + return moduleExports; + } + + // IMPORTANT: Keep this wrapper's signature in sync with ScheduleClient.create + private _patchScheduleCreate(prototype: ScheduleClient): void { + if (isWrapped(prototype.create)) { + this._unwrap(prototype, 'create'); + } + const { tracer, _getConfig: getConfig } = this; + + this._wrap(prototype, 'create', (original: ScheduleClient['create']) => { + return async function wrappedCreate( + this: ScheduleClient, + workflow: string, + input: CreateScheduledRunInput + ): Promise { + const triggerAtIso = + input.triggerAt instanceof Date + ? input.triggerAt.toISOString() + : new Date(input.triggerAt).toISOString(); + + const attributes = filterAttributes( + { + [OTelAttribute.WORKFLOW_NAME]: workflow, + [OTelAttribute.RUN_AT_TIMESTAMPS]: JSON.stringify([triggerAtIso]), + [OTelAttribute.ACTION_PAYLOAD]: JSON.stringify(input.input), + [OTelAttribute.ADDITIONAL_METADATA]: input.additionalMetadata + ? JSON.stringify(input.additionalMetadata) + : undefined, + [OTelAttribute.PRIORITY]: input.priority, + }, + getConfig().excludedAttributes + ); + + return tracer.startActiveSpan( + 'hatchet.schedule_workflow', + { + kind: SpanKind.PRODUCER, + attributes, + }, + (span: Span) => { + // Inject traceparent into additionalMetadata for context propagation + const enhancedMetadata: Carrier = { ...(input.additionalMetadata ?? {}) }; + injectContext(enhancedMetadata); + + const enhancedInput = { + ...input, + additionalMetadata: enhancedMetadata, + }; + + return original + .call(this, workflow, enhancedInput as CreateScheduledRunInput) + .catch((error: Error) => { + span.recordException(error); + span.setStatus({ code: SpanStatusCode.ERROR, message: error?.message }); + throw error; + }) + .finally(() => { + span.end(); + }); + } + ); + }; + }); + } +} + +export default HatchetInstrumentor; diff --git a/sdks/typescript/src/opentelemetry/types.ts b/sdks/typescript/src/opentelemetry/types.ts new file mode 100644 index 0000000000..86f55768ff --- /dev/null +++ b/sdks/typescript/src/opentelemetry/types.ts @@ -0,0 +1,6 @@ +export type { OpenTelemetryConfig } from '@hatchet/clients/hatchet-client/client-config'; + +export const DEFAULT_CONFIG = { + excludedAttributes: [] as string[], + includeTaskNameInSpanName: false, +}; diff --git a/sdks/typescript/src/util/config-loader/config-loader.test.ts b/sdks/typescript/src/util/config-loader/config-loader.test.ts index 13332509b1..42b663ddb9 100644 --- a/sdks/typescript/src/util/config-loader/config-loader.test.ts +++ b/sdks/typescript/src/util/config-loader/config-loader.test.ts @@ -2,6 +2,9 @@ import { ConfigLoader } from './config-loader'; describe('ConfigLoader', () => { beforeEach(() => { + // Clear env vars that might leak from other tests + delete process.env.HATCHET_CLIENT_TLS_STRATEGY; + process.env.HATCHET_CLIENT_TOKEN = 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJncnBjX2Jyb2FkY2FzdF9hZGRyZXNzIjoiMTI3LjAuMC4xOjgwODAiLCJzZXJ2ZXJfdXJsIjoiaHR0cDovL2xvY2FsaG9zdDo4MDgwIiwic3ViIjoiNzA3ZDA4NTUtODBhYi00ZTFmLWExNTYtZjFjNDU0NmNiZjUyIn0K.abcdef'; process.env.HATCHET_CLIENT_TLS_STRATEGY = 'tls'; @@ -34,6 +37,10 @@ describe('ConfigLoader', () => { enabled: true, port: 8001, }, + otel: { + excludedAttributes: [], + includeTaskNameInSpanName: false, + }, }); }); @@ -86,6 +93,10 @@ describe('ConfigLoader', () => { enabled: true, port: 8002, }, + otel: { + excludedAttributes: ['additional_metadata'], + includeTaskNameInSpanName: true, + }, }); }); diff --git a/sdks/typescript/src/util/config-loader/config-loader.ts b/sdks/typescript/src/util/config-loader/config-loader.ts index 5e166d287e..e7b97cfe31 100644 --- a/sdks/typescript/src/util/config-loader/config-loader.ts +++ b/sdks/typescript/src/util/config-loader/config-loader.ts @@ -19,7 +19,9 @@ type EnvVars = | 'HATCHET_CLIENT_LOG_LEVEL' | 'HATCHET_CLIENT_NAMESPACE' | 'HATCHET_CLIENT_WORKER_HEALTHCHECK_ENABLED' - | 'HATCHET_CLIENT_WORKER_HEALTHCHECK_PORT'; + | 'HATCHET_CLIENT_WORKER_HEALTHCHECK_PORT' + | 'HATCHET_CLIENT_OPENTELEMETRY_EXCLUDED_ATTRIBUTES' + | 'HATCHET_CLIENT_OPENTELEMETRY_INCLUDE_TASK_NAME_IN_SPAN_NAME'; type TLSStrategy = 'tls' | 'mtls'; @@ -94,6 +96,15 @@ export class ConfigLoader { namespace = `${namespace}_`; } + const otelConfig = override?.otel ?? + yaml?.otel ?? { + excludedAttributes: this.parseJsonArray( + this.env('HATCHET_CLIENT_OPENTELEMETRY_EXCLUDED_ATTRIBUTES') || '[]' + ), + includeTaskNameInSpanName: + this.env('HATCHET_CLIENT_OPENTELEMETRY_INCLUDE_TASK_NAME_IN_SPAN_NAME') === 'true', + }; + return { token: override?.token ?? yaml?.token ?? this.env('HATCHET_CLIENT_TOKEN'), host_port: grpcBroadcastAddress, @@ -107,9 +118,19 @@ export class ConfigLoader { 'INFO', tenant_id: tenantId, namespace: namespace ? `${namespace}`.toLowerCase() : '', + otel: otelConfig, }; } + private static parseJsonArray(value: string): string[] { + try { + const parsed = JSON.parse(value); + return Array.isArray(parsed) ? parsed : []; + } catch { + return []; + } + } + static get default_yaml_config_path() { return p.join(process.cwd(), DEFAULT_CONFIG_FILE); } diff --git a/sdks/typescript/src/util/config-loader/fixtures/.hatchet.yaml b/sdks/typescript/src/util/config-loader/fixtures/.hatchet.yaml index 2886047832..b267f34d46 100644 --- a/sdks/typescript/src/util/config-loader/fixtures/.hatchet.yaml +++ b/sdks/typescript/src/util/config-loader/fixtures/.hatchet.yaml @@ -9,3 +9,7 @@ tls_config: healthcheck: enabled: true port: 8002 +otel: + excludedAttributes: + - additional_metadata + includeTaskNameInSpanName: true diff --git a/sdks/typescript/src/util/opentelemetry.ts b/sdks/typescript/src/util/opentelemetry.ts new file mode 100644 index 0000000000..b738a0e3c3 --- /dev/null +++ b/sdks/typescript/src/util/opentelemetry.ts @@ -0,0 +1,59 @@ +export const OTelAttribute = { + // Shared + NAMESPACE: 'namespace', + ADDITIONAL_METADATA: 'additional_metadata', + WORKFLOW_NAME: 'workflow_name', + + PRIORITY: 'priority', + ACTION_PAYLOAD: 'payload', + + // Action + ACTION_NAME: 'action_name', + CHILD_WORKFLOW_INDEX: 'child_workflow_index', + CHILD_WORKFLOW_KEY: 'child_workflow_key', + PARENT_WORKFLOW_RUN_ID: 'parent_workflow_run_id', + RETRY_COUNT: 'retry_count', + STEP_ID: 'step_id', + STEP_RUN_ID: 'step_run_id', + TENANT_ID: 'tenant_id', + WORKER_ID: 'worker_id', + WORKFLOW_ID: 'workflow_id', + WORKFLOW_RUN_ID: 'workflow_run_id', + WORKFLOW_VERSION_ID: 'workflow_version_id', + + // Push Event + EVENT_KEY: 'event_key', + FILTER_SCOPE: 'scope', + + // Trigger Workflow + PARENT_ID: 'parent_id', + PARENT_STEP_RUN_ID: 'parent_step_run_id', + CHILD_INDEX: 'child_index', + CHILD_KEY: 'child_key', + DESIRED_WORKER_ID: 'desired_worker_id', + STICKY: 'sticky', + KEY: 'key', + + // Schedule Workflow + RUN_AT_TIMESTAMPS: 'run_at_timestamps', +} as const; + +export type OTelAttributeType = (typeof OTelAttribute)[keyof typeof OTelAttribute]; + +type ActionOTelAttributeKey = + | 'TENANT_ID' + | 'WORKER_ID' + | 'WORKFLOW_RUN_ID' + | 'STEP_ID' + | 'STEP_RUN_ID' + | 'RETRY_COUNT' + | 'PARENT_WORKFLOW_RUN_ID' + | 'CHILD_WORKFLOW_INDEX' + | 'CHILD_WORKFLOW_KEY' + | 'ACTION_PAYLOAD' + | 'WORKFLOW_NAME' + | 'ACTION_NAME' + | 'WORKFLOW_ID' + | 'WORKFLOW_VERSION_ID'; + +export type ActionOTelAttributeValue = (typeof OTelAttribute)[ActionOTelAttributeKey]; diff --git a/sdks/typescript/src/v1/client/admin.ts b/sdks/typescript/src/v1/client/admin.ts index 1c6ffc1d9b..c120179bd0 100644 --- a/sdks/typescript/src/v1/client/admin.ts +++ b/sdks/typescript/src/v1/client/admin.ts @@ -107,6 +107,9 @@ export class AdminClient { * @param input an object containing the input to the workflow * @param options an object containing the options to run the workflow * @returns the ID of the new workflow run + * + * @important This method is instrumented by HatchetInstrumentor._patchRunWorkflow. + * Keep the signature in sync with the instrumentor wrapper. */ async runWorkflow( workflowName: string, @@ -187,6 +190,9 @@ export class AdminClient { * Order is preserved in the response. * @param workflowRuns an array of objects containing the workflow name, input, and options for each workflow run * @returns an array of workflow run references + * + * @important This method is instrumented by HatchetInstrumentor._patchRunWorkflows. + * Keep the signature in sync with the instrumentor wrapper. */ async runWorkflows( workflowRuns: Array<{ diff --git a/sdks/typescript/src/v1/client/features/schedules.ts b/sdks/typescript/src/v1/client/features/schedules.ts index 6742d6b9ce..2baa971678 100644 --- a/sdks/typescript/src/v1/client/features/schedules.ts +++ b/sdks/typescript/src/v1/client/features/schedules.ts @@ -77,6 +77,9 @@ export class ScheduleClient { * @param scheduledRun - The input data for creating the Scheduled Run. * @returns A promise that resolves to the created ScheduledWorkflows object. * @throws Will throw an error if the input is invalid or the API call fails. + * + * @important This method is instrumented by HatchetInstrumentor._patchScheduleCreate. + * Keep the signature in sync with the instrumentor wrapper. */ async create( workflow: string | Workflow, diff --git a/sdks/typescript/src/v1/client/worker/worker-internal.ts b/sdks/typescript/src/v1/client/worker/worker-internal.ts index 985c17b725..da12ec6798 100644 --- a/sdks/typescript/src/v1/client/worker/worker-internal.ts +++ b/sdks/typescript/src/v1/client/worker/worker-internal.ts @@ -416,7 +416,11 @@ export class InternalWorker { this.registerActions(workflow); } - async handleStartStepRun(action: Action) { + /** + * @important This method is instrumented by HatchetInstrumentor._patchHandleStartStepRun. + * Keep the signature in sync with the instrumentor wrapper. + */ + async handleStartStepRun(action: Action): Promise { const { actionId, taskRunExternalId, taskName } = action; try { @@ -429,7 +433,7 @@ export class InternalWorker { if (!step) { this.logger.error(`Registered actions: '${Object.keys(this.action_registry).join(', ')}'`); this.logger.error(`Could not find step '${actionId}'`); - return; + return undefined; } const run = async () => { @@ -561,14 +565,15 @@ export class InternalWorker { } }; - const future = new HatchetPromise( + const future = new HatchetPromise( (async () => { let result: any; try { result = await run(); } catch (e: any) { await failure(e); - return; + // Return error for OTel instrumentor to capture + return e; } // Postcheck: user code may swallow AbortError; don't report completion after cancellation. @@ -584,6 +589,7 @@ export class InternalWorker { throwIfAborted(context.abortController.signal); await success(result); + return undefined; })() ); this.futures[taskRunExternalId] = future; @@ -601,7 +607,7 @@ export class InternalWorker { }); try { - await future.promise; + return await future.promise; } catch (e: any) { const message = e?.message || String(e); // TODO is this cased correctly... @@ -612,9 +618,11 @@ export class InternalWorker { e ); } + return e instanceof Error ? e : new Error(String(e)); } } catch (e: any) { this.logger.error('Could not send action event (outer): ', e); + return e instanceof Error ? e : new Error(String(e)); } } @@ -753,6 +761,10 @@ export class InternalWorker { }; } + /** + * @important This method is instrumented by HatchetInstrumentor._patchHandleCancelStepRun. + * Keep the signature in sync with the instrumentor wrapper. + */ async handleCancelStepRun(action: Action) { const { taskRunExternalId, taskName } = action; From e2cdadfbbd32b67b7fb9dba9bef749953e466b4e Mon Sep 17 00:00:00 2001 From: Mohammed Nafees Date: Mon, 9 Mar 2026 22:07:58 +0100 Subject: [PATCH 09/17] many many random spans --- examples/go/opentelemetry-propagation/main.go | 36 +++- examples/typescript/guides/package.json | 2 +- .../opentelemetry-propagation/main.go | 36 +++- sdks/typescript/package.json | 2 + sdks/typescript/pnpm-lock.yaml | 189 ++++++++++++++++++ .../clients/hatchet-client/client-config.ts | 4 +- .../src/opentelemetry/hatchet-exporter.ts | 75 +++++++ .../src/opentelemetry/instrumentor.ts | 93 +++++++-- 8 files changed, 414 insertions(+), 23 deletions(-) create mode 100644 sdks/typescript/src/opentelemetry/hatchet-exporter.ts diff --git a/examples/go/opentelemetry-propagation/main.go b/examples/go/opentelemetry-propagation/main.go index f162de1ddb..e431e9682b 100644 --- a/examples/go/opentelemetry-propagation/main.go +++ b/examples/go/opentelemetry-propagation/main.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "log" + "math/rand" "time" "go.opentelemetry.io/otel" @@ -48,16 +49,43 @@ func main() { tracer := otel.Tracer("otel-propagation-example") + // generateSpanTree creates a nested span subtree, returning how many spans were created. + var generateSpanTree func(ctx context.Context, count *int, limit int, depth int, prefix string) + generateSpanTree = func(ctx context.Context, count *int, limit int, depth int, prefix string) { + numChildren := 1 + rand.Intn(5) // 1-5 children at this level + for i := range numChildren { + if *count >= limit { + return + } + name := fmt.Sprintf("%s.%d", prefix, i) + childCtx, span := tracer.Start(ctx, name) + time.Sleep(time.Duration(1+rand.Intn(3)) * time.Millisecond) + *count++ + + // Recurse deeper with probability that decreases with depth + if *count < limit && depth < 8 && rand.Float64() > float64(depth)*0.12 { + generateSpanTree(childCtx, count, limit, depth+1, name) + } + + span.End() + } + } + // Child task — a standalone task that will be spawned by the parent. childTask := client.NewStandaloneTask( "otel-child-task", func(ctx hatchet.Context, input ChildInput) (ChildOutput, error) { - _, span := tracer.Start(ctx.GetContext(), "child.process") - time.Sleep(50 * time.Millisecond) - span.End() + target := 200 + rand.Intn(101) // 200-300 spans + count := 0 + // Keep spawning top-level subtrees until we hit the target. + round := 0 + for count < target { + generateSpanTree(ctx.GetContext(), &count, target, 0, fmt.Sprintf("child.r%d", round)) + round++ + } return ChildOutput{ - Message: fmt.Sprintf("Hello from child: %s", input.Greeting), + Message: fmt.Sprintf("Hello from child: %s (generated %d spans)", input.Greeting, count), }, nil }, ) diff --git a/examples/typescript/guides/package.json b/examples/typescript/guides/package.json index 969b847637..2e6d112724 100644 --- a/examples/typescript/guides/package.json +++ b/examples/typescript/guides/package.json @@ -8,7 +8,7 @@ "lint:fix": "eslint . --fix" }, "dependencies": { - "@hatchet-dev/typescript-sdk": "^1.15.2", + "@hatchet-dev/typescript-sdk": "^1.16.0", "@anthropic-ai/sdk": "^0.32.1", "@ai-sdk/openai": "^1.0.0", "@browserbasehq/sdk": "^2.7.0", diff --git a/sdks/go/examples/opentelemetry-propagation/main.go b/sdks/go/examples/opentelemetry-propagation/main.go index f162de1ddb..e431e9682b 100644 --- a/sdks/go/examples/opentelemetry-propagation/main.go +++ b/sdks/go/examples/opentelemetry-propagation/main.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "log" + "math/rand" "time" "go.opentelemetry.io/otel" @@ -48,16 +49,43 @@ func main() { tracer := otel.Tracer("otel-propagation-example") + // generateSpanTree creates a nested span subtree, returning how many spans were created. + var generateSpanTree func(ctx context.Context, count *int, limit int, depth int, prefix string) + generateSpanTree = func(ctx context.Context, count *int, limit int, depth int, prefix string) { + numChildren := 1 + rand.Intn(5) // 1-5 children at this level + for i := range numChildren { + if *count >= limit { + return + } + name := fmt.Sprintf("%s.%d", prefix, i) + childCtx, span := tracer.Start(ctx, name) + time.Sleep(time.Duration(1+rand.Intn(3)) * time.Millisecond) + *count++ + + // Recurse deeper with probability that decreases with depth + if *count < limit && depth < 8 && rand.Float64() > float64(depth)*0.12 { + generateSpanTree(childCtx, count, limit, depth+1, name) + } + + span.End() + } + } + // Child task — a standalone task that will be spawned by the parent. childTask := client.NewStandaloneTask( "otel-child-task", func(ctx hatchet.Context, input ChildInput) (ChildOutput, error) { - _, span := tracer.Start(ctx.GetContext(), "child.process") - time.Sleep(50 * time.Millisecond) - span.End() + target := 200 + rand.Intn(101) // 200-300 spans + count := 0 + // Keep spawning top-level subtrees until we hit the target. + round := 0 + for count < target { + generateSpanTree(ctx.GetContext(), &count, target, 0, fmt.Sprintf("child.r%d", round)) + round++ + } return ChildOutput{ - Message: fmt.Sprintf("Hello from child: %s", input.Greeting), + Message: fmt.Sprintf("Hello from child: %s (generated %d spans)", input.Greeting, count), }, nil }, ) diff --git a/sdks/typescript/package.json b/sdks/typescript/package.json index e1cd20adcc..7941fdda50 100644 --- a/sdks/typescript/package.json +++ b/sdks/typescript/package.json @@ -86,7 +86,9 @@ }, "optionalDependencies": { "@opentelemetry/api": "^1.9.0", + "@opentelemetry/exporter-trace-otlp-grpc": "^0.208.0", "@opentelemetry/instrumentation": "^0.208.0", + "@opentelemetry/sdk-trace-base": "^1.30.1", "prom-client": "^15.1.3" }, "packageManager": "pnpm@10.16.1", diff --git a/sdks/typescript/pnpm-lock.yaml b/sdks/typescript/pnpm-lock.yaml index 6d0af29366..3835c0a981 100644 --- a/sdks/typescript/pnpm-lock.yaml +++ b/sdks/typescript/pnpm-lock.yaml @@ -134,9 +134,15 @@ importers: '@opentelemetry/api': specifier: ^1.9.0 version: 1.9.0 + '@opentelemetry/exporter-trace-otlp-grpc': + specifier: ^0.208.0 + version: 0.208.0(@opentelemetry/api@1.9.0) '@opentelemetry/instrumentation': specifier: ^0.208.0 version: 0.208.0(@opentelemetry/api@1.9.0) + '@opentelemetry/sdk-trace-base': + specifier: ^1.30.1 + version: 1.30.1(@opentelemetry/api@1.9.0) prom-client: specifier: ^15.1.3 version: 15.1.3 @@ -524,12 +530,92 @@ packages: resolution: {integrity: sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==} engines: {node: '>=8.0.0'} + '@opentelemetry/core@1.30.1': + resolution: {integrity: sha512-OOCM2C/QIURhJMuKaekP3TRBxBKxG/TWWA0TL2J6nXUtDnuCtccy49LUJF8xPFXMX+0LMcxFpCo8M9cGY1W6rQ==} + engines: {node: '>=14'} + peerDependencies: + '@opentelemetry/api': '>=1.0.0 <1.10.0' + + '@opentelemetry/core@2.2.0': + resolution: {integrity: sha512-FuabnnUm8LflnieVxs6eP7Z383hgQU4W1e3KJS6aOG3RxWxcHyBxH8fDMHNgu/gFx/M2jvTOW/4/PHhLz6bjWw==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': '>=1.0.0 <1.10.0' + + '@opentelemetry/exporter-trace-otlp-grpc@0.208.0': + resolution: {integrity: sha512-E/eNdcqVUTAT7BC+e8VOw/krqb+5rjzYkztMZ/o+eyJl+iEY6PfczPXpwWuICwvsm0SIhBoh9hmYED5Vh5RwIw==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + '@opentelemetry/instrumentation@0.208.0': resolution: {integrity: sha512-Eju0L4qWcQS+oXxi6pgh7zvE2byogAkcsVv0OjHF/97iOz1N/aKE6etSGowYkie+YA1uo6DNwdSxaaNnLvcRlA==} engines: {node: ^18.19.0 || >=20.6.0} peerDependencies: '@opentelemetry/api': ^1.3.0 + '@opentelemetry/otlp-exporter-base@0.208.0': + resolution: {integrity: sha512-gMd39gIfVb2OgxldxUtOwGJYSH8P1kVFFlJLuut32L6KgUC4gl1dMhn+YC2mGn0bDOiQYSk/uHOdSjuKp58vvA==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + + '@opentelemetry/otlp-grpc-exporter-base@0.208.0': + resolution: {integrity: sha512-fGvAg3zb8fC0oJAzfz7PQppADI2HYB7TSt/XoCaBJFi1mSquNUjtHXEoviMgObLAa1NRIgOC1lsV1OUKi+9+lQ==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + + '@opentelemetry/otlp-transformer@0.208.0': + resolution: {integrity: sha512-DCFPY8C6lAQHUNkzcNT9R+qYExvsk6C5Bto2pbNxgicpcSWbe2WHShLxkOxIdNcBiYPdVHv/e7vH7K6TI+C+fQ==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + + '@opentelemetry/resources@1.30.1': + resolution: {integrity: sha512-5UxZqiAgLYGFjS4s9qm5mBVo433u+dSPUFWVWXmLAD4wB65oMCoXaJP1KJa9DIYYMeHu3z4BZcStG3LC593cWA==} + engines: {node: '>=14'} + peerDependencies: + '@opentelemetry/api': '>=1.0.0 <1.10.0' + + '@opentelemetry/resources@2.2.0': + resolution: {integrity: sha512-1pNQf/JazQTMA0BiO5NINUzH0cbLbbl7mntLa4aJNmCCXSj0q03T5ZXXL0zw4G55TjdL9Tz32cznGClf+8zr5A==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': '>=1.3.0 <1.10.0' + + '@opentelemetry/sdk-logs@0.208.0': + resolution: {integrity: sha512-QlAyL1jRpOeaqx7/leG1vJMp84g0xKP6gJmfELBpnI4O/9xPX+Hu5m1POk9Kl+veNkyth5t19hRlN6tNY1sjbA==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': '>=1.4.0 <1.10.0' + + '@opentelemetry/sdk-metrics@2.2.0': + resolution: {integrity: sha512-G5KYP6+VJMZzpGipQw7Giif48h6SGQ2PFKEYCybeXJsOCB4fp8azqMAAzE5lnnHK3ZVwYQrgmFbsUJO/zOnwGw==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': '>=1.9.0 <1.10.0' + + '@opentelemetry/sdk-trace-base@1.30.1': + resolution: {integrity: sha512-jVPgBbH1gCy2Lb7X0AVQ8XAfgg0pJ4nvl8/IiQA6nxOsPvS+0zMJaFSs2ltXe0J6C8dqjcnpyqINDJmU30+uOg==} + engines: {node: '>=14'} + peerDependencies: + '@opentelemetry/api': '>=1.0.0 <1.10.0' + + '@opentelemetry/sdk-trace-base@2.2.0': + resolution: {integrity: sha512-xWQgL0Bmctsalg6PaXExmzdedSp3gyKV8mQBwK/j9VGdCDu2fmXIb2gAehBKbkXCpJ4HPkgv3QfoJWRT4dHWbw==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': '>=1.3.0 <1.10.0' + + '@opentelemetry/semantic-conventions@1.28.0': + resolution: {integrity: sha512-lp4qAiMTD4sNWW4DbKLBkfiMZ4jbAboJIGOQr5DvciMRI494OapieI9qiODpOt0XBr1LjIDy1xAGAnVs5supTA==} + engines: {node: '>=14'} + + '@opentelemetry/semantic-conventions@1.40.0': + resolution: {integrity: sha512-cifvXDhcqMwwTlTK04GBNeIe7yyo28Mfby85QXFe1Yk8nmi36Ab/5UQwptOx84SsoGNRg+EVSjwzfSZMy6pmlw==} + engines: {node: '>=14'} + '@pkgr/core@0.2.9': resolution: {integrity: sha512-QNqXyfVS2wm9hweSYD2O7F0G06uurj9kZ96TRQE5Y9hU7+tgdZwIkbAKc5Ocy1HxEY2kuDQa6cQ1WRs/O5LFKA==} engines: {node: ^12.20.0 || ^14.18.0 || >=16.0.0} @@ -3363,6 +3449,30 @@ snapshots: '@opentelemetry/api@1.9.0': optional: true + '@opentelemetry/core@1.30.1(@opentelemetry/api@1.9.0)': + dependencies: + '@opentelemetry/api': 1.9.0 + '@opentelemetry/semantic-conventions': 1.28.0 + optional: true + + '@opentelemetry/core@2.2.0(@opentelemetry/api@1.9.0)': + dependencies: + '@opentelemetry/api': 1.9.0 + '@opentelemetry/semantic-conventions': 1.40.0 + optional: true + + '@opentelemetry/exporter-trace-otlp-grpc@0.208.0(@opentelemetry/api@1.9.0)': + dependencies: + '@grpc/grpc-js': 1.14.3 + '@opentelemetry/api': 1.9.0 + '@opentelemetry/core': 2.2.0(@opentelemetry/api@1.9.0) + '@opentelemetry/otlp-exporter-base': 0.208.0(@opentelemetry/api@1.9.0) + '@opentelemetry/otlp-grpc-exporter-base': 0.208.0(@opentelemetry/api@1.9.0) + '@opentelemetry/otlp-transformer': 0.208.0(@opentelemetry/api@1.9.0) + '@opentelemetry/resources': 2.2.0(@opentelemetry/api@1.9.0) + '@opentelemetry/sdk-trace-base': 2.2.0(@opentelemetry/api@1.9.0) + optional: true + '@opentelemetry/instrumentation@0.208.0(@opentelemetry/api@1.9.0)': dependencies: '@opentelemetry/api': 1.9.0 @@ -3373,6 +3483,85 @@ snapshots: - supports-color optional: true + '@opentelemetry/otlp-exporter-base@0.208.0(@opentelemetry/api@1.9.0)': + dependencies: + '@opentelemetry/api': 1.9.0 + '@opentelemetry/core': 2.2.0(@opentelemetry/api@1.9.0) + '@opentelemetry/otlp-transformer': 0.208.0(@opentelemetry/api@1.9.0) + optional: true + + '@opentelemetry/otlp-grpc-exporter-base@0.208.0(@opentelemetry/api@1.9.0)': + dependencies: + '@grpc/grpc-js': 1.14.3 + '@opentelemetry/api': 1.9.0 + '@opentelemetry/core': 2.2.0(@opentelemetry/api@1.9.0) + '@opentelemetry/otlp-exporter-base': 0.208.0(@opentelemetry/api@1.9.0) + '@opentelemetry/otlp-transformer': 0.208.0(@opentelemetry/api@1.9.0) + optional: true + + '@opentelemetry/otlp-transformer@0.208.0(@opentelemetry/api@1.9.0)': + dependencies: + '@opentelemetry/api': 1.9.0 + '@opentelemetry/api-logs': 0.208.0 + '@opentelemetry/core': 2.2.0(@opentelemetry/api@1.9.0) + '@opentelemetry/resources': 2.2.0(@opentelemetry/api@1.9.0) + '@opentelemetry/sdk-logs': 0.208.0(@opentelemetry/api@1.9.0) + '@opentelemetry/sdk-metrics': 2.2.0(@opentelemetry/api@1.9.0) + '@opentelemetry/sdk-trace-base': 2.2.0(@opentelemetry/api@1.9.0) + protobufjs: 7.5.3 + optional: true + + '@opentelemetry/resources@1.30.1(@opentelemetry/api@1.9.0)': + dependencies: + '@opentelemetry/api': 1.9.0 + '@opentelemetry/core': 1.30.1(@opentelemetry/api@1.9.0) + '@opentelemetry/semantic-conventions': 1.28.0 + optional: true + + '@opentelemetry/resources@2.2.0(@opentelemetry/api@1.9.0)': + dependencies: + '@opentelemetry/api': 1.9.0 + '@opentelemetry/core': 2.2.0(@opentelemetry/api@1.9.0) + '@opentelemetry/semantic-conventions': 1.40.0 + optional: true + + '@opentelemetry/sdk-logs@0.208.0(@opentelemetry/api@1.9.0)': + dependencies: + '@opentelemetry/api': 1.9.0 + '@opentelemetry/api-logs': 0.208.0 + '@opentelemetry/core': 2.2.0(@opentelemetry/api@1.9.0) + '@opentelemetry/resources': 2.2.0(@opentelemetry/api@1.9.0) + optional: true + + '@opentelemetry/sdk-metrics@2.2.0(@opentelemetry/api@1.9.0)': + dependencies: + '@opentelemetry/api': 1.9.0 + '@opentelemetry/core': 2.2.0(@opentelemetry/api@1.9.0) + '@opentelemetry/resources': 2.2.0(@opentelemetry/api@1.9.0) + optional: true + + '@opentelemetry/sdk-trace-base@1.30.1(@opentelemetry/api@1.9.0)': + dependencies: + '@opentelemetry/api': 1.9.0 + '@opentelemetry/core': 1.30.1(@opentelemetry/api@1.9.0) + '@opentelemetry/resources': 1.30.1(@opentelemetry/api@1.9.0) + '@opentelemetry/semantic-conventions': 1.28.0 + optional: true + + '@opentelemetry/sdk-trace-base@2.2.0(@opentelemetry/api@1.9.0)': + dependencies: + '@opentelemetry/api': 1.9.0 + '@opentelemetry/core': 2.2.0(@opentelemetry/api@1.9.0) + '@opentelemetry/resources': 2.2.0(@opentelemetry/api@1.9.0) + '@opentelemetry/semantic-conventions': 1.40.0 + optional: true + + '@opentelemetry/semantic-conventions@1.28.0': + optional: true + + '@opentelemetry/semantic-conventions@1.40.0': + optional: true + '@pkgr/core@0.2.9': {} '@protobufjs/aspromise@1.1.2': {} diff --git a/sdks/typescript/src/clients/hatchet-client/client-config.ts b/sdks/typescript/src/clients/hatchet-client/client-config.ts index f657747481..26456cdf95 100644 --- a/sdks/typescript/src/clients/hatchet-client/client-config.ts +++ b/sdks/typescript/src/clients/hatchet-client/client-config.ts @@ -24,8 +24,8 @@ export const OpenTelemetryConfigSchema = z.object({ excludedAttributes: z.array(z.string()).optional().default([]), /** - * If true, includes the task name in the span name for start_step_run spans. - * e.g., "hatchet.start_step_run.my_task" instead of "hatchet.start_step_run" + * If true, includes the task name in the span name for task run spans. + * e.g., "hatchet task run my_task" instead of "hatchet task run" */ includeTaskNameInSpanName: z.boolean().optional().default(false), }); diff --git a/sdks/typescript/src/opentelemetry/hatchet-exporter.ts b/sdks/typescript/src/opentelemetry/hatchet-exporter.ts new file mode 100644 index 0000000000..4afa4f46e9 --- /dev/null +++ b/sdks/typescript/src/opentelemetry/hatchet-exporter.ts @@ -0,0 +1,75 @@ +/** + * Hatchet OTLP Exporter + * + * Creates an OTLP gRPC trace exporter that sends spans to the Hatchet engine's + * collector endpoint, and a SpanProcessor that injects hatchet.* attributes + * into all child spans so they are queryable by the same attributes as the parent. + * + * This mirrors the Go SDK's EnableHatchetCollector() and the Python SDK's + * enable_hatchet_otel_collector option. + */ + +import type { ClientConfig } from '@hatchet/clients/hatchet-client/client-config'; + +try { + require.resolve('@opentelemetry/exporter-trace-otlp-grpc'); + require.resolve('@opentelemetry/sdk-trace-base'); +} catch { + throw new Error( + 'To use HatchetInstrumentor with enableHatchetCollector, you must install: ' + + 'npm install @opentelemetry/exporter-trace-otlp-grpc @opentelemetry/sdk-trace-base' + ); +} + +/* eslint-disable @typescript-eslint/no-require-imports */ +const { + OTLPTraceExporter, +} = require('@opentelemetry/exporter-trace-otlp-grpc') as typeof import('@opentelemetry/exporter-trace-otlp-grpc'); +const sdkTraceBase = + require('@opentelemetry/sdk-trace-base') as typeof import('@opentelemetry/sdk-trace-base'); +/* eslint-enable @typescript-eslint/no-require-imports */ + +const { BatchSpanProcessor } = sdkTraceBase; + +type SdkTracerProvider = import('@opentelemetry/sdk-trace-base').BasicTracerProvider; +type ReadableSpan = import('@opentelemetry/sdk-trace-base').ReadableSpan; + +/** + * HatchetAttributeSpanProcessor wraps a BatchSpanProcessor. + * The hatchet.* attributes are already injected by the instrumentor's + * startActiveSpan call, making them available to all child spans in context. + */ +class HatchetAttributeSpanProcessor extends BatchSpanProcessor { + onEnd(span: ReadableSpan): void { + super.onEnd(span); + } +} + +/** + * Creates an OTLP gRPC trace exporter pointing at the Hatchet engine. + */ +function createHatchetExporter(config: ClientConfig): InstanceType { + const insecure = config.tls_config.tls_strategy === 'none'; + + return new OTLPTraceExporter({ + url: `${insecure ? 'http' : 'https'}://${config.host_port}`, + metadata: { + authorization: `Bearer ${config.token}`, + } as any, + }); +} + +/** + * Adds the Hatchet OTLP exporter to the given TracerProvider. + * The exporter sends spans to the Hatchet engine's collector endpoint + * using the same connection settings as the Hatchet client. + */ +export function addHatchetExporter( + tracerProvider: SdkTracerProvider, + config: ClientConfig +): void { + const exporter = createHatchetExporter(config); + const processor = new HatchetAttributeSpanProcessor(exporter as any); + + tracerProvider.addSpanProcessor(processor); +} diff --git a/sdks/typescript/src/opentelemetry/instrumentor.ts b/sdks/typescript/src/opentelemetry/instrumentor.ts index 7032974c3e..2fc28eef39 100644 --- a/sdks/typescript/src/opentelemetry/instrumentor.ts +++ b/sdks/typescript/src/opentelemetry/instrumentor.ts @@ -21,6 +21,7 @@ import type { } from '@hatchet/clients/event/event-client'; import type { AdminClient } from '@hatchet/v1/client/admin'; import type { InternalWorker } from '@hatchet/v1/client/worker/worker-internal'; +import type { ClientConfig } from '@hatchet/clients/hatchet-client/client-config'; import { OTelAttribute, type ActionOTelAttributeValue } from '../util/opentelemetry'; import { parseJSON } from '../util/parse'; import { OpenTelemetryConfig, DEFAULT_CONFIG } from './types'; @@ -51,7 +52,23 @@ const { isWrapped, } = otelInstrumentation; -type HatchetInstrumentationConfig = OpenTelemetryConfig & InstrumentationConfig; +type HatchetInstrumentationConfig = OpenTelemetryConfig & + InstrumentationConfig & { + /** + * Enable sending traces to the Hatchet engine's OTLP collector. + * Requires @opentelemetry/exporter-trace-otlp-grpc and @opentelemetry/sdk-trace-base. + * Connection settings (endpoint, token, TLS) are read from the provided clientConfig + * or from the same environment variables used by the Hatchet client. + */ + enableHatchetCollector?: boolean; + + /** + * The Hatchet ClientConfig to use for the collector connection. + * If not provided and enableHatchetCollector is true, config will be loaded + * from environment variables / .hatchet.yaml. + */ + clientConfig?: ClientConfig; + }; type Carrier = Record; const INSTRUMENTOR_NAME = '@hatchet-dev/typescript-sdk'; @@ -136,6 +153,56 @@ function filterAttributes( export class HatchetInstrumentor extends InstrumentationBase { constructor(config: Partial = {}) { super(INSTRUMENTOR_NAME, HATCHET_VERSION, { ...DEFAULT_CONFIG, ...config }); + + if (config.enableHatchetCollector) { + this._setupHatchetCollector(config.clientConfig); + } + } + + /** + * Sets up the Hatchet OTLP exporter on the current TracerProvider. + * Loads client config from environment if not provided. + */ + private _setupHatchetCollector(clientConfig?: ClientConfig): void { + try { + // eslint-disable-next-line @typescript-eslint/no-require-imports + const { addHatchetExporter } = require('./hatchet-exporter') as typeof import('./hatchet-exporter'); + + let config = clientConfig; + if (!config) { + // Load config from environment (same as HatchetClient would) + // eslint-disable-next-line @typescript-eslint/no-require-imports + const { ConfigLoader } = require('@hatchet/util/config-loader/config-loader') as typeof import('@hatchet/util/config-loader/config-loader'); + config = ConfigLoader.loadClientConfig() as ClientConfig; + } + + // Get the SDK TracerProvider - either from the global provider or create one + let sdkTracerProvider: any; + try { + // eslint-disable-next-line @typescript-eslint/no-require-imports + const sdkTrace = require('@opentelemetry/sdk-trace-base') as typeof import('@opentelemetry/sdk-trace-base'); + + // Check if the global tracer provider is an SDK TracerProvider + const globalProvider = otelApi.trace.getTracerProvider(); + if (globalProvider instanceof sdkTrace.BasicTracerProvider) { + sdkTracerProvider = globalProvider; + } else { + // Create a new SDK TracerProvider and set it as global + sdkTracerProvider = new sdkTrace.BasicTracerProvider(); + sdkTracerProvider.register(); + } + } catch { + diag.warn( + 'hatchet instrumentation: @opentelemetry/sdk-trace-base is required for enableHatchetCollector' + ); + return; + } + + addHatchetExporter(sdkTracerProvider, config); + diag.info('hatchet instrumentation: Hatchet OTLP collector enabled'); + } catch (e) { + diag.warn(`hatchet instrumentation: Failed to set up Hatchet collector: ${e}`); + } } override setConfig(config: Partial = {}): void { @@ -239,7 +306,7 @@ export class HatchetInstrumentor extends InstrumentationBase { @@ -449,9 +516,9 @@ export class HatchetInstrumentor extends InstrumentationBase { @@ -537,9 +604,9 @@ export class HatchetInstrumentor extends InstrumentationBase { From 0c8839b614f2a0b29515d6833ff4c3dac35f3673 Mon Sep 17 00:00:00 2001 From: Mohammed Nafees Date: Mon, 9 Mar 2026 22:47:21 +0100 Subject: [PATCH 10/17] refetch polling --- .../v2components/step-run-detail/step-run-detail.tsx | 5 ++++- .../$run/v2components/step-run-detail/task-run-trace.tsx | 9 ++++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/frontend/app/src/pages/main/v1/workflow-runs-v1/$run/v2components/step-run-detail/step-run-detail.tsx b/frontend/app/src/pages/main/v1/workflow-runs-v1/$run/v2components/step-run-detail/step-run-detail.tsx index 3af311cf64..1b6ed397fc 100644 --- a/frontend/app/src/pages/main/v1/workflow-runs-v1/$run/v2components/step-run-detail/step-run-detail.tsx +++ b/frontend/app/src/pages/main/v1/workflow-runs-v1/$run/v2components/step-run-detail/step-run-detail.tsx @@ -315,7 +315,10 @@ export const TaskRunDetail = ({ {isCloudEnabled && ( - + )} diff --git a/frontend/app/src/pages/main/v1/workflow-runs-v1/$run/v2components/step-run-detail/task-run-trace.tsx b/frontend/app/src/pages/main/v1/workflow-runs-v1/$run/v2components/step-run-detail/task-run-trace.tsx index 0e27c4a6ed..e6e6669884 100644 --- a/frontend/app/src/pages/main/v1/workflow-runs-v1/$run/v2components/step-run-detail/task-run-trace.tsx +++ b/frontend/app/src/pages/main/v1/workflow-runs-v1/$run/v2components/step-run-detail/task-run-trace.tsx @@ -7,13 +7,20 @@ import { flattenSpans } from '@evilmartians/agent-prism-data'; import { useQuery } from '@tanstack/react-query'; import { useEffect, useMemo, useState } from 'react'; -export function TaskRunTrace({ taskExternalId }: { taskExternalId: string }) { +export function TaskRunTrace({ + taskExternalId, + isRunning, +}: { + taskExternalId: string; + isRunning: boolean; +}) { const tracesQuery = useQuery({ queryKey: ['cloud:traces', taskExternalId], queryFn: async () => { const res = await cloudApi.otelTracesList(taskExternalId); return res.data; }, + refetchInterval: isRunning ? 100 : false, }); const traceSpans = useMemo(() => { From 06915fb6629c0466cde1da3a83c43c12337cca76 Mon Sep 17 00:00:00 2001 From: Mohammed Nafees Date: Tue, 10 Mar 2026 15:47:25 +0100 Subject: [PATCH 11/17] otel postgres traces --- .../openapi/components/schemas/_index.yaml | 4 + .../openapi/components/schemas/v1/otel.yaml | 54 ++ api-contracts/openapi/openapi.yaml | 2 + .../openapi/paths/v1/tasks/tasks.yaml | 44 ++ api/v1/server/handlers/v1/tasks/trace.go | 46 ++ api/v1/server/oas/gen/openapi.gen.go | 743 ++++++++++-------- .../migrations/20260310150948_v1_0_84.sql | 37 + frontend/app/src/lib/api/generated/Api.ts | 18 + .../app/src/lib/api/generated/cloud/Api.ts | 18 - .../lib/api/generated/cloud/data-contracts.ts | 23 - .../src/lib/api/generated/data-contracts.ts | 23 + .../step-run-detail/otel-span-adapter.ts | 2 +- .../step-run-detail/step-run-detail.tsx | 24 +- .../step-run-detail/task-run-trace.tsx | 6 +- pkg/client/rest/gen.go | 156 ++++ pkg/repository/otelcol.go | 282 ++++++- pkg/repository/sqlcv1/models.go | 21 + pkg/repository/sqlcv1/tasks.sql | 14 +- pkg/repository/sqlcv1/tasks.sql.go | 14 +- sql/schema/v1-core.sql | 32 + 20 files changed, 1186 insertions(+), 377 deletions(-) create mode 100644 api-contracts/openapi/components/schemas/v1/otel.yaml create mode 100644 api/v1/server/handlers/v1/tasks/trace.go create mode 100644 cmd/hatchet-migrate/migrate/migrations/20260310150948_v1_0_84.sql diff --git a/api-contracts/openapi/components/schemas/_index.yaml b/api-contracts/openapi/components/schemas/_index.yaml index 1c39332c78..690128c621 100644 --- a/api-contracts/openapi/components/schemas/_index.yaml +++ b/api-contracts/openapi/components/schemas/_index.yaml @@ -400,3 +400,7 @@ V1CELDebugResponse: $ref: "./v1/cel.yaml#/V1CELDebugResponse" V1CELDebugResponseStatus: $ref: "./v1/cel.yaml#/V1CELDebugResponseStatus" +OtelSpan: + $ref: "./v1/otel.yaml#/OtelSpan" +OtelSpanList: + $ref: "./v1/otel.yaml#/OtelSpanList" diff --git a/api-contracts/openapi/components/schemas/v1/otel.yaml b/api-contracts/openapi/components/schemas/v1/otel.yaml new file mode 100644 index 0000000000..e831ea4b5f --- /dev/null +++ b/api-contracts/openapi/components/schemas/v1/otel.yaml @@ -0,0 +1,54 @@ +OtelSpan: + type: object + properties: + trace_id: + type: string + span_id: + type: string + parent_span_id: + type: string + span_name: + type: string + span_kind: + type: string + service_name: + type: string + status_code: + type: string + status_message: + type: string + duration: + type: integer + format: int64 + created_at: + type: string + format: date-time + resource_attributes: + type: object + additionalProperties: + type: string + span_attributes: + type: object + additionalProperties: + type: string + scope_name: + type: string + scope_version: + type: string + required: + - trace_id + - span_id + - span_name + - span_kind + - service_name + - status_code + - duration + - created_at + +OtelSpanList: + type: object + properties: + rows: + type: array + items: + $ref: "#/OtelSpan" diff --git a/api-contracts/openapi/openapi.yaml b/api-contracts/openapi/openapi.yaml index 30df3807f5..f33970013c 100644 --- a/api-contracts/openapi/openapi.yaml +++ b/api-contracts/openapi/openapi.yaml @@ -29,6 +29,8 @@ paths: $ref: "./paths/v1/tasks/tasks.yaml#/listTaskEvents" /api/v1/stable/tasks/{task}/logs: $ref: "./paths/v1/tasks/tasks.yaml#/listLogs" + /api/v1/stable/tasks/{task}/trace: + $ref: "./paths/v1/tasks/tasks.yaml#/getTrace" /api/v1/stable/tenants/{tenant}/tasks/cancel: $ref: "./paths/v1/tasks/tasks.yaml#/cancelTasks" /api/v1/stable/tenants/{tenant}/tasks/replay: diff --git a/api-contracts/openapi/paths/v1/tasks/tasks.yaml b/api-contracts/openapi/paths/v1/tasks/tasks.yaml index 9e4ae10c41..f35aebfdec 100644 --- a/api-contracts/openapi/paths/v1/tasks/tasks.yaml +++ b/api-contracts/openapi/paths/v1/tasks/tasks.yaml @@ -441,6 +441,50 @@ replayTasks: tags: - Task +getTrace: + get: + x-resources: ["tenant", "task"] + description: Get OTel trace for a task run + operationId: v1-task:get:trace + parameters: + - description: The task id + in: path + name: task + required: true + schema: + type: string + format: uuid + minLength: 36 + maxLength: 36 + responses: + "200": + content: + application/json: + schema: + $ref: "../../../components/schemas/_index.yaml#/OtelSpanList" + description: Successfully retrieved the OTel trace + "400": + content: + application/json: + schema: + $ref: "../../../components/schemas/_index.yaml#/APIErrors" + description: A malformed or bad request + "403": + content: + application/json: + schema: + $ref: "../../../components/schemas/_index.yaml#/APIErrors" + description: Forbidden + "404": + content: + application/json: + schema: + $ref: "../../../components/schemas/_index.yaml#/APIErrors" + description: The task was not found + summary: Get OTel trace + tags: + - Task + listLogs: get: x-resources: ["tenant", "task"] diff --git a/api/v1/server/handlers/v1/tasks/trace.go b/api/v1/server/handlers/v1/tasks/trace.go new file mode 100644 index 0000000000..27d6226389 --- /dev/null +++ b/api/v1/server/handlers/v1/tasks/trace.go @@ -0,0 +1,46 @@ +package tasks + +import ( + "github.com/labstack/echo/v4" + + "github.com/hatchet-dev/hatchet/api/v1/server/oas/gen" + "github.com/hatchet-dev/hatchet/pkg/repository" + "github.com/hatchet-dev/hatchet/pkg/repository/sqlcv1" +) + +func (t *TasksService) V1TaskGetTrace(ctx echo.Context, request gen.V1TaskGetTraceRequestObject) (gen.V1TaskGetTraceResponseObject, error) { + task := ctx.Get("task").(*sqlcv1.V1TasksOlap) + + spans, err := t.config.V1.OTelCollector().ListSpansByTaskExternalID( + ctx.Request().Context(), task.TenantID, task.ExternalID) + if err != nil { + return nil, err + } + + apiSpans := convertToAPISpans(spans) + + return gen.V1TaskGetTrace200JSONResponse(gen.OtelSpanList{Rows: &apiSpans}), nil +} + +func convertToAPISpans(spans []*repository.OtelSpanRow) []gen.OtelSpan { + result := make([]gen.OtelSpan, len(spans)) + for i, s := range spans { + result[i] = gen.OtelSpan{ + TraceId: s.TraceID, + SpanId: s.SpanID, + ParentSpanId: &s.ParentSpanID, + SpanName: s.SpanName, + SpanKind: s.SpanKind, + ServiceName: s.ServiceName, + StatusCode: s.StatusCode, + StatusMessage: &s.StatusMessage, + Duration: int64(s.Duration), //nolint:gosec + CreatedAt: s.CreatedAt, + ResourceAttributes: &s.ResourceAttributes, + SpanAttributes: &s.SpanAttributes, + ScopeName: &s.ScopeName, + ScopeVersion: &s.ScopeVersion, + } + } + return result +} diff --git a/api/v1/server/oas/gen/openapi.gen.go b/api/v1/server/oas/gen/openapi.gen.go index 5f43ce3136..7c1a16442e 100644 --- a/api/v1/server/oas/gen/openapi.gen.go +++ b/api/v1/server/oas/gen/openapi.gen.go @@ -758,6 +758,29 @@ type LogLineOrderByField string // LogLineSearch defines model for LogLineSearch. type LogLineSearch = string +// OtelSpan defines model for OtelSpan. +type OtelSpan struct { + CreatedAt time.Time `json:"created_at"` + Duration int64 `json:"duration"` + ParentSpanId *string `json:"parent_span_id,omitempty"` + ResourceAttributes *map[string]string `json:"resource_attributes,omitempty"` + ScopeName *string `json:"scope_name,omitempty"` + ScopeVersion *string `json:"scope_version,omitempty"` + ServiceName string `json:"service_name"` + SpanAttributes *map[string]string `json:"span_attributes,omitempty"` + SpanId string `json:"span_id"` + SpanKind string `json:"span_kind"` + SpanName string `json:"span_name"` + StatusCode string `json:"status_code"` + StatusMessage *string `json:"status_message,omitempty"` + TraceId string `json:"trace_id"` +} + +// OtelSpanList defines model for OtelSpanList. +type OtelSpanList struct { + Rows *[]OtelSpan `json:"rows,omitempty"` +} + // PaginationResponse defines model for PaginationResponse. type PaginationResponse struct { // CurrentPage the current page @@ -3196,6 +3219,9 @@ type ServerInterface interface { // List events for a task // (GET /api/v1/stable/tasks/{task}/task-events) V1TaskEventList(ctx echo.Context, task openapi_types.UUID, params V1TaskEventListParams) error + // Get OTel trace + // (GET /api/v1/stable/tasks/{task}/trace) + V1TaskGetTrace(ctx echo.Context, task openapi_types.UUID) error // Debug a CEL expression // (POST /api/v1/stable/tenants/{tenant}/cel/debug) V1CelDebug(ctx echo.Context, tenant openapi_types.UUID) error @@ -3949,6 +3975,26 @@ func (w *ServerInterfaceWrapper) V1TaskEventList(ctx echo.Context) error { return err } +// V1TaskGetTrace converts echo context to params. +func (w *ServerInterfaceWrapper) V1TaskGetTrace(ctx echo.Context) error { + var err error + // ------------- Path parameter "task" ------------- + var task openapi_types.UUID + + err = runtime.BindStyledParameterWithLocation("simple", false, "task", runtime.ParamLocationPath, ctx.Param("task"), &task) + if err != nil { + return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Invalid format for parameter task: %s", err)) + } + + ctx.Set(BearerAuthScopes, []string{}) + + ctx.Set(CookieAuthScopes, []string{}) + + // Invoke the callback with all the unmarshaled arguments + err = w.Handler.V1TaskGetTrace(ctx, task) + return err +} + // V1CelDebug converts echo context to params. func (w *ServerInterfaceWrapper) V1CelDebug(ctx echo.Context) error { var err error @@ -7294,6 +7340,7 @@ func RegisterHandlersWithBaseURL(router EchoRouter, si ServerInterface, baseURL router.GET(baseURL+"/api/v1/stable/tasks/:task", wrapper.V1TaskGet) router.GET(baseURL+"/api/v1/stable/tasks/:task/logs", wrapper.V1LogLineList) router.GET(baseURL+"/api/v1/stable/tasks/:task/task-events", wrapper.V1TaskEventList) + router.GET(baseURL+"/api/v1/stable/tasks/:task/trace", wrapper.V1TaskGetTrace) router.POST(baseURL+"/api/v1/stable/tenants/:tenant/cel/debug", wrapper.V1CelDebug) router.GET(baseURL+"/api/v1/stable/tenants/:tenant/events", wrapper.V1EventList) router.GET(baseURL+"/api/v1/stable/tenants/:tenant/events/keys", wrapper.V1EventKeyList) @@ -8053,6 +8100,50 @@ func (response V1TaskEventList501JSONResponse) VisitV1TaskEventListResponse(w ht return json.NewEncoder(w).Encode(response) } +type V1TaskGetTraceRequestObject struct { + Task openapi_types.UUID `json:"task"` +} + +type V1TaskGetTraceResponseObject interface { + VisitV1TaskGetTraceResponse(w http.ResponseWriter) error +} + +type V1TaskGetTrace200JSONResponse OtelSpanList + +func (response V1TaskGetTrace200JSONResponse) VisitV1TaskGetTraceResponse(w http.ResponseWriter) error { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(200) + + return json.NewEncoder(w).Encode(response) +} + +type V1TaskGetTrace400JSONResponse APIErrors + +func (response V1TaskGetTrace400JSONResponse) VisitV1TaskGetTraceResponse(w http.ResponseWriter) error { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(400) + + return json.NewEncoder(w).Encode(response) +} + +type V1TaskGetTrace403JSONResponse APIErrors + +func (response V1TaskGetTrace403JSONResponse) VisitV1TaskGetTraceResponse(w http.ResponseWriter) error { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(403) + + return json.NewEncoder(w).Encode(response) +} + +type V1TaskGetTrace404JSONResponse APIErrors + +func (response V1TaskGetTrace404JSONResponse) VisitV1TaskGetTraceResponse(w http.ResponseWriter) error { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(404) + + return json.NewEncoder(w).Encode(response) +} + type V1CelDebugRequestObject struct { Tenant openapi_types.UUID `json:"tenant"` Body *V1CelDebugJSONRequestBody @@ -12716,6 +12807,8 @@ type StrictServerInterface interface { V1TaskEventList(ctx echo.Context, request V1TaskEventListRequestObject) (V1TaskEventListResponseObject, error) + V1TaskGetTrace(ctx echo.Context, request V1TaskGetTraceRequestObject) (V1TaskGetTraceResponseObject, error) + V1CelDebug(ctx echo.Context, request V1CelDebugRequestObject) (V1CelDebugResponseObject, error) V1EventList(ctx echo.Context, request V1EventListRequestObject) (V1EventListResponseObject, error) @@ -13354,6 +13447,28 @@ func (sh *strictHandler) V1TaskEventList(ctx echo.Context, task openapi_types.UU return nil } +// V1TaskGetTrace operation +func (sh *strictHandler) V1TaskGetTrace(ctx echo.Context, task openapi_types.UUID) error { + var request V1TaskGetTraceRequestObject + + request.Task = task + + handler := func(ctx echo.Context, request interface{}) (interface{}, error) { + return sh.ssi.V1TaskGetTrace(ctx, request.(V1TaskGetTraceRequestObject)) + } + + response, err := handler(ctx, request) + + if err != nil { + return err + } else if validResponse, ok := response.(V1TaskGetTraceResponseObject); ok { + return validResponse.VisitV1TaskGetTraceResponse(ctx.Response()) + } else if response != nil { + return fmt.Errorf("Unexpected response type: %T", response) + } + return nil +} + // V1CelDebug operation func (sh *strictHandler) V1CelDebug(ctx echo.Context, tenant openapi_types.UUID) error { var request V1CelDebugRequestObject @@ -16160,326 +16275,330 @@ func (sh *strictHandler) WorkflowVersionGet(ctx echo.Context, workflow openapi_t // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/+y9e3PbOLIo/lVY+v2q7kyV5Fcmc+ak6v6h2EqiiWP7SHJy986mvBAJSxhTJJcA7WhT", - "/u638CJBEiBBvSwlrNracUQ8Go3uRqPRj+8dN1xEYQADgjtvvnewO4cLwP7s3wwHcRzG9O8oDiMYEwTZ", - "Fzf0IP2vB7Ebo4igMOi86QDHTTAJF84HQNw5JA6kvR3WuNuB38Ai8mHnzelvJyfdzn0YLwDpvOkkKCC/", - "/9bpdsgygp03HRQQOINx57mbH748m/Jv5z6MHTJHmM+pTtfpZw0foYBpATEGM5jNikmMghmbNHTxnY+C", - "B92U9HeHhA6ZQ8cL3WQBAwI0AHQddO8g4sBvCBOcA2eGyDyZHrnh4njO8dTz4KP8WwfRPYK+V4aGwsA+", - "OWQOiDK5g7ADMA5dBAj0nCdE5gweEEU+csHUz21HJwALDSKeu50Y/jtBMfQ6b/7KTf01bRxO/4YuoTBK", + "H4sIAAAAAAAC/+y9e2/buLYo/lUE/37AnQHsvDqdM6fA/cNN3NbTNMmxnfbuO6fIoSXG5kSWNCKV1LvI", + "d7/gS6IkUqL8it0K2NiTWnwsLq61uLi4Ht87briIwgAGBHfefO9gdw4XgP3ZvxkO4jiM6d9RHEYwJgiy", + "L27oQfpfD2I3RhFBYdB50wGOm2ASLpwPgLhzSBxIezuscbcDv4FF5MPOm9PfTk66nfswXgDSedNJUEB+", + "/63T7ZBlBDtvOiggcAbjznM3P3x5NuXfzn0YO2SOMJ9Tna7Tzxo+QgHTAmIMZjCbFZMYBTM2aejiOx8F", + "D7op6e8OCR0yh44XuskCBgRoAOg66N5BxIHfECY4B84MkXkyPXLDxfGc46nnwUf5tw6iewR9rwwNhYF9", + "csgcEGVyB2EHYBy6CBDoOU+IzBk8IIp85IKpn9uOTgAWGkQ8dzsx/CdBMfQ6b/7KTf01bRxO/4YuoTBK", "WsFlYoHp74jABfvj/4/hfedN5/87zmjvWBDecUp1z+k0II7BsgSSGNcAzSdIQBkW4Pvh0/kcBDN4AzB+", "CmMNYp/mkMxh7ISxE4TESTCMseOCwHFZR7r5KHYi2V/BJYkTmIIzDUMfgoDCw6eNISBwAgMQkCaTsm5O", "AJ8cwvpi6xmHwSMifOGWkyHWwwnZV/4zo3aEHRRgAgIXWs8+RrMgiRpMjtEscJIoY6VGUyZkbkFalCz6", - "tOlztxOFmMzDmWWvG9Gadlz6YdCPoqGBK2/od8puzvCCrSbBkPWhXE+piDg4iaIwJjlGPD179dvr3//r", - "jx79o/B/9Pf/Pjk90zKqif77Aid5HmDr0lEFBV3ABT2HDoqd8N6hmIUBQS4TdCrEf3WmACO30+3MwnDm", + "tOlztxOFmMzDmWWvG9Gadlz6YdCPoqGBK2/od8puzvCCrSbBkPWhXE+piDg4iaIwJjlGPD179dvr3//j", + "jx79o/B/9Pf/PDk90zKqif77Aid5HmDr0lEFBV3ABT2HDoqd8N6hmIUBQS4TdCrEf3WmACO30+3MwnDm", "Q8qLKY+XxFiJmU1gD+kJEAMp9gvSJKACrIJrBeWkQ1BpKDo5YcAkt0JXZUJi4lCLG/qFIoQPkcFYlu61", "4lTIXLmYChl2kxFpQZRF6EOIiYECQ0w+hDOnfzN05rSVCuOckAi/OT4W9H8kvlDi1B0/IEIf4bJ+nge4", "zE0TzR/uMtIFU9eD99bkO4I4TGIX6sU4l4le37B6ghZQORRjMZbzBLAQpzmp3Tk7OTvrnZ71Tl9NTl+/", "Ofn9zW9/HP3xxx+vXv/RO3n95uSko6grHiCwRyfQoQoZBALyON0owHQdFDi3t1xA0KFVgKbTs9Pf/jj5", - "r97Zb7/D3m+vwOseOHvt9X47/a/fT71T9/7+v+n8C/DtEgYzyuSvfteAk0TeqmjyASaO6L8NXBX4AdFJ", + "j97Zb7/D3m+vwOseOHvt9X47/Y/fT71T9/7+P+n8C/DtEgYzyuSvfteAk0TeqmjyASaO6L8NXBX4AdFJ", "sl1VQTfwxiR8gDrx8C1CMcS6JX+ZQ87+lFgJ7e6I1kfWG7yABHiAk2TNmZGjYKNcmRTkSgrbUX5/z16/", - "rsNhCls3FS8pMrRIdF0YEa4jjOC/E8iFSR6fXCHgmF2POhcoMBNrt/OtF4II9ehlYQaDHvxGYtAjYMag", - "eAQ+ovvSeZOuuJskyOs8lwiJw6tb79vEf+A62OARBsS4ZPgo70JW+qpmyFrNlc/w9bnbOafnkG8B0NDL", - "g9R4O7ILV8K4rcn2WC2IQsiWFAZuEscwcJeXaIHImMSAwNmSn97JgnY471+dDy7vhld3N6Pr96PBeNzp", - "di5G1zd3V4Mvg/Gk0+38z+3gdpD98/3o+vbmbnR9e3VxN7p+O7xS9jiDUpl77IYRVOf8cj36+O7y+kun", - "25n0xx9r+0NC6K86ERNDjLWXUsrObjaGk7XtUiXQo9r0DAaQYsQB9Mh07uNw4RCAHxwURAnBXUcycteB", - "xD3SiSG/iNdKAjXtxzMjglESYP1CFuAbWiQLJ0gWU6qv32dLI85TGD/c++GTEydBXoCigLw6097nsdwS", - "S3D5FtKOBEYjCDyqLemUbgptLL6nhy10aDeK8ac5cuf8kFM3B/Md5vdifgrUSFiBreIGdFWakMvUiSB1", - "bQSQOtoq7fsDXHLdz/MQXTrwb3Ld1T0w2FRKMPEfvtvoZVzUycPXLK/4sTM08IeXxJnJRG4NFCcywg4T", - "9uXNsD8iwgUiAfK7ciK2GP3x2+eHL79xrnX6svG/WiANR2GAYRlrRCo0ZYzlwKoGg49ihuM8DoMvgnUn", - "MZrNYGzcx4zKPilqT2lgNw6DQTXd0iZXYgPKSjMVe9qRoxiFMSLLImkz8SKkU+fNK3Z48b9PyyRfUhDo", - "bF3d4hQ4S6v6mmKw+qzW46xAdGmbVNSnFMhOUmWbM2Tox2IMZTfAg+4SR/uzU8jQPdsmdTPKY8ivUvSm", - "4zQ5FsrDsk8MODagc498AilE9ZzAr6MMa9nmja/GinXBuIskjJDbj03suAD/CQNHKvgOpRjnl/7o6le5", - "+vHV2GFjrCPGUk13gYL/fdpdgG//++z172WVNwXWzPXc6Nj3YUwGC4D893GYRGb5TZtgnbD0ESZ0jbyF", - "NG3F9ES0tPussHwPPcIum7G8dgFq3cprLjl8cO1es09yW+laqT7BLxkb2Vu5rm4nDv1a3Yiv5hOk+tiI", - "ttfioyMGq8OKGR/BDAXwM4ylQK+HSTZ+7nZg8IjiMFhAbuau7ztQOlhflLktfBN7wJAYBtMQxB4KZhdC", - "zup1LG5+NsrzbBgulUnoYBLGkD3C6OHO9gb7ycwgBv1ktvmFd8WbEzvxng0mSgaUnpIyTQLbHoRVSNUq", - "FlqJotiAy/bbVJ1oNNcahp0FJPPQqzcTKOj6xLsoxF553K6s+3Q7nFqGnnYOeYer+WzU3GQDwfzaYcxG", - "qhQ03UCF2XOwCsrI6CDdg1o6vUQ6eReBGQrS94aqXbxJW6aKPBPdT03sRSrfWL2L6GhHMWxcDN71by8n", - "HWYX1Zs11AGuYw/Gb5fv5KuyHCaQii8sWV6zkZj2u0u1d02tdQ2+JulLbf0RVmS1MrjDi7wAL77Qi/d7", - "40Ik/Y+SYJwsFiCutfuwrfpS7lbBklxnThfyVW64PBPzm97kRuL88uf4+sqZLgnEv9Yr76nazqb/uB4N", - "yDH2gPnT5ZT5XgK6L1BWgCgkyAWKoStBklIEYLfDFSSz/DBJIAvRM4Ygdufa08hE7+XXQ2Zz1z4iMy0z", - "M3fKhlojp8HAdg+QxdC8VZNxIxh4wh5dNbBo1mTkfycwqYeYt2oybpwEgQXEolmTkXHiuhB69UCnDe1H", - "T6kcVz0NaW6K7NuRehVegcfWOLHMYl15b/oznGoEeZWfHZPniqedOMX+DqdHW3ohLY2JCYzspdeYwEiH", - "2EpVmKAFDBOiX774WLf0x3XV4EdF/ZXXL7Z0nV77ZzgdJUGFdONv4Hbv2mmn1OHT3GQEATZczO5RgPC8", - "2dR/c4qs2lFKtLylYffWILoY4sTXm58xATFpthhMAEmwxXro+cTbyuct8QxnTeJ085tTufsA42oWaLJc", - "RSmtA1k5mAs917828kEkgaS7YOaacbpNUvW4GVxdDK/ed7qd0e3VFf9rfHt+PhhcDC463c67/vCS/cFf", - "rvnfb/vnH6/fvdNqK1SN0/uz2XrBFrtqNltMwl6WsPlpaafKY+qbo9UfKcR5Izx+YXjz0NS6OiiwiYl0", - "ZMaW6QP34QuczsPw4cUXqcCyqSWGs0sUwEbOecw9gn6migSVLPJI9cOZ46MANvHE4h782jnocKJBrZJi", - "6s1baGwSBWypXmtZWEE6w9cMVZfwEfp5w83bWypohlfvrjvdzpf+6KrT7QxGo+uRXqYo46SXJ6v9z0Gg", - "EyTi+8vfPSVZ6aUH/7jG/TM/QsMbqOhccQfVIED11freEe4zdxGj3bNuJ4Df5L9edTtBsmD/wJ03pyfM", - "CpzjrFxnnUun9M6JOBWmE59ZXasUWLT+z/BbeeRXdiNn69J6ooYE+OolljZllh0fYcJfN7L4oRObW5xG", - "Yv0PvcF+giRGrkYeB8nixu6KzehYXrSPTOv9H6tbNR8LccdUdsU2Djiyu07zEcWl+qhT6xCRgZqbpasi", - "RCf/R4BA5k9WRqWVzZa5wDE/Kr2LG8BkBO+Rb3iYZQ7KwoNZHYx5L8esI2ReRFtw82YTfQZ+Am0d52L+", - "1IodFhkjTL5i159Q4IVP+m3fhE25BtGP5nVIaaJZxwJ40HYR/Jt+Cv6NLYPuJQoUj7AMzTyG4z6MXejZ", - "en4o9wRlv+R6U6hylPZVpes9OAwzHtMeh+nnNQ7E4hilI5FjU2JNQaV2NOjCgIyV+2zhnYiBZ6Jn/tXR", - "ef+pBogmN9RVLBJrWBO2ZjIQKM1sBqULdNG/u5pH0o3oqndrAUtxdK34hzOECYyhJ2/2muABwz6nvsPI", - "c+J0HB6dhTD7DOMjjc96CXl2viKZr3LVZBYRKkYnzxGkf/08sRQjGPlg+UOFLfAlKWYqbFxZjjtedn1K", - "89cnJzXrLcBtWrXJjKR0tz/CCnY/W/gkdDGVeUz0VbCV3n9Y6/hLRy1YfDQDziAmt7FB87wdXTK/Lhh4", - "zNFTXPqxQ8LtuCCYjsskQP+mupEHA4LuEYxT3VqogyK2j/ujqiGxU+iHwUxCXCtlt+gOa2forXRxHbtz", - "6CU+VChtXZf2LbukdzuEu97b6wlNvNizwb8q6PE2Z/dmoVn0j/H5h8HFLf1RpwymM2/XTXBPHf7Kq8+8", - "/nbh3NeYxDbnDzhKgnPVCNz4MYkDsOuzVAHAZoljK8X9S6nDSzpOZkRR6TNZpt23if9wAX1I4DsWgrGi", - "C2AaQZB6AD7ApcMul04EEE83woM8nOkyn2viAS5P37Cmp9xV7Yz/66xJ2oluJwJxdkXVX50a0g0f8Uvd", - "hWxFatzAYM8Nt9h4fN6ne99M8pWoh0X9FFpptOn1leMhH4rpxgsUiH+e2njdVmPIpCR77Lu34ZUUibhh", - "TiX9UuzSLCkL6lblXKqaQ58TSh8NuBFyL+WJagDyLUs9QSnFZNFYdzPXVf7ykrzpyozczXNurEtVCvqa", - "sqC6SAlM89WZOHObPJMmLNkq32tRtApn7oFpW3M5eF5NKq8SEFEexWT+VjWm6tfhMVyAaB7GcOyHZMO2", - "75xdWe+gyM2Z2A/5E5joYe9QsaIdGquKlCZyjcDIiRO5sHpTg+qEVr9Q5PvSO9N+paWLRoWF2hr0Am9m", - "aOmqtvaCXZ1SjeqZU/almYMggL4JTPHZQZ7+6Q/TwZ0nPrr+UYWPcGW0o8spmD19xUnWsoCBhWn19Nsa", - "S6fdzetmg6+z6L2w3dlZ1yQiUnTn6aKrkKH2fCEwMok7vSvxHPleDPPekLU6L8IXScyyl+pSe4ncf0Li", - "IMzSjEx91Z1CCRLdiisxvwfiZquKV0troz8l/FA+x1SmianJtNuhB1j21C8JkyH2FzrHHR3A6f0zOTl5", - "xUiZ5GK6lNwym3K5NyzZTN4KWnO0Ll2EBXVyn6cKut6Ci32fDKIw5z+mbMSGHPEZh30xvdfUEmWuOz4P", - "k4DowTXf41Z5eM/6VGCoaJvPRRJYOKKLuIm0/eblQJgQE4grigjmGNa/F7YXO2RuPLCBd6nYmTU0SNuY", - "HtrWJE4sZE2TFaddKlbMnQdWN9+mFJiurDJ4QaCuH7tz9AgPUi41fxbYKxET0luivlMF18eQxMsKKbo1", - "flSuZrthiYpbkIIEiUf9jdpE7/tgtMgzoNYpT7QxJEpwzVRgfo329B2UEAgNyUketFiP8ONhPSjdwEco", - "Xydte49lHyu6e4diTMaQ3wDsae8SNO3VMMyMX6FyABZmTjGroEmN++D7W0HM+xLjnyPTWkLORLq0i40G", - "3Avg7ur67sv16ONg1OlmP476k8Hd5fDTcJJ5CQyv3t9Nhp8GF3fXt8w2Nx4P319xP4JJfzRhf/XPP15d", - "f7kcXLzn7gfDq+H4Q94TYTSYjP7BPRVUpwQ69PXt5G40eDcaiD6jgTKJOvf48pq2vBz0x+mYw8HF3dt/", - "3N2O2VJktti70e3VHU8++3HwjzvVN8LQRACqNRHqOEZBqhIIJBY4Gk6G5/3LqtGqnDrEX3ccDZ8GVwXE", - "N3D6EH/z1lWRjxOAH/TZTbNEA5UZVUT/BLNR8okEmnTUmY9lm8r7sc0knarRBQQa6Z/mf7XPF1TIGau5", - "IIS+Jx507KQi24fNJ5INCfCtOmtRl2bbKVZsgbHIeDcw5CVMrT+hw1pLE9qC9cJ6CxAIgL8kyMXXEblO", - "SLVNSQw4B9gJIwI9R5gm0kH0c6ybCW/r2eZNueTWTkaXZUNomD6wNqc9gysb/auRlArZLneb5nJLeTzM", - "2S61a94DNUO/F7qsoLOwx4m2M2KPgc/5VaFgJjKy490JCZ5gbvAtQnSXWVg7A6Z6fN6LT4OdJ1Z2gkXo", - "OyCGDoiiOATuHAUzXn+CIbhqfpmtkxMJC9ZZEQq+ZFnoowwPi+6pxIViU3wHkJ/E0AIU5iqtApJLHs9y", - "Ienn9AHmSzU/fWZxgCAQO8ueP4vph6sjfsA3SWTvmLVNHNDa0D7nXjZxAJHhaoKqNvv8ZZYEWoDNcmGQ", - "P4iknuiHLvBZcNgj9MOIfWYxx17iFiq9KeqdklF3e6l0n9PqJZUPwbJ2jahbtst6Lqvl6617FxQsanrV", - "lJ/NWOMtqt412Qi5tPfGU7zmKJKJhrO9UpMHGqmR087eHE6ClJudSXxPy/C/GEHZ56mkrFfX+hbDmPe4", - "SaY+cqtIgY1XkXJahXlvNl3s3yqbPhL7JKXo9ZcrZjHoX3waXnW6nU+DT28HowrZWZ2HoP5y1uQuVoWJ", - "HByKsWzVq3FxvGI8VooASfnF6jyp4WUwuhtfXk863c7gM7dZTPrjj3ej2ytmE7m+UmJPWP6U8+tPw6v3", - "d18Gbz9cX3+swH1Oi9IpkiBeVET2s+/CX10roHkOAhI6TyBmufJK6hXvrY+Ub5b0QJ/vYDMpDPjY5iXq", - "4V8vD1tKE/Xsm1KQXQKDug1rnrdgAQmMZfYCeY7ysZxf0BE8ck4dDyy7zqnzBOED/e8iDMj81xV9dFL0", - "aLMZmMWuRNRN6CNXkwuVa/xVl+C0LCBvqlEaGojdPPvVObgK4MyrExZQW4FqFEhKqQMpjz6fdLqdz6d6", - "UcJ9QncQcGiMYeXOzk2q+VQkzX9OBxxrgjLMVVLW9GOvdmHnAP2MlUvUldekFNhI0RCj5qYCIvq/PCBm", - "VjtoS3FraHpJQ9MWDUBbqVvXwJC/sh3ewIVfmM+TORsDvgEJ1iU+U9mEO045CDsRa+2AwHNcEAQhcQCr", - "S8sK3suk3aUDSwcd1t3Ha+1RwPNiiLFql8pp0dLQUTZP0Q8fAJ7rjps5wHN1yP+FC9OJA4grorxe/JiX", - "XnfO56watH7CzzBG96gOvcy6RmXQo2hOf0VxHgY9J8wBvgEYP4Wx7RzAiUQHB0Ni4K9tvGR5CEc+WOYY", - "Qe5fY0NWHrtfDQR2PgfBDEoEGZkggE9mJDLehU8Z1qRGrYd9Bb1DjszWHVUCkgJRib/1YCillxVfujk8", - "mVB+Gc5QsHrpttX4e61KbnuHcbnGqA7XMqnXQaHb7oQ0CIY93C1ZvN1201S1Gs9RhA/VyFoyOu/wNN/G", - "KcMn023b59PzweUFnCazTReS7Qp9FKNF4gMCcZZpg72WuWHie84UsgdSrn2AQJRoCmMH5DRmXUhPXfXz", - "88GlUvWc3Q8egZ9Q6te6Y/sExjdg6YfAwIEiG0jE25TXB+Qnqn04YUB/iOEjChPcE+7FYoxOVfKg8sTs", - "U3k+UgoPFbmYqk03uQrhwo5TQxmVgewGLqCfZEYyB8lavGwDWMltXpxIsxOZ+7ouBg0nfhqLVdjhbPQu", - "nZBV3MH4PvG1iqBdjEgZCzJcpORgbgyWMI5hiFOm33JLTNfFKg1yqyDzkhyPK7O0fz49Z5EQE4AfKmqk", - "ExgHwBf5AozmKtHMGV5gSYouCJwY3ovLN+IKOcAPlH9zhKl2Vu1cG85FYpcU5vMpxYdM//Ks3zAZQUKb", - "Yl32DWx6reDoYmhIl408zIXeE4xhVsxra6h45otgMocvtKpEfqUUVfhL3g6KMkzVYCrEpxSOpmGUqLC6", - "Mtza9xM+3pFzS2/xdBKcTDF31KIo95jiI1phBxBVGtnlIavMH7tu1i9DBkoeicUQkjvyDIKGbbkI4lf2", - "PAzg9X3nzV+1wk7T/y3AyO0nZN557q7Sv38z5HUGV+n84VP/vPP81bg4MTgzuvrrLBEyAAuaD110rTQR", - "Q3FIBJ5Y18nSRMUsajm8d2grGBDkCioMmSVGMoiI6VeEfv9mePdx8A+NsC8mVZbTc0g01GJGKUOGPofu", - "R7gcNNa61CVx9e4BLo+cCfOWwg4zupGQV2OB+VbOfRwuVFxIIXK0RgrmFKtlT2PKZustkA1RXpxQHVlh", - "jDCOoUsy0UFCR7w/6d2fmQ1KulFZkeI46yIUHeRWara8SSqh9XRYXhU3chd7pwXGyzkpDBqpdKjOwO7q", - "6M1e5GUiaw8Egyo/tyQX3vbHw/PtSgUmiPcAmxSO7SKTrXRjuLwAs3MlyUgxqY4m/Ui97prWTy6rwB6Y", - "2Sbh1/DST1RT20pVDe8zANRLzxQ6IFg6f46vr3oYxgj46D/s6ZGv7GglpbZissIxEsaOCwichTH6j1ru", - "tXx2QBhUJbDCBCwi8VCanrvcaR0G9i5c+1WfXBymLM20qUKuch+Vk7F32eySlo7iTJeFGS05lTHTRAFG", - "WySTf0fBTMi3qyZKjPA7T0HN4GQWEBBFPnILuYfWKeQuFrVWKXftvF8z8bMHNmMpCA336vLGGtLT1lN4", - "qhcWtpGRo2YPa7PWWWSUU4g/ZbR06jgJjrZ0kzVXcDGT1Q9SMb2ta16RmiNO67z9W1Z/y2ZP90TJ6iKk", - "hSn9dwOLVpUdScddCF9A1wcxICLrjdkpQXA2wo6XdXF+IXECf6UHeBSHsxgsFuzq9Ms98DH8ddMOC0Yd", - "R1HWpKrDFLYyPg7DTLcJhaJi2xtYAevG3qRgrU7EbzIcZmShstFenLpZSvb67LufT41lfwEhcBEZ1F7x", - "UZFgxaq/mpRTO6kj7MuivNVIKhbQfbnyw8V0UrrnOhIvHZaJxgbTzesZF9CxRkXjbKR94ITK2sPp56pi", - "i/3xeafbuRiMzw3L5fW22qfBpk+DHG/beRmMxdhbfhikoJtMPc1lJ12QXm4yH4BPFbnB2FVVWvDqN2aQ", - "Nl8xF5l1ZrxKbRqSGEFcv3z65YL77Bir+NA2VgY7nv2LGWyaJR2T19Bm+Zl5Ew6cOrW6Zxmu9Ze6dMv2", - "QqRmRF/HFZIgt5xgrGFGMTlWLpNYMXuYPvVYMaPYeHA1uZuoi0nXcMdPyFL6s/PRoD8plFz7OLy54R+v", - "by8pdiZ348HVhTKy/uRRZKylqdk+1w1GAY/cbFJqADaloyxlbKn4RkCQv0qds+piHc3KcXAkmJnyJkQB", - "4VGK5R0QtKiVrVlqNn3wN1rAVSPweCNN7jerZWgOYu4q1nRnVdRY3kOYCpUEJny6ldlWrVzQVJLTu51V", - "pXssQNgUI9nSNOSeg00RmamQyNL6nV9/urkcTErZ/CqSFOZfu1YrY6Jc/vMHdTbNus9bTKMThtMS9jeq", - "UKnvhWYNU7ZiA2H7B4uap8WaW3D2npTi5Alg4dbRIB+Al9eY7NygNVugjJhk9XU1w4mvxaG6DgqcBfJ9", - "hKEbBh6203HrPGELszi/pCH7gEBM6G+/1pePt0I/HV52s8d/nR9yBcoF1QuvevljBAMQoaOrMLhKfB9M", - "ffjnmOXNSFv10CIKYzapcMUvN44AveJ0ZojMk+mRGy6O54C4c0h6HnyUfx+DCB0/nh5jGD/C+DgE7Iz+", - "1gvEWJ03zNC6ZhhYshhH4CmA3nklOyo2ct68zJhVubvLA/JvDSnogPaElyRganhqeLB+wuKdU9lZq0Bt", - "4b5nURpLw6FbKo9VVFSzegWG0ljlg3Jdy8NqG7nB2S0eBCov78MAw7j5kYdEt6YOFLbvF0dqNdodVSQm", - "VkaaNAOIsNHI6815GNyjmTbfSHVx2bUqP69AfIWYI2twcuWTyzOJyHfNROsUzlLt46rW1OXWGxkNpDmv", - "0nOmm10gCuyqWn/yrJCv2MUNQblHJ/0WfC0q9Nu1ClUbYDelFZcCilPgBSTmC9kELcTT/RYtsB6MyNyg", - "99JPOWUCcS+wJ0BgfA98Xz/kzhTRteufbUeTaCg4uU9DQ2TRU4R3tEfXz6bQaKzrG7grtkrLD6S0rOYM", - "p+oAaxW25MK3cMRe5A7qVQ7dr4Uj5CXPUUpNLBF6o+NUHH0bO013lgSv24liFMpaKRq/cfHVREr6um2q", - "Plvj9ita10f858btKun6Pp/y5EltWOjK/mb6ZwCRk6oUenlwYXT7GQW3l0FsGjIwFtBWg5XsAkplh+fu", - "IZDN1kvEtIGbL0/zm3G1NRcrt/JwtQwXVeIDv6qsqQRql5k0Qh9NIWn9myHjCgXJ+dBCHRHMIfBgbHe6", - "87bFTRTT1uJKmakr1/G1SkT1FYGUDyTtpoHmXVM0pDJOLti2qIRaZbSiS53SURhCdWhMMFWRTUiUX2sH", - "KqAsHbUmt1U++NSfUR1vvlDxNv7QP+106X/OXv/O/3h9etbpdj5dvK7GXhrPqskiq0xkHxub9mIJTN3Q", - "s6hXlxthIDsxd5pZAEgSww9r0zEd2knH0wpMNAtYcSU3hobLK2bfGBumsgzNAqsJigG8KaIUPOlXXASt", - "lkYGCt7TsOLB/2HlCscDFhTD/7gdXVaTx164zkmdxtIhJtWBTWmj3DnwfRhUOYU2CNGrdICXz+6FI9GJ", - "JXCW2n35gFa29v3gajBicvP9cPLh9i1z8xsNbwbMQ69//rHT7VwOrwZ95nz3efh/THue3WA3H4Rd6aXS", - "3LdDmilb/47Wv+PH8u9oXTDKDydrGmL3+yHhYOzYDd/Iax6lNRZv8U69ltWbtc5M3tm1Lf9EnXsxTl+j", - "VTulchpeQCLLKBScfJPA3itBpGDAc1BvhVFj0Wn7d2GsgUc+GLEMHDZhP6xhpozkvQ3WD2Tg4ODNpZOp", - "deAox3J3cjiR6JaQlbc2rw7kt9eriZ7ZQj1LdcoqYF/q1UXVjho8uxgwvqknmC86lw+JIvNidhT8V3BM", - "UqNE++9FSTetSi60fl5oYqMlFhuZPEWRCq3ym8SGNNSybxL7jQxtwiBCx9XtdQ4lPMmXufjAphaJ7UwC", - "VK6KDOf0FHOG904QEieKw0fkQa/rACcGgRcuZKcn5PvOFDozGMBYXmNU6jrbGsabo9nbTwJcbW92Tcop", - "nLXIplLLbLrYqeUlL36srC+5LkbGFJf2O2DYN/YkCgIvq/AY86FWu/IvIJmHXqPVCtA/8Z6pbn8eegaq", - "/TCZ3Mjc2W7opRQsDT32+QbuAE84wGbOTfzVEuHVJCRQWXPOZ4Yq3to68ZiWAlamnU/p1mXGrkmn27m5", - "HrP/3E6YlmQ6IXkQFq6K0MLiTYjXYXJB4EQwpnR1ZF8RT5iV2G1Xm3WLUgJK8/kBjNEsgJ6TdWLWoNvb", - "4YUjSHr3tzwfTKGPq8uHsjaMzHM+IVw025EHF3J0HB0afYDJBwhiMoWAVN3Xc7vGqsGyOg7Amcve+Zvy", - "2cnZWe/0rHf6anL6+s3J729+++Pojz/+ePX6j97J6zcnJ/ZpUgBnMHpkDzABU58ZwPYQ0u2fzuZTOYYu", - "TKuSYlN2FtqGR3/wqnRhvApJjfJzaagqFtV8smqeuDaTEnayXk4YqLvYALLivFrokoBu4TC4D+24Z6R0", - "oEeTH5LsgrxKvWo+7Dgbh7luq8ih3xzwCJAPpshHZMmO51wh3IzIf6EQ3bH8t71/Jicnr6DzXXb2YVeU", - "VH7+VZ+n1A9NZxOGCxDNwxg6tJEQQysSzViONWbz6UL5rYtkZFOn2W/OJ8PPvCB3+udN/3ZsCDS2iW7h", - "e5RGtvCz0pgRTJze/DwpAFlvwOO9b+v04dvRpWb4puoxa69VbZSjonSyV2bulbmeaNdNOwtVFNnmxbVr", - "Jq9OVFqBh5d/iTVeBFIgR3lRVqiwDYJZIp6xrIXc+OIj5scu76wUqC6n19GrakK+Dr6RGGgbYO/BPGxp", - "cQwiVSG9vuyzDAM3/5h8YI8ik3/cDMbno+ENy5dy+/YfeuNOUeiWaKpW6AIuCOnQlNIKuq8UuHXxGGlD", - "Jwly4jw3uKZGPytdbqoNixbJQpmkydC64ucVnFE0qo0Hl+8+XI95qodP/as+TyHzZfD2w/X1R+NesOO5", - "bAJW16aPY0p/sXCT7jYoC8sVEVkYVl9O9O9wajii6BcdQFac/mc41R2JO9EojZiTRQQ1ajaYrb7W1DYL", - "tBe76uc54WaY3e0qVyDet5pJXOUpTSKz0mauOWHT0A0DD4knFn7LczWpTmaQKN9Z6XmN90cgk57wHHwz", - "SLDwXE27OjPaN9UaFLu9FmGM9cckBgTOarODKxBe5vrxstbmi0hZWqUQk3zh7GJ+4Vdn9eJLTl1cTVeL", - "1aotGl7o0iCmAA4vtDiUvT+iIGdIeXd7dT4ZsgPr4nbUf3tJldSL/vtKAUkHkZpIIwpms2vYS37Xqzdr", - "xWDuWDPSX++eK/bTmDuKMclHWBVOSUICfB3Fpjz2AJcGnyI5PCVLu4hNeTsHDo6gi+6Rm03i/BIBjKHn", - "PCIg/NV/1XOFERENHM7011sSJ1Azft37req5lRpgTk9OToyeWNph8r5TDd2gGi3o73AqxZjtOW4o/LB2", - "dDM/EXdtpORzC1vPy4CQcybapGOQ6vOh9Q4ylxp5u2ww+ETpVXbXaaiSGB1+1skdng2kuvIoYH+tFiZ7", - "cldWnH7sD4VREqyRV7k8yjsE/dy5r6btyGg5J8UUyVgzyVg6M7Wyu5Xdrex+KdltmOMHFO0V3pAriGY2", - "2pDAhdm/0nBfqe9srLY3ZqnQqhPurulxlmVb23gStQ0MaJDpxZS8xdwUYlHdEiKVUeuop5Qp9mZwdcET", - "xGapYjVZgPM5Y9P0sm/75x+v372rPSXZtCvdm/MCxUyMk7w4KfrbhMGNIvlLsNIGY3cOvcSvCIoydF77", - "OPpSzJNiKWBqNhvzKupGL6RcepYtsmNVPTJcuwijkYBlXG5CR3Koc96xTgstNC/NnzGENrl0VR5vyXTa", - "j4K5tN8kjzbPDl612AmY6dDrc5VxfZN/sOHkKsKsyyGsoh8hFM5jepG518sFLUtzvrxDBm6sm5A532tn", - "ZHLkTjzebnparF9hc82ggDeN5IVpyMUqA6f42axyz9UtPfoyDexOvEI0RzNPMWOUp5t82aoCQ9Fmiyyb", - "e8Kw2RD11YM5vdyDxCc3lVmWRCNjtiWrRwJxi/wT84N3YSiB9ef4+srhQJfDdtgIWica+Sz4Qo99Yexx", - "b0wLNGChdkzQAoaG4jiYIPdhaXLFod8cLJ5V7F4SFXnRgG2ZDvZ4Wngps8Kx0mfMkz/pUP6YUbY5iavN", - "Ap+U92zbd4vGyXKtr4FyWZIwcgN9red0RlabfBtqQp97sSe7Qjh3qMgehQpVhWMIub+KsaTIAnyrafHU", - "TNk31RXhkR8Jlb9MfnIIpxDEMJb5TBhG2bHCfs42ZU5IxK49YfiAoGyO6K7yn+Tb+ZuOCGHO+orUNrR3", - "gkm4sJzsmUl87haliR7gszj9myErd0WYTSz/a0qIndOjk6MTRsc8iLvzpvPq6PToRMRjM0ywmGtflImd", - "6QJk3svnedoqgBg7qT2GbjqQxU06l+L7e4YGGdDAZjk7OSkP/AECn8wZil7z724YEJFUQ9STpk2P/8ac", - "r3B6ANbw8SCOQyqFn0v+qVchSdeRI47Om7++djtY1nChq84aSp+SvwTM7hy6D52vtD/DXwyBt6xHIG2G", - "qjA4kg32HYVswQ4JHeC6MCIOicH9PXJrMZpioBalj6fHwKciJZj14AIgv8cekvHxd/az+tszx4sPieb2", - "dMF+xw5IM33R7g7rzt+mS7vQpy0GtAFzteAjMJ6JwQISpg/8VeHkU5rBEXnOO294HoRUaJSW0lGFGn8f", - "yHZsvZq8X0v09JvGkzBxXYjxfeL7S4ej1MulSSsh77nb+W1XlNd3FsCnWICewzJoeTLsiIPxauNg6KB4", - "F8ZT5HmQ3z4y+uZ0UkVmkuInrAk9rL71YqFysA+8b6erIYyv7NpLXE2WdH7dWofE+Qg/Bokzengbcnm8", - "EWLg2OGbVkBcGrdWJpNKbJHQSSTO89h41ov9jSxEuwQd7DkxwAFtxYClGODUsj0xoB6QEeqR8AEG9FSU", - "f7PTMAp1KQ1G8DF8gA4IWLJG1lp4a6UzFsREhCa0lTTo0O42UiId3iATJKx7ddzFbHmCzhl0PzZR4yZU", - "LUiHbuxE7Jwk4+y3KkpOtzxHwa4fJt6xekM3a9ClTHHy2sMGcVCACQhcWCLic/pZupeYFevt45YB4iRB", - "FnCxLwRWo7VzBKvv9WLrPykvbN96coheGHFnF3GiKfvNzeHH39l/n6v2m0op1uqotKHMKs43slYS8STR", - "JuWEp3DcpRDa3GaL1Eo1hzevofIoxBrHBtuxVrblSFzBTEbeHMUVUo3Tz1czhR/XiTW2LalUq6H5i1SA", - "/ex0f8FIuKX9/aL9BVz5DDee3rs7uEXGtSY0lR6JB3KQb+IIp2McMzs93yVs3PFLhOkFyHdyrU0bTFsP", - "8w23ttt0LrHjypQNN19mwMmtbp8IId16thGFTSjvf26TwwCRkErz4++c45+PozicQvPlUr59OiBXeYLZ", - "dXnlilwuBDPDp1PfhJiMkuCGzWtvmzIdeqnk2vGpV0FQ8Bt0E2lbYfg92umpcBUSVoEgjNF/eJZ6kdOI", - "B1/zKM2SmZMA5EPP4XZ7h22P807I82G2rfqDI0dm2Afuw/F39h8LK74zpg2VAit5ymFfRXIoe6N9bkwj", - "8TAQ99I6n8fJPqk2p7sB4zbISJhP/Ho3E/OcYyx1I/D98IlOr3sRKFKtFL3s9yoVixNdnmMCfPwdB9iK", - "W67GqtQv80uAG7BJfjAzo4iTe+/YpICMllH2kFFKBJuyytW4klECrGETqbgo1ia96kLnlVfiEos0fht7", - "Mf2jazYE8MJMK1kCFBjOXr/OAXG6CR0oikP6D+i1Z9gesabpEsnqNzggiiS1l4813qbAjwRMfXjsgRk+", - "TlO/Gy+NmN0aWTuHzAFxptAPg5maVSBNMw5m5Svl59MLwMrNTkQJ9XpzmUzwnSVo4Sm3Gcv8O4HxMuMZ", - "D8zukFd9zG0rQsRK7hTgfamLjzX1bqwG/gWYnYuYL332sQo5RKeUr39s1p/bStjtvN6V8KO3ULSIfLiA", - "ASnpBsx4IekgfToH+EErYVjD4+/0PzXPS7zSxXTJ+aYoQOgElqZ2No7x0KeA7vjIB4TARUREXhaDUBCN", - "OiospViobdrxCzU9GpneGFZ/dv78jd99tj/rRK2/TzWF+zDhSZr2RERk/FwSEeY7A7ERIcd+OKvTVfxw", - "5vgogDLzkYCjKFEuw9klCng9lkOUKiLLEwlFWt7p0iBZeBpGLTQoIKyoZDno0pA8NyYiNXbozCChqGZY", - "NsyMEbc8amauSN1guDelVQWspk4CgvwNTN13qLzrEfiNOBiC2J07bCalxnPF+lkHnUivXiujYPgI/V/w", - "r3QiFLh+4kHT/tKWuKPVdqsFvmQBOoCtcuvJ5DYUMBalYqY89vluurxLO+WgtAKulFPH6pC12p49OHJV", - "IdRAIRZRrO27eV4rTSW/cuxchrP1Tx36/70sdNj8uqoUajMePGkdth/g6MEPKDIx//09hhs5d7Z60m1f", - "pc72egUHmfba26rVORmnkzDrq9ishWKid6F/7MFpMjMb6QePwE9YvSfnfHDpwG9RDDELqgUzgAKc1U8T", - "9YE9QMCRRh6eQ/+CTXUoLgWbj2j5fHo+uGRIqAlgYZjEVBSyesFUTOiRv9M4FhV8mXaxRtRBQT2eZg2t", - "XqO+xE2TWYnFFJ4/H1yaWd6K1y30Gv4AkBc9aVXjIj8302328Y3uR9JvNDdaacx/gEusXJSM09J2za+X", - "jAxExH3dxfI8DDCiV0lBYuyRKXRZ5g3PAfcEivIT4tq+TWNDNSxTeB/GsBaYTZkf3vGtIWEOGhCzWnSh", - "i5gEfUJkrr7FFctDa+DL0koYdnbLz2T268ql8XcWgLhzxJ4eXRgTgIIsdL9qnWk2PriSoaRQ+N16cemW", - "iFVOl/S4Q7HDnyt1EIuEfS+6LdOlk2XIzXzEWTG19F5isKmUEwhrF6Ip5iCneYDLHq/IFAEUY+cXDzLB", - "R7lv6QDnX2/+9WtRbFU6QdgZtrAbRtBKHvKWtutirdeDd7t3VPv7aWuBqrNApbxhGbbRQEE7ZsewpZbG", - "z3YrTe0jXB6Ksrb1MCaJi6aMwNDdMoOOGRyhPW6BIb4/nvYaBK4y3wKC9f4FTWJY99iv0ASTxNSBMqfY", - "n/aA2khoIW4SVphSjhVnch3H5pgSLWvPKK6StuaEfTUnlKovWyjQtbfPyilKV0R2GedzHq1faqLZXQEn", - "UwyJ44LAQyzPjKTrjd4eqlbs3GLoMTbisBB6PS7DA4i0ubK3e0PRjJ1ePBTWbiDYpYhpJXte25J4yWQ7", - "x2+VrtU1vO2cs1JDDnAC+CQGNopm3vbnfrxhKODosHnAYe83KSk7rLATt+rv8s1GkEcd64myUwrA7ZP0", - "rp6kr7JX6BzDp/yZ8qY9z9trceyCxf+2CW8EdZKiceLO/VLjBLciFpftybXoL1spJg7ztmUpGmQsZysW", - "XlIs2LJ+VyFMevRXhGKkCrzZYMJnO2SLScrPPzkXz0LSHu5Gi8kKZ2yR0SrTBNcfmwce8Jw7NtMkuy/J", - "cNu4AvBNWvkK8ALJh63lg8w33MqHwzvlLZR95tu+yIrVVagFQjLKQGAnTgJH9KzOW8w9KC4RJtyLQtbG", - "O1SZVo6EUtBQ459kAejawVH10GzKQalom2XW38Dj3jrm6dOaZuiFPF0o3LxqHSPl/4XVlAMGoEWVO9r+", - "Tra+Y623SmxZCgT+xsdcpdLKu1n8rSHZAG+Igtkdr+G3I8j7Ggeih96j8OmxeCTIPInuFpWuRC9rxKaC", - "bZQEUqI1j5tWpWib42B/ApjZ3izSg8ouxsL+xI1CFBDLc3eBgoRAeh2Xf8UQPHjhU5AexQ2O4feQ3NDJ", - "D/0QZgee9A1WQneEwbrTVarUn52cnfZO6P8mJydv2P/+r0HuiO79e34T2cQBySBNPYdVUEMK3xrA3qMA", - "4Tn03rLBm4O7fdmYI7UVpCPjk1Y+7ql8zO/OxqUkPnZZKXBzFBovFZ7mo9HJO97k536gZChgqkpNgSSe", - "4yt0XIm0nUaRsUl96PE8YbUvk7J5mySqjZYtyaiCZNi4ZIph5INlVXEn+r1SMvEmP7Vk4ihoIpliibRd", - "SiYOpq1gikXrVi61cqkklwpyYYNySaT+tPG+lenV67xvRfb21v12n91vObk4dFi7+DXW/oo2XyUYUtDE", - "OB3F1t4qic4aUNGhAtLqSV7cw1VlnwYurikjt2/xeR/XFDGZ3BQoXtvL1VTEIt3E1s9V+LkKfDR55ZZM", - "+UKerpJGmri67mPy85/b17Wc2dyC9xuoTczdVfzDzt+1VmYcuMcrnVy+PUoWrvd9zbBiBna3dmhb/pf+", - "rC3v74WrSy17d1Vyq3FplfQrfFqFemjg20N2ay0owD8aj0pv1ZZHDe6qNcckDOgp2IsBgT12A6WbK/be", - "ksvq/Flrj8UD92jdLodtzzv1x1XcpYtqKxj2SHHXyIPVT3b9Df4mxCy/BwrccIGCWUqvC4gxmFWc8CPo", - "QvTYyqAmMihIfL9E+cHSicDSD4HnoMABwdIRq+12CPxGjiMfoAKlFafciQyxyEyaw9M98DFslQtD3THO", - "eBp2W5XDbe7pwme4FydB3RtHPmtg7StHliWwfenY/7ylWGRytHrr2FnWR+aHD2IfQcxyXUMr8LYYFOAD", - "0gSUjVVM2RvHb8tcNQcSrUCBSOPobDLswHjLLv5f5pDMuQAQFWqci/57TE+vMPCX6u9p3UCdQAr85Z1s", - "UKuoTMPQhyCwiOnIFZG0wNkLhXdoSl0a4zwsMvu+WLyHc++DGTtqnwRdhDFzwFDJIL1fgsBzwoTQP4X6", - "iKn+SBtIXfDIuYD3IPF5vvt/UXr4l4PunSTAkB3juuWLme7koJ1KEtpZPb2mL8Ct09C+1d3IaZSqoit/", - "H9Hf13yJUjXcYw/hyAfLHnOXqNF3RVs6rHCvCO8rlOBqHfiCD8bcLg5aH1ZEK07fsXJIEfGSAn0CdWZF", - "QJGlL1JueMsmeC0JtKKrFV1NRZfkkx7lk2rJleNRpj3oE/5n6e0qJNdADDb0Dldwtffc9p77k9xzd3ac", - "ZXKhPc1+pNMsd3rs5GQT12tz2M+EN5BepfkLe8XR1bqXngrUKUipearOkQIJhf/mrt+oFa0ZEoB83MzP", - "VKWQ9r2p6PZZYKANMHien5nPp/JLTSmJPMmBwGPOZOn5T8L0KimKJf2z4zGi+GfHiQwP0hn9WLqd5WDg", - "ts0Z62l4BVaWd7C5DFfgsvYU3+NTvBj+ZsnQ3RJBr8Dix6JkXBWnE57liyTMcJTn+6NaLh7LmnQr8rI6", - "vaKu/5isrV4/W5beUyev8zDxPR5PSy+SOs1lj3KT5LgqLRD5IrKGJXuyKLHLwnJ5kDu31NtfHdIi89ZG", - "r5+nIk0mVrUGkB9Xoq5U1bEVqq2eVJRdBC1QMKvXlkS7xtLrPSQTMcXB3n20MsiDEZnzjCU8q5njzpHv", - "xdDkusE6NJR+2xckfHNaSXLwkqSKPzctXmAkZIr88/kYxO4cPcI6LUi0EmDS7loRMiYwEu66fTmwhfiQ", - "4xmtpxLe1nV3dY1smzJJ7LvYcyuplE8q2dYF3X0+ppTrCjmZykIqx/4K80v5RLefyqYq0ZSycL1MsrmX", - "idL99vJoIGusttLoJ5FG9netVhYdjixSGH/7ksgPZ3WeUn44c3wUlHSjsjn6MpxdogDaWoNaMfSy8Uw+", - "fIS+lcsQb5mbuYoZJB3QXu8Q9D1jBjlID16HzabAUVHMhHVoCsiY99KGkgAWKBDGXtX62ee3S76WhpNf", - "q30NeODTeyiGroh2r4DiQmm2CiRZ/+0eUqo0aAvor5uCLpXCyllwGc6aHwPC0agitTnzgMDCk8jguD9h", - "P5+rji+bdszhg/OJ6pL0ctekl3HF4RA2cr4RSP2xaXwFr5uU2NLstMKfpkjkOopOXedqTcbcNUa8sFcS", - "eNOETGlgh5jB+OSzG2+5l6V4mTKppfbd3jY4MXoh5BcN+I2fwKVCGrbMlstoWp2DKeCzoWBWzVeHk4lp", - "S16nHAFNDrcopogkiMdlvEDhzvacW/+cE3yyAutVnHfHwKeEEcx6cAGQ35vFYRJVPpxS5U7eAgV5sTEc", - "NoAjBiiybp82GdAW72mDQ4l02v5JqENMw5JTxk1oeSf/mlhBrY3OMeurT3muOsb46UMq1JtbATd2Z10J", - "5Y2udqfbZe8VTkANDbV8rb37ablts6fkMYaE1LkWYbZ7sosju1RnM1DIBQWzsehzIEl9d3RMKohZ44xU", - "96RlJc21ToOmjfFRhHokfIA1yfCc/s3Q4e2quaYfoQlt1uqT+Jj5Fd0MGT6wRepIHZ9I/6jWhl5UHilF", - "ctQqzJD+uE4plyCjdjtib3VEhgBJ64pauE0TRnHSlr82HDabMVNDBqs6cCy8pXh1uZzLlCntauY006Zb", - "3Wv3hAe4tHJOoO2ap59hZPARLm3ymmQwpe7Lwwtsmw+Ty4rGAEqX6OHFiiBmMWhrpPKxgXCUBDyOUhi+", - "XsTVg+3nyzh6sKn3wM1DhUN18qggliyDEFw6j8BPoD6PEPwGFpEPqch+gMvTN6zpaadL/3XG/3VGxXt1", - "vqFPm003lC2DJy5NMw5V0zlrPDz8TEMrRdq13jWB2edSUVoYctc3IbNxDTpIewVgCGC4qDELi8TEL+Le", - "wymhic0X8h4/u3f12X/vZtaR4E+hnsJvLoQeNJRz5HvTgM/rLybH08R/MLvTvU18UccI4kwm4EqhQPv8", - "xIKBLr+hcMAvKR1wc/HQRl/smXxgbKoKCbxhKeGCwIV+hdst+84NGUri7JyKa5Ia3K2Ej/AzKxQMAfYK", - "hbgwxDDywXLjYiNz2KL/esouy0OenHhbRTzkD+H0b+haaC4MaTDLUdIKqb0VUiNGqduRT8yMZmlj5bY5", - "CzvrR7hsn/UyY+NKt3WG7PbGrruxO8L2u0k+EKeB8ZzmPIibHc0jecT8rEczR8C+HM2bMatx4Fqt/ic9", - "ML+z//aeEJn35Cdm3a4NPwIE8MMzqDQQXgAC3kPyBZH5RLJ9rfyQ7KMXHyWQd/12+cOf8nTTVknHwKii", - "PeXzvmwKZqx5t6sh8mp+RsEjIrBpwITspXcCHbKvre4rfT8VfKzk9Smx3fp66sIhMlrcUgwEn6CS1tvn", - "LCXqgaPELtiB4/ZFIxw4uKsENgjC+Nlje8/OdqT1AmL3zlXkW51cgAGY+rAXAwJ7bEzKHoLXVtGLhRSS", - "P/T4v5+5iPEhgWVhc8F+x6kZyUbQ8D4H672X5/pq2HopOg795K+VLZxC9lm25NiME2FGriZdNL+PtRH0", - "zTjhcKLoD4UTthvov5pW8GKh/pacy+E7GM4VIfiNObfq5FvAxZQxX6MbpOylZ/FP7Gt7g5TUqOBjpRuk", - "xHZ7g9TdIDNa3EyQoBjv+Dv/w0IJdIAAwrmPw0VdkC2nhh9DFRTLNsHGP++Ud3/bCu+uogP+HFy7R7lq", - "rwypaVMmzW1MA3nRlYRskUaqNIlZBPwYOvBeiIDtKr98u+yUX4GOPUl5ZSm9NHqw2LdWeL2w8DLKlRWE", - "V5XWE8XhApI5THBvQXVQt758UdbFEV1SH7y6zJQ3addPYrIf4qJA4DdyHPkAFaiiOFKTO0AZyy1TvjRT", - "Ug7Q7MumbiD/TmACrdmQtW7Mgf9Dex0Q8x12ZPMhBatu3x6So73VMlg4jzDGKAxambhPMjHdnbJElJyz", - "qkzMnvpsXL3j9LGxztd7BAi8pA3bvBr7XJ12EzkYajG5zUwLKZ3tQbaFIiy7KquR57UGwQQKO7d+hgUr", - "uIqbTNwyb4tL/uuqElf06EWhj9xlfcpJ2cHhHWwSTkpX6BvWo003eaxDy2qPRoXdaB+Pdp61FfvAfahO", - "NDmmTZwnOJ2H4UP5OZV9/sK/ts+pPMekipMmt4cCqveJHXZU8fg2AAmZhzH6D/T4xK93M/EnSOahxyp6", - "AN8Pn/TVlvkGMT2Qs4B6nrGPazHiMSYgJkZ2HNOv/By77idk7rDLSpEhb7F8tmEAXVOEsp6HyJmvTs40", - "eFC5h6FMHCs5rMwh8ITXiB9ygqmxeLINh24SI7Jk+HHD8AFBOigrivRVpQeG0vyMkhDoDqxMB3V5f8dX", - "4yIBFgRygFs5LOTw1XiooqqBJC5iuZXFeyeLy4yQSuKr8RrphgsD6xisjcZgCMjzV2WW4c3RbH5S66iK", - "4q62DL1HDG3kPEuOrjxRRZ3O3i6erETp8EN7udq+uUCHmGY2g7SedW5n2keVfXhUSfdm08/Muqrqlayb", - "FVB3pkvOUIXTmxPigdjxuvta2X2bEkNs0YryoZUIOyuFqtLiE+D1UOtEhHqo05/oRq9aZbtaTtTmBOwT", - "AheRSG7J2iriwyQ4Di0ZYCtBqlziEWa+0kKEcCLw9++C8MKPeHWMsiuGjiHtWJE7jCVZtOVh1rxl4X3M", - "ZhYngdiqGo92FEQJ84fgj7u65T7vhabS5jKrkC9sw19CoGRrqrQF8GbCWaBOuLyHZMyHbUXLy2kHzbL0", - "GiwNYrj2QrHPFwq5S1uRGgTghx4mgNQYDAF+YNWghKWwxko4AfhhzAa1FxHDix/RNpgiogGHanHd8uge", - "mAFNbLCL9EjCa6b3FMYPVckiMgdso0tT682UBZNwVHxhSKUIqarqSZGRBrzwjo7cjva5bd/ezxXyXz2J", - "oRjExEI//Tt5jn84NnZUjFczs9coBaHc2pZz9++hXGW8lQ5LRhXVD2n0hOTCu9pLPjsbfvrDMsNEW/N6", - "IxmqpfaQj9Fb3btSIpobgprXolCr/2pKUigle9vCFEphCgUvuMagm6uv/HJlKnRwW5ezV2y9OYJpL6l7", - "Wb4iv0flcOBqU1ITgfNd/WedH0uOE2pPYEGmh+zWUmB9PWgqBg9YTRDbtWpmgdbNxRzXn39Bqo/p7+Zp", - "anV+PmaPkbWPSfzJkjO0CvRRDV8P2egtc788c2dZTG6UIpQcxnXenfI4YtvdmrV3ZNb+ouI+sMkfkm1S", - "U5VhcxIHz0EEt6RHjNnYrbw5GGWCb1irUfxAGkUauyJ8hiojQ0Wldsbivp++j2ONrlHF+ixwkruyDGRh", - "v1YGbBzAS4CJM7xgCevn0PGB3EFTmiKAydAz5il6dabLU7QDH9smBT1LZflak8j++dasIEvsHW/sZCG2", - "eplgLe00mp8ycZoH70Hik86bk25OVOwihVo69+tVJh/zTGrTpcMm0E8qPpnzOexC7Wofezavb20yJWM6", - "Zm0w0LmMa5gC4s5Ljz1VGtPhBANty8tBeSfhyLB12xfRJOWnkk0/9kSKpeZ7qvSNkmDo4Vzq2bUQXM63", - "29AgJCKQ2tejmvRonGx28XKDj904DOo1EtrK+TucZkCRGM1mte4T53EY/NRqysHkd003Fnl02hkkqUp8", - "VJPG23Rx28Jdl87cFLyrOlVKOyWj+CbT0Q7NpzrMDOUVOXOnS+de5OXdWOpeVYpg+/S90+X2MvgqSsGO", - "c/jmkLGGht4euxotvXTObUldp4fu8Xf6n5781a7MXfkgtn74oIRz4EXv0tWbwMphdPdl7yzr02k3sc0P", - "XKwXp0dTs7eKPEF8fe5WPSauyVyH7J60x5y1paOzPTYPwbDf6LDeiHyoKy/JZk1ntBYOB15rcr/kw7aq", - "TaoCYsINHFa2PkoFvISjjW2vTlVQi0G2qkK1HBBsuQ1RYKfKs+PA9kFPfWWsd1NqDWb7bDBjj8gNrGWs", - "/Q5NZftox4tATJFmcF0pgMUbf1EfM3YEnyZFjBY24SSyXbj62vgsloggwdCq3qJsu4p1a8z6CjuTDXAP", - "KPCsoGING4P0EQVePTQHb0wlaAEdcE8BLTlPPwEsY5nVJXTOTs5Oeyf0f5OTkzfsf//XaKxm3ft0Aj3x", - "0mO1R6Ho2FYjpxBP4X0Yw22C/JbNsEmYK7B8jwKE56vDLPvvFM+bAnqjmN7e40DZEv/TPg0UdcfWwrEV", - "d+ntvAkwD2mb/P3AEaDRgy7P/mpCf8tAiEOuQN2q4a0avns1vNUtW93yRUKg8JoV25kAaiuL1J/vW6ie", - "np3zFFQv8enxWGM1TFuuYj8cy86tFXGfrYjbuxelBHBQnlOtMtUqUwejTGXLyET1RmyzKUhWDJ5aaTUw", - "bzVGsiRhWqvDZrUSgwawXb3keJr4D73ME1EfUfQ28R+EU9uGFBU64uH4J27JD6HMUxlabMOOpvVbs9s6", - "IpVrMieeU0ksTtu1EkJKiLdW+7x1ScHdVWokBW/k/BJD2fvXDYqNw3Gu2qnYkGk6G4gNsU/7KzbkmmrE", - "hlhHKzYMYqN2n7cpNr6nf/ZKOSNrIyD0IDcUGgceB6HBgbGakRbVexsaod/d1uGxGBthwFMzj0cDbdRE", - "SWyEAQ+6QvFBcd82D+T2rn/oMRTbliPV0RS568CGJMuBB1rsvXDZVuxFSbo0qI+akVE57+PLXllqJaQa", - "7PFTKj8HUP3ttuqytClZaXeJSlNoPmeZW6rKWDnACeCTOX+LffoWEQ91OEWv6jOJVOfMrARtR6KRY3vV", - "sDRROdq4+TuVjc2Cb9VaXWb4W8m4e8m4d4VOhKCrovLtpM5SZHHOqUcvj6VuICSyvYarU4xaKbxLKSx3", - "YAXNtEKt23PFVJXArWLail+T+BUKSZ1OvHGRy6vn9dwwCUhNvARrI3ORy7KP4BEgH0x9yKSvIm709oX3", - "kPDqfPiczXjworcuZfyBl4zIbdaKZkpOKpx82hdEg8N0DkmrFZLIs3+CYYyP3SSOYTVnY3474A0d2q3E", - "vbcYxu8hOReDbZHu6EwN6YxB3BYgfvkCxNBNYkSWTIy7YfiAYD+hsuuvr1RUFZIO5clNkjvbfg0ZzxCZ", - "J9NjF/j+FLgPRnI+DxeRDwnkNH1N53e05xGdiNuj3rOhrykuz+XwBQJ/dXJW8/bqinm98rxzCDx2uH3v", - "+CHfjPw+FMX6cwGZOdzJBebnsEQfJiA2i4Ix/boa4ljX5lhj8GwfZwy6hggLw5kPt0NvbOgfnN44+jZM", - "bxnifjh6Q8EjIrC6dhNm0UxSG+YdmNJtdXzTESas71DMtcVTXJ3IypndR1huTH6Brb5ofayymjwF7GWU", - "N9HcEHO0dwxcF0bEbHnrs+84tbCJSUrUpm4+79PZjj2JD84nUgxJBgNQBfXxlevor/WYSsmLY7u09/b0", - "FUNW3aKikj793oy+eJ/OturS08E3QF985S19VdIXx/YK9OWHMxSYyeoynGEHBQ5gZ+NRhYJxyQbaknMG", - "PYLp+PWEtLt7tB/OZtBzUNBen1/4+tzt/HZ2tqt1R3FIaYAZbQcBQWTp9JxH4COPTUY3RTRBwcyBciSz", - "wssIW3+V73a+9WBAp+rFgMAes4FTHZq/1eiYOUxIDTeHCbFj5zB5eWOVYLJwzwp1t0aqGm2aUY+tfWoB", - "F1MY4zmKGtzhlE529zh+Bn7KuomkFFslcP2kzS90KoraS90qlzoVg/UkGQGMn8K4wpUizcVOOziyfZVI", - "vZFjbk9JOp+DYJZOtE/akssg81JEteK8VZqaKU3VrM4pP8+Ma+tTMZxRSRxXXbt5C1ypUqWeUtviewnG", - "PnG8RF770Ngy/WZuSpLKN3NZwj5wH7bySDWmI+/xG1WNJG34aPUIYyxAMLo/0TWIdtIFCsP4UaOlD4P7", - "8D0kn8WgG61JrECaZWg8PTo5OtHlgFQ8j/5Ku361KDc8qVhswduygti/QCeGJImDHPIKNx0qZpMgoPyT", - "TvGtJ4fshRFPOVVmgSc4nYfhQ084oh1/Fz9YhL/To060Ljuq8d/tI9vFQGZHsHSiHfuBWYaKS/jag+3l", - "jRPF8HSVTI3eX6LFVyvmOBZ4tjFTyKbCr76GY4Tihm0TZe4t32zGf5JDz90nBWooZqoyrlCspHVABHbS", - "7WrZc4/Yk1llSlvUlEdT3mR/PNd4X/NWWsdq5pxpxXPcybTKZ1lzxh+Ox3Jj31Gx4tYeWXJKLgV8yQuK", - "2QeZqdX1lR8rCdk+7cBe0PK2ovhz54bprBAYSCTKdhcHZclralB+y2mGmovrMFvhNCkG91glAmtWg7XB", - "vWgvI2SaJNFKAWwD9F44c4QgVoViVoyP6dZpWPac0EDl+hkCxVYMDmt566V5S41CW4exbNQ+e+5qpgfu", - "BYNtXhfMI8M2Vl7kJM1x2a6VQyuJUFQPW3lgVBDXY84aNdGqXB7dpHxdvJTxHtOXDuNJ2aA83j7ws6ZE", - "BS8wsYH6watXD9YDNovDJGJ1PzIQ5EYZQWGdPsJlpzYNyJaFxJq1uOSjUluOaw+1iZXqfzUSXDI1kdG5", - "RWbVaJosaKUcQXspuSYadjlyhvfMuo0TSh3Q6zKu8gGBmKQ8hbBzD4k7h56pOlQm+PdckRJksGLioRdL", - "N6TA2yjPUJtdqM0utIXsQo1Es5AN2OJVK3eSW4ll4VtzQCaYH0Eub1nKSYep9VTBVt7tlQqYkeKqKmDR", - "8W8KQQzj1PGvq3UFZJ5kXB4ksd950+k8f33+fwEAAP//y98GQS44AwA=", + "rsNhCls3FS8pMrRIdF0YEa4jjOA/CeTCJI9PrhBwzK5HnQsUmIm12/nWC0GEevSyMINBD34jMegRMGNQ", + "PAIf0X3pvElX3E0S5HWeS4TE4dWt923iP3AdbPAIA2JcMnyUdyErfVUzZK3mymf4+tztnNNzyLcAaOjl", + "QWq8HdmFK2Hc1mR7rBZEIWRLCgM3iWMYuMtLtEBkTGJA4GzJT+9kQTuc96/OB5d3w6u7m9H1+9FgPO50", + "Oxej65u7q8GXwXjS6Xb+63ZwO8j++X50fXtzN7q+vbq4G12/HV4pe5xBqcw9dsMIqnN+uR59fHd5/aXT", + "7Uz644+1/SEh9FediIkhxtpLKWVnNxvDydp2qRLoUW16BgNIMeIAemQ693G4cAjADw4KooTgriMZuetA", + "4h7pxJBfxGslgZr245kRwSgJsH4hC/ANLZKFEySLKdXX77OlEecpjB/u/fDJiZMgL0BRQF6dae/zWG6J", + "Jbh8C2lHAqMRBB7VlnRKN4U2Ft/TwxY6tBvF+NMcuXN+yKmbg/kO83sxPwVqJKzAVnEDuipNyGXqRJC6", + "NgJIHW2V9v0BLrnu53mILh34N7nu6h4YbColmPgP3230Mi7q5OFrllf82Bka+MNL4sxkIrcGihMZYYcJ", + "+/Jm2B8R4QKRAPldORFbjP747fPDl9841zp92fhfLZCGozDAsIw1IhWaMsZyYFWDwUcxw3Eeh8EXwbqT", + "GM1mMDbuY0ZlnxS1pzSwG4fBoJpuaZMrsQFlpZmKPe3IUYzCGJFlkbSZeBHSqfPmFTu8+N+nZZIvKQh0", + "tq5ucQqcpVV9TTFYfVbrcVYgurRNKupTCmQnqbLNGTL0YzGGshvgQXeJo/3ZKWTonm2TuhnlMeRXKXrT", + "cZocC+Vh2ScGHBvQuUc+gRSiek7g11GGtWzzxldjxbpg3EUSRsjtxyZ2XIB/h4EjFXyHUozzS3909atc", + "/fhq7LAx1hFjqaa7QMH/Pu0uwLf/ffb697LKmwJr5npudOz7MCaDBUD++zhMIrP8pk2wTlj6CBO6Rt5C", + "mrZieiJa2n1WWL6HHmGXzVheuwC1buU1lxw+uHav2Se5rXStVJ/gl4yN7K1cV7cTh36tbsRX8wlSfWxE", + "22vx0RGD1WHFjI9ghgL4GcZSoNfDJBs/dzsweERxGCwgN3PX9x0oHawvytwWvok9YEgMg2kIYg8Fswsh", + "Z/U6Fjc/G+V5NgyXyiR0MAljyB5h9HBne4P9ZGYQg34y2/zCu+LNiZ14zwYTJQNKT0mZJoFtD8IqpGoV", + "C61EUWzAZfttqk40mmsNw84Cknno1ZsJFHR94l0UYq88blfWfbodTi1DTzuHvMPVfDZqbrKBYH7tMGYj", + "VQqabqDC7DlYBWVkdJDuQS2dXiKdvIvADAXpe0PVLt6kLVNFnonupyb2IpVvrN5FdLSjGDYuBu/6t5eT", + "DrOL6s0a6gDXsQfjt8t38lVZDhNIxReWLK/ZSEz73aXau6bWugZfk/Sltv4IK7JaGdzhRV6AF1/oxfu9", + "cSGS/kdJME4WCxDX2n3YVn0pd6tgSa4zpwv5Kjdcnon5TW9yI3F++XN8feVMlwTiX+uV91RtZ9N/XI8G", + "5Bh7wPzpcsp8LwHdFygrQBQS5ALF0JUgSSkCsNvhCpJZfpgkkIXoGUMQu3PtaWSi9/LrIbO5ax+RmZaZ", + "mTtlQ62R02BguwfIYmjeqsm4EQw8YY+uGlg0azLyPwlM6iHmrZqMGydBYAGxaNZkZJy4LoRePdBpQ/vR", + "UyrHVU9Dmpsi+3akXoVX4LE1TiyzWFfem/4MpxpBXuVnx+S54mknTrG/w+nRll5IS2NiAiN76TUmMNIh", + "tlIVJmgBw4Toly8+1i39cV01+FFRf+X1iy1dp9f+GU5HSVAh3fgbuN27dtopdfg0NxlBgA0Xs3sUIDxv", + "NvXfnCKrdpQSLW9p2L01iC6GOPH15mdMQEyaLQYTQBJssR56PvG28nlLPMNZkzjd/OZU7j7AuJoFmixX", + "UUrrQFYO5kLP9a+NfBBJIOkumLlmnG6TVD1uBlcXw6v3nW5ndHt1xf8a356fDwYXg4tOt/OuP7xkf/CX", + "a/732/75x+t377TaClXj9P5stl6wxa6azRaTsJclbH5a2qnymPrmaPVHCnHeCI9fGN48NLWuDgpsYiId", + "mbFl+sB9+AKn8zB8ePFFKrBsaonh7BIFsJFzHnOPoJ+pIkElizxS/XDm+CiATTyxuAe/dg46nGhQq6SY", + "evMWGptEAVuq11oWVpDO8DVD1SV8hH7ecPP2lgqa4dW7606386U/uup0O4PR6HqklynKOOnlyWr/cxDo", + "BIn4/vJ3T0lWeunBP65x/8yP0PAGKjpX3EGvCfTHEQiMXHEHGhyvqQODnWtHBGIYkDscgeAO6c946eN5", + "BwiJ0TQhsNKfxPSWp+hIbhjBO7Pmzj4/Zi9J5RYwfkRu1RB0ORsDtwI37NsDCiq+VtxQqEJxJ+OiTN8V", + "oVWGNQYu1MNWfPCVLbP1qPCpKyngNw+oQmBdlT510l4Stl5CNOLwlEesbN8akaJ6P37vCIe0u4gh9qzb", + "CeA3+a9X3U6QLNg/cOfN6Ql7V8lxZa6zzkla+rtFXK6nE59ZGSoUWLQRBfBbeeRXdiNn69L6docE+KpZ", + "iDZltlIfYcLfC7OIvBMbu4hmc/4rgQm9ScXI1Wg4QbK4sTNaMbqRpqsj03r/y8pOxcdC3NWbGa2MA47s", + "DFR8RGGmOurUuhhloOZm6aoI0fHYCBDIPDTLqLR6BWFOpcwzUe80CjAZwXvkG1wdmMu/iAlQB2PxADHr", + "CJlf3hYCJ9hEn4GfQFtX1Jg7L2CHxZqJRxSx608o8MIn/bZv4pWmBtGP5nVIaaJZxwJ40HYR/Jt+Cv6N", + "LYPuJQoUH8sMzTwq6j6MXejZ+lIpN29lv+R6U6hylPZVpes9UC8zHtMqmOnnNVTM4hglJZNjU2JNQaV2", + "NOjCgIwVC1Hh5ZWBZ6Jn/tXR+dOqJr0mNp9VbHxr2Oe2ZoQTKM2scCWTVDFioppH0o3oqtYqAUtxdK34", + "hzOECYyhJ21lmnAcwz6n3vjIc+J0HB7viDD7DOMjTRRICXl23leZ93/VZBYxX0a36RGkf/080UkjGPlg", + "+UMFAvElKYZfbFxZjjtedn1K89cnJzXrLcBtWrXJMKt0tz/CCpZ0W/gkdDGVeUz0VbCV3iNf60pPRy3Y", + "UDUDziAmt7FB87wdXTJPSRh4zHVaXJOxQ8LtOPWYjsskQP9Q3ciDAUH3CMapbi3UQREtyz281SDzKfTD", + "YCYhrpWyW3Qwt3s6qXQaH7tz6CU+VCht3SCRLQd5dDuEB7PY6wlN4kKywb8q6PE295LEgh3pH+PzD4OL", + "W/qjThlMZ96u4+2eutCWV5/50e7CXbYxiW3Ow3aUBOfqs0rj51kOwK7PUgUAmyWOrRT3L6UOL+mKnBFF", + "pRdymXbfJv7DBfQhge9YUNOKTrVpTE7qU/sAlw67XDoRQDyBDw+bcqbLfPaWB7g8fcOannLnzzP+r7Mm", + "iVzku4NQKvRXp4Z0w0f8UnchW5EaNzDYc8MtNh6f9+neN5N8Jephjy2FVhpten3leMiHYrrxAgXin6c2", + "tvxqDJmUZI999za8kiIRN8xSpl+KXeIyZUHdqixmVXPos6zp42s3Qu6lzGsNQL5lyVwopZgsGutu5rrK", + "X16SN12Zkbt5Fpt1qUpBX1MWVBcpgWm+OhNnbpNn0hRAW+V7LYpW4cw9MG1rLgfPq0nlVUKMyqOYzN+q", + "xlTtbzGGCxDNwxiO/ZBs2PadsyvrXX65ORP7IX8CEz3sXZRWtENjVZHSxIISGDlxIhdWb2pQ3TrrF4p8", + "X/o726+0dNGosFBbg17gzQwtXdXWXrCrU6pRfd3KfjhzEATQN4EpPjvI0z/9YTq488RH1z+q8BGujHZ0", + "OQWzp684yVoWMLAwrZ5+W2PptLt53WzwdRa9F7Y7O+uaRESK7jxddBUy1J4vBEYmcad3zp8j34th3r+4", + "VudF+CKJWT5gXbI8kU1TSByEWeKeqa+6Uyhh11txzuf3QNxsVfFqiaL0p4QfyucYG88zU+7qDj3Asqd+", + "SZgMsb/QOe7oAE7vv5OTk1eMlEkuSlLJ1rSpIBbDks3kraA1R+vS6V5QJ/d5qqDrLQSt9MkgCnMemcpG", + "bCi0hXHYF9N7TS1R5rrj8zAJiB5c8z1ulYf3rE8Fhoq2+VxsjkVoh4hESttvXg6ECTGBuKKIYI5h/Xth", + "e7FD5sZDhXiXip1ZQ4O0jZKjbU3ixELWNFlx2qVixdx5YHXzbUqB6coqw4EE6vqxO0eP8CDlUvNngb0S", + "MSG9Jeo7VXB9DEm8rJCiW+NH5Wq2G5aouAUpSJB41N+oTfS+D0aLPANqnfJEG0PqEddMBebXaE/foco/", + "P0550GI9wo+H9WBBDo9Qvk7a9h7LPlZ09w7FmIwhDJrR3iVo2qth4Ca/QuUALMycYlZBkxpJxfe3gpj3", + "JWtGjkxrCTkT6dIuNhpwL4C7q+u7L9ejj4NRp5v9OOpPBneXw0/DSeYlMLx6fzcZfhpc3F3fMtvceDx8", + "f8X9CCb90YT91T//eHX95XJw8Z67HwyvhuMPeU+E0WAy+hf3VFCdEujQ17eTu9Hg3Wgg+owGyiTq3OPL", + "a9ryctAfp2MOBxd3b/91dztmS5H5l+9Gt1d3PJ3zx8G/7lTfCEMTAajWRKjjGAWpSmidWOBoOBme9y+r", + "Rqty6hB/3XE0fBpcFRDfwOlD/M1bV8USTwB+0OcLzlJ3VOYoEv0TzEbJp+Zo0lFnPpZtKu/HNpN0qkYX", + "EGikf5pR2T4DVyELs+aCEPqeeNCxk4psHzafmjkkwLfqrEVdmr+qWAMJxiKH5MCQ6TO1/oQOay1NaAvW", + "C+stQCAA/pIgF19H5Doh1TYlMeAcYCeMCPQcYZpIB9HPsW5uya3XbzBlZ1w7vWOWX6RhQs7aKhEMrmz0", + "r0ZSKuSP3W3i2C1lxjHnj9WueQ/UDP1e6PLszsIeJ9rOiD0GPudXhYKZqHGAdyckeMrGwbcI0V1miSIY", + "MNXj8158Guw8sUIuLOeFA2LogCiKQ+DOUTDjFV0Ygqvml/lvOZGwYJ0VoeBLlmHVZXhYdE8lLhSb4juA", + "/CSGFqAwV2kVkFw5BpZdTD+nDzBfqvnpM4sDBIHYWfb8WUzoXR3xA75JInvHrG3igNaG9jn3sokDiAxX", + "E1S12ecvsyTQAmyWC4P8QST1RD90gc+Cwx6hH0bsM4s59hK3UDtRUe+UHNXbS079nNYDqnwIltWgRCXA", + "XVZIWi0Ddt27oGBR06um/GzGGm9R9a7JRsgVkjCe4jVHkUzdne2Vmo7TSI2cdvbmcBKk3OxM4ntahv/F", + "CMo+8ytlvbrWtxjGvMdNMvWRW0UKbLyKJO4qzHuz6WL/Vtn0kdgnKUWvv1wxi0H/4tPwqtPtfBp8ejsY", + "VcjO6jwE9ZezJnexKkzk4FCMZatejYvjFeOxUgRIyi/Wu0oNL4PR3fjyetLpdgafuc1i0h9/vBvdXjGb", + "yPWVEnvCMhKdX38aXr2/+zJ4++H6+mMF7nNalE6RBPGiIrKffRf+6loBzXMQkNB5AjHLPllSr3hvfaR8", + "s6QH+nwHm0lhwMc2L1EP/3qZDVOaqGfflILsEhjUbVjzvAULSGAssxfIc5SP5fyCjuCRc+p4YNl1Tp0n", + "CB/ofxdhQOa/ruijk6JHm83ALHYlom5CH7ma7MJc46+6BKeFNnlTjdLQQOzm2a/OwVUAZ16dsIDaClSj", + "QFKKh0h59Pmk0+18PtWLEu4TuoOAQ2MMK3d2blIfq6IMxXM64FgTlGGuO7SmH3u1CzsH6GesBaSuvCal", + "wEbK8Bg1NxUQ0f/lATGz2kFbiltD00samrZoANpKJcgGhvyV7fAGLvzCfJ7M2RjwDUiwLvGZyibcccpB", + "2IlYawcEnuOCIAiJA1ilZyeAWbXW0oGlgw7r7uO19ijgeTHEWLVL5bRoaegom6fohw8Az3XHzRzguTrk", + "/8KF6cQBxBXRm6UfBs44iaIwJs75nNVX10/4GcboHtWhl1nXqAx6FM3pryjOw6DnhDnANwDjpzC2nQM4", + "kejgYEgM/LWNlywP4cgHyxwjyP1rbMjKY/ergcDO5yCYQYkgIxME8MmMRMa78CnDmtSo9bCvoHfIkdm6", + "o0pAUiAq8bceDKWEzeJLN4cnE8ovwxkKVi+GuBp/r1Ubce8wLtcY1eFaJvU6KHTbnZAGwbCHuyUeuq03", + "TVWr8RxF+FCNrCWj8w5P822cMnwy3bZ9Pj0fXF7AaTLbdGnmrtBHMVokPiAQZ5k22GuZGya+50wheyDl", + "2gcIRNGzMHZATmPWhfTka76X0XU+uHSyNux+8Aj8hFK/1h3bJzC+AUs/BAYOFNlAIt6mvD4gP1HtwwkD", + "+kMMH1GY4J5wLxZjdKqSB5UnZp/K85FSeKjIxVRtusnV3Bd2nBrKqAxkN3AB/SQzkjlIVrdmG8CK2PNy", + "X5qdyNzXdTFoOPHTWKzCDmejd+mErIYVxveJr1UE7WJEyliQ4SIlB3NjsIRxDEOcMv2WW2K6Lla7k1sF", + "mZfkeFxZ9+Dz6TmLhJgA/GA+SL8RGAfAF/kCjOYq0cwZXmBJii4InBjei8s34go5wA+Uf3OEqXZW7Vwb", + "zkVilxTm8ynFh0z/8qzfMBlBQptiXfYNbHqt4OhiaEiXjTzMhd4TjGFWHm9rqHjmi2Ayhy+0YvurpajC", + "X/J2UJRhqgZTIT6lcDQNUyiOUFXYXvt+wsc7cm7pLZ5OgpMp5o5aFOUeU3xEK+wAokojuzxklflj1836", + "ZchAySOxGEJyR55B0LAtF0H8yp6HAby+77z5q1bYafq/BRi5/YTMO8/dVfr3b4a8cucqnT986p93nr8a", + "FycGZ0ZXf50lQgZgQfOhi66VJmIoDonAE+s6WZqomEUth/cObQUDglxBhSGzxEgGETH9itDv3wzvPg7+", + "pRH2xaTKcnoOiYZazChlyNDn0P0Il4PGWpe6JK7ePcDlkTNh3lLYYUY3EvL6RjDfyrmPw4WKCylEjtZI", + "wZxiVV91Zb0FsiHKixOqIyuMEcYxdEkmOkjoiPcnvfszs0FJNyorUhxnXYSig9xKzZY3SSW0ng7Lq+JG", + "7mLvtGR/OSeFQSOVDtUZ2F0dvdmLvExk7YFgUOXnluTC2/54eL5dqcAE8R5gk8KxXWSylW4Mlxdgdq4k", + "GSkm1dGkH6nXXdOK5GUV2AMz2yT8Gl76iarUW6mq4X0GgHrpmUIHBEvnz/H1VQ/DGAEf/Zs9PfKVHa2k", + "1FZMVjhGwthxAYGzMEb/Vgsoa4qEwaAqgRUmYBGJh9L03OVO6zCwd+Har4r/4jBlaaZNNaeV+6icjL3L", + "Zpe0dBRnuizMaMmpjJkmCjDasrP8OwpmQr5dNVFihN95CmoGJ7OAgCjykVvIPaTPIJzVfLdYlKZYfMW7", + "VqkIjmber5n42QObsRSEhnt1eWMN6WnrKTzVCwvbyMhRs4e1WessMsopxJ8yWjp1nARHW7rJmiu4mMnK", + "nFikrvhY2rBJfXxhlKwZmrdqMu4/VhXYeKsm48Z2hdhEsyYjM+Mp9OqBThvaj170tUrrvP0jq79ls6d7", + "omR1EdLClP67gUWryo6k4y6EL6DrgxgQkfXG7JQgOBthx8u6OL+QOIG/0gM8isNZDBYLdnX65R74GP66", + "aYcFo46jKGtS1WEKWxkfh2Gm24RCUbHtDayAdWNvUrBWJ+I3GQ4zslDZaC9O3Swle3323c+nxkLagBC4", + "iAxqr/ioSLBiHW1NyqmdVOb2ZZnraiQVS1K/XEHvYjop3XMdiZcOy0Rjg+nmFcIL6FijRng20j5wQmU1", + "7/RzVbHF/vi80+1cDMbnhuXyelvt02DTp0GOt+28DMZi7C0/DFLQTaae5rKTLkgvN5kPwKeK3GDsqiot", + "ePUbM0ibr5iLzDozXqU2DUmMIK5fPv1ywX12jFV8aBsrgx3P/sUMNs2SjslraLP8zLwJB06dWt2zDNf6", + "S126ZXshUjOir+MKSZBbTjDWMKOYHCuXSayYPUyfeqyYUWw8uJrcTdTFpGu44ydkKf3Z+WjQnxRKrn0c", + "3tzwj9e3lxQ7k7vx4OpCGVl/8igy1tLUbJ/rBqOAR242KTUAm9JRljK2VHwjIMhfpc5ZdbGOZuU4OBLM", + "THkTooDwKMXyDgha1MrWLDWbPvgbLeCqEXi8kSb3m9UyNAcxdxVrurMqaizvIUyFSgITPt3KbKtWLmgq", + "yendzqrSPRYgbIqRbGkacs/BpojMVEhkaf3Orz/dXA4mpWx+FUkK869dq5UxUS7/+YM6m2bd5y2m0QnD", + "aQn7G1Wo1PdCs4YpW7GBsP2DRc3TYs0tOHtPSnHyBLBw62iQD8DLa0x2btCaLVBGTLL6uprhxNfiUF0H", + "Bc4C+T7C0A0DD9vpuHWesIVZnF/SkH1AICb0t1/ry8dboZ8OL7vZ47/OD7kC5YLqhVe9/DGCAYjQ0VUY", + "XCW+D6Y+/HPM8makrXpoEYUxm1S44pcbR4BecTozRObJ9MgNF8dzQNw5JD0PPsq/j0GEjh9PjzGMH2F8", + "HAJ2Rn/rBWKszhtmaF0zDCxZjCPwFEDvvJIdFRs5b15mzKrc3eUB+beGFHRAe8JLEjA1PDU8WD9h8c6p", + "7KxVoLZw37MojaXh0C2Vxyoqqlm9AkNprPJBua7lYbWN3ODsFg8ClZf3YYBh3PzIQ6JbUwcK2/eLI7Ua", + "7Y4qEhMrI02aAUTYaOT15jwM7tFMm2+kurjsWpWfVyC+QsyRNTi58snlmUTku2aidQpnqfZxVWvqcuuN", + "jAbSnFfpOdPNLhAFdlWtP3lWyFfs4oag3KOTfgu+FhX67VqFqg2wm9KKSwHFKfACEvOFbIIW4ul+ixZY", + "D0ZkbtB76aecMoG4F9gTIDC+B76vH3Jniuja9c+2o0k0FJzcp6Ehsugpwjvao+tnU2g01vUN3BVbpeUH", + "UlpWc4ZTdYC1Clty4Vs4Yi9yB/Uqh+7XwhHykucopSaWCL3RcSqOvo2dpjtLgtftRDEKZa0Ujd+4+Goi", + "JX3dNlWfrXH7Fa3rI/5z43aVdH2fT3nypDYsdGV/M/0zgMhJVQq9PLgwuv2MgtvLIDYNGRgLaKvBSnYB", + "pbLDc/cQyGbrJWLawM2Xp/nNuNqai5Vbebhahosq8YFfVdZUArXLTBqhj6aQtP7NkHGFguR8aKGOCOYQ", + "eDC2O9152+ImimlrcaXM1JXr+FolovqKQMoHknbTQPOuKRpSGScXbFtUQq0yWtGlTukoDKE6NCaYqsgm", + "JMqvtQMVUJaOWpPbKh986s+ojjdfqHgbf+ifdrr0P2evf+d/vD4963Q7ny5eV2MvjWfVZJFVJrKPjU17", + "sQSmbuhZ1KvLjTCQnZg7zSwAJInhh7XpmA7tpONpBSaaBay4khtDw+UVs2+MDVNZhmaB1QTFAN4UUQqe", + "9CsuglZLIwMF72lY8eD/sHKF4wELiuF/3I4uq8ljL1znpE5j6RCT6sCmtFHuHPg+DKqcQhuE6FU6wMtn", + "98KR6MQSOEvtvnxAK1v7fnA1GDG5+X44+XD7lrn5jYY3A+ah1z//2Ol2LodXgz5zvvs8/D+mPc9usJsP", + "wq70Umnu2yHNlK1/R+vf8WP5d7QuGOWHkzUNsfv9kHAwduyGb+Q1j9Iai7d4p17L6s1aZybv7NqWf6LO", + "vRinr9GqnVI5DS8gkWUUCk6+SWDvlSBSMOA5qLfCqLHotP27MNbAIx+MWAYOm7Af1jBTRvLeBusHMnBw", + "8ObSydQ6cJRjuTs5nEh0S8jKW5tXB/Lb69VEz2yhnqU6ZRWwL/XqompHDZ5dDBjf1BPMF53Lh0SReTE7", + "Cv4rOCapUaL996Kkm1YlF1o/LzSx0RKLjUyeokiFVvlNYkMaatk3if1GhjZhEKHj6vY6hxKe5MtcfGBT", + "i8R2JgEqV0WGc3qKOcN7JwiJE8XhI/Kg13WAE4PACxey0xPyfWcKnRkMYCyvMSp1nW0N483R7O0nAa62", + "N7sm5RTOWmRTqWU2XezU8pIXP1bWl1wXI2OKS/sdMOwbexIFgZdVeIz5UKtd+ReQzEOv0WoF6J94z1S3", + "Pw89A9V+mExuZO5sN/RSCpaGHvt8A3eAJxxgM+cm/mqJ8GoSEqisOeczQxVvbZ14TEsBK9POp3TrMmPX", + "pNPt3FyP2X9uJ0xLMp2QPAgLV0VoYfEmxOswuSBwIhhTujqyr4gnzErstqvNukUpAaX5/ADGaBZAz8k6", + "MWvQ7e3wwhEkvftbng+m0MfV5UNZG0bmOZ8QLprtyIMLOTqODo0+wOQDBDGZQkCq7uu5XWPVYFkdB+DM", + "Ze/8Tfns5Oysd3rWO301OX395uT3N7/9cfTHH3+8ev1H7+T1m5MT+zQpgDMYPbIHmICpzwxgewjp9k9n", + "86kcQxemVUmxKTsLbcOjP3hVujBehaRG+bk0VBWLaj5ZNU9cm0kJO1kvJwzUXWwAWXFeLXRJQLdwGNyH", + "dtwzUjrQo8kPSXZBXqVeNR92nI3DXLdV5NBvDngEyAdT5COyZMdzrhBuRuS/UIjuWP7b3n8nJyevoPNd", + "dvZhV5RUfv5Vn6fUD01nE4YLEM3DGDq0kRBDKxLNWI41ZvPpQvmti2RkU6fZb84nw8+8IHf6503/dmwI", + "NLaJbuF7lEa28LPSmBFMnN78PCkAWW/A471v6/Th29GlZvim6jFrr1VtlKOidLJXZu6VuZ5o1007C1UU", + "2ebFtWsmr05UWoGHl3+JNV4EUiBHeVFWqLANglkinrGshdz44iPmxy7vrBSoLqfX0atqQr4OvpEYaBtg", + "78E8bGlxDCJVIb2+7LMMAzf/mnxgjyKTf90Mxuej4Q3Ll3L79l96405R6JZoqlboAi4I6dCU0gq6rxS4", + "dfEYaUMnCXLiPDe4pkY/K11uqg2LFslCmaTJ0Lri5xWcUTSqjQeX7z5cj3mqh0/9qz5PIfNl8PbD9fVH", + "416w47lsAlbXpo9jSn+xcJPuNigLyxURWRhWX07073BqOKLoFx1AVpz+ZzjVHYk70SiNmJNFBDVqNpit", + "vtbUNgu0F7vq5znhZpjd7SpXIN63mklc5SlNIrPSZq45YdPQDQMPiScWfstzNalOZpAo31npeY33RyCT", + "nvAcfDNIsPBcTbs6M9o31RoUu70WYYz1xyQGBM5qs4MrEF7m+vGy1uaLSFlapRCTfOHsYn7hV2f14ktO", + "XVxNV4vVqi0aXujSIKYADi+0OJS9P6IgZ0h5d3t1PhmyA+vidtR/e0mV1Iv++0oBSQeRmkgjCmaza9hL", + "fterN2vFYO5YM9Jf754r9tOYO4oxyUdYFU5JQgJ8HcWmPPYAlwafIjk8JUu7iE15OwcOjqCL7pGbTeL8", + "EgGMoec8IiD81X/Vc4UREQ0czvTXWxInUDN+3fut6rmVGmBOT05OjJ5Y2mHyvlMN3aAaLejvcCrFmO05", + "bij8sHZ0Mz8Rd22k5HMLW8/LgJBzJtqkY5Dq86H1DjKXGnm7bDD4ROlVdtdpqJIYHX7WyR2eDaS68ihg", + "f60WJntyV1acfuwPhVESrJFXuTzKOwT93Lmvpu3IaDknxRTJWDPJWDoztbK7ld2t7H4p2W2Y4wcU7RXe", + "kCuIZjbakMCF2b/ScF+p72ystjdmqdCqE+6u6XGWZVvbeBK1DQxokOnFlLzF3BRiUd0SIpVR66inlCn2", + "ZnB1wRPEZqliNVmA8zlj0/Syb/vnH6/fvas9Jdm0K92b8wLFTIyTvDgp+tuEwY0i+Uuw0gZjdw69xK8I", + "ijJ0Xvs4+lLMk2IpYGo2G/Mq6kYvpFx6li2yY1U9Mly7CKORgGVcbkJHcqhz3rFOCy00L82fMYQ2uXRV", + "Hm/JdNqPgrm03ySPNs8OXrXYCZjp0OtzlXF9k3+w4eQqwqzLIayiHyEUzmN6kbnXywUtS3O+vEMGbqyb", + "kDnfa2dkcuROPN5uelqsX2FzzaCAN43khWnIxSoDp/jZrHLP1S09+jIN7E68QjRHM08xY5Snm3zZqgJD", + "0WaLLJt7wrDZEPXVgzm93IPEJzeVWZZEI2O2JatHAnGL/BPzg3dhKIH15/j6yuFAl8N22AhaJxr5LPhC", + "j31h7HFvTAs0YKF2TNAChobiOJgg92FpcsWh3xwsnlXsXhIVedGAbZkO9nhaeCmzwrHSZ8yTP+lQ/phR", + "tjmJq80Cn5T3bNt3i8bJcq2vgXJZkjByA32t53RGVpt8G2pCn3uxJ7tCOHeoyB6FClWFYwi5v4qxpMgC", + "fKtp8dRM2TfVFeGRHwmVv0x+cginEMQwlvlMGEbZscJ+zjZlTkjErj1h+ICgbI7orvKf5Nv5m44IYc76", + "itQ2tHeCSbiwnOyZSXzuFqWJHuCzOP2bISt3RZhNLP9rSoid06OToxNGxzyIu/Om8+ro9OhExGMzTLCY", + "a1+UiZ3pAmTey+d52iqAGDupPYZuOpDFTTqX4vt7hgYZ0MBmOTs5KQ/8AQKfzBmKXvPvbhgQkVRD1JOm", + "TY//xpyvcHoA1vDxII5DKoWfS/6pVyFJ15Ejjs6bv752O1jWcKGrzhpKn5K/BMzuHLoPna+0P8NfDIG3", + "rEcgbYaqMDiSDfYdhWzBDgkd4LowIg6Jwf09cmsxmmKgFqWPp8fApyIlmPXgAiC/xx6S8fF39rP62zPH", + "iw+J5vZ0wX7HDkgzfdHuDuvO36ZLu9CnLQa0AXO14CMwnonBAhKmD/xV4eRTmsERec47b3gehFRolJbS", + "UYUafx/Idmy9mrxfS/T0m8aTMHFdiPF94vtLh6PUy6VJKyHvudv5bVeU13cWwKdYgJ7DMmh5MuyIg/Fq", + "42DooHgXxlPkeZDfPjL65nRSRWaS4iesCT2svvVioXKwD7xvp6shjK/s2ktcTZZ0ft1ah8T5CD8GiTN6", + "eBtyebwRYuDY4ZtWQFwat1Ymk0pskdBJJM7z2HjWi/2NLES7BB3sOTHAAW3FgKUY4NSyPTGgHpAR6pHw", + "AQb0VJR/s9MwCnUpDUbwMXyADghYskbWWnhrpTMWxESEJrSVNOjQ7jZSIh3eIBMkrHt13MVseYLOGXQ/", + "NlHjJlQtSIdu7ETsnCTj7LcqSk63PEfBrh8m3rF6Qzdr0KVMcfLawwZxUIAJCFxYIuJz+lm6l5gV6+3j", + "lgHiJEEWcLEvBFajtXMEq+/1Yus/KS9s33pyiF4YcWcXcaIp+83N4cff2X+fq/abSinW6qi0ocwqzjey", + "VhLxJNEm5YSncNylENrcZovUSjWHN6+h8ijEGscG27FWtuVIXMFMRt4cxRVSjdPPVzOFH9eJNbYtqVSr", + "ofmLVID97HR/wUi4pf39ov0FXPkMN57euzu4Rca1JjSVHokHcpBv4ginYxwzOz3fJWzc8UuE6QXId3Kt", + "TRtMWw/zDbe223QusePKlA03X2bAya1unwgh3Xq2EYVNKO9/bpPDAJGQSvPj75zjn4+jOJxC8+VSvn06", + "IFd5gtl1eeWKXC4EM8OnU9+EmIyS4IbNa2+bMh16qeTa8alXQVDwG3QTaVth+D3a6alwFRJWgSCM0b95", + "lnqR04gHX/MozZKZkwDkQ8/hdnuHbY/zTsjzYbat+oMjR2bYB+7D8Xf2HwsrvjOmDZUCK3nKYV9Fcih7", + "o31uTCPxMBD30jqfx8k+qTanuwHjNshImE/8ejcT85xjLHUj8P3wiU6vexEoUq0Uvez3KhWLE12eYwJ8", + "/B0H2Ipbrsaq1C/zS4AbsEl+MDOjiJN779ikgIyWUfaQUUoEm7LK1biSUQKsYROpuCjWJr3qQueVV+IS", + "izR+G3sx/aNrNgTwwkwrWQIUGM5ev84BcboJHSiKQ/oP6LVn2B6xpukSyeo3OCCKJLWXjzXepsCPBEx9", + "eOyBGT5OU78bL42Y3RpZO4fMAXGm0A+DmZpVIE0zDmblK+Xn0wvAys1ORAn1enOZTPCdJWjhKbcZy/yT", + "wHiZ8YwHZnfIqz7mthUhYiV3CvC+1MXHmno3VgP/AszORcyXPvtYhRyiU8rXPzbrz20l7HZe70r40Vso", + "WkQ+XMCAlHQDZryQdJA+nQP8oJUwrOHxd/qfmuclXuliuuR8UxQgdAJLUzsbx3joU0B3fOQDQuAiIiIv", + "i0EoiEYdFZZSLNQ27fiFmh6NTG8Mqz87f/7G7z7bn3Wi1t+nmsJ9mPAkTXsiIjJ+LokI852B2IiQYz+c", + "1ekqfjhzfBRAmflIwFGUKJfh7BIFvB7LIUoVkeWJhCIt73RpkCw8DaMWGhQQVlSyHHRpSJ4bE5EaO3Rm", + "kFBUMywbZsaIWx41M1ekbjDcm9KqAlZTJwFB/gam7jtU3vUI/EYcDEHszh02k1LjuWL9rINOpFevlVEw", + "fIT+L/hXOhEKXD/xoGl/aUvc0Wq71QJfsgAdwFa59WRyGwoYi1IxUx77fDdd3qWdclBaAVfKqWN1yFpt", + "zx4cuaoQaqAQiyjW9t08r5Wmkl85di7D2fqnDv3/XhY6bH5dVQq1GQ+etA7bD3D04AcUmZj//h7DjZw7", + "Wz3ptq9SZ3u9goNMe+1t1eqcjNNJmM2q2CQGbmX0oXM9gb7DmilwiGKUhov7hA2658Jum7LgmkB/HIFg", + "BUGQobsVBi8tDEr3XGVzNsGIrIXyVuZC/9iD02Rmfi0bPAI/YYXXnPPBpQO/RTHELLodzAAKcFbIUBTq", + "9gABRxpWPYf+BZvqUHx7Nh9a9vn0fHDJkFATScYwialOwgp3UzbVI3+nAWUq+DL/aY2ogYJ6PM0a2guG", + "+iQ+TWYlFlN4/nxwaWZ5K163uGDwl7i8DpCWFy/yc7NLxj4+lv9IFw2NaUm+qj3AJVYsFsZpabvmdh5G", + "BiL1RZ2F5zwMMPJgLEmMvfaGLkuB4zngnkBRB0bYz7Zp9auGZQrvwxjWArMpO+A7vjUkzEEDYlYUMnQR", + "k6BPiMzVR/FinXYNfFl+F8PObvm92n5duXoazgIQd46YD4ALYwJQkOXQqFpnmhYTrmSxZE9UhrSaVYtL", + "t0Sscrqkxx2KHe43oINYZM580W2ZLp0sVXUWrMGqGqYGAoNxs5zJW7sQTVUVOc0DXPZ4abQIoBg7v3iQ", + "CT7KfUsHOP/z5n9+LYqtSm8kOwszdsMIWslD3tJ2Xaz1evBu11hkbyhqTcF1puCUNyzjpxooaMfsGLbU", + "0vjZbqWpfYTLQ1HWth5PKHHRlBEYultm0DGDI7THLTDE98fTXoMIcubkQ7De0adJMPkeO/iaYJKYOlDm", + "FPvTHlAbifHFTeJ7U8qx4kyu49gcU6Jl7RnFVdLWnLCv5oRSGXQLBbr29lk5RemKyC7jfM6j9Wu+NLsr", + "4GSKIXFcEHiIJXySdL3R20PVip1bDD3GRhwWQq/HZXgAkTZX5kRjqF6z04uHwtoNBLsUMa1kz2tbEi+Z", + "bOf4rdK1uoa3nXNW88sBTgCfxMBG0czb/tyPNwwFHB02Dzjs/SYlZYdVWONW/V2+2QjyqGM9Uf9NAbh9", + "Dt7Vc/CV4QU45c+UN+153l6LYxcs/rdNnDGokxSNM+julxonuBWxBAmeXIv+spVi4jBvW5aiQQZVt2Lh", + "JcWCLet3FcKkR39FTFSqwJsNJny2Q7aYpPz8k3PxLCTt4W60mKxwxhYZrTJfd/2xeeCZB3LHZprt+iUZ", + "bhtXAL5JK18BXiALuLV8kIm/W/lweKe8hbLPgkwWWdXICrVASEYZke/ESeCIntUJxLkHxSXChHtRyCKV", + "hyrTyiGJChpq/JMsAF07SrEemk05KBVts8z6G3jcW8c8fVpcEL2QpwuFm5ePZKT8v7Ca+8MAtCg3Sdvf", + "ydZ3rPVWiS3LRcLf+JirVFoCOwuEN2T94A1RMLvjxTR3BHlf40D00HsUPj0WjwSZJ9HdotKV6GWN2FSw", + "jZJASrTmCQxUKdomG9mfTAJsbxbpQWUXY2F/4kYhCojlubtAQUIgvY7Lv2IIHrzwKUiP4gbH8HtIbujk", + "h34IswNP+gYrYTPCYN3pduA3QLe486ZzdnJ22juh/5ucnLxh//u/Brkjuvfv+U1kEwckgzT1HFZBDSl8", + "awB7jwKE59B7ywZvDu72ZWOO1FaQjoxPWvm4p/Ixvzsbl5L42GU1+c1RaLxmf5oYSifveJOf+4GSoYCp", + "KjWVyniyvdBxJdJ2GkXGJvWhxxP21b5MyuZttrY2bL0kowqSYeOSKYaRD5ZVVdbo90rJxJv81JKJo6CJ", + "ZIol0nYpmTiYtoIpFq1budTKpZJcKsiFDcolkYPXxvtW1jmo874VZRRa99t9dr/l5OLQYe3i11j7K9p8", + "lWBIQRPjdBRbe6skOmtARYcKSKsneXEPV5V9Gri4pozcvsXnfVxTxGRyU6B4bS9XUzWZdBNbP1fh5yrw", + "0eSVWzLlC3m6Shpp4uq6j1UIfm5f13KJAQveb6A2MXdX8Q87f9damXHgHq90cvn2KFm43vc1w4oZ2N3a", + "oW35X/qztry/F64utezdVcmtxqVV0q/waRXqoYFvD9mttaAA/2g8Kr1VWx41uKvWHJMwoKdgLwYE9tgN", + "lG6u2HtLLqvzZ609Fg/co3W7HLY979QfV3GXLqqtYNgjxV0jD1Y/2fU3+JsQs/weKHDDBQpmKb0uIMZg", + "VnHCj6AL0WMrg5rIoCDx/RLlB0snAks/BJ6DAgcES0esttsh8Bs5jnyACpRWnHInMsQiM2kOT/fAx7BV", + "LgwFADnjadhtVQ63uacLn+FenAR1bxz5rIG1rxxZlsD2pWP/85ZikcnR6q1jZ1kfmR8+iH0EMct1Da3A", + "22JQgA9IE1A2Vrpobxy/LXPVHEi0AgUijaOzybAD4y27+H+ZQzLnAkCUinIu+u8xPb3CwF+qv6cFPHUC", + "KfCXd7JBraIyDUMfgsAipiNXzdUCZy8U3qGpOWuM87DI7Pti8R7OvQ9m7Kh9EnQRxswBQyWD9H4JAs8J", + "E0L/FOojpvojbSB1wSPnAt6DxOf57v+H0sP/OOjeSQIM2TGuW76Y6U4O2qkkoZ0Vtmz6Atw6De1bAZyc", + "RqkquvL3Ef19zZcoVcM99hCOfLDsMXeJGn1XtKXDCveK8L5CCa7WgS/4YMzt4qD1YUW04vQdK4cUES8p", + "0CdQZ1YEFFn6InW/t2yC15JAK7pa0dVUdEk+6VE+qZZcOR5l2oM+4X+W3q5Ccg3EYEPvcAVXe89t77k/", + "yT13Z8dZJhfa0+xHOs1yp8dOTjZxvTaH/Ux4A+lVmr+wVxxdrXvpqUCdgpSap+ocKZBQ+G/u+o1a0Zoh", + "AcjHzfxMVQpp35uKbp8FBtoAg+f5mfl8Kr/UlJLIkxwIPOZMlp7/JEyvkqJY0n93PEYU/91xIsODdEY/", + "lm5nORi4bXPGehpegZXlHWwuwxW4rD3F9/gUL4a/WTJ0t0TQK7D4sSgZV8XphGf5IgkzHOX5/qiWi8ey", + "Jt2KvKxOr6jrPyZrq9fPlqX31MnrPEx8j8fT0oukTnPZo9wkOa5KC0S+iKxhyZ4sSuyysFwe5M4t9fZX", + "B8pArI6PtdHr56lIk4lVrQHkx5WoK1V1bIVqqycVZRdBCxTM6rUl0a6x9HoPyURMcbB3H60M8mBE5jxj", + "Cc9q5rhz5HsxNLlusA4Npd/2BQnfnFaSHLwkqeLPTYsXGAmZIv98PgaxO0ePsE4LEq0EmLS7VoSMCYyE", + "u25fDmwhPuR4RuuphLd13V1dI9umTBL7LvbcSirlk0q2dUF3n48p5bpCTqaykMqxv8L8Uj7R7aeyqUo0", + "pSxcL5Ns7mWidL+9PBrIGqutNPpJpJH9XauVRYcjixTG374k8sNZnaeUH84cHwUl3ahsjr4MZ5cogLbW", + "oFYMvWw8kw8foW/lMsRb5mauYgZJB7TXOwR9z5hBDtKD12GzKXBUFDNhHZoCMua9tKEkgAUKhLFXtX72", + "+e2Sr6Xh5NdqXwMe+PQeiqErot0roLhQmq0CSdZ/u4eUKg3aAvrrpqBLpbByFlyGs+bHgHA0qkhtzjwg", + "sPAkMjjuT9jP56rjy6Ydc/jgfKK6JL3cNellXHE4hI2cbwRSf2waX8HrJiW2NDut8KcpErmOolPXuVqT", + "MXeNES/slQTeNCFTGtghZjA++ezGW+5lKV6mTGqpfbe3DU6MXgj5RQN+4ydwqZCGLbPlMppW52AK+Gwo", + "mFXz1eFkYtqS1ylHQJPDLYopIgnicRkvULizPefWP+cEn6zAehXn3THwKWEEsx5cAOT3ZnGYRJUPp1S5", + "k7dAQV5sDIcN4IgBiqzbp00GtMV72uBQIp22fxLqENOw5JRxE1reyb8mVlBro3PM+upTnquOMX76kAr1", + "5lbAjd1ZV0J5o6vd6XbZe4UTUENDLV9r735abtvsKXmMISF1rkWY7Z7s4sgu1dkMFHJBwWws+hxIUt8d", + "HZMKYtY4I9U9aVlJc63ToGljfBShHgkfYE0yPKd/M3R4u2qu6UdoQpu1+iQ+Zn5FN0OGD2yROlLHJ9I/", + "qrWhF5VHSpEctQozpD+uU8olyKjdjthbHZEhQNK6ohZu04RRnLTlrw2HzWbM1JDBqg4cC28pXl0u5zJl", + "SruaOc206Vb32j3hAS6tnBNou+bpZxgZfIRLm7wmGUyp+/LwAtvmw+SyojGA0iV6eLEiiFkM2hqpfGwg", + "HCUBj6MUhq8XcfVg+/kyjh5s6j1w81DhUJ08KoglyyAEl84j8BOozyMEv4FF5EMqsh/g8vQNa3ra6dJ/", + "nfF/nVHxXp1v6NNm0w1ly+CJS9OMQ9V0zhoPDz/T0EqRdq13TWD2uVSUFobc9U3IbFyDDtJeARgCGC5q", + "zMIiMfGLuPdwSmhi84W8x8/uXX32n7uZdST4U6in8JsLoQcN5Rz53jTg8/qLyfE08R/M7nRvE1/UMYI4", + "kwm4UijQPj+xYKDLbygc8EtKB9xcPLTRF3smHxibqkICb1hKuCBwoV/hdsu+c0OGkjg7p+KapAZ3K+Ej", + "/MwKBUOAvUIhLgwxjHyw3LjYyBy26L+essvykCcn3lYRD/lDOP0buhaaC0MazHKUtEJqb4XUiFHqduQT", + "M6NZ2li5bc7CzvoRLttnvczYuNJtnSG7vbHrbuyOsP1ukg/EaWA8pzkP4mZH80geMT/r0cwRsC9H82bM", + "ahy4Vqv/SQ/M7+y/vSdE5j35iVm3a8OPAAH88AwqDYQXgID3kHxBZD6RbF8rPyT76MVHCeRdv13+8Kc8", + "3bRV0jEwqmhP+bwvm4IZa97taoi8mp9R8IgIbBowIXvpnUCH7Gur+0rfTwUfK3l9Smy3vp66cIiMFrcU", + "A8EnqKT19jlLiXrgKLELduC4fdEIBw7uKoENgjB+9tjes7Mdab2A2L1zFflWJxdgAKY+7MWAwB4bk7KH", + "4LVV9GIhheQPPf7vZy5ifEhgWdhcsN9xakayETS8z8F67+W5vhq2XoqOQz/5a2ULp5B9li05NuNEmJGr", + "SRfN72NtBH0zTjicKPpD4YTtBvqvphW8WKi/Jedy+A6Gc0UIfmPOrTr5FnAxZczX6AYpe+lZ/BP72t4g", + "JTUq+FjpBimx3d4gdTfIjBY3EyQoxjv+zv+wUAIdIIBw7uNwURdky6nhx1AFxbJNsPHPO+Xd37bCu6vo", + "gD8H1+5RrtorQ2ralElzG9NAXnQlIVukkSpNYhYBP4YOvBciYLvKL98uO+VXoGNPUl5ZSi+NHiz2rRVe", + "Lyy8jHJlBeFVpfVEcbiAZA4T3FtQHdStL1+UdXFEl9QHry4z5U3a9ZOY7Ie4KBD4jRxHPkAFqiiO1OQO", + "UMZyy5QvzZSUAzT7sqkbyD8JTKA1G7LWjTnwv2ivA2K+w45sPqRg1e3bQ3K0t1oGC+cRxhiFQSsT90km", + "prtTloiSc1aVidlTn42rd5w+Ntb5eo8AgZe0YZtXY5+r024iB0MtJreZaSGlsz3ItlCEZVdlNfK81iCY", + "QGHn1s+wYAVXcZOJW+Ztccl/XVXiih69KPSRu6xPOSk7OLyDTcJJ6Qp9w3q06SaPdWhZ7dGosBvt49HO", + "s7ZiH7gP1Ykmx7SJ8wSn8zB8KD+nss9f+Nf2OZXnmFRx0uT2UED1PrHDjioe3wYgIfMwRv+GHp/49W4m", + "/gTJPPRYRQ/g++GTvtoy3yCmB3IWUM8z9nEtRjzGBMTEyI5j+pWfY9f9hMwddlkpMuQtls82DKBrilDW", + "8xA589XJmQYPKvcwlIljJYeVOQSe8BrxQ04wNRZPtuHQTWJElgw/bhg+IEgHZUWRvqr0wFCan1ESAt2B", + "lemgLu/v+GpcJMCCQA5wK4eFHL4aD1VUNZDERSy3snjvZHGZEVJJfDVeI91wYWAdg7XRGAwBef6qzDK8", + "OZrNT2odVVHc1Zah94ihjZxnydGVJ6qo09nbxZOVKB1+aC9X2zcX6BDTzGaQ1rPO7Uz7qLIPjyrp3mz6", + "mVlXVb2SdbMC6s50yRmqcHpzQjwQO153Xyu7b1NiiC1aUT60EmFnpVBVWnwCvB5qnYhQD3X6E93oVats", + "V8uJ2pyAfULgIhLJLVlbRXyYBMehJQNsJUiVSzzCzFdaiBBOBP7+XRBe+BGvjlF2xdAxpB0rcoexJIu2", + "PMyatyy8j9nM4iQQW1Xj0Y6CKGH+EPxxV7fc573QVNpcZhXyhW34SwiUbE2VtgDeTDgL1AmX95CM+bCt", + "aHk57aBZll6DpUEM114o9vlCIXdpK1KDAPzQwwSQGoMhwA+sGpSwFNZYCScAP4zZoPYiYnjxI9oGU0Q0", + "4FAtrlse3QMzoIkNdpEeSXjN9J7C+KEqWUTmgG10aWq9mbJgEo6KLwypFCFVVT0pMtKAF97RkdvRPrft", + "2/u5Qv6rJzEUg5hY6Kd/J8/xD8fGjorxamb2GqUglFvbcu7+PZSrjLfSYcmoovohjZ6QXHhXe8lnZ8NP", + "f1hmmGhrXm8kQ7XUHvIxeqt7V0pEc0NQ81oUavVfTUkKpWRvW5hCKUyh4AXXGHRz9ZVfrkyFDm7rcvaK", + "rTdHMO0ldS/LV+T3qBwOXG1KaiJwvqv/rPNjyXFC7QksyPSQ3VoKrK8HTcXgAasJYrtWzSzQurmY4/rz", + "L0j1Mf3dPE2tzs/H7DGy9jGJP1lyhlaBPqrh6yEbvWXul2fuLIvJjVKEksO4zrtTHkdsu1uz9o7M2l9U", + "3Ac2+UOyTWqqMmxO4uA5iOCW9IgxG7uVNwejTPANazWKH0ijSGNXhM9QZWSoqNTOWNz30/dxrNE1qlif", + "BU5yV5aBLOzXyoCNA3gJMHGGFyxh/Rw6PpA7aEpTBDAZesY8Ra/OdHmKduBj26SgZ6ksX2sS2T/fmhVk", + "ib3jjZ0sxFYvE6ylnUbzUyZO8+A9SHzSeXPSzYmKXaRQS+d+vcrkY55Jbbp02AT6ScUncz6HXahd7WPP", + "5vWtTaZkTMesDQY6l3ENU0Dceemxp0pjOpxgoG15OSjvJBwZtm77Ipqk/FSy6ceeSLHUfE+VvlESDD2c", + "Sz27FoLL+XYbGoREBFL7elSTHo2TzS5ebvCxG4dBvUZCWzl/h9MMKBKj2azWfeI8DoOfWk05mPyu6cYi", + "j047gyRViY9q0nibLm5buOvSmZuCd1WnSmmnZBTfZDraoflUh5mhvCJn7nTp3Iu8vBtL3atKEWyfvne6", + "3F4GX0Up2HEO3xwy1tDQ22NXo6WXzrktqev00D3+Tv/Tk7/albkrH8TWDx+UcA686F26ehNYOYzuvuyd", + "ZX067Sa2+YGL9eL0aGr2VpEniK/P3arHxDWZ65Ddk/aYs7Z0dLbH5iEY9hsd1huRD3XlJdms6YzWwuHA", + "a03ul3zYVrVJVUBMuIHDytZHqYCXcLSx7dWpCmoxyFZVqJYDgi23IQrsVHl2HNg+6KmvjPVuSq3BbJ8N", + "ZuwRuYG1jLXfoalsH+14EYgp0gyuKwWweOMv6mPGjuDTpIjRwiacRLYLV18bn8USESQYWtVblG1XsW6N", + "WV9hZ7IB7gEFnhVUrGFjkD6iwKuH5uCNqQQtoAPuKaAl5+kngGUss7qEztnJ2WnvhP5vcnLyhv3v/xqN", + "1ax7n06gJ156rPYoFB3bauQU4im8D2O4TZDfshk2CXMFlu9RgPB8dZhl/53ieVNAbxTT23scKFvif9qn", + "gaLu2Fo4tuIuvZ03AeYhbZO/HzgCNHrQ5dlfTehvGQhxyBWoWzW8VcN3r4a3umWrW75ICBRes2I7E0Bt", + "ZZH6830L1dOzc56C6iU+PR5rrIZpy1Xsh2PZubUi7rMVcXv3opQADspzqlWmWmXqYJSpbBmZqN6IbTYF", + "yYrBUyutBuatxkiWJExrddisVmLQALarlxxPE/+hl3ki6iOK3ib+g3Bq25CiQkc8HP/ELfkhlHkqQ4tt", + "2NG0fmt2W0ekck3mxHMqicVpu1ZCSAnx1mqfty4puLtKjaTgjZxfYih7/7pBsXE4zlU7FRsyTWcDsSH2", + "aX/FhlxTjdgQ62jFhkFs1O7zNsXG9/TPXilnZG0EhB7khkLjwOMgNDgwVjPSonpvQyP0u9s6PBZjIwx4", + "aubxaKCNmiiJjTDgQVcoPiju2+aB3N71Dz2GYttypDqaIncd2JBkOfBAi70XLtuKvShJlwb1UTMyKud9", + "fNkrS62EVIM9fkrl5wCqv91WXZY2JSvtLlFpCs3nLHNLVRkrBzgBfDLnb7FP3yLioQ6n6FV9JpHqnJmV", + "oO1INHJsrxqWJipHGzd/p7KxWfCtWqvLDH8rGXcvGfeu0IkQdFVUvp3UWYoszjn16OWx1A2ERLbXcHWK", + "USuFdymF5Q6soJlWqHV7rpiqErhVTFvxaxK/QiGp04k3LnJ59byeGyYBqYmXYG1kLnJZ9hE8AuSDqQ+Z", + "9FXEjd6+8B4SXp0Pn7MZD1701qWMP/CSEbnNWtFMyUmFk0/7gmhwmM4habVCEnn2TzCM8bGbxDGs5mzM", + "bwe8oUO7lbj3FsP4PSTnYrAt0h2dqSGdMYjbAsQvX4AYukmMyJKJcTcMHxDsJ1R2/fWViqpC0qE8uUly", + "Z9uvIeMZIvNkeuwC358C98FIzufhIvIhgZymr+n8jvY8ohNxe9R7NvQ1xeW5HL5A4K9OzmreXl0xr1ee", + "dw6Bxw637x0/5JuR34eiWH8uIDOHO7nA/ByW6MMExGZRMKZfV0Mc69ocawye7eOMQdcQYWE48+F26I0N", + "/YPTG0ffhuktQ9wPR28oeEQEVtduwiyaSWrDvANTuq2ObzrChPUdirm2eIqrE1k5s/sIy43JL7DVF62P", + "VVaTp4C9jPImmhtijvaOgevCiJgtb332HacWNjFJidrUzed9OtuxJ/HB+USKIclgAKqgPr5yHf21HlMp", + "eXFsl/benr5iyKpbVFTSp9+b0Rfv09lWXXo6+Aboi6+8pa9K+uLYXoG+/HCGAjNZXYYz7KDAAexsPKpQ", + "MC7ZQFtyzqBHMB2/npB2d4/2w9kMeg4K2uvzC1+fu53fzs52te4oDikNMKPtICCILJ2e8wh85LHJ6KaI", + "JiiYOVCOZFZ4GWHrr/LdzrceDOhUvRgQ2GM2cKpD87caHTOHCanh5jAhduwcJi9vrBJMFu5Zoe7WSFWj", + "TTPqsbVPLeBiCmM8R1GDO5zSye4ex8/AT1k3kZRiqwSun7T5hU5FUXupW+VSp2KwniQjgPFTGFe4UqS5", + "2GkHR7avEqk3csztKUnncxDM0on2SVtyGWReiqhWnLdKUzOlqZrVOeXnmXFtfSqGMyqJ46prN2+BK1Wq", + "1FNqW3wvwdgnjpfIax8aW6bfzE1JUvlmLkvYB+7DVh6pxnTkPX6jqpGkDR+tHmGMBQhG9ye6BtFOukBh", + "GD9qtPRhcB++h+SzGHSjNYkVSLMMjadHJ0cnuhyQiufRX2nXrxblhicViy14W1YQ+xfoxJAkcZBDXuGm", + "Q8VsEgSUf9IpvvXkkL0w4imnyizwBKfzMHzoCUe04+/iB4vwd3rUidZlRzX+u31kuxjI7AiWTrRjPzDL", + "UHEJX3uwvbxxohierpKp0ftLtPhqxRzHAs82ZgrZVPjV13CMUNywbaLMveWbzfhPcui5+6RADcVMVcYV", + "ipW0DojATrpdLXvuEXsyq0xpi5ryaMqb7I/nGu9r3krrWM2cM614jjuZVvksa874w/FYbuw7Klbc2iNL", + "TsmlgC95QTH7IDO1ur7yYyUh26cd2Ata3lYUf+7cMJ0VAgOJRNnu4qAseU0Nym85zVBzcR1mK5wmxeAe", + "q0RgzWqwNrgX7WWETJMkWimAbYDeC2eOEMSqUMyK8THdOg3LnhMaqFw/Q6DYisFhLW+9NG+pUWjrMJaN", + "2mfPXc30wL1gsM3rgnlk2MbKi5ykOS7btXJoJRGK6mErD4wK4nrMWaMmWpXLo5uUr4uXMt5j+tJhPCkb", + "lMfbB37WlKjgBSY2UD949erBesBmcZhErO5HBoLcKCMorNNHuOzUpgHZspBYsxaXfFRqy3HtoTaxUv2v", + "RoJLpiYyOrfIrBpNkwWtlCNoLyXXRMMuR87wnlm3cUKpA3pdxlU+IBCTlKcQdu4hcefQM1WHygT/nitS", + "ggxWTDz0YumGFHgb5Rlqswu12YW2kF2okWgWsgFbvGrlTnIrsSx8aw7IBPMjyOUtSznpMLWeKtjKu71S", + "ATNSXFUFLDr+TSGIYZw6/nW1roDMk4zLgyT2O286neevz/8vAAD//yTV7zcJPwMA", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/cmd/hatchet-migrate/migrate/migrations/20260310150948_v1_0_84.sql b/cmd/hatchet-migrate/migrate/migrations/20260310150948_v1_0_84.sql new file mode 100644 index 0000000000..212b0cc3a3 --- /dev/null +++ b/cmd/hatchet-migrate/migrate/migrations/20260310150948_v1_0_84.sql @@ -0,0 +1,37 @@ +-- +goose Up + +CREATE TABLE v1_otel_traces ( + id BIGINT GENERATED ALWAYS AS IDENTITY, + tenant_id UUID NOT NULL, + trace_id TEXT NOT NULL, + span_id TEXT NOT NULL, + parent_span_id TEXT NOT NULL DEFAULT '', + span_name TEXT NOT NULL, + span_kind TEXT NOT NULL DEFAULT 'INTERNAL', + service_name TEXT NOT NULL DEFAULT 'unknown', + status_code TEXT NOT NULL DEFAULT 'UNSET', + status_message TEXT NOT NULL DEFAULT '', + duration_ns BIGINT NOT NULL DEFAULT 0, + resource_attributes JSONB NOT NULL DEFAULT '{}', + span_attributes JSONB NOT NULL DEFAULT '{}', + scope_name TEXT NOT NULL DEFAULT '', + scope_version TEXT NOT NULL DEFAULT '', + task_run_external_id UUID, + workflow_run_external_id UUID, + start_time TIMESTAMPTZ NOT NULL, + PRIMARY KEY (id, start_time) +) PARTITION BY RANGE (start_time); + +CREATE INDEX idx_v1_otel_traces_task_lookup + ON v1_otel_traces (tenant_id, task_run_external_id) + WHERE task_run_external_id IS NOT NULL; + +CREATE INDEX idx_v1_otel_traces_trace + ON v1_otel_traces (tenant_id, trace_id, start_time); + +-- +goose StatementBegin +SELECT create_v1_range_partition('v1_otel_traces'::text, CURRENT_DATE::date); +-- +goose StatementEnd + +-- +goose Down +DROP TABLE IF EXISTS v1_otel_traces; diff --git a/frontend/app/src/lib/api/generated/Api.ts b/frontend/app/src/lib/api/generated/Api.ts index 640e71940f..b44e020e0c 100644 --- a/frontend/app/src/lib/api/generated/Api.ts +++ b/frontend/app/src/lib/api/generated/Api.ts @@ -46,6 +46,7 @@ import { LogLineOrderByDirection, LogLineOrderByField, LogLineSearch, + OtelSpanList, RateLimitList, RateLimitOrderByDirection, RateLimitOrderByField, @@ -258,6 +259,23 @@ export class Api< format: "json", ...params, }); + /** + * @description Get OTel trace for a task run + * + * @tags Task + * @name V1TaskGetTrace + * @summary Get OTel trace + * @request GET:/api/v1/stable/tasks/{task}/trace + * @secure + */ + v1TaskGetTrace = (task: string, params: RequestParams = {}) => + this.request({ + path: `/api/v1/stable/tasks/${task}/trace`, + method: "GET", + secure: true, + format: "json", + ...params, + }); /** * @description Cancel tasks * diff --git a/frontend/app/src/lib/api/generated/cloud/Api.ts b/frontend/app/src/lib/api/generated/cloud/Api.ts index 3db0b7c591..0e4ce7b191 100644 --- a/frontend/app/src/lib/api/generated/cloud/Api.ts +++ b/frontend/app/src/lib/api/generated/cloud/Api.ts @@ -45,7 +45,6 @@ import { OrganizationForUserList, OrganizationInviteList, OrganizationTenant, - OtelSpanList, RejectOrganizationInviteRequest, RemoveOrganizationMembersRequest, RuntimeConfigActionsResponse, @@ -740,23 +739,6 @@ export class Api< type: ContentType.Json, ...params, }); - /** - * @description Lists OTel spans for a task - * - * @tags Observability - * @name OtelTracesList - * @summary List OTel Traces - * @request GET:/api/v1/cloud/tasks/{task}/traces - * @secure - */ - otelTracesList = (task: string, params: RequestParams = {}) => - this.request({ - path: `/api/v1/cloud/tasks/${task}/traces`, - method: "GET", - secure: true, - format: "json", - ...params, - }); /** * @description Receive a webhook message from Autumn * diff --git a/frontend/app/src/lib/api/generated/cloud/data-contracts.ts b/frontend/app/src/lib/api/generated/cloud/data-contracts.ts index 980e967a3e..dc0be99dc5 100644 --- a/frontend/app/src/lib/api/generated/cloud/data-contracts.ts +++ b/frontend/app/src/lib/api/generated/cloud/data-contracts.ts @@ -474,29 +474,6 @@ export interface LogLineList { pagination?: PaginationResponse; } -export interface OtelSpan { - trace_id: string; - span_id: string; - parent_span_id?: string; - span_name: string; - span_kind: string; - service_name: string; - status_code: string; - status_message?: string; - /** @format int64 */ - duration: number; - /** @format date-time */ - created_at: string; - resource_attributes?: Record; - span_attributes?: Record; - scope_name?: string; - scope_version?: string; -} - -export interface OtelSpanList { - rows?: OtelSpan[]; -} - export type Matrix = SampleStream[]; export interface SampleStream { diff --git a/frontend/app/src/lib/api/generated/data-contracts.ts b/frontend/app/src/lib/api/generated/data-contracts.ts index ef9bff9026..e81c6219ff 100644 --- a/frontend/app/src/lib/api/generated/data-contracts.ts +++ b/frontend/app/src/lib/api/generated/data-contracts.ts @@ -516,6 +516,29 @@ export interface V1LogLineList { rows?: V1LogLine[]; } +export interface OtelSpan { + trace_id: string; + span_id: string; + parent_span_id?: string; + span_name: string; + span_kind: string; + service_name: string; + status_code: string; + status_message?: string; + /** @format int64 */ + duration: number; + /** @format date-time */ + created_at: string; + resource_attributes?: Record; + span_attributes?: Record; + scope_name?: string; + scope_version?: string; +} + +export interface OtelSpanList { + rows?: OtelSpan[]; +} + export interface V1TaskFilter { /** @format date-time */ since: string; diff --git a/frontend/app/src/pages/main/v1/workflow-runs-v1/$run/v2components/step-run-detail/otel-span-adapter.ts b/frontend/app/src/pages/main/v1/workflow-runs-v1/$run/v2components/step-run-detail/otel-span-adapter.ts index e6e6bc3c29..e037d7c069 100644 --- a/frontend/app/src/pages/main/v1/workflow-runs-v1/$run/v2components/step-run-detail/otel-span-adapter.ts +++ b/frontend/app/src/pages/main/v1/workflow-runs-v1/$run/v2components/step-run-detail/otel-span-adapter.ts @@ -1,4 +1,4 @@ -import type { OtelSpan } from '@/lib/api/generated/cloud/data-contracts'; +import type { OtelSpan } from '@/lib/api/generated/data-contracts'; import type { OpenTelemetrySpan, OpenTelemetrySpanKind, diff --git a/frontend/app/src/pages/main/v1/workflow-runs-v1/$run/v2components/step-run-detail/step-run-detail.tsx b/frontend/app/src/pages/main/v1/workflow-runs-v1/$run/v2components/step-run-detail/step-run-detail.tsx index 1b6ed397fc..355d2e3ead 100644 --- a/frontend/app/src/pages/main/v1/workflow-runs-v1/$run/v2components/step-run-detail/step-run-detail.tsx +++ b/frontend/app/src/pages/main/v1/workflow-runs-v1/$run/v2components/step-run-detail/step-run-detail.tsx @@ -23,7 +23,6 @@ import { import { useSidePanel } from '@/hooks/use-side-panel'; import { V1TaskStatus, V1TaskSummary, queries } from '@/lib/api'; import { emptyGolangUUID, formatDuration } from '@/lib/utils'; -import useCloud from '@/pages/auth/hooks/use-cloud'; import { TaskRunActionButton } from '@/pages/main/v1/task-runs-v1/actions'; import { WorkflowDefinitionLink } from '@/pages/main/workflow-runs/$run/v2components/workflow-definition'; import { appRoutes } from '@/router'; @@ -109,7 +108,6 @@ export const TaskRunDetail = ({ showViewTaskRunButton, }: TaskRunDetailProps) => { const { open } = useSidePanel(); - const { isCloudEnabled } = useCloud(); const [logsResetKey, setLogsResetKey] = useState(0); const handleTaskRunExpand = useCallback( (taskRunId: string) => { @@ -252,11 +250,9 @@ export const TaskRunDetail = ({ Logs - {isCloudEnabled && ( - - Trace - - )} + + Trace + - {isCloudEnabled && ( - - - - )} + + + { - const res = await cloudApi.otelTracesList(taskExternalId); + const res = await api.v1TaskGetTrace(taskExternalId); return res.data; }, refetchInterval: isRunning ? 100 : false, diff --git a/pkg/client/rest/gen.go b/pkg/client/rest/gen.go index d413f0c92a..7e6e44d6ad 100644 --- a/pkg/client/rest/gen.go +++ b/pkg/client/rest/gen.go @@ -755,6 +755,29 @@ type LogLineOrderByField string // LogLineSearch defines model for LogLineSearch. type LogLineSearch = string +// OtelSpan defines model for OtelSpan. +type OtelSpan struct { + CreatedAt time.Time `json:"created_at"` + Duration int64 `json:"duration"` + ParentSpanId *string `json:"parent_span_id,omitempty"` + ResourceAttributes *map[string]string `json:"resource_attributes,omitempty"` + ScopeName *string `json:"scope_name,omitempty"` + ScopeVersion *string `json:"scope_version,omitempty"` + ServiceName string `json:"service_name"` + SpanAttributes *map[string]string `json:"span_attributes,omitempty"` + SpanId string `json:"span_id"` + SpanKind string `json:"span_kind"` + SpanName string `json:"span_name"` + StatusCode string `json:"status_code"` + StatusMessage *string `json:"status_message,omitempty"` + TraceId string `json:"trace_id"` +} + +// OtelSpanList defines model for OtelSpanList. +type OtelSpanList struct { + Rows *[]OtelSpan `json:"rows,omitempty"` +} + // PaginationResponse defines model for PaginationResponse. type PaginationResponse struct { // CurrentPage the current page @@ -3266,6 +3289,9 @@ type ClientInterface interface { // V1TaskEventList request V1TaskEventList(ctx context.Context, task openapi_types.UUID, params *V1TaskEventListParams, reqEditors ...RequestEditorFn) (*http.Response, error) + // V1TaskGetTrace request + V1TaskGetTrace(ctx context.Context, task openapi_types.UUID, reqEditors ...RequestEditorFn) (*http.Response, error) + // V1CelDebugWithBody request with any body V1CelDebugWithBody(ctx context.Context, tenant openapi_types.UUID, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) @@ -3919,6 +3945,18 @@ func (c *Client) V1TaskEventList(ctx context.Context, task openapi_types.UUID, p return c.Client.Do(req) } +func (c *Client) V1TaskGetTrace(ctx context.Context, task openapi_types.UUID, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewV1TaskGetTraceRequest(c.Server, task) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + if err := c.applyEditors(ctx, req, reqEditors); err != nil { + return nil, err + } + return c.Client.Do(req) +} + func (c *Client) V1CelDebugWithBody(ctx context.Context, tenant openapi_types.UUID, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) { req, err := NewV1CelDebugRequestWithBody(c.Server, tenant, contentType, body) if err != nil { @@ -6565,6 +6603,40 @@ func NewV1TaskEventListRequest(server string, task openapi_types.UUID, params *V return req, nil } +// NewV1TaskGetTraceRequest generates requests for V1TaskGetTrace +func NewV1TaskGetTraceRequest(server string, task openapi_types.UUID) (*http.Request, error) { + var err error + + var pathParam0 string + + pathParam0, err = runtime.StyleParamWithLocation("simple", false, "task", runtime.ParamLocationPath, task) + if err != nil { + return nil, err + } + + serverURL, err := url.Parse(server) + if err != nil { + return nil, err + } + + operationPath := fmt.Sprintf("/api/v1/stable/tasks/%s/trace", pathParam0) + if operationPath[0] == '/' { + operationPath = "." + operationPath + } + + queryURL, err := serverURL.Parse(operationPath) + if err != nil { + return nil, err + } + + req, err := http.NewRequest("GET", queryURL.String(), nil) + if err != nil { + return nil, err + } + + return req, nil +} + // NewV1CelDebugRequest calls the generic V1CelDebug builder with application/json body func NewV1CelDebugRequest(server string, tenant openapi_types.UUID, body V1CelDebugJSONRequestBody) (*http.Request, error) { var bodyReader io.Reader @@ -13268,6 +13340,9 @@ type ClientWithResponsesInterface interface { // V1TaskEventListWithResponse request V1TaskEventListWithResponse(ctx context.Context, task openapi_types.UUID, params *V1TaskEventListParams, reqEditors ...RequestEditorFn) (*V1TaskEventListResponse, error) + // V1TaskGetTraceWithResponse request + V1TaskGetTraceWithResponse(ctx context.Context, task openapi_types.UUID, reqEditors ...RequestEditorFn) (*V1TaskGetTraceResponse, error) + // V1CelDebugWithBodyWithResponse request with any body V1CelDebugWithBodyWithResponse(ctx context.Context, tenant openapi_types.UUID, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*V1CelDebugResponse, error) @@ -14119,6 +14194,31 @@ func (r V1TaskEventListResponse) StatusCode() int { return 0 } +type V1TaskGetTraceResponse struct { + Body []byte + HTTPResponse *http.Response + JSON200 *OtelSpanList + JSON400 *APIErrors + JSON403 *APIErrors + JSON404 *APIErrors +} + +// Status returns HTTPResponse.Status +func (r V1TaskGetTraceResponse) Status() string { + if r.HTTPResponse != nil { + return r.HTTPResponse.Status + } + return http.StatusText(0) +} + +// StatusCode returns HTTPResponse.StatusCode +func (r V1TaskGetTraceResponse) StatusCode() int { + if r.HTTPResponse != nil { + return r.HTTPResponse.StatusCode + } + return 0 +} + type V1CelDebugResponse struct { Body []byte HTTPResponse *http.Response @@ -17116,6 +17216,15 @@ func (c *ClientWithResponses) V1TaskEventListWithResponse(ctx context.Context, t return ParseV1TaskEventListResponse(rsp) } +// V1TaskGetTraceWithResponse request returning *V1TaskGetTraceResponse +func (c *ClientWithResponses) V1TaskGetTraceWithResponse(ctx context.Context, task openapi_types.UUID, reqEditors ...RequestEditorFn) (*V1TaskGetTraceResponse, error) { + rsp, err := c.V1TaskGetTrace(ctx, task, reqEditors...) + if err != nil { + return nil, err + } + return ParseV1TaskGetTraceResponse(rsp) +} + // V1CelDebugWithBodyWithResponse request with arbitrary body returning *V1CelDebugResponse func (c *ClientWithResponses) V1CelDebugWithBodyWithResponse(ctx context.Context, tenant openapi_types.UUID, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*V1CelDebugResponse, error) { rsp, err := c.V1CelDebugWithBody(ctx, tenant, contentType, body, reqEditors...) @@ -19142,6 +19251,53 @@ func ParseV1TaskEventListResponse(rsp *http.Response) (*V1TaskEventListResponse, return response, nil } +// ParseV1TaskGetTraceResponse parses an HTTP response from a V1TaskGetTraceWithResponse call +func ParseV1TaskGetTraceResponse(rsp *http.Response) (*V1TaskGetTraceResponse, error) { + bodyBytes, err := io.ReadAll(rsp.Body) + defer func() { _ = rsp.Body.Close() }() + if err != nil { + return nil, err + } + + response := &V1TaskGetTraceResponse{ + Body: bodyBytes, + HTTPResponse: rsp, + } + + switch { + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 200: + var dest OtelSpanList + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON200 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 400: + var dest APIErrors + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON400 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 403: + var dest APIErrors + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON403 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 404: + var dest APIErrors + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON404 = &dest + + } + + return response, nil +} + // ParseV1CelDebugResponse parses an HTTP response from a V1CelDebugWithResponse call func ParseV1CelDebugResponse(rsp *http.Response) (*V1CelDebugResponse, error) { bodyBytes, err := io.ReadAll(rsp.Body) diff --git a/pkg/repository/otelcol.go b/pkg/repository/otelcol.go index 001c06b2cb..4f81c3143a 100644 --- a/pkg/repository/otelcol.go +++ b/pkg/repository/otelcol.go @@ -2,9 +2,14 @@ package repository import ( "context" + "encoding/hex" + "encoding/json" + "fmt" "time" "github.com/google/uuid" + "github.com/jackc/pgx/v5" + tracev1 "go.opentelemetry.io/proto/otlp/trace/v1" ) type SpanData struct { @@ -64,12 +69,283 @@ func newOTelCollectorRepository(s *sharedRepository) OTelCollectorRepository { } } +// transformedSpan holds pre-processed span data ready for insertion. +type transformedSpan struct { + startTime time.Time + taskRunExternalID *uuid.UUID + workflowRunExternalID *uuid.UUID + statusMessage string + scopeVersion string + spanKind string + serviceName string + statusCode string + traceID string + spanID string + parentSpanID string + spanName string + scopeName string + spanAttributes []byte + resourceAttributes []byte + durationNs int64 + tenantID uuid.UUID +} + +// spanCopyFromSource implements pgx.CopyFromSource for batch inserts. +type spanCopyFromSource struct { + spans []transformedSpan + idx int +} + +func (s *spanCopyFromSource) Next() bool { + s.idx++ + return s.idx < len(s.spans) +} + +func (s *spanCopyFromSource) Values() ([]interface{}, error) { + span := s.spans[s.idx] + return []interface{}{ + span.tenantID, + span.traceID, + span.spanID, + span.parentSpanID, + span.spanName, + span.spanKind, + span.serviceName, + span.statusCode, + span.statusMessage, + span.durationNs, + span.resourceAttributes, + span.spanAttributes, + span.scopeName, + span.scopeVersion, + span.taskRunExternalID, + span.workflowRunExternalID, + span.startTime, + }, nil +} + +func (s *spanCopyFromSource) Err() error { + return nil +} + func (o *otelCollectorRepositoryImpl) CreateSpans(ctx context.Context, tenantId uuid.UUID, opts *CreateSpansOpts) error { - // intentional no-op, intended to be overridden + if opts == nil { + return fmt.Errorf("opts cannot be nil") + } + + if err := o.v.Validate(opts); err != nil { + return fmt.Errorf("validation error: %w", err) + } + + if len(opts.Spans) == 0 { + return nil + } + + transformed := make([]transformedSpan, 0, len(opts.Spans)) + for _, sd := range opts.Spans { + ts := transformedSpan{ + tenantID: tenantId, + traceID: hex.EncodeToString(sd.TraceID), + spanID: hex.EncodeToString(sd.SpanID), + spanName: sd.Name, + spanKind: spanKindToString(sd.Kind), + serviceName: extractServiceName(sd.ResourceAttributes), + statusCode: spanStatusCodeToString(sd.StatusCode), + statusMessage: sd.StatusMessage, + durationNs: int64(sd.EndTimeUnixNano - sd.StartTimeUnixNano), //nolint:gosec + scopeName: sd.InstrumentationScope, + startTime: time.Unix(0, int64(sd.StartTimeUnixNano)), //nolint:gosec + } + + if len(sd.ParentSpanID) > 0 { + ts.parentSpanID = hex.EncodeToString(sd.ParentSpanID) + } + + ts.resourceAttributes = jsonBytesToJSONB(sd.ResourceAttributes) + ts.spanAttributes = jsonBytesToJSONB(sd.Attributes) + + if sd.TaskRunExternalID != nil && *sd.TaskRunExternalID != uuid.Nil { + id := *sd.TaskRunExternalID + ts.taskRunExternalID = &id + } + + if sd.WorkflowRunID != nil && *sd.WorkflowRunID != uuid.Nil { + id := *sd.WorkflowRunID + ts.workflowRunExternalID = &id + } + + transformed = append(transformed, ts) + } + + _, err := o.pool.CopyFrom( + ctx, + pgx.Identifier{"v1_otel_traces"}, + []string{ + "tenant_id", "trace_id", "span_id", "parent_span_id", + "span_name", "span_kind", "service_name", "status_code", + "status_message", "duration_ns", "resource_attributes", + "span_attributes", "scope_name", "scope_version", + "task_run_external_id", "workflow_run_external_id", "start_time", + }, + &spanCopyFromSource{spans: transformed, idx: -1}, + ) + + if err != nil { + return fmt.Errorf("error copying spans to v1_otel_traces: %w", err) + } + return nil } func (o *otelCollectorRepositoryImpl) ListSpansByTaskExternalID(ctx context.Context, tenantId, taskExternalID uuid.UUID) ([]*OtelSpanRow, error) { - // intentional no-op, intended to be overridden - return nil, nil + query := ` + SELECT + trace_id, span_id, parent_span_id, span_name, span_kind, + service_name, status_code, status_message, duration_ns, start_time, + resource_attributes, span_attributes, scope_name, scope_version + FROM v1_otel_traces + WHERE tenant_id = $1 AND trace_id IN ( + SELECT DISTINCT trace_id FROM v1_otel_traces + WHERE tenant_id = $1 AND task_run_external_id = $2 + ) + ORDER BY start_time ASC + LIMIT 1000 + ` + + rows, err := o.pool.Query(ctx, query, tenantId, taskExternalID) + if err != nil { + return nil, fmt.Errorf("error querying v1_otel_traces: %w", err) + } + defer rows.Close() + + var result []*OtelSpanRow + for rows.Next() { + row := &OtelSpanRow{} + var durationNs int64 + var resourceAttrsJSON, spanAttrsJSON []byte + + if err := rows.Scan( + &row.TraceID, + &row.SpanID, + &row.ParentSpanID, + &row.SpanName, + &row.SpanKind, + &row.ServiceName, + &row.StatusCode, + &row.StatusMessage, + &durationNs, + &row.CreatedAt, + &resourceAttrsJSON, + &spanAttrsJSON, + &row.ScopeName, + &row.ScopeVersion, + ); err != nil { + return nil, fmt.Errorf("error scanning v1_otel_traces row: %w", err) + } + + row.Duration = uint64(durationNs) //nolint:gosec + row.ResourceAttributes = jsonbToStringMap(resourceAttrsJSON) + row.SpanAttributes = jsonbToStringMap(spanAttrsJSON) + + result = append(result, row) + } + + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("error iterating v1_otel_traces rows: %w", err) + } + + return result, nil +} + +// Helper functions for data transformation + +func extractServiceName(resourceAttrsJSON []byte) string { + if len(resourceAttrsJSON) == 0 { + return "unknown" + } + + var attrs map[string]interface{} + if err := json.Unmarshal(resourceAttrsJSON, &attrs); err != nil { + return "unknown" + } + + if serviceName, ok := attrs["service.name"].(string); ok { + return serviceName + } + + return "unknown" +} + +func jsonBytesToJSONB(jsonBytes []byte) []byte { + if len(jsonBytes) == 0 { + return []byte("{}") + } + // Validate it's valid JSON; if not, return empty object + var raw json.RawMessage + if err := json.Unmarshal(jsonBytes, &raw); err != nil { + return []byte("{}") + } + return jsonBytes +} + +func jsonbToStringMap(jsonBytes []byte) map[string]string { + if len(jsonBytes) == 0 { + return make(map[string]string) + } + + var jsonMap map[string]interface{} + if err := json.Unmarshal(jsonBytes, &jsonMap); err != nil { + return make(map[string]string) + } + + result := make(map[string]string, len(jsonMap)) + for k, v := range jsonMap { + switch val := v.(type) { + case string: + result[k] = val + case nil: + result[k] = "" + default: + b, err := json.Marshal(v) + if err != nil { + result[k] = fmt.Sprintf("%v", v) + } else { + result[k] = string(b) + } + } + } + + return result +} + +func spanKindToString(kind int32) string { + switch tracev1.Span_SpanKind(kind) { + case tracev1.Span_SPAN_KIND_UNSPECIFIED: + return "UNSPECIFIED" + case tracev1.Span_SPAN_KIND_INTERNAL: + return "INTERNAL" + case tracev1.Span_SPAN_KIND_SERVER: + return "SERVER" + case tracev1.Span_SPAN_KIND_CLIENT: + return "CLIENT" + case tracev1.Span_SPAN_KIND_PRODUCER: + return "PRODUCER" + case tracev1.Span_SPAN_KIND_CONSUMER: + return "CONSUMER" + default: + return "UNKNOWN" + } +} + +func spanStatusCodeToString(code int32) string { + switch tracev1.Status_StatusCode(code) { + case tracev1.Status_STATUS_CODE_UNSET: + return "UNSET" + case tracev1.Status_STATUS_CODE_OK: + return "OK" + case tracev1.Status_STATUS_CODE_ERROR: + return "ERROR" + default: + return "UNKNOWN" + } } diff --git a/pkg/repository/sqlcv1/models.go b/pkg/repository/sqlcv1/models.go index bc6d5c5583..5c8410f8ad 100644 --- a/pkg/repository/sqlcv1/models.go +++ b/pkg/repository/sqlcv1/models.go @@ -3203,6 +3203,27 @@ type V1OperationIntervalSettings struct { IntervalNanoseconds int64 `json:"interval_nanoseconds"` } +type V1OtelTraces struct { + ID int64 `json:"id"` + TenantID uuid.UUID `json:"tenant_id"` + TraceID string `json:"trace_id"` + SpanID string `json:"span_id"` + ParentSpanID string `json:"parent_span_id"` + SpanName string `json:"span_name"` + SpanKind string `json:"span_kind"` + ServiceName string `json:"service_name"` + StatusCode string `json:"status_code"` + StatusMessage string `json:"status_message"` + DurationNs int64 `json:"duration_ns"` + ResourceAttributes []byte `json:"resource_attributes"` + SpanAttributes []byte `json:"span_attributes"` + ScopeName string `json:"scope_name"` + ScopeVersion string `json:"scope_version"` + TaskRunExternalID *uuid.UUID `json:"task_run_external_id"` + WorkflowRunExternalID *uuid.UUID `json:"workflow_run_external_id"` + StartTime pgtype.Timestamptz `json:"start_time"` +} + type V1Payload struct { TenantID uuid.UUID `json:"tenant_id"` ID int64 `json:"id"` diff --git a/pkg/repository/sqlcv1/tasks.sql b/pkg/repository/sqlcv1/tasks.sql index 4639bfcb18..6a8b35935f 100644 --- a/pkg/repository/sqlcv1/tasks.sql +++ b/pkg/repository/sqlcv1/tasks.sql @@ -7,7 +7,8 @@ SELECT create_v1_range_partition('v1_payload', @date::date), create_v1_range_partition('v1_event', @date::date), create_v1_weekly_range_partition('v1_event_lookup_table', @date::date), - create_v1_range_partition('v1_event_to_run', @date::date); + create_v1_range_partition('v1_event_to_run', @date::date), + create_v1_range_partition('v1_otel_traces', @date::date); -- name: EnsureTablePartitionsExist :one WITH tomorrow_date AS ( @@ -25,6 +26,8 @@ WITH tomorrow_date AS ( SELECT 'v1_payload_' || to_char((SELECT date FROM tomorrow_date), 'YYYYMMDD') UNION ALL SELECT 'v1_event_' || to_char((SELECT date FROM tomorrow_date), 'YYYYMMDD') + UNION ALL + SELECT 'v1_otel_traces_' || to_char((SELECT date FROM tomorrow_date), 'YYYYMMDD') ), partition_check AS ( SELECT COUNT(*) AS total_tables, @@ -56,6 +59,8 @@ WITH task_partitions AS ( SELECT 'v1_event_lookup_table' AS parent_table, p::text as partition_name FROM get_v1_weekly_partitions_before_date('v1_event_lookup_table', @date::date) AS p ), event_to_run_partitions AS ( SELECT 'v1_event_to_run' AS parent_table, p::text as partition_name FROM get_v1_partitions_before_date('v1_event_to_run', @date::date) AS p +), otel_traces_partitions AS ( + SELECT 'v1_otel_traces' AS parent_table, p::text as partition_name FROM get_v1_partitions_before_date('v1_otel_traces', @date::date) AS p ) SELECT @@ -111,6 +116,13 @@ SELECT * FROM event_to_run_partitions + +UNION ALL + +SELECT + * +FROM + otel_traces_partitions ; -- name: DefaultTaskActivityGauge :one diff --git a/pkg/repository/sqlcv1/tasks.sql.go b/pkg/repository/sqlcv1/tasks.sql.go index 673c2b6243..ce43061fef 100644 --- a/pkg/repository/sqlcv1/tasks.sql.go +++ b/pkg/repository/sqlcv1/tasks.sql.go @@ -233,7 +233,8 @@ SELECT create_v1_range_partition('v1_payload', $1::date), create_v1_range_partition('v1_event', $1::date), create_v1_weekly_range_partition('v1_event_lookup_table', $1::date), - create_v1_range_partition('v1_event_to_run', $1::date) + create_v1_range_partition('v1_event_to_run', $1::date), + create_v1_range_partition('v1_otel_traces', $1::date) ` func (q *Queries) CreatePartitions(ctx context.Context, db DBTX, date pgtype.Date) error { @@ -329,6 +330,8 @@ WITH tomorrow_date AS ( SELECT 'v1_payload_' || to_char((SELECT date FROM tomorrow_date), 'YYYYMMDD') UNION ALL SELECT 'v1_event_' || to_char((SELECT date FROM tomorrow_date), 'YYYYMMDD') + UNION ALL + SELECT 'v1_otel_traces_' || to_char((SELECT date FROM tomorrow_date), 'YYYYMMDD') ), partition_check AS ( SELECT COUNT(*) AS total_tables, @@ -1242,6 +1245,8 @@ WITH task_partitions AS ( SELECT 'v1_event_lookup_table' AS parent_table, p::text as partition_name FROM get_v1_weekly_partitions_before_date('v1_event_lookup_table', $1::date) AS p ), event_to_run_partitions AS ( SELECT 'v1_event_to_run' AS parent_table, p::text as partition_name FROM get_v1_partitions_before_date('v1_event_to_run', $1::date) AS p +), otel_traces_partitions AS ( + SELECT 'v1_otel_traces' AS parent_table, p::text as partition_name FROM get_v1_partitions_before_date('v1_otel_traces', $1::date) AS p ) SELECT @@ -1297,6 +1302,13 @@ SELECT parent_table, partition_name FROM event_to_run_partitions + +UNION ALL + +SELECT + parent_table, partition_name +FROM + otel_traces_partitions ` type ListPartitionsBeforeDateRow struct { diff --git a/sql/schema/v1-core.sql b/sql/schema/v1-core.sql index e2c1cd3abe..96d76ef370 100644 --- a/sql/schema/v1-core.sql +++ b/sql/schema/v1-core.sql @@ -2274,3 +2274,35 @@ CREATE TABLE v1_event_to_run ( PRIMARY KEY (event_id, event_seen_at, run_external_id) ) PARTITION BY RANGE(event_seen_at); + +-- OTEL TRACES -- +CREATE TABLE v1_otel_traces ( + id BIGINT GENERATED ALWAYS AS IDENTITY, + tenant_id UUID NOT NULL, + trace_id TEXT NOT NULL, + span_id TEXT NOT NULL, + parent_span_id TEXT NOT NULL DEFAULT '', + span_name TEXT NOT NULL, + span_kind TEXT NOT NULL DEFAULT 'INTERNAL', + service_name TEXT NOT NULL DEFAULT 'unknown', + status_code TEXT NOT NULL DEFAULT 'UNSET', + status_message TEXT NOT NULL DEFAULT '', + duration_ns BIGINT NOT NULL DEFAULT 0, + resource_attributes JSONB NOT NULL DEFAULT '{}', + span_attributes JSONB NOT NULL DEFAULT '{}', + scope_name TEXT NOT NULL DEFAULT '', + scope_version TEXT NOT NULL DEFAULT '', + task_run_external_id UUID, + workflow_run_external_id UUID, + start_time TIMESTAMPTZ NOT NULL, + PRIMARY KEY (id, start_time) +) PARTITION BY RANGE (start_time); + +CREATE INDEX idx_v1_otel_traces_task_lookup + ON v1_otel_traces (tenant_id, task_run_external_id) + WHERE task_run_external_id IS NOT NULL; + +CREATE INDEX idx_v1_otel_traces_trace + ON v1_otel_traces (tenant_id, trace_id, start_time); + +SELECT create_v1_range_partition('v1_otel_traces', CURRENT_DATE); From 9396872e0a7a9011c3e7794548e73af156276096 Mon Sep 17 00:00:00 2001 From: Mohammed Nafees Date: Tue, 10 Mar 2026 15:58:50 +0100 Subject: [PATCH 12/17] add to rbac.yaml --- api/v1/server/rbac/rbac.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/api/v1/server/rbac/rbac.yaml b/api/v1/server/rbac/rbac.yaml index 7b129dff07..4dd3070ed8 100644 --- a/api/v1/server/rbac/rbac.yaml +++ b/api/v1/server/rbac/rbac.yaml @@ -140,3 +140,4 @@ roles: - WorkflowCronDelete - WorkflowCronGet - WorkflowCronUpdate + - V1TaskGetTrace From 35c59b9ba783a999718d3ff0f01500fa346156f9 Mon Sep 17 00:00:00 2001 From: Mohammed Nafees Date: Wed, 11 Mar 2026 00:50:52 +0100 Subject: [PATCH 13/17] some refactor --- .../openapi/components/schemas/v1/otel.yaml | 2 + .../openapi/paths/v1/tasks/tasks.yaml | 14 + api/v1/server/handlers/v1/tasks/trace.go | 43 +- api/v1/server/oas/gen/openapi.gen.go | 590 +++++++++--------- api/v1/server/oas/transformers/v1/trace.go | 86 +++ .../migrations/20260310150948_v1_0_84.sql | 11 +- examples/go/opentelemetry-propagation/main.go | 147 ----- examples/go/opentelemetry/main.go | 165 ++--- frontend/app/src/lib/api/generated/Api.ts | 18 +- .../src/lib/api/generated/data-contracts.ts | 1 + .../step-run-detail/task-run-trace.tsx | 37 +- internal/services/otelcol/server.go | 4 +- pkg/client/rest/gen.go | 64 +- pkg/repository/otelcol.go | 319 +++------- pkg/repository/sqlcv1/copyfrom.go | 48 ++ pkg/repository/sqlcv1/models.go | 93 ++- pkg/repository/sqlcv1/otel.sql | 43 ++ pkg/repository/sqlcv1/otel.sql.go | 132 ++++ pkg/repository/sqlcv1/sqlc.yaml | 1 + .../opentelemetry-propagation/main.go | 147 ----- sdks/go/examples/opentelemetry/main.go | 165 ++--- sql/schema/v1-core.sql | 11 +- 22 files changed, 1086 insertions(+), 1055 deletions(-) create mode 100644 api/v1/server/oas/transformers/v1/trace.go delete mode 100644 examples/go/opentelemetry-propagation/main.go create mode 100644 pkg/repository/sqlcv1/otel.sql create mode 100644 pkg/repository/sqlcv1/otel.sql.go delete mode 100644 sdks/go/examples/opentelemetry-propagation/main.go diff --git a/api-contracts/openapi/components/schemas/v1/otel.yaml b/api-contracts/openapi/components/schemas/v1/otel.yaml index e831ea4b5f..3ddd61c480 100644 --- a/api-contracts/openapi/components/schemas/v1/otel.yaml +++ b/api-contracts/openapi/components/schemas/v1/otel.yaml @@ -48,6 +48,8 @@ OtelSpan: OtelSpanList: type: object properties: + pagination: + $ref: "../metadata.yaml#/PaginationResponse" rows: type: array items: diff --git a/api-contracts/openapi/paths/v1/tasks/tasks.yaml b/api-contracts/openapi/paths/v1/tasks/tasks.yaml index f35aebfdec..0952d91e65 100644 --- a/api-contracts/openapi/paths/v1/tasks/tasks.yaml +++ b/api-contracts/openapi/paths/v1/tasks/tasks.yaml @@ -456,6 +456,20 @@ getTrace: format: uuid minLength: 36 maxLength: 36 + - description: The number to skip + in: query + name: offset + required: false + schema: + type: integer + format: int64 + - description: The number to limit by + in: query + name: limit + required: false + schema: + type: integer + format: int64 responses: "200": content: diff --git a/api/v1/server/handlers/v1/tasks/trace.go b/api/v1/server/handlers/v1/tasks/trace.go index 27d6226389..3028fd4eee 100644 --- a/api/v1/server/handlers/v1/tasks/trace.go +++ b/api/v1/server/handlers/v1/tasks/trace.go @@ -4,43 +4,28 @@ import ( "github.com/labstack/echo/v4" "github.com/hatchet-dev/hatchet/api/v1/server/oas/gen" - "github.com/hatchet-dev/hatchet/pkg/repository" + transformers "github.com/hatchet-dev/hatchet/api/v1/server/oas/transformers/v1" "github.com/hatchet-dev/hatchet/pkg/repository/sqlcv1" ) func (t *TasksService) V1TaskGetTrace(ctx echo.Context, request gen.V1TaskGetTraceRequestObject) (gen.V1TaskGetTraceResponseObject, error) { task := ctx.Get("task").(*sqlcv1.V1TasksOlap) - spans, err := t.config.V1.OTelCollector().ListSpansByTaskExternalID( - ctx.Request().Context(), task.TenantID, task.ExternalID) - if err != nil { - return nil, err - } + limit := int64(1000) + offset := int64(0) - apiSpans := convertToAPISpans(spans) + if request.Params.Limit != nil { + limit = *request.Params.Limit + } - return gen.V1TaskGetTrace200JSONResponse(gen.OtelSpanList{Rows: &apiSpans}), nil -} + if request.Params.Offset != nil { + offset = *request.Params.Offset + } -func convertToAPISpans(spans []*repository.OtelSpanRow) []gen.OtelSpan { - result := make([]gen.OtelSpan, len(spans)) - for i, s := range spans { - result[i] = gen.OtelSpan{ - TraceId: s.TraceID, - SpanId: s.SpanID, - ParentSpanId: &s.ParentSpanID, - SpanName: s.SpanName, - SpanKind: s.SpanKind, - ServiceName: s.ServiceName, - StatusCode: s.StatusCode, - StatusMessage: &s.StatusMessage, - Duration: int64(s.Duration), //nolint:gosec - CreatedAt: s.CreatedAt, - ResourceAttributes: &s.ResourceAttributes, - SpanAttributes: &s.SpanAttributes, - ScopeName: &s.ScopeName, - ScopeVersion: &s.ScopeVersion, - } + result, err := t.config.V1.OTelCollector().ListSpansByTaskExternalID(ctx.Request().Context(), task.TenantID, task.ExternalID, offset, limit) + if err != nil { + return nil, err } - return result + + return gen.V1TaskGetTrace200JSONResponse(transformers.ToV1OtelSpanList(result.Rows, limit, offset, result.Total)), nil } diff --git a/api/v1/server/oas/gen/openapi.gen.go b/api/v1/server/oas/gen/openapi.gen.go index 7c1a16442e..6aaffc3f54 100644 --- a/api/v1/server/oas/gen/openapi.gen.go +++ b/api/v1/server/oas/gen/openapi.gen.go @@ -778,7 +778,8 @@ type OtelSpan struct { // OtelSpanList defines model for OtelSpanList. type OtelSpanList struct { - Rows *[]OtelSpan `json:"rows,omitempty"` + Pagination *PaginationResponse `json:"pagination,omitempty"` + Rows *[]OtelSpan `json:"rows,omitempty"` } // PaginationResponse defines model for PaginationResponse. @@ -2520,6 +2521,15 @@ type V1TaskEventListParams struct { Limit *int64 `form:"limit,omitempty" json:"limit,omitempty"` } +// V1TaskGetTraceParams defines parameters for V1TaskGetTrace. +type V1TaskGetTraceParams struct { + // Offset The number to skip + Offset *int64 `form:"offset,omitempty" json:"offset,omitempty"` + + // Limit The number to limit by + Limit *int64 `form:"limit,omitempty" json:"limit,omitempty"` +} + // V1EventListParams defines parameters for V1EventList. type V1EventListParams struct { // Offset The number to skip @@ -3221,7 +3231,7 @@ type ServerInterface interface { V1TaskEventList(ctx echo.Context, task openapi_types.UUID, params V1TaskEventListParams) error // Get OTel trace // (GET /api/v1/stable/tasks/{task}/trace) - V1TaskGetTrace(ctx echo.Context, task openapi_types.UUID) error + V1TaskGetTrace(ctx echo.Context, task openapi_types.UUID, params V1TaskGetTraceParams) error // Debug a CEL expression // (POST /api/v1/stable/tenants/{tenant}/cel/debug) V1CelDebug(ctx echo.Context, tenant openapi_types.UUID) error @@ -3990,8 +4000,24 @@ func (w *ServerInterfaceWrapper) V1TaskGetTrace(ctx echo.Context) error { ctx.Set(CookieAuthScopes, []string{}) + // Parameter object where we will unmarshal all parameters from the context + var params V1TaskGetTraceParams + // ------------- Optional query parameter "offset" ------------- + + err = runtime.BindQueryParameter("form", true, false, "offset", ctx.QueryParams(), ¶ms.Offset) + if err != nil { + return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Invalid format for parameter offset: %s", err)) + } + + // ------------- Optional query parameter "limit" ------------- + + err = runtime.BindQueryParameter("form", true, false, "limit", ctx.QueryParams(), ¶ms.Limit) + if err != nil { + return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Invalid format for parameter limit: %s", err)) + } + // Invoke the callback with all the unmarshaled arguments - err = w.Handler.V1TaskGetTrace(ctx, task) + err = w.Handler.V1TaskGetTrace(ctx, task, params) return err } @@ -8101,7 +8127,8 @@ func (response V1TaskEventList501JSONResponse) VisitV1TaskEventListResponse(w ht } type V1TaskGetTraceRequestObject struct { - Task openapi_types.UUID `json:"task"` + Task openapi_types.UUID `json:"task"` + Params V1TaskGetTraceParams } type V1TaskGetTraceResponseObject interface { @@ -13448,10 +13475,11 @@ func (sh *strictHandler) V1TaskEventList(ctx echo.Context, task openapi_types.UU } // V1TaskGetTrace operation -func (sh *strictHandler) V1TaskGetTrace(ctx echo.Context, task openapi_types.UUID) error { +func (sh *strictHandler) V1TaskGetTrace(ctx echo.Context, task openapi_types.UUID, params V1TaskGetTraceParams) error { var request V1TaskGetTraceRequestObject request.Task = task + request.Params = params handler := func(ctx echo.Context, request interface{}) (interface{}, error) { return sh.ssi.V1TaskGetTrace(ctx, request.(V1TaskGetTraceRequestObject)) @@ -16323,282 +16351,282 @@ var swaggerSpec = []string{ "evMWGptEAVuq11oWVpDO8DVD1SV8hH7ecPP2lgqa4dW7606386U/uup0O4PR6HqklynKOOnlyWr/cxDo", "BIn4/vJ3T0lWeunBP65x/8yP0PAGKjpX3EGvCfTHEQiMXHEHGhyvqQODnWtHBGIYkDscgeAO6c946eN5", "BwiJ0TQhsNKfxPSWp+hIbhjBO7Pmzj4/Zi9J5RYwfkRu1RB0ORsDtwI37NsDCiq+VtxQqEJxJ+OiTN8V", - "oVWGNQYu1MNWfPCVLbP1qPCpKyngNw+oQmBdlT510l4Stl5CNOLwlEesbN8akaJ6P37vCIe0u4gh9qzb", - "CeA3+a9X3U6QLNg/cOfN6Ql7V8lxZa6zzkla+rtFXK6nE59ZGSoUWLQRBfBbeeRXdiNn69L6docE+KpZ", - "iDZltlIfYcLfC7OIvBMbu4hmc/4rgQm9ScXI1Wg4QbK4sTNaMbqRpqsj03r/y8pOxcdC3NWbGa2MA47s", - "DFR8RGGmOurUuhhloOZm6aoI0fHYCBDIPDTLqLR6BWFOpcwzUe80CjAZwXvkG1wdmMu/iAlQB2PxADHr", - "CJlf3hYCJ9hEn4GfQFtX1Jg7L2CHxZqJRxSx608o8MIn/bZv4pWmBtGP5nVIaaJZxwJ40HYR/Jt+Cv6N", - "LYPuJQoUH8sMzTwq6j6MXejZ+lIpN29lv+R6U6hylPZVpes9UC8zHtMqmOnnNVTM4hglJZNjU2JNQaV2", - "NOjCgIwVC1Hh5ZWBZ6Jn/tXR+dOqJr0mNp9VbHxr2Oe2ZoQTKM2scCWTVDFioppH0o3oqtYqAUtxdK34", - "hzOECYyhJ21lmnAcwz6n3vjIc+J0HB7viDD7DOMjTRRICXl23leZ93/VZBYxX0a36RGkf/080UkjGPlg", - "+UMFAvElKYZfbFxZjjtedn1K89cnJzXrLcBtWrXJMKt0tz/CCpZ0W/gkdDGVeUz0VbCV3iNf60pPRy3Y", - "UDUDziAmt7FB87wdXTJPSRh4zHVaXJOxQ8LtOPWYjsskQP9Q3ciDAUH3CMapbi3UQREtyz281SDzKfTD", - "YCYhrpWyW3Qwt3s6qXQaH7tz6CU+VCht3SCRLQd5dDuEB7PY6wlN4kKywb8q6PE295LEgh3pH+PzD4OL", - "W/qjThlMZ96u4+2eutCWV5/50e7CXbYxiW3Ow3aUBOfqs0rj51kOwK7PUgUAmyWOrRT3L6UOL+mKnBFF", - "pRdymXbfJv7DBfQhge9YUNOKTrVpTE7qU/sAlw67XDoRQDyBDw+bcqbLfPaWB7g8fcOannLnzzP+r7Mm", - "iVzku4NQKvRXp4Z0w0f8UnchW5EaNzDYc8MtNh6f9+neN5N8Jephjy2FVhpten3leMiHYrrxAgXin6c2", - "tvxqDJmUZI999za8kiIRN8xSpl+KXeIyZUHdqixmVXPos6zp42s3Qu6lzGsNQL5lyVwopZgsGutu5rrK", - "X16SN12Zkbt5Fpt1qUpBX1MWVBcpgWm+OhNnbpNn0hRAW+V7LYpW4cw9MG1rLgfPq0nlVUKMyqOYzN+q", - "xlTtbzGGCxDNwxiO/ZBs2PadsyvrXX65ORP7IX8CEz3sXZRWtENjVZHSxIISGDlxIhdWb2pQ3TrrF4p8", - "X/o726+0dNGosFBbg17gzQwtXdXWXrCrU6pRfd3KfjhzEATQN4EpPjvI0z/9YTq488RH1z+q8BGujHZ0", - "OQWzp684yVoWMLAwrZ5+W2PptLt53WzwdRa9F7Y7O+uaRESK7jxddBUy1J4vBEYmcad3zp8j34th3r+4", - "VudF+CKJWT5gXbI8kU1TSByEWeKeqa+6Uyhh11txzuf3QNxsVfFqiaL0p4QfyucYG88zU+7qDj3Asqd+", - "SZgMsb/QOe7oAE7vv5OTk1eMlEkuSlLJ1rSpIBbDks3kraA1R+vS6V5QJ/d5qqDrLQSt9MkgCnMemcpG", - "bCi0hXHYF9N7TS1R5rrj8zAJiB5c8z1ulYf3rE8Fhoq2+VxsjkVoh4hESttvXg6ECTGBuKKIYI5h/Xth", - "e7FD5sZDhXiXip1ZQ4O0jZKjbU3ixELWNFlx2qVixdx5YHXzbUqB6coqw4EE6vqxO0eP8CDlUvNngb0S", - "MSG9Jeo7VXB9DEm8rJCiW+NH5Wq2G5aouAUpSJB41N+oTfS+D0aLPANqnfJEG0PqEddMBebXaE/foco/", - "P0550GI9wo+H9WBBDo9Qvk7a9h7LPlZ09w7FmIwhDJrR3iVo2qth4Ca/QuUALMycYlZBkxpJxfe3gpj3", - "JWtGjkxrCTkT6dIuNhpwL4C7q+u7L9ejj4NRp5v9OOpPBneXw0/DSeYlMLx6fzcZfhpc3F3fMtvceDx8", - "f8X9CCb90YT91T//eHX95XJw8Z67HwyvhuMPeU+E0WAy+hf3VFCdEujQ17eTu9Hg3Wgg+owGyiTq3OPL", - "a9ryctAfp2MOBxd3b/91dztmS5H5l+9Gt1d3PJ3zx8G/7lTfCEMTAajWRKjjGAWpSmidWOBoOBme9y+r", - "Rqty6hB/3XE0fBpcFRDfwOlD/M1bV8USTwB+0OcLzlJ3VOYoEv0TzEbJp+Zo0lFnPpZtKu/HNpN0qkYX", - "EGikf5pR2T4DVyELs+aCEPqeeNCxk4psHzafmjkkwLfqrEVdmr+qWAMJxiKH5MCQ6TO1/oQOay1NaAvW", - "C+stQCAA/pIgF19H5Doh1TYlMeAcYCeMCPQcYZpIB9HPsW5uya3XbzBlZ1w7vWOWX6RhQs7aKhEMrmz0", - "r0ZSKuSP3W3i2C1lxjHnj9WueQ/UDP1e6PLszsIeJ9rOiD0GPudXhYKZqHGAdyckeMrGwbcI0V1miSIY", - "MNXj8158Guw8sUIuLOeFA2LogCiKQ+DOUTDjFV0Ygqvml/lvOZGwYJ0VoeBLlmHVZXhYdE8lLhSb4juA", - "/CSGFqAwV2kVkFw5BpZdTD+nDzBfqvnpM4sDBIHYWfb8WUzoXR3xA75JInvHrG3igNaG9jn3sokDiAxX", - "E1S12ecvsyTQAmyWC4P8QST1RD90gc+Cwx6hH0bsM4s59hK3UDtRUe+UHNXbS079nNYDqnwIltWgRCXA", - "XVZIWi0Ddt27oGBR06um/GzGGm9R9a7JRsgVkjCe4jVHkUzdne2Vmo7TSI2cdvbmcBKk3OxM4ntahv/F", - "CMo+8ytlvbrWtxjGvMdNMvWRW0UKbLyKJO4qzHuz6WL/Vtn0kdgnKUWvv1wxi0H/4tPwqtPtfBp8ejsY", - "VcjO6jwE9ZezJnexKkzk4FCMZatejYvjFeOxUgRIyi/Wu0oNL4PR3fjyetLpdgafuc1i0h9/vBvdXjGb", - "yPWVEnvCMhKdX38aXr2/+zJ4++H6+mMF7nNalE6RBPGiIrKffRf+6loBzXMQkNB5AjHLPllSr3hvfaR8", - "s6QH+nwHm0lhwMc2L1EP/3qZDVOaqGfflILsEhjUbVjzvAULSGAssxfIc5SP5fyCjuCRc+p4YNl1Tp0n", - "CB/ofxdhQOa/ruijk6JHm83ALHYlom5CH7ma7MJc46+6BKeFNnlTjdLQQOzm2a/OwVUAZ16dsIDaClSj", - "QFKKh0h59Pmk0+18PtWLEu4TuoOAQ2MMK3d2blIfq6IMxXM64FgTlGGuO7SmH3u1CzsH6GesBaSuvCal", - "wEbK8Bg1NxUQ0f/lATGz2kFbiltD00samrZoANpKJcgGhvyV7fAGLvzCfJ7M2RjwDUiwLvGZyibcccpB", - "2IlYawcEnuOCIAiJA1ilZyeAWbXW0oGlgw7r7uO19ijgeTHEWLVL5bRoaegom6fohw8Az3XHzRzguTrk", - "/8KF6cQBxBXRm6UfBs44iaIwJs75nNVX10/4GcboHtWhl1nXqAx6FM3pryjOw6DnhDnANwDjpzC2nQM4", - "kejgYEgM/LWNlywP4cgHyxwjyP1rbMjKY/ergcDO5yCYQYkgIxME8MmMRMa78CnDmtSo9bCvoHfIkdm6", - "o0pAUiAq8bceDKWEzeJLN4cnE8ovwxkKVi+GuBp/r1Ubce8wLtcY1eFaJvU6KHTbnZAGwbCHuyUeuq03", - "TVWr8RxF+FCNrCWj8w5P822cMnwy3bZ9Pj0fXF7AaTLbdGnmrtBHMVokPiAQZ5k22GuZGya+50wheyDl", - "2gcIRNGzMHZATmPWhfTka76X0XU+uHSyNux+8Aj8hFK/1h3bJzC+AUs/BAYOFNlAIt6mvD4gP1HtwwkD", - "+kMMH1GY4J5wLxZjdKqSB5UnZp/K85FSeKjIxVRtusnV3Bd2nBrKqAxkN3AB/SQzkjlIVrdmG8CK2PNy", - "X5qdyNzXdTFoOPHTWKzCDmejd+mErIYVxveJr1UE7WJEyliQ4SIlB3NjsIRxDEOcMv2WW2K6Lla7k1sF", - "mZfkeFxZ9+Dz6TmLhJgA/GA+SL8RGAfAF/kCjOYq0cwZXmBJii4InBjei8s34go5wA+Uf3OEqXZW7Vwb", - "zkVilxTm8ynFh0z/8qzfMBlBQptiXfYNbHqt4OhiaEiXjTzMhd4TjGFWHm9rqHjmi2Ayhy+0YvurpajC", - "X/J2UJRhqgZTIT6lcDQNUyiOUFXYXvt+wsc7cm7pLZ5OgpMp5o5aFOUeU3xEK+wAokojuzxklflj1836", - "ZchAySOxGEJyR55B0LAtF0H8yp6HAby+77z5q1bYafq/BRi5/YTMO8/dVfr3b4a8cucqnT986p93nr8a", - "FycGZ0ZXf50lQgZgQfOhi66VJmIoDonAE+s6WZqomEUth/cObQUDglxBhSGzxEgGETH9itDv3wzvPg7+", - "pRH2xaTKcnoOiYZazChlyNDn0P0Il4PGWpe6JK7ePcDlkTNh3lLYYUY3EvL6RjDfyrmPw4WKCylEjtZI", - "wZxiVV91Zb0FsiHKixOqIyuMEcYxdEkmOkjoiPcnvfszs0FJNyorUhxnXYSig9xKzZY3SSW0ng7Lq+JG", - "7mLvtGR/OSeFQSOVDtUZ2F0dvdmLvExk7YFgUOXnluTC2/54eL5dqcAE8R5gk8KxXWSylW4Mlxdgdq4k", - "GSkm1dGkH6nXXdOK5GUV2AMz2yT8Gl76iarUW6mq4X0GgHrpmUIHBEvnz/H1VQ/DGAEf/Zs9PfKVHa2k", - "1FZMVjhGwthxAYGzMEb/Vgsoa4qEwaAqgRUmYBGJh9L03OVO6zCwd+Har4r/4jBlaaZNNaeV+6icjL3L", - "Zpe0dBRnuizMaMmpjJkmCjDasrP8OwpmQr5dNVFihN95CmoGJ7OAgCjykVvIPaTPIJzVfLdYlKZYfMW7", - "VqkIjmber5n42QObsRSEhnt1eWMN6WnrKTzVCwvbyMhRs4e1WessMsopxJ8yWjp1nARHW7rJmiu4mMnK", - "nFikrvhY2rBJfXxhlKwZmrdqMu4/VhXYeKsm48Z2hdhEsyYjM+Mp9OqBThvaj170tUrrvP0jq79ls6d7", - "omR1EdLClP67gUWryo6k4y6EL6DrgxgQkfXG7JQgOBthx8u6OL+QOIG/0gM8isNZDBYLdnX65R74GP66", - "aYcFo46jKGtS1WEKWxkfh2Gm24RCUbHtDayAdWNvUrBWJ+I3GQ4zslDZaC9O3Swle3323c+nxkLagBC4", - "iAxqr/ioSLBiHW1NyqmdVOb2ZZnraiQVS1K/XEHvYjop3XMdiZcOy0Rjg+nmFcIL6FijRng20j5wQmU1", - "7/RzVbHF/vi80+1cDMbnhuXyelvt02DTp0GOt+28DMZi7C0/DFLQTaae5rKTLkgvN5kPwKeK3GDsqiot", - "ePUbM0ibr5iLzDozXqU2DUmMIK5fPv1ywX12jFV8aBsrgx3P/sUMNs2SjslraLP8zLwJB06dWt2zDNf6", - "S126ZXshUjOir+MKSZBbTjDWMKOYHCuXSayYPUyfeqyYUWw8uJrcTdTFpGu44ydkKf3Z+WjQnxRKrn0c", - "3tzwj9e3lxQ7k7vx4OpCGVl/8igy1tLUbJ/rBqOAR242KTUAm9JRljK2VHwjIMhfpc5ZdbGOZuU4OBLM", - "THkTooDwKMXyDgha1MrWLDWbPvgbLeCqEXi8kSb3m9UyNAcxdxVrurMqaizvIUyFSgITPt3KbKtWLmgq", - "yendzqrSPRYgbIqRbGkacs/BpojMVEhkaf3Orz/dXA4mpWx+FUkK869dq5UxUS7/+YM6m2bd5y2m0QnD", - "aQn7G1Wo1PdCs4YpW7GBsP2DRc3TYs0tOHtPSnHyBLBw62iQD8DLa0x2btCaLVBGTLL6uprhxNfiUF0H", - "Bc4C+T7C0A0DD9vpuHWesIVZnF/SkH1AICb0t1/ry8dboZ8OL7vZ47/OD7kC5YLqhVe9/DGCAYjQ0VUY", - "XCW+D6Y+/HPM8makrXpoEYUxm1S44pcbR4BecTozRObJ9MgNF8dzQNw5JD0PPsq/j0GEjh9PjzGMH2F8", - "HAJ2Rn/rBWKszhtmaF0zDCxZjCPwFEDvvJIdFRs5b15mzKrc3eUB+beGFHRAe8JLEjA1PDU8WD9h8c6p", - "7KxVoLZw37MojaXh0C2Vxyoqqlm9AkNprPJBua7lYbWN3ODsFg8ClZf3YYBh3PzIQ6JbUwcK2/eLI7Ua", - "7Y4qEhMrI02aAUTYaOT15jwM7tFMm2+kurjsWpWfVyC+QsyRNTi58snlmUTku2aidQpnqfZxVWvqcuuN", - "jAbSnFfpOdPNLhAFdlWtP3lWyFfs4oag3KOTfgu+FhX67VqFqg2wm9KKSwHFKfACEvOFbIIW4ul+ixZY", - "D0ZkbtB76aecMoG4F9gTIDC+B76vH3Jniuja9c+2o0k0FJzcp6Ehsugpwjvao+tnU2g01vUN3BVbpeUH", - "UlpWc4ZTdYC1Clty4Vs4Yi9yB/Uqh+7XwhHykucopSaWCL3RcSqOvo2dpjtLgtftRDEKZa0Ujd+4+Goi", - "JX3dNlWfrXH7Fa3rI/5z43aVdH2fT3nypDYsdGV/M/0zgMhJVQq9PLgwuv2MgtvLIDYNGRgLaKvBSnYB", - "pbLDc/cQyGbrJWLawM2Xp/nNuNqai5Vbebhahosq8YFfVdZUArXLTBqhj6aQtP7NkHGFguR8aKGOCOYQ", - "eDC2O9152+ImimlrcaXM1JXr+FolovqKQMoHknbTQPOuKRpSGScXbFtUQq0yWtGlTukoDKE6NCaYqsgm", - "JMqvtQMVUJaOWpPbKh986s+ojjdfqHgbf+ifdrr0P2evf+d/vD4963Q7ny5eV2MvjWfVZJFVJrKPjU17", - "sQSmbuhZ1KvLjTCQnZg7zSwAJInhh7XpmA7tpONpBSaaBay4khtDw+UVs2+MDVNZhmaB1QTFAN4UUQqe", - "9CsuglZLIwMF72lY8eD/sHKF4wELiuF/3I4uq8ljL1znpE5j6RCT6sCmtFHuHPg+DKqcQhuE6FU6wMtn", - "98KR6MQSOEvtvnxAK1v7fnA1GDG5+X44+XD7lrn5jYY3A+ah1z//2Ol2LodXgz5zvvs8/D+mPc9usJsP", - "wq70Umnu2yHNlK1/R+vf8WP5d7QuGOWHkzUNsfv9kHAwduyGb+Q1j9Iai7d4p17L6s1aZybv7NqWf6LO", - "vRinr9GqnVI5DS8gkWUUCk6+SWDvlSBSMOA5qLfCqLHotP27MNbAIx+MWAYOm7Af1jBTRvLeBusHMnBw", - "8ObSydQ6cJRjuTs5nEh0S8jKW5tXB/Lb69VEz2yhnqU6ZRWwL/XqompHDZ5dDBjf1BPMF53Lh0SReTE7", - "Cv4rOCapUaL996Kkm1YlF1o/LzSx0RKLjUyeokiFVvlNYkMaatk3if1GhjZhEKHj6vY6hxKe5MtcfGBT", - "i8R2JgEqV0WGc3qKOcN7JwiJE8XhI/Kg13WAE4PACxey0xPyfWcKnRkMYCyvMSp1nW0N483R7O0nAa62", - "N7sm5RTOWmRTqWU2XezU8pIXP1bWl1wXI2OKS/sdMOwbexIFgZdVeIz5UKtd+ReQzEOv0WoF6J94z1S3", - "Pw89A9V+mExuZO5sN/RSCpaGHvt8A3eAJxxgM+cm/mqJ8GoSEqisOeczQxVvbZ14TEsBK9POp3TrMmPX", - "pNPt3FyP2X9uJ0xLMp2QPAgLV0VoYfEmxOswuSBwIhhTujqyr4gnzErstqvNukUpAaX5/ADGaBZAz8k6", - "MWvQ7e3wwhEkvftbng+m0MfV5UNZG0bmOZ8QLprtyIMLOTqODo0+wOQDBDGZQkCq7uu5XWPVYFkdB+DM", - "Ze/8Tfns5Oysd3rWO301OX395uT3N7/9cfTHH3+8ev1H7+T1m5MT+zQpgDMYPbIHmICpzwxgewjp9k9n", - "86kcQxemVUmxKTsLbcOjP3hVujBehaRG+bk0VBWLaj5ZNU9cm0kJO1kvJwzUXWwAWXFeLXRJQLdwGNyH", - "dtwzUjrQo8kPSXZBXqVeNR92nI3DXLdV5NBvDngEyAdT5COyZMdzrhBuRuS/UIjuWP7b3n8nJyevoPNd", - "dvZhV5RUfv5Vn6fUD01nE4YLEM3DGDq0kRBDKxLNWI41ZvPpQvmti2RkU6fZb84nw8+8IHf6503/dmwI", - "NLaJbuF7lEa28LPSmBFMnN78PCkAWW/A471v6/Th29GlZvim6jFrr1VtlKOidLJXZu6VuZ5o1007C1UU", - "2ebFtWsmr05UWoGHl3+JNV4EUiBHeVFWqLANglkinrGshdz44iPmxy7vrBSoLqfX0atqQr4OvpEYaBtg", - "78E8bGlxDCJVIb2+7LMMAzf/mnxgjyKTf90Mxuej4Q3Ll3L79l96405R6JZoqlboAi4I6dCU0gq6rxS4", - "dfEYaUMnCXLiPDe4pkY/K11uqg2LFslCmaTJ0Lri5xWcUTSqjQeX7z5cj3mqh0/9qz5PIfNl8PbD9fVH", - "416w47lsAlbXpo9jSn+xcJPuNigLyxURWRhWX07073BqOKLoFx1AVpz+ZzjVHYk70SiNmJNFBDVqNpit", - "vtbUNgu0F7vq5znhZpjd7SpXIN63mklc5SlNIrPSZq45YdPQDQMPiScWfstzNalOZpAo31npeY33RyCT", - "nvAcfDNIsPBcTbs6M9o31RoUu70WYYz1xyQGBM5qs4MrEF7m+vGy1uaLSFlapRCTfOHsYn7hV2f14ktO", - "XVxNV4vVqi0aXujSIKYADi+0OJS9P6IgZ0h5d3t1PhmyA+vidtR/e0mV1Iv++0oBSQeRmkgjCmaza9hL", - "fterN2vFYO5YM9Jf754r9tOYO4oxyUdYFU5JQgJ8HcWmPPYAlwafIjk8JUu7iE15OwcOjqCL7pGbTeL8", - "EgGMoec8IiD81X/Vc4UREQ0czvTXWxInUDN+3fut6rmVGmBOT05OjJ5Y2mHyvlMN3aAaLejvcCrFmO05", - "bij8sHZ0Mz8Rd22k5HMLW8/LgJBzJtqkY5Dq86H1DjKXGnm7bDD4ROlVdtdpqJIYHX7WyR2eDaS68ihg", - "f60WJntyV1acfuwPhVESrJFXuTzKOwT93Lmvpu3IaDknxRTJWDPJWDoztbK7ld2t7H4p2W2Y4wcU7RXe", - "kCuIZjbakMCF2b/ScF+p72ystjdmqdCqE+6u6XGWZVvbeBK1DQxokOnFlLzF3BRiUd0SIpVR66inlCn2", - "ZnB1wRPEZqliNVmA8zlj0/Syb/vnH6/fvas9Jdm0K92b8wLFTIyTvDgp+tuEwY0i+Uuw0gZjdw69xK8I", - "ijJ0Xvs4+lLMk2IpYGo2G/Mq6kYvpFx6li2yY1U9Mly7CKORgGVcbkJHcqhz3rFOCy00L82fMYQ2uXRV", - "Hm/JdNqPgrm03ySPNs8OXrXYCZjp0OtzlXF9k3+w4eQqwqzLIayiHyEUzmN6kbnXywUtS3O+vEMGbqyb", - "kDnfa2dkcuROPN5uelqsX2FzzaCAN43khWnIxSoDp/jZrHLP1S09+jIN7E68QjRHM08xY5Snm3zZqgJD", - "0WaLLJt7wrDZEPXVgzm93IPEJzeVWZZEI2O2JatHAnGL/BPzg3dhKIH15/j6yuFAl8N22AhaJxr5LPhC", - "j31h7HFvTAs0YKF2TNAChobiOJgg92FpcsWh3xwsnlXsXhIVedGAbZkO9nhaeCmzwrHSZ8yTP+lQ/phR", - "tjmJq80Cn5T3bNt3i8bJcq2vgXJZkjByA32t53RGVpt8G2pCn3uxJ7tCOHeoyB6FClWFYwi5v4qxpMgC", - "fKtp8dRM2TfVFeGRHwmVv0x+cginEMQwlvlMGEbZscJ+zjZlTkjErj1h+ICgbI7orvKf5Nv5m44IYc76", - "itQ2tHeCSbiwnOyZSXzuFqWJHuCzOP2bISt3RZhNLP9rSoid06OToxNGxzyIu/Om8+ro9OhExGMzTLCY", - "a1+UiZ3pAmTey+d52iqAGDupPYZuOpDFTTqX4vt7hgYZ0MBmOTs5KQ/8AQKfzBmKXvPvbhgQkVRD1JOm", - "TY//xpyvcHoA1vDxII5DKoWfS/6pVyFJ15Ejjs6bv752O1jWcKGrzhpKn5K/BMzuHLoPna+0P8NfDIG3", - "rEcgbYaqMDiSDfYdhWzBDgkd4LowIg6Jwf09cmsxmmKgFqWPp8fApyIlmPXgAiC/xx6S8fF39rP62zPH", - "iw+J5vZ0wX7HDkgzfdHuDuvO36ZLu9CnLQa0AXO14CMwnonBAhKmD/xV4eRTmsERec47b3gehFRolJbS", - "UYUafx/Idmy9mrxfS/T0m8aTMHFdiPF94vtLh6PUy6VJKyHvudv5bVeU13cWwKdYgJ7DMmh5MuyIg/Fq", - "42DooHgXxlPkeZDfPjL65nRSRWaS4iesCT2svvVioXKwD7xvp6shjK/s2ktcTZZ0ft1ah8T5CD8GiTN6", - "eBtyebwRYuDY4ZtWQFwat1Ymk0pskdBJJM7z2HjWi/2NLES7BB3sOTHAAW3FgKUY4NSyPTGgHpAR6pHw", - "AQb0VJR/s9MwCnUpDUbwMXyADghYskbWWnhrpTMWxESEJrSVNOjQ7jZSIh3eIBMkrHt13MVseYLOGXQ/", - "NlHjJlQtSIdu7ETsnCTj7LcqSk63PEfBrh8m3rF6Qzdr0KVMcfLawwZxUIAJCFxYIuJz+lm6l5gV6+3j", - "lgHiJEEWcLEvBFajtXMEq+/1Yus/KS9s33pyiF4YcWcXcaIp+83N4cff2X+fq/abSinW6qi0ocwqzjey", - "VhLxJNEm5YSncNylENrcZovUSjWHN6+h8ijEGscG27FWtuVIXMFMRt4cxRVSjdPPVzOFH9eJNbYtqVSr", - "ofmLVID97HR/wUi4pf39ov0FXPkMN57euzu4Rca1JjSVHokHcpBv4ginYxwzOz3fJWzc8UuE6QXId3Kt", - "TRtMWw/zDbe223QusePKlA03X2bAya1unwgh3Xq2EYVNKO9/bpPDAJGQSvPj75zjn4+jOJxC8+VSvn06", - "IFd5gtl1eeWKXC4EM8OnU9+EmIyS4IbNa2+bMh16qeTa8alXQVDwG3QTaVth+D3a6alwFRJWgSCM0b95", - "lnqR04gHX/MozZKZkwDkQ8/hdnuHbY/zTsjzYbat+oMjR2bYB+7D8Xf2HwsrvjOmDZUCK3nKYV9Fcih7", - "o31uTCPxMBD30jqfx8k+qTanuwHjNshImE/8ejcT85xjLHUj8P3wiU6vexEoUq0Uvez3KhWLE12eYwJ8", - "/B0H2Ipbrsaq1C/zS4AbsEl+MDOjiJN779ikgIyWUfaQUUoEm7LK1biSUQKsYROpuCjWJr3qQueVV+IS", - "izR+G3sx/aNrNgTwwkwrWQIUGM5ev84BcboJHSiKQ/oP6LVn2B6xpukSyeo3OCCKJLWXjzXepsCPBEx9", - "eOyBGT5OU78bL42Y3RpZO4fMAXGm0A+DmZpVIE0zDmblK+Xn0wvAys1ORAn1enOZTPCdJWjhKbcZy/yT", - "wHiZ8YwHZnfIqz7mthUhYiV3CvC+1MXHmno3VgP/AszORcyXPvtYhRyiU8rXPzbrz20l7HZe70r40Vso", - "WkQ+XMCAlHQDZryQdJA+nQP8oJUwrOHxd/qfmuclXuliuuR8UxQgdAJLUzsbx3joU0B3fOQDQuAiIiIv", - "i0EoiEYdFZZSLNQ27fiFmh6NTG8Mqz87f/7G7z7bn3Wi1t+nmsJ9mPAkTXsiIjJ+LokI852B2IiQYz+c", - "1ekqfjhzfBRAmflIwFGUKJfh7BIFvB7LIUoVkeWJhCIt73RpkCw8DaMWGhQQVlSyHHRpSJ4bE5EaO3Rm", - "kFBUMywbZsaIWx41M1ekbjDcm9KqAlZTJwFB/gam7jtU3vUI/EYcDEHszh02k1LjuWL9rINOpFevlVEw", - "fIT+L/hXOhEKXD/xoGl/aUvc0Wq71QJfsgAdwFa59WRyGwoYi1IxUx77fDdd3qWdclBaAVfKqWN1yFpt", - "zx4cuaoQaqAQiyjW9t08r5Wmkl85di7D2fqnDv3/XhY6bH5dVQq1GQ+etA7bD3D04AcUmZj//h7DjZw7", - "Wz3ptq9SZ3u9goNMe+1t1eqcjNNJmM2q2CQGbmX0oXM9gb7DmilwiGKUhov7hA2658Jum7LgmkB/HIFg", - "BUGQobsVBi8tDEr3XGVzNsGIrIXyVuZC/9iD02Rmfi0bPAI/YYXXnPPBpQO/RTHELLodzAAKcFbIUBTq", - "9gABRxpWPYf+BZvqUHx7Nh9a9vn0fHDJkFATScYwialOwgp3UzbVI3+nAWUq+DL/aY2ogYJ6PM0a2guG", - "+iQ+TWYlFlN4/nxwaWZ5K163uGDwl7i8DpCWFy/yc7NLxj4+lv9IFw2NaUm+qj3AJVYsFsZpabvmdh5G", - "BiL1RZ2F5zwMMPJgLEmMvfaGLkuB4zngnkBRB0bYz7Zp9auGZQrvwxjWArMpO+A7vjUkzEEDYlYUMnQR", - "k6BPiMzVR/FinXYNfFl+F8PObvm92n5duXoazgIQd46YD4ALYwJQkOXQqFpnmhYTrmSxZE9UhrSaVYtL", - "t0Sscrqkxx2KHe43oINYZM580W2ZLp0sVXUWrMGqGqYGAoNxs5zJW7sQTVUVOc0DXPZ4abQIoBg7v3iQ", - "CT7KfUsHOP/z5n9+LYqtSm8kOwszdsMIWslD3tJ2Xaz1evBu11hkbyhqTcF1puCUNyzjpxooaMfsGLbU", - "0vjZbqWpfYTLQ1HWth5PKHHRlBEYultm0DGDI7THLTDE98fTXoMIcubkQ7De0adJMPkeO/iaYJKYOlDm", - "FPvTHlAbifHFTeJ7U8qx4kyu49gcU6Jl7RnFVdLWnLCv5oRSGXQLBbr29lk5RemKyC7jfM6j9Wu+NLsr", - "4GSKIXFcEHiIJXySdL3R20PVip1bDD3GRhwWQq/HZXgAkTZX5kRjqF6z04uHwtoNBLsUMa1kz2tbEi+Z", - "bOf4rdK1uoa3nXNW88sBTgCfxMBG0czb/tyPNwwFHB02Dzjs/SYlZYdVWONW/V2+2QjyqGM9Uf9NAbh9", - "Dt7Vc/CV4QU45c+UN+153l6LYxcs/rdNnDGokxSNM+julxonuBWxBAmeXIv+spVi4jBvW5aiQQZVt2Lh", - "JcWCLet3FcKkR39FTFSqwJsNJny2Q7aYpPz8k3PxLCTt4W60mKxwxhYZrTJfd/2xeeCZB3LHZprt+iUZ", - "bhtXAL5JK18BXiALuLV8kIm/W/lweKe8hbLPgkwWWdXICrVASEYZke/ESeCIntUJxLkHxSXChHtRyCKV", - "hyrTyiGJChpq/JMsAF07SrEemk05KBVts8z6G3jcW8c8fVpcEL2QpwuFm5ePZKT8v7Ca+8MAtCg3Sdvf", - "ydZ3rPVWiS3LRcLf+JirVFoCOwuEN2T94A1RMLvjxTR3BHlf40D00HsUPj0WjwSZJ9HdotKV6GWN2FSw", - "jZJASrTmCQxUKdomG9mfTAJsbxbpQWUXY2F/4kYhCojlubtAQUIgvY7Lv2IIHrzwKUiP4gbH8HtIbujk", - "h34IswNP+gYrYTPCYN3pduA3QLe486ZzdnJ22juh/5ucnLxh//u/Brkjuvfv+U1kEwckgzT1HFZBDSl8", - "awB7jwKE59B7ywZvDu72ZWOO1FaQjoxPWvm4p/Ixvzsbl5L42GU1+c1RaLxmf5oYSifveJOf+4GSoYCp", - "KjWVyniyvdBxJdJ2GkXGJvWhxxP21b5MyuZttrY2bL0kowqSYeOSKYaRD5ZVVdbo90rJxJv81JKJo6CJ", - "ZIol0nYpmTiYtoIpFq1budTKpZJcKsiFDcolkYPXxvtW1jmo874VZRRa99t9dr/l5OLQYe3i11j7K9p8", - "lWBIQRPjdBRbe6skOmtARYcKSKsneXEPV5V9Gri4pozcvsXnfVxTxGRyU6B4bS9XUzWZdBNbP1fh5yrw", - "0eSVWzLlC3m6Shpp4uq6j1UIfm5f13KJAQveb6A2MXdX8Q87f9damXHgHq90cvn2KFm43vc1w4oZ2N3a", - "oW35X/qztry/F64utezdVcmtxqVV0q/waRXqoYFvD9mttaAA/2g8Kr1VWx41uKvWHJMwoKdgLwYE9tgN", - "lG6u2HtLLqvzZ609Fg/co3W7HLY979QfV3GXLqqtYNgjxV0jD1Y/2fU3+JsQs/weKHDDBQpmKb0uIMZg", - "VnHCj6AL0WMrg5rIoCDx/RLlB0snAks/BJ6DAgcES0esttsh8Bs5jnyACpRWnHInMsQiM2kOT/fAx7BV", - "LgwFADnjadhtVQ63uacLn+FenAR1bxz5rIG1rxxZlsD2pWP/85ZikcnR6q1jZ1kfmR8+iH0EMct1Da3A", - "22JQgA9IE1A2Vrpobxy/LXPVHEi0AgUijaOzybAD4y27+H+ZQzLnAkCUinIu+u8xPb3CwF+qv6cFPHUC", - "KfCXd7JBraIyDUMfgsAipiNXzdUCZy8U3qGpOWuM87DI7Pti8R7OvQ9m7Kh9EnQRxswBQyWD9H4JAs8J", - "E0L/FOojpvojbSB1wSPnAt6DxOf57v+H0sP/OOjeSQIM2TGuW76Y6U4O2qkkoZ0Vtmz6Atw6De1bAZyc", - "RqkquvL3Ef19zZcoVcM99hCOfLDsMXeJGn1XtKXDCveK8L5CCa7WgS/4YMzt4qD1YUW04vQdK4cUES8p", - "0CdQZ1YEFFn6InW/t2yC15JAK7pa0dVUdEk+6VE+qZZcOR5l2oM+4X+W3q5Ccg3EYEPvcAVXe89t77k/", - "yT13Z8dZJhfa0+xHOs1yp8dOTjZxvTaH/Ux4A+lVmr+wVxxdrXvpqUCdgpSap+ocKZBQ+G/u+o1a0Zoh", - "AcjHzfxMVQpp35uKbp8FBtoAg+f5mfl8Kr/UlJLIkxwIPOZMlp7/JEyvkqJY0n93PEYU/91xIsODdEY/", - "lm5nORi4bXPGehpegZXlHWwuwxW4rD3F9/gUL4a/WTJ0t0TQK7D4sSgZV8XphGf5IgkzHOX5/qiWi8ey", - "Jt2KvKxOr6jrPyZrq9fPlqX31MnrPEx8j8fT0oukTnPZo9wkOa5KC0S+iKxhyZ4sSuyysFwe5M4t9fZX", - "B8pArI6PtdHr56lIk4lVrQHkx5WoK1V1bIVqqycVZRdBCxTM6rUl0a6x9HoPyURMcbB3H60M8mBE5jxj", - "Cc9q5rhz5HsxNLlusA4Npd/2BQnfnFaSHLwkqeLPTYsXGAmZIv98PgaxO0ePsE4LEq0EmLS7VoSMCYyE", - "u25fDmwhPuR4RuuphLd13V1dI9umTBL7LvbcSirlk0q2dUF3n48p5bpCTqaykMqxv8L8Uj7R7aeyqUo0", - "pSxcL5Ns7mWidL+9PBrIGqutNPpJpJH9XauVRYcjixTG374k8sNZnaeUH84cHwUl3ahsjr4MZ5cogLbW", - "oFYMvWw8kw8foW/lMsRb5mauYgZJB7TXOwR9z5hBDtKD12GzKXBUFDNhHZoCMua9tKEkgAUKhLFXtX72", - "+e2Sr6Xh5NdqXwMe+PQeiqErot0roLhQmq0CSdZ/u4eUKg3aAvrrpqBLpbByFlyGs+bHgHA0qkhtzjwg", - "sPAkMjjuT9jP56rjy6Ydc/jgfKK6JL3cNellXHE4hI2cbwRSf2waX8HrJiW2NDut8KcpErmOolPXuVqT", - "MXeNES/slQTeNCFTGtghZjA++ezGW+5lKV6mTGqpfbe3DU6MXgj5RQN+4ydwqZCGLbPlMppW52AK+Gwo", - "mFXz1eFkYtqS1ylHQJPDLYopIgnicRkvULizPefWP+cEn6zAehXn3THwKWEEsx5cAOT3ZnGYRJUPp1S5", - "k7dAQV5sDIcN4IgBiqzbp00GtMV72uBQIp22fxLqENOw5JRxE1reyb8mVlBro3PM+upTnquOMX76kAr1", - "5lbAjd1ZV0J5o6vd6XbZe4UTUENDLV9r735abtvsKXmMISF1rkWY7Z7s4sgu1dkMFHJBwWws+hxIUt8d", - "HZMKYtY4I9U9aVlJc63ToGljfBShHgkfYE0yPKd/M3R4u2qu6UdoQpu1+iQ+Zn5FN0OGD2yROlLHJ9I/", - "qrWhF5VHSpEctQozpD+uU8olyKjdjthbHZEhQNK6ohZu04RRnLTlrw2HzWbM1JDBqg4cC28pXl0u5zJl", - "SruaOc206Vb32j3hAS6tnBNou+bpZxgZfIRLm7wmGUyp+/LwAtvmw+SyojGA0iV6eLEiiFkM2hqpfGwg", - "HCUBj6MUhq8XcfVg+/kyjh5s6j1w81DhUJ08KoglyyAEl84j8BOozyMEv4FF5EMqsh/g8vQNa3ra6dJ/", - "nfF/nVHxXp1v6NNm0w1ly+CJS9OMQ9V0zhoPDz/T0EqRdq13TWD2uVSUFobc9U3IbFyDDtJeARgCGC5q", - "zMIiMfGLuPdwSmhi84W8x8/uXX32n7uZdST4U6in8JsLoQcN5Rz53jTg8/qLyfE08R/M7nRvE1/UMYI4", - "kwm4UijQPj+xYKDLbygc8EtKB9xcPLTRF3smHxibqkICb1hKuCBwoV/hdsu+c0OGkjg7p+KapAZ3K+Ej", - "/MwKBUOAvUIhLgwxjHyw3LjYyBy26L+essvykCcn3lYRD/lDOP0buhaaC0MazHKUtEJqb4XUiFHqduQT", - "M6NZ2li5bc7CzvoRLttnvczYuNJtnSG7vbHrbuyOsP1ukg/EaWA8pzkP4mZH80geMT/r0cwRsC9H82bM", - "ahy4Vqv/SQ/M7+y/vSdE5j35iVm3a8OPAAH88AwqDYQXgID3kHxBZD6RbF8rPyT76MVHCeRdv13+8Kc8", - "3bRV0jEwqmhP+bwvm4IZa97taoi8mp9R8IgIbBowIXvpnUCH7Gur+0rfTwUfK3l9Smy3vp66cIiMFrcU", - "A8EnqKT19jlLiXrgKLELduC4fdEIBw7uKoENgjB+9tjes7Mdab2A2L1zFflWJxdgAKY+7MWAwB4bk7KH", - "4LVV9GIhheQPPf7vZy5ifEhgWdhcsN9xakayETS8z8F67+W5vhq2XoqOQz/5a2ULp5B9li05NuNEmJGr", - "SRfN72NtBH0zTjicKPpD4YTtBvqvphW8WKi/Jedy+A6Gc0UIfmPOrTr5FnAxZczX6AYpe+lZ/BP72t4g", - "JTUq+FjpBimx3d4gdTfIjBY3EyQoxjv+zv+wUAIdIIBw7uNwURdky6nhx1AFxbJNsPHPO+Xd37bCu6vo", - "gD8H1+5RrtorQ2ralElzG9NAXnQlIVukkSpNYhYBP4YOvBciYLvKL98uO+VXoGNPUl5ZSi+NHiz2rRVe", - "Lyy8jHJlBeFVpfVEcbiAZA4T3FtQHdStL1+UdXFEl9QHry4z5U3a9ZOY7Ie4KBD4jRxHPkAFqiiO1OQO", - "UMZyy5QvzZSUAzT7sqkbyD8JTKA1G7LWjTnwv2ivA2K+w45sPqRg1e3bQ3K0t1oGC+cRxhiFQSsT90km", - "prtTloiSc1aVidlTn42rd5w+Ntb5eo8AgZe0YZtXY5+r024iB0MtJreZaSGlsz3ItlCEZVdlNfK81iCY", - "QGHn1s+wYAVXcZOJW+Ztccl/XVXiih69KPSRu6xPOSk7OLyDTcJJ6Qp9w3q06SaPdWhZ7dGosBvt49HO", - "s7ZiH7gP1Ykmx7SJ8wSn8zB8KD+nss9f+Nf2OZXnmFRx0uT2UED1PrHDjioe3wYgIfMwRv+GHp/49W4m", - "/gTJPPRYRQ/g++GTvtoy3yCmB3IWUM8z9nEtRjzGBMTEyI5j+pWfY9f9hMwddlkpMuQtls82DKBrilDW", - "8xA589XJmQYPKvcwlIljJYeVOQSe8BrxQ04wNRZPtuHQTWJElgw/bhg+IEgHZUWRvqr0wFCan1ESAt2B", - "lemgLu/v+GpcJMCCQA5wK4eFHL4aD1VUNZDERSy3snjvZHGZEVJJfDVeI91wYWAdg7XRGAwBef6qzDK8", - "OZrNT2odVVHc1Zah94ihjZxnydGVJ6qo09nbxZOVKB1+aC9X2zcX6BDTzGaQ1rPO7Uz7qLIPjyrp3mz6", - "mVlXVb2SdbMC6s50yRmqcHpzQjwQO153Xyu7b1NiiC1aUT60EmFnpVBVWnwCvB5qnYhQD3X6E93oVats", - "V8uJ2pyAfULgIhLJLVlbRXyYBMehJQNsJUiVSzzCzFdaiBBOBP7+XRBe+BGvjlF2xdAxpB0rcoexJIu2", - "PMyatyy8j9nM4iQQW1Xj0Y6CKGH+EPxxV7fc573QVNpcZhXyhW34SwiUbE2VtgDeTDgL1AmX95CM+bCt", - "aHk57aBZll6DpUEM114o9vlCIXdpK1KDAPzQwwSQGoMhwA+sGpSwFNZYCScAP4zZoPYiYnjxI9oGU0Q0", - "4FAtrlse3QMzoIkNdpEeSXjN9J7C+KEqWUTmgG10aWq9mbJgEo6KLwypFCFVVT0pMtKAF97RkdvRPrft", - "2/u5Qv6rJzEUg5hY6Kd/J8/xD8fGjorxamb2GqUglFvbcu7+PZSrjLfSYcmoovohjZ6QXHhXe8lnZ8NP", - "f1hmmGhrXm8kQ7XUHvIxeqt7V0pEc0NQ81oUavVfTUkKpWRvW5hCKUyh4AXXGHRz9ZVfrkyFDm7rcvaK", - "rTdHMO0ldS/LV+T3qBwOXG1KaiJwvqv/rPNjyXFC7QksyPSQ3VoKrK8HTcXgAasJYrtWzSzQurmY4/rz", - "L0j1Mf3dPE2tzs/H7DGy9jGJP1lyhlaBPqrh6yEbvWXul2fuLIvJjVKEksO4zrtTHkdsu1uz9o7M2l9U", - "3Ac2+UOyTWqqMmxO4uA5iOCW9IgxG7uVNwejTPANazWKH0ijSGNXhM9QZWSoqNTOWNz30/dxrNE1qlif", - "BU5yV5aBLOzXyoCNA3gJMHGGFyxh/Rw6PpA7aEpTBDAZesY8Ra/OdHmKduBj26SgZ6ksX2sS2T/fmhVk", - "ib3jjZ0sxFYvE6ylnUbzUyZO8+A9SHzSeXPSzYmKXaRQS+d+vcrkY55Jbbp02AT6ScUncz6HXahd7WPP", - "5vWtTaZkTMesDQY6l3ENU0Dceemxp0pjOpxgoG15OSjvJBwZtm77Ipqk/FSy6ceeSLHUfE+VvlESDD2c", - "Sz27FoLL+XYbGoREBFL7elSTHo2TzS5ebvCxG4dBvUZCWzl/h9MMKBKj2azWfeI8DoOfWk05mPyu6cYi", - "j047gyRViY9q0nibLm5buOvSmZuCd1WnSmmnZBTfZDraoflUh5mhvCJn7nTp3Iu8vBtL3atKEWyfvne6", - "3F4GX0Up2HEO3xwy1tDQ22NXo6WXzrktqev00D3+Tv/Tk7/albkrH8TWDx+UcA686F26ehNYOYzuvuyd", - "ZX067Sa2+YGL9eL0aGr2VpEniK/P3arHxDWZ65Ddk/aYs7Z0dLbH5iEY9hsd1huRD3XlJdms6YzWwuHA", - "a03ul3zYVrVJVUBMuIHDytZHqYCXcLSx7dWpCmoxyFZVqJYDgi23IQrsVHl2HNg+6KmvjPVuSq3BbJ8N", - "ZuwRuYG1jLXfoalsH+14EYgp0gyuKwWweOMv6mPGjuDTpIjRwiacRLYLV18bn8USESQYWtVblG1XsW6N", - "WV9hZ7IB7gEFnhVUrGFjkD6iwKuH5uCNqQQtoAPuKaAl5+kngGUss7qEztnJ2WnvhP5vcnLyhv3v/xqN", - "1ax7n06gJ156rPYoFB3bauQU4im8D2O4TZDfshk2CXMFlu9RgPB8dZhl/53ieVNAbxTT23scKFvif9qn", - "gaLu2Fo4tuIuvZ03AeYhbZO/HzgCNHrQ5dlfTehvGQhxyBWoWzW8VcN3r4a3umWrW75ICBRes2I7E0Bt", - "ZZH6830L1dOzc56C6iU+PR5rrIZpy1Xsh2PZubUi7rMVcXv3opQADspzqlWmWmXqYJSpbBmZqN6IbTYF", - "yYrBUyutBuatxkiWJExrddisVmLQALarlxxPE/+hl3ki6iOK3ib+g3Bq25CiQkc8HP/ELfkhlHkqQ4tt", - "2NG0fmt2W0ekck3mxHMqicVpu1ZCSAnx1mqfty4puLtKjaTgjZxfYih7/7pBsXE4zlU7FRsyTWcDsSH2", - "aX/FhlxTjdgQ62jFhkFs1O7zNsXG9/TPXilnZG0EhB7khkLjwOMgNDgwVjPSonpvQyP0u9s6PBZjIwx4", - "aubxaKCNmiiJjTDgQVcoPiju2+aB3N71Dz2GYttypDqaIncd2JBkOfBAi70XLtuKvShJlwb1UTMyKud9", - "fNkrS62EVIM9fkrl5wCqv91WXZY2JSvtLlFpCs3nLHNLVRkrBzgBfDLnb7FP3yLioQ6n6FV9JpHqnJmV", - "oO1INHJsrxqWJipHGzd/p7KxWfCtWqvLDH8rGXcvGfeu0IkQdFVUvp3UWYoszjn16OWx1A2ERLbXcHWK", - "USuFdymF5Q6soJlWqHV7rpiqErhVTFvxaxK/QiGp04k3LnJ59byeGyYBqYmXYG1kLnJZ9hE8AuSDqQ+Z", - "9FXEjd6+8B4SXp0Pn7MZD1701qWMP/CSEbnNWtFMyUmFk0/7gmhwmM4habVCEnn2TzCM8bGbxDGs5mzM", - "bwe8oUO7lbj3FsP4PSTnYrAt0h2dqSGdMYjbAsQvX4AYukmMyJKJcTcMHxDsJ1R2/fWViqpC0qE8uUly", - "Z9uvIeMZIvNkeuwC358C98FIzufhIvIhgZymr+n8jvY8ohNxe9R7NvQ1xeW5HL5A4K9OzmreXl0xr1ee", - "dw6Bxw637x0/5JuR34eiWH8uIDOHO7nA/ByW6MMExGZRMKZfV0Mc69ocawye7eOMQdcQYWE48+F26I0N", - "/YPTG0ffhuktQ9wPR28oeEQEVtduwiyaSWrDvANTuq2ObzrChPUdirm2eIqrE1k5s/sIy43JL7DVF62P", - "VVaTp4C9jPImmhtijvaOgevCiJgtb332HacWNjFJidrUzed9OtuxJ/HB+USKIclgAKqgPr5yHf21HlMp", - "eXFsl/benr5iyKpbVFTSp9+b0Rfv09lWXXo6+Aboi6+8pa9K+uLYXoG+/HCGAjNZXYYz7KDAAexsPKpQ", - "MC7ZQFtyzqBHMB2/npB2d4/2w9kMeg4K2uvzC1+fu53fzs52te4oDikNMKPtICCILJ2e8wh85LHJ6KaI", - "JiiYOVCOZFZ4GWHrr/LdzrceDOhUvRgQ2GM2cKpD87caHTOHCanh5jAhduwcJi9vrBJMFu5Zoe7WSFWj", - "TTPqsbVPLeBiCmM8R1GDO5zSye4ex8/AT1k3kZRiqwSun7T5hU5FUXupW+VSp2KwniQjgPFTGFe4UqS5", - "2GkHR7avEqk3csztKUnncxDM0on2SVtyGWReiqhWnLdKUzOlqZrVOeXnmXFtfSqGMyqJ46prN2+BK1Wq", - "1FNqW3wvwdgnjpfIax8aW6bfzE1JUvlmLkvYB+7DVh6pxnTkPX6jqpGkDR+tHmGMBQhG9ye6BtFOukBh", - "GD9qtPRhcB++h+SzGHSjNYkVSLMMjadHJ0cnuhyQiufRX2nXrxblhicViy14W1YQ+xfoxJAkcZBDXuGm", - "Q8VsEgSUf9IpvvXkkL0w4imnyizwBKfzMHzoCUe04+/iB4vwd3rUidZlRzX+u31kuxjI7AiWTrRjPzDL", - "UHEJX3uwvbxxohierpKp0ftLtPhqxRzHAs82ZgrZVPjV13CMUNywbaLMveWbzfhPcui5+6RADcVMVcYV", - "ipW0DojATrpdLXvuEXsyq0xpi5ryaMqb7I/nGu9r3krrWM2cM614jjuZVvksa874w/FYbuw7Klbc2iNL", - "TsmlgC95QTH7IDO1ur7yYyUh26cd2Ata3lYUf+7cMJ0VAgOJRNnu4qAseU0Nym85zVBzcR1mK5wmxeAe", - "q0RgzWqwNrgX7WWETJMkWimAbYDeC2eOEMSqUMyK8THdOg3LnhMaqFw/Q6DYisFhLW+9NG+pUWjrMJaN", - "2mfPXc30wL1gsM3rgnlk2MbKi5ykOS7btXJoJRGK6mErD4wK4nrMWaMmWpXLo5uUr4uXMt5j+tJhPCkb", - "lMfbB37WlKjgBSY2UD949erBesBmcZhErO5HBoLcKCMorNNHuOzUpgHZspBYsxaXfFRqy3HtoTaxUv2v", - "RoJLpiYyOrfIrBpNkwWtlCNoLyXXRMMuR87wnlm3cUKpA3pdxlU+IBCTlKcQdu4hcefQM1WHygT/nitS", - "ggxWTDz0YumGFHgb5Rlqswu12YW2kF2okWgWsgFbvGrlTnIrsSx8aw7IBPMjyOUtSznpMLWeKtjKu71S", - "ATNSXFUFLDr+TSGIYZw6/nW1roDMk4zLgyT2O286neevz/8vAAD//yTV7zcJPwMA", + "oVWGNQYu1MNWfPCVLbP1qPCpKyngNw+oQmBdlT510l4S9h5IiJTHrGznmglV78nvHeHQdhexjTnrdgL4", + "Tf7rVbcTJAv2D9x5c3rC3mVyXJ3rrHOylv5yET8X0onPrAwdCizaiAT4rTzyK7uRs3VpfcNDAnzVrESb", + "MlurjzDh741ZRN+JjV1Fszn/lcCE3sRi5Go0pCBZ3NgZvRjdSNPXkWm9/2Vl5+JjIe4qzoxexgFHdgYu", + "PqIwcx11al2UMlBzs3RVhOh4dAQIZB6eZVRavaIwp1Tm2ah3OgWYjOA98g2uEixkQMQUqIOxeIKYdYTM", + "r28LgRdsos/AT6CtK2vMnR+ww2LVxCOM2PUnFHjhk37bN/HKU4PoR/M6pDTRrGMBPGi7CP5NPwX/xpZB", + "9xIFio9mhmYeVXUfxi70bH2xlJu7sl9yvSlUOUr7qtL1Hhw+GY9pFdT08xoqanGMkpLKsSmxpqBSOxp0", + "YUDGioWp8HLLwDPRM//q6PxxVZNgE5vRKjbCNex7WzPiCZRmVrySSasYcVHNI+lGdFVrl4ClOLpW/MMZ", + "wgTG0JO2Nk04j2GfU29+5DlxOg6Pl0SYfYbxkSaKpIQ8O++tLHqgajKLmDGj2/UI0r9+nuimEYx8sPyh", + "Aon4khTDMTauLMcdL7s+pfnrk5Oa9RbgNq3aZNhVutsfYQVLvC18ErqYyjwm+irYSu/Rr3XFp6MWbLCa", + "AWcQk9vYoHneji6ZpyUMPOZ6La7Z2CHhdpyCTMdlEqB/qG7kwYCgewTjVLcW6qCItuUe4mqQ+hT6YTCT", + "ENdK2S06qNs9vVQ6nY/dOfQSHyqUtm6QyZaDRLodwoNh7PWEJnEl2eBfFfR4m3uJYsGS9I/x+YfBxS39", + "UacMpjNv13F3T11wy6vP/HB34W7bmMQ256E7SoJz9Vmm8fMuB2DXZ6kCgM0Sx1aK+5dSh5d0Zc6IotKL", + "uUy7bxP/4QL6kMB3LChqRafcNKYn9cl9gEuHXS6dCCCeAIiHXTnTZT77ywNcnr5hTU+58+gZ/9dZk0Qw", + "8t1CKBX6q1NDuuEjfqm7kK1IjRsY7LnhFhuPz/t075tJvhL1sMeaQiuNNr2+cjzkQzHdeIEC8c9TG1t+", + "NYZMSrLHvnsbXkmRiBtmOdMvxS7xmbKgblUWtKo59Fna9PG5GyH3Uua2BiDfsmQwlFJMFo11N3Nd5S8v", + "yZuuzMjdPAvOulSloK8pC6qLlMA0X52JM7fJM2kKoa3yvRZFq3DmHpi2NZeD59Wk8iohSuVRTOZvVWOq", + "9tcYwwWI5mEMx35INmz7ztmV9S7D3JyJ/ZA/gYke9i5OK9qhsapIaWJJCYycOJELqzc1qG6h9QtFvi/9", + "pe1XWrpoVFiorUEv8GaGlq5qay/Y1SnVqL5yZT+eOQgC6JvAFJ8d5Omf/jAd3Hnio+sfVfgIV0Y7upyC", + "2dNXnGQtCxhYmFZPv62xdNrdvG42+DqL3gvbnZ11TSIiRXeeLroKGWrPFwIjk7jTO/fPke/FMO+fXKvz", + "InyRxCyfsC7ZnsjGKSQOwizxz9RX3SmUsO2tOPfzeyButqp4tURT+lPCD+VzjI3nmin3dYceYNlTvyRM", + "hthf6Bx3dACn99/JyckrRsokF2WpZHvaVBCMYclm8lbQmqN16bQvqJP7PFXQ9RaCXvpkEIU5j05lIzYU", + "GsM47IvpvaaWKHPd8XmYBEQPrvket8rDe9anAkNF23wutsciNEREMqXtNy8HwoSYQFxRRDDHsP69sL3Y", + "IXPjoUa8S8XOrKFB2kbZ0bYmcWIha5qsOO1SsWLuPLC6+TalwHRlleFEAnX92J2jR3iQcqn5s8BeiZiQ", + "3hL1nSq4PoYkXlZI0a3xo3I12w1LVNyCFCRIPOpv1CZ63wejRZ4BtU55oo0hdYlrpgLza7Sn71Dl3x+n", + "PGixHuHHw3qwIIlHKF8nbXuPZR8runuHYkzGEAbNaO8SNO3VMPCTX6FyABZmTjGroEmNxOL7W0HM+5J1", + "I0emtYSciXRpFxsNuBfA3dX13Zfr0cfBqNPNfhz1J4O7y+Gn4STzEhhevb+bDD8NLu6ub5ltbjwevr/i", + "fgST/mjC/uqff7y6/nI5uHjP3Q+GV8Pxh7wnwmgwGf2LeyqoTgl06Ovbyd1o8G40EH1GA2USde7x5TVt", + "eTnoj9Mxh4OLu7f/ursds6XI/M13o9urO54O+uPgX3eqb4ShiQBUayLUcYyCVCU0TyxwNJwMz/uXVaNV", + "OXWIv+44Gj4NrgqIb+D0If7mratikScAP+jzDWepPypzHIn+CWaj5FN7NOmoMx/LNpX3Y5tJOlWjCwg0", + "0j/NyGyfwauQxVlzQQh9Tzzo2ElFtg+bT+0cEuBbddaiLs1/VayhBGORg3JgyBSaWn9Ch7WWJrQF64X1", + "FiAQAH9JkIuvI3KdkGqbkhhwDrATRgR6jjBNpIPo51g3N+XW6z+YsjuunR4yy0/SMKFnbZUJBlc2+lcj", + "KRXyz+428eyWMuuY889q17wHaoZ+L3R5emdhjxNtZ8QeA5/zq0LBTNRIwLsTEjzl4+BbhOgus0QTDJjq", + "8XkvPg12nlghGJYzwwExdEAUxSFw5yiY8YowDMFV88v8uZxIWLDOilDwJcuw7DI8LLqnEheKTfEdQH4S", + "QwtQmKu0CkiunAPLTqaf0weYL9X89JnFAYJA7Cx7/iwmBK+O+AHfJJG9Y9Y2cUBrQ/uce9nEAUSGqwmq", + "2uzzl1kSaAE2y4VB/iCSeqIfusBnwWGP0A8j9pnFHHuJW6i9qKh3So7r7SW3fk7rCVU+BMtqUqKS4C4r", + "LK2WQbvuXVCwqOlVU342Y423qHrXZCPkClEYT/Gao0im/s72Sk3naaRGTjt7czgJUm52JvE9LcP/YgRl", + "nzmWsl5d61sMY97jJpn6yK0iBTZeRRJ4Fea92XSxf6ts+kjsk5Si11+umMWgf/FpeNXpdj4NPr0djCpk", + "Z3UegvrLWZO7WBUmcnAoxrJVr8bF8YrxWCkCJOUX62WlhpfB6G58eT3pdDuDz9xmMemPP96Nbq+YTeT6", + "Sok9YRmNzq8/Da/e330ZvP1wff2xAvc5LUqnSIJ4URHZz74Lf3WtgOY5CEjoPIGYZa8sqVe8tz5SvlnS", + "A32+g82kMOBjm5eoh3+9zIgpTdSzb0pBdgkM6jased6CBSQwltkL5DnKx3J+QUfwyDl1PLDsOqfOE4QP", + "9L+LMCDzX1f00UnRo81mYBa7ElE3oY9cTXZirvFXXYLTQp28qUZpaCB28+xX5+AqgDOvTlhAbQWqUSAp", + "xUekPPp80ul2Pp/qRQn3Cd1BwKExhpU7Ozepr1VRxuI5HXCsCcow1y1a04+92oWdA/Qz1hJSV16TUmAj", + "ZXyMmpsKiOj/8oCYWe2gLcWtoeklDU1bNABtpZJkA0P+ynZ4Axd+YT5P5mwM+AYkWJf4TGUT7jjlIOxE", + "rLUDAs9xQRCExAGsUrQTwKzaa+nA0kGHdffxWnsU8LwYYqzapXJatDR0lM1T9MMHgOe642YO8Fwd8n/h", + "wnTiAOKK6M3SDwNnnERRGBPnfM7qs+sn/AxjdI/q0Musa1QGPYrm9FcU52HQc8Ic4BuA8VMY284BnEh0", + "cDAkBv7axkuWh3Dkg2WOEeT+NTZk5bH71UBg53MQzKBEkJEJAvhkRiLjXfiUYU1q1HrYV9A75Mhs3VEl", + "ICkQlfhbD4ZSwmfxpZvDkwnll+EMBasXU1yNv9eqrbh3GJdrjOpwLZN6HRS67U5Ig2DYw90SD93Wm6aq", + "1XiOInyoRtaS0XmHp/k2Thk+mW7bPp+eDy4v4DSZbbq0c1fooxgtEh8QiLNMG+y1zA0T33OmkD2Qcu0D", + "BKJoWhg7IKcx60J68jXjy+g6H1w6WRt2P3gEfkKpX+uO7RMY34ClHwIDB4psIBFvU14fkJ+o9uGEAf0h", + "ho8oTHBPuBeLMTpVyYPKE7NP5flIKTxU5GKqNt3kavYLO04NZVQGshu4gH6SGckcJKtjsw1gRfB5uTDN", + "TmTu67oYNJz4aSxWYYez0bt0QlYDC+P7xNcqgnYxImUsyHCRkoO5MVjCOIYhTpl+yy0xXRer/cmtgsxL", + "cjyurJvw+fScRUJMAH4wH6TfCIwD4It8AUZzlWjmDC+wJEUXBE4M78XlG3GFHOAHyr85wlQ7q3auDeci", + "sUsK8/mU4kOmf3nWb5iMIKFNsS77Bja9VnB0MTSky0Ye5kLvCcYwK6+3NVQ880UwmcMXWrH91VJU4S95", + "OyjKMFWDqRCfUjiahikUV6gqjK99P+HjHTm39BZPJ8HJFHNHLYpyjyk+ohV2AFGlkV0essr8setm/TJk", + "oOSRWAwhuSPPIGjYlosgfmXPwwBe33fe/FUr7DT93wKM3H5C5p3n7ir9+zdDXvlzlc4fPvXPO89fjYsT", + "gzOjq7/OEiEDsKD50EXXShMxFIdE4Il1nSxNVMyilsN7h7aCAUGuoMKQWWIkg4iYfkXo92+Gdx8H/9II", + "+2JSZTk9h0RDLWaUMmToc+h+hMtBY61LXRJX7x7g8siZMG8p7DCjGwl5fSSYb+Xcx+FCxYUUIkdrpGBO", + "saqv2rLeAtkQ5cUJ1ZEVxgjjGLokEx0kdMT7k979mdmgpBuVFSmOsy5C0UFupWbLm6QSWk+H5VVxI3ex", + "d1ryv5yTwqCRSofqDOyujt7sRV4msvZAMKjyc0ty4W1/PDzfrlRggngPsEnh2C4y2Uo3hssLMDtXkowU", + "k+po0o/U665pRfOyCuyBmW0Sfg0v/URV7q1U1fA+A0C99EyhA4Kl8+f4+qqHYYyAj/7Nnh75yo5WUmor", + "JiscI2HsuIDAWRijf6sFmDVFxmBQlcAKE7CIxENpeu5yp3UY2LtwNfX73EQtGTNRiMOUpZk21axW7qNy", + "MvYum13S0lGc6bIwoyWnMmaaKMBoy9by7yiYCfl21USJEX7nKagZnMwCAqLIR24h95A+g3BWM95iUZpi", + "8xXvWqUiOJp5v2biZw9sxlIQGu7V5Y01pKetp/BULyxsIyNHzR7WZq2zyCinEH/KaOnUcRIcbekma67g", + "YiYrc2KRuuJjacMm9fWFUbJmaN6qybj/WFVg462ajBvbFWITzZqMzIyn0KsHOm1oP3rR1yqt8/aPrP6W", + "zZ7uiZLVRUgLU/rvBhatKjuSjrsQvoCuD2JARNYbs1OC4GyEHS/r4vxC4gT+Sg/wKA5nMVgs2NXpl3vg", + "Y/jrph0WjDqOoqxJVYcpbGV8HIaZbhMKRcW2N7AC1o29ScFanYjfZDjMyEJlo704dbOU7PXZdz+fGgtx", + "A0LgIjKoveKjIsGKdbg1Kad2Utnbl2Wyq5FULGn9cgXBi+mkdM91JF46LBONDaabVxgvoGONGuPZSPvA", + "CZXVwNPPVcUW++PzTrdzMRifG5bL6221T4NNnwY53rbzMhiLsbf8MEhBN5l6mstOuiC93GQ+AJ8qcoOx", + "q6q04NVvzCBtvmIuMuvMeJXaNCQxgrh++fTLBffZMVbxoW2sDHY8+xcz2DRLOiavoc3yM/MmHDh1anXP", + "MlzrL3Xplu2FSM2Ivo4rJEFuOcFYw4xicqxcJrFi9jB96rFiRrHx4GpyN1EXk67hjp+QpfRn56NBf1Io", + "ufZxeHPDP17fXlLsTO7Gg6sLZWT9yaPIWEtTs32uG4wCHrnZpNQAbEpHWcrYUvGNgCB/lTpn1cU6mpXj", + "4EgwM+VNiALCoxTLOyBoUStbs9Rs+uBvtICrRuDxRprcb1bL0BzE3FWs6c6qqLG8hzAVKglM+HQrs61a", + "uaCpJKd3O6tK91iAsClGsqVpyD0HmyIyUyGRpfU7v/50czmYlLL5VSQpzL92rVbGRLn85w/qbJp1n7eY", + "RicMpyXsb1ShUt8LzRqmbMUGwvYPFjVPizW34Ow9KcXJE8DCraNBPgAvrzHZuUFrtkAZMcnq62qGE1+L", + "Q3UdFDgL5PsIQzcMPGyn49Z5whZmcX5JQ/YBgZjQ336tLx9vhX46vOxmj/86P+QKlAuqF1718scIBiBC", + "R1dhcJX4Ppj68M8xy5uRtuqhRRTGbFLhil9uHAF6xenMEJkn0yM3XBzPAXHnkPQ8+Cj/PgYROn48PcYw", + "foTxcQjYGf2tF4ixOm+YoXXNMLBkMY7AUwC980p2VGzkvHmZMatyd5cH5N8aUtAB7QkvScDU8NTwYP2E", + "xTunsrNWgdrCfc+iNJaGQ7dUHquoqGb1CgylscoH5bqWh9U2coOzWzwIVF7ehwGGcfMjD4luTR0obN8v", + "jtRqtDuqSEysjDRpBhBho5HXm/MwuEczbb6R6uKya1V+XoH4CjFH1uDkyieXZxKR75qJ1imcpdrHVa2p", + "y603MhpIc16l50w3u0AU2FW1/uRZIV+xixuCco9O+i34WlTot2sVqjbAbkorLgUUp8ALSMwXsglaiKf7", + "LVpgPRiRuUHvpZ9yygTiXmBPgMD4Hvi+fsidKaJr1z/bjibRUHByn4aGyKKnCO9oj66fTaHRWNc3cFds", + "lZYfSGlZzRlO1QHWKmzJhW/hiL3IHdSrHLpfC0fIS56jlJpYIvRGx6k4+jZ2mu4sCV63E8UolLVSNH7j", + "4quJlPR121R9tsbtV7Suj/jPjdtV0vV9PuXJk9qw0JX9zfTPACInVSn08uDC6PYzCm4vg9g0ZGAsoK0G", + "K9kFlMoOz91DIJutl4hpAzdfnuY342prLlZu5eFqGS6qxAd+VVlTCdQuM2mEPppC0vo3Q8YVCpLzoYU6", + "IphD4MHY7nTnbYubKKatxZUyU1eu42uViOorAikfSNpNA827pmhIZZxcsG1RCbXKaEWXOqWjMITq0Jhg", + "qiKbkCi/1g5UQFk6ak1uq3zwqT+jOt58oeJt/KF/2unS/5y9/p3/8fr0rNPtfLp4XY29NJ5Vk0VWmcg+", + "NjbtxRKYuqFnUa8uN8JAdmLuNLMAkCSGH9amYzq0k46nFZhoFrDiSm4MDZdXzL4xNkxlGZoFVhMUA3hT", + "RCl40q+4CFotjQwUvKdhxYP/w8oVjgcsKIb/cTu6rCaPvXCdkzqNpUNMqgOb0ka5c+D7MKhyCm0Qolfp", + "AC+f3QtHohNL4Cy1+/IBrWzt+8HVYMTk5vvh5MPtW+bmNxreDJiHXv/8Y6fbuRxeDfrM+e7z8P+Y9jy7", + "wW4+CLvSS6W5b4c0U7b+Ha1/x4/l39G6YJQfTtY0xO73Q8LB2LEbvpHXPEprLN7inXotqzdrnZm8s2tb", + "/ok692KcvkardkrlNLyARJZRKDj5JoG9V4JIwYDnoN4Ko8ai0/bvwlgDj3wwYhk4bMJ+WMNMGcl7G6wf", + "yMDBwZtLJ1PrwFGO5e7kcCLRLSErb21eHchvr1cTPbOFepbqlFXAvtSri6odNXh2MWB8U08wX3QuHxJF", + "5sXsKPiv4JikRon234uSblqVXGj9vNDERkssNjJ5iiIVWuU3iQ1pqGXfJPYbGdqEQYSOq9vrHEp4ki9z", + "8YFNLRLbmQSoXBUZzukp5gzvnSAkThSHj8iDXtcBTgwCL1zITk/I950pdGYwgLG8xqjUdbY1jDdHs7ef", + "BLja3uyalFM4a5FNpZbZdLFTy0te/FhZX3JdjIwpLu13wLBv7EkUBF5W4THmQ6125V9AMg+9RqsVoH/i", + "PVPd/jz0DFT7YTK5kbmz3dBLKVgaeuzzDdwBnnCAzZyb+KslwqtJSKCy5pzPDFW8tXXiMS0FrEw7n9Kt", + "y4xdk063c3M9Zv+5nTAtyXRC8iAsXBWhhcWbEK/D5ILAiWBM6erIviKeMCux26426xalBJTm8wMYo1kA", + "PSfrxKxBt7fDC0eQ9O5veT6YQh9Xlw9lbRiZ53xCuGi2Iw8u5Og4OjT6AJMPEMRkCgGpuq/ndo1Vg2V1", + "HIAzl73zN+Wzk7Oz3ulZ7/TV5PT1m5Pf3/z2x9Eff/zx6vUfvZPXb05O7NOkAM5g9MgeYAKmPjOA7SGk", + "2z+dzadyDF2YViXFpuwstA2P/uBV6cJ4FZIa5efSUFUsqvlk1TxxbSYl7GS9nDBQd7EBZMV5tdAlAd3C", + "YXAf2nHPSOlAjyY/JNkFeZV61XzYcTYOc91WkUO/OeARIB9MkY/Ikh3PuUK4GZH/QiG6Y/lve/+dnJy8", + "gs532dmHXVFS+flXfZ5SPzSdTRguQDQPY+jQRkIMrUg0YznWmM2nC+W3LpKRTZ1mvzmfDD/zgtzpnzf9", + "27Eh0NgmuoXvURrZws9KY0YwcXrz86QAZL0Bj/e+rdOHb0eXmuGbqsesvVa1UY6K0slemblX5nqiXTft", + "LFRRZJsX166ZvDpRaQUeXv4l1ngRSIEc5UVZocI2CGaJeMayFnLji4+YH7u8s1KgupxeR6+qCfk6+EZi", + "oG2AvQfzsKXFMYhUhfT6ss8yDNz8a/KBPYpM/nUzGJ+PhjcsX8rt23/pjTtFoVuiqVqhC7ggpENTSivo", + "vlLg1sVjpA2dJMiJ89zgmhr9rHS5qTYsWiQLZZImQ+uKn1dwRtGoNh5cvvtwPeapHj71r/o8hcyXwdsP", + "19cfjXvBjueyCVhdmz6OKf3Fwk2626AsLFdEZGFYfTnRv8Op4YiiX3QAWXH6n+FUdyTuRKM0Yk4WEdSo", + "2WC2+lpT2yzQXuyqn+eEm2F2t6tcgXjfaiZxlac0icxKm7nmhE1DNww8JJ5Y+C3P1aQ6mUGifGel5zXe", + "H4FMesJz8M0gwcJzNe3qzGjfVGtQ7PZahDHWH5MYEDirzQ6uQHiZ68fLWpsvImVplUJM8oWzi/mFX53V", + "iy85dXE1XS1Wq7ZoeKFLg5gCOLzQ4lD2/oiCnCHl3e3V+WTIDqyL21H/7SVVUi/67ysFJB1EaiKNKJjN", + "rmEv+V2v3qwVg7ljzUh/vXuu2E9j7ijGJB9hVTglCQnwdRSb8tgDXBp8iuTwlCztIjbl7Rw4OIIuukdu", + "NonzSwQwhp7ziIDwV/9VzxVGRDRwONNfb0mcQM34de+3qudWaoA5PTk5MXpiaYfJ+041dINqtKC/w6kU", + "Y7bnuKHww9rRzfxE3LWRks8tbD0vA0LOmWiTjkGqz4fWO8hcauTtssHgE6VX2V2noUpidPhZJ3d4NpDq", + "yqOA/bVamOzJXVlx+rE/FEZJsEZe5fIo7xD0c+e+mrYjo+WcFFMkY80kY+nM1MruVna3svulZLdhjh9Q", + "tFd4Q64gmtloQwIXZv9Kw32lvrOx2t6YpUKrTri7psdZlm1t40nUNjCgQaYXU/IWc1OIRXVLiFRGraOe", + "UqbYm8HVBU8Qm6WK1WQBzueMTdPLvu2ff7x+9672lGTTrnRvzgsUMzFO8uKk6G8TBjeK5C/BShuM3Tn0", + "Er8iKMrQee3j6EsxT4qlgKnZbMyrqBu9kHLpWbbIjlX1yHDtIoxGApZxuQkdyaHOecc6LbTQvDR/xhDa", + "5NJVebwl02k/CubSfpM82jw7eNViJ2CmQ6/PVcb1Tf7BhpOrCLMuh7CKfoRQOI/pReZeLxe0LM358g4Z", + "uLFuQuZ8r52RyZE78Xi76WmxfoXNNYMC3jSSF6YhF6sMnOJns8o9V7f06Ms0sDvxCtEczTzFjFGebvJl", + "qwoMRZstsmzuCcNmQ9RXD+b0cg8Sn9xUZlkSjYzZlqweCcQt8k/MD96FoQTWn+PrK4cDXQ7bYSNonWjk", + "s+ALPfaFsce9MS3QgIXaMUELGBqK42CC3IelyRWHfnOweFaxe0lU5EUDtmU62ONp4aXMCsdKnzFP/qRD", + "+WNG2eYkrjYLfFLes23fLRony7W+BsplScLIDfS1ntMZWW3ybagJfe7FnuwK4dyhInsUKlQVjiHk/irG", + "kiIL8K2mxVMzZd9UV4RHfiRU/jL5ySGcQhDDWOYzYRhlxwr7OduUOSERu/aE4QOCsjmiu8p/km/nbzoi", + "hDnrK1Lb0N4JJuHCcrJnJvG5W5QmeoDP4vRvhqzcFWE2sfyvKSF2To9Ojk4YHfMg7s6bzquj06MTEY/N", + "MMFirn1RJnamC5B5L5/naasAYuyk9hi66UAWN+lciu/vGRpkQAOb5ezkpDzwBwh8Mmcoes2/u2FARFIN", + "UU+aNj3+G3O+wukBWMPHgzgOqRR+LvmnXoUkXUeOODpv/vra7WBZw4WuOmsofUr+EjC7c+g+dL7S/gx/", + "MQTesh6BtBmqwuBINth3FLIFOyR0gOvCiDgkBvf3yK3FaIqBWpQ+nh4Dn4qUYNaDC4D8HntIxsff2c/q", + "b88cLz4kmtvTBfsdOyDN9EW7O6w7f5su7UKfthjQBszVgo/AeCYGC0iYPvBXhZNPaQZH5DnvvOF5EFKh", + "UVpKRxVq/H0g27H1avJ+LdHTbxpPwsR1Icb3ie8vHY5SL5cmrYS8527nt11RXt9ZAJ9iAXoOy6DlybAj", + "DsarjYOhg+JdGE+R50F++8jom9NJFZlJip+wJvSw+taLhcrBPvC+na6GML6yay9xNVnS+XVrHRLnI/wY", + "JM7o4W3I5fFGiIFjh29aAXFp3FqZTCqxRUInkTjPY+NZL/Y3shDtEnSw58QAB7QVA5ZigFPL9sSAekBG", + "qEfCBxjQU1H+zU7DKNSlNBjBx/ABOiBgyRpZa+Gtlc5YEBMRmtBW0qBDu9tIiXR4g0yQsO7VcRez5Qk6", + "Z9D92ESNm1C1IB26sROxc5KMs9+qKDnd8hwFu36YeMfqDd2sQZcyxclrDxvEQQEmIHBhiYjP6WfpXmJW", + "rLePWwaIkwRZwMW+EFiN1s4RrL7Xi63/pLywfevJIXphxJ1dxImm7Dc3hx9/Z/99rtpvKqVYq6PShjKr", + "ON/IWknEk0SblBOewnGXQmhzmy1SK9Uc3ryGyqMQaxwbbMda2ZYjcQUzGXlzFFdINU4/X80Uflwn1ti2", + "pFKthuYvUgH2s9P9BSPhlvb3i/YXcOUz3Hh67+7gFhnXmtBUeiQeyEG+iSOcjnHM7PR8l7Bxxy8Rphcg", + "38m1Nm0wbT3MN9zabtO5xI4rUzbcfJkBJ7e6fSKEdOvZRhQ2obz/uU0OA0RCKs2Pv3OOfz6O4nAKzZdL", + "+fbpgFzlCWbX5ZUrcrkQzAyfTn0TYjJKghs2r71tynTopZJrx6deBUHBb9BNpG2F4fdop6fCVUhYBYIw", + "Rv/mWepFTiMefM2jNEtmTgKQDz2H2+0dtj3OOyHPh9m26g+OHJlhH7gPx9/Zfyys+M6YNlQKrOQph30V", + "yaHsjfa5MY3Ew0DcS+t8Hif7pNqc7gaM2yAjYT7x691MzHOOsdSNwPfDJzq97kWgSLVS9LLfq1QsTnR5", + "jgnw8XccYCtuuRqrUr/MLwFuwCb5wcyMIk7uvWOTAjJaRtlDRikRbMoqV+NKRgmwhk2k4qJYm/SqC51X", + "XolLLNL4bezF9I+u2RDACzOtZAlQYDh7/ToHxOkmdKAoDuk/oNeeYXvEmqZLJKvf4IAoktRePtZ4mwI/", + "EjD14bEHZvg4Tf1uvDRidmtk7RwyB8SZQj8MZmpWgTTNOJiVr5SfTy8AKzc7ESXU681lMsF3lqCFp9xm", + "LPNPAuNlxjMemN0hr/qY21aEiJXcKcD7Uhcfa+rdWA38CzA7FzFf+uxjFXKITilf/9isP7eVsNt5vSvh", + "R2+haBH5cAEDUtINmPFC0kH6dA7wg1bCsIbH3+l/ap6XeKWL6ZLzTVGA0AksTe1sHOOhTwHd8ZEPCIGL", + "iIi8LAahIBp1VFhKsVDbtOMXano0Mr0xrP7s/Pkbv/tsf9aJWn+fagr3YcKTNO2JiMj4uSQizHcGYiNC", + "jv1wVqer+OHM8VEAZeYjAUdRolyGs0sU8HoshyhVRJYnEoq0vNOlQbLwNIxaaFBAWFHJctClIXluTERq", + "7NCZQUJRzbBsmBkjbnnUzFyRusFwb0qrClhNnQQE+RuYuu9Qedcj8BtxMASxO3fYTEqN54r1sw46kV69", + "VkbB8BH6v+Bf6UQocP3Eg6b9pS1xR6vtVgt8yQJ0AFvl1pPJbShgLErFTHns8910eZd2ykFpBVwpp47V", + "IWu1PXtw5KpCqIFCLKJY23fzvFaaSn7l2LkMZ+ufOvT/e1nosPl1VSnUZjx40jpsP8DRgx9QZGL++3sM", + "N3LubPWk275Kne31Cg4y7bW3VatzMk4nYTarYpMYuJXRh871BPoOa6bAIYpRGi7uEzZoK+x+XGF3TaA/", + "jkCwgqTL6KmVdi8t7UoXeWVzNiFpWAvlMdCF/rEHp8nM/Bw4eAR+wirLOeeDSwd+i2KIWfg+mAEU4KxS", + "o6hE7gECjjSy6Bz6F2yqQ3Fe2nzs3OfT88ElQ0JNqBzDJKZyiFUmp2yqR/5OI+ZU8GWC1xpRAwX1eJo1", + "tDco9c1/msxKLKbw/Png0szyVrxucYPiT415JSetn17k52a3qH30BviRlAuN7Uw+Gz7AJVZMMsZpabvm", + "hixGBiK3R50J6zwMMPJgLEmMPWeHLsvx4zngnkBR6EYYCLdp1qyGZQrvwxjWArMpQ+c7vjUkzEEDYlb1", + "MnQRk6BPiMzVV/9iIXoNfFkCG8PObvlB3n5duYIhzgIQd46Yk4MLYwJQkCUJqVpnmvcTrmSSZW9whryh", + "VYtLt0Sscrqkxx2KHe4YoYNYpAZ90W2ZLp0sF3cWjcLKNqYWEIP1tpyqXLsQTdkYOc0DXPZ47bcIoBg7", + "v3iQCT7KfUsHOP/z5n9+LYqtSncrOxM6dsMIWslD3tJ2Xaz1evBu1xpmbwlrbd11tu6UNywDxBooaMfs", + "GLbU0vjZbqWpfYTLQ1HWth4wKXHRlBEYultm0DGDI7THLTDE98fTXoMQeebFRLDek6lJtPweezCbYJKY", + "OlDmFPvTHlAbCWLGTQKYU8qx4kyu49gcU6Jl7RnFVdLWnLCv5oRSnXcLBbr29lk5RemKyC7jfM6j9Yva", + "NLsr4GSKIXFcEHiIZbSSdL3R20PVip1bDD3GRhwWQq/HZXgAkTZX5iVkKM+z04uHwtoNBLsUMa1kz2tb", + "Ei+ZbOf4rdK1uoa3nXNW1MwBTgCfxMBG0czb/tyPNwwFHB02Dzjs/SYlZYeVkONW/V2+2QjyqGM9UeBO", + "Abh9Dt7Vc/CV4QU45c+UN+153l6LYxcs/rdNIDWokxSNUwTvlxonuBWxDBCeXIv+spVi4jBvW5aiQUaN", + "t2LhJcWCLet3FcKkR39F0FeqwJsNJny2Q7aYpPz8k3PxLCTt4W60mKxwxhYZrTIhef2xeeCpFXLHZprO", + "+yUZbhtXAL5JK18BXiDNubV8kJnNW/lweKe8hbLPomgWWVnMCrVASEaZcsCJk8ARPaszpHMPikuECfei", + "kFU4D1WmlWMuFTTU+CdZALp2GGY9NJtyUCraZpn1N/C4t455+rR6InohTxcKN6+PyUj5f2E1uYkBaFFP", + "k7a/k63vWOutEluWbIW/8TFXqbTGdxbpb0hrwhuiYHbHq4XuCPK+xoHoofcofHosHgkyT6K7RaUr0csa", + "salgGyWBlGjNMzSoUrTNprI/qRLY3izSg8ouxsL+xI1CFBDLc3eBgoRAeh2Xf8UQPHjhU5AexQ2O4feQ", + "3NDJD/0QZgee9A1WwmaEwbrT7cBvgG5x503n7OTstHdC/zc5OXnD/vd/DXJHdO/f85vIJg5IBmnqOayC", + "GlL41gD2HgUIz6H3lg3eHNzty8Ycqa0gHRmftPJxT+Vjfnc2LiXxsQsCF/rmKLRz9j3NfKWTd7zJz/1A", + "yVDAVJWaUmw8m2DouBJpO40iY5P60OMZCWtfJmXzNh1dG5dfklEFybBxyRTDyAfLqjJy9HulZOJNfmrJ", + "xFHQRDLFEmm7lEwcTFvBFIvWrVxq5VJJLhXkwgblkkgybON9Kws51HnfijoRrfvtPrvfcnJx6LB28Wus", + "/RVtvkowpKCJcTqKrb1VEp01oKJDBaTVk7y4h6vKPg1cXFNGbt/i8z6uKWIyuSlQvLaXq6lcTrqJrZ+r", + "8HMV+Gjyyi2Z8oU8XSWNNHF13ccyCz+3r2u5hoIF7zdQm5i7q/iHnb9rrcw4cI9XOrl8e5QsXO/7mmHF", + "DOxu7dC2/C/9WVve3wtXl1r27qrkVuPSKulX+LQK9dDAt4fs1lpQgH80HpXeqi2PGtxVa45JGNBTsBcD", + "AnvsBko3V+y9JZfV+bPWHosH7tG6XQ7bnnfqj6u4SxfVVjDskeKukQern+z6G/xNiFl+DxS44QIFs5Re", + "FxBjMKs44UfQheixlUFNZFCQ+H6J8oOlE4GlHwLPQYEDgqUjVtvtEPiNHEc+QAVKK065ExlikZk0h6d7", + "4GPYKheGCoec8TTstiqH29zThc9wL06CujeOfNbA2leOLEtg+9Kx/3lLscjkaPXWsbOsj8wPH8Q+gpjl", + "uoZW4G0xKMAHpAkoG6vNtDeO35a5ag4kWoECkcbR2WTYgfGWXfy/zCGZcwEgamE5F/33mJ5eYeAv1d/T", + "CqU6gRT4yzvZoFZRmYahD0FgEdORK1drgbMXCu/QFNU1xnlYZPZ9sXgP594HM3bUPgm6CGPmgKGSQXq/", + "BIHnhAmhfwr1EVP9kTaQuuCRcwHvQeLzfPf/Q+nhfxx07yQBhuwY1y1fzHQnB+1UktDOKnc2fQFunYb2", + "rcJPTqNUFV35+4j+vuZLlKrhHnsIRz5Y9pi7RI2+K9rSYYV7RXhfoQRX68AXfDDmdnHQ+rAiWnH6jpVD", + "ioiXFOgTqDMrAoosfZHC5ls2wWtJoBVdrehqKrokn/Qon1RLrhyPMu1Bn/A/S29XIbkGYrChd7iCq73n", + "tvfcn+Seu7PjLJML7Wn2I51mudNjJyebuF6bw34mvIH0Ks1f2CuOrta99FSgTkFKzVN1jhRIKPw3d/1G", + "rWjNkADk42Z+piqFtO9NRbfPAgNtgMHz/Mx8PpVfakpJ5EkOBB5zJkvPfxKmV0lRLOm/Ox4jiv/uOJHh", + "QTqjH0u3sxwM3LY5Yz0Nr8DK8g42l+EKXNae4nt8ihfD3ywZulsi6BVY/FiUjKvidMKzfJGEGY7yfH9U", + "y8VjWZNuRV5Wp1fU9R+TtdXrZ8vSe+rkdR4mvsfjaelFUqe57FFukhxXpQUiX0TWsGRPFiV2WVguD3Ln", + "lnr7qwNlIFbHx9ro9fNUpMnEqtYA8uNK1JWqOrZCtdWTirKLoAUKZvXakmjXWHq9h2QipjjYu49WBnkw", + "InOesYRnNXPcOfK9GJpcN1iHhtJv+4KEb04rSQ5eklTx56bFC4yETJF/Ph+D2J2jR1inBYlWAkzaXStC", + "xgRGwl23Lwe2EB9yPKP1VMLbuu6urpFtUyaJfRd7biWV8kkl27qgu8/HlHJdISdTWUjl2F9hfimf6PZT", + "2VQlmlIWrpdJNvcyUbrfXh4NZI3VVhr9JNLI/q7VyqLDkUUK429fEvnhrM5Tyg9njo+Ckm5UNkdfhrNL", + "FEBba1Arhl42nsmHj9C3chniLXMzVzGDpAPa6x2CvmfMIAfpweuw2RQ4KoqZsA5NARnzXtpQEsACBcLY", + "q1o/+/x2ydfScPJrta8BD3x6D8XQFdHuFVBcKM1WgSTrv91DSpUGbQH9dVPQpVJYOQsuw1nzY0A4GlWk", + "NmceEFh4Ehkc9yfs53PV8WXTjjl8cD5RXZJe7pr0Mq44HMJGzjcCqT82ja/gdZMSW5qdVvjTFIlcR9Gp", + "61ytyZi7xogX9koCb5qQKQ3sEDMYn3x24y33shQvUya11L7b2wYnRi+E/KIBv/ETuFRIw5bZchlNq3Mw", + "BXw2FMyq+epwMjFtyeuUI6DJ4RbFFJEE8biMFyjc2Z5z659zgk9WYL2K8+4Y+JQwglkPLgDye7M4TKLK", + "h1Oq3MlboCAvNobDBnDEAEXW7dMmA9riPW1wKJFO2z8JdYhpWHLKuAkt7+RfEyuotdE5Zn31Kc9Vxxg/", + "fUiFenMr4MburCuhvNHV7nS77L3CCaihoZavtXc/Lbdt9pQ8xpCQOtcizHZPdnFkl+psBgq5oGA2Fn0O", + "JKnvjo5JBTFrnJHqnrSspLnWadC0MT6KUI+ED7AmGZ7Tvxk6vF011/QjNKHNWn0SHzO/opshwwe2SB2p", + "4xPpH9Xa0IvKI6VIjlqFGdIf1ynlEmTUbkfsrY7IECBpXVELt2nCKE7a8teGw2YzZmrIYFUHjoW3FK8u", + "l3OZMqVdzZxm2nSre+2e8ACXVs4JtF3z9DOMDD7CpU1ekwym1H15eIFt82FyWdEYQOkSPbxYEcQsBm2N", + "VD42EI6SgMdRCsPXi7h6sP18GUcPNvUeuHmocKhOHhXEkmUQgkvnEfgJ1OcRgt/AIvIhFdkPcHn6hjU9", + "7XTpv874v86oeK/ON/Rps+mGsmXwxKVpxqFqOmeNh4efaWilSLvWuyYw+1wqSgtD7vomZDauQQdprwAM", + "AQwXNWZhkZj4Rdx7OCU0sflC3uNn964++8/dzDoS/CnUU/jNhdCDhnKOfG8a8Hn9xeR4mvgPZne6t4kv", + "6hhBnMkEXCkUaJ+fWDDQ5TcUDvglpQNuLh7a6Is9kw+MTVUhgTcsJVwQuNCvcLtl37khQ0mcnVNxTVKD", + "u5XwEX5mhYIhwF6hEBeGGEY+WG5cbGQOW/RfT9lleciTE2+riIf8IZz+DV0LzYUhDWY5SlohtbdCasQo", + "dTvyiZnRLG2s3DZnYWf9CJfts15mbFzpts6Q3d7YdTd2R9h+N8kH4jQwntOcB3Gzo3kkj5if9WjmCNiX", + "o3kzZjUOXKvV/6QH5nf2394TIvOe/MSs27XhR4AAfngGlQbCC0DAe0i+IDKfSLavlR+SffTiowTyrt8u", + "f/hTnm7aKukYGFW0p3zel03BjDXvdjVEXs3PKHhEBDYNmJC99E6gQ/a11X2l76eCj5W8PiW2W19PXThE", + "RotbioHgE1TSevucpUQ9cJTYBTtw3L5ohAMHd5XABkEYP3ts79nZjrReQOzeuYp8q5MLMABTH/ZiQGCP", + "jUnZQ/DaKnqxkELyhx7/9zMXMT4ksCxsLtjvODUj2Qga3udgvffyXF8NWy9Fx6Gf/LWyhVPIPsuWHJtx", + "IszI1aSL5vexNoK+GSccThT9oXDCdgP9V9MKXizU35JzOXwHw7kiBL8x51adfAu4mDLma3SDlL30LP6J", + "fW1vkJIaFXysdIOU2G5vkLobZEaLmwkSFOMdf+d/WCiBDhBAOPdxuKgLsuXU8GOogmLZJtj4553y7m9b", + "4d1VdMCfg2v3KFftlSE1bcqkuY1pIC+6kpAt0kiVJjGLgB9DB94LEbBd5Zdvl53yK9CxJymvLKWXRg8W", + "+9YKrxcWXka5soLwqtJ6ojhcQDKHCe4tqA7q1pcvyro4okvqg1eXmfIm7fpJTPZDXBQI/EaOIx+gAlUU", + "R2pyByhjuWXKl2ZKygGafdnUDeSfBCbQmg1Z68Yc+F+01wEx32FHNh9SsOr27SE52lstg4XzCGOMwqCV", + "ifskE9PdKUtEyTmrysTsqc/G1TtOHxvrfL1HgMBL2rDNq7HP1Wk3kYOhFpPbzLSQ0tkeZFsowrKrshp5", + "XmsQTKCwc+tnWLCCq7jJxC3ztrjkv64qcUWPXhT6yF3Wp5yUHRzewSbhpHSFvmE92nSTxzq0rPZoVNiN", + "9vFo51lbsQ/ch+pEk2PaxHmC03kYPpSfU9nnL/xr+5zKc0yqOGlyeyigep/YYUcVj28DkJB5GKN/Q49P", + "/Ho3E3+CZB56rKIH8P3wSV9tmW8Q0wM5C6jnGfu4FiMeYwJiYmTHMf3Kz7HrfkLmDrusFBnyFstnGwbQ", + "NUUo63mInPnq5EyDB5V7GMrEsZLDyhwCT3iN+CEnmBqLJ9tw6CYxIkuGHzcMHxCkg7KiSF9VemAozc8o", + "CYHuwMp0UJf3d3w1LhJgQSAHuJXDQg5fjYcqqhpI4iKWW1m8d7K4zAipJL4ar5FuuDCwjsHaaAyGgDx/", + "VWYZ3hzN5ie1jqoo7mrL0HvE0EbOs+ToyhNV1Ons7eLJSpQOP7SXq+2bC3SIaWYzSOtZ53amfVTZh0eV", + "dG82/cysq6peybpZAXVnuuQMVTi9OSEeiB2vu6+V3bcpMcQWrSgfWomws1KoKi0+AV4PtU5EqIc6/Ylu", + "9KpVtqvlRG1OwD4hcBGJ5JasrSI+TILj0JIBthKkyiUeYeYrLUQIJwJ//y4IL/yIV8cou2LoGNKOFbnD", + "WJJFWx5mzVsW3sdsZnESiK2q8WhHQZQwfwj+uKtb7vNeaCptLrMK+cI2/CUESramSlsAbyacBeqEy3tI", + "xnzYVrS8nHbQLEuvwdIghmsvFPt8oZC7tBWpQQB+6GECSI3BEOAHVg1KWAprrIQTgB/GbFB7ETG8+BFt", + "gykiGnCoFtctj+6BGdDEBrtIjyS8ZnpPYfxQlSwic8A2ujS13kxZMAlHxReGVIqQqqqeFBlpwAvv6Mjt", + "aJ/b9u39XCH/1ZMYikFMLPTTv5Pn+IdjY0fFeDUze41SEMqtbTl3/x7KVcZb6bBkVFH9kEZPSC68q73k", + "s7Phpz8sM0y0Na83kqFaag/5GL3VvSslorkhqHktCrX6r6YkhVKyty1MoRSmUPCCawy6ufrKL1emQge3", + "dTl7xdabI5j2krqX5Svye1QOB642JTURON/Vf9b5seQ4ofYEFmR6yG4tBdbXg6Zi8IDVBLFdq2YWaN1c", + "zHH9+Rek+pj+bp6mVufnY/YYWfuYxJ8sOUOrQB/V8PWQjd4y98szd5bF5EYpQslhXOfdKY8jtt2tWXtH", + "Zu0vKu4Dm/wh2SY1VRk2J3HwHERwS3rEmI3dypuDUSb4hrUaxQ+kUaSxK8JnqDIyVFRqZyzu++n7ONbo", + "GlWszwInuSvLQBb2a2XAxgG8BJg4wwuWsH4OHR/IHTSlKQKYDD1jnqJXZ7o8RTvwsW1S0LNUlq81ieyf", + "b80KssTe8cZOFmKrlwnW0k6j+SkTp3nwHiQ+6bw56eZExS5SqKVzv15l8jHPpDZdOmwC/aTikzmfwy7U", + "rvaxZ/P61iZTMqZj1gYDncu4hikg7rz02FOlMR1OMNC2vByUdxKODFu3fRFNUn4q2fRjT6RYar6nSt8o", + "CYYezqWeXQvB5Xy7DQ1CIgKpfT2qSY/GyWYXLzf42I3DoF4joa2cv8NpBhSJ0WxW6z5xHofBT62mHEx+", + "13RjkUennUGSqsRHNWm8TRe3Ldx16cxNwbuqU6W0UzKKbzId7dB8qsPMUF6RM3e6dO5FXt6Npe5VpQi2", + "T987XW4vg6+iFOw4h28OGWto6O2xq9HSS+fcltR1eugef6f/6clf7crclQ9i64cPSjgHXvQuXb0JrBxG", + "d1/2zrI+nXYT2/zAxXpxejQ1e6vIE8TX527VY+KazHXI7kl7zFlbOjrbY/MQDPuNDuuNyIe68pJs1nRG", + "a+Fw4LUm90s+bKvapCogJtzAYWXro1TASzja2PbqVAW1GGSrKlTLAcGW2xAFdqo8Ow5sH/TUV8Z6N6XW", + "YLbPBjP2iNzAWsba79BUto92vAjEFGkG15UCWLzxF/UxY0fwaVLEaGETTiLbhauvjc9iiQgSDK3qLcq2", + "q1i3xqyvsDPZAPeAAs8KKtawMUgfUeDVQ3PwxlSCFtAB9xTQkvP0E8AyllldQufs5Oy0d0L/Nzk5ecP+", + "93+NxmrWvU8n0BMvPVZ7FIqObTVyCvEU3ocx3CbIb9kMm4S5Asv3KEB4vjrMsv9O8bwpoDeK6e09DpQt", + "8T/t00BRd2wtHFtxl97OmwDzkLbJ3w8cARo96PLsryb0twyEOOQK1K0a3qrhu1fDW92y1S1fJAQKr1mx", + "nQmgtrJI/fm+herp2TlPQfUSnx6PNVbDtOUq9sOx7NxaEffZiri9e1FKAAflOdUqU60ydTDKVLaMTFRv", + "xDabgmTF4KmVVgPzVmMkSxKmtTpsVisxaADb1UuOp4n/0Ms8EfURRW8T/0E4tW1IUaEjHo5/4pb8EMo8", + "laHFNuxoWr81u60jUrkmc+I5lcTitF0rIaSEeGu1z1uXFNxdpUZS8EbOLzGUvX/doNg4HOeqnYoNmaaz", + "gdgQ+7S/YkOuqUZsiHW0YsMgNmr3eZti43v6Z6+UM7I2AkIPckOhceBxEBocGKsZaVG9t6ER+t1tHR6L", + "sREGPDXzeDTQRk2UxEYY8KArFB8U923zQG7v+oceQ7FtOVIdTZG7DmxIshx4oMXeC5dtxV6UpEuD+qgZ", + "GZXzPr7slaVWQqrBHj+l8nMA1d9uqy5Lm5KVdpeoNIXmc5a5paqMlQOcAD6Z87fYp28R8VCHU/SqPpNI", + "dc7MStB2JBo5tlcNSxOVo42bv1PZ2Cz4Vq3VZYa/lYy7l4x7V+hECLoqKt9O6ixFFuecevTyWOoGQiLb", + "a7g6xaiVwruUwnIHVtBMK9S6PVdMVQncKqat+DWJX6GQ1OnEGxe5vHpezw2TgNTES7A2Mhe5LPsIHgHy", + "wdSHTPoq4kZvX3gPCa/Oh8/ZjAcveutSxh94yYjcZq1opuSkwsmnfUE0OEznkLRaIYk8+ycYxvjYTeIY", + "VnM25rcD3tCh3Urce4th/B6SczHYFumOztSQzhjEbQHily9ADN0kRmTJxLgbhg8I9hMqu/76SkVVIelQ", + "ntwkubPt15DxDJF5Mj12ge9PgftgJOfzcBH5kEBO09d0fkd7HtGJuD3qPRv6muLyXA5fIPBXJ2c1b6+u", + "mNcrzzuHwGOH2/eOH/LNyO9DUaw/F5CZw51cYH4OS/RhAmKzKBjTr6shjnVtjjUGz/ZxxqBriLAwnPlw", + "O/TGhv7B6Y2jb8P0liHuh6M3FDwiAqtrN2EWzSS1Yd6BKd1WxzcdYcL6DsVcWzzF1YmsnNl9hOXG5BfY", + "6ovWxyqryVPAXkZ5E80NMUd7x8B1YUTMlrc++45TC5uYpERt6ubzPp3t2JP44HwixZBkMABVUB9fuY7+", + "Wo+plLw4tkt7b09fMWTVLSoq6dPvzeiL9+lsqy49HXwD9MVX3tJXJX1xbK9AX344Q4GZrC7DGXZQ4AB2", + "Nh5VKBiXbKAtOWfQI5iOX09Iu7tH++FsBj0HBe31+YWvz93Ob2dnu1p3FIeUBpjRdhAQRJZOz3kEPvLY", + "ZHRTRBMUzBwoRzIrvIyw9Vf5budbDwZ0ql4MCOwxGzjVoflbjY6Zw4TUcHOYEDt2DpOXN1YJJgv3rFB3", + "a6Sq0aYZ9djapxZwMYUxnqOowR1O6WR3j+Nn4Kesm0hKsVUC10/a/EKnoqi91K1yqVMxWE+SEcD4KYwr", + "XCnSXOy0gyPbV4nUGznm9pSk8zkIZulE+6QtuQwyL0VUK85bpamZ0lTN6pzy88y4tj4VwxmVxHHVtZu3", + "wJUqVeoptS2+l2DsE8dL5LUPjS3Tb+amJKl8M5cl7AP3YSuPVGM68h6/UdVI0oaPVo8wxgIEo/sTXYNo", + "J12gMIwfNVr6MLgP30PyWQy60ZrECqRZhsbTo5OjE10OSMXz6K+061eLcsOTisUWvC0riP0LdGJIkjjI", + "Ia9w06FiNgkCyj/pFN96csheGPGUU2UWeILTeRg+9IQj2vF38YNF+Ds96kTrsqMa/90+sl0MZHYESyfa", + "sR+YZai4hK892F7eOFEMT1fJ1Oj9JVp8tWKOY4FnGzOFbCr86ms4Rihu2DZR5t7yzWb8Jzn03H1SoIZi", + "pirjCsVKWgdEYCfdrpY994g9mVWmtEVNeTTlTfbHc433NW+ldaxmzplWPMedTKt8ljVn/OF4LDf2HRUr", + "bu2RJafkUsCXvKCYfZCZWl1f+bGSkO3TDuwFLW8rij93bpjOCoGBRKJsd3FQlrymBuW3nGaoubgOsxVO", + "k2Jwj1UisGY1WBvci/YyQqZJEq0UwDZA74UzRwhiVShmxfiYbp2GZc8JDVSunyFQbMXgsJa3Xpq31Ci0", + "dRjLRu2z565meuBeMNjmdcE8Mmxj5UVO0hyX7Vo5tJIIRfWwlQdGBXE95qxRE63K5dFNytfFSxnvMX3p", + "MJ6UDcrj7QM/a0pU8AITG6gfvHr1YD1gszhMIlb3IwNBbpQRFNbpI1x2atOAbFlIrFmLSz4qteW49lCb", + "WKn+VyPBJVMTGZ1bZFaNpsmCVsoRtJeSa6JhlyNneM+s2zih1AG9LuMqHxCIScpTCDv3kLhz6JmqQ2WC", + "f88VKUEGKyYeerF0Qwq8jfIMtdmF2uxCW8gu1Eg0C9mALV61cie5lVgWvjUHZIL5EeTylqWcdJhaTxVs", + "5d1eqYAZKa6qAhYd/6YQxDBOHf+6WldA5knG5UES+503nc7z1+f/FwAA///hbZopKkADAA==", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/api/v1/server/oas/transformers/v1/trace.go b/api/v1/server/oas/transformers/v1/trace.go new file mode 100644 index 0000000000..35cd6ee90a --- /dev/null +++ b/api/v1/server/oas/transformers/v1/trace.go @@ -0,0 +1,86 @@ +package transformers + +import ( + "encoding/json" + "math" + + "github.com/hatchet-dev/hatchet/api/v1/server/oas/gen" + "github.com/hatchet-dev/hatchet/pkg/repository/sqlcv1" +) + +func ToV1OtelSpanList(spans []*sqlcv1.ListSpansByTaskExternalIDRow, limit, offset, total int64) gen.OtelSpanList { + apiSpans := ToV1OtelSpan(spans) + + numPages := int64(math.Ceil(float64(total) / float64(limit))) + currentPage := (offset / limit) + 1 + + var nextPage int64 + if total < offset+limit { + nextPage = currentPage + } else { + nextPage = currentPage + 1 + } + + return gen.OtelSpanList{ + Rows: &apiSpans, + Pagination: &gen.PaginationResponse{ + CurrentPage: ¤tPage, + NextPage: &nextPage, + NumPages: &numPages, + }, + } +} + +func ToV1OtelSpan(spans []*sqlcv1.ListSpansByTaskExternalIDRow) []gen.OtelSpan { + result := make([]gen.OtelSpan, len(spans)) + + for i, s := range spans { + resourceAttrs := jsonbToStringMap(s.ResourceAttributes) + spanAttrs := jsonbToStringMap(s.SpanAttributes) + + result[i] = gen.OtelSpan{ + TraceId: s.TraceID, + SpanId: s.SpanID, + ParentSpanId: &s.ParentSpanID, + SpanName: s.SpanName, + SpanKind: string(s.SpanKind), + ServiceName: s.ServiceName, + StatusCode: string(s.StatusCode), + StatusMessage: &s.StatusMessage, + Duration: s.DurationNs, + CreatedAt: s.StartTime.Time, + ResourceAttributes: &resourceAttrs, + SpanAttributes: &spanAttrs, + ScopeName: &s.ScopeName, + ScopeVersion: &s.ScopeVersion, + } + } + + return result +} + +func jsonbToStringMap(data []byte) map[string]string { + if len(data) == 0 { + return nil + } + + var raw map[string]any + if err := json.Unmarshal(data, &raw); err != nil { + return nil + } + + result := make(map[string]string, len(raw)) + for k, v := range raw { + switch val := v.(type) { + case string: + result[k] = val + default: + b, err := json.Marshal(val) + if err == nil { + result[k] = string(b) + } + } + } + + return result +} diff --git a/cmd/hatchet-migrate/migrate/migrations/20260310150948_v1_0_84.sql b/cmd/hatchet-migrate/migrate/migrations/20260310150948_v1_0_84.sql index 212b0cc3a3..8bb869631c 100644 --- a/cmd/hatchet-migrate/migrate/migrations/20260310150948_v1_0_84.sql +++ b/cmd/hatchet-migrate/migrate/migrations/20260310150948_v1_0_84.sql @@ -1,5 +1,8 @@ -- +goose Up +CREATE TYPE v1_otel_span_kind AS ENUM ('UNSPECIFIED', 'INTERNAL', 'SERVER', 'CLIENT', 'PRODUCER', 'CONSUMER'); +CREATE TYPE v1_otel_status_code AS ENUM ('UNSET', 'OK', 'ERROR'); + CREATE TABLE v1_otel_traces ( id BIGINT GENERATED ALWAYS AS IDENTITY, tenant_id UUID NOT NULL, @@ -7,9 +10,9 @@ CREATE TABLE v1_otel_traces ( span_id TEXT NOT NULL, parent_span_id TEXT NOT NULL DEFAULT '', span_name TEXT NOT NULL, - span_kind TEXT NOT NULL DEFAULT 'INTERNAL', + span_kind v1_otel_span_kind NOT NULL DEFAULT 'INTERNAL', service_name TEXT NOT NULL DEFAULT 'unknown', - status_code TEXT NOT NULL DEFAULT 'UNSET', + status_code v1_otel_status_code NOT NULL DEFAULT 'UNSET', status_message TEXT NOT NULL DEFAULT '', duration_ns BIGINT NOT NULL DEFAULT 0, resource_attributes JSONB NOT NULL DEFAULT '{}', @@ -29,9 +32,9 @@ CREATE INDEX idx_v1_otel_traces_task_lookup CREATE INDEX idx_v1_otel_traces_trace ON v1_otel_traces (tenant_id, trace_id, start_time); --- +goose StatementBegin SELECT create_v1_range_partition('v1_otel_traces'::text, CURRENT_DATE::date); --- +goose StatementEnd -- +goose Down DROP TABLE IF EXISTS v1_otel_traces; +DROP TYPE IF EXISTS v1_otel_status_code; +DROP TYPE IF EXISTS v1_otel_span_kind; diff --git a/examples/go/opentelemetry-propagation/main.go b/examples/go/opentelemetry-propagation/main.go deleted file mode 100644 index e431e9682b..0000000000 --- a/examples/go/opentelemetry-propagation/main.go +++ /dev/null @@ -1,147 +0,0 @@ -package main - -import ( - "context" - "fmt" - "log" - "math/rand" - "time" - - "go.opentelemetry.io/otel" - - "github.com/hatchet-dev/hatchet/pkg/cmdutils" - hatchet "github.com/hatchet-dev/hatchet/sdks/go" - hatchetotel "github.com/hatchet-dev/hatchet/sdks/go/opentelemetry" -) - -// This example demonstrates cross-workflow trace propagation. -// A parent task spawns a child task via .Run(), and both tasks' spans -// appear under the same trace in the UI — even if they run on different workers. - -type ParentInput struct { - Name string `json:"name"` -} - -type ParentOutput struct { - ChildResult string `json:"child_result"` -} - -type ChildInput struct { - Greeting string `json:"greeting"` -} - -type ChildOutput struct { - Message string `json:"message"` -} - -func main() { - client, err := hatchet.NewClient() - if err != nil { - log.Fatalf("failed to create client: %v", err) - } - - instrumentor, err := hatchetotel.NewInstrumentor( - hatchetotel.EnableHatchetCollector(), - ) - if err != nil { - log.Fatalf("failed to create instrumentor: %v", err) - } - - tracer := otel.Tracer("otel-propagation-example") - - // generateSpanTree creates a nested span subtree, returning how many spans were created. - var generateSpanTree func(ctx context.Context, count *int, limit int, depth int, prefix string) - generateSpanTree = func(ctx context.Context, count *int, limit int, depth int, prefix string) { - numChildren := 1 + rand.Intn(5) // 1-5 children at this level - for i := range numChildren { - if *count >= limit { - return - } - name := fmt.Sprintf("%s.%d", prefix, i) - childCtx, span := tracer.Start(ctx, name) - time.Sleep(time.Duration(1+rand.Intn(3)) * time.Millisecond) - *count++ - - // Recurse deeper with probability that decreases with depth - if *count < limit && depth < 8 && rand.Float64() > float64(depth)*0.12 { - generateSpanTree(childCtx, count, limit, depth+1, name) - } - - span.End() - } - } - - // Child task — a standalone task that will be spawned by the parent. - childTask := client.NewStandaloneTask( - "otel-child-task", - func(ctx hatchet.Context, input ChildInput) (ChildOutput, error) { - target := 200 + rand.Intn(101) // 200-300 spans - count := 0 - // Keep spawning top-level subtrees until we hit the target. - round := 0 - for count < target { - generateSpanTree(ctx.GetContext(), &count, target, 0, fmt.Sprintf("child.r%d", round)) - round++ - } - - return ChildOutput{ - Message: fmt.Sprintf("Hello from child: %s (generated %d spans)", input.Greeting, count), - }, nil - }, - ) - - // Parent task — spawns the child task via .Run(), which propagates the traceparent. - parentTask := client.NewStandaloneTask( - "otel-parent-task", - func(ctx hatchet.Context, input ParentInput) (ParentOutput, error) { - _, span := tracer.Start(ctx.GetContext(), "parent.prepare") - time.Sleep(30 * time.Millisecond) - span.End() - - // This .Run() call automatically injects traceparent into AdditionalMetadata, - // so the child task's spans will appear under the same trace. - result, err := childTask.Run(ctx, ChildInput{ - Greeting: fmt.Sprintf("greetings from %s", input.Name), - }) - if err != nil { - return ParentOutput{}, fmt.Errorf("child task failed: %w", err) - } - - var childOutput ChildOutput - if err := result.Into(&childOutput); err != nil { - return ParentOutput{}, fmt.Errorf("failed to parse child output: %w", err) - } - - return ParentOutput{ - ChildResult: childOutput.Message, - }, nil - }, - ) - - worker, err := client.NewWorker( - "otel-propagation-worker", - hatchet.WithWorkflows(parentTask, childTask), - ) - if err != nil { - log.Fatalf("failed to create worker: %v", err) - } - - worker.Use(instrumentor.Middleware()) - - interruptCtx, cancel := cmdutils.NewInterruptContext() - defer cancel() - - fmt.Println("Starting worker with OTel trace propagation...") - fmt.Println("Trigger the parent task to see linked parent → child traces in the UI.") - - go func() { - <-interruptCtx.Done() - if shutdownErr := instrumentor.Shutdown(context.Background()); shutdownErr != nil { - log.Printf("failed to shutdown instrumentor: %v", shutdownErr) - } - }() - - if startErr := worker.StartBlocking(interruptCtx); startErr != nil { - log.Printf("worker error: %v", startErr) - } -} diff --git a/examples/go/opentelemetry/main.go b/examples/go/opentelemetry/main.go index 3b6fd8c75b..4a9920aaf4 100644 --- a/examples/go/opentelemetry/main.go +++ b/examples/go/opentelemetry/main.go @@ -2,10 +2,9 @@ package main import ( "context" - "encoding/json" "fmt" "log" - "math/rand/v2" //nolint:gosec // G404: example code, not security-sensitive + "math/rand" "time" "go.opentelemetry.io/otel" @@ -15,31 +14,20 @@ import ( hatchetotel "github.com/hatchet-dev/hatchet/sdks/go/opentelemetry" ) -type PipelineInput struct { - URL string `json:"url"` +type ParentInput struct { + Name string `json:"name"` } -type FetchOutput struct { - Data string `json:"data"` +type ParentOutput struct { + ChildResult string `json:"child_result"` } -type ValidateOutput struct { - Valid bool `json:"valid"` - FieldCount int `json:"field_count"` +type ChildInput struct { + Greeting string `json:"greeting"` } -type ProcessOutput struct { - ProcessedData string `json:"processed_data"` - RecordCount int `json:"record_count"` -} - -type SaveOutput struct { - Location string `json:"location"` - RecordsSaved int `json:"records_saved"` -} - -func randMillis(base, jitter int) time.Duration { - return time.Duration(base+rand.IntN(jitter)) * time.Millisecond //nolint:gosec // G404 +type ChildOutput struct { + Message string `json:"message"` } func main() { @@ -48,9 +36,6 @@ func main() { log.Fatalf("failed to create client: %v", err) } - // Set up OpenTelemetry instrumentation. - // EnableHatchetCollector() auto-configures from the same env vars as the client - // (HATCHET_CLIENT_HOST_PORT, HATCHET_CLIENT_TOKEN, HATCHET_CLIENT_TLS_STRATEGY). instrumentor, err := hatchetotel.NewInstrumentor( hatchetotel.EnableHatchetCollector(), ) @@ -60,83 +45,72 @@ func main() { tracer := otel.Tracer("otel-example") - // Create a multi-task workflow - workflow := client.NewWorkflow("otel-data-pipeline") - - fetchData := workflow.NewTask("fetch-data", func(ctx hatchet.Context, input PipelineInput) (*FetchOutput, error) { - _, span := tracer.Start(ctx.GetContext(), fmt.Sprintf("GET %s", input.URL)) - time.Sleep(randMillis(10, 20)) - span.End() - - _, parseSpan := tracer.Start(ctx.GetContext(), "json.parse") - time.Sleep(randMillis(5, 10)) - parseSpan.End() + var generateSpanTree func(ctx context.Context, count *int, limit int, depth int, prefix string) + generateSpanTree = func(ctx context.Context, count *int, limit int, depth int, prefix string) { + numChildren := 1 + rand.Intn(5) + for i := range numChildren { + if *count >= limit { + return + } + name := fmt.Sprintf("%s.%d", prefix, i) + childCtx, span := tracer.Start(ctx, name) + time.Sleep(time.Duration(1+rand.Intn(3)) * time.Millisecond) + *count++ + + if *count < limit && depth < 8 && rand.Float64() > float64(depth)*0.12 { + generateSpanTree(childCtx, count, limit, depth+1, name) + } - return &FetchOutput{ - Data: `{"users": [{"name": "Alice"}, {"name": "Bob"}]}`, - }, nil - }) - - validateData := workflow.NewTask("validate-data", func(ctx hatchet.Context, input PipelineInput) (*ValidateOutput, error) { - var parentOutput FetchOutput - if parentErr := ctx.ParentOutput(fetchData, &parentOutput); parentErr != nil { - return nil, parentErr - } - - _, span := tracer.Start(ctx.GetContext(), "schema.validate") - time.Sleep(randMillis(5, 10)) - - var parsed map[string]any - if unmarshalErr := json.Unmarshal([]byte(parentOutput.Data), &parsed); unmarshalErr != nil { span.End() - return nil, fmt.Errorf("invalid JSON: %w", unmarshalErr) - } - span.End() - - return &ValidateOutput{ - Valid: true, - FieldCount: len(parsed), - }, nil - }, hatchet.WithParents(fetchData)) - - processData := workflow.NewTask("process-data", func(ctx hatchet.Context, input PipelineInput) (*ProcessOutput, error) { - var validateOutput ValidateOutput - if parentErr := ctx.ParentOutput(validateData, &validateOutput); parentErr != nil { - return nil, parentErr } + } - _, span := tracer.Start(ctx.GetContext(), "data.transform") - time.Sleep(randMillis(10, 15)) - span.End() - - _, enrichSpan := tracer.Start(ctx.GetContext(), "data.enrich") - time.Sleep(randMillis(5, 10)) - enrichSpan.End() - - return &ProcessOutput{ - ProcessedData: "transformed_and_enriched", - RecordCount: validateOutput.FieldCount, - }, nil - }, hatchet.WithParents(validateData)) - - workflow.NewTask("save-results", func(ctx hatchet.Context, input PipelineInput) (*SaveOutput, error) { - var processOutput ProcessOutput - if parentErr := ctx.ParentOutput(processData, &processOutput); parentErr != nil { - return nil, parentErr - } + childTask := client.NewStandaloneTask( + "otel-child-task", + func(ctx hatchet.Context, input ChildInput) (ChildOutput, error) { + target := 200 + rand.Intn(101) + count := 0 + round := 0 + for count < target { + generateSpanTree(ctx.GetContext(), &count, target, 0, fmt.Sprintf("child.r%d", round)) + round++ + } + + return ChildOutput{ + Message: fmt.Sprintf("Hello from child: %s (generated %d spans)", input.Greeting, count), + }, nil + }, + ) - _, span := tracer.Start(ctx.GetContext(), "db.insert") - time.Sleep(randMillis(10, 20)) - span.End() + parentTask := client.NewStandaloneTask( + "otel-parent-task", + func(ctx hatchet.Context, input ParentInput) (ParentOutput, error) { + _, span := tracer.Start(ctx.GetContext(), "parent.prepare") + time.Sleep(30 * time.Millisecond) + span.End() - return &SaveOutput{ - RecordsSaved: processOutput.RecordCount, - Location: "postgresql://localhost/pipeline_results", - }, nil - }, hatchet.WithParents(processData)) + result, err := childTask.Run(ctx, ChildInput{ + Greeting: fmt.Sprintf("greetings from %s", input.Name), + }) + if err != nil { + return ParentOutput{}, fmt.Errorf("child task failed: %w", err) + } + + var childOutput ChildOutput + if err := result.Into(&childOutput); err != nil { + return ParentOutput{}, fmt.Errorf("failed to parse child output: %w", err) + } + + return ParentOutput{ + ChildResult: childOutput.Message, + }, nil + }, + ) - // Create worker and register the OTel middleware - worker, err := client.NewWorker("otel-worker", hatchet.WithWorkflows(workflow)) + worker, err := client.NewWorker( + "otel-worker", + hatchet.WithWorkflows(parentTask, childTask), + ) if err != nil { log.Fatalf("failed to create worker: %v", err) } @@ -146,11 +120,8 @@ func main() { interruptCtx, cancel := cmdutils.NewInterruptContext() defer cancel() - fmt.Println("Starting worker with OpenTelemetry instrumentation...") - go func() { <-interruptCtx.Done() - // Flush remaining spans before exit if shutdownErr := instrumentor.Shutdown(context.Background()); shutdownErr != nil { log.Printf("failed to shutdown instrumentor: %v", shutdownErr) } diff --git a/frontend/app/src/lib/api/generated/Api.ts b/frontend/app/src/lib/api/generated/Api.ts index b44e020e0c..d7bf4a896e 100644 --- a/frontend/app/src/lib/api/generated/Api.ts +++ b/frontend/app/src/lib/api/generated/Api.ts @@ -268,10 +268,26 @@ export class Api< * @request GET:/api/v1/stable/tasks/{task}/trace * @secure */ - v1TaskGetTrace = (task: string, params: RequestParams = {}) => + v1TaskGetTrace = ( + task: string, + query?: { + /** + * The number to skip + * @format int64 + */ + offset?: number; + /** + * The number to limit by + * @format int64 + */ + limit?: number; + }, + params: RequestParams = {}, + ) => this.request({ path: `/api/v1/stable/tasks/${task}/trace`, method: "GET", + query: query, secure: true, format: "json", ...params, diff --git a/frontend/app/src/lib/api/generated/data-contracts.ts b/frontend/app/src/lib/api/generated/data-contracts.ts index e81c6219ff..2ef371de5c 100644 --- a/frontend/app/src/lib/api/generated/data-contracts.ts +++ b/frontend/app/src/lib/api/generated/data-contracts.ts @@ -536,6 +536,7 @@ export interface OtelSpan { } export interface OtelSpanList { + pagination?: PaginationResponse; rows?: OtelSpan[]; } diff --git a/frontend/app/src/pages/main/v1/workflow-runs-v1/$run/v2components/step-run-detail/task-run-trace.tsx b/frontend/app/src/pages/main/v1/workflow-runs-v1/$run/v2components/step-run-detail/task-run-trace.tsx index 39aacd6fe4..9362c7aa1b 100644 --- a/frontend/app/src/pages/main/v1/workflow-runs-v1/$run/v2components/step-run-detail/task-run-trace.tsx +++ b/frontend/app/src/pages/main/v1/workflow-runs-v1/$run/v2components/step-run-detail/task-run-trace.tsx @@ -2,11 +2,41 @@ import { convertOtelSpans } from './otel-span-adapter'; import { TreeView } from '@/components/v1/agent-prism/TreeView'; import { Loading } from '@/components/v1/ui/loading'; import api from '@/lib/api/api'; +import { OtelSpan } from '@/lib/api/generated/data-contracts'; import { openTelemetrySpanAdapter } from '@evilmartians/agent-prism-data'; import { flattenSpans } from '@evilmartians/agent-prism-data'; import { useQuery } from '@tanstack/react-query'; import { useEffect, useMemo, useState } from 'react'; +const PAGE_SIZE = 200; + +async function fetchAllSpans(taskExternalId: string): Promise { + const allSpans: OtelSpan[] = []; + let offset = 0; + + // eslint-disable-next-line no-constant-condition + while (true) { + const res = await api.v1TaskGetTrace(taskExternalId, { + offset, + limit: PAGE_SIZE, + }); + + const rows = res.data.rows ?? []; + allSpans.push(...rows); + + const numPages = res.data.pagination?.num_pages ?? 1; + const currentPage = res.data.pagination?.current_page ?? 1; + + if (currentPage >= numPages || rows.length === 0) { + break; + } + + offset += PAGE_SIZE; + } + + return allSpans; +} + export function TaskRunTrace({ taskExternalId, isRunning, @@ -16,15 +46,12 @@ export function TaskRunTrace({ }) { const tracesQuery = useQuery({ queryKey: ['task:trace', taskExternalId], - queryFn: async () => { - const res = await api.v1TaskGetTrace(taskExternalId); - return res.data; - }, + queryFn: () => fetchAllSpans(taskExternalId), refetchInterval: isRunning ? 100 : false, }); const traceSpans = useMemo(() => { - const rows = tracesQuery.data?.rows; + const rows = tracesQuery.data; if (!rows || rows.length === 0) { return []; } diff --git a/internal/services/otelcol/server.go b/internal/services/otelcol/server.go index 26422ed40e..089c9ac6e1 100644 --- a/internal/services/otelcol/server.go +++ b/internal/services/otelcol/server.go @@ -83,10 +83,10 @@ func (oc *otelCollectorImpl) convertOTLPToSpanData(resourceSpans []*tracev1.Reso SpanID: span.GetSpanId(), ParentSpanID: span.GetParentSpanId(), Name: span.GetName(), - Kind: int32(span.GetKind()), + Kind: span.GetKind(), StartTimeUnixNano: span.GetStartTimeUnixNano(), EndTimeUnixNano: span.GetEndTimeUnixNano(), - StatusCode: int32(span.GetStatus().GetCode()), + StatusCode: span.GetStatus().GetCode(), StatusMessage: span.GetStatus().GetMessage(), Attributes: oc.serializeAttributes(span.GetAttributes()), Events: oc.serializeEvents(span.GetEvents()), diff --git a/pkg/client/rest/gen.go b/pkg/client/rest/gen.go index 7e6e44d6ad..5c4312f74b 100644 --- a/pkg/client/rest/gen.go +++ b/pkg/client/rest/gen.go @@ -775,7 +775,8 @@ type OtelSpan struct { // OtelSpanList defines model for OtelSpanList. type OtelSpanList struct { - Rows *[]OtelSpan `json:"rows,omitempty"` + Pagination *PaginationResponse `json:"pagination,omitempty"` + Rows *[]OtelSpan `json:"rows,omitempty"` } // PaginationResponse defines model for PaginationResponse. @@ -2517,6 +2518,15 @@ type V1TaskEventListParams struct { Limit *int64 `form:"limit,omitempty" json:"limit,omitempty"` } +// V1TaskGetTraceParams defines parameters for V1TaskGetTrace. +type V1TaskGetTraceParams struct { + // Offset The number to skip + Offset *int64 `form:"offset,omitempty" json:"offset,omitempty"` + + // Limit The number to limit by + Limit *int64 `form:"limit,omitempty" json:"limit,omitempty"` +} + // V1EventListParams defines parameters for V1EventList. type V1EventListParams struct { // Offset The number to skip @@ -3290,7 +3300,7 @@ type ClientInterface interface { V1TaskEventList(ctx context.Context, task openapi_types.UUID, params *V1TaskEventListParams, reqEditors ...RequestEditorFn) (*http.Response, error) // V1TaskGetTrace request - V1TaskGetTrace(ctx context.Context, task openapi_types.UUID, reqEditors ...RequestEditorFn) (*http.Response, error) + V1TaskGetTrace(ctx context.Context, task openapi_types.UUID, params *V1TaskGetTraceParams, reqEditors ...RequestEditorFn) (*http.Response, error) // V1CelDebugWithBody request with any body V1CelDebugWithBody(ctx context.Context, tenant openapi_types.UUID, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) @@ -3945,8 +3955,8 @@ func (c *Client) V1TaskEventList(ctx context.Context, task openapi_types.UUID, p return c.Client.Do(req) } -func (c *Client) V1TaskGetTrace(ctx context.Context, task openapi_types.UUID, reqEditors ...RequestEditorFn) (*http.Response, error) { - req, err := NewV1TaskGetTraceRequest(c.Server, task) +func (c *Client) V1TaskGetTrace(ctx context.Context, task openapi_types.UUID, params *V1TaskGetTraceParams, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewV1TaskGetTraceRequest(c.Server, task, params) if err != nil { return nil, err } @@ -6604,7 +6614,7 @@ func NewV1TaskEventListRequest(server string, task openapi_types.UUID, params *V } // NewV1TaskGetTraceRequest generates requests for V1TaskGetTrace -func NewV1TaskGetTraceRequest(server string, task openapi_types.UUID) (*http.Request, error) { +func NewV1TaskGetTraceRequest(server string, task openapi_types.UUID, params *V1TaskGetTraceParams) (*http.Request, error) { var err error var pathParam0 string @@ -6629,6 +6639,44 @@ func NewV1TaskGetTraceRequest(server string, task openapi_types.UUID) (*http.Req return nil, err } + if params != nil { + queryValues := queryURL.Query() + + if params.Offset != nil { + + if queryFrag, err := runtime.StyleParamWithLocation("form", true, "offset", runtime.ParamLocationQuery, *params.Offset); err != nil { + return nil, err + } else if parsed, err := url.ParseQuery(queryFrag); err != nil { + return nil, err + } else { + for k, v := range parsed { + for _, v2 := range v { + queryValues.Add(k, v2) + } + } + } + + } + + if params.Limit != nil { + + if queryFrag, err := runtime.StyleParamWithLocation("form", true, "limit", runtime.ParamLocationQuery, *params.Limit); err != nil { + return nil, err + } else if parsed, err := url.ParseQuery(queryFrag); err != nil { + return nil, err + } else { + for k, v := range parsed { + for _, v2 := range v { + queryValues.Add(k, v2) + } + } + } + + } + + queryURL.RawQuery = queryValues.Encode() + } + req, err := http.NewRequest("GET", queryURL.String(), nil) if err != nil { return nil, err @@ -13341,7 +13389,7 @@ type ClientWithResponsesInterface interface { V1TaskEventListWithResponse(ctx context.Context, task openapi_types.UUID, params *V1TaskEventListParams, reqEditors ...RequestEditorFn) (*V1TaskEventListResponse, error) // V1TaskGetTraceWithResponse request - V1TaskGetTraceWithResponse(ctx context.Context, task openapi_types.UUID, reqEditors ...RequestEditorFn) (*V1TaskGetTraceResponse, error) + V1TaskGetTraceWithResponse(ctx context.Context, task openapi_types.UUID, params *V1TaskGetTraceParams, reqEditors ...RequestEditorFn) (*V1TaskGetTraceResponse, error) // V1CelDebugWithBodyWithResponse request with any body V1CelDebugWithBodyWithResponse(ctx context.Context, tenant openapi_types.UUID, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*V1CelDebugResponse, error) @@ -17217,8 +17265,8 @@ func (c *ClientWithResponses) V1TaskEventListWithResponse(ctx context.Context, t } // V1TaskGetTraceWithResponse request returning *V1TaskGetTraceResponse -func (c *ClientWithResponses) V1TaskGetTraceWithResponse(ctx context.Context, task openapi_types.UUID, reqEditors ...RequestEditorFn) (*V1TaskGetTraceResponse, error) { - rsp, err := c.V1TaskGetTrace(ctx, task, reqEditors...) +func (c *ClientWithResponses) V1TaskGetTraceWithResponse(ctx context.Context, task openapi_types.UUID, params *V1TaskGetTraceParams, reqEditors ...RequestEditorFn) (*V1TaskGetTraceResponse, error) { + rsp, err := c.V1TaskGetTrace(ctx, task, params, reqEditors...) if err != nil { return nil, err } diff --git a/pkg/repository/otelcol.go b/pkg/repository/otelcol.go index 4f81c3143a..fc7b8a066f 100644 --- a/pkg/repository/otelcol.go +++ b/pkg/repository/otelcol.go @@ -8,28 +8,31 @@ import ( "time" "github.com/google/uuid" - "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgtype" tracev1 "go.opentelemetry.io/proto/otlp/trace/v1" + + "github.com/hatchet-dev/hatchet/pkg/repository/sqlchelpers" + "github.com/hatchet-dev/hatchet/pkg/repository/sqlcv1" ) type SpanData struct { + TenantID uuid.UUID WorkflowRunID *uuid.UUID TaskRunExternalID *uuid.UUID StatusMessage string InstrumentationScope string Name string - ResourceAttributes []byte - Attributes []byte - Events []byte - Links []byte + ResourceAttributes json.RawMessage + Attributes json.RawMessage + Events json.RawMessage + Links json.RawMessage TraceID []byte ParentSpanID []byte SpanID []byte EndTimeUnixNano uint64 StartTimeUnixNano uint64 - StatusCode int32 - Kind int32 - TenantID uuid.UUID + StatusCode tracev1.Status_StatusCode + Kind tracev1.Span_SpanKind } type CreateSpansOpts struct { @@ -37,26 +40,14 @@ type CreateSpansOpts struct { Spans []*SpanData } -type OtelSpanRow struct { - CreatedAt time.Time - SpanAttributes map[string]string - ResourceAttributes map[string]string - SpanName string - SpanKind string - ServiceName string - StatusCode string - StatusMessage string - TraceID string - ParentSpanID string - SpanID string - ScopeName string - ScopeVersion string - Duration uint64 +type ListSpansResult struct { + Rows []*sqlcv1.ListSpansByTaskExternalIDRow + Total int64 } type OTelCollectorRepository interface { CreateSpans(ctx context.Context, tenantId uuid.UUID, opts *CreateSpansOpts) error - ListSpansByTaskExternalID(ctx context.Context, tenantId, taskExternalID uuid.UUID) ([]*OtelSpanRow, error) + ListSpansByTaskExternalID(ctx context.Context, tenantId, taskExternalID uuid.UUID, offset, limit int64) (*ListSpansResult, error) } type otelCollectorRepositoryImpl struct { @@ -69,65 +60,6 @@ func newOTelCollectorRepository(s *sharedRepository) OTelCollectorRepository { } } -// transformedSpan holds pre-processed span data ready for insertion. -type transformedSpan struct { - startTime time.Time - taskRunExternalID *uuid.UUID - workflowRunExternalID *uuid.UUID - statusMessage string - scopeVersion string - spanKind string - serviceName string - statusCode string - traceID string - spanID string - parentSpanID string - spanName string - scopeName string - spanAttributes []byte - resourceAttributes []byte - durationNs int64 - tenantID uuid.UUID -} - -// spanCopyFromSource implements pgx.CopyFromSource for batch inserts. -type spanCopyFromSource struct { - spans []transformedSpan - idx int -} - -func (s *spanCopyFromSource) Next() bool { - s.idx++ - return s.idx < len(s.spans) -} - -func (s *spanCopyFromSource) Values() ([]interface{}, error) { - span := s.spans[s.idx] - return []interface{}{ - span.tenantID, - span.traceID, - span.spanID, - span.parentSpanID, - span.spanName, - span.spanKind, - span.serviceName, - span.statusCode, - span.statusMessage, - span.durationNs, - span.resourceAttributes, - span.spanAttributes, - span.scopeName, - span.scopeVersion, - span.taskRunExternalID, - span.workflowRunExternalID, - span.startTime, - }, nil -} - -func (s *spanCopyFromSource) Err() error { - return nil -} - func (o *otelCollectorRepositoryImpl) CreateSpans(ctx context.Context, tenantId uuid.UUID, opts *CreateSpansOpts) error { if opts == nil { return fmt.Errorf("opts cannot be nil") @@ -141,125 +73,98 @@ func (o *otelCollectorRepositoryImpl) CreateSpans(ctx context.Context, tenantId return nil } - transformed := make([]transformedSpan, 0, len(opts.Spans)) - for _, sd := range opts.Spans { - ts := transformedSpan{ - tenantID: tenantId, - traceID: hex.EncodeToString(sd.TraceID), - spanID: hex.EncodeToString(sd.SpanID), - spanName: sd.Name, - spanKind: spanKindToString(sd.Kind), - serviceName: extractServiceName(sd.ResourceAttributes), - statusCode: spanStatusCodeToString(sd.StatusCode), - statusMessage: sd.StatusMessage, - durationNs: int64(sd.EndTimeUnixNano - sd.StartTimeUnixNano), //nolint:gosec - scopeName: sd.InstrumentationScope, - startTime: time.Unix(0, int64(sd.StartTimeUnixNano)), //nolint:gosec + params := make([]sqlcv1.InsertOtelSpansParams, len(opts.Spans)) + for i, sd := range opts.Spans { + var parentSpanID string + if len(sd.ParentSpanID) > 0 { + parentSpanID = hex.EncodeToString(sd.ParentSpanID) } - if len(sd.ParentSpanID) > 0 { - ts.parentSpanID = hex.EncodeToString(sd.ParentSpanID) + resourceAttrs := []byte(sd.ResourceAttributes) + if len(resourceAttrs) == 0 { + resourceAttrs = []byte("{}") } - ts.resourceAttributes = jsonBytesToJSONB(sd.ResourceAttributes) - ts.spanAttributes = jsonBytesToJSONB(sd.Attributes) + spanAttrs := []byte(sd.Attributes) + if len(spanAttrs) == 0 { + spanAttrs = []byte("{}") + } + var taskRunExternalID *uuid.UUID if sd.TaskRunExternalID != nil && *sd.TaskRunExternalID != uuid.Nil { id := *sd.TaskRunExternalID - ts.taskRunExternalID = &id + taskRunExternalID = &id } + var workflowRunExternalID *uuid.UUID if sd.WorkflowRunID != nil && *sd.WorkflowRunID != uuid.Nil { id := *sd.WorkflowRunID - ts.workflowRunExternalID = &id + workflowRunExternalID = &id } - transformed = append(transformed, ts) + startTime := time.Unix(0, int64(sd.StartTimeUnixNano)) //nolint:gosec + + params[i] = sqlcv1.InsertOtelSpansParams{ + TenantID: tenantId, + TraceID: hex.EncodeToString(sd.TraceID), + SpanID: hex.EncodeToString(sd.SpanID), + ParentSpanID: parentSpanID, + SpanName: sd.Name, + SpanKind: protoSpanKindToDB(sd.Kind), + ServiceName: extractServiceName(sd.ResourceAttributes), + StatusCode: protoStatusCodeToDB(sd.StatusCode), + StatusMessage: sd.StatusMessage, + DurationNs: int64(sd.EndTimeUnixNano - sd.StartTimeUnixNano), //nolint:gosec + ResourceAttributes: resourceAttrs, + SpanAttributes: spanAttrs, + ScopeName: sd.InstrumentationScope, + TaskRunExternalID: taskRunExternalID, + WorkflowRunExternalID: workflowRunExternalID, + StartTime: pgtype.Timestamptz{Time: startTime, Valid: true}, + } } - _, err := o.pool.CopyFrom( - ctx, - pgx.Identifier{"v1_otel_traces"}, - []string{ - "tenant_id", "trace_id", "span_id", "parent_span_id", - "span_name", "span_kind", "service_name", "status_code", - "status_message", "duration_ns", "resource_attributes", - "span_attributes", "scope_name", "scope_version", - "task_run_external_id", "workflow_run_external_id", "start_time", - }, - &spanCopyFromSource{spans: transformed, idx: -1}, - ) - + _, err := o.queries.InsertOtelSpans(ctx, o.pool, params) if err != nil { - return fmt.Errorf("error copying spans to v1_otel_traces: %w", err) + return fmt.Errorf("error inserting otel spans: %w", err) } return nil } -func (o *otelCollectorRepositoryImpl) ListSpansByTaskExternalID(ctx context.Context, tenantId, taskExternalID uuid.UUID) ([]*OtelSpanRow, error) { - query := ` - SELECT - trace_id, span_id, parent_span_id, span_name, span_kind, - service_name, status_code, status_message, duration_ns, start_time, - resource_attributes, span_attributes, scope_name, scope_version - FROM v1_otel_traces - WHERE tenant_id = $1 AND trace_id IN ( - SELECT DISTINCT trace_id FROM v1_otel_traces - WHERE tenant_id = $1 AND task_run_external_id = $2 - ) - ORDER BY start_time ASC - LIMIT 1000 - ` - - rows, err := o.pool.Query(ctx, query, tenantId, taskExternalID) +func (o *otelCollectorRepositoryImpl) ListSpansByTaskExternalID(ctx context.Context, tenantId, taskExternalID uuid.UUID, offset, limit int64) (*ListSpansResult, error) { + tx, commit, rollback, err := sqlchelpers.PrepareTx(ctx, o.pool, o.l) if err != nil { - return nil, fmt.Errorf("error querying v1_otel_traces: %w", err) + return nil, fmt.Errorf("error starting transaction: %w", err) } - defer rows.Close() - - var result []*OtelSpanRow - for rows.Next() { - row := &OtelSpanRow{} - var durationNs int64 - var resourceAttrsJSON, spanAttrsJSON []byte - - if err := rows.Scan( - &row.TraceID, - &row.SpanID, - &row.ParentSpanID, - &row.SpanName, - &row.SpanKind, - &row.ServiceName, - &row.StatusCode, - &row.StatusMessage, - &durationNs, - &row.CreatedAt, - &resourceAttrsJSON, - &spanAttrsJSON, - &row.ScopeName, - &row.ScopeVersion, - ); err != nil { - return nil, fmt.Errorf("error scanning v1_otel_traces row: %w", err) - } + defer rollback() - row.Duration = uint64(durationNs) //nolint:gosec - row.ResourceAttributes = jsonbToStringMap(resourceAttrsJSON) - row.SpanAttributes = jsonbToStringMap(spanAttrsJSON) + total, err := o.queries.CountSpansByTaskExternalID(ctx, tx, sqlcv1.CountSpansByTaskExternalIDParams{ + Tenantid: tenantId, + Taskexternalid: taskExternalID, + }) + if err != nil { + return nil, fmt.Errorf("error counting otel spans: %w", err) + } - result = append(result, row) + rows, err := o.queries.ListSpansByTaskExternalID(ctx, tx, sqlcv1.ListSpansByTaskExternalIDParams{ + Tenantid: tenantId, + Taskexternalid: taskExternalID, + Spanoffset: offset, + Spanlimit: limit, + }) + if err != nil { + return nil, fmt.Errorf("error listing otel spans: %w", err) } - if err := rows.Err(); err != nil { - return nil, fmt.Errorf("error iterating v1_otel_traces rows: %w", err) + if err := commit(ctx); err != nil { + return nil, fmt.Errorf("error committing transaction: %w", err) } - return result, nil + return &ListSpansResult{Rows: rows, Total: total}, nil } -// Helper functions for data transformation - -func extractServiceName(resourceAttrsJSON []byte) string { +func extractServiceName(resourceAttrsJSON json.RawMessage) string { if len(resourceAttrsJSON) == 0 { return "unknown" } @@ -276,76 +181,30 @@ func extractServiceName(resourceAttrsJSON []byte) string { return "unknown" } -func jsonBytesToJSONB(jsonBytes []byte) []byte { - if len(jsonBytes) == 0 { - return []byte("{}") - } - // Validate it's valid JSON; if not, return empty object - var raw json.RawMessage - if err := json.Unmarshal(jsonBytes, &raw); err != nil { - return []byte("{}") - } - return jsonBytes -} - -func jsonbToStringMap(jsonBytes []byte) map[string]string { - if len(jsonBytes) == 0 { - return make(map[string]string) - } - - var jsonMap map[string]interface{} - if err := json.Unmarshal(jsonBytes, &jsonMap); err != nil { - return make(map[string]string) - } - - result := make(map[string]string, len(jsonMap)) - for k, v := range jsonMap { - switch val := v.(type) { - case string: - result[k] = val - case nil: - result[k] = "" - default: - b, err := json.Marshal(v) - if err != nil { - result[k] = fmt.Sprintf("%v", v) - } else { - result[k] = string(b) - } - } - } - - return result -} - -func spanKindToString(kind int32) string { - switch tracev1.Span_SpanKind(kind) { - case tracev1.Span_SPAN_KIND_UNSPECIFIED: - return "UNSPECIFIED" +func protoSpanKindToDB(kind tracev1.Span_SpanKind) sqlcv1.V1OtelSpanKind { + switch kind { case tracev1.Span_SPAN_KIND_INTERNAL: - return "INTERNAL" + return sqlcv1.V1OtelSpanKindINTERNAL case tracev1.Span_SPAN_KIND_SERVER: - return "SERVER" + return sqlcv1.V1OtelSpanKindSERVER case tracev1.Span_SPAN_KIND_CLIENT: - return "CLIENT" + return sqlcv1.V1OtelSpanKindCLIENT case tracev1.Span_SPAN_KIND_PRODUCER: - return "PRODUCER" + return sqlcv1.V1OtelSpanKindPRODUCER case tracev1.Span_SPAN_KIND_CONSUMER: - return "CONSUMER" + return sqlcv1.V1OtelSpanKindCONSUMER default: - return "UNKNOWN" + return sqlcv1.V1OtelSpanKindUNSPECIFIED } } -func spanStatusCodeToString(code int32) string { - switch tracev1.Status_StatusCode(code) { - case tracev1.Status_STATUS_CODE_UNSET: - return "UNSET" +func protoStatusCodeToDB(code tracev1.Status_StatusCode) sqlcv1.V1OtelStatusCode { + switch code { case tracev1.Status_STATUS_CODE_OK: - return "OK" + return sqlcv1.V1OtelStatusCodeOK case tracev1.Status_STATUS_CODE_ERROR: - return "ERROR" + return sqlcv1.V1OtelStatusCodeERROR default: - return "UNKNOWN" + return sqlcv1.V1OtelStatusCodeUNSET } } diff --git a/pkg/repository/sqlcv1/copyfrom.go b/pkg/repository/sqlcv1/copyfrom.go index f277818094..c0dce3b85a 100644 --- a/pkg/repository/sqlcv1/copyfrom.go +++ b/pkg/repository/sqlcv1/copyfrom.go @@ -371,3 +371,51 @@ func (r iteratorForInsertLogLine) Err() error { func (q *Queries) InsertLogLine(ctx context.Context, db DBTX, arg []InsertLogLineParams) (int64, error) { return db.CopyFrom(ctx, []string{"v1_log_line"}, []string{"tenant_id", "task_id", "task_inserted_at", "message", "metadata", "retry_count", "level"}, &iteratorForInsertLogLine{rows: arg}) } + +// iteratorForInsertOtelSpans implements pgx.CopyFromSource. +type iteratorForInsertOtelSpans struct { + rows []InsertOtelSpansParams + skippedFirstNextCall bool +} + +func (r *iteratorForInsertOtelSpans) Next() bool { + if len(r.rows) == 0 { + return false + } + if !r.skippedFirstNextCall { + r.skippedFirstNextCall = true + return true + } + r.rows = r.rows[1:] + return len(r.rows) > 0 +} + +func (r iteratorForInsertOtelSpans) Values() ([]interface{}, error) { + return []interface{}{ + r.rows[0].TenantID, + r.rows[0].TraceID, + r.rows[0].SpanID, + r.rows[0].ParentSpanID, + r.rows[0].SpanName, + r.rows[0].SpanKind, + r.rows[0].ServiceName, + r.rows[0].StatusCode, + r.rows[0].StatusMessage, + r.rows[0].DurationNs, + r.rows[0].ResourceAttributes, + r.rows[0].SpanAttributes, + r.rows[0].ScopeName, + r.rows[0].ScopeVersion, + r.rows[0].TaskRunExternalID, + r.rows[0].WorkflowRunExternalID, + r.rows[0].StartTime, + }, nil +} + +func (r iteratorForInsertOtelSpans) Err() error { + return nil +} + +func (q *Queries) InsertOtelSpans(ctx context.Context, db DBTX, arg []InsertOtelSpansParams) (int64, error) { + return db.CopyFrom(ctx, []string{"v1_otel_traces"}, []string{"tenant_id", "trace_id", "span_id", "parent_span_id", "span_name", "span_kind", "service_name", "status_code", "status_message", "duration_ns", "resource_attributes", "span_attributes", "scope_name", "scope_version", "task_run_external_id", "workflow_run_external_id", "start_time"}, &iteratorForInsertOtelSpans{rows: arg}) +} diff --git a/pkg/repository/sqlcv1/models.go b/pkg/repository/sqlcv1/models.go index 5c8410f8ad..b26b90095b 100644 --- a/pkg/repository/sqlcv1/models.go +++ b/pkg/repository/sqlcv1/models.go @@ -1397,6 +1397,95 @@ func (ns NullV1MatchKind) Value() (driver.Value, error) { return string(ns.V1MatchKind), nil } +type V1OtelSpanKind string + +const ( + V1OtelSpanKindUNSPECIFIED V1OtelSpanKind = "UNSPECIFIED" + V1OtelSpanKindINTERNAL V1OtelSpanKind = "INTERNAL" + V1OtelSpanKindSERVER V1OtelSpanKind = "SERVER" + V1OtelSpanKindCLIENT V1OtelSpanKind = "CLIENT" + V1OtelSpanKindPRODUCER V1OtelSpanKind = "PRODUCER" + V1OtelSpanKindCONSUMER V1OtelSpanKind = "CONSUMER" +) + +func (e *V1OtelSpanKind) Scan(src interface{}) error { + switch s := src.(type) { + case []byte: + *e = V1OtelSpanKind(s) + case string: + *e = V1OtelSpanKind(s) + default: + return fmt.Errorf("unsupported scan type for V1OtelSpanKind: %T", src) + } + return nil +} + +type NullV1OtelSpanKind struct { + V1OtelSpanKind V1OtelSpanKind `json:"v1_otel_span_kind"` + Valid bool `json:"valid"` // Valid is true if V1OtelSpanKind is not NULL +} + +// Scan implements the Scanner interface. +func (ns *NullV1OtelSpanKind) Scan(value interface{}) error { + if value == nil { + ns.V1OtelSpanKind, ns.Valid = "", false + return nil + } + ns.Valid = true + return ns.V1OtelSpanKind.Scan(value) +} + +// Value implements the driver Valuer interface. +func (ns NullV1OtelSpanKind) Value() (driver.Value, error) { + if !ns.Valid { + return nil, nil + } + return string(ns.V1OtelSpanKind), nil +} + +type V1OtelStatusCode string + +const ( + V1OtelStatusCodeUNSET V1OtelStatusCode = "UNSET" + V1OtelStatusCodeOK V1OtelStatusCode = "OK" + V1OtelStatusCodeERROR V1OtelStatusCode = "ERROR" +) + +func (e *V1OtelStatusCode) Scan(src interface{}) error { + switch s := src.(type) { + case []byte: + *e = V1OtelStatusCode(s) + case string: + *e = V1OtelStatusCode(s) + default: + return fmt.Errorf("unsupported scan type for V1OtelStatusCode: %T", src) + } + return nil +} + +type NullV1OtelStatusCode struct { + V1OtelStatusCode V1OtelStatusCode `json:"v1_otel_status_code"` + Valid bool `json:"valid"` // Valid is true if V1OtelStatusCode is not NULL +} + +// Scan implements the Scanner interface. +func (ns *NullV1OtelStatusCode) Scan(value interface{}) error { + if value == nil { + ns.V1OtelStatusCode, ns.Valid = "", false + return nil + } + ns.Valid = true + return ns.V1OtelStatusCode.Scan(value) +} + +// Value implements the driver Valuer interface. +func (ns NullV1OtelStatusCode) Value() (driver.Value, error) { + if !ns.Valid { + return nil, nil + } + return string(ns.V1OtelStatusCode), nil +} + type V1PayloadLocation string const ( @@ -3210,9 +3299,9 @@ type V1OtelTraces struct { SpanID string `json:"span_id"` ParentSpanID string `json:"parent_span_id"` SpanName string `json:"span_name"` - SpanKind string `json:"span_kind"` + SpanKind V1OtelSpanKind `json:"span_kind"` ServiceName string `json:"service_name"` - StatusCode string `json:"status_code"` + StatusCode V1OtelStatusCode `json:"status_code"` StatusMessage string `json:"status_message"` DurationNs int64 `json:"duration_ns"` ResourceAttributes []byte `json:"resource_attributes"` diff --git a/pkg/repository/sqlcv1/otel.sql b/pkg/repository/sqlcv1/otel.sql new file mode 100644 index 0000000000..a8b6c9ccd1 --- /dev/null +++ b/pkg/repository/sqlcv1/otel.sql @@ -0,0 +1,43 @@ +-- name: InsertOtelSpans :copyfrom +INSERT INTO v1_otel_traces ( + tenant_id, + trace_id, + span_id, + parent_span_id, + span_name, + span_kind, + service_name, + status_code, + status_message, + duration_ns, + resource_attributes, + span_attributes, + scope_name, + scope_version, + task_run_external_id, + workflow_run_external_id, + start_time +) VALUES ( + $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17 +); + +-- name: CountSpansByTaskExternalID :one +SELECT COUNT(*) FROM v1_otel_traces +WHERE tenant_id = @tenantId::UUID AND trace_id IN ( + SELECT DISTINCT trace_id FROM v1_otel_traces + WHERE tenant_id = @tenantId::UUID AND task_run_external_id = @taskExternalId::UUID +); + +-- name: ListSpansByTaskExternalID :many +SELECT + trace_id, span_id, parent_span_id, span_name, span_kind, + service_name, status_code, status_message, duration_ns, start_time, + resource_attributes, span_attributes, scope_name, scope_version +FROM v1_otel_traces +WHERE tenant_id = @tenantId::UUID AND trace_id IN ( + SELECT DISTINCT trace_id FROM v1_otel_traces + WHERE tenant_id = @tenantId::UUID AND task_run_external_id = @taskExternalId::UUID +) +ORDER BY start_time ASC +OFFSET COALESCE(@spanOffset::BIGINT, 0) +LIMIT COALESCE(@spanLimit::BIGINT, 1000); diff --git a/pkg/repository/sqlcv1/otel.sql.go b/pkg/repository/sqlcv1/otel.sql.go new file mode 100644 index 0000000000..de79ad0552 --- /dev/null +++ b/pkg/repository/sqlcv1/otel.sql.go @@ -0,0 +1,132 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.29.0 +// source: otel.sql + +package sqlcv1 + +import ( + "context" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgtype" +) + +const countSpansByTaskExternalID = `-- name: CountSpansByTaskExternalID :one +SELECT COUNT(*) FROM v1_otel_traces +WHERE tenant_id = $1::UUID AND trace_id IN ( + SELECT DISTINCT trace_id FROM v1_otel_traces + WHERE tenant_id = $1::UUID AND task_run_external_id = $2::UUID +) +` + +type CountSpansByTaskExternalIDParams struct { + Tenantid uuid.UUID `json:"tenantid"` + Taskexternalid uuid.UUID `json:"taskexternalid"` +} + +func (q *Queries) CountSpansByTaskExternalID(ctx context.Context, db DBTX, arg CountSpansByTaskExternalIDParams) (int64, error) { + row := db.QueryRow(ctx, countSpansByTaskExternalID, arg.Tenantid, arg.Taskexternalid) + var count int64 + err := row.Scan(&count) + return count, err +} + +type InsertOtelSpansParams struct { + TenantID uuid.UUID `json:"tenant_id"` + TraceID string `json:"trace_id"` + SpanID string `json:"span_id"` + ParentSpanID string `json:"parent_span_id"` + SpanName string `json:"span_name"` + SpanKind V1OtelSpanKind `json:"span_kind"` + ServiceName string `json:"service_name"` + StatusCode V1OtelStatusCode `json:"status_code"` + StatusMessage string `json:"status_message"` + DurationNs int64 `json:"duration_ns"` + ResourceAttributes []byte `json:"resource_attributes"` + SpanAttributes []byte `json:"span_attributes"` + ScopeName string `json:"scope_name"` + ScopeVersion string `json:"scope_version"` + TaskRunExternalID *uuid.UUID `json:"task_run_external_id"` + WorkflowRunExternalID *uuid.UUID `json:"workflow_run_external_id"` + StartTime pgtype.Timestamptz `json:"start_time"` +} + +const listSpansByTaskExternalID = `-- name: ListSpansByTaskExternalID :many +SELECT + trace_id, span_id, parent_span_id, span_name, span_kind, + service_name, status_code, status_message, duration_ns, start_time, + resource_attributes, span_attributes, scope_name, scope_version +FROM v1_otel_traces +WHERE tenant_id = $1::UUID AND trace_id IN ( + SELECT DISTINCT trace_id FROM v1_otel_traces + WHERE tenant_id = $1::UUID AND task_run_external_id = $2::UUID +) +ORDER BY start_time ASC +OFFSET COALESCE($3::BIGINT, 0) +LIMIT COALESCE($4::BIGINT, 1000) +` + +type ListSpansByTaskExternalIDParams struct { + Tenantid uuid.UUID `json:"tenantid"` + Taskexternalid uuid.UUID `json:"taskexternalid"` + Spanoffset int64 `json:"spanoffset"` + Spanlimit int64 `json:"spanlimit"` +} + +type ListSpansByTaskExternalIDRow struct { + TraceID string `json:"trace_id"` + SpanID string `json:"span_id"` + ParentSpanID string `json:"parent_span_id"` + SpanName string `json:"span_name"` + SpanKind V1OtelSpanKind `json:"span_kind"` + ServiceName string `json:"service_name"` + StatusCode V1OtelStatusCode `json:"status_code"` + StatusMessage string `json:"status_message"` + DurationNs int64 `json:"duration_ns"` + StartTime pgtype.Timestamptz `json:"start_time"` + ResourceAttributes []byte `json:"resource_attributes"` + SpanAttributes []byte `json:"span_attributes"` + ScopeName string `json:"scope_name"` + ScopeVersion string `json:"scope_version"` +} + +func (q *Queries) ListSpansByTaskExternalID(ctx context.Context, db DBTX, arg ListSpansByTaskExternalIDParams) ([]*ListSpansByTaskExternalIDRow, error) { + rows, err := db.Query(ctx, listSpansByTaskExternalID, + arg.Tenantid, + arg.Taskexternalid, + arg.Spanoffset, + arg.Spanlimit, + ) + if err != nil { + return nil, err + } + defer rows.Close() + var items []*ListSpansByTaskExternalIDRow + for rows.Next() { + var i ListSpansByTaskExternalIDRow + if err := rows.Scan( + &i.TraceID, + &i.SpanID, + &i.ParentSpanID, + &i.SpanName, + &i.SpanKind, + &i.ServiceName, + &i.StatusCode, + &i.StatusMessage, + &i.DurationNs, + &i.StartTime, + &i.ResourceAttributes, + &i.SpanAttributes, + &i.ScopeName, + &i.ScopeVersion, + ); err != nil { + return nil, err + } + items = append(items, &i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} diff --git a/pkg/repository/sqlcv1/sqlc.yaml b/pkg/repository/sqlcv1/sqlc.yaml index 966444d156..9417ee6c57 100644 --- a/pkg/repository/sqlcv1/sqlc.yaml +++ b/pkg/repository/sqlcv1/sqlc.yaml @@ -20,6 +20,7 @@ sql: - olap.sql - rate_limits.sql - log_line.sql + - otel.sql - security_check.sql - slack.sql - sleep.sql diff --git a/sdks/go/examples/opentelemetry-propagation/main.go b/sdks/go/examples/opentelemetry-propagation/main.go deleted file mode 100644 index e431e9682b..0000000000 --- a/sdks/go/examples/opentelemetry-propagation/main.go +++ /dev/null @@ -1,147 +0,0 @@ -package main - -import ( - "context" - "fmt" - "log" - "math/rand" - "time" - - "go.opentelemetry.io/otel" - - "github.com/hatchet-dev/hatchet/pkg/cmdutils" - hatchet "github.com/hatchet-dev/hatchet/sdks/go" - hatchetotel "github.com/hatchet-dev/hatchet/sdks/go/opentelemetry" -) - -// This example demonstrates cross-workflow trace propagation. -// A parent task spawns a child task via .Run(), and both tasks' spans -// appear under the same trace in the UI — even if they run on different workers. - -type ParentInput struct { - Name string `json:"name"` -} - -type ParentOutput struct { - ChildResult string `json:"child_result"` -} - -type ChildInput struct { - Greeting string `json:"greeting"` -} - -type ChildOutput struct { - Message string `json:"message"` -} - -func main() { - client, err := hatchet.NewClient() - if err != nil { - log.Fatalf("failed to create client: %v", err) - } - - instrumentor, err := hatchetotel.NewInstrumentor( - hatchetotel.EnableHatchetCollector(), - ) - if err != nil { - log.Fatalf("failed to create instrumentor: %v", err) - } - - tracer := otel.Tracer("otel-propagation-example") - - // generateSpanTree creates a nested span subtree, returning how many spans were created. - var generateSpanTree func(ctx context.Context, count *int, limit int, depth int, prefix string) - generateSpanTree = func(ctx context.Context, count *int, limit int, depth int, prefix string) { - numChildren := 1 + rand.Intn(5) // 1-5 children at this level - for i := range numChildren { - if *count >= limit { - return - } - name := fmt.Sprintf("%s.%d", prefix, i) - childCtx, span := tracer.Start(ctx, name) - time.Sleep(time.Duration(1+rand.Intn(3)) * time.Millisecond) - *count++ - - // Recurse deeper with probability that decreases with depth - if *count < limit && depth < 8 && rand.Float64() > float64(depth)*0.12 { - generateSpanTree(childCtx, count, limit, depth+1, name) - } - - span.End() - } - } - - // Child task — a standalone task that will be spawned by the parent. - childTask := client.NewStandaloneTask( - "otel-child-task", - func(ctx hatchet.Context, input ChildInput) (ChildOutput, error) { - target := 200 + rand.Intn(101) // 200-300 spans - count := 0 - // Keep spawning top-level subtrees until we hit the target. - round := 0 - for count < target { - generateSpanTree(ctx.GetContext(), &count, target, 0, fmt.Sprintf("child.r%d", round)) - round++ - } - - return ChildOutput{ - Message: fmt.Sprintf("Hello from child: %s (generated %d spans)", input.Greeting, count), - }, nil - }, - ) - - // Parent task — spawns the child task via .Run(), which propagates the traceparent. - parentTask := client.NewStandaloneTask( - "otel-parent-task", - func(ctx hatchet.Context, input ParentInput) (ParentOutput, error) { - _, span := tracer.Start(ctx.GetContext(), "parent.prepare") - time.Sleep(30 * time.Millisecond) - span.End() - - // This .Run() call automatically injects traceparent into AdditionalMetadata, - // so the child task's spans will appear under the same trace. - result, err := childTask.Run(ctx, ChildInput{ - Greeting: fmt.Sprintf("greetings from %s", input.Name), - }) - if err != nil { - return ParentOutput{}, fmt.Errorf("child task failed: %w", err) - } - - var childOutput ChildOutput - if err := result.Into(&childOutput); err != nil { - return ParentOutput{}, fmt.Errorf("failed to parse child output: %w", err) - } - - return ParentOutput{ - ChildResult: childOutput.Message, - }, nil - }, - ) - - worker, err := client.NewWorker( - "otel-propagation-worker", - hatchet.WithWorkflows(parentTask, childTask), - ) - if err != nil { - log.Fatalf("failed to create worker: %v", err) - } - - worker.Use(instrumentor.Middleware()) - - interruptCtx, cancel := cmdutils.NewInterruptContext() - defer cancel() - - fmt.Println("Starting worker with OTel trace propagation...") - fmt.Println("Trigger the parent task to see linked parent → child traces in the UI.") - - go func() { - <-interruptCtx.Done() - if shutdownErr := instrumentor.Shutdown(context.Background()); shutdownErr != nil { - log.Printf("failed to shutdown instrumentor: %v", shutdownErr) - } - }() - - if startErr := worker.StartBlocking(interruptCtx); startErr != nil { - log.Printf("worker error: %v", startErr) - } -} diff --git a/sdks/go/examples/opentelemetry/main.go b/sdks/go/examples/opentelemetry/main.go index 3b6fd8c75b..4a9920aaf4 100644 --- a/sdks/go/examples/opentelemetry/main.go +++ b/sdks/go/examples/opentelemetry/main.go @@ -2,10 +2,9 @@ package main import ( "context" - "encoding/json" "fmt" "log" - "math/rand/v2" //nolint:gosec // G404: example code, not security-sensitive + "math/rand" "time" "go.opentelemetry.io/otel" @@ -15,31 +14,20 @@ import ( hatchetotel "github.com/hatchet-dev/hatchet/sdks/go/opentelemetry" ) -type PipelineInput struct { - URL string `json:"url"` +type ParentInput struct { + Name string `json:"name"` } -type FetchOutput struct { - Data string `json:"data"` +type ParentOutput struct { + ChildResult string `json:"child_result"` } -type ValidateOutput struct { - Valid bool `json:"valid"` - FieldCount int `json:"field_count"` +type ChildInput struct { + Greeting string `json:"greeting"` } -type ProcessOutput struct { - ProcessedData string `json:"processed_data"` - RecordCount int `json:"record_count"` -} - -type SaveOutput struct { - Location string `json:"location"` - RecordsSaved int `json:"records_saved"` -} - -func randMillis(base, jitter int) time.Duration { - return time.Duration(base+rand.IntN(jitter)) * time.Millisecond //nolint:gosec // G404 +type ChildOutput struct { + Message string `json:"message"` } func main() { @@ -48,9 +36,6 @@ func main() { log.Fatalf("failed to create client: %v", err) } - // Set up OpenTelemetry instrumentation. - // EnableHatchetCollector() auto-configures from the same env vars as the client - // (HATCHET_CLIENT_HOST_PORT, HATCHET_CLIENT_TOKEN, HATCHET_CLIENT_TLS_STRATEGY). instrumentor, err := hatchetotel.NewInstrumentor( hatchetotel.EnableHatchetCollector(), ) @@ -60,83 +45,72 @@ func main() { tracer := otel.Tracer("otel-example") - // Create a multi-task workflow - workflow := client.NewWorkflow("otel-data-pipeline") - - fetchData := workflow.NewTask("fetch-data", func(ctx hatchet.Context, input PipelineInput) (*FetchOutput, error) { - _, span := tracer.Start(ctx.GetContext(), fmt.Sprintf("GET %s", input.URL)) - time.Sleep(randMillis(10, 20)) - span.End() - - _, parseSpan := tracer.Start(ctx.GetContext(), "json.parse") - time.Sleep(randMillis(5, 10)) - parseSpan.End() + var generateSpanTree func(ctx context.Context, count *int, limit int, depth int, prefix string) + generateSpanTree = func(ctx context.Context, count *int, limit int, depth int, prefix string) { + numChildren := 1 + rand.Intn(5) + for i := range numChildren { + if *count >= limit { + return + } + name := fmt.Sprintf("%s.%d", prefix, i) + childCtx, span := tracer.Start(ctx, name) + time.Sleep(time.Duration(1+rand.Intn(3)) * time.Millisecond) + *count++ + + if *count < limit && depth < 8 && rand.Float64() > float64(depth)*0.12 { + generateSpanTree(childCtx, count, limit, depth+1, name) + } - return &FetchOutput{ - Data: `{"users": [{"name": "Alice"}, {"name": "Bob"}]}`, - }, nil - }) - - validateData := workflow.NewTask("validate-data", func(ctx hatchet.Context, input PipelineInput) (*ValidateOutput, error) { - var parentOutput FetchOutput - if parentErr := ctx.ParentOutput(fetchData, &parentOutput); parentErr != nil { - return nil, parentErr - } - - _, span := tracer.Start(ctx.GetContext(), "schema.validate") - time.Sleep(randMillis(5, 10)) - - var parsed map[string]any - if unmarshalErr := json.Unmarshal([]byte(parentOutput.Data), &parsed); unmarshalErr != nil { span.End() - return nil, fmt.Errorf("invalid JSON: %w", unmarshalErr) - } - span.End() - - return &ValidateOutput{ - Valid: true, - FieldCount: len(parsed), - }, nil - }, hatchet.WithParents(fetchData)) - - processData := workflow.NewTask("process-data", func(ctx hatchet.Context, input PipelineInput) (*ProcessOutput, error) { - var validateOutput ValidateOutput - if parentErr := ctx.ParentOutput(validateData, &validateOutput); parentErr != nil { - return nil, parentErr } + } - _, span := tracer.Start(ctx.GetContext(), "data.transform") - time.Sleep(randMillis(10, 15)) - span.End() - - _, enrichSpan := tracer.Start(ctx.GetContext(), "data.enrich") - time.Sleep(randMillis(5, 10)) - enrichSpan.End() - - return &ProcessOutput{ - ProcessedData: "transformed_and_enriched", - RecordCount: validateOutput.FieldCount, - }, nil - }, hatchet.WithParents(validateData)) - - workflow.NewTask("save-results", func(ctx hatchet.Context, input PipelineInput) (*SaveOutput, error) { - var processOutput ProcessOutput - if parentErr := ctx.ParentOutput(processData, &processOutput); parentErr != nil { - return nil, parentErr - } + childTask := client.NewStandaloneTask( + "otel-child-task", + func(ctx hatchet.Context, input ChildInput) (ChildOutput, error) { + target := 200 + rand.Intn(101) + count := 0 + round := 0 + for count < target { + generateSpanTree(ctx.GetContext(), &count, target, 0, fmt.Sprintf("child.r%d", round)) + round++ + } + + return ChildOutput{ + Message: fmt.Sprintf("Hello from child: %s (generated %d spans)", input.Greeting, count), + }, nil + }, + ) - _, span := tracer.Start(ctx.GetContext(), "db.insert") - time.Sleep(randMillis(10, 20)) - span.End() + parentTask := client.NewStandaloneTask( + "otel-parent-task", + func(ctx hatchet.Context, input ParentInput) (ParentOutput, error) { + _, span := tracer.Start(ctx.GetContext(), "parent.prepare") + time.Sleep(30 * time.Millisecond) + span.End() - return &SaveOutput{ - RecordsSaved: processOutput.RecordCount, - Location: "postgresql://localhost/pipeline_results", - }, nil - }, hatchet.WithParents(processData)) + result, err := childTask.Run(ctx, ChildInput{ + Greeting: fmt.Sprintf("greetings from %s", input.Name), + }) + if err != nil { + return ParentOutput{}, fmt.Errorf("child task failed: %w", err) + } + + var childOutput ChildOutput + if err := result.Into(&childOutput); err != nil { + return ParentOutput{}, fmt.Errorf("failed to parse child output: %w", err) + } + + return ParentOutput{ + ChildResult: childOutput.Message, + }, nil + }, + ) - // Create worker and register the OTel middleware - worker, err := client.NewWorker("otel-worker", hatchet.WithWorkflows(workflow)) + worker, err := client.NewWorker( + "otel-worker", + hatchet.WithWorkflows(parentTask, childTask), + ) if err != nil { log.Fatalf("failed to create worker: %v", err) } @@ -146,11 +120,8 @@ func main() { interruptCtx, cancel := cmdutils.NewInterruptContext() defer cancel() - fmt.Println("Starting worker with OpenTelemetry instrumentation...") - go func() { <-interruptCtx.Done() - // Flush remaining spans before exit if shutdownErr := instrumentor.Shutdown(context.Background()); shutdownErr != nil { log.Printf("failed to shutdown instrumentor: %v", shutdownErr) } diff --git a/sql/schema/v1-core.sql b/sql/schema/v1-core.sql index 96d76ef370..e975dc2185 100644 --- a/sql/schema/v1-core.sql +++ b/sql/schema/v1-core.sql @@ -2275,7 +2275,10 @@ CREATE TABLE v1_event_to_run ( PRIMARY KEY (event_id, event_seen_at, run_external_id) ) PARTITION BY RANGE(event_seen_at); --- OTEL TRACES -- +CREATE TYPE v1_otel_span_kind AS ENUM ('UNSPECIFIED', 'INTERNAL', 'SERVER', 'CLIENT', 'PRODUCER', 'CONSUMER'); + +CREATE TYPE v1_otel_status_code AS ENUM ('UNSET', 'OK', 'ERROR'); + CREATE TABLE v1_otel_traces ( id BIGINT GENERATED ALWAYS AS IDENTITY, tenant_id UUID NOT NULL, @@ -2283,9 +2286,9 @@ CREATE TABLE v1_otel_traces ( span_id TEXT NOT NULL, parent_span_id TEXT NOT NULL DEFAULT '', span_name TEXT NOT NULL, - span_kind TEXT NOT NULL DEFAULT 'INTERNAL', + span_kind v1_otel_span_kind NOT NULL DEFAULT 'INTERNAL', service_name TEXT NOT NULL DEFAULT 'unknown', - status_code TEXT NOT NULL DEFAULT 'UNSET', + status_code v1_otel_status_code NOT NULL DEFAULT 'UNSET', status_message TEXT NOT NULL DEFAULT '', duration_ns BIGINT NOT NULL DEFAULT 0, resource_attributes JSONB NOT NULL DEFAULT '{}', @@ -2304,5 +2307,3 @@ CREATE INDEX idx_v1_otel_traces_task_lookup CREATE INDEX idx_v1_otel_traces_trace ON v1_otel_traces (tenant_id, trace_id, start_time); - -SELECT create_v1_range_partition('v1_otel_traces', CURRENT_DATE); From c3eec009936465269f38e85828e0b83f1915dbaa Mon Sep 17 00:00:00 2001 From: Mohammed Nafees Date: Wed, 11 Mar 2026 00:57:36 +0100 Subject: [PATCH 14/17] oAPI side typing impl --- .../openapi/components/schemas/_index.yaml | 4 + .../openapi/components/schemas/v1/otel.yaml | 21 +- api/v1/server/oas/gen/openapi.gen.go | 587 +++++++++--------- api/v1/server/oas/transformers/v1/cel.go | 4 +- api/v1/server/oas/transformers/v1/trace.go | 4 +- .../src/lib/api/generated/data-contracts.ts | 19 +- pkg/client/rest/gen.go | 31 +- pkg/v1/features/cel.go | 6 +- 8 files changed, 380 insertions(+), 296 deletions(-) diff --git a/api-contracts/openapi/components/schemas/_index.yaml b/api-contracts/openapi/components/schemas/_index.yaml index 690128c621..c25cc8ab77 100644 --- a/api-contracts/openapi/components/schemas/_index.yaml +++ b/api-contracts/openapi/components/schemas/_index.yaml @@ -402,5 +402,9 @@ V1CELDebugResponseStatus: $ref: "./v1/cel.yaml#/V1CELDebugResponseStatus" OtelSpan: $ref: "./v1/otel.yaml#/OtelSpan" +OtelSpanKind: + $ref: "./v1/otel.yaml#/OtelSpanKind" +OtelStatusCode: + $ref: "./v1/otel.yaml#/OtelStatusCode" OtelSpanList: $ref: "./v1/otel.yaml#/OtelSpanList" diff --git a/api-contracts/openapi/components/schemas/v1/otel.yaml b/api-contracts/openapi/components/schemas/v1/otel.yaml index 3ddd61c480..c8708ff799 100644 --- a/api-contracts/openapi/components/schemas/v1/otel.yaml +++ b/api-contracts/openapi/components/schemas/v1/otel.yaml @@ -10,11 +10,11 @@ OtelSpan: span_name: type: string span_kind: - type: string + $ref: "#/OtelSpanKind" service_name: type: string status_code: - type: string + $ref: "#/OtelStatusCode" status_message: type: string duration: @@ -45,6 +45,23 @@ OtelSpan: - duration - created_at +OtelSpanKind: + type: string + enum: + - UNSPECIFIED + - INTERNAL + - SERVER + - CLIENT + - PRODUCER + - CONSUMER + +OtelStatusCode: + type: string + enum: + - UNSET + - OK + - ERROR + OtelSpanList: type: object properties: diff --git a/api/v1/server/oas/gen/openapi.gen.go b/api/v1/server/oas/gen/openapi.gen.go index 6aaffc3f54..c84a0481ac 100644 --- a/api/v1/server/oas/gen/openapi.gen.go +++ b/api/v1/server/oas/gen/openapi.gen.go @@ -93,6 +93,23 @@ const ( LogLineOrderByFieldCreatedAt LogLineOrderByField = "createdAt" ) +// Defines values for OtelSpanKind. +const ( + CLIENT OtelSpanKind = "CLIENT" + CONSUMER OtelSpanKind = "CONSUMER" + INTERNAL OtelSpanKind = "INTERNAL" + PRODUCER OtelSpanKind = "PRODUCER" + SERVER OtelSpanKind = "SERVER" + UNSPECIFIED OtelSpanKind = "UNSPECIFIED" +) + +// Defines values for OtelStatusCode. +const ( + OtelStatusCodeERROR OtelStatusCode = "ERROR" + OtelStatusCodeOK OtelStatusCode = "OK" + OtelStatusCodeUNSET OtelStatusCode = "UNSET" +) + // Defines values for RateLimitOrderByDirection. const ( Asc RateLimitOrderByDirection = "asc" @@ -203,8 +220,8 @@ const ( // Defines values for V1CELDebugResponseStatus. const ( - V1CELDebugResponseStatusERROR V1CELDebugResponseStatus = "ERROR" - V1CELDebugResponseStatusSUCCESS V1CELDebugResponseStatus = "SUCCESS" + ERROR V1CELDebugResponseStatus = "ERROR" + SUCCESS V1CELDebugResponseStatus = "SUCCESS" ) // Defines values for V1CreateWebhookRequestAPIKeyAuthType. @@ -769,19 +786,25 @@ type OtelSpan struct { ServiceName string `json:"service_name"` SpanAttributes *map[string]string `json:"span_attributes,omitempty"` SpanId string `json:"span_id"` - SpanKind string `json:"span_kind"` + SpanKind OtelSpanKind `json:"span_kind"` SpanName string `json:"span_name"` - StatusCode string `json:"status_code"` + StatusCode OtelStatusCode `json:"status_code"` StatusMessage *string `json:"status_message,omitempty"` TraceId string `json:"trace_id"` } +// OtelSpanKind defines model for OtelSpanKind. +type OtelSpanKind string + // OtelSpanList defines model for OtelSpanList. type OtelSpanList struct { Pagination *PaginationResponse `json:"pagination,omitempty"` Rows *[]OtelSpan `json:"rows,omitempty"` } +// OtelStatusCode defines model for OtelStatusCode. +type OtelStatusCode string + // PaginationResponse defines model for PaginationResponse. type PaginationResponse struct { // CurrentPage the current page @@ -16350,283 +16373,285 @@ var swaggerSpec = []string{ "mbFl+sB9+AKn8zB8ePFFKrBsaonh7BIFsJFzHnOPoJ+pIkElizxS/XDm+CiATTyxuAe/dg46nGhQq6SY", "evMWGptEAVuq11oWVpDO8DVD1SV8hH7ecPP2lgqa4dW7606386U/uup0O4PR6HqklynKOOnlyWr/cxDo", "BIn4/vJ3T0lWeunBP65x/8yP0PAGKjpX3EGvCfTHEQiMXHEHGhyvqQODnWtHBGIYkDscgeAO6c946eN5", - "BwiJ0TQhsNKfxPSWp+hIbhjBO7Pmzj4/Zi9J5RYwfkRu1RB0ORsDtwI37NsDCiq+VtxQqEJxJ+OiTN8V", - "oVWGNQYu1MNWfPCVLbP1qPCpKyngNw+oQmBdlT510l4S9h5IiJTHrGznmglV78nvHeHQdhexjTnrdgL4", - "Tf7rVbcTJAv2D9x5c3rC3mVyXJ3rrHOylv5yET8X0onPrAwdCizaiAT4rTzyK7uRs3VpfcNDAnzVrESb", - "MlurjzDh741ZRN+JjV1Fszn/lcCE3sRi5Go0pCBZ3NgZvRjdSNPXkWm9/2Vl5+JjIe4qzoxexgFHdgYu", - "PqIwcx11al2UMlBzs3RVhOh4dAQIZB6eZVRavaIwp1Tm2ah3OgWYjOA98g2uEixkQMQUqIOxeIKYdYTM", - "r28LgRdsos/AT6CtK2vMnR+ww2LVxCOM2PUnFHjhk37bN/HKU4PoR/M6pDTRrGMBPGi7CP5NPwX/xpZB", - "9xIFio9mhmYeVXUfxi70bH2xlJu7sl9yvSlUOUr7qtL1Hhw+GY9pFdT08xoqanGMkpLKsSmxpqBSOxp0", - "YUDGioWp8HLLwDPRM//q6PxxVZNgE5vRKjbCNex7WzPiCZRmVrySSasYcVHNI+lGdFVrl4ClOLpW/MMZ", - "wgTG0JO2Nk04j2GfU29+5DlxOg6Pl0SYfYbxkSaKpIQ8O++tLHqgajKLmDGj2/UI0r9+nuimEYx8sPyh", - "Aon4khTDMTauLMcdL7s+pfnrk5Oa9RbgNq3aZNhVutsfYQVLvC18ErqYyjwm+irYSu/Rr3XFp6MWbLCa", - "AWcQk9vYoHneji6ZpyUMPOZ6La7Z2CHhdpyCTMdlEqB/qG7kwYCgewTjVLcW6qCItuUe4mqQ+hT6YTCT", - "ENdK2S06qNs9vVQ6nY/dOfQSHyqUtm6QyZaDRLodwoNh7PWEJnEl2eBfFfR4m3uJYsGS9I/x+YfBxS39", - "UacMpjNv13F3T11wy6vP/HB34W7bmMQ256E7SoJz9Vmm8fMuB2DXZ6kCgM0Sx1aK+5dSh5d0Zc6IotKL", - "uUy7bxP/4QL6kMB3LChqRafcNKYn9cl9gEuHXS6dCCCeAIiHXTnTZT77ywNcnr5hTU+58+gZ/9dZk0Qw", - "8t1CKBX6q1NDuuEjfqm7kK1IjRsY7LnhFhuPz/t075tJvhL1sMeaQiuNNr2+cjzkQzHdeIEC8c9TG1t+", - "NYZMSrLHvnsbXkmRiBtmOdMvxS7xmbKgblUWtKo59Fna9PG5GyH3Uua2BiDfsmQwlFJMFo11N3Nd5S8v", - "yZuuzMjdPAvOulSloK8pC6qLlMA0X52JM7fJM2kKoa3yvRZFq3DmHpi2NZeD59Wk8iohSuVRTOZvVWOq", - "9tcYwwWI5mEMx35INmz7ztmV9S7D3JyJ/ZA/gYke9i5OK9qhsapIaWJJCYycOJELqzc1qG6h9QtFvi/9", - "pe1XWrpoVFiorUEv8GaGlq5qay/Y1SnVqL5yZT+eOQgC6JvAFJ8d5Omf/jAd3Hnio+sfVfgIV0Y7upyC", - "2dNXnGQtCxhYmFZPv62xdNrdvG42+DqL3gvbnZ11TSIiRXeeLroKGWrPFwIjk7jTO/fPke/FMO+fXKvz", - "InyRxCyfsC7ZnsjGKSQOwizxz9RX3SmUsO2tOPfzeyButqp4tURT+lPCD+VzjI3nmin3dYceYNlTvyRM", - "hthf6Bx3dACn99/JyckrRsokF2WpZHvaVBCMYclm8lbQmqN16bQvqJP7PFXQ9RaCXvpkEIU5j05lIzYU", - "GsM47IvpvaaWKHPd8XmYBEQPrvket8rDe9anAkNF23wutsciNEREMqXtNy8HwoSYQFxRRDDHsP69sL3Y", - "IXPjoUa8S8XOrKFB2kbZ0bYmcWIha5qsOO1SsWLuPLC6+TalwHRlleFEAnX92J2jR3iQcqn5s8BeiZiQ", - "3hL1nSq4PoYkXlZI0a3xo3I12w1LVNyCFCRIPOpv1CZ63wejRZ4BtU55oo0hdYlrpgLza7Sn71Dl3x+n", - "PGixHuHHw3qwIIlHKF8nbXuPZR8runuHYkzGEAbNaO8SNO3VMPCTX6FyABZmTjGroEmNxOL7W0HM+5J1", - "I0emtYSciXRpFxsNuBfA3dX13Zfr0cfBqNPNfhz1J4O7y+Gn4STzEhhevb+bDD8NLu6ub5ltbjwevr/i", - "fgST/mjC/uqff7y6/nI5uHjP3Q+GV8Pxh7wnwmgwGf2LeyqoTgl06Ovbyd1o8G40EH1GA2USde7x5TVt", - "eTnoj9Mxh4OLu7f/ursds6XI/M13o9urO54O+uPgX3eqb4ShiQBUayLUcYyCVCU0TyxwNJwMz/uXVaNV", - "OXWIv+44Gj4NrgqIb+D0If7mratikScAP+jzDWepPypzHIn+CWaj5FN7NOmoMx/LNpX3Y5tJOlWjCwg0", - "0j/NyGyfwauQxVlzQQh9Tzzo2ElFtg+bT+0cEuBbddaiLs1/VayhBGORg3JgyBSaWn9Ch7WWJrQF64X1", - "FiAQAH9JkIuvI3KdkGqbkhhwDrATRgR6jjBNpIPo51g3N+XW6z+YsjuunR4yy0/SMKFnbZUJBlc2+lcj", - "KRXyz+428eyWMuuY889q17wHaoZ+L3R5emdhjxNtZ8QeA5/zq0LBTNRIwLsTEjzl4+BbhOgus0QTDJjq", - "8XkvPg12nlghGJYzwwExdEAUxSFw5yiY8YowDMFV88v8uZxIWLDOilDwJcuw7DI8LLqnEheKTfEdQH4S", - "QwtQmKu0CkiunAPLTqaf0weYL9X89JnFAYJA7Cx7/iwmBK+O+AHfJJG9Y9Y2cUBrQ/uce9nEAUSGqwmq", - "2uzzl1kSaAE2y4VB/iCSeqIfusBnwWGP0A8j9pnFHHuJW6i9qKh3So7r7SW3fk7rCVU+BMtqUqKS4C4r", - "LK2WQbvuXVCwqOlVU342Y423qHrXZCPkClEYT/Gao0im/s72Sk3naaRGTjt7czgJUm52JvE9LcP/YgRl", - "nzmWsl5d61sMY97jJpn6yK0iBTZeRRJ4Fea92XSxf6ts+kjsk5Si11+umMWgf/FpeNXpdj4NPr0djCpk", - "Z3UegvrLWZO7WBUmcnAoxrJVr8bF8YrxWCkCJOUX62WlhpfB6G58eT3pdDuDz9xmMemPP96Nbq+YTeT6", - "Sok9YRmNzq8/Da/e330ZvP1wff2xAvc5LUqnSIJ4URHZz74Lf3WtgOY5CEjoPIGYZa8sqVe8tz5SvlnS", - "A32+g82kMOBjm5eoh3+9zIgpTdSzb0pBdgkM6jased6CBSQwltkL5DnKx3J+QUfwyDl1PLDsOqfOE4QP", - "9L+LMCDzX1f00UnRo81mYBa7ElE3oY9cTXZirvFXXYLTQp28qUZpaCB28+xX5+AqgDOvTlhAbQWqUSAp", - "xUekPPp80ul2Pp/qRQn3Cd1BwKExhpU7Ozepr1VRxuI5HXCsCcow1y1a04+92oWdA/Qz1hJSV16TUmAj", - "ZXyMmpsKiOj/8oCYWe2gLcWtoeklDU1bNABtpZJkA0P+ynZ4Axd+YT5P5mwM+AYkWJf4TGUT7jjlIOxE", - "rLUDAs9xQRCExAGsUrQTwKzaa+nA0kGHdffxWnsU8LwYYqzapXJatDR0lM1T9MMHgOe642YO8Fwd8n/h", - "wnTiAOKK6M3SDwNnnERRGBPnfM7qs+sn/AxjdI/q0Musa1QGPYrm9FcU52HQc8Ic4BuA8VMY284BnEh0", - "cDAkBv7axkuWh3Dkg2WOEeT+NTZk5bH71UBg53MQzKBEkJEJAvhkRiLjXfiUYU1q1HrYV9A75Mhs3VEl", - "ICkQlfhbD4ZSwmfxpZvDkwnll+EMBasXU1yNv9eqrbh3GJdrjOpwLZN6HRS67U5Ig2DYw90SD93Wm6aq", - "1XiOInyoRtaS0XmHp/k2Thk+mW7bPp+eDy4v4DSZbbq0c1fooxgtEh8QiLNMG+y1zA0T33OmkD2Qcu0D", - "BKJoWhg7IKcx60J68jXjy+g6H1w6WRt2P3gEfkKpX+uO7RMY34ClHwIDB4psIBFvU14fkJ+o9uGEAf0h", - "ho8oTHBPuBeLMTpVyYPKE7NP5flIKTxU5GKqNt3kavYLO04NZVQGshu4gH6SGckcJKtjsw1gRfB5uTDN", - "TmTu67oYNJz4aSxWYYez0bt0QlYDC+P7xNcqgnYxImUsyHCRkoO5MVjCOIYhTpl+yy0xXRer/cmtgsxL", - "cjyurJvw+fScRUJMAH4wH6TfCIwD4It8AUZzlWjmDC+wJEUXBE4M78XlG3GFHOAHyr85wlQ7q3auDeci", - "sUsK8/mU4kOmf3nWb5iMIKFNsS77Bja9VnB0MTSky0Ye5kLvCcYwK6+3NVQ880UwmcMXWrH91VJU4S95", - "OyjKMFWDqRCfUjiahikUV6gqjK99P+HjHTm39BZPJ8HJFHNHLYpyjyk+ohV2AFGlkV0essr8setm/TJk", - "oOSRWAwhuSPPIGjYlosgfmXPwwBe33fe/FUr7DT93wKM3H5C5p3n7ir9+zdDXvlzlc4fPvXPO89fjYsT", - "gzOjq7/OEiEDsKD50EXXShMxFIdE4Il1nSxNVMyilsN7h7aCAUGuoMKQWWIkg4iYfkXo92+Gdx8H/9II", - "+2JSZTk9h0RDLWaUMmToc+h+hMtBY61LXRJX7x7g8siZMG8p7DCjGwl5fSSYb+Xcx+FCxYUUIkdrpGBO", - "saqv2rLeAtkQ5cUJ1ZEVxgjjGLokEx0kdMT7k979mdmgpBuVFSmOsy5C0UFupWbLm6QSWk+H5VVxI3ex", - "d1ryv5yTwqCRSofqDOyujt7sRV4msvZAMKjyc0ty4W1/PDzfrlRggngPsEnh2C4y2Uo3hssLMDtXkowU", - "k+po0o/U665pRfOyCuyBmW0Sfg0v/URV7q1U1fA+A0C99EyhA4Kl8+f4+qqHYYyAj/7Nnh75yo5WUmor", - "JiscI2HsuIDAWRijf6sFmDVFxmBQlcAKE7CIxENpeu5yp3UY2LtwNfX73EQtGTNRiMOUpZk21axW7qNy", - "MvYum13S0lGc6bIwoyWnMmaaKMBoy9by7yiYCfl21USJEX7nKagZnMwCAqLIR24h95A+g3BWM95iUZpi", - "8xXvWqUiOJp5v2biZw9sxlIQGu7V5Y01pKetp/BULyxsIyNHzR7WZq2zyCinEH/KaOnUcRIcbekma67g", - "YiYrc2KRuuJjacMm9fWFUbJmaN6qybj/WFVg462ajBvbFWITzZqMzIyn0KsHOm1oP3rR1yqt8/aPrP6W", - "zZ7uiZLVRUgLU/rvBhatKjuSjrsQvoCuD2JARNYbs1OC4GyEHS/r4vxC4gT+Sg/wKA5nMVgs2NXpl3vg", - "Y/jrph0WjDqOoqxJVYcpbGV8HIaZbhMKRcW2N7AC1o29ScFanYjfZDjMyEJlo704dbOU7PXZdz+fGgtx", - "A0LgIjKoveKjIsGKdbg1Kad2Utnbl2Wyq5FULGn9cgXBi+mkdM91JF46LBONDaabVxgvoGONGuPZSPvA", - "CZXVwNPPVcUW++PzTrdzMRifG5bL6221T4NNnwY53rbzMhiLsbf8MEhBN5l6mstOuiC93GQ+AJ8qcoOx", - "q6q04NVvzCBtvmIuMuvMeJXaNCQxgrh++fTLBffZMVbxoW2sDHY8+xcz2DRLOiavoc3yM/MmHDh1anXP", - "MlzrL3Xplu2FSM2Ivo4rJEFuOcFYw4xicqxcJrFi9jB96rFiRrHx4GpyN1EXk67hjp+QpfRn56NBf1Io", - "ufZxeHPDP17fXlLsTO7Gg6sLZWT9yaPIWEtTs32uG4wCHrnZpNQAbEpHWcrYUvGNgCB/lTpn1cU6mpXj", - "4EgwM+VNiALCoxTLOyBoUStbs9Rs+uBvtICrRuDxRprcb1bL0BzE3FWs6c6qqLG8hzAVKglM+HQrs61a", - "uaCpJKd3O6tK91iAsClGsqVpyD0HmyIyUyGRpfU7v/50czmYlLL5VSQpzL92rVbGRLn85w/qbJp1n7eY", - "RicMpyXsb1ShUt8LzRqmbMUGwvYPFjVPizW34Ow9KcXJE8DCraNBPgAvrzHZuUFrtkAZMcnq62qGE1+L", - "Q3UdFDgL5PsIQzcMPGyn49Z5whZmcX5JQ/YBgZjQ336tLx9vhX46vOxmj/86P+QKlAuqF1718scIBiBC", - "R1dhcJX4Ppj68M8xy5uRtuqhRRTGbFLhil9uHAF6xenMEJkn0yM3XBzPAXHnkPQ8+Cj/PgYROn48PcYw", - "foTxcQjYGf2tF4ixOm+YoXXNMLBkMY7AUwC980p2VGzkvHmZMatyd5cH5N8aUtAB7QkvScDU8NTwYP2E", - "xTunsrNWgdrCfc+iNJaGQ7dUHquoqGb1CgylscoH5bqWh9U2coOzWzwIVF7ehwGGcfMjD4luTR0obN8v", - "jtRqtDuqSEysjDRpBhBho5HXm/MwuEczbb6R6uKya1V+XoH4CjFH1uDkyieXZxKR75qJ1imcpdrHVa2p", - "y603MhpIc16l50w3u0AU2FW1/uRZIV+xixuCco9O+i34WlTot2sVqjbAbkorLgUUp8ALSMwXsglaiKf7", - "LVpgPRiRuUHvpZ9yygTiXmBPgMD4Hvi+fsidKaJr1z/bjibRUHByn4aGyKKnCO9oj66fTaHRWNc3cFds", - "lZYfSGlZzRlO1QHWKmzJhW/hiL3IHdSrHLpfC0fIS56jlJpYIvRGx6k4+jZ2mu4sCV63E8UolLVSNH7j", - "4quJlPR121R9tsbtV7Suj/jPjdtV0vV9PuXJk9qw0JX9zfTPACInVSn08uDC6PYzCm4vg9g0ZGAsoK0G", - "K9kFlMoOz91DIJutl4hpAzdfnuY342prLlZu5eFqGS6qxAd+VVlTCdQuM2mEPppC0vo3Q8YVCpLzoYU6", - "IphD4MHY7nTnbYubKKatxZUyU1eu42uViOorAikfSNpNA827pmhIZZxcsG1RCbXKaEWXOqWjMITq0Jhg", - "qiKbkCi/1g5UQFk6ak1uq3zwqT+jOt58oeJt/KF/2unS/5y9/p3/8fr0rNPtfLp4XY29NJ5Vk0VWmcg+", - "NjbtxRKYuqFnUa8uN8JAdmLuNLMAkCSGH9amYzq0k46nFZhoFrDiSm4MDZdXzL4xNkxlGZoFVhMUA3hT", - "RCl40q+4CFotjQwUvKdhxYP/w8oVjgcsKIb/cTu6rCaPvXCdkzqNpUNMqgOb0ka5c+D7MKhyCm0Qolfp", - "AC+f3QtHohNL4Cy1+/IBrWzt+8HVYMTk5vvh5MPtW+bmNxreDJiHXv/8Y6fbuRxeDfrM+e7z8P+Y9jy7", - "wW4+CLvSS6W5b4c0U7b+Ha1/x4/l39G6YJQfTtY0xO73Q8LB2LEbvpHXPEprLN7inXotqzdrnZm8s2tb", - "/ok692KcvkardkrlNLyARJZRKDj5JoG9V4JIwYDnoN4Ko8ai0/bvwlgDj3wwYhk4bMJ+WMNMGcl7G6wf", - "yMDBwZtLJ1PrwFGO5e7kcCLRLSErb21eHchvr1cTPbOFepbqlFXAvtSri6odNXh2MWB8U08wX3QuHxJF", - "5sXsKPiv4JikRon234uSblqVXGj9vNDERkssNjJ5iiIVWuU3iQ1pqGXfJPYbGdqEQYSOq9vrHEp4ki9z", - "8YFNLRLbmQSoXBUZzukp5gzvnSAkThSHj8iDXtcBTgwCL1zITk/I950pdGYwgLG8xqjUdbY1jDdHs7ef", - "BLja3uyalFM4a5FNpZbZdLFTy0te/FhZX3JdjIwpLu13wLBv7EkUBF5W4THmQ6125V9AMg+9RqsVoH/i", - "PVPd/jz0DFT7YTK5kbmz3dBLKVgaeuzzDdwBnnCAzZyb+KslwqtJSKCy5pzPDFW8tXXiMS0FrEw7n9Kt", - "y4xdk063c3M9Zv+5nTAtyXRC8iAsXBWhhcWbEK/D5ILAiWBM6erIviKeMCux26426xalBJTm8wMYo1kA", - "PSfrxKxBt7fDC0eQ9O5veT6YQh9Xlw9lbRiZ53xCuGi2Iw8u5Og4OjT6AJMPEMRkCgGpuq/ndo1Vg2V1", - "HIAzl73zN+Wzk7Oz3ulZ7/TV5PT1m5Pf3/z2x9Eff/zx6vUfvZPXb05O7NOkAM5g9MgeYAKmPjOA7SGk", - "2z+dzadyDF2YViXFpuwstA2P/uBV6cJ4FZIa5efSUFUsqvlk1TxxbSYl7GS9nDBQd7EBZMV5tdAlAd3C", - "YXAf2nHPSOlAjyY/JNkFeZV61XzYcTYOc91WkUO/OeARIB9MkY/Ikh3PuUK4GZH/QiG6Y/lve/+dnJy8", - "gs532dmHXVFS+flXfZ5SPzSdTRguQDQPY+jQRkIMrUg0YznWmM2nC+W3LpKRTZ1mvzmfDD/zgtzpnzf9", - "27Eh0NgmuoXvURrZws9KY0YwcXrz86QAZL0Bj/e+rdOHb0eXmuGbqsesvVa1UY6K0slemblX5nqiXTft", - "LFRRZJsX166ZvDpRaQUeXv4l1ngRSIEc5UVZocI2CGaJeMayFnLji4+YH7u8s1KgupxeR6+qCfk6+EZi", - "oG2AvQfzsKXFMYhUhfT6ss8yDNz8a/KBPYpM/nUzGJ+PhjcsX8rt23/pjTtFoVuiqVqhC7ggpENTSivo", - "vlLg1sVjpA2dJMiJ89zgmhr9rHS5qTYsWiQLZZImQ+uKn1dwRtGoNh5cvvtwPeapHj71r/o8hcyXwdsP", - "19cfjXvBjueyCVhdmz6OKf3Fwk2626AsLFdEZGFYfTnRv8Op4YiiX3QAWXH6n+FUdyTuRKM0Yk4WEdSo", - "2WC2+lpT2yzQXuyqn+eEm2F2t6tcgXjfaiZxlac0icxKm7nmhE1DNww8JJ5Y+C3P1aQ6mUGifGel5zXe", - "H4FMesJz8M0gwcJzNe3qzGjfVGtQ7PZahDHWH5MYEDirzQ6uQHiZ68fLWpsvImVplUJM8oWzi/mFX53V", - "iy85dXE1XS1Wq7ZoeKFLg5gCOLzQ4lD2/oiCnCHl3e3V+WTIDqyL21H/7SVVUi/67ysFJB1EaiKNKJjN", - "rmEv+V2v3qwVg7ljzUh/vXuu2E9j7ijGJB9hVTglCQnwdRSb8tgDXBp8iuTwlCztIjbl7Rw4OIIuukdu", - "NonzSwQwhp7ziIDwV/9VzxVGRDRwONNfb0mcQM34de+3qudWaoA5PTk5MXpiaYfJ+041dINqtKC/w6kU", - "Y7bnuKHww9rRzfxE3LWRks8tbD0vA0LOmWiTjkGqz4fWO8hcauTtssHgE6VX2V2noUpidPhZJ3d4NpDq", - "yqOA/bVamOzJXVlx+rE/FEZJsEZe5fIo7xD0c+e+mrYjo+WcFFMkY80kY+nM1MruVna3svulZLdhjh9Q", - "tFd4Q64gmtloQwIXZv9Kw32lvrOx2t6YpUKrTri7psdZlm1t40nUNjCgQaYXU/IWc1OIRXVLiFRGraOe", - "UqbYm8HVBU8Qm6WK1WQBzueMTdPLvu2ff7x+9672lGTTrnRvzgsUMzFO8uKk6G8TBjeK5C/BShuM3Tn0", - "Er8iKMrQee3j6EsxT4qlgKnZbMyrqBu9kHLpWbbIjlX1yHDtIoxGApZxuQkdyaHOecc6LbTQvDR/xhDa", - "5NJVebwl02k/CubSfpM82jw7eNViJ2CmQ6/PVcb1Tf7BhpOrCLMuh7CKfoRQOI/pReZeLxe0LM358g4Z", - "uLFuQuZ8r52RyZE78Xi76WmxfoXNNYMC3jSSF6YhF6sMnOJns8o9V7f06Ms0sDvxCtEczTzFjFGebvJl", - "qwoMRZstsmzuCcNmQ9RXD+b0cg8Sn9xUZlkSjYzZlqweCcQt8k/MD96FoQTWn+PrK4cDXQ7bYSNonWjk", - "s+ALPfaFsce9MS3QgIXaMUELGBqK42CC3IelyRWHfnOweFaxe0lU5EUDtmU62ONp4aXMCsdKnzFP/qRD", - "+WNG2eYkrjYLfFLes23fLRony7W+BsplScLIDfS1ntMZWW3ybagJfe7FnuwK4dyhInsUKlQVjiHk/irG", - "kiIL8K2mxVMzZd9UV4RHfiRU/jL5ySGcQhDDWOYzYRhlxwr7OduUOSERu/aE4QOCsjmiu8p/km/nbzoi", - "hDnrK1Lb0N4JJuHCcrJnJvG5W5QmeoDP4vRvhqzcFWE2sfyvKSF2To9Ojk4YHfMg7s6bzquj06MTEY/N", - "MMFirn1RJnamC5B5L5/naasAYuyk9hi66UAWN+lciu/vGRpkQAOb5ezkpDzwBwh8Mmcoes2/u2FARFIN", - "UU+aNj3+G3O+wukBWMPHgzgOqRR+LvmnXoUkXUeOODpv/vra7WBZw4WuOmsofUr+EjC7c+g+dL7S/gx/", - "MQTesh6BtBmqwuBINth3FLIFOyR0gOvCiDgkBvf3yK3FaIqBWpQ+nh4Dn4qUYNaDC4D8HntIxsff2c/q", - "b88cLz4kmtvTBfsdOyDN9EW7O6w7f5su7UKfthjQBszVgo/AeCYGC0iYPvBXhZNPaQZH5DnvvOF5EFKh", - "UVpKRxVq/H0g27H1avJ+LdHTbxpPwsR1Icb3ie8vHY5SL5cmrYS8527nt11RXt9ZAJ9iAXoOy6DlybAj", - "DsarjYOhg+JdGE+R50F++8jom9NJFZlJip+wJvSw+taLhcrBPvC+na6GML6yay9xNVnS+XVrHRLnI/wY", - "JM7o4W3I5fFGiIFjh29aAXFp3FqZTCqxRUInkTjPY+NZL/Y3shDtEnSw58QAB7QVA5ZigFPL9sSAekBG", - "qEfCBxjQU1H+zU7DKNSlNBjBx/ABOiBgyRpZa+Gtlc5YEBMRmtBW0qBDu9tIiXR4g0yQsO7VcRez5Qk6", - "Z9D92ESNm1C1IB26sROxc5KMs9+qKDnd8hwFu36YeMfqDd2sQZcyxclrDxvEQQEmIHBhiYjP6WfpXmJW", - "rLePWwaIkwRZwMW+EFiN1s4RrL7Xi63/pLywfevJIXphxJ1dxImm7Dc3hx9/Z/99rtpvKqVYq6PShjKr", - "ON/IWknEk0SblBOewnGXQmhzmy1SK9Uc3ryGyqMQaxwbbMda2ZYjcQUzGXlzFFdINU4/X80Uflwn1ti2", - "pFKthuYvUgH2s9P9BSPhlvb3i/YXcOUz3Hh67+7gFhnXmtBUeiQeyEG+iSOcjnHM7PR8l7Bxxy8Rphcg", - "38m1Nm0wbT3MN9zabtO5xI4rUzbcfJkBJ7e6fSKEdOvZRhQ2obz/uU0OA0RCKs2Pv3OOfz6O4nAKzZdL", - "+fbpgFzlCWbX5ZUrcrkQzAyfTn0TYjJKghs2r71tynTopZJrx6deBUHBb9BNpG2F4fdop6fCVUhYBYIw", - "Rv/mWepFTiMefM2jNEtmTgKQDz2H2+0dtj3OOyHPh9m26g+OHJlhH7gPx9/Zfyys+M6YNlQKrOQph30V", - "yaHsjfa5MY3Ew0DcS+t8Hif7pNqc7gaM2yAjYT7x691MzHOOsdSNwPfDJzq97kWgSLVS9LLfq1QsTnR5", - "jgnw8XccYCtuuRqrUr/MLwFuwCb5wcyMIk7uvWOTAjJaRtlDRikRbMoqV+NKRgmwhk2k4qJYm/SqC51X", - "XolLLNL4bezF9I+u2RDACzOtZAlQYDh7/ToHxOkmdKAoDuk/oNeeYXvEmqZLJKvf4IAoktRePtZ4mwI/", - "EjD14bEHZvg4Tf1uvDRidmtk7RwyB8SZQj8MZmpWgTTNOJiVr5SfTy8AKzc7ESXU681lMsF3lqCFp9xm", - "LPNPAuNlxjMemN0hr/qY21aEiJXcKcD7Uhcfa+rdWA38CzA7FzFf+uxjFXKITilf/9isP7eVsNt5vSvh", - "R2+haBH5cAEDUtINmPFC0kH6dA7wg1bCsIbH3+l/ap6XeKWL6ZLzTVGA0AksTe1sHOOhTwHd8ZEPCIGL", - "iIi8LAahIBp1VFhKsVDbtOMXano0Mr0xrP7s/Pkbv/tsf9aJWn+fagr3YcKTNO2JiMj4uSQizHcGYiNC", - "jv1wVqer+OHM8VEAZeYjAUdRolyGs0sU8HoshyhVRJYnEoq0vNOlQbLwNIxaaFBAWFHJctClIXluTERq", - "7NCZQUJRzbBsmBkjbnnUzFyRusFwb0qrClhNnQQE+RuYuu9Qedcj8BtxMASxO3fYTEqN54r1sw46kV69", - "VkbB8BH6v+Bf6UQocP3Eg6b9pS1xR6vtVgt8yQJ0AFvl1pPJbShgLErFTHns8910eZd2ykFpBVwpp47V", - "IWu1PXtw5KpCqIFCLKJY23fzvFaaSn7l2LkMZ+ufOvT/e1nosPl1VSnUZjx40jpsP8DRgx9QZGL++3sM", - "N3LubPWk275Kne31Cg4y7bW3VatzMk4nYTarYpMYuJXRh871BPoOa6bAIYpRGi7uEzZoK+x+XGF3TaA/", - "jkCwgqTL6KmVdi8t7UoXeWVzNiFpWAvlMdCF/rEHp8nM/Bw4eAR+wirLOeeDSwd+i2KIWfg+mAEU4KxS", - "o6hE7gECjjSy6Bz6F2yqQ3Fe2nzs3OfT88ElQ0JNqBzDJKZyiFUmp2yqR/5OI+ZU8GWC1xpRAwX1eJo1", - "tDco9c1/msxKLKbw/Png0szyVrxucYPiT415JSetn17k52a3qH30BviRlAuN7Uw+Gz7AJVZMMsZpabvm", - "hixGBiK3R50J6zwMMPJgLEmMPWeHLsvx4zngnkBR6EYYCLdp1qyGZQrvwxjWArMpQ+c7vjUkzEEDYlb1", - "MnQRk6BPiMzVV/9iIXoNfFkCG8PObvlB3n5duYIhzgIQd46Yk4MLYwJQkCUJqVpnmvcTrmSSZW9whryh", - "VYtLt0Sscrqkxx2KHe4YoYNYpAZ90W2ZLp0sF3cWjcLKNqYWEIP1tpyqXLsQTdkYOc0DXPZ47bcIoBg7", - "v3iQCT7KfUsHOP/z5n9+LYqtSncrOxM6dsMIWslD3tJ2Xaz1evBu1xpmbwlrbd11tu6UNywDxBooaMfs", - "GLbU0vjZbqWpfYTLQ1HWth4wKXHRlBEYultm0DGDI7THLTDE98fTXoMQeebFRLDek6lJtPweezCbYJKY", - "OlDmFPvTHlAbCWLGTQKYU8qx4kyu49gcU6Jl7RnFVdLWnLCv5oRSnXcLBbr29lk5RemKyC7jfM6j9Yva", - "NLsr4GSKIXFcEHiIZbSSdL3R20PVip1bDD3GRhwWQq/HZXgAkTZX5iVkKM+z04uHwtoNBLsUMa1kz2tb", - "Ei+ZbOf4rdK1uoa3nXNW1MwBTgCfxMBG0czb/tyPNwwFHB02Dzjs/SYlZYeVkONW/V2+2QjyqGM9UeBO", - "Abh9Dt7Vc/CV4QU45c+UN+153l6LYxcs/rdNIDWokxSNUwTvlxonuBWxDBCeXIv+spVi4jBvW5aiQUaN", - "t2LhJcWCLet3FcKkR39F0FeqwJsNJny2Q7aYpPz8k3PxLCTt4W60mKxwxhYZrTIhef2xeeCpFXLHZprO", - "+yUZbhtXAL5JK18BXiDNubV8kJnNW/lweKe8hbLPomgWWVnMCrVASEaZcsCJk8ARPaszpHMPikuECfei", - "kFU4D1WmlWMuFTTU+CdZALp2GGY9NJtyUCraZpn1N/C4t455+rR6InohTxcKN6+PyUj5f2E1uYkBaFFP", - "k7a/k63vWOutEluWbIW/8TFXqbTGdxbpb0hrwhuiYHbHq4XuCPK+xoHoofcofHosHgkyT6K7RaUr0csa", - "salgGyWBlGjNMzSoUrTNprI/qRLY3izSg8ouxsL+xI1CFBDLc3eBgoRAeh2Xf8UQPHjhU5AexQ2O4feQ", - "3NDJD/0QZgee9A1WwmaEwbrT7cBvgG5x503n7OTstHdC/zc5OXnD/vd/DXJHdO/f85vIJg5IBmnqOayC", - "GlL41gD2HgUIz6H3lg3eHNzty8Ycqa0gHRmftPJxT+Vjfnc2LiXxsQsCF/rmKLRz9j3NfKWTd7zJz/1A", - "yVDAVJWaUmw8m2DouBJpO40iY5P60OMZCWtfJmXzNh1dG5dfklEFybBxyRTDyAfLqjJy9HulZOJNfmrJ", - "xFHQRDLFEmm7lEwcTFvBFIvWrVxq5VJJLhXkwgblkkgybON9Kws51HnfijoRrfvtPrvfcnJx6LB28Wus", - "/RVtvkowpKCJcTqKrb1VEp01oKJDBaTVk7y4h6vKPg1cXFNGbt/i8z6uKWIyuSlQvLaXq6lcTrqJrZ+r", - "8HMV+Gjyyi2Z8oU8XSWNNHF13ccyCz+3r2u5hoIF7zdQm5i7q/iHnb9rrcw4cI9XOrl8e5QsXO/7mmHF", - "DOxu7dC2/C/9WVve3wtXl1r27qrkVuPSKulX+LQK9dDAt4fs1lpQgH80HpXeqi2PGtxVa45JGNBTsBcD", - "AnvsBko3V+y9JZfV+bPWHosH7tG6XQ7bnnfqj6u4SxfVVjDskeKukQern+z6G/xNiFl+DxS44QIFs5Re", - "FxBjMKs44UfQheixlUFNZFCQ+H6J8oOlE4GlHwLPQYEDgqUjVtvtEPiNHEc+QAVKK065ExlikZk0h6d7", - "4GPYKheGCoec8TTstiqH29zThc9wL06CujeOfNbA2leOLEtg+9Kx/3lLscjkaPXWsbOsj8wPH8Q+gpjl", - "uoZW4G0xKMAHpAkoG6vNtDeO35a5ag4kWoECkcbR2WTYgfGWXfy/zCGZcwEgamE5F/33mJ5eYeAv1d/T", - "CqU6gRT4yzvZoFZRmYahD0FgEdORK1drgbMXCu/QFNU1xnlYZPZ9sXgP594HM3bUPgm6CGPmgKGSQXq/", - "BIHnhAmhfwr1EVP9kTaQuuCRcwHvQeLzfPf/Q+nhfxx07yQBhuwY1y1fzHQnB+1UktDOKnc2fQFunYb2", - "rcJPTqNUFV35+4j+vuZLlKrhHnsIRz5Y9pi7RI2+K9rSYYV7RXhfoQRX68AXfDDmdnHQ+rAiWnH6jpVD", - "ioiXFOgTqDMrAoosfZHC5ls2wWtJoBVdrehqKrokn/Qon1RLrhyPMu1Bn/A/S29XIbkGYrChd7iCq73n", - "tvfcn+Seu7PjLJML7Wn2I51mudNjJyebuF6bw34mvIH0Ks1f2CuOrta99FSgTkFKzVN1jhRIKPw3d/1G", - "rWjNkADk42Z+piqFtO9NRbfPAgNtgMHz/Mx8PpVfakpJ5EkOBB5zJkvPfxKmV0lRLOm/Ox4jiv/uOJHh", - "QTqjH0u3sxwM3LY5Yz0Nr8DK8g42l+EKXNae4nt8ihfD3ywZulsi6BVY/FiUjKvidMKzfJGEGY7yfH9U", - "y8VjWZNuRV5Wp1fU9R+TtdXrZ8vSe+rkdR4mvsfjaelFUqe57FFukhxXpQUiX0TWsGRPFiV2WVguD3Ln", - "lnr7qwNlIFbHx9ro9fNUpMnEqtYA8uNK1JWqOrZCtdWTirKLoAUKZvXakmjXWHq9h2QipjjYu49WBnkw", - "InOesYRnNXPcOfK9GJpcN1iHhtJv+4KEb04rSQ5eklTx56bFC4yETJF/Ph+D2J2jR1inBYlWAkzaXStC", - "xgRGwl23Lwe2EB9yPKP1VMLbuu6urpFtUyaJfRd7biWV8kkl27qgu8/HlHJdISdTWUjl2F9hfimf6PZT", - "2VQlmlIWrpdJNvcyUbrfXh4NZI3VVhr9JNLI/q7VyqLDkUUK429fEvnhrM5Tyg9njo+Ckm5UNkdfhrNL", - "FEBba1Arhl42nsmHj9C3chniLXMzVzGDpAPa6x2CvmfMIAfpweuw2RQ4KoqZsA5NARnzXtpQEsACBcLY", - "q1o/+/x2ydfScPJrta8BD3x6D8XQFdHuFVBcKM1WgSTrv91DSpUGbQH9dVPQpVJYOQsuw1nzY0A4GlWk", - "NmceEFh4Ehkc9yfs53PV8WXTjjl8cD5RXZJe7pr0Mq44HMJGzjcCqT82ja/gdZMSW5qdVvjTFIlcR9Gp", - "61ytyZi7xogX9koCb5qQKQ3sEDMYn3x24y33shQvUya11L7b2wYnRi+E/KIBv/ETuFRIw5bZchlNq3Mw", - "BXw2FMyq+epwMjFtyeuUI6DJ4RbFFJEE8biMFyjc2Z5z659zgk9WYL2K8+4Y+JQwglkPLgDye7M4TKLK", - "h1Oq3MlboCAvNobDBnDEAEXW7dMmA9riPW1wKJFO2z8JdYhpWHLKuAkt7+RfEyuotdE5Zn31Kc9Vxxg/", - "fUiFenMr4MburCuhvNHV7nS77L3CCaihoZavtXc/Lbdt9pQ8xpCQOtcizHZPdnFkl+psBgq5oGA2Fn0O", - "JKnvjo5JBTFrnJHqnrSspLnWadC0MT6KUI+ED7AmGZ7Tvxk6vF011/QjNKHNWn0SHzO/opshwwe2SB2p", - "4xPpH9Xa0IvKI6VIjlqFGdIf1ynlEmTUbkfsrY7IECBpXVELt2nCKE7a8teGw2YzZmrIYFUHjoW3FK8u", - "l3OZMqVdzZxm2nSre+2e8ACXVs4JtF3z9DOMDD7CpU1ekwym1H15eIFt82FyWdEYQOkSPbxYEcQsBm2N", - "VD42EI6SgMdRCsPXi7h6sP18GUcPNvUeuHmocKhOHhXEkmUQgkvnEfgJ1OcRgt/AIvIhFdkPcHn6hjU9", - "7XTpv874v86oeK/ON/Rps+mGsmXwxKVpxqFqOmeNh4efaWilSLvWuyYw+1wqSgtD7vomZDauQQdprwAM", - "AQwXNWZhkZj4Rdx7OCU0sflC3uNn964++8/dzDoS/CnUU/jNhdCDhnKOfG8a8Hn9xeR4mvgPZne6t4kv", - "6hhBnMkEXCkUaJ+fWDDQ5TcUDvglpQNuLh7a6Is9kw+MTVUhgTcsJVwQuNCvcLtl37khQ0mcnVNxTVKD", - "u5XwEX5mhYIhwF6hEBeGGEY+WG5cbGQOW/RfT9lleciTE2+riIf8IZz+DV0LzYUhDWY5SlohtbdCasQo", - "dTvyiZnRLG2s3DZnYWf9CJfts15mbFzpts6Q3d7YdTd2R9h+N8kH4jQwntOcB3Gzo3kkj5if9WjmCNiX", - "o3kzZjUOXKvV/6QH5nf2394TIvOe/MSs27XhR4AAfngGlQbCC0DAe0i+IDKfSLavlR+SffTiowTyrt8u", - "f/hTnm7aKukYGFW0p3zel03BjDXvdjVEXs3PKHhEBDYNmJC99E6gQ/a11X2l76eCj5W8PiW2W19PXThE", - "RotbioHgE1TSevucpUQ9cJTYBTtw3L5ohAMHd5XABkEYP3ts79nZjrReQOzeuYp8q5MLMABTH/ZiQGCP", - "jUnZQ/DaKnqxkELyhx7/9zMXMT4ksCxsLtjvODUj2Qga3udgvffyXF8NWy9Fx6Gf/LWyhVPIPsuWHJtx", - "IszI1aSL5vexNoK+GSccThT9oXDCdgP9V9MKXizU35JzOXwHw7kiBL8x51adfAu4mDLma3SDlL30LP6J", - "fW1vkJIaFXysdIOU2G5vkLobZEaLmwkSFOMdf+d/WCiBDhBAOPdxuKgLsuXU8GOogmLZJtj4553y7m9b", - "4d1VdMCfg2v3KFftlSE1bcqkuY1pIC+6kpAt0kiVJjGLgB9DB94LEbBd5Zdvl53yK9CxJymvLKWXRg8W", - "+9YKrxcWXka5soLwqtJ6ojhcQDKHCe4tqA7q1pcvyro4okvqg1eXmfIm7fpJTPZDXBQI/EaOIx+gAlUU", - "R2pyByhjuWXKl2ZKygGafdnUDeSfBCbQmg1Z68Yc+F+01wEx32FHNh9SsOr27SE52lstg4XzCGOMwqCV", - "ifskE9PdKUtEyTmrysTsqc/G1TtOHxvrfL1HgMBL2rDNq7HP1Wk3kYOhFpPbzLSQ0tkeZFsowrKrshp5", - "XmsQTKCwc+tnWLCCq7jJxC3ztrjkv64qcUWPXhT6yF3Wp5yUHRzewSbhpHSFvmE92nSTxzq0rPZoVNiN", - "9vFo51lbsQ/ch+pEk2PaxHmC03kYPpSfU9nnL/xr+5zKc0yqOGlyeyigep/YYUcVj28DkJB5GKN/Q49P", - "/Ho3E3+CZB56rKIH8P3wSV9tmW8Q0wM5C6jnGfu4FiMeYwJiYmTHMf3Kz7HrfkLmDrusFBnyFstnGwbQ", - "NUUo63mInPnq5EyDB5V7GMrEsZLDyhwCT3iN+CEnmBqLJ9tw6CYxIkuGHzcMHxCkg7KiSF9VemAozc8o", - "CYHuwMp0UJf3d3w1LhJgQSAHuJXDQg5fjYcqqhpI4iKWW1m8d7K4zAipJL4ar5FuuDCwjsHaaAyGgDx/", - "VWYZ3hzN5ie1jqoo7mrL0HvE0EbOs+ToyhNV1Ons7eLJSpQOP7SXq+2bC3SIaWYzSOtZ53amfVTZh0eV", - "dG82/cysq6peybpZAXVnuuQMVTi9OSEeiB2vu6+V3bcpMcQWrSgfWomws1KoKi0+AV4PtU5EqIc6/Ylu", - "9KpVtqvlRG1OwD4hcBGJ5JasrSI+TILj0JIBthKkyiUeYeYrLUQIJwJ//y4IL/yIV8cou2LoGNKOFbnD", - "WJJFWx5mzVsW3sdsZnESiK2q8WhHQZQwfwj+uKtb7vNeaCptLrMK+cI2/CUESramSlsAbyacBeqEy3tI", - "xnzYVrS8nHbQLEuvwdIghmsvFPt8oZC7tBWpQQB+6GECSI3BEOAHVg1KWAprrIQTgB/GbFB7ETG8+BFt", - "gykiGnCoFtctj+6BGdDEBrtIjyS8ZnpPYfxQlSwic8A2ujS13kxZMAlHxReGVIqQqqqeFBlpwAvv6Mjt", - "aJ/b9u39XCH/1ZMYikFMLPTTv5Pn+IdjY0fFeDUze41SEMqtbTl3/x7KVcZb6bBkVFH9kEZPSC68q73k", - "s7Phpz8sM0y0Na83kqFaag/5GL3VvSslorkhqHktCrX6r6YkhVKyty1MoRSmUPCCawy6ufrKL1emQge3", - "dTl7xdabI5j2krqX5Svye1QOB642JTURON/Vf9b5seQ4ofYEFmR6yG4tBdbXg6Zi8IDVBLFdq2YWaN1c", - "zHH9+Rek+pj+bp6mVufnY/YYWfuYxJ8sOUOrQB/V8PWQjd4y98szd5bF5EYpQslhXOfdKY8jtt2tWXtH", - "Zu0vKu4Dm/wh2SY1VRk2J3HwHERwS3rEmI3dypuDUSb4hrUaxQ+kUaSxK8JnqDIyVFRqZyzu++n7ONbo", - "GlWszwInuSvLQBb2a2XAxgG8BJg4wwuWsH4OHR/IHTSlKQKYDD1jnqJXZ7o8RTvwsW1S0LNUlq81ieyf", - "b80KssTe8cZOFmKrlwnW0k6j+SkTp3nwHiQ+6bw56eZExS5SqKVzv15l8jHPpDZdOmwC/aTikzmfwy7U", - "rvaxZ/P61iZTMqZj1gYDncu4hikg7rz02FOlMR1OMNC2vByUdxKODFu3fRFNUn4q2fRjT6RYar6nSt8o", - "CYYezqWeXQvB5Xy7DQ1CIgKpfT2qSY/GyWYXLzf42I3DoF4joa2cv8NpBhSJ0WxW6z5xHofBT62mHEx+", - "13RjkUennUGSqsRHNWm8TRe3Ldx16cxNwbuqU6W0UzKKbzId7dB8qsPMUF6RM3e6dO5FXt6Npe5VpQi2", - "T987XW4vg6+iFOw4h28OGWto6O2xq9HSS+fcltR1eugef6f/6clf7crclQ9i64cPSjgHXvQuXb0JrBxG", - "d1/2zrI+nXYT2/zAxXpxejQ1e6vIE8TX527VY+KazHXI7kl7zFlbOjrbY/MQDPuNDuuNyIe68pJs1nRG", - "a+Fw4LUm90s+bKvapCogJtzAYWXro1TASzja2PbqVAW1GGSrKlTLAcGW2xAFdqo8Ow5sH/TUV8Z6N6XW", - "YLbPBjP2iNzAWsba79BUto92vAjEFGkG15UCWLzxF/UxY0fwaVLEaGETTiLbhauvjc9iiQgSDK3qLcq2", - "q1i3xqyvsDPZAPeAAs8KKtawMUgfUeDVQ3PwxlSCFtAB9xTQkvP0E8AyllldQufs5Oy0d0L/Nzk5ecP+", - "93+NxmrWvU8n0BMvPVZ7FIqObTVyCvEU3ocx3CbIb9kMm4S5Asv3KEB4vjrMsv9O8bwpoDeK6e09DpQt", - "8T/t00BRd2wtHFtxl97OmwDzkLbJ3w8cARo96PLsryb0twyEOOQK1K0a3qrhu1fDW92y1S1fJAQKr1mx", - "nQmgtrJI/fm+herp2TlPQfUSnx6PNVbDtOUq9sOx7NxaEffZiri9e1FKAAflOdUqU60ydTDKVLaMTFRv", - "xDabgmTF4KmVVgPzVmMkSxKmtTpsVisxaADb1UuOp4n/0Ms8EfURRW8T/0E4tW1IUaEjHo5/4pb8EMo8", - "laHFNuxoWr81u60jUrkmc+I5lcTitF0rIaSEeGu1z1uXFNxdpUZS8EbOLzGUvX/doNg4HOeqnYoNmaaz", - "gdgQ+7S/YkOuqUZsiHW0YsMgNmr3eZti43v6Z6+UM7I2AkIPckOhceBxEBocGKsZaVG9t6ER+t1tHR6L", - "sREGPDXzeDTQRk2UxEYY8KArFB8U923zQG7v+oceQ7FtOVIdTZG7DmxIshx4oMXeC5dtxV6UpEuD+qgZ", - "GZXzPr7slaVWQqrBHj+l8nMA1d9uqy5Lm5KVdpeoNIXmc5a5paqMlQOcAD6Z87fYp28R8VCHU/SqPpNI", - "dc7MStB2JBo5tlcNSxOVo42bv1PZ2Cz4Vq3VZYa/lYy7l4x7V+hECLoqKt9O6ixFFuecevTyWOoGQiLb", - "a7g6xaiVwruUwnIHVtBMK9S6PVdMVQncKqat+DWJX6GQ1OnEGxe5vHpezw2TgNTES7A2Mhe5LPsIHgHy", - "wdSHTPoq4kZvX3gPCa/Oh8/ZjAcveutSxh94yYjcZq1opuSkwsmnfUE0OEznkLRaIYk8+ycYxvjYTeIY", - "VnM25rcD3tCh3Urce4th/B6SczHYFumOztSQzhjEbQHily9ADN0kRmTJxLgbhg8I9hMqu/76SkVVIelQ", - "ntwkubPt15DxDJF5Mj12ge9PgftgJOfzcBH5kEBO09d0fkd7HtGJuD3qPRv6muLyXA5fIPBXJ2c1b6+u", - "mNcrzzuHwGOH2/eOH/LNyO9DUaw/F5CZw51cYH4OS/RhAmKzKBjTr6shjnVtjjUGz/ZxxqBriLAwnPlw", - "O/TGhv7B6Y2jb8P0liHuh6M3FDwiAqtrN2EWzSS1Yd6BKd1WxzcdYcL6DsVcWzzF1YmsnNl9hOXG5BfY", - "6ovWxyqryVPAXkZ5E80NMUd7x8B1YUTMlrc++45TC5uYpERt6ubzPp3t2JP44HwixZBkMABVUB9fuY7+", - "Wo+plLw4tkt7b09fMWTVLSoq6dPvzeiL9+lsqy49HXwD9MVX3tJXJX1xbK9AX344Q4GZrC7DGXZQ4AB2", - "Nh5VKBiXbKAtOWfQI5iOX09Iu7tH++FsBj0HBe31+YWvz93Ob2dnu1p3FIeUBpjRdhAQRJZOz3kEPvLY", - "ZHRTRBMUzBwoRzIrvIyw9Vf5budbDwZ0ql4MCOwxGzjVoflbjY6Zw4TUcHOYEDt2DpOXN1YJJgv3rFB3", - "a6Sq0aYZ9djapxZwMYUxnqOowR1O6WR3j+Nn4Kesm0hKsVUC10/a/EKnoqi91K1yqVMxWE+SEcD4KYwr", - "XCnSXOy0gyPbV4nUGznm9pSk8zkIZulE+6QtuQwyL0VUK85bpamZ0lTN6pzy88y4tj4VwxmVxHHVtZu3", - "wJUqVeoptS2+l2DsE8dL5LUPjS3Tb+amJKl8M5cl7AP3YSuPVGM68h6/UdVI0oaPVo8wxgIEo/sTXYNo", - "J12gMIwfNVr6MLgP30PyWQy60ZrECqRZhsbTo5OjE10OSMXz6K+061eLcsOTisUWvC0riP0LdGJIkjjI", - "Ia9w06FiNgkCyj/pFN96csheGPGUU2UWeILTeRg+9IQj2vF38YNF+Ds96kTrsqMa/90+sl0MZHYESyfa", - "sR+YZai4hK892F7eOFEMT1fJ1Oj9JVp8tWKOY4FnGzOFbCr86ms4Rihu2DZR5t7yzWb8Jzn03H1SoIZi", - "pirjCsVKWgdEYCfdrpY994g9mVWmtEVNeTTlTfbHc433NW+ldaxmzplWPMedTKt8ljVn/OF4LDf2HRUr", - "bu2RJafkUsCXvKCYfZCZWl1f+bGSkO3TDuwFLW8rij93bpjOCoGBRKJsd3FQlrymBuW3nGaoubgOsxVO", - "k2Jwj1UisGY1WBvci/YyQqZJEq0UwDZA74UzRwhiVShmxfiYbp2GZc8JDVSunyFQbMXgsJa3Xpq31Ci0", - "dRjLRu2z565meuBeMNjmdcE8Mmxj5UVO0hyX7Vo5tJIIRfWwlQdGBXE95qxRE63K5dFNytfFSxnvMX3p", - "MJ6UDcrj7QM/a0pU8AITG6gfvHr1YD1gszhMIlb3IwNBbpQRFNbpI1x2atOAbFlIrFmLSz4qteW49lCb", - "WKn+VyPBJVMTGZ1bZFaNpsmCVsoRtJeSa6JhlyNneM+s2zih1AG9LuMqHxCIScpTCDv3kLhz6JmqQ2WC", - "f88VKUEGKyYeerF0Qwq8jfIMtdmF2uxCW8gu1Eg0C9mALV61cie5lVgWvjUHZIL5EeTylqWcdJhaTxVs", - "5d1eqYAZKa6qAhYd/6YQxDBOHf+6WldA5knG5UES+503nc7z1+f/FwAA///hbZopKkADAA==", + "BwiJ0TQhsNKfxPSWp+hIbhjBO7Pmzj4/Zi9J5RYwfkRu1RB0ORsDtwI37NsDCmpN+XKTP9K2smPF5YXq", + "GncyZKp2YNb8nLbOOivCrrzGGLhQv6biQ7FsmeFBBV7FQGFf8qtQCLOr0rXulMjhSmG026vxzeB8+G7I", + "FKnh1WQwuupfUqVrMPo8GFFF63I4uJp0up2b0fXF7Tn/7fpqfPtpoJeOcqo9EGKpGLAy7xe2PY+lAUXB", + "9cfKc0EDseoh+r0jnPbuIkZEZ91OAL/Jf73qdoJkwf6BO29OT9jbU05y5TrrHMmlT2DEz7504jMrY44C", + "izbqAn4rj/zKbuRsXVr/95AAXzWd0abMnuwjTPibaha1eGJjO9Ls7n8lMKG3zRi5Gi0wSBY3doY9RnjS", + "vHdkWu9/Wdny+FiIu8Mzw55xwJGdEY+PKEx5R51aN6wM1NwsXRUhOnkyAgQyL9YyKq1eipjjLfPe1DvW", + "AkxG8B75BncQFhYh4ibUwVjMRMw6Qua7uIXgEjbRZ+An0NZdN+YOHthh8XjioUns+hMKvPBJv+2beMmq", + "QfSjeR1SmmjWsQAetF0E/6afgn9jy6B7iQLFDzVDM48cuw9jF3q2/maKdULZL7neFKocpX1V6XoPTq+M", + "x7RKePp5DTW8OEZJEefYlFhTUKkdDbowIGPFilZ4nWbgmeiZf3V0Pseq2bOJXWwVO+gaNsytGSoFSjNL", + "ZclsV4wqqeaRdCO6qkVPwFIcXSv+4QxhAmPoSXuiJmTJsM9pxALynDgdh8eEIsw+w/hIEylTQp6dh1oW", + "IVE1mUVcnNG1fATpXz9PBNcIRj5Y/lDBUnxJinEcG1eW446XXZ/S/PXJSc16C3CbVm0yXivd7Y+wwmuD", + "LXwSupjKPCb6KthKH7WgDTegoxbszJoBZxCT29iged6OLpk3KQw85l4uTALYIeF2HJ9Mx2USoH+obuTB", + "gKB7BONUtxbqoIgo5l7waiD+FPphMJMQ10rZLTrh2z0vVTrWj9059BIfKpS2biDNlgNhuh3CA37s9YQm", + "sTPZ4F8V9Hibe21jAaH0j/H5h8HFLf1RpwymM2/XOXlP3YzLq898jXfhUtyYxDbnhTxKgnP16anxEzYH", + "YNdnqQKAzRLHVor7l1KHl3TXzoii0lO7TLtvE//hAvqQwHcs8GtFx+M0bin1O36AS4ddLp0IIJ7kiIeW", + "OdNlPsPNA1yevmFNT7mD7Bn/11mTZDfybUYoFfqrU0O64SN+qbuQrUiNGxjsueEWG4/P+3Tvm0m+EvWw", + "B6lCK402vb5yPORDMd14gQLxz1Obx4BqDJmUZI999za8kiIRN8zkpl+KXXI3ZUHdqkxvVXPoM9HpY5A3", + "Qu6l7HQNQL5lCW8opZgsGutu5rrKX16SN12Zkbt5pp91qUpBX1MWVBcpgWm+OhNnbpNn0jRJW+V7LYpW", + "4cw9MG1rLgfPq0nlVcKwyqOYzN+qxlTtkzKGCxDNwxiO/ZBs2Padsyvr3aK5ORP7IX8CEz3s3bhWtENj", + "VZHSxMsSGDlxIhdWb2pQXV/rF4p8X/qE26+0dNGosFBbg17gzQwtXdXWXrCrU6pR/QHLvkpzEATQN4Ep", + "PjvI0z/9YTq488RH1z+q8BGujHZ0OQWzp684yVoWMLAwrZ5+W2PptLt53WzwdRa9F7Y7O+uaRESK7jxd", + "dBUy1J4vBEYmcacPYJgj34th3ge7VudF+CKJWc5kXUJBkXFUSByEWXKjqa+6Uyih6VsJYOD3QNxsVfFq", + "ybT0p4QfyucYG+88U37vDj3Asqd+SZgMsb/QOe7oAE7vv5OTk1eMlEkuklTJaLWpQB/Dks3kraA1R+sy", + "MEFQJ/d5qqDrLQT29MkgCnNeq8pGbCj8h3HYF9N7TS1R5rrj8zAJiB5c8z1ulYf3rE8Fhoq2+Vz8kkX4", + "i4jWSttvXg6ECTGBuKKIYI5h/Xthe7FD5sbDqXiXip1ZQ4O0jSSkbU3ixELWNFlx2qVixdx5YHXzbUqB", + "6coqQ6YE6vqxO0eP8CDlUvNngb0SMSG9Jeo7VXB9DEm8rJCiW+NH5Wq2G5aouAUpSJB41N+oTfS+D0aL", + "PANqnfJEG0N6FtdMBebXaE/foSoWIU550GI9wo+H9WCBII9Qvk7a9h7LPlZ09w7FmIwhDJrR3iVo2qth", + "cCu/QuUALMycYlZBkxptxve3gpj3JbNIjkxrCTkT6dIuNhpwL4C7q+u7L9ejjywqJP1x1J8M7i6Hn4aT", + "zEtgePX+bjL8NLi4u75ltrnxePj+ivsRTPqjCfurf/7x6vrL5eDiPXc/GF4Nxx/yngijwWT0L+6poDol", + "0KGvbyd3o8G70UD0GQ2USdS5x5fXtOXloD9OxxwOLu7e/uvudsyWInNU341ur+54yuuPg3/dqb4RhiYC", + "UK2JUMcxClKV8EOxwNFwMjzvX1aNVuXUIf6642j4xMN4FJw0cPoQf/PWVfHWE4Af9DmVs/QmlXmcRP8E", + "s1Hy6UuadNSZj2WbyvuxzSSdqtEFBBrpn2adts9SVshUrbkghL4nHnTspCLbh82nrw4J8K06a1GX5vgq", + "1omCscizOTBkQ02tP6HDWksT2oL1wnoLEAiAvyTIxdcRuU5ItU1JDDgH2AkjAj1HmCbSQfRzrJt/c+s1", + "LkwZLNdOgZnlYGmYtLS2kgaDKxv9q5GUCjl2d5tcd0vZg8w5drVr3gM1Q78XulzEs7DHibYzYo+Bz/lV", + "oWAm6kDg3QkJntZy8C1CdJdZMg0GTPX4vBefBjtPrNgNywvigBg6IIriELhzFMx41RuG4Kr5ZY5gTiQs", + "WGdFKPiSZeh5GR4W3VOJC8Wm+A4gP4mhBSjMVVoFJFeygmVg08/pA8yXan76zOIAQSB2lj1/FpOeV0f8", + "gG+SyN4xa5s4oLWhfc69bOIAIsPVBFVt9vnLLAm0AJvlwiB/EEk90Q9d4LPgsEfohxH7zGKOvcQt1JdU", + "1Dslj/f2Eng/pzWTKh+CZcUsUS1xl1WkVssSXvcuKFjU9KopP5uxxltUvWuyEXLFNoyneM1RJNObZ3ul", + "piw1UiOnnb05nAQpNzuT+J6W4X8xgrLPjktZr671LYYx73GTTH3kVpECG68i0b0K895suti/VTZ9JPZJ", + "StHrL1fMYtC/+DS86nQ7nwaf3hoSY/BhqvMQ1F/OmtzFqjCRg0Mxlq16NS6OV4zHShEgKb9YEyw1vAxG", + "d+PL60mn2xl85jaLSX/88W50e8VsItdXSuwJS1pyfv1pePX+7svg7Yfr648VuM9pUTpFEsSLish+9l34", + "q2sFNM9BQELnCcQsQ2dJveK99ZHyzZIe6PMdbCaFAR/bvEQ9/Otlf0xpop59UwqyS2BQt2HN8xYsIIGx", + "zF4gz1E+lvMLOoJHzqnjgWXXOXWeIHyg/12EAZn/uqKPTooebTYDs9iViLoJfeRqMjBzjb/qEpwWI+VN", + "NUpDA7GbZ786B1cBnHl1wgJqK1CNAkkpsCLl0eeTTrfz+VQvSrhP6A4CDo0xrNzZuUkNsYpSHc/pgGNN", + "UIa5NtOafuzVLuwcoJ+xXpK68pqUAhspVWTU3FRARP+XB8TMagdtKW4NTS9paNqiAWgr1TIbGPJXtsMb", + "uPAL83kyZ2PANyDBusRnKptwxykHYSdirR0QeI4LgiAkDmDVsJ0AZhVtSweWDjqsu4/X2qOA58UQY9Uu", + "ldOipaGjbJ6iHz4APNcdN3OA5+qQ/wsXphMHEFdEb5Z+GDjjJIrCmDjnc1aDXj/hZxije1SHXmZdozLo", + "UTSnv6I4D4OeE+YA3wCMn8LYdg7gRKKDgyEx8Nc2XrI8hCMfLHOMIPevsSErj92vBgI7n4NgBiWCjEwQ", + "wCczEhnvwqcMa1Kj1sO+gt4hR2brjioBSYGoxN96MJSSWosv3RyeTCi/DGcoWL1g5Gr8vVb9yL3DuFxj", + "VIdrmdTroNBtd0IaBMMe7pZ46LbeNFWtxnMU4UM1spaMzjs8zbdxyvDJdNv2+fR8cHkBp8ls0+Wru0If", + "xWiR+IBAnGXaYK9lbpj4njOF7IGUax8gEIXhwtgBOY1ZF9KTr4tfRtf54NLJ2rD7wSPwE0r9Wndsn8D4", + "Biz9EBg4UGQDiXib8vqA/ES1DycM6A8xfERhgnvCvViM0alKHlSemH0qz0dK4aEiF1O16UbBm5y1jjIq", + "A9kNXEA/yYxkDpIVwNkGsEL/vCSaZicy93VdDBpO/DQWq7DD2ehdOiGr84XxfeJrFUG7GJEyFmS4SMnB", + "3BgsYRzDEKdMv+WWmK6L1TflVkHmJTkeV+YA/3x6ziIhJgA/mA/SbwTGAfBFvgCjuUo0c4YXWJKiCwIn", + "hvfi8o24Qg7wA+XfHGGqnVU714Zzkdglhfl8SvEh07886zdMRpDQpliXfQObXis4uhga0mUjD3Oh9wRj", + "mJUQ3BoqnvkimMzhC63Y/mopqvCXvB0UZZiqwVSITykcTcMUCkhUFf/Xvp/w8Y6cW3qLp5PgZIq5oxZF", + "uccUH9EKO4Co0sguD1ll/th1s34ZMlDySCyGkNyRZxA0bMtFEL+y52EAr+87b/6qFXaa/m8BRm4/IfPO", + "c3eV/v2bIa9uukrnD5/6553nr8bFicGZ0dVfZ4mQAVjQfOiia6WJGIpDIvDEuk6WJipmUcvhvUNbwYAg", + "V1BhyCwxkkFETL8i9Ps3w7uPg39phH0xqbKcnkOioRYzShky9Dl0P8LloLHWpS6Jq3cPcHnkTJi3FHaY", + "0Y2EvAYUzLdy7uNwoeJCCpGjNVIwp1jVV6ZZb4FsiPLihOrICmOEcQxdkokOEjri/Unv/sxsUNKNyooU", + "x1kXoeggt1Kz5U1SCa2nw/KquJG72FtIZKjJSWHQSKVDdQZ2V0dv9iIvE1l7IBhU+bklufC2Px6eb1cq", + "MEG8B9ikcGwXmWylG8PlBZidK0lGikl1NOlH6nXXtGp7WQX2wMw2Cb+Gl36iSv5Wqmp4nwGgXnqm0AHB", + "0vlzfH3VwzBGwEf/Zk+PfGVHKym1FZMVjpEwdlxA4CyM0b/VItOaQmowqEpghQlYROKhND13udM6DOxd", + "uJr6fW6iloyZKMRhytJMm+pyK/dRORl7l80uaekoznRZmNGSUxkzTRRgtKV5+XcUzIR8u2qixAi/8xTU", + "DE5mAQFR5CO3kHtIn0E4q4tvsShNQf2Kd61SERzNvF8z8bMHNmMpCA336vLGGtLT1lN4qhcWtpGRo2YP", + "a7PWWWSUU4g/ZbR06jgJjrZ0kzVXcDGTlTmxSF3xsbShfZX/bkcYJWuG5q2ajPuPVQU23qrJuLFdITbR", + "rMnIzHgKvXqg04b2oxd9rdI6b//I6m/Z7OmeKFldhLQwpf9uYNGqsiPpuAvhC+j6IAZEZL0xOyUIzkbY", + "8bIuzi8kTuCv9ACP4nAWg8WCXZ1+uQc+hr9u2mHBqOMoyppUdZjCVsbHYZjpNqFQVGx7Aytg3dibFKzV", + "ifhNhsOMLFQ22otTN0vJXp999/Opsdg4IAQuIoPaKz4qEqxYa1yTcmon1ct9WQq8GknFst0vV/S8mE5K", + "91xH4qXDMtHYYLp5FfUCOtaoo56NtA+cUFnxPP1cVWyxPz7vdDsXg/G5Ybm83lb7NNj0aZDjbTsvg7EY", + "e8sPgxR0k6mnueykC9LLTeYD8KkiNxi7qkoLXv3GDNLmK+Yis86MV6lNQxIjiOuXT79ccJ8dYxUf2sbK", + "YMezfzGDTbOkY/Ia2iw/M2/CgVOnVvcsw7X+Updu2V6I1Izo67hCEuSWE4w1zCgmx8plEitmD9OnHitm", + "FBsPriZ3E3Ux6Rru+AlZSn92Phr0J4WSax+HNzf84/XtJcXO5G48uLpQRtafPIqMtTQ12+e6wSjgkZtN", + "Sg3ApnSUpYwtFd8ICPJXqXNWXayjWTkOjgQzU96EKCA8SrG8A4IWtbI1S82mD/5GC7hqBB5vpMn9ZrUM", + "zUHMXcWa7qyKGst7CFOhksCET7cy26qVC5pKcnq3s6p0jwUIm2IkW5qG3HOwKSIzFRJZWr/z6083l4NJ", + "KZtfRZLC/GvXamVMlMt//qDOpln3eYtpdMJwWsL+RhUq9b3QrGHKVmwgbP9gUfO0WHMLzt6TUpw8ASzc", + "OhrkA/DyGpOdG7RmC5QRk6y+rmY48bU4VNdBgbNAvo8wdMPAw3Y6bp0nbGEW55c0ZB8QiAn97df68vFW", + "6KfDy272+K/zQ65AuaB64VUvf4xgACJ0dBUGV4nvg6kP/xyzvBlpqx5aRGHMJhWu+OXGEaBXnM4MkXky", + "PXLDxfEcEHcOSc+Dj/LvYxCh48fTYwzjRxgfh4Cd0d96gRir84YZWtcMA0sW4wg8BdA7r2RHxUbOm5cZ", + "syp3d3lA/q0hBR3QnvCSBEwNTw0P1k9YvHMqO2sVqC3c9yxKY2k4dEvlsYqKalavwFAaq3xQrmt5WG0j", + "Nzi7xYNA5eV9GGAYNz/ykOjW1IHC9v3iSK1Gu6OKxMTKSJNmABE2Gnm9OQ+DezTT5hupLi67VuXnFYiv", + "EHNkDU6ufHJ5JhH5rploncJZqn1c1Zq63Hojo4E051V6znSzC0SBXVXrT54V8hW7uCEo9+ik34KvRYV+", + "u1ahagPsprTiUkBxCryAxHwhm6CFeLrfogXWgxGZG/Re+imnTCDuBfYECIzvge/rh9yZIrp2/bPtaBIN", + "BSf3aWiILHqK8I726PrZFBqNdX0Dd8VWafmBlJbVnOFUHWCtwpZc+BaO2IvcQb3Kofu1cIS85DlKqYkl", + "Qm90nIqjb2On6c6S4HU7UYxCWStF4zcuvppISV+3TdVna9x+Rev6iP/cuF0lXd/nU548qQ0LXdnfTP8M", + "IHJSlUIvDy6Mbj+j4PYyiE1DBsYC2mqwkl1Aqezw3D0Estl6iZg2cPPlaX4zrrbmYuVWHq6W4aJKfOBX", + "lTWVQO0yk0booykkrX8zZFyhIDkfWqgjgjkEHoztTnfetriJYtpaXCkzdeU6vlaJqL4ikPKBpN000Lxr", + "ioZUxskF2xaVUKuMVnSpUzoKQ6gOjQmmKrIJifJr7UAFlKWj1uS2ygef+jOq480XKt7GH/qnnS79z9nr", + "3/kfr0/POt3Op4vX1dhL41k1WWSViexjY9NeLIGpG3oW9epyIwxkJ+ZOMwsASWL4YW06pkM76XhagYlm", + "ASuu5MbQcHnF7Btjw1SWoVlgNUExgDdFlIIn/YqLoNXSyEDBexpWPPg/rFzheMCCYvgft6PLavLYC9c5", + "qdNYOsSkOrApbZQ7B74Pgyqn0AYhepUO8PLZvXAkOrEEzlK7Lx/Qyta+H1wNRkxuvh9OPty+ZW5+o+HN", + "gHno9c8/drqdy+HVoM+c7z4P/49pz7Mb7OaDsCu9VJr7dkgzZevf0fp3/Fj+Ha0LRvnhZE1D7H4/JByM", + "HbvhG3nNo7TG4i3eqdeyerPWmck7u7bln6hzL8bpa7Rqp1ROwwtIZBmFgpNvEth7JYgUDHgO6q0waiw6", + "bf8ujDXwyAcjloHDJuyHNcyUkby3wfqBDBwcvLl0MrUOHOVY7k4OJxLdErLy1ubVgfz2ejXRM1uoZ6lO", + "WQXsS726qNpRg2cXA8Y39QTzRefyIVFkXsyOgv8KjklqlGj/vSjpplXJhdbPC01stMRiI5OnKFKhVX6T", + "2JCGWvZNYr+RoU0YROi4ur3OoYQn+TIXH9jUIrGdSYDKVZHhnJ5izvDeCULiRHH4iDzodR3gxCDwwoXs", + "9IR835lCZwYDGMtrjEpdZ1vDeHM0e/tJgKvtza5JOYWzFtlUaplNFzu1vOTFj5X1JdfFyJji0n4HDPvG", + "nkRB4GUVHmM+1GpX/gUk89BrtFoB+ifeM9Xtz0PPQLUfJpMbmTvbDb2UgqWhxz7fwB3gCQfYzLmJv1oi", + "vJqEBCprzvnMUMVbWyce01LAyrTzKd26zNg16XQ7N9dj9p/bCdOSTCckD8LCVRFaWLwJ8TpMLgicCMaU", + "ro7sK+IJsxK77WqzblFKQGk+P4AxmgXQc7JOzBp0ezu8cARJ7/6W54Mp9HF1+VDWhpF5zieEi2Y78uBC", + "jo6jQ6MPMPkAQUymEJCq+3pu11g1WFbHAThz2Tt/Uz47OTvrnZ71Tl9NTl+/Ofn9zW9/HP3xxx+vXv/R", + "O3n95uTEPk0K4AxGj+wBJmDqMwPYHkK6/dPZfCrH0IVpVVJsys5C2/DoD16VLoxXIalRfi4NVcWimk9W", + "zRPXZlLCTtbLCQN1FxtAVpxXC10S0C0cBvehHfeMlA70aPJDkl2QV6lXzYcdZ+Mw120VOfSbAx4B8sEU", + "+Ygs2fGcK4SbEfkvFKI7lv+299/Jyckr6HyXnX3YFSWVn3/V5yn1Q9PZhOECRPMwhg5tJMTQikQzlmON", + "2Xy6UH7rIhnZ1Gn2m/PJ8DMvyJ3+edO/HRsCjW2iW/gepZEt/Kw0ZgQTpzc/TwpA1hvweO/bOn34dnSp", + "Gb6peszaa1Ub5agoneyVmXtlrifaddPOQhVFtnlx7ZrJqxOVVuDh5V9ijReBFMhRXpQVKmyDYJaIZyxr", + "ITe++Ij5scs7KwWqy+l19KqakK+DbyQG2gbYezAPW1ocg0hVSK8v+yzDwM2/Jh/Yo8jkXzeD8floeMPy", + "pdy+/ZfeuFMUuiWaqhW6gAtCOjSltILuKwVuXTxG2tBJgpw4zw2uqdHPSpebasOiRbJQJmkytK74eQVn", + "FI1q48Hluw/XY57q4VP/qs9TyHwZvP1wff3RuBfseC6bgNW16eOY0l8s3KS7DcrCckVEFobVlxP9O5wa", + "jij6RQeQFaf/GU51R+JONEoj5mQRQY2aDWarrzW1zQLtxa76eU64GWZ3u8oViPetZhJXeUqTyKy0mWtO", + "2DR0w8BD4omF3/JcTaqTGSTKd1Z6XuP9EcikJzwH3wwSLDxX067OjPZNtQbFbq9FGGP9MYkBgbPa7OAK", + "hJe5frystfkiUpZWKcQkXzi7mF/41Vm9+JJTF1fT1WK1aouGF7o0iCmAwwstDmXvjyjIGVLe3V6dT4bs", + "wLq4HfXfXlIl9aL/vlJA0kGkJtKIgtnsGvaS3/XqzVoxmDvWjPTXu+eK/TTmjmJM8hFWhVOSkABfR7Ep", + "jz3ApcGnSA5PydIuYlPezoGDI+iie+Rmkzi/RABj6DmPCAh/9V/1XGFERAOHM/31lsQJ1Ixf936rem6l", + "BpjTk5MToyeWdpi871RDN6hGC/o7nEoxZnuOGwo/rB3dzE/EXRsp+dzC1vMyIOSciTbpGKT6fGi9g8yl", + "Rt4uGww+UXqV3XUaqiRGh591codnA6muPArYX6uFyZ7clRWnH/tDYZQEa+RVLo/yDkE/d+6raTsyWs5J", + "MUUy1kwyls5MrexuZXcru19Kdhvm+AFFe4U35AqimY02JHBh9q803FfqOxur7Y1ZKrTqhLtrepxl2dY2", + "nkRtAwMaZHoxJW8xN4VYVLeESGXUOuopZYq9GVxd8ASxWapYTRbgfM7YNL3s2/75x+t372pPSTbtSvfm", + "vEAxE+MkL06K/jZhcKNI/hKstMHYnUMv8SuCogyd1z6OvhTzpFgKmJrNxryKutELKZeeZYvsWFWPDNcu", + "wmgkYBmXm9CRHOqcd6zTQgvNS/NnDKFNLl2Vx1synfajYC7tN8mjzbODVy12AmY69PpcZVzf5B9sOLmK", + "MOtyCKvoRwiF85heZO71ckHL0pwv75CBG+smZM732hmZHLkTj7ebnhbrV9hcMyjgTSN5YRpyscrAKX42", + "q9xzdUuPvkwDuxOvEM3RzFPMGOXpJl+2qsBQtNkiy+aeMGw2RH31YE4v9yDxyU1lliXRyJhtyeqRQNwi", + "/8T84F0YSmD9Ob6+cjjQ5bAdNoLWiUY+C77QY18Ye9wb0wINWKgdE7SAoaE4DibIfViaXHHoNweLZxW7", + "l0RFXjRgW6aDPZ4WXsqscKz0GfPkTzqUP2aUbU7iarPAJ+U92/bdonGyXOtroFyWJIzcQF/rOZ2R1Sbf", + "hprQ517sya4Qzh0qskehQlXhGELur2IsKbIA32paPDVT9k11RXjkR0LlL5OfHMIpBDGMZT4ThlF2rLCf", + "s02ZExKxa08YPiAomyO6q/wn+Xb+piNCmLO+IrUN7Z1gEi4sJ3tmEp+7RWmiB/gsTv9myMpdEWYTy/+a", + "EmLn9Ojk6ITRMQ/i7rzpvDo6PToR8dgMEyzm2hdlYme6AJn38nmetgogxk5qj6GbDmRxk86l+P6eoUEG", + "NLBZzk5OygN/gMAnc4ai1/y7GwZEJNUQ9aRp0+O/MecrnB6ANXw8iOOQSuHnkn/qVUjSdeSIo/Pmr6/d", + "DpY1XOiqs4bSp+QvAbM7h+5D5yvtz/AXQ+At6xFIm6EqDI5kg31HIVuwQ0IHuC6MiENicH+P3FqMphio", + "Renj6THwqUgJZj24AMjvsYdkfPyd/az+9szx4kOiuT1dsN+xA9JMX7S7w7rzt+nSLvRpiwFtwFwt+AiM", + "Z2KwgITpA39VOPmUZnBEnvPOG54HIRUapaV0VKHG3weyHVuvJu/XEj39pvEkTFwXYnyf+P7S4Sj1cmnS", + "Ssh77nZ+2xXl9Z0F8CkWoOewDFqeDDviYLzaOBg6KN6F8RR5HuS3j4y+OZ1UkZmk+AlrQg+rb71YqBzs", + "A+/b6WoI4yu79hJXkyWdX7fWIXE+wo9B4owe3oZcHm+EGDh2+KYVEJfGrZXJpBJbJHQSifM8Np71Yn8j", + "C9EuQQd7TgxwQFsxYCkGOLVsTwyoB2SEeiR8gAE9FeXf7DSMQl1KgxF8DB+gAwKWrJG1Ft5a6YwFMRGh", + "CW0lDTq0u42USIc3yAQJ614ddzFbnqBzBt2PTdS4CVUL0qEbOxE7J8k4+62KktMtz1Gw64eJd6ze0M0a", + "dClTnLz2sEEcFGACAheWiPicfpbuJWbFevu4ZYA4SZAFXOwLgdVo7RzB6nu92PpPygvbt54cohdG3NlF", + "nGjKfnNz+PF39t/nqv2mUoq1OiptKLOK842slUQ8SbRJOeEpHHcphDa32SK1Us3hzWuoPAqxxrHBdqyV", + "bTkSVzCTkTdHcYVU4/Tz1Uzhx3VijW1LKtVqaP4iFWA/O91fMBJuaX+/aH8BVz7Djaf37g5ukXGtCU2l", + "R+KBHOSbOMLpGMfMTs93CRt3/BJhegHynVxr0wbT1sN8w63tNp1L7LgyZcPNlxlwcqvbJ0JIt55tRGET", + "yvuf2+QwQCSk0vz4O+f45+MoDqfQfLmUb58OyFWeYHZdXrkilwvBzPDp1DchJqMkuGHz2tumTIdeKrl2", + "fOpVEBT8Bt1E2lYYfo92eipchYRVIAhj9G+epV7kNOLB1zxKs2TmJAD50HO43d5h2+O8E/J8mG2r/uDI", + "kRn2gftw/J39x8KK74xpQ6XASp5y2FeRHMreaJ8b00g8DMS9tM7ncbJPqs3pbsC4DTIS5hO/3s3EPOcY", + "S90IfD98otPrXgSKVCtFL/u9SsXiRJfnmAAff8cBtuKWq7Eq9cv8EuAGbJIfzMwo4uTeOzYpIKNllD1k", + "lBLBpqxyNa5klABr2EQqLoq1Sa+60HnllbjEIo3fxl5M/+iaDQG8MNNKlgAFhrPXr3NAnG5CB4rikP4D", + "eu0ZtkesabpEsvoNDogiSe3lY423KfAjAVMfHntgho/T1O/GSyNmt0bWziFzQJwp9MNgpmYVSNOMg1n5", + "Svn59AKwcrMTUUK93lwmE3xnCVp4ym3GMv8kMF5mPOOB2R3yqo+5bUWIWMmdArwvdfGxpt6N1cC/ALNz", + "EfOlzz5WIYfolPL1j836c1sJu53XuxJ+9BaKFpEPFzAgJd2AGS8kHaRP5wA/aCUMa3j8nf6n5nmJV7qY", + "LjnfFAUIncDS1M7GMR76FNAdH/mAELiIiMjLYhAKolFHhaUUC7VNO36hpkcj0xvD6s/On7/xu8/2Z52o", + "9feppnAfJjxJ056IiIyfSyLCfGcgNiLk2A9ndbqKH84cHwVQZj4ScBQlymU4u0QBr8dyiFJFZHkioUjL", + "O10aJAtPw6iFBgWEFZUsB10akufGRKTGDp0ZJBTVDMuGmTHilkfNzBWpGwz3prSqgNXUSUCQv4Gp+w6V", + "dz0CvxEHQxC7c4fNpNR4rlg/66AT6dVrZRQMH6H/C/6VToQC1088aNpf2hJ3tNputcCXLEAHsFVuPZnc", + "hgLGolTMlMc+302Xd2mnHJRWwJVy6lgdslbbswdHriqEGijEIoq1fTfPa6Wp5FeOnctwtv6pQ/+/l4UO", + "m19XlUJtxoMnrcP2Axw9+AFFJua/v8dwI+fOVk+67avU2V6v4CDTXntbtTon43QSZrMqNomBWxl96FxP", + "oO+wZgocohil4eI+YYO2wu7HFXbXBPrjCAQrSLqMnlpp99LSrnSRVzZnE5KGtVAeA13oH3twmszMz4GD", + "R+AnrLKccz64dOC3KIaYhe+DGUABzio1ikrkHiDgSCOLzqF/waY6FOelzcfOfT49H1wyJNSEyjFMYiqH", + "WGVyyqZ65O80Yk4FXyZ4rRE1UFCPp1lDe4NS3/ynyazEYgrPnw8uzSxvxesWNyj+1JhXctL66UV+bnaL", + "2kdvgB9JudDYzuSz4QNcYsUkY5yWtmtuyGJkIHJ71JmwzsMAIw/GksTYc3boshw/ngPuCRSFboSBcJtm", + "zWpYpvA+jGEtMJsydL7jW0PCHDQgZlUvQxcxCfqEyFx99S8WotfAlyWwMezslh/k7deVKxjiLABx54g5", + "ObgwJgAFWZKQqnWmeT/hSiZZ9gZnyBtatbh0S8Qqp0t63KHY4Y4ROohFatAX3Zbp0slycWfRKKxsY2oB", + "MVhvy6nKtQvRlI2R0zzAZY/XfosAirHziweZ4KPct3SA8z9v/ufXotiqdLeyM6FjN4yglTzkLW3XxVqv", + "B+92rWH2lrDW1l1n6055wzJArIGCdsyOYUstjZ/tVpraR7g8FGVt6wGTEhdNGYGhu2UGHTM4QnvcAkN8", + "fzztNQiRZ15MBOs9mZpEy++xB7MJJompA2VOsT/tAbWRIGbcJIA5pRwrzuQ6js0xJVrWnlFcJW3NCftq", + "TijVebdQoGtvn5VTlK6I7DLO5zxav6hNs7sCTqYYEscFgYdYRitJ1xu9PVSt2LnF0GNsxGEh9HpchgcQ", + "aXNlXkKG8jw7vXgorN1AsEsR00r2vLYl8ZLJdo7fKl2ra3jbOWdFzRzgBPBJDGwUzbztz/14w1DA0WHz", + "gMPeb1JSdlgJOW7V3+WbjSCPOtYTBe4UgNvn4F09B18ZXoBT/kx5057n7bU4dsHif9sEUoM6SdE4RfB+", + "qXGCWxHLAOHJtegvWykmDvO2ZSkaZNR4KxZeUizYsn5XIUx69FcEfaUKvNlgwmc7ZItJys8/ORfPQtIe", + "7kaLyQpnbJHRKhOS1x+bB55aIXdspum8X5LhtnEF4Ju08hXgBdKcW8sHmdm8lQ+Hd8pbKPssimaRlcWs", + "UAuEZJQpB5w4CRzRszpDOveguESYcC8KWYXzUGVaOeZSQUONf5IFoGuHYdZDsykHpaJtlll/A49765in", + "T6snohfydKFw8/qYjJT/F1aTmxiAFvU0afs72fqOtd4qsWXJVvgbH3OVSmt8Z5H+hrQmvCEKZne8WuiO", + "IO9rHIgeeo/Cp8fikSDzJLpbVLoSvawRmwq2URJIidY8Q4MqRdtsKvuTKoHtzSI9qOxiLOxP3ChEAbE8", + "dxcoSAik13H5VwzBgxc+BelR3OAYfg/JDZ380A9hduBJ32AlbEYYrDvdDvwG6BZ33nTOTs5Oeyf0f5OT", + "kzfsf//XIHdE9/49v4ls4oBkkKaewyqoIYVvDWDvUYDwHHpv2eDNwd2+bMyR2grSkfFJKx/3VD7md2fj", + "UhIfuyBwoW+OQjtn39PMVzp5x5v83A+UDAVMVakpxcazCYaOK5G20ygyNqkPPZ6RsPZlUjZv09G1cfkl", + "GVWQDBuXTDGMfLCsKiNHv1dKJt7kp5ZMHAVNJFMskbZLycTBtBVMsWjdyqVWLpXkUkEubFAuiSTDNt63", + "spBDnfetqBPRut/us/stJxeHDmsXv8baX9HmqwRDCpoYp6PY2lsl0VkDKjpUQFo9yYt7uKrs08DFNWXk", + "9i0+7+OaIiaTmwLFa3u5msrlpJvY+rkKP1eBjyav3JIpX8jTVdJIE1fXfSyz8HP7upZrKFjwfgO1ibm7", + "in/Y+bvWyowD93ilk8u3R8nC9b6vGVbMwO7WDm3L/9KfteX9vXB1qWXvrkpuNS6tkn6FT6tQDw18e8hu", + "rQUF+EfjUemt2vKowV215piEAT0FezEgsMduoHRzxd5bclmdP2vtsXjgHq3b5bDteaf+uIq7dFFtBcMe", + "Ke4aebD6ya6/wd+EmOX3QIEbLlAwS+l1ATEGs4oTfgRdiB5bGdREBgWJ75coP1g6EVj6IfAcFDggWDpi", + "td0Ogd/IceQDVKC04pQ7kSEWmUlzeLoHPoatcmGocMgZT8Nuq3K4zT1d+Az34iSoe+PIZw2sfeXIsgS2", + "Lx37n7cUi0yOVm8dO8v6yPzwQewjiFmua2gF3haDAnxAmoCysdpMe+P4bZmr5kCiFSgQaRydTYYdGG/Z", + "xf/LHJI5FwCiFpZz0X+P6ekVBv5S/T2tUKoTSIG/vJMNahWVaRj6EAQWMR25crUWOHuh8A5NUV1jnIdF", + "Zt8Xi/dw7n0wY0ftk6CLMGYOGCoZpPdLEHhOmBD6p1AfMdUfaQOpCx45F/AeJD7Pd/8/lB7+x0H3ThJg", + "yI5x3fLFTHdy0E4lCe2scmfTF+DWaWjfKvzkNEpV0ZW/j+jva75EqRrusYdw5INlj7lL1Oi7oi0dVrhX", + "hPcVSnC1DnzBB2NuFwetDyuiFafvWDmkiHhJgT6BOrMioMjSFylsvmUTvJYEWtHViq6mokvySY/ySbXk", + "yvEo0x70Cf+z9HYVkmsgBht6hyu42ntue8/9Se65OzvOMrnQnmY/0mmWOz12crKJ67U57GfCG0iv0vyF", + "veLoat1LTwXqFKTUPFXnSIGEwn9z12/UitYMCUA+buZnqlJI+95UdPssMNAGGDzPz8znU/mlppREnuRA", + "4DFnsvT8J2F6lRTFkv674zGi+O+OExkepDP6sXQ7y8HAbZsz1tPwCqws72BzGa7AZe0pvseneDH8zZKh", + "uyWCXoHFj0XJuCpOJzzLF0mY4SjP90e1XDyWNelW5GV1ekVd/zFZW71+tiy9p05e52Hiezyell4kdZrL", + "HuUmyXFVWiDyRWQNS/ZkUWKXheXyIHduqbe/OlAGYnV8rI1eP09Fmkysag0gP65EXamqYytUWz2pKLsI", + "WqBgVq8tiXaNpdd7SCZiioO9+2hlkAcjMucZS3hWM8edI9+Locl1g3VoKP22L0j45rSS5OAlSRV/blq8", + "wEjIFPnn8zGI3Tl6hHVakGglwKTdtSJkTGAk3HX7cmAL8SHHM1pPJbyt6+7qGtk2ZZLYd7HnVlIpn1Sy", + "rQu6+3xMKdcVcjKVhVSO/RXml/KJbj+VTVWiKWXheplkcy8Tpfvt5dFA1lhtpdFPIo3s71qtLDocWaQw", + "/vYlkR/O6jyl/HDm+Cgo6UZlc/RlOLtEAbS1BrVi6GXjmXz4CH0rlyHeMjdzFTNIOqC93iHoe8YMcpAe", + "vA6bTYGjopgJ69AUkDHvpQ0lASxQIIy9qvWzz2+XfC0NJ79W+xrwwKf3UAxdEe1eAcWF0mwVSLL+2z2k", + "VGnQFtBfNwVdKoWVs+AynDU/BoSjUUVqc+YBgYUnkcFxf8J+PlcdXzbtmMMH5xPVJenlrkkv44rDIWzk", + "fCOQ+mPT+ApeNymxpdlphT9Nkch1FJ26ztWajLlrjHhhryTwpgmZ0sAOMYPxyWc33nIvS/EyZVJL7bu9", + "bXBi9ELILxrwGz+BS4U0bJktl9G0OgdTwGdDwayarw4nE9OWvE45ApocblFMEUkQj8t4gcKd7Tm3/jkn", + "+GQF1qs4746BTwkjmPXgAiC/N4vDJKp8OKXKnbwFCvJiYzhsAEcMUGTdPm0yoC3e0waHEum0/ZNQh5iG", + "JaeMm9DyTv41sYJaG51j1lef8lx1jPHTh1SoN7cCbuzOuhLKG13tTrfL3iucgBoaavlae/fTcttmT8lj", + "DAmpcy3CbPdkF0d2qc5moJALCmZj0edAkvru6JhUELPGGanuSctKmmudBk0b46MI9Uj4AGuS4Tn9m6HD", + "21VzTT9CE9qs1SfxMfMruhkyfGCL1JE6PpH+Ua0Nvag8UorkqFWYIf1xnVIuQUbtdsTe6ogMAZLWFbVw", + "myaM4qQtf204bDZjpoYMVnXgWHhL8epyOZcpU9rVzGmmTbe61+4JD3Bp5ZxA2zVPP8PI4CNc2uQ1yWBK", + "3ZeHF9g2HyaXFY0BlC7Rw4sVQcxi0NZI5WMD4SgJeBylMHy9iKsH28+XcfRgU++Bm4cKh+rkUUEsWQYh", + "uHQegZ9AfR4h+A0sIh9Skf0Al6dvWNPTTpf+64z/64yK9+p8Q582m24oWwZPXJpmHKqmc9Z4ePiZhlaK", + "tGu9awKzz6WitDDkrm9CZuMadJD2CsAQwHBRYxYWiYlfxL2HU0ITmy/kPX527+qz/9zNrCPBn0I9hd9c", + "CD1oKOfI96YBn9dfTI6nif9gdqd7m/iijhHEmUzAlUKB9vmJBQNdfkPhgF9SOuDm4qGNvtgz+cDYVBUS", + "eMNSwgWBC/0Kt1v2nRsylMTZORXXJDW4Wwkf4WdWKBgC7BUKcWGIYeSD5cbFRuawRf/1lF2Whzw58baK", + "eMgfwunf0LXQXBjSYJajpBVSeyukRoxStyOfmBnN0sbKbXMWdtaPcNk+62XGxpVu6wzZ7Y1dd2N3hO13", + "k3wgTgPjOc15EDc7mkfyiPlZj2aOgH05mjdjVuPAtVr9T3pgfmf/7T0hMu/JT8y6XRt+BAjgh2dQaSC8", + "AAS8h+QLIvOJZPta+SHZRy8+SiDv+u3yhz/l6aatko6BUUV7yud92RTMWPNuV0Pk1fyMgkdEYNOACdlL", + "7wQ6ZF9b3Vf6fir4WMnrU2K79fXUhUNktLilGAg+QSWtt89ZStQDR4ldsAPH7YtGOHBwVwlsEITxs8f2", + "np3tSOsFxO6dq8i3OrkAAzD1YS8GBPbYmJQ9BK+tohcLKSR/6PF/P3MR40MCy8Lmgv2OUzOSjaDhfQ7W", + "ey/P9dWw9VJ0HPrJXytbOIXss2zJsRknwoxcTbpofh9rI+ibccLhRNEfCidsN9B/Na3gxUL9LTmXw3cw", + "nCtC8BtzbtXJt4CLKWO+RjdI2UvP4p/Y1/YGKalRwcdKN0iJ7fYGqbtBZrS4mSBBMd7xd/6HhRLoAAGE", + "cx+Hi7ogW04NP4YqKJZtgo1/3inv/rYV3l1FB/w5uHaPctVeGVLTpkya25gG8qIrCdkijVRpErMI+DF0", + "4L0QAdtVfvl22Sm/Ah17kvLKUnpp9GCxb63wemHhZZQrKwivKq0nisMFJHOY4N6C6qBuffmirIsjuqQ+", + "eHWZKW/Srp/EZD/ERYHAb+Q48gEqUEVxpCZ3gDKWW6Z8aaakHKDZl03dQP5JYAKt2ZC1bsyB/0V7HRDz", + "HXZk8yEFq27fHpKjvdUyWDiPMMYoDFqZuE8yMd2dskSUnLOqTMye+mxcveP0sbHO13sECLykDdu8Gvtc", + "nXYTORhqMbnNTAspne1BtoUiLLsqq5HntQbBBAo7t36GBSu4iptM3DJvi0v+66oSV/ToRaGP3GV9yknZ", + "weEdbBJOSlfoG9ajTTd5rEPLao9Ghd1oH492nrUV+8B9qE40OaZNnCc4nYfhQ/k5lX3+wr+2z6k8x6SK", + "kya3hwKq94kddlTx+DYACZmHMfo39PjEr3cz8SdI5qHHKnoA3w+f9NWW+QYxPZCzgHqesY9rMeIxJiAm", + "RnYc06/8HLvuJ2TusMtKkSFvsXy2YQBdU4SynofIma9OzjR4ULmHoUwcKzmszCHwhNeIH3KCqbF4sg2H", + "bhIjsmT4ccPwAUE6KCuK9FWlB4bS/IySEOgOrEwHdXl/x1fjIgEWBHKAWzks5PDVeKiiqoEkLmK5lcV7", + "J4vLjJBK4qvxGumGCwPrGKyNxmAIyPNXZZbhzdFsflLrqIrirrYMvUcMbeQ8S46uPFFFnc7eLp6sROnw", + "Q3u52r65QIeYZjaDtJ51bmfaR5V9eFRJ92bTz8y6quqVrJsVUHemS85QhdObE+KB2PG6+1rZfZsSQ2zR", + "ivKhlQg7K4Wq0uIT4PVQ60SEeqjTn+hGr1plu1pO1OYE7BMCF5FIbsnaKuLDJDgOLRlgK0GqXOIRZr7S", + "QoRwIvD374Lwwo94dYyyK4aOIe1YkTuMJVm05WHWvGXhfcxmFieB2Koaj3YURAnzh+CPu7rlPu+FptLm", + "MquQL2zDX0KgZGuqtAXwZsJZoE64vIdkzIdtRcvLaQfNsvQaLA1iuPZCsc8XCrlLW5EaBOCHHiaA1BgM", + "AX5g1aCEpbDGSjgB+GHMBrUXEcOLH9E2mCKiAYdqcd3y6B6YAU1ssIv0SMJrpvcUxg9VySIyB2yjS1Pr", + "zZQFk3BUfGFIpQipqupJkZEGvPCOjtyO9rlt397PFfJfPYmhGMTEQj/9O3mOfzg2dlSMVzOz1ygFodza", + "lnP376FcZbyVDktGFdUPafSE5MK72ks+Oxt++sMyw0Rb83ojGaql9pCP0Vvdu1IimhuCmteiUKv/akpS", + "KCV728IUSmEKBS+4xqCbq6/8cmUqdHBbl7NXbL05gmkvqXtZviK/R+Vw4GpTUhOB8139Z50fS44Tak9g", + "QaaH7NZSYH09aCoGD1hNENu1amaB1s3FHNeff0Gqj+nv5mlqdX4+Zo+RtY9J/MmSM7QK9FENXw/Z6C1z", + "vzxzZ1lMbpQilBzGdd6d8jhi292atXdk1v6i4j6wyR+SbVJTlWFzEgfPQQS3pEeM2ditvDkYZYJvWKtR", + "/EAaRRq7InyGKiNDRaV2xuK+n76PY42uUcX6LHCSu7IMZGG/VgZsHMBLgIkzvGAJ6+fQ8YHcQVOaIoDJ", + "0DPmKXp1pstTtAMf2yYFPUtl+VqTyP751qwgS+wdb+xkIbZ6mWAt7TSanzJxmgfvQeKTzpuTbk5U7CKF", + "Wjr361UmH/NMatOlwybQTyo+mfM57ELtah97Nq9vbTIlYzpmbTDQuYxrmALizkuPPVUa0+EEA23Ly0F5", + "J+HIsHXbF9Ek5aeSTT/2RIql5nuq9I2SYOjhXOrZtRBczrfb0CAkIpDa16Oa9GicbHbxcoOP3TgM6jUS", + "2sr5O5xmQJEYzWa17hPncRj81GrKweR3TTcWeXTaGSSpSnxUk8bbdHHbwl2XztwUvKs6VUo7JaP4JtPR", + "Ds2nOswM5RU5c6dL517k5d1Y6l5VimD79L3T5fYy+CpKwY5z+OaQsYaG3h67Gi29dM5tSV2nh+7xd/qf", + "nvzVrsxd+SC2fvighHPgRe/S1ZvAymF092XvLOvTaTexzQ9crBenR1Ozt4o8QXx97lY9Jq7JXIfsnrTH", + "nLWlo7M9Ng/BsN/osN6IfKgrL8lmTWe0Fg4HXmtyv+TDtqpNqgJiwg0cVrY+SgW8hKONba9OVVCLQbaq", + "QrUcEGy5DVFgp8qz48D2QU99Zax3U2oNZvtsMGOPyA2sZaz9Dk1l+2jHi0BMkWZwXSmAxRt/UR8zdgSf", + "JkWMFjbhJLJduPra+CyWiCDB0Kreomy7inVrzPoKO5MNcA8o8KygYg0bg/QRBV49NAdvTCVoAR1wTwEt", + "OU8/ASxjmdUldM5Ozk57J/R/k5OTN+x//9dorGbd+3QCPfHSY7VHoejYViOnEE/hfRjDbYL8ls2wSZgr", + "sHyPAoTnq8Ms++8Uz5sCeqOY3t7jQNkS/9M+DRR1x9bCsRV36e28CTAPaZv8/cARoNGDLs/+akJ/y0CI", + "Q65A3arhrRq+ezW81S1b3fJFQqDwmhXbmQBqK4vUn+9bqJ6enfMUVC/x6fFYYzVMW65iPxzLzq0VcZ+t", + "iNu7F6UEcFCeU60y1SpTB6NMZcvIRPVGbLMpSFYMnlppNTBvNUayJGFaq8NmtRKDBrBdveR4mvgPvcwT", + "UR9R9DbxH4RT24YUFTri4fgnbskPocxTGVpsw46m9Vuz2zoilWsyJ55TSSxO27USQkqIt1b7vHVJwd1V", + "aiQFb+T8EkPZ+9cNio3Dca7aqdiQaTobiA2xT/srNuSaasSGWEcrNgxio3aftyk2vqd/9ko5I2sjIPQg", + "NxQaBx4HocGBsZqRFtV7Gxqh393W4bEYG2HAUzOPRwNt1ERJbIQBD7pC8UFx3zYP5Pauf+gxFNuWI9XR", + "FLnrwIYky4EHWuy9cNlW7EVJujSoj5qRUTnv48teWWolpBrs8VMqPwdQ/e226rK0KVlpd4lKU2g+Z5lb", + "qspYOcAJ4JM5f4t9+hYRD3U4Ra/qM4lU58ysBG1HopFje9WwNFE52rj5O5WNzYJv1VpdZvhbybh7ybh3", + "hU6EoKui8u2kzlJkcc6pRy+PpW4gJLK9hqtTjFopvEspLHdgBc20Qq3bc8VUlcCtYtqKX5P4FQpJnU68", + "cZHLq+f13DAJSE28BGsjc5HLso/gESAfTH3IpK8ibvT2hfeQ8Op8+JzNePCity5l/IGXjMht1opmSk4q", + "nHzaF0SDw3QOSasVksizf4JhjI/dJI5hNWdjfjvgDR3arcS9txjG7yE5F4Ntke7oTA3pjEHcFiB++QLE", + "0E1iRJZMjLth+IBgP6Gy66+vVFQVkg7lyU2SO9t+DRnPEJkn02MX+P4UuA9Gcj4PF5EPCeQ0fU3nd7Tn", + "EZ2I26Pes6GvKS7P5fAFAn91clbz9uqKeb3yvHMIPHa4fe/4Id+M/D4UxfpzAZk53MkF5uewRB8mIDaL", + "gjH9uhriWNfmWGPwbB9nDLqGCAvDmQ+3Q29s6B+c3jj6NkxvGeJ+OHpDwSMisLp2E2bRTFIb5h2Y0m11", + "fNMRJqzvUMy1xVNcncjKmd1HWG5MfoGtvmh9rLKaPAXsZZQ30dwQc7R3DFwXRsRseeuz7zi1sIlJStSm", + "bj7v09mOPYkPzidSDEkGA1AF9fGV6+iv9ZhKyYtju7T39vQVQ1bdoqKSPv3ejL54n8626tLTwTdAX3zl", + "LX1V0hfH9gr05YczFJjJ6jKcYQcFDmBn41GFgnHJBtqScwY9gun49YS0u3u0H85m0HNQ0F6fX/j63O38", + "dna2q3VHcUhpgBltBwFBZOn0nEfgI49NRjdFNEHBzIFyJLPCywhbf5Xvdr71YECn6sWAwB6zgVMdmr/V", + "6Jg5TEgNN4cJsWPnMHl5Y5VgsnDPCnW3RqoabZpRj619agEXUxjjOYoa3OGUTnb3OH4Gfsq6iaQUWyVw", + "/aTNL3QqitpL3SqXOhWD9SQZAYyfwrjClSLNxU47OLJ9lUi9kWNuT0k6n4Nglk60T9qSyyDzUkS14rxV", + "mpopTdWszik/z4xr61MxnFFJHFddu3kLXKlSpZ5S2+J7CcY+cbxEXvvQ2DL9Zm5Kkso3c1nCPnAftvJI", + "NaYj7/EbVY0kbfho9QhjLEAwuj/RNYh20gUKw/hRo6UPg/vwPSSfxaAbrUmsQJplaDw9Ojk60eWAVDyP", + "/kq7frUoNzypWGzB27KC2L9AJ4YkiYMc8go3HSpmkyCg/JNO8a0nh+yFEU85VWaBJzidh+FDTziiHX8X", + "P1iEv9OjTrQuO6rx3+0j28VAZkewdKId+4FZhopL+NqD7eWNE8XwdJVMjd5fosVXK+Y4Fni2MVPIpsKv", + "voZjhOKGbRNl7i3fbMZ/kkPP3ScFaihmqjKuUKykdUAEdtLtatlzj9iTWWVKW9SUR1PeZH8813hf81Za", + "x2rmnGnFc9zJtMpnWXPGH47HcmPfUbHi1h5ZckouBXzJC4rZB5mp1fWVHysJ2T7twF7Q8rai+HPnhums", + "EBhIJMp2FwdlyWtqUH7LaYaai+swW+E0KQb3WCUCa1aDtcG9aC8jZJok0UoBbAP0XjhzhCBWhWJWjI/p", + "1mlY9pzQQOX6GQLFVgwOa3nrpXlLjUJbh7Fs1D577mqmB+4Fg21eF8wjwzZWXuQkzXHZrpVDK4lQVA9b", + "eWBUENdjzho10apcHt2kfF28lPEe05cO40nZoDzePvCzpkQFLzCxgfrBq1cP1gM2i8MkYnU/MhDkRhlB", + "YZ0+wmWnNg3IloXEmrW45KNSW45rD7WJlep/NRJcMjWR0blFZtVomixopRxBeym5Jhp2OXKG98y6jRNK", + "HdDrMq7yAYGYpDyFsHMPiTuHnqk6VCb491yREmSwYuKhF0s3pMDbKM9Qm12ozS60hexCjUSzkA3Y4lUr", + "d5JbiWXhW3NAJpgfQS5vWcpJh6n1VMFW3u2VCpiR4qoqYNHxbwpBDOPU8a+rdQVknmRcHiSx33nT6Tx/", + "ff5/AQAA///yYOzZDkEDAA==", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/api/v1/server/oas/transformers/v1/cel.go b/api/v1/server/oas/transformers/v1/cel.go index 9ace017860..0a7c20443b 100644 --- a/api/v1/server/oas/transformers/v1/cel.go +++ b/api/v1/server/oas/transformers/v1/cel.go @@ -9,9 +9,9 @@ func ToV1CELDebugResponse(success bool, output *bool, err *string) gen.V1CELDebu } if success { - response.Status = gen.V1CELDebugResponseStatusSUCCESS + response.Status = gen.SUCCESS } else { - response.Status = gen.V1CELDebugResponseStatusERROR + response.Status = gen.ERROR } return response diff --git a/api/v1/server/oas/transformers/v1/trace.go b/api/v1/server/oas/transformers/v1/trace.go index 35cd6ee90a..d139658aed 100644 --- a/api/v1/server/oas/transformers/v1/trace.go +++ b/api/v1/server/oas/transformers/v1/trace.go @@ -43,9 +43,9 @@ func ToV1OtelSpan(spans []*sqlcv1.ListSpansByTaskExternalIDRow) []gen.OtelSpan { SpanId: s.SpanID, ParentSpanId: &s.ParentSpanID, SpanName: s.SpanName, - SpanKind: string(s.SpanKind), + SpanKind: gen.OtelSpanKind(s.SpanKind), ServiceName: s.ServiceName, - StatusCode: string(s.StatusCode), + StatusCode: gen.OtelStatusCode(s.StatusCode), StatusMessage: &s.StatusMessage, Duration: s.DurationNs, CreatedAt: s.StartTime.Time, diff --git a/frontend/app/src/lib/api/generated/data-contracts.ts b/frontend/app/src/lib/api/generated/data-contracts.ts index 2ef371de5c..aabc2cebd2 100644 --- a/frontend/app/src/lib/api/generated/data-contracts.ts +++ b/frontend/app/src/lib/api/generated/data-contracts.ts @@ -256,6 +256,21 @@ export enum TenantVersion { V1 = "V1", } +export enum OtelStatusCode { + UNSET = "UNSET", + OK = "OK", + ERROR = "ERROR", +} + +export enum OtelSpanKind { + UNSPECIFIED = "UNSPECIFIED", + INTERNAL = "INTERNAL", + SERVER = "SERVER", + CLIENT = "CLIENT", + PRODUCER = "PRODUCER", + CONSUMER = "CONSUMER", +} + export enum V1LogLineOrderByDirection { ASC = "ASC", DESC = "DESC", @@ -521,9 +536,9 @@ export interface OtelSpan { span_id: string; parent_span_id?: string; span_name: string; - span_kind: string; + span_kind: OtelSpanKind; service_name: string; - status_code: string; + status_code: OtelStatusCode; status_message?: string; /** @format int64 */ duration: number; diff --git a/pkg/client/rest/gen.go b/pkg/client/rest/gen.go index 5c4312f74b..10c88199a8 100644 --- a/pkg/client/rest/gen.go +++ b/pkg/client/rest/gen.go @@ -90,6 +90,23 @@ const ( LogLineOrderByFieldCreatedAt LogLineOrderByField = "createdAt" ) +// Defines values for OtelSpanKind. +const ( + CLIENT OtelSpanKind = "CLIENT" + CONSUMER OtelSpanKind = "CONSUMER" + INTERNAL OtelSpanKind = "INTERNAL" + PRODUCER OtelSpanKind = "PRODUCER" + SERVER OtelSpanKind = "SERVER" + UNSPECIFIED OtelSpanKind = "UNSPECIFIED" +) + +// Defines values for OtelStatusCode. +const ( + OtelStatusCodeERROR OtelStatusCode = "ERROR" + OtelStatusCodeOK OtelStatusCode = "OK" + OtelStatusCodeUNSET OtelStatusCode = "UNSET" +) + // Defines values for RateLimitOrderByDirection. const ( Asc RateLimitOrderByDirection = "asc" @@ -200,8 +217,8 @@ const ( // Defines values for V1CELDebugResponseStatus. const ( - V1CELDebugResponseStatusERROR V1CELDebugResponseStatus = "ERROR" - V1CELDebugResponseStatusSUCCESS V1CELDebugResponseStatus = "SUCCESS" + ERROR V1CELDebugResponseStatus = "ERROR" + SUCCESS V1CELDebugResponseStatus = "SUCCESS" ) // Defines values for V1CreateWebhookRequestAPIKeyAuthType. @@ -766,19 +783,25 @@ type OtelSpan struct { ServiceName string `json:"service_name"` SpanAttributes *map[string]string `json:"span_attributes,omitempty"` SpanId string `json:"span_id"` - SpanKind string `json:"span_kind"` + SpanKind OtelSpanKind `json:"span_kind"` SpanName string `json:"span_name"` - StatusCode string `json:"status_code"` + StatusCode OtelStatusCode `json:"status_code"` StatusMessage *string `json:"status_message,omitempty"` TraceId string `json:"trace_id"` } +// OtelSpanKind defines model for OtelSpanKind. +type OtelSpanKind string + // OtelSpanList defines model for OtelSpanList. type OtelSpanList struct { Pagination *PaginationResponse `json:"pagination,omitempty"` Rows *[]OtelSpan `json:"rows,omitempty"` } +// OtelStatusCode defines model for OtelStatusCode. +type OtelStatusCode string + // PaginationResponse defines model for PaginationResponse. type PaginationResponse struct { // CurrentPage the current page diff --git a/pkg/v1/features/cel.go b/pkg/v1/features/cel.go index 93bba02396..b8b71eeeba 100644 --- a/pkg/v1/features/cel.go +++ b/pkg/v1/features/cel.go @@ -64,15 +64,15 @@ func (c *celClientImpl) Debug(ctx context.Context, expression string, input map[ return nil, err } - if resp.JSON200.Status == rest.V1CELDebugResponseStatus(gen.V1CELDebugResponseStatusERROR) { + if resp.JSON200.Status == rest.V1CELDebugResponseStatus(gen.ERROR) { return &CELEvaluationResult{ - status: gen.V1CELDebugResponseStatusERROR, + status: gen.ERROR, err: resp.JSON200.Error, }, nil } return &CELEvaluationResult{ - status: gen.V1CELDebugResponseStatusSUCCESS, + status: gen.SUCCESS, output: resp.JSON200.Output, }, nil } From 9e49f85fcc901b60f34a88358d43d23aa765930b Mon Sep 17 00:00:00 2001 From: Mohammed Nafees Date: Wed, 11 Mar 2026 01:06:49 +0100 Subject: [PATCH 15/17] TS SDK example --- examples/typescript/opentelemetry/run.ts | 43 +++++ .../tracer.ts | 8 - .../worker.ts | 14 +- .../opentelemetry_instrumentation/client.ts | 7 - .../opentelemetry_instrumentation/triggers.ts | 162 ------------------ .../src/v1/examples/opentelemetry/run.ts | 43 +++++ .../src/v1/examples/opentelemetry/tracer.ts | 62 +++++++ .../src/v1/examples/opentelemetry/worker.ts | 70 ++++++++ 8 files changed, 219 insertions(+), 190 deletions(-) create mode 100644 examples/typescript/opentelemetry/run.ts rename examples/typescript/{opentelemetry_instrumentation => opentelemetry}/tracer.ts (84%) rename examples/typescript/{opentelemetry_instrumentation => opentelemetry}/worker.ts (80%) delete mode 100644 examples/typescript/opentelemetry_instrumentation/client.ts delete mode 100644 examples/typescript/opentelemetry_instrumentation/triggers.ts create mode 100644 sdks/typescript/src/v1/examples/opentelemetry/run.ts create mode 100644 sdks/typescript/src/v1/examples/opentelemetry/tracer.ts create mode 100644 sdks/typescript/src/v1/examples/opentelemetry/worker.ts diff --git a/examples/typescript/opentelemetry/run.ts b/examples/typescript/opentelemetry/run.ts new file mode 100644 index 0000000000..420a15d30c --- /dev/null +++ b/examples/typescript/opentelemetry/run.ts @@ -0,0 +1,43 @@ +import { getTracer } from './tracer'; + +import { SpanStatusCode, type Span } from '@opentelemetry/api'; +import { hatchet } from '../hatchet-client'; +import { otelWorkflow } from './worker'; + +const tracer = getTracer('opentelemetry-triggers'); + +async function runWorkflow() { + return tracer.startActiveSpan('run_workflow', async (span: Span) => { + try { + const workflowRun = await hatchet.admin.runWorkflow(otelWorkflow.name, {}); + const runId = await workflowRun.getWorkflowRunId(); + console.log(`Started workflow run: ${runId}`); + + const result = await workflowRun.output; + span.setStatus({ code: SpanStatusCode.OK, message: 'Workflow completed' }); + console.log(`Workflow completed with result:`, result); + } catch (error: any) { + const errorMessage = Array.isArray(error) + ? error.join(', ') + : error?.message || String(error); + console.error('Workflow failed:', errorMessage); + span.recordException(error); + span.setStatus({ code: SpanStatusCode.ERROR, message: errorMessage }); + } finally { + span.end(); + } + }); +} + +async function main() { + await runWorkflow(); + + console.log('Waiting for spans to be exported...'); + await new Promise((resolve) => setTimeout(resolve, 5000)); + + process.exit(0); +} + +if (require.main === module) { + main().catch(console.error); +} diff --git a/examples/typescript/opentelemetry_instrumentation/tracer.ts b/examples/typescript/opentelemetry/tracer.ts similarity index 84% rename from examples/typescript/opentelemetry_instrumentation/tracer.ts rename to examples/typescript/opentelemetry/tracer.ts index 2ef7267ebc..0e62d95f7b 100644 --- a/examples/typescript/opentelemetry_instrumentation/tracer.ts +++ b/examples/typescript/opentelemetry/tracer.ts @@ -25,7 +25,6 @@ if (isCI) { process.env.HATCHET_CLIENT_OTEL_SERVICE_NAME || 'hatchet-typescript-example', }); - // Parse headers from environment variable in format "key=value" const headersEnv = process.env.HATCHET_CLIENT_OTEL_EXPORTER_OTLP_HEADERS; const headers: Record | undefined = headersEnv ? { [headersEnv.split('=')[0]]: headersEnv.split('=')[1] } @@ -46,23 +45,16 @@ if (isCI) { traceProvider = provider; - - // NOTE: Instrumentation has to be registered before the instrumented libraries are imported registerInstrumentations({ tracerProvider: traceProvider, instrumentations: [ new HatchetInstrumentor({ - // Optional: exclude sensitive attributes from spans - // excludedAttributes: ['payload', 'additional_metadata'], - - // Optional: include task name in span names for better filtering includeTaskNameInSpanName: true, }), ], }); } - function getTracer(name: string): Tracer { return trace.getTracer(name); } diff --git a/examples/typescript/opentelemetry_instrumentation/worker.ts b/examples/typescript/opentelemetry/worker.ts similarity index 80% rename from examples/typescript/opentelemetry_instrumentation/worker.ts rename to examples/typescript/opentelemetry/worker.ts index 3b49e05d92..443bc9b556 100644 --- a/examples/typescript/opentelemetry_instrumentation/worker.ts +++ b/examples/typescript/opentelemetry/worker.ts @@ -1,4 +1,4 @@ -import { hatchet } from './client'; +import { hatchet } from '../hatchet-client'; import { getTracer } from './tracer'; import { SpanStatusCode } from '@opentelemetry/api'; @@ -24,9 +24,6 @@ otelWorkflow.task({ }, }); -/** - * Task demonstrating that span hierarchy is preserved even when errors occur. - */ otelWorkflow.task({ name: 'step-with-error', fn: async () => { @@ -44,9 +41,6 @@ otelWorkflow.task({ }, }); -/** - * Task that is automatically instrumented without any manual span creation. - */ otelWorkflow.task({ name: 'auto-instrumented-step', fn: async () => { @@ -55,9 +49,6 @@ otelWorkflow.task({ }, }); -/** - * Task demonstrating error handling in auto-instrumented steps. - */ otelWorkflow.task({ name: 'auto-instrumented-step-with-error', fn: async () => { @@ -66,9 +57,6 @@ otelWorkflow.task({ }); async function main() { - console.log('Starting OpenTelemetry instrumented worker...'); - console.log('Instrumentation is automatic via module patching.'); - const worker = await hatchet.worker('otel-example-worker-ts', { slots: 1, workflows: [otelWorkflow], diff --git a/examples/typescript/opentelemetry_instrumentation/client.ts b/examples/typescript/opentelemetry_instrumentation/client.ts deleted file mode 100644 index 91e61a2eec..0000000000 --- a/examples/typescript/opentelemetry_instrumentation/client.ts +++ /dev/null @@ -1,7 +0,0 @@ -import './tracer'; - -import Hatchet from '@hatchet-dev/typescript-sdk'; - -export const hatchet = Hatchet.init({ - log_level: 'DEBUG', -}); diff --git a/examples/typescript/opentelemetry_instrumentation/triggers.ts b/examples/typescript/opentelemetry_instrumentation/triggers.ts deleted file mode 100644 index 39a779fb74..0000000000 --- a/examples/typescript/opentelemetry_instrumentation/triggers.ts +++ /dev/null @@ -1,162 +0,0 @@ -import { getTracer } from './tracer'; - -import { SpanStatusCode, type Span } from '@opentelemetry/api'; -import { hatchet } from './client'; -import { otelWorkflow } from './worker'; - -const tracer = getTracer('opentelemetry-triggers'); - -const ADDITIONAL_METADATA = { source: 'otel-example', version: '1.0' }; - -async function pushEvent() { - console.log('\n--- Push Event ---'); - - return tracer.startActiveSpan('push_event', async (span: Span) => { - try { - await hatchet.events.push( - 'otel:event', - { message: 'Hello from instrumented trigger!' }, - { additionalMetadata: ADDITIONAL_METADATA } - ); - console.log('Event pushed successfully'); - } finally { - span.end(); - } - }); -} - -async function bulkPushEvents() { - console.log('\n--- Bulk Push Events ---'); - - return tracer.startActiveSpan('bulk_push_event', async (span: Span) => { - try { - await hatchet.events.bulkPush('otel:event', [ - { - payload: { message: 'Bulk event 1' }, - additionalMetadata: ADDITIONAL_METADATA, - }, - { - payload: { message: 'Bulk event 2' }, - additionalMetadata: ADDITIONAL_METADATA, - }, - { - payload: { message: 'Bulk event 3' }, - additionalMetadata: ADDITIONAL_METADATA, - }, - ]); - console.log('Bulk events pushed successfully'); - } finally { - span.end(); - } - }); -} - -async function runWorkflow() { - console.log('\n--- Run Workflow ---'); - - return tracer.startActiveSpan('run_workflow', async (span: Span) => { - try { - const workflowRun = await hatchet.admin.runWorkflow(otelWorkflow.name, {}, { - additionalMetadata: ADDITIONAL_METADATA, - }); - const runId = await workflowRun.getWorkflowRunId(); - console.log(`Started workflow run: ${runId}`); - - const result = await workflowRun.output; - span.setStatus({ code: SpanStatusCode.OK, message: 'Workflow completed' }); - console.log(`Workflow completed with result:`, result); - } catch (error: any) { - const errorMessage = Array.isArray(error) ? error.join(', ') : error?.message || String(error); - console.error('Workflow failed:', errorMessage); - span.recordException(error); - span.setStatus({ code: SpanStatusCode.ERROR, message: errorMessage }); - } finally { - span.end(); - } - }); -} - -async function runWorkflows() { - console.log('\n--- Run Workflows (Bulk) ---'); - - return tracer.startActiveSpan('run_workflows', async (span: Span) => { - try { - const refs = await hatchet.admin.runWorkflows([ - { - workflowName: otelWorkflow.name, - input: {}, - options: { additionalMetadata: ADDITIONAL_METADATA }, - }, - { - workflowName: otelWorkflow.name, - input: {}, - options: { additionalMetadata: ADDITIONAL_METADATA }, - }, - ]); - console.log(`Started ${refs.length} workflow runs`); - - const results = await Promise.all(refs.map((ref: { result: () => Promise }) => ref.result())); - span.setStatus({ code: SpanStatusCode.OK, message: 'Workflows completed' }); - console.log(`Workflows completed with results:`, results); - } catch (error: any) { - const errorMessage = Array.isArray(error) ? error.join(', ') : error?.message || String(error); - console.error('Workflows failed:', errorMessage); - span.recordException(error); - span.setStatus({ code: SpanStatusCode.ERROR, message: errorMessage }); - } finally { - span.end(); - } - }); -} - -async function scheduleWorkflow() { - console.log('\n--- Schedule Workflow ---'); - - return tracer.startActiveSpan('schedule_workflow', async (span: Span) => { - try { - // Schedule workflow to run 10 seconds from now - const triggerAt = new Date(Date.now() + 10 * 1000); - - const scheduledRun = await hatchet.schedules.create(otelWorkflow.name, { - triggerAt, - input: { message: 'Hello from scheduled workflow!' }, - additionalMetadata: ADDITIONAL_METADATA, - }); - - console.log(`Scheduled workflow run: ${scheduledRun.metadata.id}`); - console.log(`Will trigger at: ${triggerAt.toISOString()}`); - - span.setStatus({ code: SpanStatusCode.OK, message: 'Workflow scheduled' }); - } catch (error: any) { - const errorMessage = Array.isArray(error) ? error.join(', ') : error?.message || String(error); - console.error('Schedule workflow failed:', errorMessage); - span.recordException(error); - span.setStatus({ code: SpanStatusCode.ERROR, message: errorMessage }); - } finally { - span.end(); - } - }); -} - - - -async function main() { - console.log('OpenTelemetry Triggers Example'); - console.log('==============================\n'); - - await pushEvent(); - await bulkPushEvents(); - await runWorkflow(); - await runWorkflows(); - await scheduleWorkflow(); - - console.log('\n--- Waiting for spans to be exported... ---'); - await new Promise((resolve) => setTimeout(resolve, 5000)); - console.log('Done!'); - - process.exit(0); -} - -if (require.main === module) { - main().catch(console.error); -} diff --git a/sdks/typescript/src/v1/examples/opentelemetry/run.ts b/sdks/typescript/src/v1/examples/opentelemetry/run.ts new file mode 100644 index 0000000000..420a15d30c --- /dev/null +++ b/sdks/typescript/src/v1/examples/opentelemetry/run.ts @@ -0,0 +1,43 @@ +import { getTracer } from './tracer'; + +import { SpanStatusCode, type Span } from '@opentelemetry/api'; +import { hatchet } from '../hatchet-client'; +import { otelWorkflow } from './worker'; + +const tracer = getTracer('opentelemetry-triggers'); + +async function runWorkflow() { + return tracer.startActiveSpan('run_workflow', async (span: Span) => { + try { + const workflowRun = await hatchet.admin.runWorkflow(otelWorkflow.name, {}); + const runId = await workflowRun.getWorkflowRunId(); + console.log(`Started workflow run: ${runId}`); + + const result = await workflowRun.output; + span.setStatus({ code: SpanStatusCode.OK, message: 'Workflow completed' }); + console.log(`Workflow completed with result:`, result); + } catch (error: any) { + const errorMessage = Array.isArray(error) + ? error.join(', ') + : error?.message || String(error); + console.error('Workflow failed:', errorMessage); + span.recordException(error); + span.setStatus({ code: SpanStatusCode.ERROR, message: errorMessage }); + } finally { + span.end(); + } + }); +} + +async function main() { + await runWorkflow(); + + console.log('Waiting for spans to be exported...'); + await new Promise((resolve) => setTimeout(resolve, 5000)); + + process.exit(0); +} + +if (require.main === module) { + main().catch(console.error); +} diff --git a/sdks/typescript/src/v1/examples/opentelemetry/tracer.ts b/sdks/typescript/src/v1/examples/opentelemetry/tracer.ts new file mode 100644 index 0000000000..f393c437a1 --- /dev/null +++ b/sdks/typescript/src/v1/examples/opentelemetry/tracer.ts @@ -0,0 +1,62 @@ +const { NodeTracerProvider } = require('@opentelemetry/sdk-trace-node'); +const { OTLPTraceExporter } = require('@opentelemetry/exporter-trace-otlp-http'); +const { resourceFromAttributes } = require('@opentelemetry/resources'); +const { SEMRESATTRS_SERVICE_NAME } = require('@opentelemetry/semantic-conventions'); +const { BatchSpanProcessor } = require('@opentelemetry/sdk-trace-base'); +const { registerInstrumentations } = require('@opentelemetry/instrumentation'); +const { trace } = require('@opentelemetry/api'); + +import type { TracerProvider, Tracer } from '@opentelemetry/api'; +import { HatchetInstrumentor } from '@hatchet/opentelemetry'; + +const isCI = process.env.CI === 'true'; + +let traceProvider: TracerProvider; + +if (isCI) { + traceProvider = trace.getTracerProvider(); + registerInstrumentations({ + tracerProvider: traceProvider, + instrumentations: [new HatchetInstrumentor()], + }); +} else { + const resource = resourceFromAttributes({ + [SEMRESATTRS_SERVICE_NAME]: + process.env.HATCHET_CLIENT_OTEL_SERVICE_NAME || 'hatchet-typescript-example', + }); + + const headersEnv = process.env.HATCHET_CLIENT_OTEL_EXPORTER_OTLP_HEADERS; + const headers: Record | undefined = headersEnv + ? { [headersEnv.split('=')[0]]: headersEnv.split('=')[1] } + : undefined; + + const exporter = new OTLPTraceExporter({ + url: + process.env.HATCHET_CLIENT_OTEL_EXPORTER_OTLP_ENDPOINT || 'http://localhost:4318/v1/traces', + headers, + }); + + const provider = new NodeTracerProvider({ + resource, + spanProcessors: [new BatchSpanProcessor(exporter)], + }); + + provider.register(); + + traceProvider = provider; + + registerInstrumentations({ + tracerProvider: traceProvider, + instrumentations: [ + new HatchetInstrumentor({ + includeTaskNameInSpanName: true, + }), + ], + }); +} + +function getTracer(name: string): Tracer { + return trace.getTracer(name); +} + +export { traceProvider, getTracer }; diff --git a/sdks/typescript/src/v1/examples/opentelemetry/worker.ts b/sdks/typescript/src/v1/examples/opentelemetry/worker.ts new file mode 100644 index 0000000000..443bc9b556 --- /dev/null +++ b/sdks/typescript/src/v1/examples/opentelemetry/worker.ts @@ -0,0 +1,70 @@ +import { hatchet } from '../hatchet-client'; +import { getTracer } from './tracer'; +import { SpanStatusCode } from '@opentelemetry/api'; + +const tracer = getTracer('opentelemetry-worker'); + +export const otelWorkflow = hatchet.workflow({ + name: 'otelworkflowtypescript', +}); + +otelWorkflow.task({ + name: 'step-with-custom-spans', + fn: async () => { + return tracer.startActiveSpan('custom-business-logic', async (span) => { + try { + console.log('Executing step with custom tracing...'); + await new Promise((resolve) => setTimeout(resolve, 100)); + + return { result: 'success' }; + } finally { + span.end(); + } + }); + }, +}); + +otelWorkflow.task({ + name: 'step-with-error', + fn: async () => { + return tracer.startActiveSpan('custom-span-with-error', async (span) => { + try { + throw new Error('Intentional error for demonstration'); + } catch (error: any) { + span.recordException(error); + span.setStatus({ code: SpanStatusCode.ERROR, message: error?.message }); + throw error; + } finally { + span.end(); + } + }); + }, +}); + +otelWorkflow.task({ + name: 'auto-instrumented-step', + fn: async () => { + console.log('This step is automatically traced without manual span code'); + return { automatically: 'instrumented' }; + }, +}); + +otelWorkflow.task({ + name: 'auto-instrumented-step-with-error', + fn: async () => { + throw new Error('Auto-instrumented step error'); + }, +}); + +async function main() { + const worker = await hatchet.worker('otel-example-worker-ts', { + slots: 1, + workflows: [otelWorkflow], + }); + + await worker.start(); +} + +if (require.main === module) { + main().catch(console.error); +} From e1e59bf393937e749d6e301c51bcaeb0fecef819 Mon Sep 17 00:00:00 2001 From: Mohammed Nafees Date: Wed, 11 Mar 2026 01:15:07 +0100 Subject: [PATCH 16/17] span names --- sdks/go/opentelemetry/middleware.go | 4 ++-- .../hatchet_sdk/opentelemetry/instrumentor.py | 20 ++++++++--------- .../src/opentelemetry/instrumentor.ts | 22 +++++++++---------- 3 files changed, 23 insertions(+), 23 deletions(-) diff --git a/sdks/go/opentelemetry/middleware.go b/sdks/go/opentelemetry/middleware.go index 8da529b41d..e3cd932572 100644 --- a/sdks/go/opentelemetry/middleware.go +++ b/sdks/go/opentelemetry/middleware.go @@ -11,7 +11,7 @@ import ( // NewMiddleware creates a Hatchet middleware that wraps each step run execution // with an OpenTelemetry span. It: // - Extracts W3C traceparent from AdditionalMetadata for distributed trace propagation -// - Creates a "hatchet task run" span with hatchet.* attributes +// - Creates a "hatchet.start_step_run" span with hatchet.* attributes // - Stores attributes in context so HatchetAttributeSpanProcessor can inject // them into all child spans // @@ -38,7 +38,7 @@ func NewMiddleware(tracer trace.Tracer) worker.MiddlewareFunc { parentCtx = withHatchetAttributes(parentCtx, attrs) // Start span - spanCtx, span := tracer.Start(parentCtx, "hatchet task run", + spanCtx, span := tracer.Start(parentCtx, "hatchet.start_step_run", trace.WithSpanKind(trace.SpanKindConsumer), trace.WithAttributes(attrs...), ) diff --git a/sdks/python/hatchet_sdk/opentelemetry/instrumentor.py b/sdks/python/hatchet_sdk/opentelemetry/instrumentor.py index 73cbb14288..94620a5165 100644 --- a/sdks/python/hatchet_sdk/opentelemetry/instrumentor.py +++ b/sdks/python/hatchet_sdk/opentelemetry/instrumentor.py @@ -391,10 +391,10 @@ async def _wrap_handle_start_step_run( action = cast(Action, params[0]) traceparent = _parse_carrier_from_metadata(action.additional_metadata) - span_name = "hatchet task run" + span_name = "hatchet.start_step_run" if self.config.otel.include_task_name_in_start_step_run_span_name: - span_name += f" {action.action_id}" + span_name += f".{action.action_id}" hatchet_attrs = action.get_otel_attributes(self.config) token = _hatchet_span_attributes.set(hatchet_attrs) @@ -426,7 +426,7 @@ async def _wrap_handle_cancel_action( action = args[0] with self._tracer.start_as_current_span( - "hatchet cancel task run", + "hatchet.cancel_step_run", attributes={ "hatchet.step_run_id": action.step_run_id, }, @@ -467,7 +467,7 @@ def _wrap_push_event( } with self._tracer.start_as_current_span( - "hatchet push event", + "hatchet.push_event", attributes={ f"hatchet.{k.value}": v for k, v in attributes.items() @@ -509,7 +509,7 @@ def _wrap_bulk_push_event( unique_event_keys = {event.key for event in bulk_events} with self._tracer.start_as_current_span( - "hatchet push events", + "hatchet.bulk_push_event", attributes={ "hatchet.num_events": num_bulk_events, "hatchet.unique_event_keys": json.dumps(unique_event_keys, default=str), @@ -569,7 +569,7 @@ def _wrap_run_workflow( } with self._tracer.start_as_current_span( - "hatchet trigger task", + "hatchet.run_workflow", attributes={ f"hatchet.{k.value}": v for k, v in attributes.items() @@ -627,7 +627,7 @@ async def _wrap_async_run_workflow( } with self._tracer.start_as_current_span( - "hatchet trigger task", + "hatchet.run_workflow", attributes={ f"hatchet.{k.value}": v for k, v in attributes.items() @@ -709,7 +709,7 @@ def _wrap_schedule_workflow( } with self._tracer.start_as_current_span( - "hatchet schedule task", + "hatchet.schedule_workflow", attributes={ f"hatchet.{k.value}": v for k, v in attributes.items() @@ -749,7 +749,7 @@ def _wrap_run_workflows( } with self._tracer.start_as_current_span( - "hatchet trigger tasks", + "hatchet.run_workflows", attributes={ "hatchet.num_workflows": num_workflows, "hatchet.unique_workflow_names": json.dumps( @@ -792,7 +792,7 @@ async def _wrap_async_run_workflows( } with self._tracer.start_as_current_span( - "hatchet trigger tasks", + "hatchet.run_workflows", attributes={ "hatchet.num_workflows": num_workflows, "hatchet.unique_workflow_names": json.dumps( diff --git a/sdks/typescript/src/opentelemetry/instrumentor.ts b/sdks/typescript/src/opentelemetry/instrumentor.ts index 2fc28eef39..9de0fa313d 100644 --- a/sdks/typescript/src/opentelemetry/instrumentor.ts +++ b/sdks/typescript/src/opentelemetry/instrumentor.ts @@ -306,7 +306,7 @@ export class HatchetInstrumentor extends InstrumentationBase { @@ -516,9 +516,9 @@ export class HatchetInstrumentor extends InstrumentationBase { @@ -604,9 +604,9 @@ export class HatchetInstrumentor extends InstrumentationBase { From adbf0ea225afa6879f9c63164157db9463642c0d Mon Sep 17 00:00:00 2001 From: Mohammed Nafees Date: Wed, 11 Mar 2026 01:50:48 +0100 Subject: [PATCH 17/17] fix lint --- .../hatchet/worker.py | 6 +++--- examples/typescript/opentelemetry/tracer.ts | 1 + .../hatchet/worker.py | 6 +++--- .../src/opentelemetry/hatchet-exporter.ts | 20 ++++++++----------- .../src/opentelemetry/instrumentor.ts | 15 ++++++++------ .../src/v1/examples/opentelemetry/tracer.ts | 2 ++ 6 files changed, 26 insertions(+), 24 deletions(-) diff --git a/examples/python/opentelemetry_instrumentation/hatchet/worker.py b/examples/python/opentelemetry_instrumentation/hatchet/worker.py index 42846868f3..af26c16ee3 100644 --- a/examples/python/opentelemetry_instrumentation/hatchet/worker.py +++ b/examples/python/opentelemetry_instrumentation/hatchet/worker.py @@ -13,7 +13,7 @@ from opentelemetry.sdk.resources import SERVICE_NAME, Resource from opentelemetry.sdk.trace import TracerProvider from opentelemetry.sdk.trace.export import ConsoleSpanExporter, SimpleSpanProcessor -from opentelemetry.trace import StatusCode +from opentelemetry.trace import StatusCode, Tracer from hatchet_sdk import Context, EmptyModel, Hatchet from hatchet_sdk.opentelemetry.instrumentor import HatchetInstrumentor @@ -25,10 +25,10 @@ # Module-level tracer — will be set in main() before the worker starts. # Tasks use this to create custom child spans inside the auto-instrumented # hatchet task run parent span. -_tracer = None +_tracer: Tracer | None = None -def _get_tracer(): +def _get_tracer() -> Tracer: global _tracer if _tracer is None: from opentelemetry.trace import get_tracer diff --git a/examples/typescript/opentelemetry/tracer.ts b/examples/typescript/opentelemetry/tracer.ts index 0e62d95f7b..88bb8b6182 100644 --- a/examples/typescript/opentelemetry/tracer.ts +++ b/examples/typescript/opentelemetry/tracer.ts @@ -5,6 +5,7 @@ const { SEMRESATTRS_SERVICE_NAME } = require('@opentelemetry/semantic-convention const { BatchSpanProcessor } = require('@opentelemetry/sdk-trace-base'); const { registerInstrumentations } = require('@opentelemetry/instrumentation'); const { trace } = require('@opentelemetry/api'); +/* eslint-enable @typescript-eslint/no-require-imports */ import type { TracerProvider, Tracer } from '@opentelemetry/api'; import { HatchetInstrumentor } from '@hatchet-dev/typescript-sdk/opentelemetry'; diff --git a/sdks/python/examples/opentelemetry_instrumentation/hatchet/worker.py b/sdks/python/examples/opentelemetry_instrumentation/hatchet/worker.py index 42846868f3..af26c16ee3 100644 --- a/sdks/python/examples/opentelemetry_instrumentation/hatchet/worker.py +++ b/sdks/python/examples/opentelemetry_instrumentation/hatchet/worker.py @@ -13,7 +13,7 @@ from opentelemetry.sdk.resources import SERVICE_NAME, Resource from opentelemetry.sdk.trace import TracerProvider from opentelemetry.sdk.trace.export import ConsoleSpanExporter, SimpleSpanProcessor -from opentelemetry.trace import StatusCode +from opentelemetry.trace import StatusCode, Tracer from hatchet_sdk import Context, EmptyModel, Hatchet from hatchet_sdk.opentelemetry.instrumentor import HatchetInstrumentor @@ -25,10 +25,10 @@ # Module-level tracer — will be set in main() before the worker starts. # Tasks use this to create custom child spans inside the auto-instrumented # hatchet task run parent span. -_tracer = None +_tracer: Tracer | None = None -def _get_tracer(): +def _get_tracer() -> Tracer: global _tracer if _tracer is None: from opentelemetry.trace import get_tracer diff --git a/sdks/typescript/src/opentelemetry/hatchet-exporter.ts b/sdks/typescript/src/opentelemetry/hatchet-exporter.ts index 4afa4f46e9..f0d8d2e8f7 100644 --- a/sdks/typescript/src/opentelemetry/hatchet-exporter.ts +++ b/sdks/typescript/src/opentelemetry/hatchet-exporter.ts @@ -22,11 +22,10 @@ try { } /* eslint-disable @typescript-eslint/no-require-imports */ -const { - OTLPTraceExporter, -} = require('@opentelemetry/exporter-trace-otlp-grpc') as typeof import('@opentelemetry/exporter-trace-otlp-grpc'); -const sdkTraceBase = - require('@opentelemetry/sdk-trace-base') as typeof import('@opentelemetry/sdk-trace-base'); +// prettier-ignore +const { OTLPTraceExporter } = require('@opentelemetry/exporter-trace-otlp-grpc') as typeof import('@opentelemetry/exporter-trace-otlp-grpc'); +// prettier-ignore +const sdkTraceBase = require('@opentelemetry/sdk-trace-base') as typeof import('@opentelemetry/sdk-trace-base'); /* eslint-enable @typescript-eslint/no-require-imports */ const { BatchSpanProcessor } = sdkTraceBase; @@ -53,9 +52,8 @@ function createHatchetExporter(config: ClientConfig): InstanceType