diff --git a/api-contracts/openapi/components/schemas/_index.yaml b/api-contracts/openapi/components/schemas/_index.yaml index 1c39332c78..c25cc8ab77 100644 --- a/api-contracts/openapi/components/schemas/_index.yaml +++ b/api-contracts/openapi/components/schemas/_index.yaml @@ -400,3 +400,11 @@ V1CELDebugResponse: $ref: "./v1/cel.yaml#/V1CELDebugResponse" V1CELDebugResponseStatus: $ref: "./v1/cel.yaml#/V1CELDebugResponseStatus" +OtelSpan: + $ref: "./v1/otel.yaml#/OtelSpan" +OtelSpanKind: + $ref: "./v1/otel.yaml#/OtelSpanKind" +OtelStatusCode: + $ref: "./v1/otel.yaml#/OtelStatusCode" +OtelSpanList: + $ref: "./v1/otel.yaml#/OtelSpanList" diff --git a/api-contracts/openapi/components/schemas/v1/otel.yaml b/api-contracts/openapi/components/schemas/v1/otel.yaml new file mode 100644 index 0000000000..c8708ff799 --- /dev/null +++ b/api-contracts/openapi/components/schemas/v1/otel.yaml @@ -0,0 +1,73 @@ +OtelSpan: + type: object + properties: + trace_id: + type: string + span_id: + type: string + parent_span_id: + type: string + span_name: + type: string + span_kind: + $ref: "#/OtelSpanKind" + service_name: + type: string + status_code: + $ref: "#/OtelStatusCode" + status_message: + type: string + duration: + type: integer + format: int64 + created_at: + type: string + format: date-time + resource_attributes: + type: object + additionalProperties: + type: string + span_attributes: + type: object + additionalProperties: + type: string + scope_name: + type: string + scope_version: + type: string + required: + - trace_id + - span_id + - span_name + - span_kind + - service_name + - status_code + - duration + - created_at + +OtelSpanKind: + type: string + enum: + - UNSPECIFIED + - INTERNAL + - SERVER + - CLIENT + - PRODUCER + - CONSUMER + +OtelStatusCode: + type: string + enum: + - UNSET + - OK + - ERROR + +OtelSpanList: + type: object + properties: + pagination: + $ref: "../metadata.yaml#/PaginationResponse" + rows: + type: array + items: + $ref: "#/OtelSpan" diff --git a/api-contracts/openapi/openapi.yaml b/api-contracts/openapi/openapi.yaml index 30df3807f5..f33970013c 100644 --- a/api-contracts/openapi/openapi.yaml +++ b/api-contracts/openapi/openapi.yaml @@ -29,6 +29,8 @@ paths: $ref: "./paths/v1/tasks/tasks.yaml#/listTaskEvents" /api/v1/stable/tasks/{task}/logs: $ref: "./paths/v1/tasks/tasks.yaml#/listLogs" + /api/v1/stable/tasks/{task}/trace: + $ref: "./paths/v1/tasks/tasks.yaml#/getTrace" /api/v1/stable/tenants/{tenant}/tasks/cancel: $ref: "./paths/v1/tasks/tasks.yaml#/cancelTasks" /api/v1/stable/tenants/{tenant}/tasks/replay: diff --git a/api-contracts/openapi/paths/v1/tasks/tasks.yaml b/api-contracts/openapi/paths/v1/tasks/tasks.yaml index 9e4ae10c41..0952d91e65 100644 --- a/api-contracts/openapi/paths/v1/tasks/tasks.yaml +++ b/api-contracts/openapi/paths/v1/tasks/tasks.yaml @@ -441,6 +441,64 @@ replayTasks: tags: - Task +getTrace: + get: + x-resources: ["tenant", "task"] + description: Get OTel trace for a task run + operationId: v1-task:get:trace + parameters: + - description: The task id + in: path + name: task + required: true + schema: + type: string + format: uuid + minLength: 36 + maxLength: 36 + - description: The number to skip + in: query + name: offset + required: false + schema: + type: integer + format: int64 + - description: The number to limit by + in: query + name: limit + required: false + schema: + type: integer + format: int64 + responses: + "200": + content: + application/json: + schema: + $ref: "../../../components/schemas/_index.yaml#/OtelSpanList" + description: Successfully retrieved the OTel trace + "400": + content: + application/json: + schema: + $ref: "../../../components/schemas/_index.yaml#/APIErrors" + description: A malformed or bad request + "403": + content: + application/json: + schema: + $ref: "../../../components/schemas/_index.yaml#/APIErrors" + description: Forbidden + "404": + content: + application/json: + schema: + $ref: "../../../components/schemas/_index.yaml#/APIErrors" + description: The task was not found + summary: Get OTel trace + tags: + - Task + listLogs: get: x-resources: ["tenant", "task"] diff --git a/api/v1/server/handlers/v1/tasks/trace.go b/api/v1/server/handlers/v1/tasks/trace.go new file mode 100644 index 0000000000..3028fd4eee --- /dev/null +++ b/api/v1/server/handlers/v1/tasks/trace.go @@ -0,0 +1,31 @@ +package tasks + +import ( + "github.com/labstack/echo/v4" + + "github.com/hatchet-dev/hatchet/api/v1/server/oas/gen" + transformers "github.com/hatchet-dev/hatchet/api/v1/server/oas/transformers/v1" + "github.com/hatchet-dev/hatchet/pkg/repository/sqlcv1" +) + +func (t *TasksService) V1TaskGetTrace(ctx echo.Context, request gen.V1TaskGetTraceRequestObject) (gen.V1TaskGetTraceResponseObject, error) { + task := ctx.Get("task").(*sqlcv1.V1TasksOlap) + + limit := int64(1000) + offset := int64(0) + + if request.Params.Limit != nil { + limit = *request.Params.Limit + } + + if request.Params.Offset != nil { + offset = *request.Params.Offset + } + + result, err := t.config.V1.OTelCollector().ListSpansByTaskExternalID(ctx.Request().Context(), task.TenantID, task.ExternalID, offset, limit) + if err != nil { + return nil, err + } + + return gen.V1TaskGetTrace200JSONResponse(transformers.ToV1OtelSpanList(result.Rows, limit, offset, result.Total)), nil +} diff --git a/api/v1/server/oas/gen/openapi.gen.go b/api/v1/server/oas/gen/openapi.gen.go index 5f43ce3136..c84a0481ac 100644 --- a/api/v1/server/oas/gen/openapi.gen.go +++ b/api/v1/server/oas/gen/openapi.gen.go @@ -93,6 +93,23 @@ const ( LogLineOrderByFieldCreatedAt LogLineOrderByField = "createdAt" ) +// Defines values for OtelSpanKind. +const ( + CLIENT OtelSpanKind = "CLIENT" + CONSUMER OtelSpanKind = "CONSUMER" + INTERNAL OtelSpanKind = "INTERNAL" + PRODUCER OtelSpanKind = "PRODUCER" + SERVER OtelSpanKind = "SERVER" + UNSPECIFIED OtelSpanKind = "UNSPECIFIED" +) + +// Defines values for OtelStatusCode. +const ( + OtelStatusCodeERROR OtelStatusCode = "ERROR" + OtelStatusCodeOK OtelStatusCode = "OK" + OtelStatusCodeUNSET OtelStatusCode = "UNSET" +) + // Defines values for RateLimitOrderByDirection. const ( Asc RateLimitOrderByDirection = "asc" @@ -203,8 +220,8 @@ const ( // Defines values for V1CELDebugResponseStatus. const ( - V1CELDebugResponseStatusERROR V1CELDebugResponseStatus = "ERROR" - V1CELDebugResponseStatusSUCCESS V1CELDebugResponseStatus = "SUCCESS" + ERROR V1CELDebugResponseStatus = "ERROR" + SUCCESS V1CELDebugResponseStatus = "SUCCESS" ) // Defines values for V1CreateWebhookRequestAPIKeyAuthType. @@ -758,6 +775,36 @@ type LogLineOrderByField string // LogLineSearch defines model for LogLineSearch. type LogLineSearch = string +// OtelSpan defines model for OtelSpan. +type OtelSpan struct { + CreatedAt time.Time `json:"created_at"` + Duration int64 `json:"duration"` + ParentSpanId *string `json:"parent_span_id,omitempty"` + ResourceAttributes *map[string]string `json:"resource_attributes,omitempty"` + ScopeName *string `json:"scope_name,omitempty"` + ScopeVersion *string `json:"scope_version,omitempty"` + ServiceName string `json:"service_name"` + SpanAttributes *map[string]string `json:"span_attributes,omitempty"` + SpanId string `json:"span_id"` + SpanKind OtelSpanKind `json:"span_kind"` + SpanName string `json:"span_name"` + StatusCode OtelStatusCode `json:"status_code"` + StatusMessage *string `json:"status_message,omitempty"` + TraceId string `json:"trace_id"` +} + +// OtelSpanKind defines model for OtelSpanKind. +type OtelSpanKind string + +// OtelSpanList defines model for OtelSpanList. +type OtelSpanList struct { + Pagination *PaginationResponse `json:"pagination,omitempty"` + Rows *[]OtelSpan `json:"rows,omitempty"` +} + +// OtelStatusCode defines model for OtelStatusCode. +type OtelStatusCode string + // PaginationResponse defines model for PaginationResponse. type PaginationResponse struct { // CurrentPage the current page @@ -2497,6 +2544,15 @@ type V1TaskEventListParams struct { Limit *int64 `form:"limit,omitempty" json:"limit,omitempty"` } +// V1TaskGetTraceParams defines parameters for V1TaskGetTrace. +type V1TaskGetTraceParams struct { + // Offset The number to skip + Offset *int64 `form:"offset,omitempty" json:"offset,omitempty"` + + // Limit The number to limit by + Limit *int64 `form:"limit,omitempty" json:"limit,omitempty"` +} + // V1EventListParams defines parameters for V1EventList. type V1EventListParams struct { // Offset The number to skip @@ -3196,6 +3252,9 @@ type ServerInterface interface { // List events for a task // (GET /api/v1/stable/tasks/{task}/task-events) V1TaskEventList(ctx echo.Context, task openapi_types.UUID, params V1TaskEventListParams) error + // Get OTel trace + // (GET /api/v1/stable/tasks/{task}/trace) + V1TaskGetTrace(ctx echo.Context, task openapi_types.UUID, params V1TaskGetTraceParams) error // Debug a CEL expression // (POST /api/v1/stable/tenants/{tenant}/cel/debug) V1CelDebug(ctx echo.Context, tenant openapi_types.UUID) error @@ -3949,6 +4008,42 @@ func (w *ServerInterfaceWrapper) V1TaskEventList(ctx echo.Context) error { return err } +// V1TaskGetTrace converts echo context to params. +func (w *ServerInterfaceWrapper) V1TaskGetTrace(ctx echo.Context) error { + var err error + // ------------- Path parameter "task" ------------- + var task openapi_types.UUID + + err = runtime.BindStyledParameterWithLocation("simple", false, "task", runtime.ParamLocationPath, ctx.Param("task"), &task) + if err != nil { + return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Invalid format for parameter task: %s", err)) + } + + ctx.Set(BearerAuthScopes, []string{}) + + ctx.Set(CookieAuthScopes, []string{}) + + // Parameter object where we will unmarshal all parameters from the context + var params V1TaskGetTraceParams + // ------------- Optional query parameter "offset" ------------- + + err = runtime.BindQueryParameter("form", true, false, "offset", ctx.QueryParams(), ¶ms.Offset) + if err != nil { + return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Invalid format for parameter offset: %s", err)) + } + + // ------------- Optional query parameter "limit" ------------- + + err = runtime.BindQueryParameter("form", true, false, "limit", ctx.QueryParams(), ¶ms.Limit) + if err != nil { + return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Invalid format for parameter limit: %s", err)) + } + + // Invoke the callback with all the unmarshaled arguments + err = w.Handler.V1TaskGetTrace(ctx, task, params) + return err +} + // V1CelDebug converts echo context to params. func (w *ServerInterfaceWrapper) V1CelDebug(ctx echo.Context) error { var err error @@ -7294,6 +7389,7 @@ func RegisterHandlersWithBaseURL(router EchoRouter, si ServerInterface, baseURL router.GET(baseURL+"/api/v1/stable/tasks/:task", wrapper.V1TaskGet) router.GET(baseURL+"/api/v1/stable/tasks/:task/logs", wrapper.V1LogLineList) router.GET(baseURL+"/api/v1/stable/tasks/:task/task-events", wrapper.V1TaskEventList) + router.GET(baseURL+"/api/v1/stable/tasks/:task/trace", wrapper.V1TaskGetTrace) router.POST(baseURL+"/api/v1/stable/tenants/:tenant/cel/debug", wrapper.V1CelDebug) router.GET(baseURL+"/api/v1/stable/tenants/:tenant/events", wrapper.V1EventList) router.GET(baseURL+"/api/v1/stable/tenants/:tenant/events/keys", wrapper.V1EventKeyList) @@ -8053,6 +8149,51 @@ func (response V1TaskEventList501JSONResponse) VisitV1TaskEventListResponse(w ht return json.NewEncoder(w).Encode(response) } +type V1TaskGetTraceRequestObject struct { + Task openapi_types.UUID `json:"task"` + Params V1TaskGetTraceParams +} + +type V1TaskGetTraceResponseObject interface { + VisitV1TaskGetTraceResponse(w http.ResponseWriter) error +} + +type V1TaskGetTrace200JSONResponse OtelSpanList + +func (response V1TaskGetTrace200JSONResponse) VisitV1TaskGetTraceResponse(w http.ResponseWriter) error { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(200) + + return json.NewEncoder(w).Encode(response) +} + +type V1TaskGetTrace400JSONResponse APIErrors + +func (response V1TaskGetTrace400JSONResponse) VisitV1TaskGetTraceResponse(w http.ResponseWriter) error { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(400) + + return json.NewEncoder(w).Encode(response) +} + +type V1TaskGetTrace403JSONResponse APIErrors + +func (response V1TaskGetTrace403JSONResponse) VisitV1TaskGetTraceResponse(w http.ResponseWriter) error { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(403) + + return json.NewEncoder(w).Encode(response) +} + +type V1TaskGetTrace404JSONResponse APIErrors + +func (response V1TaskGetTrace404JSONResponse) VisitV1TaskGetTraceResponse(w http.ResponseWriter) error { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(404) + + return json.NewEncoder(w).Encode(response) +} + type V1CelDebugRequestObject struct { Tenant openapi_types.UUID `json:"tenant"` Body *V1CelDebugJSONRequestBody @@ -12716,6 +12857,8 @@ type StrictServerInterface interface { V1TaskEventList(ctx echo.Context, request V1TaskEventListRequestObject) (V1TaskEventListResponseObject, error) + V1TaskGetTrace(ctx echo.Context, request V1TaskGetTraceRequestObject) (V1TaskGetTraceResponseObject, error) + V1CelDebug(ctx echo.Context, request V1CelDebugRequestObject) (V1CelDebugResponseObject, error) V1EventList(ctx echo.Context, request V1EventListRequestObject) (V1EventListResponseObject, error) @@ -13354,6 +13497,29 @@ func (sh *strictHandler) V1TaskEventList(ctx echo.Context, task openapi_types.UU return nil } +// V1TaskGetTrace operation +func (sh *strictHandler) V1TaskGetTrace(ctx echo.Context, task openapi_types.UUID, params V1TaskGetTraceParams) error { + var request V1TaskGetTraceRequestObject + + request.Task = task + request.Params = params + + handler := func(ctx echo.Context, request interface{}) (interface{}, error) { + return sh.ssi.V1TaskGetTrace(ctx, request.(V1TaskGetTraceRequestObject)) + } + + response, err := handler(ctx, request) + + if err != nil { + return err + } else if validResponse, ok := response.(V1TaskGetTraceResponseObject); ok { + return validResponse.VisitV1TaskGetTraceResponse(ctx.Response()) + } else if response != nil { + return fmt.Errorf("Unexpected response type: %T", response) + } + return nil +} + // V1CelDebug operation func (sh *strictHandler) V1CelDebug(ctx echo.Context, tenant openapi_types.UUID) error { var request V1CelDebugRequestObject @@ -16160,326 +16326,332 @@ func (sh *strictHandler) WorkflowVersionGet(ctx echo.Context, workflow openapi_t // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/+y9e3PbOLIo/lVY+v2q7kyV5Fcmc+ak6v6h2EqiiWP7SHJy986mvBAJSxhTJJcA7WhT", - "/u638CJBEiBBvSwlrNracUQ8Go3uRqPRj+8dN1xEYQADgjtvvnewO4cLwP7s3wwHcRzG9O8oDiMYEwTZ", - "Fzf0IP2vB7Ebo4igMOi86QDHTTAJF84HQNw5JA6kvR3WuNuB38Ai8mHnzelvJyfdzn0YLwDpvOkkKCC/", - "/9bpdsgygp03HRQQOINx57mbH748m/Jv5z6MHTJHmM+pTtfpZw0foYBpATEGM5jNikmMghmbNHTxnY+C", - "B92U9HeHhA6ZQ8cL3WQBAwI0AHQddO8g4sBvCBOcA2eGyDyZHrnh4njO8dTz4KP8WwfRPYK+V4aGwsA+", - "OWQOiDK5g7ADMA5dBAj0nCdE5gweEEU+csHUz21HJwALDSKeu50Y/jtBMfQ6b/7KTf01bRxO/4YuoTBK", + "H4sIAAAAAAAC/+y9e2/buLYo/lUE/37AnQHsvDqdM6fA/cNN3NbTNMmxnfbuO6fIoSXG5kSWNCKV1LvI", + "d7/gS6IkUqL8it0K2NiTWnwsLq61uLi4Ht87briIwgAGBHfefO9gdw4XgP3ZvxkO4jiM6d9RHEYwJgiy", + "L27oQfpfD2I3RhFBYdB50wGOm2ASLpwPgLhzSBxIezuscbcDv4FF5MPOm9PfTk66nfswXgDSedNJUEB+", + "/63T7ZBlBDtvOiggcAbjznM3P3x5NuXfzn0YO2SOMJ9Tna7Tzxo+QgHTAmIMZjCbFZMYBTM2aejiOx8F", + "D7op6e8OCR0yh44XuskCBgRoAOg66N5BxIHfECY4B84MkXkyPXLDxfGc46nnwUf5tw6iewR9rwwNhYF9", + "csgcEGVyB2EHYBy6CBDoOU+IzBk8IIp85IKpn9uOTgAWGkQ8dzsx/CdBMfQ6b/7KTf01bRxO/4YuoTBK", "WsFlYoHp74jABfvj/4/hfedN5/87zmjvWBDecUp1z+k0II7BsgSSGNcAzSdIQBkW4Pvh0/kcBDN4AzB+", "CmMNYp/mkMxh7ISxE4TESTCMseOCwHFZR7r5KHYi2V/BJYkTmIIzDUMfgoDCw6eNISBwAgMQkCaTsm5O", "AJ8cwvpi6xmHwSMifOGWkyHWwwnZV/4zo3aEHRRgAgIXWs8+RrMgiRpMjtEscJIoY6VGUyZkbkFalCz6", - "tOlztxOFmMzDmWWvG9Gadlz6YdCPoqGBK2/od8puzvCCrSbBkPWhXE+piDg4iaIwJjlGPD179dvr3//r", - "jx79o/B/9Pf/Pjk90zKqif77Aid5HmDr0lEFBV3ABT2HDoqd8N6hmIUBQS4TdCrEf3WmACO30+3MwnDm", + "tOlztxOFmMzDmWWvG9Gadlz6YdCPoqGBK2/od8puzvCCrSbBkPWhXE+piDg4iaIwJjlGPD179dvr3//j", + "jx79o/B/9Pf/PDk90zKqif77Aid5HmDr0lEFBV3ABT2HDoqd8N6hmIUBQS4TdCrEf3WmACO30+3MwnDm", "Q8qLKY+XxFiJmU1gD+kJEAMp9gvSJKACrIJrBeWkQ1BpKDo5YcAkt0JXZUJi4lCLG/qFIoQPkcFYlu61", "4lTIXLmYChl2kxFpQZRF6EOIiYECQ0w+hDOnfzN05rSVCuOckAi/OT4W9H8kvlDi1B0/IEIf4bJ+nge4", "zE0TzR/uMtIFU9eD99bkO4I4TGIX6sU4l4le37B6ghZQORRjMZbzBLAQpzmp3Tk7OTvrnZ71Tl9NTl+/", "Ofn9zW9/HP3xxx+vXv/RO3n95uSko6grHiCwRyfQoQoZBALyON0owHQdFDi3t1xA0KFVgKbTs9Pf/jj5", - "r97Zb7/D3m+vwOseOHvt9X47/a/fT71T9/7+v+n8C/DtEgYzyuSvfteAk0TeqmjyASaO6L8NXBX4AdFJ", + "j97Zb7/D3m+vwOseOHvt9X47/Y/fT71T9/7+P+n8C/DtEgYzyuSvfteAk0TeqmjyASaO6L8NXBX4AdFJ", "sl1VQTfwxiR8gDrx8C1CMcS6JX+ZQ87+lFgJ7e6I1kfWG7yABHiAk2TNmZGjYKNcmRTkSgrbUX5/z16/", - "rsNhCls3FS8pMrRIdF0YEa4jjOC/E8iFSR6fXCHgmF2POhcoMBNrt/OtF4II9ehlYQaDHvxGYtAjYMag", - "eAQ+ovvSeZOuuJskyOs8lwiJw6tb79vEf+A62OARBsS4ZPgo70JW+qpmyFrNlc/w9bnbOafnkG8B0NDL", - "g9R4O7ILV8K4rcn2WC2IQsiWFAZuEscwcJeXaIHImMSAwNmSn97JgnY471+dDy7vhld3N6Pr96PBeNzp", - "di5G1zd3V4Mvg/Gk0+38z+3gdpD98/3o+vbmbnR9e3VxN7p+O7xS9jiDUpl77IYRVOf8cj36+O7y+kun", - "25n0xx9r+0NC6K86ERNDjLWXUsrObjaGk7XtUiXQo9r0DAaQYsQB9Mh07uNw4RCAHxwURAnBXUcycteB", - "xD3SiSG/iNdKAjXtxzMjglESYP1CFuAbWiQLJ0gWU6qv32dLI85TGD/c++GTEydBXoCigLw6097nsdwS", - "S3D5FtKOBEYjCDyqLemUbgptLL6nhy10aDeK8ac5cuf8kFM3B/Md5vdifgrUSFiBreIGdFWakMvUiSB1", - "bQSQOtoq7fsDXHLdz/MQXTrwb3Ld1T0w2FRKMPEfvtvoZVzUycPXLK/4sTM08IeXxJnJRG4NFCcywg4T", - "9uXNsD8iwgUiAfK7ciK2GP3x2+eHL79xrnX6svG/WiANR2GAYRlrRCo0ZYzlwKoGg49ihuM8DoMvgnUn", - "MZrNYGzcx4zKPilqT2lgNw6DQTXd0iZXYgPKSjMVe9qRoxiFMSLLImkz8SKkU+fNK3Z48b9PyyRfUhDo", - "bF3d4hQ4S6v6mmKw+qzW46xAdGmbVNSnFMhOUmWbM2Tox2IMZTfAg+4SR/uzU8jQPdsmdTPKY8ivUvSm", - "4zQ5FsrDsk8MODagc498AilE9ZzAr6MMa9nmja/GinXBuIskjJDbj03suAD/CQNHKvgOpRjnl/7o6le5", - "+vHV2GFjrCPGUk13gYL/fdpdgG//++z172WVNwXWzPXc6Nj3YUwGC4D893GYRGb5TZtgnbD0ESZ0jbyF", - "NG3F9ES0tPussHwPPcIum7G8dgFq3cprLjl8cO1es09yW+laqT7BLxkb2Vu5rm4nDv1a3Yiv5hOk+tiI", - "ttfioyMGq8OKGR/BDAXwM4ylQK+HSTZ+7nZg8IjiMFhAbuau7ztQOlhflLktfBN7wJAYBtMQxB4KZhdC", - "zup1LG5+NsrzbBgulUnoYBLGkD3C6OHO9gb7ycwgBv1ktvmFd8WbEzvxng0mSgaUnpIyTQLbHoRVSNUq", - "FlqJotiAy/bbVJ1oNNcahp0FJPPQqzcTKOj6xLsoxF553K6s+3Q7nFqGnnYOeYer+WzU3GQDwfzaYcxG", - "qhQ03UCF2XOwCsrI6CDdg1o6vUQ6eReBGQrS94aqXbxJW6aKPBPdT03sRSrfWL2L6GhHMWxcDN71by8n", - "HWYX1Zs11AGuYw/Gb5fv5KuyHCaQii8sWV6zkZj2u0u1d02tdQ2+JulLbf0RVmS1MrjDi7wAL77Qi/d7", - "40Ik/Y+SYJwsFiCutfuwrfpS7lbBklxnThfyVW64PBPzm97kRuL88uf4+sqZLgnEv9Yr76nazqb/uB4N", - "yDH2gPnT5ZT5XgK6L1BWgCgkyAWKoStBklIEYLfDFSSz/DBJIAvRM4Ygdufa08hE7+XXQ2Zz1z4iMy0z", - "M3fKhlojp8HAdg+QxdC8VZNxIxh4wh5dNbBo1mTkfycwqYeYt2oybpwEgQXEolmTkXHiuhB69UCnDe1H", - "T6kcVz0NaW6K7NuRehVegcfWOLHMYl15b/oznGoEeZWfHZPniqedOMX+DqdHW3ohLY2JCYzspdeYwEiH", - "2EpVmKAFDBOiX774WLf0x3XV4EdF/ZXXL7Z0nV77ZzgdJUGFdONv4Hbv2mmn1OHT3GQEATZczO5RgPC8", - "2dR/c4qs2lFKtLylYffWILoY4sTXm58xATFpthhMAEmwxXro+cTbyuct8QxnTeJ085tTufsA42oWaLJc", - "RSmtA1k5mAs917828kEkgaS7YOaacbpNUvW4GVxdDK/ed7qd0e3VFf9rfHt+PhhcDC463c67/vCS/cFf", - "rvnfb/vnH6/fvdNqK1SN0/uz2XrBFrtqNltMwl6WsPlpaafKY+qbo9UfKcR5Izx+YXjz0NS6OiiwiYl0", - "ZMaW6QP34QuczsPw4cUXqcCyqSWGs0sUwEbOecw9gn6migSVLPJI9cOZ46MANvHE4h782jnocKJBrZJi", - "6s1baGwSBWypXmtZWEE6w9cMVZfwEfp5w83bWypohlfvrjvdzpf+6KrT7QxGo+uRXqYo46SXJ6v9z0Gg", - "EyTi+8vfPSVZ6aUH/7jG/TM/QsMbqOhccQfVIED11freEe4zdxGj3bNuJ4Df5L9edTtBsmD/wJ03pyfM", - "CpzjrFxnnUun9M6JOBWmE59ZXasUWLT+z/BbeeRXdiNn69J6ooYE+OolljZllh0fYcJfN7L4oRObW5xG", - "Yv0PvcF+giRGrkYeB8nixu6KzehYXrSPTOv9H6tbNR8LccdUdsU2Djiyu07zEcWl+qhT6xCRgZqbpasi", - "RCf/R4BA5k9WRqWVzZa5wDE/Kr2LG8BkBO+Rb3iYZQ7KwoNZHYx5L8esI2ReRFtw82YTfQZ+Am0d52L+", - "1IodFhkjTL5i159Q4IVP+m3fhE25BtGP5nVIaaJZxwJ40HYR/Jt+Cv6NLYPuJQoUj7AMzTyG4z6MXejZ", - "en4o9wRlv+R6U6hylPZVpes9OAwzHtMeh+nnNQ7E4hilI5FjU2JNQaV2NOjCgIyV+2zhnYiBZ6Jn/tXR", - "ef+pBogmN9RVLBJrWBO2ZjIQKM1sBqULdNG/u5pH0o3oqndrAUtxdK34hzOECYyhJ2/2muABwz6nvsPI", - "c+J0HB6dhTD7DOMjjc96CXl2viKZr3LVZBYRKkYnzxGkf/08sRQjGPlg+UOFLfAlKWYqbFxZjjtedn1K", - "89cnJzXrLcBtWrXJjKR0tz/CCnY/W/gkdDGVeUz0VbCV3n9Y6/hLRy1YfDQDziAmt7FB87wdXTK/Lhh4", - "zNFTXPqxQ8LtuCCYjsskQP+mupEHA4LuEYxT3VqogyK2j/ujqiGxU+iHwUxCXCtlt+gOa2forXRxHbtz", - "6CU+VChtXZf2LbukdzuEu97b6wlNvNizwb8q6PE2Z/dmoVn0j/H5h8HFLf1RpwymM2/XTXBPHf7Kq8+8", - "/nbh3NeYxDbnDzhKgnPVCNz4MYkDsOuzVAHAZoljK8X9S6nDSzpOZkRR6TNZpt23if9wAX1I4DsWgrGi", - "C2AaQZB6AD7ApcMul04EEE83woM8nOkyn2viAS5P37Cmp9xV7Yz/66xJ2oluJwJxdkXVX50a0g0f8Uvd", - "hWxFatzAYM8Nt9h4fN6ne99M8pWoh0X9FFpptOn1leMhH4rpxgsUiH+e2njdVmPIpCR77Lu34ZUUibhh", - "TiX9UuzSLCkL6lblXKqaQ58TSh8NuBFyL+WJagDyLUs9QSnFZNFYdzPXVf7ykrzpyozczXNurEtVCvqa", - "sqC6SAlM89WZOHObPJMmLNkq32tRtApn7oFpW3M5eF5NKq8SEFEexWT+VjWm6tfhMVyAaB7GcOyHZMO2", - "75xdWe+gyM2Z2A/5E5joYe9QsaIdGquKlCZyjcDIiRO5sHpTg+qEVr9Q5PvSO9N+paWLRoWF2hr0Am9m", - "aOmqtvaCXZ1SjeqZU/almYMggL4JTPHZQZ7+6Q/TwZ0nPrr+UYWPcGW0o8spmD19xUnWsoCBhWn19Nsa", - "S6fdzetmg6+z6L2w3dlZ1yQiUnTn6aKrkKH2fCEwMok7vSvxHPleDPPekLU6L8IXScyyl+pSe4ncf0Li", - "IMzSjEx91Z1CCRLdiisxvwfiZquKV0troz8l/FA+x1SmianJtNuhB1j21C8JkyH2FzrHHR3A6f0zOTl5", - "xUiZ5GK6lNwym3K5NyzZTN4KWnO0Ll2EBXVyn6cKut6Ci32fDKIw5z+mbMSGHPEZh30xvdfUEmWuOz4P", - "k4DowTXf41Z5eM/6VGCoaJvPRRJYOKKLuIm0/eblQJgQE4grigjmGNa/F7YXO2RuPLCBd6nYmTU0SNuY", - "HtrWJE4sZE2TFaddKlbMnQdWN9+mFJiurDJ4QaCuH7tz9AgPUi41fxbYKxET0luivlMF18eQxMsKKbo1", - "flSuZrthiYpbkIIEiUf9jdpE7/tgtMgzoNYpT7QxJEpwzVRgfo329B2UEAgNyUketFiP8ONhPSjdwEco", - "Xydte49lHyu6e4diTMaQ3wDsae8SNO3VMMyMX6FyABZmTjGroEmN++D7W0HM+xLjnyPTWkLORLq0i40G", - "3Avg7ur67sv16ONg1OlmP476k8Hd5fDTcJJ5CQyv3t9Nhp8GF3fXt8w2Nx4P319xP4JJfzRhf/XPP15d", - "f7kcXLzn7gfDq+H4Q94TYTSYjP7BPRVUpwQ69PXt5G40eDcaiD6jgTKJOvf48pq2vBz0x+mYw8HF3dt/", - "3N2O2VJktti70e3VHU8++3HwjzvVN8LQRACqNRHqOEZBqhIIJBY4Gk6G5/3LqtGqnDrEX3ccDZ8GVwXE", - "N3D6EH/z1lWRjxOAH/TZTbNEA5UZVUT/BLNR8okEmnTUmY9lm8r7sc0knarRBQQa6Z/mf7XPF1TIGau5", - "IIS+Jx507KQi24fNJ5INCfCtOmtRl2bbKVZsgbHIeDcw5CVMrT+hw1pLE9qC9cJ6CxAIgL8kyMXXEblO", - "SLVNSQw4B9gJIwI9R5gm0kH0c6ybCW/r2eZNueTWTkaXZUNomD6wNqc9gysb/auRlArZLneb5nJLeTzM", - "2S61a94DNUO/F7qsoLOwx4m2M2KPgc/5VaFgJjKy490JCZ5gbvAtQnSXWVg7A6Z6fN6LT4OdJ1Z2gkXo", - "OyCGDoiiOATuHAUzXn+CIbhqfpmtkxMJC9ZZEQq+ZFnoowwPi+6pxIViU3wHkJ/E0AIU5iqtApJLHs9y", - "Ienn9AHmSzU/fWZxgCAQO8ueP4vph6sjfsA3SWTvmLVNHNDa0D7nXjZxAJHhaoKqNvv8ZZYEWoDNcmGQ", - "P4iknuiHLvBZcNgj9MOIfWYxx17iFiq9KeqdklF3e6l0n9PqJZUPwbJ2jahbtst6Lqvl6617FxQsanrV", - "lJ/NWOMtqt412Qi5tPfGU7zmKJKJhrO9UpMHGqmR087eHE6ClJudSXxPy/C/GEHZ56mkrFfX+hbDmPe4", - "SaY+cqtIgY1XkXJahXlvNl3s3yqbPhL7JKXo9ZcrZjHoX3waXnW6nU+DT28HowrZWZ2HoP5y1uQuVoWJ", - "HByKsWzVq3FxvGI8VooASfnF6jyp4WUwuhtfXk863c7gM7dZTPrjj3ej2ytmE7m+UmJPWP6U8+tPw6v3", - "d18Gbz9cX3+swH1Oi9IpkiBeVET2s+/CX10roHkOAhI6TyBmufJK6hXvrY+Ub5b0QJ/vYDMpDPjY5iXq", - "4V8vD1tKE/Xsm1KQXQKDug1rnrdgAQmMZfYCeY7ysZxf0BE8ck4dDyy7zqnzBOED/e8iDMj81xV9dFL0", - "aLMZmMWuRNRN6CNXkwuVa/xVl+C0LCBvqlEaGojdPPvVObgK4MyrExZQW4FqFEhKqQMpjz6fdLqdz6d6", - "UcJ9QncQcGiMYeXOzk2q+VQkzX9OBxxrgjLMVVLW9GOvdmHnAP2MlUvUldekFNhI0RCj5qYCIvq/PCBm", - "VjtoS3FraHpJQ9MWDUBbqVvXwJC/sh3ewIVfmM+TORsDvgEJ1iU+U9mEO045CDsRa+2AwHNcEAQhcQCr", - "S8sK3suk3aUDSwcd1t3Ha+1RwPNiiLFql8pp0dLQUTZP0Q8fAJ7rjps5wHN1yP+FC9OJA4grorxe/JiX", - "XnfO56watH7CzzBG96gOvcy6RmXQo2hOf0VxHgY9J8wBvgEYP4Wx7RzAiUQHB0Ni4K9tvGR5CEc+WOYY", - "Qe5fY0NWHrtfDQR2PgfBDEoEGZkggE9mJDLehU8Z1qRGrYd9Bb1DjszWHVUCkgJRib/1YCillxVfujk8", - "mVB+Gc5QsHrpttX4e61KbnuHcbnGqA7XMqnXQaHb7oQ0CIY93C1ZvN1201S1Gs9RhA/VyFoyOu/wNN/G", - "KcMn023b59PzweUFnCazTReS7Qp9FKNF4gMCcZZpg72WuWHie84UsgdSrn2AQJRoCmMH5DRmXUhPXfXz", - "88GlUvWc3Q8egZ9Q6te6Y/sExjdg6YfAwIEiG0jE25TXB+Qnqn04YUB/iOEjChPcE+7FYoxOVfKg8sTs", - "U3k+UgoPFbmYqk03uQrhwo5TQxmVgewGLqCfZEYyB8lavGwDWMltXpxIsxOZ+7ouBg0nfhqLVdjhbPQu", - "nZBV3MH4PvG1iqBdjEgZCzJcpORgbgyWMI5hiFOm33JLTNfFKg1yqyDzkhyPK7O0fz49Z5EQE4AfKmqk", - "ExgHwBf5AozmKtHMGV5gSYouCJwY3ovLN+IKOcAPlH9zhKl2Vu1cG85FYpcU5vMpxYdM//Ks3zAZQUKb", - "Yl32DWx6reDoYmhIl408zIXeE4xhVsxra6h45otgMocvtKpEfqUUVfhL3g6KMkzVYCrEpxSOpmGUqLC6", - "Mtza9xM+3pFzS2/xdBKcTDF31KIo95jiI1phBxBVGtnlIavMH7tu1i9DBkoeicUQkjvyDIKGbbkI4lf2", - "PAzg9X3nzV+1wk7T/y3AyO0nZN557q7Sv38z5HUGV+n84VP/vPP81bg4MTgzuvrrLBEyAAuaD110rTQR", - "Q3FIBJ5Y18nSRMUsajm8d2grGBDkCioMmSVGMoiI6VeEfv9mePdx8A+NsC8mVZbTc0g01GJGKUOGPofu", - "R7gcNNa61CVx9e4BLo+cCfOWwg4zupGQV2OB+VbOfRwuVFxIIXK0RgrmFKtlT2PKZustkA1RXpxQHVlh", - "jDCOoUsy0UFCR7w/6d2fmQ1KulFZkeI46yIUHeRWara8SSqh9XRYXhU3chd7pwXGyzkpDBqpdKjOwO7q", - "6M1e5GUiaw8Egyo/tyQX3vbHw/PtSgUmiPcAmxSO7SKTrXRjuLwAs3MlyUgxqY4m/Ui97prWTy6rwB6Y", - "2Sbh1/DST1RT20pVDe8zANRLzxQ6IFg6f46vr3oYxgj46D/s6ZGv7GglpbZissIxEsaOCwichTH6j1ru", - "tXx2QBhUJbDCBCwi8VCanrvcaR0G9i5c+1WfXBymLM20qUKuch+Vk7F32eySlo7iTJeFGS05lTHTRAFG", - "WySTf0fBTMi3qyZKjPA7T0HN4GQWEBBFPnILuYfWKeQuFrVWKXftvF8z8bMHNmMpCA336vLGGtLT1lN4", - "qhcWtpGRo2YPa7PWWWSUU4g/ZbR06jgJjrZ0kzVXcDGT1Q9SMb2ta16RmiNO67z9W1Z/y2ZP90TJ6iKk", - "hSn9dwOLVpUdScddCF9A1wcxICLrjdkpQXA2wo6XdXF+IXECf6UHeBSHsxgsFuzq9Ms98DH8ddMOC0Yd", - "R1HWpKrDFLYyPg7DTLcJhaJi2xtYAevG3qRgrU7EbzIcZmShstFenLpZSvb67LufT41lfwEhcBEZ1F7x", - "UZFgxaq/mpRTO6kj7MuivNVIKhbQfbnyw8V0UrrnOhIvHZaJxgbTzesZF9CxRkXjbKR94ITK2sPp56pi", - "i/3xeafbuRiMzw3L5fW22qfBpk+DHG/beRmMxdhbfhikoJtMPc1lJ12QXm4yH4BPFbnB2FVVWvDqN2aQ", - "Nl8xF5l1ZrxKbRqSGEFcv3z65YL77Bir+NA2VgY7nv2LGWyaJR2T19Bm+Zl5Ew6cOrW6Zxmu9Ze6dMv2", - "QqRmRF/HFZIgt5xgrGFGMTlWLpNYMXuYPvVYMaPYeHA1uZuoi0nXcMdPyFL6s/PRoD8plFz7OLy54R+v", - "by8pdiZ348HVhTKy/uRRZKylqdk+1w1GAY/cbFJqADaloyxlbKn4RkCQv0qds+piHc3KcXAkmJnyJkQB", - "4VGK5R0QtKiVrVlqNn3wN1rAVSPweCNN7jerZWgOYu4q1nRnVdRY3kOYCpUEJny6ldlWrVzQVJLTu51V", - "pXssQNgUI9nSNOSeg00RmamQyNL6nV9/urkcTErZ/CqSFOZfu1YrY6Jc/vMHdTbNus9bTKMThtMS9jeq", - "UKnvhWYNU7ZiA2H7B4uap8WaW3D2npTi5Alg4dbRIB+Al9eY7NygNVugjJhk9XU1w4mvxaG6DgqcBfJ9", - "hKEbBh6203HrPGELszi/pCH7gEBM6G+/1pePt0I/HV52s8d/nR9yBcoF1QuvevljBAMQoaOrMLhKfB9M", - "ffjnmOXNSFv10CIKYzapcMUvN44AveJ0ZojMk+mRGy6O54C4c0h6HnyUfx+DCB0/nh5jGD/C+DgE7Iz+", - "1gvEWJ03zNC6ZhhYshhH4CmA3nklOyo2ct68zJhVubvLA/JvDSnogPaElyRganhqeLB+wuKdU9lZq0Bt", - "4b5nURpLw6FbKo9VVFSzegWG0ljlg3Jdy8NqG7nB2S0eBCov78MAw7j5kYdEt6YOFLbvF0dqNdodVSQm", - "VkaaNAOIsNHI6815GNyjmTbfSHVx2bUqP69AfIWYI2twcuWTyzOJyHfNROsUzlLt46rW1OXWGxkNpDmv", - "0nOmm10gCuyqWn/yrJCv2MUNQblHJ/0WfC0q9Nu1ClUbYDelFZcCilPgBSTmC9kELcTT/RYtsB6MyNyg", - "99JPOWUCcS+wJ0BgfA98Xz/kzhTRteufbUeTaCg4uU9DQ2TRU4R3tEfXz6bQaKzrG7grtkrLD6S0rOYM", - "p+oAaxW25MK3cMRe5A7qVQ7dr4Uj5CXPUUpNLBF6o+NUHH0bO013lgSv24liFMpaKRq/cfHVREr6um2q", - "Plvj9ita10f858btKun6Pp/y5EltWOjK/mb6ZwCRk6oUenlwYXT7GQW3l0FsGjIwFtBWg5XsAkplh+fu", - "IZDN1kvEtIGbL0/zm3G1NRcrt/JwtQwXVeIDv6qsqQRql5k0Qh9NIWn9myHjCgXJ+dBCHRHMIfBgbHe6", - "87bFTRTT1uJKmakr1/G1SkT1FYGUDyTtpoHmXVM0pDJOLti2qIRaZbSiS53SURhCdWhMMFWRTUiUX2sH", - "KqAsHbUmt1U++NSfUR1vvlDxNv7QP+106X/OXv/O/3h9etbpdj5dvK7GXhrPqskiq0xkHxub9mIJTN3Q", - "s6hXlxthIDsxd5pZAEgSww9r0zEd2knH0wpMNAtYcSU3hobLK2bfGBumsgzNAqsJigG8KaIUPOlXXASt", - "lkYGCt7TsOLB/2HlCscDFhTD/7gdXVaTx164zkmdxtIhJtWBTWmj3DnwfRhUOYU2CNGrdICXz+6FI9GJ", - "JXCW2n35gFa29v3gajBicvP9cPLh9i1z8xsNbwbMQ69//rHT7VwOrwZ95nz3efh/THue3WA3H4Rd6aXS", - "3LdDmilb/47Wv+PH8u9oXTDKDydrGmL3+yHhYOzYDd/Iax6lNRZv8U69ltWbtc5M3tm1Lf9EnXsxTl+j", - "VTulchpeQCLLKBScfJPA3itBpGDAc1BvhVFj0Wn7d2GsgUc+GLEMHDZhP6xhpozkvQ3WD2Tg4ODNpZOp", - "deAox3J3cjiR6JaQlbc2rw7kt9eriZ7ZQj1LdcoqYF/q1UXVjho8uxgwvqknmC86lw+JIvNidhT8V3BM", - "UqNE++9FSTetSi60fl5oYqMlFhuZPEWRCq3ym8SGNNSybxL7jQxtwiBCx9XtdQ4lPMmXufjAphaJ7UwC", - "VK6KDOf0FHOG904QEieKw0fkQa/rACcGgRcuZKcn5PvOFDozGMBYXmNU6jrbGsabo9nbTwJcbW92Tcop", - "nLXIplLLbLrYqeUlL36srC+5LkbGFJf2O2DYN/YkCgIvq/AY86FWu/IvIJmHXqPVCtA/8Z6pbn8eegaq", - "/TCZ3Mjc2W7opRQsDT32+QbuAE84wGbOTfzVEuHVJCRQWXPOZ4Yq3to68ZiWAlamnU/p1mXGrkmn27m5", - "HrP/3E6YlmQ6IXkQFq6K0MLiTYjXYXJB4EQwpnR1ZF8RT5iV2G1Xm3WLUgJK8/kBjNEsgJ6TdWLWoNvb", - "4YUjSHr3tzwfTKGPq8uHsjaMzHM+IVw025EHF3J0HB0afYDJBwhiMoWAVN3Xc7vGqsGyOg7Amcve+Zvy", - "2cnZWe/0rHf6anL6+s3J729+++Pojz/+ePX6j97J6zcnJ/ZpUgBnMHpkDzABU58ZwPYQ0u2fzuZTOYYu", - "TKuSYlN2FtqGR3/wqnRhvApJjfJzaagqFtV8smqeuDaTEnayXk4YqLvYALLivFrokoBu4TC4D+24Z6R0", - "oEeTH5LsgrxKvWo+7Dgbh7luq8ih3xzwCJAPpshHZMmO51wh3IzIf6EQ3bH8t71/Jicnr6DzXXb2YVeU", - "VH7+VZ+n1A9NZxOGCxDNwxg6tJEQQysSzViONWbz6UL5rYtkZFOn2W/OJ8PPvCB3+udN/3ZsCDS2iW7h", - "e5RGtvCz0pgRTJze/DwpAFlvwOO9b+v04dvRpWb4puoxa69VbZSjonSyV2bulbmeaNdNOwtVFNnmxbVr", - "Jq9OVFqBh5d/iTVeBFIgR3lRVqiwDYJZIp6xrIXc+OIj5scu76wUqC6n19GrakK+Dr6RGGgbYO/BPGxp", - "cQwiVSG9vuyzDAM3/5h8YI8ik3/cDMbno+ENy5dy+/YfeuNOUeiWaKpW6AIuCOnQlNIKuq8UuHXxGGlD", - "Jwly4jw3uKZGPytdbqoNixbJQpmkydC64ucVnFE0qo0Hl+8+XI95qodP/as+TyHzZfD2w/X1R+NesOO5", - "bAJW16aPY0p/sXCT7jYoC8sVEVkYVl9O9O9wajii6BcdQFac/mc41R2JO9EojZiTRQQ1ajaYrb7W1DYL", - "tBe76uc54WaY3e0qVyDet5pJXOUpTSKz0mauOWHT0A0DD4knFn7LczWpTmaQKN9Z6XmN90cgk57wHHwz", - "SLDwXE27OjPaN9UaFLu9FmGM9cckBgTOarODKxBe5vrxstbmi0hZWqUQk3zh7GJ+4Vdn9eJLTl1cTVeL", - "1aotGl7o0iCmAA4vtDiUvT+iIGdIeXd7dT4ZsgPr4nbUf3tJldSL/vtKAUkHkZpIIwpms2vYS37Xqzdr", - "xWDuWDPSX++eK/bTmDuKMclHWBVOSUICfB3Fpjz2AJcGnyI5PCVLu4hNeTsHDo6gi+6Rm03i/BIBjKHn", - "PCIg/NV/1XOFERENHM7011sSJ1Azft37req5lRpgTk9OToyeWNph8r5TDd2gGi3o73AqxZjtOW4o/LB2", - "dDM/EXdtpORzC1vPy4CQcybapGOQ6vOh9Q4ylxp5u2ww+ETpVXbXaaiSGB1+1skdng2kuvIoYH+tFiZ7", - "cldWnH7sD4VREqyRV7k8yjsE/dy5r6btyGg5J8UUyVgzyVg6M7Wyu5Xdrex+KdltmOMHFO0V3pAriGY2", - "2pDAhdm/0nBfqe9srLY3ZqnQqhPurulxlmVb23gStQ0MaJDpxZS8xdwUYlHdEiKVUeuop5Qp9mZwdcET", - "xGapYjVZgPM5Y9P0sm/75x+v372rPSXZtCvdm/MCxUyMk7w4KfrbhMGNIvlLsNIGY3cOvcSvCIoydF77", - "OPpSzJNiKWBqNhvzKupGL6RcepYtsmNVPTJcuwijkYBlXG5CR3Koc96xTgstNC/NnzGENrl0VR5vyXTa", - "j4K5tN8kjzbPDl612AmY6dDrc5VxfZN/sOHkKsKsyyGsoh8hFM5jepG518sFLUtzvrxDBm6sm5A532tn", - "ZHLkTjzebnparF9hc82ggDeN5IVpyMUqA6f42axyz9UtPfoyDexOvEI0RzNPMWOUp5t82aoCQ9Fmiyyb", - "e8Kw2RD11YM5vdyDxCc3lVmWRCNjtiWrRwJxi/wT84N3YSiB9ef4+srhQJfDdtgIWica+Sz4Qo99Yexx", - "b0wLNGChdkzQAoaG4jiYIPdhaXLFod8cLJ5V7F4SFXnRgG2ZDvZ4Wngps8Kx0mfMkz/pUP6YUbY5iavN", - "Ap+U92zbd4vGyXKtr4FyWZIwcgN9red0RlabfBtqQp97sSe7Qjh3qMgehQpVhWMIub+KsaTIAnyrafHU", - "TNk31RXhkR8Jlb9MfnIIpxDEMJb5TBhG2bHCfs42ZU5IxK49YfiAoGyO6K7yn+Tb+ZuOCGHO+orUNrR3", - "gkm4sJzsmUl87haliR7gszj9myErd0WYTSz/a0qIndOjk6MTRsc8iLvzpvPq6PToRMRjM0ywmGtflImd", - "6QJk3svnedoqgBg7qT2GbjqQxU06l+L7e4YGGdDAZjk7OSkP/AECn8wZil7z724YEJFUQ9STpk2P/8ac", - "r3B6ANbw8SCOQyqFn0v+qVchSdeRI47Om7++djtY1nChq84aSp+SvwTM7hy6D52vtD/DXwyBt6xHIG2G", - "qjA4kg32HYVswQ4JHeC6MCIOicH9PXJrMZpioBalj6fHwKciJZj14AIgv8cekvHxd/az+tszx4sPieb2", - "dMF+xw5IM33R7g7rzt+mS7vQpy0GtAFzteAjMJ6JwQISpg/8VeHkU5rBEXnOO294HoRUaJSW0lGFGn8f", - "yHZsvZq8X0v09JvGkzBxXYjxfeL7S4ej1MulSSsh77nb+W1XlNd3FsCnWICewzJoeTLsiIPxauNg6KB4", - "F8ZT5HmQ3z4y+uZ0UkVmkuInrAk9rL71YqFysA+8b6erIYyv7NpLXE2WdH7dWofE+Qg/Bokzengbcnm8", - "EWLg2OGbVkBcGrdWJpNKbJHQSSTO89h41ov9jSxEuwQd7DkxwAFtxYClGODUsj0xoB6QEeqR8AEG9FSU", - "f7PTMAp1KQ1G8DF8gA4IWLJG1lp4a6UzFsREhCa0lTTo0O42UiId3iATJKx7ddzFbHmCzhl0PzZR4yZU", - "LUiHbuxE7Jwk4+y3KkpOtzxHwa4fJt6xekM3a9ClTHHy2sMGcVCACQhcWCLic/pZupeYFevt45YB4iRB", - "FnCxLwRWo7VzBKvv9WLrPykvbN96coheGHFnF3GiKfvNzeHH39l/n6v2m0op1uqotKHMKs43slYS8STR", - "JuWEp3DcpRDa3GaL1Eo1hzevofIoxBrHBtuxVrblSFzBTEbeHMUVUo3Tz1czhR/XiTW2LalUq6H5i1SA", - "/ex0f8FIuKX9/aL9BVz5DDee3rs7uEXGtSY0lR6JB3KQb+IIp2McMzs93yVs3PFLhOkFyHdyrU0bTFsP", - "8w23ttt0LrHjypQNN19mwMmtbp8IId16thGFTSjvf26TwwCRkErz4++c45+PozicQvPlUr59OiBXeYLZ", - "dXnlilwuBDPDp1PfhJiMkuCGzWtvmzIdeqnk2vGpV0FQ8Bt0E2lbYfg92umpcBUSVoEgjNF/eJZ6kdOI", - "B1/zKM2SmZMA5EPP4XZ7h22P807I82G2rfqDI0dm2Afuw/F39h8LK74zpg2VAit5ymFfRXIoe6N9bkwj", - "8TAQ99I6n8fJPqk2p7sB4zbISJhP/Ho3E/OcYyx1I/D98IlOr3sRKFKtFL3s9yoVixNdnmMCfPwdB9iK", - "W67GqtQv80uAG7BJfjAzo4iTe+/YpICMllH2kFFKBJuyytW4klECrGETqbgo1ia96kLnlVfiEos0fht7", - "Mf2jazYE8MJMK1kCFBjOXr/OAXG6CR0oikP6D+i1Z9gesabpEsnqNzggiiS1l4813qbAjwRMfXjsgRk+", - "TlO/Gy+NmN0aWTuHzAFxptAPg5maVSBNMw5m5Svl59MLwMrNTkQJ9XpzmUzwnSVo4Sm3Gcv8O4HxMuMZ", - "D8zukFd9zG0rQsRK7hTgfamLjzX1bqwG/gWYnYuYL332sQo5RKeUr39s1p/bStjtvN6V8KO3ULSIfLiA", - "ASnpBsx4IekgfToH+EErYVjD4+/0PzXPS7zSxXTJ+aYoQOgElqZ2No7x0KeA7vjIB4TARUREXhaDUBCN", - "OiospViobdrxCzU9GpneGFZ/dv78jd99tj/rRK2/TzWF+zDhSZr2RERk/FwSEeY7A7ERIcd+OKvTVfxw", - "5vgogDLzkYCjKFEuw9klCng9lkOUKiLLEwlFWt7p0iBZeBpGLTQoIKyoZDno0pA8NyYiNXbozCChqGZY", - "NsyMEbc8amauSN1guDelVQWspk4CgvwNTN13qLzrEfiNOBiC2J07bCalxnPF+lkHnUivXiujYPgI/V/w", - "r3QiFLh+4kHT/tKWuKPVdqsFvmQBOoCtcuvJ5DYUMBalYqY89vluurxLO+WgtAKulFPH6pC12p49OHJV", - "IdRAIRZRrO27eV4rTSW/cuxchrP1Tx36/70sdNj8uqoUajMePGkdth/g6MEPKDIx//09hhs5d7Z60m1f", - "pc72egUHmfba26rVORmnkzDrq9ishWKid6F/7MFpMjMb6QePwE9YvSfnfHDpwG9RDDELqgUzgAKc1U8T", - "9YE9QMCRRh6eQ/+CTXUoLgWbj2j5fHo+uGRIqAlgYZjEVBSyesFUTOiRv9M4FhV8mXaxRtRBQT2eZg2t", - "XqO+xE2TWYnFFJ4/H1yaWd6K1y30Gv4AkBc9aVXjIj8302328Y3uR9JvNDdaacx/gEusXJSM09J2za+X", - "jAxExH3dxfI8DDCiV0lBYuyRKXRZ5g3PAfcEivIT4tq+TWNDNSxTeB/GsBaYTZkf3vGtIWEOGhCzWnSh", - "i5gEfUJkrr7FFctDa+DL0koYdnbLz2T268ql8XcWgLhzxJ4eXRgTgIIsdL9qnWk2PriSoaRQ+N16cemW", - "iFVOl/S4Q7HDnyt1EIuEfS+6LdOlk2XIzXzEWTG19F5isKmUEwhrF6Ip5iCneYDLHq/IFAEUY+cXDzLB", - "R7lv6QDnX2/+9WtRbFU6QdgZtrAbRtBKHvKWtutirdeDd7t3VPv7aWuBqrNApbxhGbbRQEE7ZsewpZbG", - "z3YrTe0jXB6Ksrb1MCaJi6aMwNDdMoOOGRyhPW6BIb4/nvYaBK4y3wKC9f4FTWJY99iv0ASTxNSBMqfY", - "n/aA2khoIW4SVphSjhVnch3H5pgSLWvPKK6StuaEfTUnlKovWyjQtbfPyilKV0R2GedzHq1faqLZXQEn", - "UwyJ44LAQyzPjKTrjd4eqlbs3GLoMTbisBB6PS7DA4i0ubK3e0PRjJ1ePBTWbiDYpYhpJXte25J4yWQ7", - "x2+VrtU1vO2cs1JDDnAC+CQGNopm3vbnfrxhKODosHnAYe83KSk7rLATt+rv8s1GkEcd64myUwrA7ZP0", - "rp6kr7JX6BzDp/yZ8qY9z9trceyCxf+2CW8EdZKiceLO/VLjBLciFpftybXoL1spJg7ztmUpGmQsZysW", - "XlIs2LJ+VyFMevRXhGKkCrzZYMJnO2SLScrPPzkXz0LSHu5Gi8kKZ2yR0SrTBNcfmwce8Jw7NtMkuy/J", - "cNu4AvBNWvkK8ALJh63lg8w33MqHwzvlLZR95tu+yIrVVagFQjLKQGAnTgJH9KzOW8w9KC4RJtyLQtbG", - "O1SZVo6EUtBQ459kAejawVH10GzKQalom2XW38Dj3jrm6dOaZuiFPF0o3LxqHSPl/4XVlAMGoEWVO9r+", - "Tra+Y623SmxZCgT+xsdcpdLKu1n8rSHZAG+Igtkdr+G3I8j7Ggeih96j8OmxeCTIPInuFpWuRC9rxKaC", - "bZQEUqI1j5tWpWib42B/ApjZ3izSg8ouxsL+xI1CFBDLc3eBgoRAeh2Xf8UQPHjhU5AexQ2O4feQ3NDJ", - "D/0QZgee9A1WQneEwbrTVarUn52cnfZO6P8mJydv2P/+r0HuiO79e34T2cQBySBNPYdVUEMK3xrA3qMA", - "4Tn03rLBm4O7fdmYI7UVpCPjk1Y+7ql8zO/OxqUkPnZZKXBzFBovFZ7mo9HJO97k536gZChgqkpNgSSe", - "4yt0XIm0nUaRsUl96PE8YbUvk7J5mySqjZYtyaiCZNi4ZIph5INlVXEn+r1SMvEmP7Vk4ihoIpliibRd", - "SiYOpq1gikXrVi61cqkklwpyYYNySaT+tPG+lenV67xvRfb21v12n91vObk4dFi7+DXW/oo2XyUYUtDE", - "OB3F1t4qic4aUNGhAtLqSV7cw1VlnwYurikjt2/xeR/XFDGZ3BQoXtvL1VTEIt3E1s9V+LkKfDR55ZZM", - "+UKerpJGmri67mPy85/b17Wc2dyC9xuoTczdVfzDzt+1VmYcuMcrnVy+PUoWrvd9zbBiBna3dmhb/pf+", - "rC3v74WrSy17d1Vyq3FplfQrfFqFemjg20N2ay0owD8aj0pv1ZZHDe6qNcckDOgp2IsBgT12A6WbK/be", - "ksvq/Flrj8UD92jdLodtzzv1x1XcpYtqKxj2SHHXyIPVT3b9Df4mxCy/BwrccIGCWUqvC4gxmFWc8CPo", - "QvTYyqAmMihIfL9E+cHSicDSD4HnoMABwdIRq+12CPxGjiMfoAKlFafciQyxyEyaw9M98DFslQtD3THO", - "eBp2W5XDbe7pwme4FydB3RtHPmtg7StHliWwfenY/7ylWGRytHrr2FnWR+aHD2IfQcxyXUMr8LYYFOAD", - "0gSUjVVM2RvHb8tcNQcSrUCBSOPobDLswHjLLv5f5pDMuQAQFWqci/57TE+vMPCX6u9p3UCdQAr85Z1s", - "UKuoTMPQhyCwiOnIFZG0wNkLhXdoSl0a4zwsMvu+WLyHc++DGTtqnwRdhDFzwFDJIL1fgsBzwoTQP4X6", - "iKn+SBtIXfDIuYD3IPF5vvt/UXr4l4PunSTAkB3juuWLme7koJ1KEtpZPb2mL8Ct09C+1d3IaZSqoit/", - "H9Hf13yJUjXcYw/hyAfLHnOXqNF3RVs6rHCvCO8rlOBqHfiCD8bcLg5aH1ZEK07fsXJIEfGSAn0CdWZF", - "QJGlL1JueMsmeC0JtKKrFV1NRZfkkx7lk2rJleNRpj3oE/5n6e0qJNdADDb0Dldwtffc9p77k9xzd3ac", - "ZXKhPc1+pNMsd3rs5GQT12tz2M+EN5BepfkLe8XR1bqXngrUKUipearOkQIJhf/mrt+oFa0ZEoB83MzP", - "VKWQ9r2p6PZZYKANMHien5nPp/JLTSmJPMmBwGPOZOn5T8L0KimKJf2z4zGi+GfHiQwP0hn9WLqd5WDg", - "ts0Z62l4BVaWd7C5DFfgsvYU3+NTvBj+ZsnQ3RJBr8Dix6JkXBWnE57liyTMcJTn+6NaLh7LmnQr8rI6", - "vaKu/5isrV4/W5beUyev8zDxPR5PSy+SOs1lj3KT5LgqLRD5IrKGJXuyKLHLwnJ5kDu31NtfHdIi89ZG", - "r5+nIk0mVrUGkB9Xoq5U1bEVqq2eVJRdBC1QMKvXlkS7xtLrPSQTMcXB3n20MsiDEZnzjCU8q5njzpHv", - "xdDkusE6NJR+2xckfHNaSXLwkqSKPzctXmAkZIr88/kYxO4cPcI6LUi0EmDS7loRMiYwEu66fTmwhfiQ", - "4xmtpxLe1nV3dY1smzJJ7LvYcyuplE8q2dYF3X0+ppTrCjmZykIqx/4K80v5RLefyqYq0ZSycL1MsrmX", - "idL99vJoIGusttLoJ5FG9netVhYdjixSGH/7ksgPZ3WeUn44c3wUlHSjsjn6MpxdogDaWoNaMfSy8Uw+", - "fIS+lcsQb5mbuYoZJB3QXu8Q9D1jBjlID16HzabAUVHMhHVoCsiY99KGkgAWKBDGXtX62ee3S76WhpNf", - "q30NeODTeyiGroh2r4DiQmm2CiRZ/+0eUqo0aAvor5uCLpXCyllwGc6aHwPC0agitTnzgMDCk8jguD9h", - "P5+rji+bdszhg/OJ6pL0ctekl3HF4RA2cr4RSP2xaXwFr5uU2NLstMKfpkjkOopOXedqTcbcNUa8sFcS", - "eNOETGlgh5jB+OSzG2+5l6V4mTKppfbd3jY4MXoh5BcN+I2fwKVCGrbMlstoWp2DKeCzoWBWzVeHk4lp", - "S16nHAFNDrcopogkiMdlvEDhzvacW/+cE3yyAutVnHfHwKeEEcx6cAGQ35vFYRJVPpxS5U7eAgV5sTEc", - "NoAjBiiybp82GdAW72mDQ4l02v5JqENMw5JTxk1oeSf/mlhBrY3OMeurT3muOsb46UMq1JtbATd2Z10J", - "5Y2udqfbZe8VTkANDbV8rb37ablts6fkMYaE1LkWYbZ7sosju1RnM1DIBQWzsehzIEl9d3RMKohZ44xU", - "96RlJc21ToOmjfFRhHokfIA1yfCc/s3Q4e2quaYfoQlt1uqT+Jj5Fd0MGT6wRepIHZ9I/6jWhl5UHilF", - "ctQqzJD+uE4plyCjdjtib3VEhgBJ64pauE0TRnHSlr82HDabMVNDBqs6cCy8pXh1uZzLlCntauY006Zb", - "3Wv3hAe4tHJOoO2ap59hZPARLm3ymmQwpe7Lwwtsmw+Ty4rGAEqX6OHFiiBmMWhrpPKxgXCUBDyOUhi+", - "XsTVg+3nyzh6sKn3wM1DhUN18qggliyDEFw6j8BPoD6PEPwGFpEPqch+gMvTN6zpaadL/3XG/3VGxXt1", - "vqFPm003lC2DJy5NMw5V0zlrPDz8TEMrRdq13jWB2edSUVoYctc3IbNxDTpIewVgCGC4qDELi8TEL+Le", - "wymhic0X8h4/u3f12X/vZtaR4E+hnsJvLoQeNJRz5HvTgM/rLybH08R/MLvTvU18UccI4kwm4EqhQPv8", - "xIKBLr+hcMAvKR1wc/HQRl/smXxgbKoKCbxhKeGCwIV+hdst+84NGUri7JyKa5Ia3K2Ej/AzKxQMAfYK", - "hbgwxDDywXLjYiNz2KL/esouy0OenHhbRTzkD+H0b+haaC4MaTDLUdIKqb0VUiNGqduRT8yMZmlj5bY5", - "CzvrR7hsn/UyY+NKt3WG7PbGrruxO8L2u0k+EKeB8ZzmPIibHc0jecT8rEczR8C+HM2bMatx4Fqt/ic9", - "ML+z//aeEJn35Cdm3a4NPwIE8MMzqDQQXgAC3kPyBZH5RLJ9rfyQ7KMXHyWQd/12+cOf8nTTVknHwKii", - "PeXzvmwKZqx5t6sh8mp+RsEjIrBpwITspXcCHbKvre4rfT8VfKzk9Smx3fp66sIhMlrcUgwEn6CS1tvn", - "LCXqgaPELtiB4/ZFIxw4uKsENgjC+Nlje8/OdqT1AmL3zlXkW51cgAGY+rAXAwJ7bEzKHoLXVtGLhRSS", - "P/T4v5+5iPEhgWVhc8F+x6kZyUbQ8D4H672X5/pq2HopOg795K+VLZxC9lm25NiME2FGriZdNL+PtRH0", - "zTjhcKLoD4UTthvov5pW8GKh/pacy+E7GM4VIfiNObfq5FvAxZQxX6MbpOylZ/FP7Gt7g5TUqOBjpRuk", - "xHZ7g9TdIDNa3EyQoBjv+Dv/w0IJdIAAwrmPw0VdkC2nhh9DFRTLNsHGP++Ud3/bCu+uogP+HFy7R7lq", - "rwypaVMmzW1MA3nRlYRskUaqNIlZBPwYOvBeiIDtKr98u+yUX4GOPUl5ZSm9NHqw2LdWeL2w8DLKlRWE", - "V5XWE8XhApI5THBvQXVQt758UdbFEV1SH7y6zJQ3addPYrIf4qJA4DdyHPkAFaiiOFKTO0AZyy1TvjRT", - "Ug7Q7MumbiD/TmACrdmQtW7Mgf9Dex0Q8x12ZPMhBatu3x6So73VMlg4jzDGKAxambhPMjHdnbJElJyz", - "qkzMnvpsXL3j9LGxztd7BAi8pA3bvBr7XJ12EzkYajG5zUwLKZ3tQbaFIiy7KquR57UGwQQKO7d+hgUr", - "uIqbTNwyb4tL/uuqElf06EWhj9xlfcpJ2cHhHWwSTkpX6BvWo003eaxDy2qPRoXdaB+Pdp61FfvAfahO", - "NDmmTZwnOJ2H4UP5OZV9/sK/ts+pPMekipMmt4cCqveJHXZU8fg2AAmZhzH6D/T4xK93M/EnSOahxyp6", - "AN8Pn/TVlvkGMT2Qs4B6nrGPazHiMSYgJkZ2HNOv/By77idk7rDLSpEhb7F8tmEAXVOEsp6HyJmvTs40", - "eFC5h6FMHCs5rMwh8ITXiB9ygqmxeLINh24SI7Jk+HHD8AFBOigrivRVpQeG0vyMkhDoDqxMB3V5f8dX", - "4yIBFgRygFs5LOTw1XiooqqBJC5iuZXFeyeLy4yQSuKr8RrphgsD6xisjcZgCMjzV2WW4c3RbH5S66iK", - "4q62DL1HDG3kPEuOrjxRRZ3O3i6erETp8EN7udq+uUCHmGY2g7SedW5n2keVfXhUSfdm08/Muqrqlayb", - "FVB3pkvOUIXTmxPigdjxuvta2X2bEkNs0YryoZUIOyuFqtLiE+D1UOtEhHqo05/oRq9aZbtaTtTmBOwT", - "AheRSG7J2iriwyQ4Di0ZYCtBqlziEWa+0kKEcCLw9++C8MKPeHWMsiuGjiHtWJE7jCVZtOVh1rxl4X3M", - "ZhYngdiqGo92FEQJ84fgj7u65T7vhabS5jKrkC9sw19CoGRrqrQF8GbCWaBOuLyHZMyHbUXLy2kHzbL0", - "GiwNYrj2QrHPFwq5S1uRGgTghx4mgNQYDAF+YNWghKWwxko4AfhhzAa1FxHDix/RNpgiogGHanHd8uge", - "mAFNbLCL9EjCa6b3FMYPVckiMgdso0tT682UBZNwVHxhSKUIqarqSZGRBrzwjo7cjva5bd/ezxXyXz2J", - "oRjExEI//Tt5jn84NnZUjFczs9coBaHc2pZz9++hXGW8lQ5LRhXVD2n0hOTCu9pLPjsbfvrDMsNEW/N6", - "IxmqpfaQj9Fb3btSIpobgprXolCr/2pKUigle9vCFEphCgUvuMagm6uv/HJlKnRwW5ezV2y9OYJpL6l7", - "Wb4iv0flcOBqU1ITgfNd/WedH0uOE2pPYEGmh+zWUmB9PWgqBg9YTRDbtWpmgdbNxRzXn39Bqo/p7+Zp", - "anV+PmaPkbWPSfzJkjO0CvRRDV8P2egtc788c2dZTG6UIpQcxnXenfI4YtvdmrV3ZNb+ouI+sMkfkm1S", - "U5VhcxIHz0EEt6RHjNnYrbw5GGWCb1irUfxAGkUauyJ8hiojQ0Wldsbivp++j2ONrlHF+ixwkruyDGRh", - "v1YGbBzAS4CJM7xgCevn0PGB3EFTmiKAydAz5il6dabLU7QDH9smBT1LZflak8j++dasIEvsHW/sZCG2", - "eplgLe00mp8ycZoH70Hik86bk25OVOwihVo69+tVJh/zTGrTpcMm0E8qPpnzOexC7Wofezavb20yJWM6", - "Zm0w0LmMa5gC4s5Ljz1VGtPhBANty8tBeSfhyLB12xfRJOWnkk0/9kSKpeZ7qvSNkmDo4Vzq2bUQXM63", - "29AgJCKQ2tejmvRonGx28XKDj904DOo1EtrK+TucZkCRGM1mte4T53EY/NRqysHkd003Fnl02hkkqUp8", - "VJPG23Rx28Jdl87cFLyrOlVKOyWj+CbT0Q7NpzrMDOUVOXOnS+de5OXdWOpeVYpg+/S90+X2MvgqSsGO", - "c/jmkLGGht4euxotvXTObUldp4fu8Xf6n5781a7MXfkgtn74oIRz4EXv0tWbwMphdPdl7yzr02k3sc0P", - "XKwXp0dTs7eKPEF8fe5WPSauyVyH7J60x5y1paOzPTYPwbDf6LDeiHyoKy/JZk1ntBYOB15rcr/kw7aq", - "TaoCYsINHFa2PkoFvISjjW2vTlVQi0G2qkK1HBBsuQ1RYKfKs+PA9kFPfWWsd1NqDWb7bDBjj8gNrGWs", - "/Q5NZftox4tATJFmcF0pgMUbf1EfM3YEnyZFjBY24SSyXbj62vgsloggwdCq3qJsu4p1a8z6CjuTDXAP", - "KPCsoGING4P0EQVePTQHb0wlaAEdcE8BLTlPPwEsY5nVJXTOTs5Oeyf0f5OTkzfsf//XaKxm3ft0Aj3x", - "0mO1R6Ho2FYjpxBP4X0Yw22C/JbNsEmYK7B8jwKE56vDLPvvFM+bAnqjmN7e40DZEv/TPg0UdcfWwrEV", - "d+ntvAkwD2mb/P3AEaDRgy7P/mpCf8tAiEOuQN2q4a0avns1vNUtW93yRUKg8JoV25kAaiuL1J/vW6ie", - "np3zFFQv8enxWGM1TFuuYj8cy86tFXGfrYjbuxelBHBQnlOtMtUqUwejTGXLyET1RmyzKUhWDJ5aaTUw", - "bzVGsiRhWqvDZrUSgwawXb3keJr4D73ME1EfUfQ28R+EU9uGFBU64uH4J27JD6HMUxlabMOOpvVbs9s6", - "IpVrMieeU0ksTtu1EkJKiLdW+7x1ScHdVWokBW/k/BJD2fvXDYqNw3Gu2qnYkGk6G4gNsU/7KzbkmmrE", - "hlhHKzYMYqN2n7cpNr6nf/ZKOSNrIyD0IDcUGgceB6HBgbGakRbVexsaod/d1uGxGBthwFMzj0cDbdRE", - "SWyEAQ+6QvFBcd82D+T2rn/oMRTbliPV0RS568CGJMuBB1rsvXDZVuxFSbo0qI+akVE57+PLXllqJaQa", - "7PFTKj8HUP3ttuqytClZaXeJSlNoPmeZW6rKWDnACeCTOX+LffoWEQ91OEWv6jOJVOfMrARtR6KRY3vV", - "sDRROdq4+TuVjc2Cb9VaXWb4W8m4e8m4d4VOhKCrovLtpM5SZHHOqUcvj6VuICSyvYarU4xaKbxLKSx3", - "YAXNtEKt23PFVJXArWLail+T+BUKSZ1OvHGRy6vn9dwwCUhNvARrI3ORy7KP4BEgH0x9yKSvIm709oX3", - "kPDqfPiczXjworcuZfyBl4zIbdaKZkpOKpx82hdEg8N0DkmrFZLIs3+CYYyP3SSOYTVnY3474A0d2q3E", - "vbcYxu8hOReDbZHu6EwN6YxB3BYgfvkCxNBNYkSWTIy7YfiAYD+hsuuvr1RUFZIO5clNkjvbfg0ZzxCZ", - "J9NjF/j+FLgPRnI+DxeRDwnkNH1N53e05xGdiNuj3rOhrykuz+XwBQJ/dXJW8/bqinm98rxzCDx2uH3v", - "+CHfjPw+FMX6cwGZOdzJBebnsEQfJiA2i4Ix/boa4ljX5lhj8GwfZwy6hggLw5kPt0NvbOgfnN44+jZM", - "bxnifjh6Q8EjIrC6dhNm0UxSG+YdmNJtdXzTESas71DMtcVTXJ3IypndR1huTH6Brb5ofayymjwF7GWU", - "N9HcEHO0dwxcF0bEbHnrs+84tbCJSUrUpm4+79PZjj2JD84nUgxJBgNQBfXxlevor/WYSsmLY7u09/b0", - "FUNW3aKikj793oy+eJ/OturS08E3QF985S19VdIXx/YK9OWHMxSYyeoynGEHBQ5gZ+NRhYJxyQbaknMG", - "PYLp+PWEtLt7tB/OZtBzUNBen1/4+tzt/HZ2tqt1R3FIaYAZbQcBQWTp9JxH4COPTUY3RTRBwcyBciSz", - "wssIW3+V73a+9WBAp+rFgMAes4FTHZq/1eiYOUxIDTeHCbFj5zB5eWOVYLJwzwp1t0aqGm2aUY+tfWoB", - "F1MY4zmKGtzhlE529zh+Bn7KuomkFFslcP2kzS90KoraS90qlzoVg/UkGQGMn8K4wpUizcVOOziyfZVI", - "vZFjbk9JOp+DYJZOtE/akssg81JEteK8VZqaKU3VrM4pP8+Ma+tTMZxRSRxXXbt5C1ypUqWeUtviewnG", - "PnG8RF770Ngy/WZuSpLKN3NZwj5wH7bySDWmI+/xG1WNJG34aPUIYyxAMLo/0TWIdtIFCsP4UaOlD4P7", - "8D0kn8WgG61JrECaZWg8PTo5OtHlgFQ8j/5Ku361KDc8qVhswduygti/QCeGJImDHPIKNx0qZpMgoPyT", - "TvGtJ4fshRFPOVVmgSc4nYfhQ084oh1/Fz9YhL/To060Ljuq8d/tI9vFQGZHsHSiHfuBWYaKS/jag+3l", - "jRPF8HSVTI3eX6LFVyvmOBZ4tjFTyKbCr76GY4Tihm0TZe4t32zGf5JDz90nBWooZqoyrlCspHVABHbS", - "7WrZc4/Yk1llSlvUlEdT3mR/PNd4X/NWWsdq5pxpxXPcybTKZ1lzxh+Ox3Jj31Gx4tYeWXJKLgV8yQuK", - "2QeZqdX1lR8rCdk+7cBe0PK2ovhz54bprBAYSCTKdhcHZclralB+y2mGmovrMFvhNCkG91glAmtWg7XB", - "vWgvI2SaJNFKAWwD9F44c4QgVoViVoyP6dZpWPac0EDl+hkCxVYMDmt566V5S41CW4exbNQ+e+5qpgfu", - "BYNtXhfMI8M2Vl7kJM1x2a6VQyuJUFQPW3lgVBDXY84aNdGqXB7dpHxdvJTxHtOXDuNJ2aA83j7ws6ZE", - "BS8wsYH6watXD9YDNovDJGJ1PzIQ5EYZQWGdPsJlpzYNyJaFxJq1uOSjUluOaw+1iZXqfzUSXDI1kdG5", - "RWbVaJosaKUcQXspuSYadjlyhvfMuo0TSh3Q6zKu8gGBmKQ8hbBzD4k7h56pOlQm+PdckRJksGLioRdL", - "N6TA2yjPUJtdqM0utIXsQo1Es5AN2OJVK3eSW4ll4VtzQCaYH0Eub1nKSYep9VTBVt7tlQqYkeKqKmDR", - "8W8KQQzj1PGvq3UFZJ5kXB4ksd950+k8f33+fwEAAP//y98GQS44AwA=", + "rsNhCls3FS8pMrRIdF0YEa4jjOA/CeTCJI9PrhBwzK5HnQsUmIm12/nWC0GEevSyMINBD34jMegRMGNQ", + "PAIf0X3pvElX3E0S5HWeS4TE4dWt923iP3AdbPAIA2JcMnyUdyErfVUzZK3mymf4+tztnNNzyLcAaOjl", + "QWq8HdmFK2Hc1mR7rBZEIWRLCgM3iWMYuMtLtEBkTGJA4GzJT+9kQTuc96/OB5d3w6u7m9H1+9FgPO50", + "Oxej65u7q8GXwXjS6Xb+63ZwO8j++X50fXtzN7q+vbq4G12/HV4pe5xBqcw9dsMIqnN+uR59fHd5/aXT", + "7Uz644+1/SEh9FediIkhxtpLKWVnNxvDydp2qRLoUW16BgNIMeIAemQ693G4cAjADw4KooTgriMZuetA", + "4h7pxJBfxGslgZr245kRwSgJsH4hC/ANLZKFEySLKdXX77OlEecpjB/u/fDJiZMgL0BRQF6dae/zWG6J", + "Jbh8C2lHAqMRBB7VlnRKN4U2Ft/TwxY6tBvF+NMcuXN+yKmbg/kO83sxPwVqJKzAVnEDuipNyGXqRJC6", + "NgJIHW2V9v0BLrnu53mILh34N7nu6h4YbColmPgP3230Mi7q5OFrllf82Bka+MNL4sxkIrcGihMZYYcJ", + "+/Jm2B8R4QKRAPldORFbjP747fPDl9841zp92fhfLZCGozDAsIw1IhWaMsZyYFWDwUcxw3Eeh8EXwbqT", + "GM1mMDbuY0ZlnxS1pzSwG4fBoJpuaZMrsQFlpZmKPe3IUYzCGJFlkbSZeBHSqfPmFTu8+N+nZZIvKQh0", + "tq5ucQqcpVV9TTFYfVbrcVYgurRNKupTCmQnqbLNGTL0YzGGshvgQXeJo/3ZKWTonm2TuhnlMeRXKXrT", + "cZocC+Vh2ScGHBvQuUc+gRSiek7g11GGtWzzxldjxbpg3EUSRsjtxyZ2XIB/h4EjFXyHUozzS3909atc", + "/fhq7LAx1hFjqaa7QMH/Pu0uwLf/ffb697LKmwJr5npudOz7MCaDBUD++zhMIrP8pk2wTlj6CBO6Rt5C", + "mrZieiJa2n1WWL6HHmGXzVheuwC1buU1lxw+uHav2Se5rXStVJ/gl4yN7K1cV7cTh36tbsRX8wlSfWxE", + "22vx0RGD1WHFjI9ghgL4GcZSoNfDJBs/dzsweERxGCwgN3PX9x0oHawvytwWvok9YEgMg2kIYg8Fswsh", + "Z/U6Fjc/G+V5NgyXyiR0MAljyB5h9HBne4P9ZGYQg34y2/zCu+LNiZ14zwYTJQNKT0mZJoFtD8IqpGoV", + "C61EUWzAZfttqk40mmsNw84Cknno1ZsJFHR94l0UYq88blfWfbodTi1DTzuHvMPVfDZqbrKBYH7tMGYj", + "VQqabqDC7DlYBWVkdJDuQS2dXiKdvIvADAXpe0PVLt6kLVNFnonupyb2IpVvrN5FdLSjGDYuBu/6t5eT", + "DrOL6s0a6gDXsQfjt8t38lVZDhNIxReWLK/ZSEz73aXau6bWugZfk/Sltv4IK7JaGdzhRV6AF1/oxfu9", + "cSGS/kdJME4WCxDX2n3YVn0pd6tgSa4zpwv5Kjdcnon5TW9yI3F++XN8feVMlwTiX+uV91RtZ9N/XI8G", + "5Bh7wPzpcsp8LwHdFygrQBQS5ALF0JUgSSkCsNvhCpJZfpgkkIXoGUMQu3PtaWSi9/LrIbO5ax+RmZaZ", + "mTtlQ62R02BguwfIYmjeqsm4EQw8YY+uGlg0azLyPwlM6iHmrZqMGydBYAGxaNZkZJy4LoRePdBpQ/vR", + "UyrHVU9Dmpsi+3akXoVX4LE1TiyzWFfem/4MpxpBXuVnx+S54mknTrG/w+nRll5IS2NiAiN76TUmMNIh", + "tlIVJmgBw4Toly8+1i39cV01+FFRf+X1iy1dp9f+GU5HSVAh3fgbuN27dtopdfg0NxlBgA0Xs3sUIDxv", + "NvXfnCKrdpQSLW9p2L01iC6GOPH15mdMQEyaLQYTQBJssR56PvG28nlLPMNZkzjd/OZU7j7AuJoFmixX", + "UUrrQFYO5kLP9a+NfBBJIOkumLlmnG6TVD1uBlcXw6v3nW5ndHt1xf8a356fDwYXg4tOt/OuP7xkf/CX", + "a/732/75x+t377TaClXj9P5stl6wxa6azRaTsJclbH5a2qnymPrmaPVHCnHeCI9fGN48NLWuDgpsYiId", + "mbFl+sB9+AKn8zB8ePFFKrBsaonh7BIFsJFzHnOPoJ+pIkElizxS/XDm+CiATTyxuAe/dg46nGhQq6SY", + "evMWGptEAVuq11oWVpDO8DVD1SV8hH7ecPP2lgqa4dW7606386U/uup0O4PR6HqklynKOOnlyWr/cxDo", + "BIn4/vJ3T0lWeunBP65x/8yP0PAGKjpX3EGvCfTHEQiMXHEHGhyvqQODnWtHBGIYkDscgeAO6c946eN5", + "BwiJ0TQhsNKfxPSWp+hIbhjBO7Pmzj4/Zi9J5RYwfkRu1RB0ORsDtwI37NsDCmpN+XKTP9K2smPF5YXq", + "GncyZKp2YNb8nLbOOivCrrzGGLhQv6biQ7FsmeFBBV7FQGFf8qtQCLOr0rXulMjhSmG026vxzeB8+G7I", + "FKnh1WQwuupfUqVrMPo8GFFF63I4uJp0up2b0fXF7Tn/7fpqfPtpoJeOcqo9EGKpGLAy7xe2PY+lAUXB", + "9cfKc0EDseoh+r0jnPbuIkZEZ91OAL/Jf73qdoJkwf6BO29OT9jbU05y5TrrHMmlT2DEz7504jMrY44C", + "izbqAn4rj/zKbuRsXVr/95AAXzWd0abMnuwjTPibaha1eGJjO9Ls7n8lMKG3zRi5Gi0wSBY3doY9RnjS", + "vHdkWu9/Wdny+FiIu8Mzw55xwJGdEY+PKEx5R51aN6wM1NwsXRUhOnkyAgQyL9YyKq1eipjjLfPe1DvW", + "AkxG8B75BncQFhYh4ibUwVjMRMw6Qua7uIXgEjbRZ+An0NZdN+YOHthh8XjioUns+hMKvPBJv+2beMmq", + "QfSjeR1SmmjWsQAetF0E/6afgn9jy6B7iQLFDzVDM48cuw9jF3q2/maKdULZL7neFKocpX1V6XoPTq+M", + "x7RKePp5DTW8OEZJEefYlFhTUKkdDbowIGPFilZ4nWbgmeiZf3V0Pseq2bOJXWwVO+gaNsytGSoFSjNL", + "ZclsV4wqqeaRdCO6qkVPwFIcXSv+4QxhAmPoSXuiJmTJsM9pxALynDgdh8eEIsw+w/hIEylTQp6dh1oW", + "IVE1mUVcnNG1fATpXz9PBNcIRj5Y/lDBUnxJinEcG1eW446XXZ/S/PXJSc16C3CbVm0yXivd7Y+wwmuD", + "LXwSupjKPCb6KthKH7WgDTegoxbszJoBZxCT29iged6OLpk3KQw85l4uTALYIeF2HJ9Mx2USoH+obuTB", + "gKB7BONUtxbqoIgo5l7waiD+FPphMJMQ10rZLTrh2z0vVTrWj9059BIfKpS2biDNlgNhuh3CA37s9YQm", + "sTPZ4F8V9Hibe21jAaH0j/H5h8HFLf1RpwymM2/XOXlP3YzLq898jXfhUtyYxDbnhTxKgnP16anxEzYH", + "YNdnqQKAzRLHVor7l1KHl3TXzoii0lO7TLtvE//hAvqQwHcs8GtFx+M0bin1O36AS4ddLp0IIJ7kiIeW", + "OdNlPsPNA1yevmFNT7mD7Bn/11mTZDfybUYoFfqrU0O64SN+qbuQrUiNGxjsueEWG4/P+3Tvm0m+EvWw", + "B6lCK402vb5yPORDMd14gQLxz1Obx4BqDJmUZI999za8kiIRN8zkpl+KXXI3ZUHdqkxvVXPoM9HpY5A3", + "Qu6l7HQNQL5lCW8opZgsGutu5rrKX16SN12Zkbt5pp91qUpBX1MWVBcpgWm+OhNnbpNn0jRJW+V7LYpW", + "4cw9MG1rLgfPq0nlVcKwyqOYzN+qxlTtkzKGCxDNwxiO/ZBs2Padsyvr3aK5ORP7IX8CEz3s3bhWtENj", + "VZHSxMsSGDlxIhdWb2pQXV/rF4p8X/qE26+0dNGosFBbg17gzQwtXdXWXrCrU6pR/QHLvkpzEATQN4Ep", + "PjvI0z/9YTq488RH1z+q8BGujHZ0OQWzp684yVoWMLAwrZ5+W2PptLt53WzwdRa9F7Y7O+uaRESK7jxd", + "dBUy1J4vBEYmcacPYJgj34th3ge7VudF+CKJWc5kXUJBkXFUSByEWXKjqa+6Uyih6VsJYOD3QNxsVfFq", + "ybT0p4QfyucYG+88U37vDj3Asqd+SZgMsb/QOe7oAE7vv5OTk1eMlEkuklTJaLWpQB/Dks3kraA1R+sy", + "MEFQJ/d5qqDrLQT29MkgCnNeq8pGbCj8h3HYF9N7TS1R5rrj8zAJiB5c8z1ulYf3rE8Fhoq2+Vz8kkX4", + "i4jWSttvXg6ECTGBuKKIYI5h/Xthe7FD5sbDqXiXip1ZQ4O0jSSkbU3ixELWNFlx2qVixdx5YHXzbUqB", + "6coqQ6YE6vqxO0eP8CDlUvNngb0SMSG9Jeo7VXB9DEm8rJCiW+NH5Wq2G5aouAUpSJB41N+oTfS+D0aL", + "PANqnfJEG0N6FtdMBebXaE/foSoWIU550GI9wo+H9WCBII9Qvk7a9h7LPlZ09w7FmIwhDJrR3iVo2qth", + "cCu/QuUALMycYlZBkxptxve3gpj3JbNIjkxrCTkT6dIuNhpwL4C7q+u7L9ejjywqJP1x1J8M7i6Hn4aT", + "zEtgePX+bjL8NLi4u75ltrnxePj+ivsRTPqjCfurf/7x6vrL5eDiPXc/GF4Nxx/yngijwWT0L+6poDol", + "0KGvbyd3o8G70UD0GQ2USdS5x5fXtOXloD9OxxwOLu7e/uvudsyWInNU341ur+54yuuPg3/dqb4RhiYC", + "UK2JUMcxClKV8EOxwNFwMjzvX1aNVuXUIf6642j4xMN4FJw0cPoQf/PWVfHWE4Af9DmVs/QmlXmcRP8E", + "s1Hy6UuadNSZj2WbyvuxzSSdqtEFBBrpn2adts9SVshUrbkghL4nHnTspCLbh82nrw4J8K06a1GX5vgq", + "1omCscizOTBkQ02tP6HDWksT2oL1wnoLEAiAvyTIxdcRuU5ItU1JDDgH2AkjAj1HmCbSQfRzrJt/c+s1", + "LkwZLNdOgZnlYGmYtLS2kgaDKxv9q5GUCjl2d5tcd0vZg8w5drVr3gM1Q78XulzEs7DHibYzYo+Bz/lV", + "oWAm6kDg3QkJntZy8C1CdJdZMg0GTPX4vBefBjtPrNgNywvigBg6IIriELhzFMx41RuG4Kr5ZY5gTiQs", + "WGdFKPiSZeh5GR4W3VOJC8Wm+A4gP4mhBSjMVVoFJFeygmVg08/pA8yXan76zOIAQSB2lj1/FpOeV0f8", + "gG+SyN4xa5s4oLWhfc69bOIAIsPVBFVt9vnLLAm0AJvlwiB/EEk90Q9d4LPgsEfohxH7zGKOvcQt1JdU", + "1Dslj/f2Eng/pzWTKh+CZcUsUS1xl1WkVssSXvcuKFjU9KopP5uxxltUvWuyEXLFNoyneM1RJNObZ3ul", + "piw1UiOnnb05nAQpNzuT+J6W4X8xgrLPjktZr671LYYx73GTTH3kVpECG68i0b0K895suti/VTZ9JPZJ", + "StHrL1fMYtC/+DS86nQ7nwaf3hoSY/BhqvMQ1F/OmtzFqjCRg0Mxlq16NS6OV4zHShEgKb9YEyw1vAxG", + "d+PL60mn2xl85jaLSX/88W50e8VsItdXSuwJS1pyfv1pePX+7svg7Yfr648VuM9pUTpFEsSLish+9l34", + "q2sFNM9BQELnCcQsQ2dJveK99ZHyzZIe6PMdbCaFAR/bvEQ9/Otlf0xpop59UwqyS2BQt2HN8xYsIIGx", + "zF4gz1E+lvMLOoJHzqnjgWXXOXWeIHyg/12EAZn/uqKPTooebTYDs9iViLoJfeRqMjBzjb/qEpwWI+VN", + "NUpDA7GbZ786B1cBnHl1wgJqK1CNAkkpsCLl0eeTTrfz+VQvSrhP6A4CDo0xrNzZuUkNsYpSHc/pgGNN", + "UIa5NtOafuzVLuwcoJ+xXpK68pqUAhspVWTU3FRARP+XB8TMagdtKW4NTS9paNqiAWgr1TIbGPJXtsMb", + "uPAL83kyZ2PANyDBusRnKptwxykHYSdirR0QeI4LgiAkDmDVsJ0AZhVtSweWDjqsu4/X2qOA58UQY9Uu", + "ldOipaGjbJ6iHz4APNcdN3OA5+qQ/wsXphMHEFdEb5Z+GDjjJIrCmDjnc1aDXj/hZxije1SHXmZdozLo", + "UTSnv6I4D4OeE+YA3wCMn8LYdg7gRKKDgyEx8Nc2XrI8hCMfLHOMIPevsSErj92vBgI7n4NgBiWCjEwQ", + "wCczEhnvwqcMa1Kj1sO+gt4hR2brjioBSYGoxN96MJSSWosv3RyeTCi/DGcoWL1g5Gr8vVb9yL3DuFxj", + "VIdrmdTroNBtd0IaBMMe7pZ46LbeNFWtxnMU4UM1spaMzjs8zbdxyvDJdNv2+fR8cHkBp8ls0+Wru0If", + "xWiR+IBAnGXaYK9lbpj4njOF7IGUax8gEIXhwtgBOY1ZF9KTr4tfRtf54NLJ2rD7wSPwE0r9Wndsn8D4", + "Biz9EBg4UGQDiXib8vqA/ES1DycM6A8xfERhgnvCvViM0alKHlSemH0qz0dK4aEiF1O16UbBm5y1jjIq", + "A9kNXEA/yYxkDpIVwNkGsEL/vCSaZicy93VdDBpO/DQWq7DD2ehdOiGr84XxfeJrFUG7GJEyFmS4SMnB", + "3BgsYRzDEKdMv+WWmK6L1TflVkHmJTkeV+YA/3x6ziIhJgA/mA/SbwTGAfBFvgCjuUo0c4YXWJKiCwIn", + "hvfi8o24Qg7wA+XfHGGqnVU714Zzkdglhfl8SvEh07886zdMRpDQpliXfQObXis4uhga0mUjD3Oh9wRj", + "mJUQ3BoqnvkimMzhC63Y/mopqvCXvB0UZZiqwVSITykcTcMUCkhUFf/Xvp/w8Y6cW3qLp5PgZIq5oxZF", + "uccUH9EKO4Co0sguD1ll/th1s34ZMlDySCyGkNyRZxA0bMtFEL+y52EAr+87b/6qFXaa/m8BRm4/IfPO", + "c3eV/v2bIa9uukrnD5/6553nr8bFicGZ0dVfZ4mQAVjQfOiia6WJGIpDIvDEuk6WJipmUcvhvUNbwYAg", + "V1BhyCwxkkFETL8i9Ps3w7uPg39phH0xqbKcnkOioRYzShky9Dl0P8LloLHWpS6Jq3cPcHnkTJi3FHaY", + "0Y2EvAYUzLdy7uNwoeJCCpGjNVIwp1jVV6ZZb4FsiPLihOrICmOEcQxdkokOEjri/Unv/sxsUNKNyooU", + "x1kXoeggt1Kz5U1SCa2nw/KquJG72FtIZKjJSWHQSKVDdQZ2V0dv9iIvE1l7IBhU+bklufC2Px6eb1cq", + "MEG8B9ikcGwXmWylG8PlBZidK0lGikl1NOlH6nXXtGp7WQX2wMw2Cb+Gl36iSv5Wqmp4nwGgXnqm0AHB", + "0vlzfH3VwzBGwEf/Zk+PfGVHKym1FZMVjpEwdlxA4CyM0b/VItOaQmowqEpghQlYROKhND13udM6DOxd", + "uJr6fW6iloyZKMRhytJMm+pyK/dRORl7l80uaekoznRZmNGSUxkzTRRgtKV5+XcUzIR8u2qixAi/8xTU", + "DE5mAQFR5CO3kHtIn0E4q4tvsShNQf2Kd61SERzNvF8z8bMHNmMpCA336vLGGtLT1lN4qhcWtpGRo2YP", + "a7PWWWSUU4g/ZbR06jgJjrZ0kzVXcDGTlTmxSF3xsbShfZX/bkcYJWuG5q2ajPuPVQU23qrJuLFdITbR", + "rMnIzHgKvXqg04b2oxd9rdI6b//I6m/Z7OmeKFldhLQwpf9uYNGqsiPpuAvhC+j6IAZEZL0xOyUIzkbY", + "8bIuzi8kTuCv9ACP4nAWg8WCXZ1+uQc+hr9u2mHBqOMoyppUdZjCVsbHYZjpNqFQVGx7Aytg3dibFKzV", + "ifhNhsOMLFQ22otTN0vJXp999/Opsdg4IAQuIoPaKz4qEqxYa1yTcmon1ct9WQq8GknFst0vV/S8mE5K", + "91xH4qXDMtHYYLp5FfUCOtaoo56NtA+cUFnxPP1cVWyxPz7vdDsXg/G5Ybm83lb7NNj0aZDjbTsvg7EY", + "e8sPgxR0k6mnueykC9LLTeYD8KkiNxi7qkoLXv3GDNLmK+Yis86MV6lNQxIjiOuXT79ccJ8dYxUf2sbK", + "YMezfzGDTbOkY/Ia2iw/M2/CgVOnVvcsw7X+Updu2V6I1Izo67hCEuSWE4w1zCgmx8plEitmD9OnHitm", + "FBsPriZ3E3Ux6Rru+AlZSn92Phr0J4WSax+HNzf84/XtJcXO5G48uLpQRtafPIqMtTQ12+e6wSjgkZtN", + "Sg3ApnSUpYwtFd8ICPJXqXNWXayjWTkOjgQzU96EKCA8SrG8A4IWtbI1S82mD/5GC7hqBB5vpMn9ZrUM", + "zUHMXcWa7qyKGst7CFOhksCET7cy26qVC5pKcnq3s6p0jwUIm2IkW5qG3HOwKSIzFRJZWr/z6083l4NJ", + "KZtfRZLC/GvXamVMlMt//qDOpln3eYtpdMJwWsL+RhUq9b3QrGHKVmwgbP9gUfO0WHMLzt6TUpw8ASzc", + "OhrkA/DyGpOdG7RmC5QRk6y+rmY48bU4VNdBgbNAvo8wdMPAw3Y6bp0nbGEW55c0ZB8QiAn97df68vFW", + "6KfDy272+K/zQ65AuaB64VUvf4xgACJ0dBUGV4nvg6kP/xyzvBlpqx5aRGHMJhWu+OXGEaBXnM4MkXky", + "PXLDxfEcEHcOSc+Dj/LvYxCh48fTYwzjRxgfh4Cd0d96gRir84YZWtcMA0sW4wg8BdA7r2RHxUbOm5cZ", + "syp3d3lA/q0hBR3QnvCSBEwNTw0P1k9YvHMqO2sVqC3c9yxKY2k4dEvlsYqKalavwFAaq3xQrmt5WG0j", + "Nzi7xYNA5eV9GGAYNz/ykOjW1IHC9v3iSK1Gu6OKxMTKSJNmABE2Gnm9OQ+DezTT5hupLi67VuXnFYiv", + "EHNkDU6ufHJ5JhH5rploncJZqn1c1Zq63Hojo4E051V6znSzC0SBXVXrT54V8hW7uCEo9+ik34KvRYV+", + "u1ahagPsprTiUkBxCryAxHwhm6CFeLrfogXWgxGZG/Re+imnTCDuBfYECIzvge/rh9yZIrp2/bPtaBIN", + "BSf3aWiILHqK8I726PrZFBqNdX0Dd8VWafmBlJbVnOFUHWCtwpZc+BaO2IvcQb3Kofu1cIS85DlKqYkl", + "Qm90nIqjb2On6c6S4HU7UYxCWStF4zcuvppISV+3TdVna9x+Rev6iP/cuF0lXd/nU548qQ0LXdnfTP8M", + "IHJSlUIvDy6Mbj+j4PYyiE1DBsYC2mqwkl1Aqezw3D0Estl6iZg2cPPlaX4zrrbmYuVWHq6W4aJKfOBX", + "lTWVQO0yk0booykkrX8zZFyhIDkfWqgjgjkEHoztTnfetriJYtpaXCkzdeU6vlaJqL4ikPKBpN000Lxr", + "ioZUxskF2xaVUKuMVnSpUzoKQ6gOjQmmKrIJifJr7UAFlKWj1uS2ygef+jOq480XKt7GH/qnnS79z9nr", + "3/kfr0/POt3Op4vX1dhL41k1WWSViexjY9NeLIGpG3oW9epyIwxkJ+ZOMwsASWL4YW06pkM76XhagYlm", + "ASuu5MbQcHnF7Btjw1SWoVlgNUExgDdFlIIn/YqLoNXSyEDBexpWPPg/rFzheMCCYvgft6PLavLYC9c5", + "qdNYOsSkOrApbZQ7B74Pgyqn0AYhepUO8PLZvXAkOrEEzlK7Lx/Qyta+H1wNRkxuvh9OPty+ZW5+o+HN", + "gHno9c8/drqdy+HVoM+c7z4P/49pz7Mb7OaDsCu9VJr7dkgzZevf0fp3/Fj+Ha0LRvnhZE1D7H4/JByM", + "HbvhG3nNo7TG4i3eqdeyerPWmck7u7bln6hzL8bpa7Rqp1ROwwtIZBmFgpNvEth7JYgUDHgO6q0waiw6", + "bf8ujDXwyAcjloHDJuyHNcyUkby3wfqBDBwcvLl0MrUOHOVY7k4OJxLdErLy1ubVgfz2ejXRM1uoZ6lO", + "WQXsS726qNpRg2cXA8Y39QTzRefyIVFkXsyOgv8KjklqlGj/vSjpplXJhdbPC01stMRiI5OnKFKhVX6T", + "2JCGWvZNYr+RoU0YROi4ur3OoYQn+TIXH9jUIrGdSYDKVZHhnJ5izvDeCULiRHH4iDzodR3gxCDwwoXs", + "9IR835lCZwYDGMtrjEpdZ1vDeHM0e/tJgKvtza5JOYWzFtlUaplNFzu1vOTFj5X1JdfFyJji0n4HDPvG", + "nkRB4GUVHmM+1GpX/gUk89BrtFoB+ifeM9Xtz0PPQLUfJpMbmTvbDb2UgqWhxz7fwB3gCQfYzLmJv1oi", + "vJqEBCprzvnMUMVbWyce01LAyrTzKd26zNg16XQ7N9dj9p/bCdOSTCckD8LCVRFaWLwJ8TpMLgicCMaU", + "ro7sK+IJsxK77WqzblFKQGk+P4AxmgXQc7JOzBp0ezu8cARJ7/6W54Mp9HF1+VDWhpF5zieEi2Y78uBC", + "jo6jQ6MPMPkAQUymEJCq+3pu11g1WFbHAThz2Tt/Uz47OTvrnZ71Tl9NTl+/Ofn9zW9/HP3xxx+vXv/R", + "O3n95uTEPk0K4AxGj+wBJmDqMwPYHkK6/dPZfCrH0IVpVVJsys5C2/DoD16VLoxXIalRfi4NVcWimk9W", + "zRPXZlLCTtbLCQN1FxtAVpxXC10S0C0cBvehHfeMlA70aPJDkl2QV6lXzYcdZ+Mw120VOfSbAx4B8sEU", + "+Ygs2fGcK4SbEfkvFKI7lv+299/Jyckr6HyXnX3YFSWVn3/V5yn1Q9PZhOECRPMwhg5tJMTQikQzlmON", + "2Xy6UH7rIhnZ1Gn2m/PJ8DMvyJ3+edO/HRsCjW2iW/gepZEt/Kw0ZgQTpzc/TwpA1hvweO/bOn34dnSp", + "Gb6peszaa1Ub5agoneyVmXtlrifaddPOQhVFtnlx7ZrJqxOVVuDh5V9ijReBFMhRXpQVKmyDYJaIZyxr", + "ITe++Ij5scs7KwWqy+l19KqakK+DbyQG2gbYezAPW1ocg0hVSK8v+yzDwM2/Jh/Yo8jkXzeD8floeMPy", + "pdy+/ZfeuFMUuiWaqhW6gAtCOjSltILuKwVuXTxG2tBJgpw4zw2uqdHPSpebasOiRbJQJmkytK74eQVn", + "FI1q48Hluw/XY57q4VP/qs9TyHwZvP1wff3RuBfseC6bgNW16eOY0l8s3KS7DcrCckVEFobVlxP9O5wa", + "jij6RQeQFaf/GU51R+JONEoj5mQRQY2aDWarrzW1zQLtxa76eU64GWZ3u8oViPetZhJXeUqTyKy0mWtO", + "2DR0w8BD4omF3/JcTaqTGSTKd1Z6XuP9EcikJzwH3wwSLDxX067OjPZNtQbFbq9FGGP9MYkBgbPa7OAK", + "hJe5frystfkiUpZWKcQkXzi7mF/41Vm9+JJTF1fT1WK1aouGF7o0iCmAwwstDmXvjyjIGVLe3V6dT4bs", + "wLq4HfXfXlIl9aL/vlJA0kGkJtKIgtnsGvaS3/XqzVoxmDvWjPTXu+eK/TTmjmJM8hFWhVOSkABfR7Ep", + "jz3ApcGnSA5PydIuYlPezoGDI+iie+Rmkzi/RABj6DmPCAh/9V/1XGFERAOHM/31lsQJ1Ixf936rem6l", + "BpjTk5MToyeWdpi871RDN6hGC/o7nEoxZnuOGwo/rB3dzE/EXRsp+dzC1vMyIOSciTbpGKT6fGi9g8yl", + "Rt4uGww+UXqV3XUaqiRGh591codnA6muPArYX6uFyZ7clRWnH/tDYZQEa+RVLo/yDkE/d+6raTsyWs5J", + "MUUy1kwyls5MrexuZXcru19Kdhvm+AFFe4U35AqimY02JHBh9q803FfqOxur7Y1ZKrTqhLtrepxl2dY2", + "nkRtAwMaZHoxJW8xN4VYVLeESGXUOuopZYq9GVxd8ASxWapYTRbgfM7YNL3s2/75x+t372pPSTbtSvfm", + "vEAxE+MkL06K/jZhcKNI/hKstMHYnUMv8SuCogyd1z6OvhTzpFgKmJrNxryKutELKZeeZYvsWFWPDNcu", + "wmgkYBmXm9CRHOqcd6zTQgvNS/NnDKFNLl2Vx1synfajYC7tN8mjzbODVy12AmY69PpcZVzf5B9sOLmK", + "MOtyCKvoRwiF85heZO71ckHL0pwv75CBG+smZM732hmZHLkTj7ebnhbrV9hcMyjgTSN5YRpyscrAKX42", + "q9xzdUuPvkwDuxOvEM3RzFPMGOXpJl+2qsBQtNkiy+aeMGw2RH31YE4v9yDxyU1lliXRyJhtyeqRQNwi", + "/8T84F0YSmD9Ob6+cjjQ5bAdNoLWiUY+C77QY18Ye9wb0wINWKgdE7SAoaE4DibIfViaXHHoNweLZxW7", + "l0RFXjRgW6aDPZ4WXsqscKz0GfPkTzqUP2aUbU7iarPAJ+U92/bdonGyXOtroFyWJIzcQF/rOZ2R1Sbf", + "hprQ517sya4Qzh0qskehQlXhGELur2IsKbIA32paPDVT9k11RXjkR0LlL5OfHMIpBDGMZT4ThlF2rLCf", + "s02ZExKxa08YPiAomyO6q/wn+Xb+piNCmLO+IrUN7Z1gEi4sJ3tmEp+7RWmiB/gsTv9myMpdEWYTy/+a", + "EmLn9Ojk6ITRMQ/i7rzpvDo6PToR8dgMEyzm2hdlYme6AJn38nmetgogxk5qj6GbDmRxk86l+P6eoUEG", + "NLBZzk5OygN/gMAnc4ai1/y7GwZEJNUQ9aRp0+O/MecrnB6ANXw8iOOQSuHnkn/qVUjSdeSIo/Pmr6/d", + "DpY1XOiqs4bSp+QvAbM7h+5D5yvtz/AXQ+At6xFIm6EqDI5kg31HIVuwQ0IHuC6MiENicH+P3FqMphio", + "Renj6THwqUgJZj24AMjvsYdkfPyd/az+9szx4kOiuT1dsN+xA9JMX7S7w7rzt+nSLvRpiwFtwFwt+AiM", + "Z2KwgITpA39VOPmUZnBEnvPOG54HIRUapaV0VKHG3weyHVuvJu/XEj39pvEkTFwXYnyf+P7S4Sj1cmnS", + "Ssh77nZ+2xXl9Z0F8CkWoOewDFqeDDviYLzaOBg6KN6F8RR5HuS3j4y+OZ1UkZmk+AlrQg+rb71YqBzs", + "A+/b6WoI4yu79hJXkyWdX7fWIXE+wo9B4owe3oZcHm+EGDh2+KYVEJfGrZXJpBJbJHQSifM8Np71Yn8j", + "C9EuQQd7TgxwQFsxYCkGOLVsTwyoB2SEeiR8gAE9FeXf7DSMQl1KgxF8DB+gAwKWrJG1Ft5a6YwFMRGh", + "CW0lDTq0u42USIc3yAQJ614ddzFbnqBzBt2PTdS4CVUL0qEbOxE7J8k4+62KktMtz1Gw64eJd6ze0M0a", + "dClTnLz2sEEcFGACAheWiPicfpbuJWbFevu4ZYA4SZAFXOwLgdVo7RzB6nu92PpPygvbt54cohdG3NlF", + "nGjKfnNz+PF39t/nqv2mUoq1OiptKLOK842slUQ8SbRJOeEpHHcphDa32SK1Us3hzWuoPAqxxrHBdqyV", + "bTkSVzCTkTdHcYVU4/Tz1Uzhx3VijW1LKtVqaP4iFWA/O91fMBJuaX+/aH8BVz7Djaf37g5ukXGtCU2l", + "R+KBHOSbOMLpGMfMTs93CRt3/BJhegHynVxr0wbT1sN8w63tNp1L7LgyZcPNlxlwcqvbJ0JIt55tRGET", + "yvuf2+QwQCSk0vz4O+f45+MoDqfQfLmUb58OyFWeYHZdXrkilwvBzPDp1DchJqMkuGHz2tumTIdeKrl2", + "fOpVEBT8Bt1E2lYYfo92eipchYRVIAhj9G+epV7kNOLB1zxKs2TmJAD50HO43d5h2+O8E/J8mG2r/uDI", + "kRn2gftw/J39x8KK74xpQ6XASp5y2FeRHMreaJ8b00g8DMS9tM7ncbJPqs3pbsC4DTIS5hO/3s3EPOcY", + "S90IfD98otPrXgSKVCtFL/u9SsXiRJfnmAAff8cBtuKWq7Eq9cv8EuAGbJIfzMwo4uTeOzYpIKNllD1k", + "lBLBpqxyNa5klABr2EQqLoq1Sa+60HnllbjEIo3fxl5M/+iaDQG8MNNKlgAFhrPXr3NAnG5CB4rikP4D", + "eu0ZtkesabpEsvoNDogiSe3lY423KfAjAVMfHntgho/T1O/GSyNmt0bWziFzQJwp9MNgpmYVSNOMg1n5", + "Svn59AKwcrMTUUK93lwmE3xnCVp4ym3GMv8kMF5mPOOB2R3yqo+5bUWIWMmdArwvdfGxpt6N1cC/ALNz", + "EfOlzz5WIYfolPL1j836c1sJu53XuxJ+9BaKFpEPFzAgJd2AGS8kHaRP5wA/aCUMa3j8nf6n5nmJV7qY", + "LjnfFAUIncDS1M7GMR76FNAdH/mAELiIiMjLYhAKolFHhaUUC7VNO36hpkcj0xvD6s/On7/xu8/2Z52o", + "9feppnAfJjxJ056IiIyfSyLCfGcgNiLk2A9ndbqKH84cHwVQZj4ScBQlymU4u0QBr8dyiFJFZHkioUjL", + "O10aJAtPw6iFBgWEFZUsB10akufGRKTGDp0ZJBTVDMuGmTHilkfNzBWpGwz3prSqgNXUSUCQv4Gp+w6V", + "dz0CvxEHQxC7c4fNpNR4rlg/66AT6dVrZRQMH6H/C/6VToQC1088aNpf2hJ3tNputcCXLEAHsFVuPZnc", + "hgLGolTMlMc+302Xd2mnHJRWwJVy6lgdslbbswdHriqEGijEIoq1fTfPa6Wp5FeOnctwtv6pQ/+/l4UO", + "m19XlUJtxoMnrcP2Axw9+AFFJua/v8dwI+fOVk+67avU2V6v4CDTXntbtTon43QSZrMqNomBWxl96FxP", + "oO+wZgocohil4eI+YYO2wu7HFXbXBPrjCAQrSLqMnlpp99LSrnSRVzZnE5KGtVAeA13oH3twmszMz4GD", + "R+AnrLKccz64dOC3KIaYhe+DGUABzio1ikrkHiDgSCOLzqF/waY6FOelzcfOfT49H1wyJNSEyjFMYiqH", + "WGVyyqZ65O80Yk4FXyZ4rRE1UFCPp1lDe4NS3/ynyazEYgrPnw8uzSxvxesWNyj+1JhXctL66UV+bnaL", + "2kdvgB9JudDYzuSz4QNcYsUkY5yWtmtuyGJkIHJ71JmwzsMAIw/GksTYc3boshw/ngPuCRSFboSBcJtm", + "zWpYpvA+jGEtMJsydL7jW0PCHDQgZlUvQxcxCfqEyFx99S8WotfAlyWwMezslh/k7deVKxjiLABx54g5", + "ObgwJgAFWZKQqnWmeT/hSiZZ9gZnyBtatbh0S8Qqp0t63KHY4Y4ROohFatAX3Zbp0slycWfRKKxsY2oB", + "MVhvy6nKtQvRlI2R0zzAZY/XfosAirHziweZ4KPct3SA8z9v/ufXotiqdLeyM6FjN4yglTzkLW3XxVqv", + "B+92rWH2lrDW1l1n6055wzJArIGCdsyOYUstjZ/tVpraR7g8FGVt6wGTEhdNGYGhu2UGHTM4QnvcAkN8", + "fzztNQiRZ15MBOs9mZpEy++xB7MJJompA2VOsT/tAbWRIGbcJIA5pRwrzuQ6js0xJVrWnlFcJW3NCftq", + "TijVebdQoGtvn5VTlK6I7DLO5zxav6hNs7sCTqYYEscFgYdYRitJ1xu9PVSt2LnF0GNsxGEh9HpchgcQ", + "aXNlXkKG8jw7vXgorN1AsEsR00r2vLYl8ZLJdo7fKl2ra3jbOWdFzRzgBPBJDGwUzbztz/14w1DA0WHz", + "gMPeb1JSdlgJOW7V3+WbjSCPOtYTBe4UgNvn4F09B18ZXoBT/kx5057n7bU4dsHif9sEUoM6SdE4RfB+", + "qXGCWxHLAOHJtegvWykmDvO2ZSkaZNR4KxZeUizYsn5XIUx69FcEfaUKvNlgwmc7ZItJys8/ORfPQtIe", + "7kaLyQpnbJHRKhOS1x+bB55aIXdspum8X5LhtnEF4Ju08hXgBdKcW8sHmdm8lQ+Hd8pbKPssimaRlcWs", + "UAuEZJQpB5w4CRzRszpDOveguESYcC8KWYXzUGVaOeZSQUONf5IFoGuHYdZDsykHpaJtlll/A49765in", + "T6snohfydKFw8/qYjJT/F1aTmxiAFvU0afs72fqOtd4qsWXJVvgbH3OVSmt8Z5H+hrQmvCEKZne8WuiO", + "IO9rHIgeeo/Cp8fikSDzJLpbVLoSvawRmwq2URJIidY8Q4MqRdtsKvuTKoHtzSI9qOxiLOxP3ChEAbE8", + "dxcoSAik13H5VwzBgxc+BelR3OAYfg/JDZ380A9hduBJ32AlbEYYrDvdDvwG6BZ33nTOTs5Oeyf0f5OT", + "kzfsf//XIHdE9/49v4ls4oBkkKaewyqoIYVvDWDvUYDwHHpv2eDNwd2+bMyR2grSkfFJKx/3VD7md2fj", + "UhIfuyBwoW+OQjtn39PMVzp5x5v83A+UDAVMVakpxcazCYaOK5G20ygyNqkPPZ6RsPZlUjZv09G1cfkl", + "GVWQDBuXTDGMfLCsKiNHv1dKJt7kp5ZMHAVNJFMskbZLycTBtBVMsWjdyqVWLpXkUkEubFAuiSTDNt63", + "spBDnfetqBPRut/us/stJxeHDmsXv8baX9HmqwRDCpoYp6PY2lsl0VkDKjpUQFo9yYt7uKrs08DFNWXk", + "9i0+7+OaIiaTmwLFa3u5msrlpJvY+rkKP1eBjyav3JIpX8jTVdJIE1fXfSyz8HP7upZrKFjwfgO1ibm7", + "in/Y+bvWyowD93ilk8u3R8nC9b6vGVbMwO7WDm3L/9KfteX9vXB1qWXvrkpuNS6tkn6FT6tQDw18e8hu", + "rQUF+EfjUemt2vKowV215piEAT0FezEgsMduoHRzxd5bclmdP2vtsXjgHq3b5bDteaf+uIq7dFFtBcMe", + "Ke4aebD6ya6/wd+EmOX3QIEbLlAwS+l1ATEGs4oTfgRdiB5bGdREBgWJ75coP1g6EVj6IfAcFDggWDpi", + "td0Ogd/IceQDVKC04pQ7kSEWmUlzeLoHPoatcmGocMgZT8Nuq3K4zT1d+Az34iSoe+PIZw2sfeXIsgS2", + "Lx37n7cUi0yOVm8dO8v6yPzwQewjiFmua2gF3haDAnxAmoCysdpMe+P4bZmr5kCiFSgQaRydTYYdGG/Z", + "xf/LHJI5FwCiFpZz0X+P6ekVBv5S/T2tUKoTSIG/vJMNahWVaRj6EAQWMR25crUWOHuh8A5NUV1jnIdF", + "Zt8Xi/dw7n0wY0ftk6CLMGYOGCoZpPdLEHhOmBD6p1AfMdUfaQOpCx45F/AeJD7Pd/8/lB7+x0H3ThJg", + "yI5x3fLFTHdy0E4lCe2scmfTF+DWaWjfKvzkNEpV0ZW/j+jva75EqRrusYdw5INlj7lL1Oi7oi0dVrhX", + "hPcVSnC1DnzBB2NuFwetDyuiFafvWDmkiHhJgT6BOrMioMjSFylsvmUTvJYEWtHViq6mokvySY/ySbXk", + "yvEo0x70Cf+z9HYVkmsgBht6hyu42ntue8/9Se65OzvOMrnQnmY/0mmWOz12crKJ67U57GfCG0iv0vyF", + "veLoat1LTwXqFKTUPFXnSIGEwn9z12/UitYMCUA+buZnqlJI+95UdPssMNAGGDzPz8znU/mlppREnuRA", + "4DFnsvT8J2F6lRTFkv674zGi+O+OExkepDP6sXQ7y8HAbZsz1tPwCqws72BzGa7AZe0pvseneDH8zZKh", + "uyWCXoHFj0XJuCpOJzzLF0mY4SjP90e1XDyWNelW5GV1ekVd/zFZW71+tiy9p05e52Hiezyell4kdZrL", + "HuUmyXFVWiDyRWQNS/ZkUWKXheXyIHduqbe/OlAGYnV8rI1eP09Fmkysag0gP65EXamqYytUWz2pKLsI", + "WqBgVq8tiXaNpdd7SCZiioO9+2hlkAcjMucZS3hWM8edI9+Locl1g3VoKP22L0j45rSS5OAlSRV/blq8", + "wEjIFPnn8zGI3Tl6hHVakGglwKTdtSJkTGAk3HX7cmAL8SHHM1pPJbyt6+7qGtk2ZZLYd7HnVlIpn1Sy", + "rQu6+3xMKdcVcjKVhVSO/RXml/KJbj+VTVWiKWXheplkcy8Tpfvt5dFA1lhtpdFPIo3s71qtLDocWaQw", + "/vYlkR/O6jyl/HDm+Cgo6UZlc/RlOLtEAbS1BrVi6GXjmXz4CH0rlyHeMjdzFTNIOqC93iHoe8YMcpAe", + "vA6bTYGjopgJ69AUkDHvpQ0lASxQIIy9qvWzz2+XfC0NJ79W+xrwwKf3UAxdEe1eAcWF0mwVSLL+2z2k", + "VGnQFtBfNwVdKoWVs+AynDU/BoSjUUVqc+YBgYUnkcFxf8J+PlcdXzbtmMMH5xPVJenlrkkv44rDIWzk", + "fCOQ+mPT+ApeNymxpdlphT9Nkch1FJ26ztWajLlrjHhhryTwpgmZ0sAOMYPxyWc33nIvS/EyZVJL7bu9", + "bXBi9ELILxrwGz+BS4U0bJktl9G0OgdTwGdDwayarw4nE9OWvE45ApocblFMEUkQj8t4gcKd7Tm3/jkn", + "+GQF1qs4746BTwkjmPXgAiC/N4vDJKp8OKXKnbwFCvJiYzhsAEcMUGTdPm0yoC3e0waHEum0/ZNQh5iG", + "JaeMm9DyTv41sYJaG51j1lef8lx1jPHTh1SoN7cCbuzOuhLKG13tTrfL3iucgBoaavlae/fTcttmT8lj", + "DAmpcy3CbPdkF0d2qc5moJALCmZj0edAkvru6JhUELPGGanuSctKmmudBk0b46MI9Uj4AGuS4Tn9m6HD", + "21VzTT9CE9qs1SfxMfMruhkyfGCL1JE6PpH+Ua0Nvag8UorkqFWYIf1xnVIuQUbtdsTe6ogMAZLWFbVw", + "myaM4qQtf204bDZjpoYMVnXgWHhL8epyOZcpU9rVzGmmTbe61+4JD3Bp5ZxA2zVPP8PI4CNc2uQ1yWBK", + "3ZeHF9g2HyaXFY0BlC7Rw4sVQcxi0NZI5WMD4SgJeBylMHy9iKsH28+XcfRgU++Bm4cKh+rkUUEsWQYh", + "uHQegZ9AfR4h+A0sIh9Skf0Al6dvWNPTTpf+64z/64yK9+p8Q582m24oWwZPXJpmHKqmc9Z4ePiZhlaK", + "tGu9awKzz6WitDDkrm9CZuMadJD2CsAQwHBRYxYWiYlfxL2HU0ITmy/kPX527+qz/9zNrCPBn0I9hd9c", + "CD1oKOfI96YBn9dfTI6nif9gdqd7m/iijhHEmUzAlUKB9vmJBQNdfkPhgF9SOuDm4qGNvtgz+cDYVBUS", + "eMNSwgWBC/0Kt1v2nRsylMTZORXXJDW4Wwkf4WdWKBgC7BUKcWGIYeSD5cbFRuawRf/1lF2Whzw58baK", + "eMgfwunf0LXQXBjSYJajpBVSeyukRoxStyOfmBnN0sbKbXMWdtaPcNk+62XGxpVu6wzZ7Y1dd2N3hO13", + "k3wgTgPjOc15EDc7mkfyiPlZj2aOgH05mjdjVuPAtVr9T3pgfmf/7T0hMu/JT8y6XRt+BAjgh2dQaSC8", + "AAS8h+QLIvOJZPta+SHZRy8+SiDv+u3yhz/l6aatko6BUUV7yud92RTMWPNuV0Pk1fyMgkdEYNOACdlL", + "7wQ6ZF9b3Vf6fir4WMnrU2K79fXUhUNktLilGAg+QSWtt89ZStQDR4ldsAPH7YtGOHBwVwlsEITxs8f2", + "np3tSOsFxO6dq8i3OrkAAzD1YS8GBPbYmJQ9BK+tohcLKSR/6PF/P3MR40MCy8Lmgv2OUzOSjaDhfQ7W", + "ey/P9dWw9VJ0HPrJXytbOIXss2zJsRknwoxcTbpofh9rI+ibccLhRNEfCidsN9B/Na3gxUL9LTmXw3cw", + "nCtC8BtzbtXJt4CLKWO+RjdI2UvP4p/Y1/YGKalRwcdKN0iJ7fYGqbtBZrS4mSBBMd7xd/6HhRLoAAGE", + "cx+Hi7ogW04NP4YqKJZtgo1/3inv/rYV3l1FB/w5uHaPctVeGVLTpkya25gG8qIrCdkijVRpErMI+DF0", + "4L0QAdtVfvl22Sm/Ah17kvLKUnpp9GCxb63wemHhZZQrKwivKq0nisMFJHOY4N6C6qBuffmirIsjuqQ+", + "eHWZKW/Srp/EZD/ERYHAb+Q48gEqUEVxpCZ3gDKWW6Z8aaakHKDZl03dQP5JYAKt2ZC1bsyB/0V7HRDz", + "HXZk8yEFq27fHpKjvdUyWDiPMMYoDFqZuE8yMd2dskSUnLOqTMye+mxcveP0sbHO13sECLykDdu8Gvtc", + "nXYTORhqMbnNTAspne1BtoUiLLsqq5HntQbBBAo7t36GBSu4iptM3DJvi0v+66oSV/ToRaGP3GV9yknZ", + "weEdbBJOSlfoG9ajTTd5rEPLao9Ghd1oH492nrUV+8B9qE40OaZNnCc4nYfhQ/k5lX3+wr+2z6k8x6SK", + "kya3hwKq94kddlTx+DYACZmHMfo39PjEr3cz8SdI5qHHKnoA3w+f9NWW+QYxPZCzgHqesY9rMeIxJiAm", + "RnYc06/8HLvuJ2TusMtKkSFvsXy2YQBdU4SynofIma9OzjR4ULmHoUwcKzmszCHwhNeIH3KCqbF4sg2H", + "bhIjsmT4ccPwAUE6KCuK9FWlB4bS/IySEOgOrEwHdXl/x1fjIgEWBHKAWzks5PDVeKiiqoEkLmK5lcV7", + "J4vLjJBK4qvxGumGCwPrGKyNxmAIyPNXZZbhzdFsflLrqIrirrYMvUcMbeQ8S46uPFFFnc7eLp6sROnw", + "Q3u52r65QIeYZjaDtJ51bmfaR5V9eFRJ92bTz8y6quqVrJsVUHemS85QhdObE+KB2PG6+1rZfZsSQ2zR", + "ivKhlQg7K4Wq0uIT4PVQ60SEeqjTn+hGr1plu1pO1OYE7BMCF5FIbsnaKuLDJDgOLRlgK0GqXOIRZr7S", + "QoRwIvD374Lwwo94dYyyK4aOIe1YkTuMJVm05WHWvGXhfcxmFieB2Koaj3YURAnzh+CPu7rlPu+FptLm", + "MquQL2zDX0KgZGuqtAXwZsJZoE64vIdkzIdtRcvLaQfNsvQaLA1iuPZCsc8XCrlLW5EaBOCHHiaA1BgM", + "AX5g1aCEpbDGSjgB+GHMBrUXEcOLH9E2mCKiAYdqcd3y6B6YAU1ssIv0SMJrpvcUxg9VySIyB2yjS1Pr", + "zZQFk3BUfGFIpQipqupJkZEGvPCOjtyO9rlt397PFfJfPYmhGMTEQj/9O3mOfzg2dlSMVzOz1ygFodza", + "lnP376FcZbyVDktGFdUPafSE5MK72ks+Oxt++sMyw0Rb83ojGaql9pCP0Vvdu1IimhuCmteiUKv/akpS", + "KCV728IUSmEKBS+4xqCbq6/8cmUqdHBbl7NXbL05gmkvqXtZviK/R+Vw4GpTUhOB8139Z50fS44Tak9g", + "QaaH7NZSYH09aCoGD1hNENu1amaB1s3FHNeff0Gqj+nv5mlqdX4+Zo+RtY9J/MmSM7QK9FENXw/Z6C1z", + "vzxzZ1lMbpQilBzGdd6d8jhi292atXdk1v6i4j6wyR+SbVJTlWFzEgfPQQS3pEeM2ditvDkYZYJvWKtR", + "/EAaRRq7InyGKiNDRaV2xuK+n76PY42uUcX6LHCSu7IMZGG/VgZsHMBLgIkzvGAJ6+fQ8YHcQVOaIoDJ", + "0DPmKXp1pstTtAMf2yYFPUtl+VqTyP751qwgS+wdb+xkIbZ6mWAt7TSanzJxmgfvQeKTzpuTbk5U7CKF", + "Wjr361UmH/NMatOlwybQTyo+mfM57ELtah97Nq9vbTIlYzpmbTDQuYxrmALizkuPPVUa0+EEA23Ly0F5", + "J+HIsHXbF9Ek5aeSTT/2RIql5nuq9I2SYOjhXOrZtRBczrfb0CAkIpDa16Oa9GicbHbxcoOP3TgM6jUS", + "2sr5O5xmQJEYzWa17hPncRj81GrKweR3TTcWeXTaGSSpSnxUk8bbdHHbwl2XztwUvKs6VUo7JaP4JtPR", + "Ds2nOswM5RU5c6dL517k5d1Y6l5VimD79L3T5fYy+CpKwY5z+OaQsYaG3h67Gi29dM5tSV2nh+7xd/qf", + "nvzVrsxd+SC2fvighHPgRe/S1ZvAymF092XvLOvTaTexzQ9crBenR1Ozt4o8QXx97lY9Jq7JXIfsnrTH", + "nLWlo7M9Ng/BsN/osN6IfKgrL8lmTWe0Fg4HXmtyv+TDtqpNqgJiwg0cVrY+SgW8hKONba9OVVCLQbaq", + "QrUcEGy5DVFgp8qz48D2QU99Zax3U2oNZvtsMGOPyA2sZaz9Dk1l+2jHi0BMkWZwXSmAxRt/UR8zdgSf", + "JkWMFjbhJLJduPra+CyWiCDB0Kreomy7inVrzPoKO5MNcA8o8KygYg0bg/QRBV49NAdvTCVoAR1wTwEt", + "OU8/ASxjmdUldM5Ozk57J/R/k5OTN+x//9dorGbd+3QCPfHSY7VHoejYViOnEE/hfRjDbYL8ls2wSZgr", + "sHyPAoTnq8Ms++8Uz5sCeqOY3t7jQNkS/9M+DRR1x9bCsRV36e28CTAPaZv8/cARoNGDLs/+akJ/y0CI", + "Q65A3arhrRq+ezW81S1b3fJFQqDwmhXbmQBqK4vUn+9bqJ6enfMUVC/x6fFYYzVMW65iPxzLzq0VcZ+t", + "iNu7F6UEcFCeU60y1SpTB6NMZcvIRPVGbLMpSFYMnlppNTBvNUayJGFaq8NmtRKDBrBdveR4mvgPvcwT", + "UR9R9DbxH4RT24YUFTri4fgnbskPocxTGVpsw46m9Vuz2zoilWsyJ55TSSxO27USQkqIt1b7vHVJwd1V", + "aiQFb+T8EkPZ+9cNio3Dca7aqdiQaTobiA2xT/srNuSaasSGWEcrNgxio3aftyk2vqd/9ko5I2sjIPQg", + "NxQaBx4HocGBsZqRFtV7Gxqh393W4bEYG2HAUzOPRwNt1ERJbIQBD7pC8UFx3zYP5Pauf+gxFNuWI9XR", + "FLnrwIYky4EHWuy9cNlW7EVJujSoj5qRUTnv48teWWolpBrs8VMqPwdQ/e226rK0KVlpd4lKU2g+Z5lb", + "qspYOcAJ4JM5f4t9+hYRD3U4Ra/qM4lU58ysBG1HopFje9WwNFE52rj5O5WNzYJv1VpdZvhbybh7ybh3", + "hU6EoKui8u2kzlJkcc6pRy+PpW4gJLK9hqtTjFopvEspLHdgBc20Qq3bc8VUlcCtYtqKX5P4FQpJnU68", + "cZHLq+f13DAJSE28BGsjc5HLso/gESAfTH3IpK8ibvT2hfeQ8Op8+JzNePCity5l/IGXjMht1opmSk4q", + "nHzaF0SDw3QOSasVksizf4JhjI/dJI5hNWdjfjvgDR3arcS9txjG7yE5F4Ntke7oTA3pjEHcFiB++QLE", + "0E1iRJZMjLth+IBgP6Gy66+vVFQVkg7lyU2SO9t+DRnPEJkn02MX+P4UuA9Gcj4PF5EPCeQ0fU3nd7Tn", + "EZ2I26Pes6GvKS7P5fAFAn91clbz9uqKeb3yvHMIPHa4fe/4Id+M/D4UxfpzAZk53MkF5uewRB8mIDaL", + "gjH9uhriWNfmWGPwbB9nDLqGCAvDmQ+3Q29s6B+c3jj6NkxvGeJ+OHpDwSMisLp2E2bRTFIb5h2Y0m11", + "fNMRJqzvUMy1xVNcncjKmd1HWG5MfoGtvmh9rLKaPAXsZZQ30dwQc7R3DFwXRsRseeuz7zi1sIlJStSm", + "bj7v09mOPYkPzidSDEkGA1AF9fGV6+iv9ZhKyYtju7T39vQVQ1bdoqKSPv3ejL54n8626tLTwTdAX3zl", + "LX1V0hfH9gr05YczFJjJ6jKcYQcFDmBn41GFgnHJBtqScwY9gun49YS0u3u0H85m0HNQ0F6fX/j63O38", + "dna2q3VHcUhpgBltBwFBZOn0nEfgI49NRjdFNEHBzIFyJLPCywhbf5Xvdr71YECn6sWAwB6zgVMdmr/V", + "6Jg5TEgNN4cJsWPnMHl5Y5VgsnDPCnW3RqoabZpRj619agEXUxjjOYoa3OGUTnb3OH4Gfsq6iaQUWyVw", + "/aTNL3QqitpL3SqXOhWD9SQZAYyfwrjClSLNxU47OLJ9lUi9kWNuT0k6n4Nglk60T9qSyyDzUkS14rxV", + "mpopTdWszik/z4xr61MxnFFJHFddu3kLXKlSpZ5S2+J7CcY+cbxEXvvQ2DL9Zm5Kkso3c1nCPnAftvJI", + "NaYj7/EbVY0kbfho9QhjLEAwuj/RNYh20gUKw/hRo6UPg/vwPSSfxaAbrUmsQJplaDw9Ojk60eWAVDyP", + "/kq7frUoNzypWGzB27KC2L9AJ4YkiYMc8go3HSpmkyCg/JNO8a0nh+yFEU85VWaBJzidh+FDTziiHX8X", + "P1iEv9OjTrQuO6rx3+0j28VAZkewdKId+4FZhopL+NqD7eWNE8XwdJVMjd5fosVXK+Y4Fni2MVPIpsKv", + "voZjhOKGbRNl7i3fbMZ/kkPP3ScFaihmqjKuUKykdUAEdtLtatlzj9iTWWVKW9SUR1PeZH8813hf81Za", + "x2rmnGnFc9zJtMpnWXPGH47HcmPfUbHi1h5ZckouBXzJC4rZB5mp1fWVHysJ2T7twF7Q8rai+HPnhums", + "EBhIJMp2FwdlyWtqUH7LaYaai+swW+E0KQb3WCUCa1aDtcG9aC8jZJok0UoBbAP0XjhzhCBWhWJWjI/p", + "1mlY9pzQQOX6GQLFVgwOa3nrpXlLjUJbh7Fs1D577mqmB+4Fg21eF8wjwzZWXuQkzXHZrpVDK4lQVA9b", + "eWBUENdjzho10apcHt2kfF28lPEe05cO40nZoDzePvCzpkQFLzCxgfrBq1cP1gM2i8MkYnU/MhDkRhlB", + "YZ0+wmWnNg3IloXEmrW45KNSW45rD7WJlep/NRJcMjWR0blFZtVomixopRxBeym5Jhp2OXKG98y6jRNK", + "HdDrMq7yAYGYpDyFsHMPiTuHnqk6VCb491yREmSwYuKhF0s3pMDbKM9Qm12ozS60hexCjUSzkA3Y4lUr", + "d5JbiWXhW3NAJpgfQS5vWcpJh6n1VMFW3u2VCpiR4qoqYNHxbwpBDOPU8a+rdQVknmRcHiSx33nT6Tx/", + "ff5/AQAA///yYOzZDkEDAA==", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/api/v1/server/oas/transformers/v1/cel.go b/api/v1/server/oas/transformers/v1/cel.go index 9ace017860..0a7c20443b 100644 --- a/api/v1/server/oas/transformers/v1/cel.go +++ b/api/v1/server/oas/transformers/v1/cel.go @@ -9,9 +9,9 @@ func ToV1CELDebugResponse(success bool, output *bool, err *string) gen.V1CELDebu } if success { - response.Status = gen.V1CELDebugResponseStatusSUCCESS + response.Status = gen.SUCCESS } else { - response.Status = gen.V1CELDebugResponseStatusERROR + response.Status = gen.ERROR } return response diff --git a/api/v1/server/oas/transformers/v1/trace.go b/api/v1/server/oas/transformers/v1/trace.go new file mode 100644 index 0000000000..d139658aed --- /dev/null +++ b/api/v1/server/oas/transformers/v1/trace.go @@ -0,0 +1,86 @@ +package transformers + +import ( + "encoding/json" + "math" + + "github.com/hatchet-dev/hatchet/api/v1/server/oas/gen" + "github.com/hatchet-dev/hatchet/pkg/repository/sqlcv1" +) + +func ToV1OtelSpanList(spans []*sqlcv1.ListSpansByTaskExternalIDRow, limit, offset, total int64) gen.OtelSpanList { + apiSpans := ToV1OtelSpan(spans) + + numPages := int64(math.Ceil(float64(total) / float64(limit))) + currentPage := (offset / limit) + 1 + + var nextPage int64 + if total < offset+limit { + nextPage = currentPage + } else { + nextPage = currentPage + 1 + } + + return gen.OtelSpanList{ + Rows: &apiSpans, + Pagination: &gen.PaginationResponse{ + CurrentPage: ¤tPage, + NextPage: &nextPage, + NumPages: &numPages, + }, + } +} + +func ToV1OtelSpan(spans []*sqlcv1.ListSpansByTaskExternalIDRow) []gen.OtelSpan { + result := make([]gen.OtelSpan, len(spans)) + + for i, s := range spans { + resourceAttrs := jsonbToStringMap(s.ResourceAttributes) + spanAttrs := jsonbToStringMap(s.SpanAttributes) + + result[i] = gen.OtelSpan{ + TraceId: s.TraceID, + SpanId: s.SpanID, + ParentSpanId: &s.ParentSpanID, + SpanName: s.SpanName, + SpanKind: gen.OtelSpanKind(s.SpanKind), + ServiceName: s.ServiceName, + StatusCode: gen.OtelStatusCode(s.StatusCode), + StatusMessage: &s.StatusMessage, + Duration: s.DurationNs, + CreatedAt: s.StartTime.Time, + ResourceAttributes: &resourceAttrs, + SpanAttributes: &spanAttrs, + ScopeName: &s.ScopeName, + ScopeVersion: &s.ScopeVersion, + } + } + + return result +} + +func jsonbToStringMap(data []byte) map[string]string { + if len(data) == 0 { + return nil + } + + var raw map[string]any + if err := json.Unmarshal(data, &raw); err != nil { + return nil + } + + result := make(map[string]string, len(raw)) + for k, v := range raw { + switch val := v.(type) { + case string: + result[k] = val + default: + b, err := json.Marshal(val) + if err == nil { + result[k] = string(b) + } + } + } + + return result +} diff --git a/api/v1/server/rbac/rbac.yaml b/api/v1/server/rbac/rbac.yaml index 7b129dff07..4dd3070ed8 100644 --- a/api/v1/server/rbac/rbac.yaml +++ b/api/v1/server/rbac/rbac.yaml @@ -140,3 +140,4 @@ roles: - WorkflowCronDelete - WorkflowCronGet - WorkflowCronUpdate + - V1TaskGetTrace diff --git a/cmd/hatchet-migrate/migrate/migrations/20260310150948_v1_0_84.sql b/cmd/hatchet-migrate/migrate/migrations/20260310150948_v1_0_84.sql new file mode 100644 index 0000000000..8bb869631c --- /dev/null +++ b/cmd/hatchet-migrate/migrate/migrations/20260310150948_v1_0_84.sql @@ -0,0 +1,40 @@ +-- +goose Up + +CREATE TYPE v1_otel_span_kind AS ENUM ('UNSPECIFIED', 'INTERNAL', 'SERVER', 'CLIENT', 'PRODUCER', 'CONSUMER'); +CREATE TYPE v1_otel_status_code AS ENUM ('UNSET', 'OK', 'ERROR'); + +CREATE TABLE v1_otel_traces ( + id BIGINT GENERATED ALWAYS AS IDENTITY, + tenant_id UUID NOT NULL, + trace_id TEXT NOT NULL, + span_id TEXT NOT NULL, + parent_span_id TEXT NOT NULL DEFAULT '', + span_name TEXT NOT NULL, + span_kind v1_otel_span_kind NOT NULL DEFAULT 'INTERNAL', + service_name TEXT NOT NULL DEFAULT 'unknown', + status_code v1_otel_status_code NOT NULL DEFAULT 'UNSET', + status_message TEXT NOT NULL DEFAULT '', + duration_ns BIGINT NOT NULL DEFAULT 0, + resource_attributes JSONB NOT NULL DEFAULT '{}', + span_attributes JSONB NOT NULL DEFAULT '{}', + scope_name TEXT NOT NULL DEFAULT '', + scope_version TEXT NOT NULL DEFAULT '', + task_run_external_id UUID, + workflow_run_external_id UUID, + start_time TIMESTAMPTZ NOT NULL, + PRIMARY KEY (id, start_time) +) PARTITION BY RANGE (start_time); + +CREATE INDEX idx_v1_otel_traces_task_lookup + ON v1_otel_traces (tenant_id, task_run_external_id) + WHERE task_run_external_id IS NOT NULL; + +CREATE INDEX idx_v1_otel_traces_trace + ON v1_otel_traces (tenant_id, trace_id, start_time); + +SELECT create_v1_range_partition('v1_otel_traces'::text, CURRENT_DATE::date); + +-- +goose Down +DROP TABLE IF EXISTS v1_otel_traces; +DROP TYPE IF EXISTS v1_otel_status_code; +DROP TYPE IF EXISTS v1_otel_span_kind; diff --git a/examples/go/opentelemetry/main.go b/examples/go/opentelemetry/main.go new file mode 100644 index 0000000000..4a9920aaf4 --- /dev/null +++ b/examples/go/opentelemetry/main.go @@ -0,0 +1,133 @@ +package main + +import ( + "context" + "fmt" + "log" + "math/rand" + "time" + + "go.opentelemetry.io/otel" + + "github.com/hatchet-dev/hatchet/pkg/cmdutils" + hatchet "github.com/hatchet-dev/hatchet/sdks/go" + hatchetotel "github.com/hatchet-dev/hatchet/sdks/go/opentelemetry" +) + +type ParentInput struct { + Name string `json:"name"` +} + +type ParentOutput struct { + ChildResult string `json:"child_result"` +} + +type ChildInput struct { + Greeting string `json:"greeting"` +} + +type ChildOutput struct { + Message string `json:"message"` +} + +func main() { + client, err := hatchet.NewClient() + if err != nil { + log.Fatalf("failed to create client: %v", err) + } + + instrumentor, err := hatchetotel.NewInstrumentor( + hatchetotel.EnableHatchetCollector(), + ) + if err != nil { + log.Fatalf("failed to create instrumentor: %v", err) + } + + tracer := otel.Tracer("otel-example") + + var generateSpanTree func(ctx context.Context, count *int, limit int, depth int, prefix string) + generateSpanTree = func(ctx context.Context, count *int, limit int, depth int, prefix string) { + numChildren := 1 + rand.Intn(5) + for i := range numChildren { + if *count >= limit { + return + } + name := fmt.Sprintf("%s.%d", prefix, i) + childCtx, span := tracer.Start(ctx, name) + time.Sleep(time.Duration(1+rand.Intn(3)) * time.Millisecond) + *count++ + + if *count < limit && depth < 8 && rand.Float64() > float64(depth)*0.12 { + generateSpanTree(childCtx, count, limit, depth+1, name) + } + + span.End() + } + } + + childTask := client.NewStandaloneTask( + "otel-child-task", + func(ctx hatchet.Context, input ChildInput) (ChildOutput, error) { + target := 200 + rand.Intn(101) + count := 0 + round := 0 + for count < target { + generateSpanTree(ctx.GetContext(), &count, target, 0, fmt.Sprintf("child.r%d", round)) + round++ + } + + return ChildOutput{ + Message: fmt.Sprintf("Hello from child: %s (generated %d spans)", input.Greeting, count), + }, nil + }, + ) + + parentTask := client.NewStandaloneTask( + "otel-parent-task", + func(ctx hatchet.Context, input ParentInput) (ParentOutput, error) { + _, span := tracer.Start(ctx.GetContext(), "parent.prepare") + time.Sleep(30 * time.Millisecond) + span.End() + + result, err := childTask.Run(ctx, ChildInput{ + Greeting: fmt.Sprintf("greetings from %s", input.Name), + }) + if err != nil { + return ParentOutput{}, fmt.Errorf("child task failed: %w", err) + } + + var childOutput ChildOutput + if err := result.Into(&childOutput); err != nil { + return ParentOutput{}, fmt.Errorf("failed to parse child output: %w", err) + } + + return ParentOutput{ + ChildResult: childOutput.Message, + }, nil + }, + ) + + worker, err := client.NewWorker( + "otel-worker", + hatchet.WithWorkflows(parentTask, childTask), + ) + if err != nil { + log.Fatalf("failed to create worker: %v", err) + } + + worker.Use(instrumentor.Middleware()) + + interruptCtx, cancel := cmdutils.NewInterruptContext() + defer cancel() + + go func() { + <-interruptCtx.Done() + if shutdownErr := instrumentor.Shutdown(context.Background()); shutdownErr != nil { + log.Printf("failed to shutdown instrumentor: %v", shutdownErr) + } + }() + + if startErr := worker.StartBlocking(interruptCtx); startErr != nil { + log.Printf("worker error: %v", startErr) + } +} diff --git a/examples/python/opentelemetry_instrumentation/hatchet/trigger.py b/examples/python/opentelemetry_instrumentation/hatchet/trigger.py new file mode 100644 index 0000000000..b1282b3bd1 --- /dev/null +++ b/examples/python/opentelemetry_instrumentation/hatchet/trigger.py @@ -0,0 +1,49 @@ +""" +Trigger the OTelDataPipeline workflow. + +Make sure the worker is already running: + poetry run python -m examples.opentelemetry_instrumentation.hatchet.worker + +Then run this: + poetry run python -m examples.opentelemetry_instrumentation.hatchet.trigger +""" + +from opentelemetry.sdk.resources import SERVICE_NAME, Resource +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import ConsoleSpanExporter, SimpleSpanProcessor + +from examples.opentelemetry_instrumentation.hatchet.worker import otel_workflow +from hatchet_sdk.clients.admin import TriggerWorkflowOptions +from hatchet_sdk.opentelemetry.instrumentor import HatchetInstrumentor + +# Use the same console exporter so you can see trigger-side spans too +resource = Resource(attributes={SERVICE_NAME: "hatchet-otel-pipeline-trigger"}) +provider = TracerProvider(resource=resource) +provider.add_span_processor(SimpleSpanProcessor(ConsoleSpanExporter())) + +HatchetInstrumentor( + tracer_provider=provider, + enable_hatchet_otel_collector=True, +).instrument() + +tracer = provider.get_tracer(__name__) + + +def main() -> None: + # The run_workflow call is auto-traced with a "hatchet.run_workflow" span. + # The traceparent is automatically injected into additional_metadata, + # so the worker-side spans become children of this trigger span. + with tracer.start_as_current_span("trigger_otel_data_pipeline"): + result = otel_workflow.run( + options=TriggerWorkflowOptions( + additional_metadata={ + "source": "otel-example", + "pipeline": "data-ingest", + }, + ), + ) + print(f"Workflow result: {result}") + + +if __name__ == "__main__": + main() diff --git a/examples/python/opentelemetry_instrumentation/hatchet/worker.py b/examples/python/opentelemetry_instrumentation/hatchet/worker.py new file mode 100644 index 0000000000..af26c16ee3 --- /dev/null +++ b/examples/python/opentelemetry_instrumentation/hatchet/worker.py @@ -0,0 +1,144 @@ +""" +HatchetInstrumentor example with rich traces. + +Run the worker: + poetry run python -m examples.opentelemetry_instrumentation.hatchet.worker + +Then trigger it from another terminal: + poetry run python -m examples.opentelemetry_instrumentation.hatchet.trigger +""" + +import time + +from opentelemetry.sdk.resources import SERVICE_NAME, Resource +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import ConsoleSpanExporter, SimpleSpanProcessor +from opentelemetry.trace import StatusCode, Tracer + +from hatchet_sdk import Context, EmptyModel, Hatchet +from hatchet_sdk.opentelemetry.instrumentor import HatchetInstrumentor + +hatchet = Hatchet() + +otel_workflow = hatchet.workflow(name="OTelDataPipeline") + +# Module-level tracer — will be set in main() before the worker starts. +# Tasks use this to create custom child spans inside the auto-instrumented +# hatchet task run parent span. +_tracer: Tracer | None = None + + +def _get_tracer() -> Tracer: + global _tracer + if _tracer is None: + from opentelemetry.trace import get_tracer + + _tracer = get_tracer(__name__) + return _tracer + + +@otel_workflow.task() +def fetch_data(input: EmptyModel, ctx: Context) -> dict[str, str]: + tracer = _get_tracer() + + with tracer.start_as_current_span( + "http.request", + attributes={"http.method": "GET", "http.url": "https://api.example.com/data"}, + ) as span: + time.sleep(0.05) + span.set_attribute("http.status_code", 200) + span.set_attribute("http.response_content_length", 4096) + + with tracer.start_as_current_span("json.parse") as span: + time.sleep(0.01) + span.set_attribute("json.record_count", 42) + + return {"records_fetched": "42"} + + +@otel_workflow.task() +def validate_data(input: EmptyModel, ctx: Context) -> dict[str, str]: + tracer = _get_tracer() + + with tracer.start_as_current_span("schema.validate") as span: + time.sleep(0.02) + span.set_attribute("validation.schema", "v2.1") + span.set_attribute("validation.records_checked", 42) + span.set_attribute("validation.errors", 2) + span.set_status(StatusCode.OK, "2 records failed validation") + + with tracer.start_as_current_span("data.clean") as span: + time.sleep(0.01) + span.set_attribute("clean.records_dropped", 2) + span.set_attribute("clean.records_remaining", 40) + + return {"valid_records": "40", "dropped": "2"} + + +@otel_workflow.task() +def process_data(input: EmptyModel, ctx: Context) -> dict[str, str]: + tracer = _get_tracer() + + with tracer.start_as_current_span("transform.pipeline") as pipeline_span: + pipeline_span.set_attribute("pipeline.stages", 3) + + with tracer.start_as_current_span("transform.normalize"): + time.sleep(0.015) + + with tracer.start_as_current_span("transform.enrich") as enrich_span: + time.sleep(0.02) + enrich_span.set_attribute("enrich.source", "geocoding-api") + + with tracer.start_as_current_span("transform.aggregate") as agg_span: + time.sleep(0.03) + agg_span.set_attribute("aggregate.groups", 8) + agg_span.set_attribute("aggregate.method", "sum") + + return {"processed_groups": "8"} + + +@otel_workflow.task() +def save_results(input: EmptyModel, ctx: Context) -> dict[str, str]: + tracer = _get_tracer() + + with tracer.start_as_current_span( + "db.query", + attributes={"db.system": "postgresql", "db.operation": "INSERT"}, + ) as span: + time.sleep(0.04) + span.set_attribute("db.rows_affected", 8) + + with tracer.start_as_current_span("cache.invalidate") as span: + time.sleep(0.005) + span.set_attribute("cache.keys_invalidated", 3) + + with tracer.start_as_current_span("notification.send") as span: + time.sleep(0.01) + span.set_attribute("notification.channel", "webhook") + span.set_attribute("notification.status", "delivered") + + return {"saved": "true"} + + +def main() -> None: + resource = Resource(attributes={SERVICE_NAME: "hatchet-otel-pipeline-example"}) + provider = TracerProvider(resource=resource) + provider.add_span_processor(SimpleSpanProcessor(ConsoleSpanExporter())) + + HatchetInstrumentor( + tracer_provider=provider, + enable_hatchet_otel_collector=True, + ).instrument() + + global _tracer + _tracer = provider.get_tracer(__name__) + + worker = hatchet.worker( + "otel-pipeline-worker", + workflows=[otel_workflow], + ) + worker.start() + + +if __name__ == "__main__": + main() diff --git a/examples/typescript/guides/package.json b/examples/typescript/guides/package.json index 969b847637..2e6d112724 100644 --- a/examples/typescript/guides/package.json +++ b/examples/typescript/guides/package.json @@ -8,7 +8,7 @@ "lint:fix": "eslint . --fix" }, "dependencies": { - "@hatchet-dev/typescript-sdk": "^1.15.2", + "@hatchet-dev/typescript-sdk": "^1.16.0", "@anthropic-ai/sdk": "^0.32.1", "@ai-sdk/openai": "^1.0.0", "@browserbasehq/sdk": "^2.7.0", diff --git a/examples/typescript/opentelemetry/run.ts b/examples/typescript/opentelemetry/run.ts new file mode 100644 index 0000000000..420a15d30c --- /dev/null +++ b/examples/typescript/opentelemetry/run.ts @@ -0,0 +1,43 @@ +import { getTracer } from './tracer'; + +import { SpanStatusCode, type Span } from '@opentelemetry/api'; +import { hatchet } from '../hatchet-client'; +import { otelWorkflow } from './worker'; + +const tracer = getTracer('opentelemetry-triggers'); + +async function runWorkflow() { + return tracer.startActiveSpan('run_workflow', async (span: Span) => { + try { + const workflowRun = await hatchet.admin.runWorkflow(otelWorkflow.name, {}); + const runId = await workflowRun.getWorkflowRunId(); + console.log(`Started workflow run: ${runId}`); + + const result = await workflowRun.output; + span.setStatus({ code: SpanStatusCode.OK, message: 'Workflow completed' }); + console.log(`Workflow completed with result:`, result); + } catch (error: any) { + const errorMessage = Array.isArray(error) + ? error.join(', ') + : error?.message || String(error); + console.error('Workflow failed:', errorMessage); + span.recordException(error); + span.setStatus({ code: SpanStatusCode.ERROR, message: errorMessage }); + } finally { + span.end(); + } + }); +} + +async function main() { + await runWorkflow(); + + console.log('Waiting for spans to be exported...'); + await new Promise((resolve) => setTimeout(resolve, 5000)); + + process.exit(0); +} + +if (require.main === module) { + main().catch(console.error); +} diff --git a/examples/typescript/opentelemetry/tracer.ts b/examples/typescript/opentelemetry/tracer.ts new file mode 100644 index 0000000000..88bb8b6182 --- /dev/null +++ b/examples/typescript/opentelemetry/tracer.ts @@ -0,0 +1,63 @@ +const { NodeTracerProvider } = require('@opentelemetry/sdk-trace-node'); +const { OTLPTraceExporter } = require('@opentelemetry/exporter-trace-otlp-http'); +const { resourceFromAttributes } = require('@opentelemetry/resources'); +const { SEMRESATTRS_SERVICE_NAME } = require('@opentelemetry/semantic-conventions'); +const { BatchSpanProcessor } = require('@opentelemetry/sdk-trace-base'); +const { registerInstrumentations } = require('@opentelemetry/instrumentation'); +const { trace } = require('@opentelemetry/api'); +/* eslint-enable @typescript-eslint/no-require-imports */ + +import type { TracerProvider, Tracer } from '@opentelemetry/api'; +import { HatchetInstrumentor } from '@hatchet-dev/typescript-sdk/opentelemetry'; + +const isCI = process.env.CI === 'true'; + +let traceProvider: TracerProvider; + +if (isCI) { + traceProvider = trace.getTracerProvider(); + registerInstrumentations({ + tracerProvider: traceProvider, + instrumentations: [new HatchetInstrumentor()], + }); +} else { + const resource = resourceFromAttributes({ + [SEMRESATTRS_SERVICE_NAME]: + process.env.HATCHET_CLIENT_OTEL_SERVICE_NAME || 'hatchet-typescript-example', + }); + + const headersEnv = process.env.HATCHET_CLIENT_OTEL_EXPORTER_OTLP_HEADERS; + const headers: Record | undefined = headersEnv + ? { [headersEnv.split('=')[0]]: headersEnv.split('=')[1] } + : undefined; + + const exporter = new OTLPTraceExporter({ + url: + process.env.HATCHET_CLIENT_OTEL_EXPORTER_OTLP_ENDPOINT || 'http://localhost:4318/v1/traces', + headers, + }); + + const provider = new NodeTracerProvider({ + resource, + spanProcessors: [new BatchSpanProcessor(exporter)], + }); + + provider.register(); + + traceProvider = provider; + + registerInstrumentations({ + tracerProvider: traceProvider, + instrumentations: [ + new HatchetInstrumentor({ + includeTaskNameInSpanName: true, + }), + ], + }); +} + +function getTracer(name: string): Tracer { + return trace.getTracer(name); +} + +export { traceProvider, getTracer }; diff --git a/examples/typescript/opentelemetry/worker.ts b/examples/typescript/opentelemetry/worker.ts new file mode 100644 index 0000000000..443bc9b556 --- /dev/null +++ b/examples/typescript/opentelemetry/worker.ts @@ -0,0 +1,70 @@ +import { hatchet } from '../hatchet-client'; +import { getTracer } from './tracer'; +import { SpanStatusCode } from '@opentelemetry/api'; + +const tracer = getTracer('opentelemetry-worker'); + +export const otelWorkflow = hatchet.workflow({ + name: 'otelworkflowtypescript', +}); + +otelWorkflow.task({ + name: 'step-with-custom-spans', + fn: async () => { + return tracer.startActiveSpan('custom-business-logic', async (span) => { + try { + console.log('Executing step with custom tracing...'); + await new Promise((resolve) => setTimeout(resolve, 100)); + + return { result: 'success' }; + } finally { + span.end(); + } + }); + }, +}); + +otelWorkflow.task({ + name: 'step-with-error', + fn: async () => { + return tracer.startActiveSpan('custom-span-with-error', async (span) => { + try { + throw new Error('Intentional error for demonstration'); + } catch (error: any) { + span.recordException(error); + span.setStatus({ code: SpanStatusCode.ERROR, message: error?.message }); + throw error; + } finally { + span.end(); + } + }); + }, +}); + +otelWorkflow.task({ + name: 'auto-instrumented-step', + fn: async () => { + console.log('This step is automatically traced without manual span code'); + return { automatically: 'instrumented' }; + }, +}); + +otelWorkflow.task({ + name: 'auto-instrumented-step-with-error', + fn: async () => { + throw new Error('Auto-instrumented step error'); + }, +}); + +async function main() { + const worker = await hatchet.worker('otel-example-worker-ts', { + slots: 1, + workflows: [otelWorkflow], + }); + + await worker.start(); +} + +if (require.main === module) { + main().catch(console.error); +} diff --git a/frontend/app/package.json b/frontend/app/package.json index 28504341c2..ce38767a26 100644 --- a/frontend/app/package.json +++ b/frontend/app/package.json @@ -23,6 +23,8 @@ "docs:gen": "cd ../snippets && python3 generate.py" }, "dependencies": { + "@evilmartians/agent-prism-data": "^0.0.9", + "@evilmartians/agent-prism-types": "^0.0.9", "@heroicons/react": "^2.2.0", "@hookform/resolvers": "^3.10.0", "@lukemorales/query-key-factory": "^1.3.4", @@ -57,6 +59,7 @@ "axios": "^1.13.5", "better-emitter": "^4.0.1", "class-variance-authority": "^0.7.1", + "classnames": "^2.5.1", "clsx": "^2.1.1", "cmdk": "^0.2.1", "cronstrue": "^2.57.0", @@ -75,6 +78,8 @@ "react-dom": "^18.3.1", "react-hook-form": "^7.55.0", "react-icons": "^5.5.0", + "react-json-pretty": "^2.2.0", + "react-resizable-panels": "^2.1.9", "react-syntax-highlighter": "^15.6.1", "reactflow": "^11.11.4", "recharts": "^2.15.1", diff --git a/frontend/app/pnpm-lock.yaml b/frontend/app/pnpm-lock.yaml index d270e60d91..bd8eee8ce7 100644 --- a/frontend/app/pnpm-lock.yaml +++ b/frontend/app/pnpm-lock.yaml @@ -18,6 +18,12 @@ importers: .: dependencies: + '@evilmartians/agent-prism-data': + specifier: ^0.0.9 + version: 0.0.9 + '@evilmartians/agent-prism-types': + specifier: ^0.0.9 + version: 0.0.9 '@heroicons/react': specifier: ^2.2.0 version: 2.2.0(react@18.3.1) @@ -120,6 +126,9 @@ importers: class-variance-authority: specifier: ^0.7.1 version: 0.7.1 + classnames: + specifier: ^2.5.1 + version: 2.5.1 clsx: specifier: ^2.1.1 version: 2.1.1 @@ -174,6 +183,12 @@ importers: react-icons: specifier: ^5.5.0 version: 5.5.0(react@18.3.1) + react-json-pretty: + specifier: ^2.2.0 + version: 2.2.0(react-dom@18.3.1(react@18.3.1))(react@18.3.1) + react-resizable-panels: + specifier: ^2.1.9 + version: 2.1.9(react-dom@18.3.1(react@18.3.1))(react@18.3.1) react-syntax-highlighter: specifier: ^15.6.1 version: 15.6.1(react@18.3.1) @@ -572,6 +587,12 @@ packages: resolution: {integrity: sha512-d9zaMRSTIKDLhctzH12MtXvJKSSUhaHcjV+2Z+GK+EEY7XKpP5yR4x+N3TAcHTcu963nIr+TMcCb4DBCYX1z6Q==} engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} + '@evilmartians/agent-prism-data@0.0.9': + resolution: {integrity: sha512-CyFGIMfTfRRhr+6c/EB+6wT2irx0x6gAMK7cCnmvGJtq/Hs/EWXnxprMCMf1Ct9dErlcYXiF8Y5AZbQP82ICoA==} + + '@evilmartians/agent-prism-types@0.0.9': + resolution: {integrity: sha512-rQW62z5XGF31oMnmdzNiGwzKTYJigsxKoPWWH/ZeVAxFi0c4bi+f8TndqgAebGBdHWcn5JRaFDZ76ywlNu4c6A==} + '@floating-ui/core@1.7.3': resolution: {integrity: sha512-sGnvb5dmrJaKEZ+LDIpguvdX3bDlEllmv4/ClQ9awcmCZrlx5jQyyMWFM5kBI+EyNOCDDiKk8il0zeuX3Zlg/w==} @@ -2121,6 +2142,9 @@ packages: classcat@5.0.5: resolution: {integrity: sha512-JhZUT7JFcQy/EzW605k/ktHtncoo9vnyW/2GspNYwFlN1C/WmjuV/xtS04e9SOkL2sTdw0VAZ2UGCcQ9lR6p6w==} + classnames@2.5.1: + resolution: {integrity: sha512-saHYOzhIQs6wy2sVxTM6bUDsQO4F50V9RQ22qBpEdCW+I+/Wmke2HOl6lS6dTpdxVhb88/I6+Hs+438c3lfUow==} + clean-stack@2.2.0: resolution: {integrity: sha512-4diC9HaTE+KRAMWhDhrGOECgWZxoevMc5TlkObMqNSsVU62PYzXZ/SMTjzyGAFF1YusgxGcSWTEXBhp0CPwQ1A==} engines: {node: '>=6'} @@ -3663,6 +3687,12 @@ packages: react-is@18.3.1: resolution: {integrity: sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg==} + react-json-pretty@2.2.0: + resolution: {integrity: sha512-3UMzlAXkJ4R8S4vmkRKtvJHTewG4/rn1Q18n0zqdu/ipZbUPLVZD+QwC7uVcD/IAY3s8iNVHlgR2dMzIUS0n1A==} + peerDependencies: + react: '>=15.0' + react-dom: '>=15.0' + react-refresh@0.17.0: resolution: {integrity: sha512-z6F7K9bV85EfseRCp2bzrpyQ0Gkw1uLoCel9XBVWPg/TjRj94SkJzUTGfOa4bs7iJvBWtQG0Wq7wnI0syw3EBQ==} engines: {node: '>=0.10.0'} @@ -3697,6 +3727,12 @@ packages: '@types/react': optional: true + react-resizable-panels@2.1.9: + resolution: {integrity: sha512-z77+X08YDIrgAes4jl8xhnUu1LNIRp4+E7cv4xHmLOxxUPO/ML7PSrE813b90vj7xvQ1lcf7g2uA9GeMZonjhQ==} + peerDependencies: + react: ^16.14.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc + react-dom: ^16.14.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc + react-smooth@4.0.4: resolution: {integrity: sha512-gnGKTpYwqL0Iii09gHobNolvX4Kiq4PKx6eWBCYYix+8cdw+cGo3do906l1NBPKkSWx1DghC1dlWG9L2uGd61Q==} peerDependencies: @@ -4623,6 +4659,12 @@ snapshots: '@eslint/js@8.57.1': {} + '@evilmartians/agent-prism-data@0.0.9': + dependencies: + '@evilmartians/agent-prism-types': 0.0.9 + + '@evilmartians/agent-prism-types@0.0.9': {} + '@floating-ui/core@1.7.3': dependencies: '@floating-ui/utils': 0.2.10 @@ -6325,6 +6367,8 @@ snapshots: classcat@5.0.5: {} + classnames@2.5.1: {} + clean-stack@2.2.0: {} cli-cursor@3.1.0: @@ -7939,6 +7983,12 @@ snapshots: react-is@18.3.1: {} + react-json-pretty@2.2.0(react-dom@18.3.1(react@18.3.1))(react@18.3.1): + dependencies: + prop-types: 15.8.1 + react: 18.3.1 + react-dom: 18.3.1(react@18.3.1) + react-refresh@0.17.0: {} react-remove-scroll-bar@2.3.8(@types/react@18.3.23)(react@18.3.1): @@ -7971,6 +8021,11 @@ snapshots: optionalDependencies: '@types/react': 18.3.23 + react-resizable-panels@2.1.9(react-dom@18.3.1(react@18.3.1))(react@18.3.1): + dependencies: + react: 18.3.1 + react-dom: 18.3.1(react@18.3.1) + react-smooth@4.0.4(react-dom@18.3.1(react@18.3.1))(react@18.3.1): dependencies: fast-equals: 5.2.2 diff --git a/frontend/app/src/components/v1/agent-prism/Avatar.tsx b/frontend/app/src/components/v1/agent-prism/Avatar.tsx new file mode 100644 index 0000000000..3005f2dae2 --- /dev/null +++ b/frontend/app/src/components/v1/agent-prism/Avatar.tsx @@ -0,0 +1,143 @@ +import type { ComponentSize } from './shared'; +import { ROUNDED_CLASSES } from './shared'; +import type { TraceSpanCategory } from '@evilmartians/agent-prism-types'; +import cn from 'classnames'; +import { User } from 'lucide-react'; +import type { ComponentPropsWithRef, ReactElement } from 'react'; +import { useState } from 'react'; + +export type AvatarSize = Extract< + ComponentSize, + '4' | '6' | '8' | '9' | '10' | '11' | '12' | '16' +>; + +const sizeClasses: Record = { + '4': 'size-4 text-xs', + '6': 'size-6 text-xs', + '8': 'size-8 text-xs', + '9': 'size-9 text-sm', + '10': 'size-10 text-base', + '11': 'size-11 text-lg', + '12': 'size-12 text-xl', + '16': 'size-16 text-2xl', +}; + +const iconSizeClasses: Record = { + '4': 'size-3', + '6': 'size-4', + '8': 'size-6', + '9': 'size-7', + '10': 'size-8', + '11': 'size-9', + '12': 'size-10', + '16': 'size-12', +}; + +const bgColorClasses: Record = { + llm_call: 'bg-agentprism-avatar-llm', + tool_execution: 'bg-agentprism-avatar-tool', + agent_invocation: 'bg-agentprism-avatar-agent', + chain_operation: 'bg-agentprism-avatar-chain', + retrieval: 'bg-agentprism-avatar-retrieval', + embedding: 'bg-agentprism-avatar-embedding', + create_agent: 'bg-agentprism-avatar-create-agent', + span: 'bg-agentprism-avatar-span', + event: 'bg-agentprism-avatar-event', + guardrail: 'bg-agentprism-avatar-guardrail', + unknown: 'bg-agentprism-avatar-unknown', +}; + +export type AvatarProps = ComponentPropsWithRef<'div'> & { + /** + * The category of the span which avatar is associated with + */ + category: TraceSpanCategory; + /** + * The image source for the avatar + */ + src?: string; + /** + * The alt text for the avatar + */ + alt?: string; + /** + * The size of the avatar + * @default "md" + */ + size?: AvatarSize; + /** + * The border radius of the avatar + * @default "full" + */ + rounded?: 'none' | 'sm' | 'md' | 'lg' | 'full'; + /** + * Custom letter to display (will use first letter of alt if not provided) + */ + letter?: string; + /** + * Optional className for additional styling + */ + className?: string; +}; + +export const Avatar = ({ + category, + src, + alt = 'Avatar', + size = '10', + rounded = 'full', + letter, + children, + className = '', + ...rest +}: AvatarProps): ReactElement => { + const [error, setError] = useState(false); + + const displayLetter = letter ? letter.charAt(0) : alt.charAt(0).toUpperCase(); + + return ( +
+ {children ? ( + children + ) : error ? ( + + ) : ( + <> + {src ? ( + {alt} setError(true)} + /> + ) : ( +
+ {displayLetter} +
+ )} + + )} +
+ ); +}; diff --git a/frontend/app/src/components/v1/agent-prism/Badge.tsx b/frontend/app/src/components/v1/agent-prism/Badge.tsx new file mode 100644 index 0000000000..6840456ac6 --- /dev/null +++ b/frontend/app/src/components/v1/agent-prism/Badge.tsx @@ -0,0 +1,93 @@ +import type { ComponentSize } from './shared'; +import cn from 'classnames'; +import type { ComponentPropsWithRef, ReactElement, ReactNode } from 'react'; + +type BadgeSize = Extract; + +const sizeClasses: Record = { + '4': 'px-1 gap-1 h-4', + '5': 'px-1.5 gap-1 h-5', + '6': 'px-2 gap-1.5 h-6', + '7': 'px-2.5 gap-2 h-7', +}; + +const textSizes: Record = { + '4': 'text-xs leading-3', + '5': 'text-xs', + '6': 'text-sm', + '7': 'text-sm', +}; + +export type BadgeProps = ComponentPropsWithRef<'span'> & { + /** + * The content of the badge + */ + label: ReactNode; + + /** + * The size of the badge + * @default "md" + */ + size?: BadgeSize; + + /** + * Optional icon to display at the start of the badge + */ + iconStart?: ReactElement; + + /** + * Optional icon to display at the end of the badge + */ + iconEnd?: ReactElement; + + /** + * Optional className for additional styling + */ + className?: string; + + /** + * Whether to render the badge without any default styles + * @default false + */ + unstyled?: boolean; +}; + +/** + * An unstyled badge component that displays a label with an optional icon + */ +export const Badge = ({ + label, + size = '4', + iconStart, + iconEnd, + className = '', + unstyled = false, + ...rest +}: BadgeProps): ReactElement => { + return ( + + {iconStart && {iconStart}} + + + {label} + + + {iconEnd && {iconEnd}} + + ); +}; diff --git a/frontend/app/src/components/v1/agent-prism/BrandLogo.tsx b/frontend/app/src/components/v1/agent-prism/BrandLogo.tsx new file mode 100644 index 0000000000..413892b76e --- /dev/null +++ b/frontend/app/src/components/v1/agent-prism/BrandLogo.tsx @@ -0,0 +1,104 @@ +import { type FC } from 'react'; + +const OpenAILogo: FC<{ className?: string }> = ({ className }) => ( + + + +); + +const AnthropicLogo: FC<{ className?: string }> = ({ className }) => ( + + + +); + +const GoogleLogo: FC<{ className?: string }> = ({ className }) => ( + + + + + + +); + +const MetaLogo: FC<{ className?: string }> = ({ className }) => ( + + + +); + +const MistralLogo: FC<{ className?: string }> = ({ className }) => ( + + + + + + + + + + + + + + + + + + + + + + + +); + +const PerplexityLogo: FC<{ className?: string }> = ({ className }) => ( + + + +); + +// Logo registry +const LOGO_REGISTRY = { + openai: OpenAILogo, + anthropic: AnthropicLogo, + google: GoogleLogo, + meta: MetaLogo, + mistral: MistralLogo, + perplexity: PerplexityLogo, +} as const; + +type BrandType = keyof typeof LOGO_REGISTRY; + +type BrandLogoProps = { + brand: BrandType | string; + className?: string; + fallback?: React.ReactNode; +}; + +export const BrandLogo: FC = ({ + brand, + className = 'size-4', + fallback = null, +}) => { + const Logo = LOGO_REGISTRY[brand as BrandType]; + + if (!Logo) { + return <>{fallback}; + } + + return ; +}; diff --git a/frontend/app/src/components/v1/agent-prism/Button.tsx b/frontend/app/src/components/v1/agent-prism/Button.tsx new file mode 100644 index 0000000000..68f5a71c67 --- /dev/null +++ b/frontend/app/src/components/v1/agent-prism/Button.tsx @@ -0,0 +1,121 @@ +import type { ComponentSize } from './shared'; +import { ROUNDED_CLASSES } from './shared'; +import cn from 'classnames'; +import type { ComponentPropsWithRef, ReactElement } from 'react'; + +type ButtonSize = Extract< + ComponentSize, + '6' | '7' | '8' | '9' | '10' | '11' | '12' | '16' +>; + +type ButtonVariant = + | 'brand' + | 'primary' + | 'outlined' + | 'secondary' + | 'ghost' + | 'destructive' + | 'success'; + +const BASE_CLASSES = + 'inline-flex items-center justify-center font-medium transition-all duration-200'; + +const sizeClasses = { + '6': 'h-6 px-2 gap-1 text-xs', + '7': 'h-7 px-2 gap-1 text-xs', + '8': 'h-8 px-2 gap-1 text-xs', + '9': 'h-9 px-2.5 gap-2 text-sm', + '10': 'h-10 px-4 gap-2 text-sm', + '11': 'h-11 px-5 gap-3 text-base', + '12': 'h-12 px-5 gap-2.5 text-base', + '16': 'h-16 px-7 gap-3 text-lg', +}; + +const variantClasses: Record = { + brand: 'text-agentprism-brand-foreground bg-agentprism-brand', + primary: 'text-agentprism-primary-foreground bg-agentprism-primary', + outlined: + 'border border bg-transparent text-agentprism-foreground border-agentprism-foreground', + secondary: 'bg-agentprism-secondary text-agentprism-secondary-foreground', + ghost: 'bg-transparent text-agentprism-foreground', + destructive: 'bg-agentprism-error text-agentprism-primary-foreground', + success: 'bg-agentprism-success text-agentprism-primary-foreground', +}; + +export type ButtonProps = ComponentPropsWithRef<'button'> & { + /** + * The size of the button + * @default "6" + */ + size?: ButtonSize; + + /** + * The border radius of the button + * @default "md" + */ + rounded?: 'none' | 'sm' | 'md' | 'lg' | 'full'; + + /** + * The visual variant of the button + * @default "primary" + */ + variant?: ButtonVariant; + + /** + * Makes the button full width + * @default false + */ + fullWidth?: boolean; + + /** + * Optional icon to display at the start of the button + */ + iconStart?: ReactElement; + + /** + * Optional icon to display at the end of the button + */ + iconEnd?: ReactElement; +}; + +export const Button = ({ + children, + size = '6', + rounded = 'md', + variant = 'primary', + fullWidth = false, + disabled = false, + iconStart, + iconEnd, + type = 'button', + onClick, + className = '', + ...rest +}: ButtonProps) => { + const widthClass = fullWidth ? 'w-full' : ''; + const stateClasses = disabled + ? 'cursor-not-allowed opacity-50' + : 'hover:opacity-70'; + + return ( + + ); +}; diff --git a/frontend/app/src/components/v1/agent-prism/CollapseAndExpandControls.tsx b/frontend/app/src/components/v1/agent-prism/CollapseAndExpandControls.tsx new file mode 100644 index 0000000000..25a53ce3ef --- /dev/null +++ b/frontend/app/src/components/v1/agent-prism/CollapseAndExpandControls.tsx @@ -0,0 +1,43 @@ +import { IconButton } from './IconButton'; +import { ChevronsUpDown, ChevronsDownUp } from 'lucide-react'; +import type { ComponentPropsWithRef } from 'react'; + +export type SpanCardExpandAllButtonProps = ComponentPropsWithRef<'button'> & { + onExpandAll: () => void; +}; + +export type SpanCardCollapseAllButtonProps = ComponentPropsWithRef<'button'> & { + onCollapseAll: () => void; +}; + +export const ExpandAllButton = ({ + onExpandAll, + ...rest +}: SpanCardExpandAllButtonProps) => { + return ( + + + + ); +}; + +export const CollapseAllButton = ({ + onCollapseAll, + ...rest +}: SpanCardCollapseAllButtonProps) => { + return ( + + + + ); +}; diff --git a/frontend/app/src/components/v1/agent-prism/CollapsibleSection.tsx b/frontend/app/src/components/v1/agent-prism/CollapsibleSection.tsx new file mode 100644 index 0000000000..5371de1976 --- /dev/null +++ b/frontend/app/src/components/v1/agent-prism/CollapsibleSection.tsx @@ -0,0 +1,124 @@ +import * as Collapsible from '@radix-ui/react-collapsible'; +import cn from 'classnames'; +import { ChevronDown } from 'lucide-react'; +import * as React from 'react'; + +export interface CollapsibleSectionProps { + /** + * The title text displayed in the trigger button + */ + title: string; + + /** + * The content to display on the right side of the title + */ + rightContent?: React.ReactNode; + + /** + * The content to display when the section is expanded + */ + children: React.ReactNode; + + /** + * Whether the section starts in an open state + * @default false + */ + defaultOpen?: boolean; + + /** + * Optional className for the root container + */ + className?: string; + + /** + * Optional className for the trigger button + */ + triggerClassName?: string; + + /** + * Optional className for the content area + */ + contentClassName?: string; + + /** + * Optional callback fired when the section is expanded or collapsed + */ + onOpenChange?: (open: boolean) => void; +} + +export const CollapsibleSection: React.FC = ({ + title, + rightContent, + children, + defaultOpen = false, + className = '', + triggerClassName = '', + contentClassName = '', + onOpenChange, +}) => { + const [open, setOpen] = React.useState(defaultOpen); + + const handleOpenChange = React.useCallback( + (open: boolean): void => { + setOpen(open); + onOpenChange?.(open); + }, + [onOpenChange], + ); + + const handleKeyDown = React.useCallback( + (e: React.KeyboardEvent): void => { + if (e.key === 'Enter' || e.key === ' ') { + e.preventDefault(); + handleOpenChange(!open); + } + }, + [handleOpenChange, open], + ); + + return ( + + +
+
+ + + {title} + +
+ +
{rightContent}
+
+ + + + {children} + + + ); +}; diff --git a/frontend/app/src/components/v1/agent-prism/CopyButton.tsx b/frontend/app/src/components/v1/agent-prism/CopyButton.tsx new file mode 100644 index 0000000000..069af38ed3 --- /dev/null +++ b/frontend/app/src/components/v1/agent-prism/CopyButton.tsx @@ -0,0 +1,62 @@ +import { IconButton } from './IconButton'; +import { Check, Copy, X } from 'lucide-react'; +import { useState } from 'react'; + +type CopyButtonProps = { + label: string; + content: string; +}; + +type CopyState = 'idle' | 'success' | 'error'; + +export const CopyButton = ({ label, content }: CopyButtonProps) => { + const [copyState, setCopyState] = useState('idle'); + + const onClick = async () => { + try { + if (!navigator.clipboard) { + throw new Error('Clipboard API not supported'); + } + + await navigator.clipboard.writeText(content); + setCopyState('success'); + setTimeout(() => setCopyState('idle'), 2000); + } catch { + setCopyState('error'); + setTimeout(() => setCopyState('idle'), 2000); + } + }; + + const getIcon = () => { + switch (copyState) { + case 'success': + return ; + case 'error': + return ; + default: + return ; + } + }; + + const getAriaLabel = () => { + switch (copyState) { + case 'success': + return `${label} Copied`; + case 'error': + return `Failed to copy ${label}`; + default: + return `Copy ${label}`; + } + }; + + return ( + + {getIcon()} + + ); +}; diff --git a/frontend/app/src/components/v1/agent-prism/DetailsView/DetailsView.tsx b/frontend/app/src/components/v1/agent-prism/DetailsView/DetailsView.tsx new file mode 100644 index 0000000000..9308b19f19 --- /dev/null +++ b/frontend/app/src/components/v1/agent-prism/DetailsView/DetailsView.tsx @@ -0,0 +1,140 @@ +import type { AvatarProps } from '../Avatar'; +import { TabSelector } from '../TabSelector'; +import type { TabItem } from '../Tabs'; +import { DetailsViewAttributesTab } from './DetailsViewAttributesTab'; +import { DetailsViewHeader } from './DetailsViewHeader'; +import { DetailsViewInputOutputTab } from './DetailsViewInputOutputTab'; +import { DetailsViewRawDataTab } from './DetailsViewRawDataTab'; +import type { TraceSpan } from '@evilmartians/agent-prism-types'; +import cn from 'classnames'; +import { SquareTerminal, Tags, ArrowRightLeft } from 'lucide-react'; +import type { ReactElement, ReactNode } from 'react'; +import { useState } from 'react'; + +type DetailsViewTab = 'input-output' | 'attributes' | 'raw'; + +export interface DetailsViewProps { + /** + * The span data to display in the details view + */ + data: TraceSpan; + + /** + * Optional avatar configuration for the header + */ + avatar?: AvatarProps; + + /** + * The initially selected tab + */ + defaultTab?: DetailsViewTab; + + /** + * Optional className for the root container + */ + className?: string; + + /** + * Configuration for the copy button functionality + */ + copyButton?: { + isEnabled?: boolean; + onCopy?: (data: TraceSpan) => void; + }; + + /** + * Custom header actions to render + * Can be a ReactNode or a render function that receives the data + */ + headerActions?: ReactNode | ((data: TraceSpan) => ReactNode); + + /** + * Optional custom header component to replace the default + */ + customHeader?: ReactNode | ((props: { data: TraceSpan }) => ReactNode); + + /** + * Callback fired when the active tab changes + */ + onTabChange?: (tabValue: DetailsViewTab) => void; +} + +const TAB_ITEMS: TabItem[] = [ + { + value: 'input-output', + label: 'In/Out', + icon: , + }, + { + value: 'attributes', + label: 'Attributes', + icon: , + }, + { + value: 'raw', + label: 'RAW', + icon: , + }, +]; + +export const DetailsView = ({ + data, + avatar, + defaultTab = 'input-output', + className, + copyButton, + headerActions, + customHeader, + onTabChange, +}: DetailsViewProps): ReactElement => { + const [tab, setTab] = useState(defaultTab); + + const handleTabChange = (tabValue: DetailsViewTab) => { + setTab(tabValue); + onTabChange?.(tabValue); + }; + + const resolvedHeaderActions = + typeof headerActions === 'function' ? headerActions(data) : headerActions; + + const headerContent = customHeader ? ( + typeof customHeader === 'function' ? ( + customHeader({ data }) + ) : ( + customHeader + ) + ) : ( + + ); + + return ( +
+
{headerContent}
+
+ +
+ +
+ {tab === 'input-output' && } + {tab === 'attributes' && } + {tab === 'raw' && } +
+
+ ); +}; diff --git a/frontend/app/src/components/v1/agent-prism/DetailsView/DetailsViewAttributesTab.tsx b/frontend/app/src/components/v1/agent-prism/DetailsView/DetailsViewAttributesTab.tsx new file mode 100644 index 0000000000..46731ceef8 --- /dev/null +++ b/frontend/app/src/components/v1/agent-prism/DetailsView/DetailsViewAttributesTab.tsx @@ -0,0 +1,123 @@ +import { CollapsibleSection } from '../CollapsibleSection'; +import { TabSelector } from '../TabSelector'; +import type { TabItem } from '../Tabs'; +import { + DetailsViewContentViewer, + type DetailsViewContentViewMode, +} from './DetailsViewContentViewer'; +import { type TraceSpan } from '@evilmartians/agent-prism-types'; +import { type ReactElement, useState } from 'react'; + +interface AttributesTabProps { + data: TraceSpan; +} + +const TAB_ITEMS: TabItem[] = [ + { value: 'json', label: 'JSON' }, + { value: 'plain', label: 'Plain' }, +]; + +export const DetailsViewAttributesTab = ({ + data, +}: AttributesTabProps): ReactElement => { + if (!data.attributes || data.attributes.length === 0) { + return ( +
+

+ No attributes available for this span. +

+
+ ); + } + + return ( +
+ {data.attributes.map((attribute, index) => { + const stringValue = attribute.value.stringValue; + const simpleValue = + stringValue || + attribute.value.intValue?.toString() || + attribute.value.boolValue?.toString() || + 'N/A'; + + let parsedJson: string | null = null; + if (typeof stringValue === 'string') { + try { + parsedJson = JSON.parse(stringValue); + } catch { + parsedJson = null; + } + } + + const isComplex = parsedJson !== null; + + if (isComplex && parsedJson && stringValue) { + return ( + + ); + } + + return ( +
+
+ {attribute.key} +
+
+ {simpleValue} +
+
+ ); + })} +
+ ); +}; + +interface AttributeSectionProps { + attributeKey: string; + content: string; + parsedContent: string; + id: string; +} + +const AttributeSection = ({ + attributeKey, + content, + parsedContent, + id, +}: AttributeSectionProps): ReactElement => { + const [tab, setTab] = useState('json'); + + return ( + + items={TAB_ITEMS} + defaultValue="json" + value={tab} + onValueChange={setTab} + theme="pill" + onClick={(event) => event.stopPropagation()} + /> + } + > + + + ); +}; diff --git a/frontend/app/src/components/v1/agent-prism/DetailsView/DetailsViewContentViewer.tsx b/frontend/app/src/components/v1/agent-prism/DetailsView/DetailsViewContentViewer.tsx new file mode 100644 index 0000000000..75d5b74856 --- /dev/null +++ b/frontend/app/src/components/v1/agent-prism/DetailsView/DetailsViewContentViewer.tsx @@ -0,0 +1,50 @@ +import { CopyButton } from '../CopyButton'; +import { DetailsViewJsonOutput } from './DetailsViewJsonOutput'; +import { type ReactElement } from 'react'; + +export type DetailsViewContentViewMode = 'json' | 'plain'; + +export interface DetailsViewContentViewerProps { + content: string; + parsedContent: string | null; + mode: DetailsViewContentViewMode; + label: string; + id: string; + className?: string; +} + +export const DetailsViewContentViewer = ({ + content, + parsedContent, + mode, + label, + id, + className = '', +}: DetailsViewContentViewerProps): ReactElement => { + if (!content) { + return ( +

+ No data available +

+ ); + } + + return ( +
+
+ +
+ {mode === 'json' && parsedContent ? ( + + ) : ( +
+ 
+            {content}
+
+
+ )} +
+ ); +}; diff --git a/frontend/app/src/components/v1/agent-prism/DetailsView/DetailsViewHeader.tsx b/frontend/app/src/components/v1/agent-prism/DetailsView/DetailsViewHeader.tsx new file mode 100644 index 0000000000..bcae84e108 --- /dev/null +++ b/frontend/app/src/components/v1/agent-prism/DetailsView/DetailsViewHeader.tsx @@ -0,0 +1,97 @@ +import type { AvatarProps } from '../Avatar'; +import { Avatar } from '../Avatar'; +import { IconButton } from '../IconButton'; +import { PriceBadge } from '../PriceBadge'; +import { SpanBadge } from '../SpanBadge'; +import { SpanStatus } from '../SpanStatus'; +import { TimestampBadge } from '../TimestampBadge'; +import { TokensBadge } from '../TokensBadge'; +import { getDurationMs, formatDuration } from '@evilmartians/agent-prism-data'; +import type { TraceSpan } from '@evilmartians/agent-prism-types'; +import { Check, Copy } from 'lucide-react'; +import type { ReactNode } from 'react'; +import { useState } from 'react'; + +export interface DetailsViewHeaderProps { + data: TraceSpan; + avatar?: AvatarProps; + copyButton?: { + isEnabled?: boolean; + onCopy?: (data: TraceSpan) => void; + }; + /** + * Custom actions to render in the header + */ + actions?: ReactNode; + /** + * Optional className for the header container + */ + className?: string; +} + +export const DetailsViewHeader = ({ + data, + avatar, + copyButton, + actions, + className, +}: DetailsViewHeaderProps) => { + const [hasCopied, setHasCopied] = useState(false); + const durationMs = getDurationMs(data); + + const handleCopy = () => { + if (copyButton?.onCopy) { + copyButton.onCopy(data); + setHasCopied(true); + setTimeout(() => setHasCopied(false), 2000); + } + }; + + return ( +
+ {avatar && } + + + {data.title} + + +
+ +
+ + {copyButton && ( + + {hasCopied ? ( + + ) : ( + + )} + + )} + + + + {typeof data.tokensCount === 'number' && ( + + )} + + {typeof data.cost === 'number' && } + + + LATENCY: {formatDuration(durationMs)} + + + {typeof data.startTime === 'number' && ( + + )} + + {actions} +
+ ); +}; diff --git a/frontend/app/src/components/v1/agent-prism/DetailsView/DetailsViewHeaderActions.tsx b/frontend/app/src/components/v1/agent-prism/DetailsView/DetailsViewHeaderActions.tsx new file mode 100644 index 0000000000..f4308ac9b8 --- /dev/null +++ b/frontend/app/src/components/v1/agent-prism/DetailsView/DetailsViewHeaderActions.tsx @@ -0,0 +1,23 @@ +import type { ReactNode } from 'react'; + +export interface DetailsViewHeaderActionsProps { + /** + * Custom actions to render in the header + */ + children?: ReactNode; + /** + * Optional className for the actions container + */ + className?: string; +} + +export const DetailsViewHeaderActions = ({ + children, + className = 'flex flex-wrap items-center gap-2', +}: DetailsViewHeaderActionsProps) => { + if (!children) { + return null; + } + + return
{children}
; +}; diff --git a/frontend/app/src/components/v1/agent-prism/DetailsView/DetailsViewInputOutputTab.tsx b/frontend/app/src/components/v1/agent-prism/DetailsView/DetailsViewInputOutputTab.tsx new file mode 100644 index 0000000000..cc2e1fccb4 --- /dev/null +++ b/frontend/app/src/components/v1/agent-prism/DetailsView/DetailsViewInputOutputTab.tsx @@ -0,0 +1,123 @@ +import { CollapsibleSection } from '../CollapsibleSection'; +import { TabSelector } from '../TabSelector'; +import type { TabItem } from '../Tabs'; +import { + DetailsViewContentViewer, + type DetailsViewContentViewMode, +} from './DetailsViewContentViewer'; +import type { TraceSpan } from '@evilmartians/agent-prism-types'; +import type { ReactElement } from 'react'; +import { useState, useEffect } from 'react'; + +interface DetailsViewInputOutputTabProps { + data: TraceSpan; +} + +type IOSection = 'Input' | 'Output'; + +export const DetailsViewInputOutputTab = ({ + data, +}: DetailsViewInputOutputTabProps): ReactElement => { + const hasInput = Boolean(data.input); + const hasOutput = Boolean(data.output); + + if (!hasInput && !hasOutput) { + return ( +
+

+ No input or output data available for this span +

+
+ ); + } + + let parsedInput: string | null = null; + let parsedOutput: string | null = null; + + if (typeof data.input === 'string') { + try { + parsedInput = JSON.parse(data.input); + } catch { + parsedInput = null; + } + } + + if (typeof data.output === 'string') { + try { + parsedOutput = JSON.parse(data.output); + } catch { + parsedOutput = null; + } + } + + return ( +
+ {typeof data.input === 'string' && ( + + )} + {typeof data.output === 'string' && ( + + )} +
+ ); +}; + +interface IOSectionProps { + section: IOSection; + content: string; + parsedContent: string | null; +} + +const IOSectionView = ({ + section, + content, + parsedContent, +}: IOSectionProps): ReactElement => { + const [tab, setTab] = useState( + parsedContent ? 'json' : 'plain', + ); + + useEffect(() => { + if (tab === 'json' && !parsedContent) { + setTab('plain'); + } + }, [tab, parsedContent]); + + const tabItems: TabItem[] = [ + { value: 'json', label: 'JSON', disabled: !parsedContent }, + { value: 'plain', label: 'Plain' }, + ]; + + return ( + + items={tabItems} + defaultValue={parsedContent ? 'json' : 'plain'} + value={tab} + onValueChange={setTab} + theme="pill" + onClick={(event) => event.stopPropagation()} + /> + } + > + + + ); +}; diff --git a/frontend/app/src/components/v1/agent-prism/DetailsView/DetailsViewJsonOutput.tsx b/frontend/app/src/components/v1/agent-prism/DetailsView/DetailsViewJsonOutput.tsx new file mode 100644 index 0000000000..4ca5995d87 --- /dev/null +++ b/frontend/app/src/components/v1/agent-prism/DetailsView/DetailsViewJsonOutput.tsx @@ -0,0 +1,28 @@ +import { agentPrismPrefix } from '../theme'; +import { type FC } from 'react'; +import JSONPretty from 'react-json-pretty'; + +export interface JsonViewerProps { + content: string; + id: string; + className?: string; +} + +export const DetailsViewJsonOutput: FC = ({ + content, + id, + className = '', +}) => { + return ( + + ); +}; diff --git a/frontend/app/src/components/v1/agent-prism/DetailsView/DetailsViewRawDataTab.tsx b/frontend/app/src/components/v1/agent-prism/DetailsView/DetailsViewRawDataTab.tsx new file mode 100644 index 0000000000..0384459873 --- /dev/null +++ b/frontend/app/src/components/v1/agent-prism/DetailsView/DetailsViewRawDataTab.tsx @@ -0,0 +1,29 @@ +import { CopyButton } from '../CopyButton'; +import { DetailsViewJsonOutput } from './DetailsViewJsonOutput'; +import type { TraceSpan } from '@evilmartians/agent-prism-types'; +import { type ReactElement } from 'react'; + +interface RawDataTabProps { + data: TraceSpan; +} + +export const DetailsViewRawDataTab = ({ + data, +}: RawDataTabProps): ReactElement => ( +
+
+
+
+ +
+
+ +
+ +
+
+
+); diff --git a/frontend/app/src/components/v1/agent-prism/IconButton.tsx b/frontend/app/src/components/v1/agent-prism/IconButton.tsx new file mode 100644 index 0000000000..036eaa9a9f --- /dev/null +++ b/frontend/app/src/components/v1/agent-prism/IconButton.tsx @@ -0,0 +1,73 @@ +import type { ComponentSize } from './shared'; +import cn from 'classnames'; +import type { ComponentPropsWithRef } from 'react'; + +type IconButtonSize = Extract< + ComponentSize, + '6' | '7' | '8' | '9' | '10' | '11' | '12' | '16' +>; +type IconButtonVariant = 'default' | 'ghost'; + +export type IconButtonProps = ComponentPropsWithRef<'button'> & { + /** + * The size of the icon button + */ + size?: IconButtonSize; + + /** + * The visual variant of the icon button + */ + variant?: IconButtonVariant; + + /** + * Accessible label for screen readers + * Required for accessibility compliance + */ + 'aria-label': string; +}; + +const sizeClasses: Record = { + '6': 'h-6 min-h-6', + '7': 'h-7 min-h-7', + '8': 'h-8 min-h-8', + '9': 'h-9 min-h-9', + '10': 'h-10 min-h-10', + '11': 'h-11 min-h-11', + '12': 'h-12 min-h-12', + '16': 'h-16 min-h-16', +}; + +const variantClasses: Record = { + default: 'border border-agentprism-border bg-transparent', + ghost: 'bg-transparent', +}; + +// TODO: Remake to call Icon component directly instead of passing children +export const IconButton = ({ + children, + className, + size = '6', + variant = 'default', + type = 'button', + 'aria-label': ariaLabel, + ...rest +}: IconButtonProps) => { + return ( + + ); +}; diff --git a/frontend/app/src/components/v1/agent-prism/PriceBadge.tsx b/frontend/app/src/components/v1/agent-prism/PriceBadge.tsx new file mode 100644 index 0000000000..d4390e21da --- /dev/null +++ b/frontend/app/src/components/v1/agent-prism/PriceBadge.tsx @@ -0,0 +1,12 @@ +import type { BadgeProps } from './Badge'; +import { Badge } from './Badge'; +import type { ComponentPropsWithRef } from 'react'; + +export type PriceBadgeProps = ComponentPropsWithRef<'span'> & { + cost: number; + size?: BadgeProps['size']; +}; + +export const PriceBadge = ({ cost, size, ...rest }: PriceBadgeProps) => { + return ; +}; diff --git a/frontend/app/src/components/v1/agent-prism/SearchInput.tsx b/frontend/app/src/components/v1/agent-prism/SearchInput.tsx new file mode 100644 index 0000000000..f5cbd5b0d5 --- /dev/null +++ b/frontend/app/src/components/v1/agent-prism/SearchInput.tsx @@ -0,0 +1,16 @@ +import { TextInput, type TextInputProps } from './TextInput'; +import { Search } from 'lucide-react'; + +/** + * A simple wrapper around the TextInput component. + * It adds a search icon and a placeholder. + */ +export const SearchInput = ({ ...props }: TextInputProps) => { + return ( + } + placeholder="Filter..." + {...props} + /> + ); +}; diff --git a/frontend/app/src/components/v1/agent-prism/SpanBadge.tsx b/frontend/app/src/components/v1/agent-prism/SpanBadge.tsx new file mode 100644 index 0000000000..01d15c8185 --- /dev/null +++ b/frontend/app/src/components/v1/agent-prism/SpanBadge.tsx @@ -0,0 +1,50 @@ +import { Badge, type BadgeProps } from './Badge'; +import { getSpanCategoryIcon, getSpanCategoryLabel } from './shared'; +import type { TraceSpanCategory } from '@evilmartians/agent-prism-types'; +import cn from 'classnames'; + +export interface SpanBadgeProps + extends Omit { + category: TraceSpanCategory; +} + +const badgeClasses: Record = { + llm_call: 'bg-agentprism-badge-llm text-agentprism-badge-llm-foreground', + tool_execution: + 'bg-agentprism-badge-tool text-agentprism-badge-tool-foreground', + chain_operation: + 'bg-agentprism-badge-chain text-agentprism-badge-chain-foreground', + retrieval: + 'bg-agentprism-badge-retrieval text-agentprism-badge-retrieval-foreground', + embedding: + 'bg-agentprism-badge-embedding text-agentprism-badge-embedding-foreground', + guardrail: + 'bg-agentprism-badge-guardrail text-agentprism-badge-guardrail-foreground', + agent_invocation: + 'bg-agentprism-badge-agent text-agentprism-badge-agent-foreground', + create_agent: + 'bg-agentprism-badge-create-agent text-agentprism-badge-create-agent-foreground', + span: 'bg-agentprism-badge-span text-agentprism-badge-span-foreground', + event: 'bg-agentprism-badge-event text-agentprism-badge-event-foreground', + unknown: + 'bg-agentprism-badge-unknown text-agentprism-badge-unknown-foreground', +}; + +export const SpanBadge = ({ + category, + className, + ...props +}: SpanBadgeProps) => { + const Icon = getSpanCategoryIcon(category); + const label = getSpanCategoryLabel(category); + + return ( + } + {...props} + label={label} + unstyled + /> + ); +}; diff --git a/frontend/app/src/components/v1/agent-prism/SpanCard/SpanCard.tsx b/frontend/app/src/components/v1/agent-prism/SpanCard/SpanCard.tsx new file mode 100644 index 0000000000..083a410583 --- /dev/null +++ b/frontend/app/src/components/v1/agent-prism/SpanCard/SpanCard.tsx @@ -0,0 +1,483 @@ +import type { AvatarProps } from '../Avatar'; +import { Avatar } from '../Avatar'; +import { BrandLogo } from '../BrandLogo'; +import { SpanStatus } from '../SpanStatus'; +import { SpanCardBadges } from './SpanCardBadges'; +import type { SpanCardConnectorType } from './SpanCardConnector'; +import { SpanCardConnector } from './SpanCardConnector'; +import { SpanCardTimeline } from './SpanCardTimeline'; +import { SpanCardToggle } from './SpanCardToggle'; +import { + formatDuration, + getTimelineData, +} from '@evilmartians/agent-prism-data'; +import type { TraceSpan } from '@evilmartians/agent-prism-types'; +import * as Collapsible from '@radix-ui/react-collapsible'; +import cn from 'classnames'; +import type { FC, KeyboardEvent, MouseEvent } from 'react'; +import { useCallback } from 'react'; + +const LAYOUT_CONSTANTS = { + CONNECTOR_WIDTH: 20, + CONTENT_BASE_WIDTH: 320, +} as const; + +type ExpandButtonPlacement = 'inside' | 'outside'; + +export type SpanCardViewOptions = { + withStatus?: boolean; + expandButton?: ExpandButtonPlacement; +}; + +const DEFAULT_VIEW_OPTIONS: Required = { + withStatus: true, + expandButton: 'inside', +}; + +interface SpanCardProps { + data: TraceSpan; + level?: number; + selectedSpan?: TraceSpan; + avatar?: AvatarProps; + onSpanSelect?: (span: TraceSpan) => void; + minStart: number; + maxEnd: number; + isLastChild: boolean; + prevLevelConnectors?: SpanCardConnectorType[]; + expandedSpansIds: string[]; + onExpandSpansIdsChange: (ids: string[]) => void; + viewOptions?: SpanCardViewOptions; +} + +interface SpanCardState { + isExpanded: boolean; + hasChildren: boolean; + isSelected: boolean; +} + +const getContentWidth = ({ + level, + hasExpandButton, + contentPadding, + expandButton, +}: { + level: number; + hasExpandButton: boolean; + contentPadding: number; + expandButton: ExpandButtonPlacement; +}) => { + let width = + LAYOUT_CONSTANTS.CONTENT_BASE_WIDTH - + level * LAYOUT_CONSTANTS.CONNECTOR_WIDTH; + + if (hasExpandButton && expandButton === 'inside') { + width -= LAYOUT_CONSTANTS.CONNECTOR_WIDTH; + } + + if (expandButton === 'outside' && level === 0) { + width -= LAYOUT_CONSTANTS.CONNECTOR_WIDTH; + } + + return width - contentPadding; +}; + +const getGridTemplateColumns = ({ + connectorsColumnWidth, + expandButton, +}: { + connectorsColumnWidth: number; + expandButton: ExpandButtonPlacement; +}) => { + if (expandButton === 'inside') { + return `${connectorsColumnWidth}px 1fr`; + } + + return `${connectorsColumnWidth}px 1fr ${LAYOUT_CONSTANTS.CONNECTOR_WIDTH}px`; +}; + +const getContentPadding = ({ + level, + hasExpandButton, +}: { + level: number; + hasExpandButton: boolean; +}) => { + if (level === 0) { + return 0; + } + + if (hasExpandButton) { + return 4; + } + + return 8; +}; + +const getConnectorsLayout = ({ + level, + hasExpandButton, + isLastChild, + prevConnectors, + expandButton, +}: { + hasExpandButton: boolean; + isLastChild: boolean; + level: number; + prevConnectors: SpanCardConnectorType[]; + expandButton: ExpandButtonPlacement; +}): { + connectors: SpanCardConnectorType[]; + connectorsColumnWidth: number; +} => { + const connectors: SpanCardConnectorType[] = []; + + if (level === 0) { + return { + connectors: expandButton === 'inside' ? [] : ['vertical'], + connectorsColumnWidth: 20, + }; + } + + for (let i = 0; i < level - 1; i++) { + connectors.push('vertical'); + } + + if (!isLastChild) { + connectors.push('t-right'); + } + + if (isLastChild) { + connectors.push('corner-top-right'); + } + + let connectorsColumnWidth = + connectors.length * LAYOUT_CONSTANTS.CONNECTOR_WIDTH; + + if (hasExpandButton) { + connectorsColumnWidth += LAYOUT_CONSTANTS.CONNECTOR_WIDTH; + } + + for (let i = 0; i < prevConnectors.length; i++) { + if ( + prevConnectors[i] === 'empty' || + prevConnectors[i] === 'corner-top-right' + ) { + connectors[i] = 'empty'; + } + } + + return { + connectors, + connectorsColumnWidth, + }; +}; + +const useSpanCardEventHandlers = ( + data: TraceSpan, + onSpanSelect?: (span: TraceSpan) => void, +) => { + const handleCardClick = useCallback((): void => { + onSpanSelect?.(data); + }, [data, onSpanSelect]); + + const handleKeyDown = useCallback( + (e: KeyboardEvent): void => { + if (e.key === 'Enter' || e.key === ' ') { + e.preventDefault(); + handleCardClick(); + } + }, + [handleCardClick], + ); + + const handleToggleClick = useCallback( + (e: MouseEvent | KeyboardEvent): void => { + e.stopPropagation(); + }, + [], + ); + + return { + handleCardClick, + handleKeyDown, + handleToggleClick, + }; +}; + +const SpanCardChildren: FC<{ + data: TraceSpan; + level: number; + selectedSpan?: TraceSpan; + onSpanSelect?: (span: TraceSpan) => void; + minStart: number; + maxEnd: number; + prevLevelConnectors: SpanCardConnectorType[]; + expandedSpansIds: string[]; + onExpandSpansIdsChange: (ids: string[]) => void; + viewOptions?: SpanCardViewOptions; +}> = ({ + data, + level, + selectedSpan, + onSpanSelect, + minStart, + maxEnd, + prevLevelConnectors, + expandedSpansIds, + onExpandSpansIdsChange, + viewOptions = DEFAULT_VIEW_OPTIONS, +}) => { + if (!data.children?.length) { + return null; + } + + return ( +
+ +
    + {data.children.map((child, idx) => { + const brand = child.metadata?.brand as { type: string } | undefined; + + return ( + , + size: '4', + rounded: 'sm', + category: child.type, + } + : undefined + } + /> + ); + })} +
+ +
+ ); +}; + +export const SpanCard: FC = ({ + data, + level = 0, + selectedSpan, + onSpanSelect, + viewOptions = DEFAULT_VIEW_OPTIONS, + avatar, + minStart, + maxEnd, + isLastChild, + prevLevelConnectors = [], + expandedSpansIds, + onExpandSpansIdsChange, +}) => { + const isExpanded = expandedSpansIds.includes(data.id); + + const withStatus = viewOptions.withStatus ?? DEFAULT_VIEW_OPTIONS.withStatus; + const expandButton = + viewOptions.expandButton || DEFAULT_VIEW_OPTIONS.expandButton; + + const handleToggleClick = useCallback( + (expanded: boolean) => { + const alreadyExpanded = expandedSpansIds.includes(data.id); + + if (alreadyExpanded && !expanded) { + onExpandSpansIdsChange(expandedSpansIds.filter((id) => id !== data.id)); + } + + if (!alreadyExpanded && expanded) { + onExpandSpansIdsChange([...expandedSpansIds, data.id]); + } + }, + [expandedSpansIds, data.id, onExpandSpansIdsChange], + ); + + const state: SpanCardState = { + isExpanded, + hasChildren: Boolean(data.children?.length), + isSelected: selectedSpan?.id === data.id, + }; + + const eventHandlers = useSpanCardEventHandlers(data, onSpanSelect); + + const { durationMs } = getTimelineData({ + spanCard: data, + minStart, + maxEnd, + }); + + const hasExpandButtonAsFirstChild = + expandButton === 'inside' && state.hasChildren; + + const contentPadding = getContentPadding({ + level, + hasExpandButton: hasExpandButtonAsFirstChild, + }); + + const contentWidth = getContentWidth({ + level, + hasExpandButton: hasExpandButtonAsFirstChild, + contentPadding, + expandButton, + }); + + const { connectors, connectorsColumnWidth } = getConnectorsLayout({ + level, + hasExpandButton: hasExpandButtonAsFirstChild, + isLastChild, + prevConnectors: prevLevelConnectors, + expandButton, + }); + + const gridTemplateColumns = getGridTemplateColumns({ + connectorsColumnWidth, + expandButton, + }); + + return ( +
  • + +
    +
    + {connectors.map((connector, idx) => ( + + ))} + + {hasExpandButtonAsFirstChild && ( +
    + + + {state.isExpanded && } +
    + )} +
    +
    +
    + {avatar && } + +

    + {data.title} +

    + + +
    + +
    + {expandButton === 'outside' && withStatus && ( +
    + +
    + )} + + + +
    + + {formatDuration(durationMs)} + + + {expandButton === 'inside' && withStatus && ( +
    + +
    + )} +
    +
    +
    + + {expandButton === 'outside' && + (state.hasChildren ? ( + + ) : ( +
    + ))} +
    + + + +
    • + ); +}; diff --git a/frontend/app/src/components/v1/agent-prism/SpanCard/SpanCardBadges.tsx b/frontend/app/src/components/v1/agent-prism/SpanCard/SpanCardBadges.tsx new file mode 100644 index 0000000000..02495c57a9 --- /dev/null +++ b/frontend/app/src/components/v1/agent-prism/SpanCard/SpanCardBadges.tsx @@ -0,0 +1,14 @@ +import { SpanBadge } from '../SpanBadge'; +import type { TraceSpan } from '@evilmartians/agent-prism-types'; + +interface SpanCardBagdesProps { + data: TraceSpan; +} + +export const SpanCardBadges = ({ data }: SpanCardBagdesProps) => { + return ( +
    + +
    + ); +}; diff --git a/frontend/app/src/components/v1/agent-prism/SpanCard/SpanCardConnector.tsx b/frontend/app/src/components/v1/agent-prism/SpanCard/SpanCardConnector.tsx new file mode 100644 index 0000000000..ade076f212 --- /dev/null +++ b/frontend/app/src/components/v1/agent-prism/SpanCard/SpanCardConnector.tsx @@ -0,0 +1,38 @@ +export type SpanCardConnectorType = + | 'horizontal' + | 'vertical' + | 't-right' + | 'corner-top-right' + | 'empty'; + +interface SpanCardConnectorProps { + type: SpanCardConnectorType; +} + +export const SpanCardConnector = ({ type }: SpanCardConnectorProps) => { + if (type === 'empty') { + return
    ; + } + + return ( +
    + {(type === 'vertical' || type === 't-right') && ( +
    + )} + + {type === 't-right' && ( +
    + )} + + {type === 'corner-top-right' && ( + <> +
    + +
    + +
    + + )} +
    + ); +}; diff --git a/frontend/app/src/components/v1/agent-prism/SpanCard/SpanCardTimeline.tsx b/frontend/app/src/components/v1/agent-prism/SpanCard/SpanCardTimeline.tsx new file mode 100644 index 0000000000..b803072603 --- /dev/null +++ b/frontend/app/src/components/v1/agent-prism/SpanCard/SpanCardTimeline.tsx @@ -0,0 +1,59 @@ +import { getTimelineData } from '@evilmartians/agent-prism-data'; +import type { + TraceSpan, + TraceSpanCategory, +} from '@evilmartians/agent-prism-types'; +import cn from 'classnames'; + +interface SpanCardTimelineProps { + spanCard: TraceSpan; + minStart: number; + maxEnd: number; + className?: string; +} + +const timelineBgColors: Record = { + llm_call: 'bg-agentprism-timeline-llm', + agent_invocation: 'bg-agentprism-timeline-agent', + tool_execution: 'bg-agentprism-timeline-tool', + chain_operation: 'bg-agentprism-timeline-chain', + retrieval: 'bg-agentprism-timeline-retrieval', + embedding: 'bg-agentprism-timeline-embedding', + guardrail: 'bg-agentprism-timeline-guardrail', + create_agent: 'bg-agentprism-timeline-create-agent', + span: 'bg-agentprism-timeline-span', + event: 'bg-agentprism-timeline-event', + unknown: 'bg-agentprism-timeline-unknown', +}; + +export const SpanCardTimeline = ({ + spanCard, + minStart, + maxEnd, + className, +}: SpanCardTimelineProps) => { + const { startPercent, widthPercent } = getTimelineData({ + spanCard, + minStart, + maxEnd, + }); + + return ( + + + + + + ); +}; diff --git a/frontend/app/src/components/v1/agent-prism/SpanCard/SpanCardToggle.tsx b/frontend/app/src/components/v1/agent-prism/SpanCard/SpanCardToggle.tsx new file mode 100644 index 0000000000..9333fe52bf --- /dev/null +++ b/frontend/app/src/components/v1/agent-prism/SpanCard/SpanCardToggle.tsx @@ -0,0 +1,38 @@ +import * as Collapsible from '@radix-ui/react-collapsible'; +import { ChevronDown, ChevronRight } from 'lucide-react'; +import type { KeyboardEvent, MouseEvent } from 'react'; + +interface SpanCardToggleProps { + isExpanded: boolean; + title: string; + onToggleClick: (e: MouseEvent | KeyboardEvent) => void; +} + +export const SpanCardToggle = ({ + isExpanded, + title, + onToggleClick, +}: SpanCardToggleProps) => ( + + + +); diff --git a/frontend/app/src/components/v1/agent-prism/SpanStatus.tsx b/frontend/app/src/components/v1/agent-prism/SpanStatus.tsx new file mode 100644 index 0000000000..895b00d96c --- /dev/null +++ b/frontend/app/src/components/v1/agent-prism/SpanStatus.tsx @@ -0,0 +1,79 @@ +import type { TraceSpanStatus } from '@evilmartians/agent-prism-types'; +import cn from 'classnames'; +import { Check, Ellipsis, Info, TriangleAlert } from 'lucide-react'; +import type { ComponentPropsWithRef } from 'react'; + +type StatusVariant = 'dot' | 'badge'; + +export type StatusProps = ComponentPropsWithRef<'div'> & { + status: TraceSpanStatus; + variant?: StatusVariant; +}; + +const STATUS_COLORS_DOT: Record = { + success: 'bg-agentprism-success', + error: 'bg-agentprism-error', + pending: 'bg-agentprism-pending', + warning: 'bg-agentprism-warning', +}; + +const STATUS_COLORS_BADGE: Record = { + success: + 'bg-agentprism-success-muted text-agentprism-success-muted-foreground', + error: 'bg-agentprism-error-muted text-agentprism-error-muted-foreground', + pending: + 'bg-agentprism-pending-muted text-agentprism-pending-muted-foreground', + warning: + 'bg-agentprism-warning-muted text-agentprism-warning-muted-foreground', +}; + +export const SpanStatus = ({ + status, + variant = 'dot', + ...rest +}: StatusProps) => { + const title = `Status: ${status}`; + + return ( +
    + {variant === 'dot' ? ( + + ) : ( + + )} +
    + ); +}; + +interface StatusWithTitleProps extends StatusProps { + title: string; +} + +const SpanStatusDot = ({ status, title }: StatusWithTitleProps) => { + return ( + + ); +}; + +const SpanStatusBadge = ({ status, title }: StatusWithTitleProps) => { + return ( + + {status === 'success' && } + {status === 'error' && } + {status === 'warning' && } + {status === 'pending' && } + + ); +}; diff --git a/frontend/app/src/components/v1/agent-prism/TabSelector.tsx b/frontend/app/src/components/v1/agent-prism/TabSelector.tsx new file mode 100644 index 0000000000..054c18a5c8 --- /dev/null +++ b/frontend/app/src/components/v1/agent-prism/TabSelector.tsx @@ -0,0 +1,34 @@ +import { type TabItem, Tabs } from './Tabs'; +import { type ReactElement } from 'react'; + +export interface TabSelectorProps { + items: TabItem[]; + value: T; + onValueChange: (value: T) => void; + defaultValue?: T; + theme?: 'underline' | 'pill'; + className?: string; + onClick?: (event: React.MouseEvent) => void; +} + +export const TabSelector = ({ + items, + value, + onValueChange, + defaultValue, + theme = 'underline', + className, + onClick, +}: TabSelectorProps): ReactElement => { + return ( + + items={items} + value={value} + onValueChange={onValueChange} + defaultValue={defaultValue} + theme={theme} + className={className} + onClick={onClick} + /> + ); +}; diff --git a/frontend/app/src/components/v1/agent-prism/Tabs.tsx b/frontend/app/src/components/v1/agent-prism/Tabs.tsx new file mode 100644 index 0000000000..182a088bf8 --- /dev/null +++ b/frontend/app/src/components/v1/agent-prism/Tabs.tsx @@ -0,0 +1,139 @@ +import * as RadixTabs from '@radix-ui/react-tabs'; +import cn from 'classnames'; +import type { ComponentPropsWithRef } from 'react'; +import * as React from 'react'; + +export interface TabItem { + value: T; + label: string; + icon?: React.ReactNode; + disabled?: boolean; +} + +export type TabTheme = 'underline' | 'pill'; + +const BASE_TRIGGER = + 'text-sm font-medium transition-colors duration-200 disabled:opacity-50 disabled:cursor-not-allowed'; + +const THEMES = { + underline: { + list: 'h-9 flex border-b border-agentprism-border', + trigger: `w-full justify-center px-3 ${BASE_TRIGGER} + text-agentprism-secondary-foreground data-[state=active]:text-agentprism-foreground + border-b-2 border-transparent data-[state=active]:border-agentprism-border-inverse + -mb-[2px] + data-[state=inactive]:[&:not(:disabled)]:hover:border-agentprism-border-inverse/20 + data-[state=inactive]:[&:not(:disabled)]:hover:text-agentprism-muted-foreground`, + }, + pill: { + list: 'h-9 inline-flex gap-1 p-1 bg-agentprism-secondary rounded-lg', + trigger: `px-3 ${BASE_TRIGGER} rounded-md + text-agentprism-muted-foreground data-[state=active]:text-agentprism-foreground + data-[state=inactive]:[&:not(:disabled)]:hover:bg-agentprism-background/50 data-[state=active]:bg-agentprism-background data-[state=active]:shadow-sm + dark:data-[state=active]:shadow-none`, + }, +} as const; + +export type TabsProps = Omit< + ComponentPropsWithRef<'div'>, + 'dir' +> & { + /** + * Array of tab items to display + */ + items: TabItem[]; + + /** + * The initially selected tab value (uncontrolled) + */ + defaultValue?: T; + + /** + * The currently selected tab value (controlled) + */ + value?: T; + + /** + * Callback fired when the selected tab changes + */ + onValueChange?: (value: T) => void; + + /** + * Visual theme variant for the tabs + * @default "underline" + */ + theme?: TabTheme; + + /** + * Optional className for the root container + */ + className?: string; + + /** + * Optional className for the tabs list container + */ + tabsListClassName?: string; + + /** + * Optional className for individual tab triggers + */ + triggerClassName?: string; + + /** + * The direction of the content of the tabs + */ + dir?: 'ltr' | 'rtl'; +}; + +export const Tabs = ({ + items, + defaultValue, + value, + onValueChange, + theme = 'underline', + className = '', + tabsListClassName = '', + triggerClassName = '', + dir, + ...rest +}: TabsProps) => { + const defaultTab = defaultValue || items[0]?.value; + + const currentTheme = THEMES[theme]; + + return ( + void} + dir={dir} + {...rest} + > + + {items.map((item: TabItem) => ( + + {item.icon && ( + + {item.icon} + + )} + {item.label} + + ))} + + + ); +}; diff --git a/frontend/app/src/components/v1/agent-prism/TextInput.tsx b/frontend/app/src/components/v1/agent-prism/TextInput.tsx new file mode 100644 index 0000000000..5366cf88bc --- /dev/null +++ b/frontend/app/src/components/v1/agent-prism/TextInput.tsx @@ -0,0 +1,143 @@ +import cn from 'classnames'; +import { X } from 'lucide-react'; +import { + useRef, + type ChangeEvent, + type ComponentPropsWithRef, + type ReactNode, + type RefObject, +} from 'react'; + +export type TextInputProps = ComponentPropsWithRef<'input'> & { + /** + * Callback fired when the input value changes + */ + onValueChange?: (value: string) => void; + + /** + * Icon to display at the start of the input + */ + startIcon?: ReactNode; + + /** + * Callback fired when the clear button is clicked. If this callback is provided, + * the clear button will be shown. + */ + onClear?: () => void; + + /** + * Ref to the input element + */ + ref?: RefObject; + + /** + * Optional className for the input element + */ + inputClassName?: string; + + /** + * Unique identifier for the input (required) + */ + id: string; + + /** + * Label text for the input + */ + label?: string; + + /** + * Whether to visually hide the label while keeping it for screen readers + * @default false + */ + hideLabel?: boolean; +}; + +const iconBaseClassName = + 'absolute top-1/2 -translate-y-1/2 flex items-center justify-center text-agentprism-muted-foreground'; + +export const TextInput = ({ + className, + onChange, + onValueChange, + startIcon, + onClear, + ref, + inputClassName, + label, + hideLabel = false, + id, + ...rest +}: TextInputProps) => { + const inputRef = useRef(null); + + const handleChange = (e: ChangeEvent) => { + onChange?.(e); + onValueChange?.(e.target.value); + }; + + const handleClear = () => { + onClear?.(); + + if (ref) { + ref.current?.focus(); + return; + } + + inputRef.current?.focus(); + }; + + return ( +
    + {label && ( + + )} +
    + + {startIcon && ( +
    + {startIcon} +
    + )} + {onClear && rest.value && ( + + )} +
    +
    + ); +}; diff --git a/frontend/app/src/components/v1/agent-prism/TimestampBadge.tsx b/frontend/app/src/components/v1/agent-prism/TimestampBadge.tsx new file mode 100644 index 0000000000..91847947d1 --- /dev/null +++ b/frontend/app/src/components/v1/agent-prism/TimestampBadge.tsx @@ -0,0 +1,20 @@ +import type { BadgeProps } from './Badge'; +import { Badge } from './Badge'; +import type { ComponentPropsWithRef } from 'react'; + +export type TimestampBadgeProps = ComponentPropsWithRef<'span'> & { + timestamp: number; + size?: BadgeProps['size']; +}; + +export const TimestampBadge = ({ + timestamp, + size, + ...rest +}: TimestampBadgeProps) => { + return ; +}; + +function formatTimestamp(timestamp: number): string { + return new Date(timestamp).toLocaleString(); +} diff --git a/frontend/app/src/components/v1/agent-prism/TokensBadge.tsx b/frontend/app/src/components/v1/agent-prism/TokensBadge.tsx new file mode 100644 index 0000000000..8bbe81c275 --- /dev/null +++ b/frontend/app/src/components/v1/agent-prism/TokensBadge.tsx @@ -0,0 +1,24 @@ +import type { BadgeProps } from './Badge'; +import { Badge } from './Badge'; +import { Coins } from 'lucide-react'; +import type { ComponentPropsWithRef } from 'react'; + +export type TokensBadgeProps = ComponentPropsWithRef<'span'> & { + tokensCount: number; + size?: BadgeProps['size']; +}; + +export const TokensBadge = ({ + tokensCount, + size, + ...rest +}: TokensBadgeProps) => { + return ( + } + size={size} + {...rest} + label={tokensCount} + /> + ); +}; diff --git a/frontend/app/src/components/v1/agent-prism/TraceList/TraceList.tsx b/frontend/app/src/components/v1/agent-prism/TraceList/TraceList.tsx new file mode 100644 index 0000000000..b803cd87ea --- /dev/null +++ b/frontend/app/src/components/v1/agent-prism/TraceList/TraceList.tsx @@ -0,0 +1,84 @@ +import type { BadgeProps } from '../Badge'; +import { Badge } from '../Badge'; +import { IconButton } from '../IconButton'; +import { TraceListItem } from './TraceListItem'; +import type { TraceRecord } from '@evilmartians/agent-prism-types'; +import cn from 'classnames'; +import { ArrowLeft } from 'lucide-react'; + +type TraceRecordWithBadges = TraceRecord & { + badges?: Array; +}; + +type TraceListProps = { + traces: TraceRecordWithBadges[]; + expanded: boolean; + onExpandStateChange: (expanded: boolean) => void; + className?: string; + onTraceSelect?: (trace: TraceRecord) => void; + selectedTrace?: TraceRecord; +}; + +export const TraceList = ({ + traces, + expanded, + onExpandStateChange, + className, + onTraceSelect, + selectedTrace, +}: TraceListProps) => { + return ( +
    +
    +
    +

    Traces

    + + +
    + + onExpandStateChange(!expanded)} + > + + +
    + + {expanded && ( +
      +
      + {traces.map((trace) => ( +
    • + onTraceSelect?.(trace)} + isSelected={selectedTrace?.id === trace.id} + badges={trace.badges} + /> +
      • + ))} +
      +
    + )} +
    + ); +}; diff --git a/frontend/app/src/components/v1/agent-prism/TraceList/TraceListItem.tsx b/frontend/app/src/components/v1/agent-prism/TraceList/TraceListItem.tsx new file mode 100644 index 0000000000..ffa37f93ee --- /dev/null +++ b/frontend/app/src/components/v1/agent-prism/TraceList/TraceListItem.tsx @@ -0,0 +1,83 @@ +import type { AvatarProps } from '../Avatar'; +import type { BadgeProps } from '../Badge'; +import { Badge } from '../Badge'; +import { PriceBadge } from '../PriceBadge'; +import { TimestampBadge } from '../TimestampBadge'; +import { TokensBadge } from '../TokensBadge'; +import { TraceListItemHeader } from './TraceListItemHeader'; +import type { TraceRecord } from '@evilmartians/agent-prism-types'; +import cn from 'classnames'; +import type { KeyboardEvent } from 'react'; +import { useCallback } from 'react'; + +interface TraceListItemProps { + trace: TraceRecord; + badges?: Array; + avatar?: AvatarProps; + onClick?: () => void; + isSelected?: boolean; + showDescription?: boolean; +} + +export const TraceListItem = ({ + trace, + avatar, + onClick, + badges, + isSelected, + showDescription = true, +}: TraceListItemProps) => { + const handleKeyDown = useCallback( + (e: KeyboardEvent): void => { + if (e.key === 'Enter' || e.key === ' ') { + e.preventDefault(); + onClick?.(); + } + }, + [onClick], + ); + + const { name, agentDescription, totalCost, totalTokens, startTime } = trace; + + return ( +
    + + +
    + {showDescription && ( + + {agentDescription} + + )} + + {typeof totalCost === 'number' && } + + {typeof totalTokens === 'number' && ( + + )} + + {badges?.map((badge, index) => ( + + ))} + + {typeof startTime === 'number' && ( + + )} +
    +
    + ); +}; diff --git a/frontend/app/src/components/v1/agent-prism/TraceList/TraceListItemHeader.tsx b/frontend/app/src/components/v1/agent-prism/TraceList/TraceListItemHeader.tsx new file mode 100644 index 0000000000..c1f710e651 --- /dev/null +++ b/frontend/app/src/components/v1/agent-prism/TraceList/TraceListItemHeader.tsx @@ -0,0 +1,35 @@ +import type { AvatarProps } from '../Avatar'; +import { Avatar } from '../Avatar'; +import { Badge } from '../Badge'; +import type { TraceRecord } from '@evilmartians/agent-prism-types'; + +interface TraceListItemHeaderProps { + trace: TraceRecord; + avatar?: AvatarProps; +} + +export const TraceListItemHeader = ({ + trace, + avatar, +}: TraceListItemHeaderProps) => { + return ( +
    +
    + {avatar && } + +

    + {trace.name} +

    +
    + +
    + +
    +
    + ); +}; diff --git a/frontend/app/src/components/v1/agent-prism/TraceViewer/TraceViewer.tsx b/frontend/app/src/components/v1/agent-prism/TraceViewer/TraceViewer.tsx new file mode 100644 index 0000000000..8072a39aa7 --- /dev/null +++ b/frontend/app/src/components/v1/agent-prism/TraceViewer/TraceViewer.tsx @@ -0,0 +1,169 @@ +import { type BadgeProps } from '../Badge'; +import { type SpanCardViewOptions } from '../SpanCard/SpanCard'; +import { useIsMobile, useIsMounted } from '../shared'; +import { TraceViewerDesktopLayout } from './TraceViewerDesktopLayout'; +import { TraceViewerMobileLayout } from './TraceViewerMobileLayout'; +import { + filterSpansRecursively, + flattenSpans, +} from '@evilmartians/agent-prism-data'; +import type { TraceRecord, TraceSpan } from '@evilmartians/agent-prism-types'; +import { useCallback, useEffect, useMemo, useState } from 'react'; + +export interface TraceViewerData { + traceRecord: TraceRecord; + badges?: Array; + spans: TraceSpan[]; + spanCardViewOptions?: SpanCardViewOptions; +} + +export interface TraceViewerProps { + data: Array; + spanCardViewOptions?: SpanCardViewOptions; +} + +export const TraceViewer = ({ + data, + spanCardViewOptions, +}: TraceViewerProps) => { + const isMobile = useIsMobile(); + const isMounted = useIsMounted(); + + const [selectedSpan, setSelectedSpan] = useState(); + const [searchValue, setSearchValue] = useState(''); + const [traceListExpanded, setTraceListExpanded] = useState(true); + + const [selectedTrace, setSelectedTrace] = useState< + TraceRecordWithDisplayData | undefined + >( + data[0] + ? { + ...data[0].traceRecord, + badges: data[0].badges, + spanCardViewOptions: data[0].spanCardViewOptions, + } + : undefined, + ); + const [selectedTraceSpans, setSelectedTraceSpans] = useState( + data[0]?.spans || [], + ); + + const traceRecords: TraceRecordWithDisplayData[] = useMemo(() => { + return data.map((item) => ({ + ...item.traceRecord, + badges: item.badges, + spanCardViewOptions: item.spanCardViewOptions, + })); + }, [data]); + + const filteredSpans = useMemo(() => { + if (!searchValue.trim()) { + return selectedTraceSpans; + } + return filterSpansRecursively(selectedTraceSpans, searchValue); + }, [selectedTraceSpans, searchValue]); + + const allIds = useMemo(() => { + return flattenSpans(selectedTraceSpans).map((span) => span.id); + }, [selectedTraceSpans]); + + const [expandedSpansIds, setExpandedSpansIds] = useState(allIds); + + useEffect(() => { + setExpandedSpansIds(allIds); + }, [allIds]); + + useEffect(() => { + if (!isMounted || isMobile) { + return; + } + + if (selectedTraceSpans.length > 0 && !selectedSpan) { + setSelectedSpan(selectedTraceSpans[0]); + } + }, [selectedTraceSpans, isMobile, isMounted, selectedSpan]); + + const handleExpandAll = useCallback(() => { + setExpandedSpansIds(allIds); + }, [allIds]); + + const handleCollapseAll = useCallback(() => { + setExpandedSpansIds([]); + }, []); + + const handleTraceSelect = useCallback( + (trace: TraceRecord) => { + setSelectedSpan(undefined); + setExpandedSpansIds([]); + setSelectedTrace(trace); + setSelectedTraceSpans( + data.find((item) => item.traceRecord.id === trace.id)?.spans ?? [], + ); + }, + [data], + ); + + const handleClearTraceSelection = useCallback(() => { + setSelectedTrace(undefined); + setSelectedTraceSpans([]); + setSelectedSpan(undefined); + setExpandedSpansIds([]); + }, []); + + const props: TraceViewerLayoutProps = { + traceRecords, + traceListExpanded, + setTraceListExpanded, + selectedTrace, + selectedTraceId: selectedTrace?.id, + selectedSpan, + setSelectedSpan, + searchValue, + setSearchValue, + filteredSpans, + expandedSpansIds, + setExpandedSpansIds, + handleExpandAll, + handleCollapseAll, + handleTraceSelect, + spanCardViewOptions: + spanCardViewOptions || selectedTrace?.spanCardViewOptions, + onClearTraceSelection: handleClearTraceSelection, + }; + + return ( +
    +
    + +
    +
    + +
    +
    + ); +}; + +export interface TraceRecordWithDisplayData extends TraceRecord { + spanCardViewOptions?: SpanCardViewOptions; + badges?: BadgeProps[]; +} + +export interface TraceViewerLayoutProps { + traceRecords: TraceRecordWithDisplayData[]; + traceListExpanded: boolean; + setTraceListExpanded: (expanded: boolean) => void; + selectedTrace: TraceRecordWithDisplayData | undefined; + selectedTraceId?: string; + selectedSpan: TraceSpan | undefined; + setSelectedSpan: (span: TraceSpan | undefined) => void; + searchValue: string; + setSearchValue: (value: string) => void; + filteredSpans: TraceSpan[]; + expandedSpansIds: string[]; + setExpandedSpansIds: (ids: string[]) => void; + handleExpandAll: () => void; + handleCollapseAll: () => void; + handleTraceSelect: (trace: TraceRecord) => void; + spanCardViewOptions?: SpanCardViewOptions; + onClearTraceSelection: () => void; +} diff --git a/frontend/app/src/components/v1/agent-prism/TraceViewer/TraceViewerDesktopLayout.tsx b/frontend/app/src/components/v1/agent-prism/TraceViewer/TraceViewerDesktopLayout.tsx new file mode 100644 index 0000000000..7cdeb60043 --- /dev/null +++ b/frontend/app/src/components/v1/agent-prism/TraceViewer/TraceViewerDesktopLayout.tsx @@ -0,0 +1,97 @@ +import { DetailsView } from '../DetailsView/DetailsView'; +import { TraceList } from '../TraceList/TraceList'; +import { type TraceViewerLayoutProps } from './TraceViewer'; +import { TraceViewerPlaceholder } from './TraceViewerPlaceholder'; +import { TraceViewerTreeViewContainer } from './TraceViewerTreeViewContainer'; +import { Panel, PanelGroup, PanelResizeHandle } from 'react-resizable-panels'; + +export const TraceViewerDesktopLayout = ({ + traceRecords, + traceListExpanded, + setTraceListExpanded, + selectedTrace, + selectedTraceId, + selectedSpan, + setSelectedSpan, + searchValue, + setSearchValue, + filteredSpans, + expandedSpansIds, + setExpandedSpansIds, + handleExpandAll, + handleCollapseAll, + handleTraceSelect, + spanCardViewOptions, +}: TraceViewerLayoutProps) => { + const actualSelectedTrace = + traceRecords.find((t) => t.id === selectedTraceId) || selectedTrace; + + return ( + + + + + + + + {selectedTrace ? ( + + + + ) : ( + + + + )} + + + + + {selectedSpan ? ( + + ) : ( + + )} + + + ); +}; diff --git a/frontend/app/src/components/v1/agent-prism/TraceViewer/TraceViewerMobileLayout.tsx b/frontend/app/src/components/v1/agent-prism/TraceViewer/TraceViewerMobileLayout.tsx new file mode 100644 index 0000000000..3e7fff99f9 --- /dev/null +++ b/frontend/app/src/components/v1/agent-prism/TraceViewer/TraceViewerMobileLayout.tsx @@ -0,0 +1,101 @@ +import { Button } from '../Button'; +import { DetailsView } from '../DetailsView/DetailsView'; +import { TraceList } from '../TraceList/TraceList'; +import { type TraceViewerLayoutProps } from '../TraceViewer/TraceViewer'; +import { TraceViewerTreeViewContainer } from './TraceViewerTreeViewContainer'; +import { ArrowLeft } from 'lucide-react'; + +export const TraceViewerMobileLayout = ({ + traceRecords, + traceListExpanded, + setTraceListExpanded, + selectedTrace, + selectedTraceId, + selectedSpan, + setSelectedSpan, + searchValue, + setSearchValue, + filteredSpans, + expandedSpansIds, + setExpandedSpansIds, + handleExpandAll, + handleCollapseAll, + handleTraceSelect, + spanCardViewOptions, + onClearTraceSelection, +}: TraceViewerLayoutProps) => { + if ( + selectedTrace && + selectedTraceId && + filteredSpans.length > 0 && + selectedSpan + ) { + return ( +
    + + +
    + ); + } + + if ( + selectedTrace && + selectedTraceId && + filteredSpans.length > 0 && + !selectedSpan + ) { + return ( +
    +
    + +
    + + +
    + ); + } + + return ( +
    + t.id === selectedTraceId)} + /> +
    + ); +}; diff --git a/frontend/app/src/components/v1/agent-prism/TraceViewer/TraceViewerPlaceholder.tsx b/frontend/app/src/components/v1/agent-prism/TraceViewer/TraceViewerPlaceholder.tsx new file mode 100644 index 0000000000..cd602d7c32 --- /dev/null +++ b/frontend/app/src/components/v1/agent-prism/TraceViewer/TraceViewerPlaceholder.tsx @@ -0,0 +1,5 @@ +export const TraceViewerPlaceholder = ({ title }: { title: string }) => ( +

    + {title} +

    +); diff --git a/frontend/app/src/components/v1/agent-prism/TraceViewer/TraceViewerSearchAndControls.tsx b/frontend/app/src/components/v1/agent-prism/TraceViewer/TraceViewerSearchAndControls.tsx new file mode 100644 index 0000000000..156099e25e --- /dev/null +++ b/frontend/app/src/components/v1/agent-prism/TraceViewer/TraceViewerSearchAndControls.tsx @@ -0,0 +1,30 @@ +import { + CollapseAllButton, + ExpandAllButton, +} from '../CollapseAndExpandControls'; +import { SearchInput } from '../SearchInput'; + +export const TraceViewerSearchAndControls = ({ + searchValue, + setSearchValue, + handleExpandAll, + handleCollapseAll, +}: { + searchValue: string; + setSearchValue: (value: string) => void; + handleExpandAll: () => void; + handleCollapseAll: () => void; +}) => ( +
    + setSearchValue(e.target.value)} + placeholder="Search spans" + /> +
    + + +
    +
    +); diff --git a/frontend/app/src/components/v1/agent-prism/TraceViewer/TraceViewerTreeViewContainer.tsx b/frontend/app/src/components/v1/agent-prism/TraceViewer/TraceViewerTreeViewContainer.tsx new file mode 100644 index 0000000000..04b22dec08 --- /dev/null +++ b/frontend/app/src/components/v1/agent-prism/TraceViewer/TraceViewerTreeViewContainer.tsx @@ -0,0 +1,74 @@ +import { Badge } from '../Badge'; +import type { SpanCardViewOptions } from '../SpanCard/SpanCard'; +import { TraceListItemHeader } from '../TraceList/TraceListItemHeader'; +import { TreeView } from '../TreeView'; +import { type TraceRecordWithDisplayData } from './TraceViewer'; +import { TraceViewerSearchAndControls } from './TraceViewerSearchAndControls'; +import type { TraceSpan } from '@evilmartians/agent-prism-types'; + +export const TraceViewerTreeViewContainer = ({ + searchValue, + setSearchValue, + handleExpandAll, + handleCollapseAll, + filteredSpans, + selectedSpan, + setSelectedSpan, + expandedSpansIds, + setExpandedSpansIds, + spanCardViewOptions, + selectedTrace, + showHeader = true, +}: { + searchValue: string; + setSearchValue: (value: string) => void; + handleExpandAll: () => void; + handleCollapseAll: () => void; + filteredSpans: TraceSpan[]; + selectedSpan: TraceSpan | undefined; + setSelectedSpan: (span: TraceSpan | undefined) => void; + expandedSpansIds: string[]; + setExpandedSpansIds: (ids: string[]) => void; + spanCardViewOptions?: SpanCardViewOptions; + selectedTrace?: TraceRecordWithDisplayData; + showHeader?: boolean; +}) => ( + <> + {showHeader && selectedTrace && ( +
    + + +
    + {selectedTrace.badges?.map((badge, index) => ( + + ))} +
    +
    + )} + +
    + +
    + {filteredSpans.length === 0 ? ( +
    + No spans found +
    + ) : ( + + )} +
    +
    + +); diff --git a/frontend/app/src/components/v1/agent-prism/TreeView.tsx b/frontend/app/src/components/v1/agent-prism/TreeView.tsx new file mode 100644 index 0000000000..e6d97532b5 --- /dev/null +++ b/frontend/app/src/components/v1/agent-prism/TreeView.tsx @@ -0,0 +1,70 @@ +import { BrandLogo } from './BrandLogo'; +import type { SpanCardViewOptions } from './SpanCard/SpanCard'; +import { SpanCard } from './SpanCard/SpanCard'; +import { flattenSpans, findTimeRange } from '@evilmartians/agent-prism-data'; +import type { TraceSpan } from '@evilmartians/agent-prism-types'; +import cn from 'classnames'; +import { type FC } from 'react'; + +interface TreeViewProps { + spans: TraceSpan[]; + className?: string; + selectedSpan?: TraceSpan; + onSpanSelect?: (span: TraceSpan) => void; + expandedSpansIds: string[]; + onExpandSpansIdsChange: (ids: string[]) => void; + spanCardViewOptions?: SpanCardViewOptions; +} + +export const TreeView: FC = ({ + spans, + onSpanSelect, + className = '', + selectedSpan, + expandedSpansIds, + onExpandSpansIdsChange, + spanCardViewOptions, +}) => { + const allCards = flattenSpans(spans); + const { minStart, maxEnd } = findTimeRange(allCards); + + return ( +
    +
      + {spans.map((span, idx) => { + const brand = span.metadata?.brand as { type: string } | undefined; + + return ( + , + size: '4', + rounded: 'sm', + category: span.type, + } + : undefined + } + /> + ); + })} +
    +
    + ); +}; diff --git a/frontend/app/src/components/v1/agent-prism/shared.ts b/frontend/app/src/components/v1/agent-prism/shared.ts new file mode 100644 index 0000000000..534e8bec8b --- /dev/null +++ b/frontend/app/src/components/v1/agent-prism/shared.ts @@ -0,0 +1,169 @@ +import type { TraceSpanCategory } from '@evilmartians/agent-prism-types'; +import type { LucideIcon } from 'lucide-react'; +import { + Zap, + Wrench, + Bot, + Link, + Search, + BarChart2, + Plus, + HelpCircle, + MoveHorizontal, + CircleDot, + ShieldCheck, +} from 'lucide-react'; +import { useEffect, useState } from 'react'; + +// TYPES + +export type ColorVariant = + | 'purple' + | 'indigo' + | 'orange' + | 'teal' + | 'cyan' + | 'sky' + | 'yellow' + | 'emerald' + | 'red' + | 'gray'; + +export type ComponentSize = + | '4' + | '5' + | '6' + | '7' + | '8' + | '9' + | '10' + | '11' + | '12' + | '16'; + +// CONSTANTS + +export const ROUNDED_CLASSES = { + none: 'rounded-none', + sm: 'rounded-sm', + md: 'rounded-md', + lg: 'rounded-lg', + full: 'rounded-full', +}; + +/** + * Shared configuration for span categories containing label, theme, and icon + */ +export const SPAN_CATEGORY_CONFIG: Record< + TraceSpanCategory, + { + label: string; + theme: ColorVariant; + icon: LucideIcon; + } +> = { + llm_call: { + label: 'LLM', + theme: 'purple', + icon: Zap, + }, + tool_execution: { + label: 'TOOL', + theme: 'orange', + icon: Wrench, + }, + agent_invocation: { + label: 'AGENT INVOCATION', + theme: 'indigo', + icon: Bot, + }, + chain_operation: { + label: 'CHAIN', + theme: 'teal', + icon: Link, + }, + retrieval: { + label: 'RETRIEVAL', + theme: 'cyan', + icon: Search, + }, + embedding: { + label: 'EMBEDDING', + theme: 'emerald', + icon: BarChart2, + }, + create_agent: { + label: 'CREATE AGENT', + theme: 'sky', + icon: Plus, + }, + span: { + label: 'SPAN', + theme: 'cyan', + icon: MoveHorizontal, + }, + event: { + label: 'EVENT', + theme: 'emerald', + icon: CircleDot, + }, + guardrail: { + label: 'GUARDRAIL', + theme: 'red', + icon: ShieldCheck, + }, + unknown: { + label: 'UNKNOWN', + theme: 'gray', + icon: HelpCircle, + }, +}; + +// UTILS + +export function getSpanCategoryTheme( + category: TraceSpanCategory, +): ColorVariant { + return SPAN_CATEGORY_CONFIG[category].theme; +} + +export function getSpanCategoryLabel(category: TraceSpanCategory): string { + return SPAN_CATEGORY_CONFIG[category].label; +} + +export function getSpanCategoryIcon(category: TraceSpanCategory): LucideIcon { + return SPAN_CATEGORY_CONFIG[category].icon; +} + +export const useIsMobile = () => { + const isMounted = useIsMounted(); + + const [isMobile, setIsMobile] = useState(false); + + useEffect(() => { + // TODO: replace with something more beautiful and correct (tailwind screens?) + const mediaQuery = window.matchMedia('(max-width: 1023px)'); + + const handleChange = (e: MediaQueryListEvent | MediaQueryList) => { + setIsMobile(e.matches); + }; + + handleChange(mediaQuery); + + mediaQuery.addEventListener('change', handleChange); + + return () => mediaQuery.removeEventListener('change', handleChange); + }, []); + + return isMounted ? isMobile : false; +}; + +export const useIsMounted = () => { + const [isMounted, setIsMounted] = useState(false); + + useEffect(() => { + setIsMounted(true); + }, []); + + return isMounted; +}; diff --git a/frontend/app/src/components/v1/agent-prism/theme/index.ts b/frontend/app/src/components/v1/agent-prism/theme/index.ts new file mode 100644 index 0000000000..b0687c31ed --- /dev/null +++ b/frontend/app/src/components/v1/agent-prism/theme/index.ts @@ -0,0 +1,99 @@ +export const agentPrismPrefix = 'agentprism'; + +export const AGENT_PRISM_TOKENS = [ + 'background', + 'foreground', + 'primary', + 'primary-foreground', + 'secondary', + 'secondary-foreground', + 'muted', + 'muted-foreground', + 'accent', + 'accent-foreground', + 'brand', + 'brand-foreground', + 'brand-secondary', + 'brand-secondary-foreground', + 'border', + 'border-subtle', + 'border-strong', + 'border-inverse', + 'success', + 'success-muted', + 'success-muted-foreground', + 'error', + 'error-muted', + 'error-muted-foreground', + 'warning', + 'warning-muted', + 'warning-muted-foreground', + 'pending', + 'pending-muted', + 'pending-muted-foreground', + 'code-string', + 'code-number', + 'code-key', + 'code-base', + 'badge-default', + 'badge-default-foreground', + 'avatar-llm', + 'badge-llm', + 'badge-llm-foreground', + 'timeline-llm', + 'avatar-agent', + 'badge-agent', + 'badge-agent-foreground', + 'timeline-agent', + 'avatar-tool', + 'badge-tool', + 'badge-tool-foreground', + 'timeline-tool', + 'avatar-chain', + 'badge-chain', + 'badge-chain-foreground', + 'timeline-chain', + 'avatar-retrieval', + 'badge-retrieval', + 'badge-retrieval-foreground', + 'timeline-retrieval', + 'avatar-embedding', + 'badge-embedding', + 'badge-embedding-foreground', + 'timeline-embedding', + 'avatar-guardrail', + 'badge-guardrail', + 'badge-guardrail-foreground', + 'timeline-guardrail', + 'avatar-create-agent', + 'badge-create-agent', + 'badge-create-agent-foreground', + 'timeline-create-agent', + 'avatar-span', + 'badge-span', + 'badge-span-foreground', + 'timeline-span', + 'avatar-event', + 'badge-event', + 'badge-event-foreground', + 'timeline-event', + 'avatar-unknown', + 'badge-unknown', + 'badge-unknown-foreground', + 'timeline-unknown', +] as const; + +export type AgentPrismToken = (typeof AGENT_PRISM_TOKENS)[number]; + +export type AgentPrismColors = Record; + +export const agentPrismTailwindColors = Object.fromEntries( + AGENT_PRISM_TOKENS.map((tokenName) => [ + `agentprism-${tokenName}`, + token(tokenName), + ]), +) as AgentPrismColors; + +function token(name: string) { + return `oklch(var(--${agentPrismPrefix}-${name}) / )`; +} diff --git a/frontend/app/src/components/v1/agent-prism/theme/theme.css b/frontend/app/src/components/v1/agent-prism/theme/theme.css new file mode 100644 index 0000000000..c44c3ea3ce --- /dev/null +++ b/frontend/app/src/components/v1/agent-prism/theme/theme.css @@ -0,0 +1,245 @@ +:root { + @media (prefers-color-scheme: light) { + /* General purpose colors */ + --agentprism-background: 1 0 0; /* white */ + --agentprism-foreground: 21% 0.034 264.665; /* gray.900 */ + --agentprism-primary: 21% 0.034 264.665; /* gray.900 */ + --agentprism-primary-foreground: 98.5% 0.002 247.839; /* gray.50 */ + --agentprism-secondary: 96.7% 0.003 264.542; /* gray.100 */ + --agentprism-secondary-foreground: 55.1% 0.027 264.364; /* gray.500 */ + --agentprism-muted: 98.5% 0.002 247.839; /* gray.50 */ + --agentprism-muted-foreground: 44.6% 0.03 256.802; /* gray.600 */ + --agentprism-accent: 96.7% 0.003 264.542; /* gray.100 */ + --agentprism-accent-foreground: 96.7% 0.003 264.542; /* gray.100 */ + + /* Brand colors */ + --agentprism-brand: 60.6% 0.25 292.717; /* violet.500 */ + --agentprism-brand-foreground: 1 0 0; /* white */ + --agentprism-brand-secondary: 70.5% 0.213 47.604; /* orange.500 */ + --agentprism-brand-secondary-foreground: 1 0 0; /* white */ + + /* Borders */ + --agentprism-border: 92.8% 0.006 264.531; /* gray.200 */ + --agentprism-border-subtle: 96.7% 0.003 264.542; /* gray.100 */ + --agentprism-border-strong: 87.2% 0.01 258.338; /* gray.300 */ + --agentprism-border-inverse: 21% 0.034 264.665; /* gray.900 */ + + /* Success status color */ + --agentprism-success: 69.6% 0.17 162.48; /* emerald.500 */ + --agentprism-success-muted: 97.9% 0.021 166.113; /* emerald.50 */ + --agentprism-success-muted-foreground: 50.8% 0.118 165.612; /* emerald.700 */ + + /* Error status color */ + --agentprism-error: 63.7% 0.237 25.331; /* red.500 */ + --agentprism-error-muted: 97.1% 0.013 17.38; /* red.50 */ + --agentprism-error-muted-foreground: 57.7% 0.245 27.325; /* red.600 */ + + /* Warning status color */ + --agentprism-warning: 79.5% 0.184 86.047; /* yellow.500 */ + --agentprism-warning-muted: 98.7% 0.026 102.212; /* yellow.50 */ + --agentprism-warning-muted-foreground: 55.4% 0.135 66.442; /* yellow.700 */ + + /* Pending status color */ + --agentprism-pending: 60.6% 0.25 292.717; /* violet.500 */ + --agentprism-pending-muted: 94.3% 0.029 294.588; /* violet.100 */ + --agentprism-pending-muted-foreground: 54.1% 0.281 293.009; /* violet.600 */ + + /* Code syntax highlighting */ + --agentprism-code-string: 57.7% 0.245 27.325; /* red.600 */ + --agentprism-code-number: 57.7% 0.245 27.325; /* red.600 */ + --agentprism-code-key: 54.6% 0.245 262.881; /* blue.600 */ + --agentprism-code-base: 55.1% 0.027 264.364; /* gray.500 */ + + /* Generic badge colors */ + --agentprism-badge-default: 96.7% 0.003 264.542; /* gray.100 */ + --agentprism-badge-default-foreground: 44.6% 0.03 256.802; /* gray.600 */ + + /* Trace colors (llm) */ + --agentprism-avatar-llm: 62.7% 0.265 303.9; /* purple.500 */ + --agentprism-badge-llm: 97.7% 0.014 308.299; /* purple.50 */ + --agentprism-badge-llm-foreground: 62.7% 0.265 303.9; /* purple.500 */ + --agentprism-timeline-llm: 71.4% 0.203 305.504; /* purple.400 */ + + /* Trace colors (agent) */ + --agentprism-avatar-agent: 58.5% 0.233 277.117; /* indigo.500 */ + --agentprism-badge-agent: 96.2% 0.018 272.314; /* indigo.50 */ + --agentprism-badge-agent-foreground: 58.5% 0.233 277.117; /* indigo.500 */ + --agentprism-timeline-agent: 67.3% 0.182 276.935; /* indigo.400 */ + + /* Trace colors (tool) */ + --agentprism-avatar-tool: 70.5% 0.213 47.604; /* orange.500 */ + --agentprism-badge-tool: 98% 0.016 73.684; /* orange.50 */ + --agentprism-badge-tool-foreground: 70.5% 0.213 47.604; /* orange.500 */ + --agentprism-timeline-tool: 75% 0.183 55.934; /* orange.400 */ + + /* Trace colors (chain) */ + --agentprism-avatar-chain: 70.4% 0.14 182.503; /* teal.500 */ + --agentprism-badge-chain: 98.4% 0.014 180.72; /* teal.50 */ + --agentprism-badge-chain-foreground: 70.4% 0.14 182.503; /* teal.500 */ + --agentprism-timeline-chain: 77.7% 0.152 181.912; /* teal.400 */ + + /* Trace colors (retrieval) */ + --agentprism-avatar-retrieval: 71.5% 0.143 215.221; /* cyan.500 */ + --agentprism-badge-retrieval: 98.4% 0.019 200.873; /* cyan.50 */ + --agentprism-badge-retrieval-foreground: 71.5% 0.143 215.221; /* cyan.500 */ + --agentprism-timeline-retrieval: 78.9% 0.154 211.53; /* cyan.400 */ + + /* Trace colors (embedding) */ + --agentprism-avatar-embedding: 69.6% 0.17 162.48; /* emerald.500 */ + --agentprism-badge-embedding: 97.9% 0.021 166.113; /* emerald.50 */ + --agentprism-badge-embedding-foreground: 69.6% 0.17 162.48; /* emerald.500 */ + --agentprism-timeline-embedding: 76.5% 0.177 163.223; /* emerald.400 */ + + /* Trace colors (guardrail) */ + --agentprism-avatar-guardrail: 63.7% 0.237 25.331; /* red.500 */ + --agentprism-badge-guardrail: 97.1% 0.013 17.38; /* red.50 */ + --agentprism-badge-guardrail-foreground: 63.7% 0.237 25.331; /* red.500 */ + --agentprism-timeline-guardrail: 70.4% 0.191 22.216; /* red.400 */ + + /* Trace colors (create agent) */ + --agentprism-avatar-create-agent: 68.5% 0.169 237.323; /* sky.500 */ + --agentprism-badge-create-agent: 97.7% 0.013 236.62; /* sky.50 */ + --agentprism-badge-create-agent-foreground: 68.5% 0.169 237.323; /* sky.500 */ + --agentprism-timeline-create-agent: 74.6% 0.16 232.661; /* sky.400 */ + + /* Trace colors (span) */ + --agentprism-avatar-span: 71.5% 0.143 215.221; /* cyan.500 */ + --agentprism-badge-span: 98.4% 0.019 200.873; /* cyan.50 */ + --agentprism-badge-span-foreground: 71.5% 0.143 215.221; /* cyan.500 */ + --agentprism-timeline-span: 78.9% 0.154 211.53; /* cyan.400 */ + + /* Trace colors (event) */ + --agentprism-avatar-event: 69.6% 0.17 162.48; /* emerald.500 */ + --agentprism-badge-event: 97.9% 0.021 166.113; /* emerald.50 */ + --agentprism-badge-event-foreground: 69.6% 0.17 162.48; /* emerald.500 */ + --agentprism-timeline-event: 76.5% 0.177 163.223; /* emerald.400 */ + + /* Trace colors (unknown) */ + --agentprism-avatar-unknown: 55.1% 0.027 264.364; /* gray.500 */ + --agentprism-badge-unknown: 98.5% 0.002 247.839; /* gray.50 */ + --agentprism-badge-unknown-foreground: 55.1% 0.027 264.364; /* gray.500 */ + --agentprism-timeline-unknown: 70.7% 0.022 261.325; /* gray.400 */ + } + + @media (prefers-color-scheme: dark) { + /* General purpose colors */ + --agentprism-background: 13% 0.028 261.692; /* gray.950 */ + --agentprism-foreground: 96.7% 0.003 264.542; /* gray.100 */ + --agentprism-primary: 96.7% 0.003 264.542; /* gray.100 */ + --agentprism-primary-foreground: 13% 0.028 261.692; /* gray.950 */ + --agentprism-secondary: 27.8% 0.033 256.848; /* gray.800 */ + --agentprism-secondary-foreground: 55.1% 0.027 264.364; /* gray.500 */ + --agentprism-muted: 21% 0.034 264.665; /* gray.900 */ + --agentprism-muted-foreground: 70.7% 0.022 261.325; /* gray.400 */ + --agentprism-accent: 96.7% 0.003 264.542; /* gray.100 */ + --agentprism-accent-foreground: 21% 0.034 264.665; /* gray.900 */ + + /* Brand colors */ + --agentprism-brand: 60.6% 0.25 292.717; /* violet.500 */ + --agentprism-brand-foreground: 1 0 0; /* white */ + --agentprism-brand-secondary: 70.5% 0.213 47.604; /* orange.500 */ + --agentprism-brand-secondary-foreground: 1 0 0; /* white */ + + /* Borders */ + --agentprism-border: 44.6% 0.03 256.802; /* gray.600 */ + --agentprism-border-subtle: 37.3% 0.034 259.733; /* gray.700 */ + --agentprism-border-strong: 55.1% 0.027 264.364; /* gray.500 */ + --agentprism-border-inverse: 92.8% 0.006 264.531; /* gray.200 */ + + /* Success status color */ + --agentprism-success: 72.3% 0.219 149.579; /* green.500 */ + --agentprism-success-muted: 26.6% 0.065 152.934; /* green.950 */ + --agentprism-success-muted-foreground: 87.1% 0.15 154.449; /* green.300 */ + + /* Error status color */ + --agentprism-error: 63.7% 0.237 25.331; /* red.500 */ + --agentprism-error-muted: 25.8% 0.092 26.042; /* red.950 */ + --agentprism-error-muted-foreground: 80.8% 0.114 19.571; /* red.300 */ + + /* Warning status color */ + --agentprism-warning: 79.5% 0.184 86.047; /* yellow.500 */ + --agentprism-warning-muted: 28.6% 0.066 53.813; /* yellow.950 */ + --agentprism-warning-muted-foreground: 90.5% 0.182 98.111; /* yellow.300 */ + + /* Pending status color */ + --agentprism-pending: 60.6% 0.25 292.717; /* violet.500 */ + --agentprism-pending-muted: 28.3% 0.141 291.089; /* violet.950 */ + --agentprism-pending-muted-foreground: 70.2% 0.183 293.541; /* violet.400 */ + + /* Code syntax highlighting */ + --agentprism-code-string: 70.4% 0.191 22.216; /* red.400 */ + --agentprism-code-number: 70.4% 0.191 22.216; /* red.400 */ + --agentprism-code-key: 80.9% 0.105 251.813; /* blue.300 */ + --agentprism-code-base: 70.7% 0.022 261.325; /* gray.400 */ + + /* Generic badge colors */ + --agentprism-badge-default: 21% 0.034 264.665; /* gray.900 */ + --agentprism-badge-default-foreground: 70.7% 0.022 261.325; /* gray.400 */ + + /* Trace colors (llm) */ + --agentprism-avatar-llm: 82.7% 0.119 306.383; /* purple.300 */ + --agentprism-badge-llm: 29.1% 0.149 302.717; /* purple.950 */ + --agentprism-badge-llm-foreground: 82.7% 0.119 306.383; /* purple.300 */ + --agentprism-timeline-llm: 71.4% 0.203 305.504; /* purple.400 */ + + /* Trace colors (agent) */ + --agentprism-avatar-agent: 78.5% 0.115 274.713; /* indigo.300 */ + --agentprism-badge-agent: 25.7% 0.09 281.288; /* indigo.950 */ + --agentprism-badge-agent-foreground: 78.5% 0.115 274.713; /* indigo.300 */ + --agentprism-timeline-agent: 67.3% 0.182 276.935; /* indigo.400 */ + + /* Trace colors (tool) */ + --agentprism-avatar-tool: 83.7% 0.128 66.29; /* orange.300 */ + --agentprism-badge-tool: 26.6% 0.079 36.259; /* orange.950 */ + --agentprism-badge-tool-foreground: 83.7% 0.128 66.29; /* orange.300 */ + --agentprism-timeline-tool: 75% 0.183 55.934; /* orange.400 */ + + /* Trace colors (chain) */ + --agentprism-avatar-chain: 85.5% 0.138 181.071; /* teal.300 */ + --agentprism-badge-chain: 27.7% 0.046 192.524; /* teal.950 */ + --agentprism-badge-chain-foreground: 85.5% 0.138 181.071; /* teal.300 */ + --agentprism-timeline-chain: 77.7% 0.152 181.912; /* teal.400 */ + + /* Trace colors (retrieval) */ + --agentprism-avatar-retrieval: 86.5% 0.127 207.078; /* cyan.300 */ + --agentprism-badge-retrieval: 30.2% 0.056 229.695; /* cyan.950 */ + --agentprism-badge-retrieval-foreground: 86.5% 0.127 207.078; /* cyan.300 */ + --agentprism-timeline-retrieval: 78.9% 0.154 211.53; /* cyan.400 */ + + /* Trace colors (embedding) */ + --agentprism-avatar-embedding: 84.5% 0.143 164.978; /* emerald.300 */ + --agentprism-badge-embedding: 26.2% 0.051 172.552; /* emerald.950 */ + --agentprism-badge-embedding-foreground: 84.5% 0.143 164.978; /* emerald.300 */ + --agentprism-timeline-embedding: 76.5% 0.177 163.223; /* emerald.400 */ + + /* Trace colors (guardrail) */ + --agentprism-avatar-guardrail: 80.8% 0.114 19.571; /* red.300 */ + --agentprism-badge-guardrail: 25.8% 0.092 26.042; /* red.950 */ + --agentprism-badge-guardrail-foreground: 80.8% 0.114 19.571; /* red.300 */ + --agentprism-timeline-guardrail: 70.4% 0.191 22.216; /* red.400 */ + + /* Trace colors (create agent) */ + --agentprism-avatar-create-agent: 82.8% 0.111 230.318; /* sky.300 */ + --agentprism-badge-create-agent: 29.3% 0.066 243.157; /* sky.950 */ + --agentprism-badge-create-agent-foreground: 82.8% 0.111 230.318; /* sky.300 */ + --agentprism-timeline-create-agent: 74.6% 0.16 232.661; /* sky.400 */ + + /* Trace colors (span) */ + --agentprism-avatar-span: 86.5% 0.127 207.078; /* cyan.300 */ + --agentprism-badge-span: 30.2% 0.056 229.695; /* cyan.950 */ + --agentprism-badge-span-foreground: 86.5% 0.127 207.078; /* cyan.300 */ + --agentprism-timeline-span: 78.9% 0.154 211.53; /* cyan.400 */ + + /* Trace colors (event) */ + --agentprism-avatar-event: 84.5% 0.143 164.978; /* emerald.300 */ + --agentprism-badge-event: 26.2% 0.051 172.552; /* emerald.950 */ + --agentprism-badge-event-foreground: 84.5% 0.143 164.978; /* emerald.300 */ + --agentprism-timeline-event: 76.5% 0.177 163.223; /* emerald.400 */ + + /* Trace colors (unknown) */ + --agentprism-avatar-unknown: 87.2% 0.01 258.338; /* gray.300 */ + --agentprism-badge-unknown: 13% 0.028 261.692; /* gray.950 */ + --agentprism-badge-unknown-foreground: 87.2% 0.01 258.338; /* gray.300 */ + --agentprism-timeline-unknown: 70.7% 0.022 261.325; /* gray.400 */ + } +} diff --git a/frontend/app/src/lib/api/generated/Api.ts b/frontend/app/src/lib/api/generated/Api.ts index 640e71940f..d7bf4a896e 100644 --- a/frontend/app/src/lib/api/generated/Api.ts +++ b/frontend/app/src/lib/api/generated/Api.ts @@ -46,6 +46,7 @@ import { LogLineOrderByDirection, LogLineOrderByField, LogLineSearch, + OtelSpanList, RateLimitList, RateLimitOrderByDirection, RateLimitOrderByField, @@ -258,6 +259,39 @@ export class Api< format: "json", ...params, }); + /** + * @description Get OTel trace for a task run + * + * @tags Task + * @name V1TaskGetTrace + * @summary Get OTel trace + * @request GET:/api/v1/stable/tasks/{task}/trace + * @secure + */ + v1TaskGetTrace = ( + task: string, + query?: { + /** + * The number to skip + * @format int64 + */ + offset?: number; + /** + * The number to limit by + * @format int64 + */ + limit?: number; + }, + params: RequestParams = {}, + ) => + this.request({ + path: `/api/v1/stable/tasks/${task}/trace`, + method: "GET", + query: query, + secure: true, + format: "json", + ...params, + }); /** * @description Cancel tasks * diff --git a/frontend/app/src/lib/api/generated/data-contracts.ts b/frontend/app/src/lib/api/generated/data-contracts.ts index ef9bff9026..aabc2cebd2 100644 --- a/frontend/app/src/lib/api/generated/data-contracts.ts +++ b/frontend/app/src/lib/api/generated/data-contracts.ts @@ -256,6 +256,21 @@ export enum TenantVersion { V1 = "V1", } +export enum OtelStatusCode { + UNSET = "UNSET", + OK = "OK", + ERROR = "ERROR", +} + +export enum OtelSpanKind { + UNSPECIFIED = "UNSPECIFIED", + INTERNAL = "INTERNAL", + SERVER = "SERVER", + CLIENT = "CLIENT", + PRODUCER = "PRODUCER", + CONSUMER = "CONSUMER", +} + export enum V1LogLineOrderByDirection { ASC = "ASC", DESC = "DESC", @@ -516,6 +531,30 @@ export interface V1LogLineList { rows?: V1LogLine[]; } +export interface OtelSpan { + trace_id: string; + span_id: string; + parent_span_id?: string; + span_name: string; + span_kind: OtelSpanKind; + service_name: string; + status_code: OtelStatusCode; + status_message?: string; + /** @format int64 */ + duration: number; + /** @format date-time */ + created_at: string; + resource_attributes?: Record; + span_attributes?: Record; + scope_name?: string; + scope_version?: string; +} + +export interface OtelSpanList { + pagination?: PaginationResponse; + rows?: OtelSpan[]; +} + export interface V1TaskFilter { /** @format date-time */ since: string; diff --git a/frontend/app/src/main.tsx b/frontend/app/src/main.tsx index 280a85eaf5..98c71f4d85 100644 --- a/frontend/app/src/main.tsx +++ b/frontend/app/src/main.tsx @@ -1,3 +1,4 @@ +import './components/v1/agent-prism/theme/theme.css'; import './index.css'; import queryClient from './query-client.tsx'; import Router from './router.tsx'; diff --git a/frontend/app/src/pages/main/v1/workflow-runs-v1/$run/v2components/step-run-detail/otel-span-adapter.ts b/frontend/app/src/pages/main/v1/workflow-runs-v1/$run/v2components/step-run-detail/otel-span-adapter.ts new file mode 100644 index 0000000000..e037d7c069 --- /dev/null +++ b/frontend/app/src/pages/main/v1/workflow-runs-v1/$run/v2components/step-run-detail/otel-span-adapter.ts @@ -0,0 +1,130 @@ +import type { OtelSpan } from '@/lib/api/generated/data-contracts'; +import type { + OpenTelemetrySpan, + OpenTelemetrySpanKind, + OpenTelemetryStatusCode, + TraceSpanAttribute, +} from '@evilmartians/agent-prism-types'; + +const SPAN_KIND_MAP: Record = { + INTERNAL: 'SPAN_KIND_INTERNAL', + SERVER: 'SPAN_KIND_SERVER', + CLIENT: 'SPAN_KIND_CLIENT', + PRODUCER: 'SPAN_KIND_PRODUCER', + CONSUMER: 'SPAN_KIND_CONSUMER', +}; + +const STATUS_CODE_MAP: Record = { + OK: 'STATUS_CODE_OK', + ERROR: 'STATUS_CODE_ERROR', + UNSET: 'STATUS_CODE_UNSET', +}; + +function recordToAttributes( + record: Record | undefined, +): TraceSpanAttribute[] { + if (!record) { + return []; + } + return Object.entries(record).map(([key, value]) => ({ + key, + value: { stringValue: value }, + })); +} + +/** + * Converts our API's flat OtelSpan format to the OTLP-style OpenTelemetrySpan + * expected by @evilmartians/agent-prism-data adapters. + */ +export function convertOtelSpanToOpenTelemetrySpan( + span: OtelSpan, +): OpenTelemetrySpan { + const startNanos = BigInt(new Date(span.created_at).getTime()) * 1_000_000n; + const durationNanos = BigInt(span.duration); + const endNanos = startNanos + durationNanos; + + return { + traceId: span.trace_id, + spanId: span.span_id, + parentSpanId: span.parent_span_id || undefined, + name: span.span_name, + kind: SPAN_KIND_MAP[span.span_kind] || 'SPAN_KIND_INTERNAL', + startTimeUnixNano: startNanos.toString(), + endTimeUnixNano: endNanos.toString(), + attributes: [ + ...recordToAttributes(span.span_attributes), + ...recordToAttributes(span.resource_attributes), + { key: 'service.name', value: { stringValue: span.service_name } }, + ], + status: { + code: STATUS_CODE_MAP[span.status_code] || 'STATUS_CODE_UNSET', + message: span.status_message, + }, + flags: 0, + }; +} + +/** + * Filters spans to only include those belonging to a specific task and its + * descendants. Finds the task's root span via hatchet.step_run_id attribute, + * then collects all descendant spans by following parent_span_id chains. + */ +function filterSpansForTask( + spans: OtelSpan[], + taskExternalId: string, +): OtelSpan[] { + // Find the root span for this task (the "hatchet task run" span) + const taskRootSpan = spans.find( + (s) => s.span_attributes?.['hatchet.step_run_id'] === taskExternalId, + ); + + if (!taskRootSpan) { + // Fallback: if no root span found, return all spans + return spans; + } + + // Build a parent->children index + const childrenByParent = new Map(); + for (const s of spans) { + const pid = s.parent_span_id; + if (pid) { + const children = childrenByParent.get(pid) || []; + children.push(s); + childrenByParent.set(pid, children); + } + } + + // BFS from the task root span to collect all descendants + const result: OtelSpan[] = [taskRootSpan]; + const queue = [taskRootSpan.span_id]; + while (queue.length > 0) { + const parentId = queue.shift()!; + const children = childrenByParent.get(parentId) || []; + for (const child of children) { + result.push(child); + queue.push(child.span_id); + } + } + + return result; +} + +export function convertOtelSpans( + spans: OtelSpan[], + taskExternalId?: string, +): OpenTelemetrySpan[] { + const filtered = taskExternalId + ? filterSpansForTask(spans, taskExternalId) + : spans; + const converted = filtered.map(convertOtelSpanToOpenTelemetrySpan); + + // Promote orphaned spans to root spans: if a span's parentSpanId + // references a span not in this set, clear it so the tree builder + // treats it as a root instead of silently dropping it. + const spanIdSet = new Set(converted.map((s) => s.spanId)); + return converted.map((s) => + s.parentSpanId && !spanIdSet.has(s.parentSpanId) + ? { ...s, parentSpanId: undefined } + : s, + ); +} diff --git a/frontend/app/src/pages/main/v1/workflow-runs-v1/$run/v2components/step-run-detail/step-run-detail.tsx b/frontend/app/src/pages/main/v1/workflow-runs-v1/$run/v2components/step-run-detail/step-run-detail.tsx index b3f566abc0..355d2e3ead 100644 --- a/frontend/app/src/pages/main/v1/workflow-runs-v1/$run/v2components/step-run-detail/step-run-detail.tsx +++ b/frontend/app/src/pages/main/v1/workflow-runs-v1/$run/v2components/step-run-detail/step-run-detail.tsx @@ -8,6 +8,7 @@ import { StepRunEvents } from '../step-run-events-for-workflow-run'; import { Waterfall } from '../waterfall'; import { V1StepRunOutput } from './step-run-output'; import { TaskRunLogs } from './task-run-logs'; +import { TaskRunTrace } from './task-run-trace'; import RelativeDate from '@/components/v1/molecules/relative-date'; import { CopyWorkflowConfigButton } from '@/components/v1/shared/copy-workflow-config'; import { Button } from '@/components/v1/ui/button'; @@ -35,6 +36,7 @@ export enum TabOption { ChildWorkflowRuns = 'child-workflow-runs', Input = 'input', Logs = 'logs', + Trace = 'trace', Waterfall = 'waterfall', AdditionalMetadata = 'additional-metadata', Activity = 'activity', @@ -248,6 +250,9 @@ export const TaskRunDetail = ({ Logs + + Trace + + + + { + const allSpans: OtelSpan[] = []; + let offset = 0; + + // eslint-disable-next-line no-constant-condition + while (true) { + const res = await api.v1TaskGetTrace(taskExternalId, { + offset, + limit: PAGE_SIZE, + }); + + const rows = res.data.rows ?? []; + allSpans.push(...rows); + + const numPages = res.data.pagination?.num_pages ?? 1; + const currentPage = res.data.pagination?.current_page ?? 1; + + if (currentPage >= numPages || rows.length === 0) { + break; + } + + offset += PAGE_SIZE; + } + + return allSpans; +} + +export function TaskRunTrace({ + taskExternalId, + isRunning, +}: { + taskExternalId: string; + isRunning: boolean; +}) { + const tracesQuery = useQuery({ + queryKey: ['task:trace', taskExternalId], + queryFn: () => fetchAllSpans(taskExternalId), + refetchInterval: isRunning ? 100 : false, + }); + + const traceSpans = useMemo(() => { + const rows = tracesQuery.data; + if (!rows || rows.length === 0) { + return []; + } + + const otlpSpans = convertOtelSpans(rows, taskExternalId); + return openTelemetrySpanAdapter.convertRawSpansToSpanTree(otlpSpans); + }, [tracesQuery.data, taskExternalId]); + + const allIds = useMemo( + () => flattenSpans(traceSpans).map((s) => s.id), + [traceSpans], + ); + + const [expandedSpansIds, setExpandedSpansIds] = useState([]); + + // Auto-expand all spans when data loads + useEffect(() => { + if (allIds.length > 0) { + setExpandedSpansIds(allIds); + } + }, [allIds]); + + if (tracesQuery.isLoading) { + return ; + } + + if (tracesQuery.isError) { + return ( +
    + Failed to load traces. +
    + ); + } + + if (traceSpans.length === 0) { + return ( +
    + No trace found for this task run. To collect traces, use the{' '} + + HatchetInstrumentor + {' '} + in your SDK. +
    + ); + } + + return ( +
    +
    +
    +

    + OpenTelemetry Traces +

    +
    +
    +
    + +
    +
    + ); +} diff --git a/frontend/app/tailwind.config.js b/frontend/app/tailwind.config.js index fa3db6977b..93f70cd30a 100644 --- a/frontend/app/tailwind.config.js +++ b/frontend/app/tailwind.config.js @@ -1,3 +1,34 @@ +// Generate agent-prism Tailwind color mappings from CSS custom properties +const agentPrismTokens = [ + "background", "foreground", "primary", "primary-foreground", "secondary", + "secondary-foreground", "muted", "muted-foreground", "accent", "accent-foreground", + "brand", "brand-foreground", "brand-secondary", "brand-secondary-foreground", + "border", "border-subtle", "border-strong", "border-inverse", + "success", "success-muted", "success-muted-foreground", + "error", "error-muted", "error-muted-foreground", + "warning", "warning-muted", "warning-muted-foreground", + "pending", "pending-muted", "pending-muted-foreground", + "code-string", "code-number", "code-key", "code-base", + "badge-default", "badge-default-foreground", + "avatar-llm", "badge-llm", "badge-llm-foreground", "timeline-llm", + "avatar-agent", "badge-agent", "badge-agent-foreground", "timeline-agent", + "avatar-tool", "badge-tool", "badge-tool-foreground", "timeline-tool", + "avatar-chain", "badge-chain", "badge-chain-foreground", "timeline-chain", + "avatar-retrieval", "badge-retrieval", "badge-retrieval-foreground", "timeline-retrieval", + "avatar-embedding", "badge-embedding", "badge-embedding-foreground", "timeline-embedding", + "avatar-guardrail", "badge-guardrail", "badge-guardrail-foreground", "timeline-guardrail", + "avatar-create-agent", "badge-create-agent", "badge-create-agent-foreground", "timeline-create-agent", + "avatar-span", "badge-span", "badge-span-foreground", "timeline-span", + "avatar-event", "badge-event", "badge-event-foreground", "timeline-event", + "avatar-unknown", "badge-unknown", "badge-unknown-foreground", "timeline-unknown", +]; +const agentPrismColors = Object.fromEntries( + agentPrismTokens.map((name) => [ + `agentprism-${name}`, + `oklch(var(--agentprism-${name}) / )`, + ]) +); + /** @type {import('tailwindcss').Config} */ module.exports = { darkMode: ["class"], @@ -49,6 +80,7 @@ module.exports = { }, extend: { colors: { + ...agentPrismColors, "purple": { "50": "hsl(252, 82%, 95%)", "100": "hsl(252, 82%, 90%)", diff --git a/internal/services/otelcol/server.go b/internal/services/otelcol/server.go index 26422ed40e..089c9ac6e1 100644 --- a/internal/services/otelcol/server.go +++ b/internal/services/otelcol/server.go @@ -83,10 +83,10 @@ func (oc *otelCollectorImpl) convertOTLPToSpanData(resourceSpans []*tracev1.Reso SpanID: span.GetSpanId(), ParentSpanID: span.GetParentSpanId(), Name: span.GetName(), - Kind: int32(span.GetKind()), + Kind: span.GetKind(), StartTimeUnixNano: span.GetStartTimeUnixNano(), EndTimeUnixNano: span.GetEndTimeUnixNano(), - StatusCode: int32(span.GetStatus().GetCode()), + StatusCode: span.GetStatus().GetCode(), StatusMessage: span.GetStatus().GetMessage(), Attributes: oc.serializeAttributes(span.GetAttributes()), Events: oc.serializeEvents(span.GetEvents()), diff --git a/pkg/client/rest/gen.go b/pkg/client/rest/gen.go index d413f0c92a..10c88199a8 100644 --- a/pkg/client/rest/gen.go +++ b/pkg/client/rest/gen.go @@ -90,6 +90,23 @@ const ( LogLineOrderByFieldCreatedAt LogLineOrderByField = "createdAt" ) +// Defines values for OtelSpanKind. +const ( + CLIENT OtelSpanKind = "CLIENT" + CONSUMER OtelSpanKind = "CONSUMER" + INTERNAL OtelSpanKind = "INTERNAL" + PRODUCER OtelSpanKind = "PRODUCER" + SERVER OtelSpanKind = "SERVER" + UNSPECIFIED OtelSpanKind = "UNSPECIFIED" +) + +// Defines values for OtelStatusCode. +const ( + OtelStatusCodeERROR OtelStatusCode = "ERROR" + OtelStatusCodeOK OtelStatusCode = "OK" + OtelStatusCodeUNSET OtelStatusCode = "UNSET" +) + // Defines values for RateLimitOrderByDirection. const ( Asc RateLimitOrderByDirection = "asc" @@ -200,8 +217,8 @@ const ( // Defines values for V1CELDebugResponseStatus. const ( - V1CELDebugResponseStatusERROR V1CELDebugResponseStatus = "ERROR" - V1CELDebugResponseStatusSUCCESS V1CELDebugResponseStatus = "SUCCESS" + ERROR V1CELDebugResponseStatus = "ERROR" + SUCCESS V1CELDebugResponseStatus = "SUCCESS" ) // Defines values for V1CreateWebhookRequestAPIKeyAuthType. @@ -755,6 +772,36 @@ type LogLineOrderByField string // LogLineSearch defines model for LogLineSearch. type LogLineSearch = string +// OtelSpan defines model for OtelSpan. +type OtelSpan struct { + CreatedAt time.Time `json:"created_at"` + Duration int64 `json:"duration"` + ParentSpanId *string `json:"parent_span_id,omitempty"` + ResourceAttributes *map[string]string `json:"resource_attributes,omitempty"` + ScopeName *string `json:"scope_name,omitempty"` + ScopeVersion *string `json:"scope_version,omitempty"` + ServiceName string `json:"service_name"` + SpanAttributes *map[string]string `json:"span_attributes,omitempty"` + SpanId string `json:"span_id"` + SpanKind OtelSpanKind `json:"span_kind"` + SpanName string `json:"span_name"` + StatusCode OtelStatusCode `json:"status_code"` + StatusMessage *string `json:"status_message,omitempty"` + TraceId string `json:"trace_id"` +} + +// OtelSpanKind defines model for OtelSpanKind. +type OtelSpanKind string + +// OtelSpanList defines model for OtelSpanList. +type OtelSpanList struct { + Pagination *PaginationResponse `json:"pagination,omitempty"` + Rows *[]OtelSpan `json:"rows,omitempty"` +} + +// OtelStatusCode defines model for OtelStatusCode. +type OtelStatusCode string + // PaginationResponse defines model for PaginationResponse. type PaginationResponse struct { // CurrentPage the current page @@ -2494,6 +2541,15 @@ type V1TaskEventListParams struct { Limit *int64 `form:"limit,omitempty" json:"limit,omitempty"` } +// V1TaskGetTraceParams defines parameters for V1TaskGetTrace. +type V1TaskGetTraceParams struct { + // Offset The number to skip + Offset *int64 `form:"offset,omitempty" json:"offset,omitempty"` + + // Limit The number to limit by + Limit *int64 `form:"limit,omitempty" json:"limit,omitempty"` +} + // V1EventListParams defines parameters for V1EventList. type V1EventListParams struct { // Offset The number to skip @@ -3266,6 +3322,9 @@ type ClientInterface interface { // V1TaskEventList request V1TaskEventList(ctx context.Context, task openapi_types.UUID, params *V1TaskEventListParams, reqEditors ...RequestEditorFn) (*http.Response, error) + // V1TaskGetTrace request + V1TaskGetTrace(ctx context.Context, task openapi_types.UUID, params *V1TaskGetTraceParams, reqEditors ...RequestEditorFn) (*http.Response, error) + // V1CelDebugWithBody request with any body V1CelDebugWithBody(ctx context.Context, tenant openapi_types.UUID, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) @@ -3919,6 +3978,18 @@ func (c *Client) V1TaskEventList(ctx context.Context, task openapi_types.UUID, p return c.Client.Do(req) } +func (c *Client) V1TaskGetTrace(ctx context.Context, task openapi_types.UUID, params *V1TaskGetTraceParams, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewV1TaskGetTraceRequest(c.Server, task, params) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + if err := c.applyEditors(ctx, req, reqEditors); err != nil { + return nil, err + } + return c.Client.Do(req) +} + func (c *Client) V1CelDebugWithBody(ctx context.Context, tenant openapi_types.UUID, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) { req, err := NewV1CelDebugRequestWithBody(c.Server, tenant, contentType, body) if err != nil { @@ -6565,6 +6636,78 @@ func NewV1TaskEventListRequest(server string, task openapi_types.UUID, params *V return req, nil } +// NewV1TaskGetTraceRequest generates requests for V1TaskGetTrace +func NewV1TaskGetTraceRequest(server string, task openapi_types.UUID, params *V1TaskGetTraceParams) (*http.Request, error) { + var err error + + var pathParam0 string + + pathParam0, err = runtime.StyleParamWithLocation("simple", false, "task", runtime.ParamLocationPath, task) + if err != nil { + return nil, err + } + + serverURL, err := url.Parse(server) + if err != nil { + return nil, err + } + + operationPath := fmt.Sprintf("/api/v1/stable/tasks/%s/trace", pathParam0) + if operationPath[0] == '/' { + operationPath = "." + operationPath + } + + queryURL, err := serverURL.Parse(operationPath) + if err != nil { + return nil, err + } + + if params != nil { + queryValues := queryURL.Query() + + if params.Offset != nil { + + if queryFrag, err := runtime.StyleParamWithLocation("form", true, "offset", runtime.ParamLocationQuery, *params.Offset); err != nil { + return nil, err + } else if parsed, err := url.ParseQuery(queryFrag); err != nil { + return nil, err + } else { + for k, v := range parsed { + for _, v2 := range v { + queryValues.Add(k, v2) + } + } + } + + } + + if params.Limit != nil { + + if queryFrag, err := runtime.StyleParamWithLocation("form", true, "limit", runtime.ParamLocationQuery, *params.Limit); err != nil { + return nil, err + } else if parsed, err := url.ParseQuery(queryFrag); err != nil { + return nil, err + } else { + for k, v := range parsed { + for _, v2 := range v { + queryValues.Add(k, v2) + } + } + } + + } + + queryURL.RawQuery = queryValues.Encode() + } + + req, err := http.NewRequest("GET", queryURL.String(), nil) + if err != nil { + return nil, err + } + + return req, nil +} + // NewV1CelDebugRequest calls the generic V1CelDebug builder with application/json body func NewV1CelDebugRequest(server string, tenant openapi_types.UUID, body V1CelDebugJSONRequestBody) (*http.Request, error) { var bodyReader io.Reader @@ -13268,6 +13411,9 @@ type ClientWithResponsesInterface interface { // V1TaskEventListWithResponse request V1TaskEventListWithResponse(ctx context.Context, task openapi_types.UUID, params *V1TaskEventListParams, reqEditors ...RequestEditorFn) (*V1TaskEventListResponse, error) + // V1TaskGetTraceWithResponse request + V1TaskGetTraceWithResponse(ctx context.Context, task openapi_types.UUID, params *V1TaskGetTraceParams, reqEditors ...RequestEditorFn) (*V1TaskGetTraceResponse, error) + // V1CelDebugWithBodyWithResponse request with any body V1CelDebugWithBodyWithResponse(ctx context.Context, tenant openapi_types.UUID, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*V1CelDebugResponse, error) @@ -14119,6 +14265,31 @@ func (r V1TaskEventListResponse) StatusCode() int { return 0 } +type V1TaskGetTraceResponse struct { + Body []byte + HTTPResponse *http.Response + JSON200 *OtelSpanList + JSON400 *APIErrors + JSON403 *APIErrors + JSON404 *APIErrors +} + +// Status returns HTTPResponse.Status +func (r V1TaskGetTraceResponse) Status() string { + if r.HTTPResponse != nil { + return r.HTTPResponse.Status + } + return http.StatusText(0) +} + +// StatusCode returns HTTPResponse.StatusCode +func (r V1TaskGetTraceResponse) StatusCode() int { + if r.HTTPResponse != nil { + return r.HTTPResponse.StatusCode + } + return 0 +} + type V1CelDebugResponse struct { Body []byte HTTPResponse *http.Response @@ -17116,6 +17287,15 @@ func (c *ClientWithResponses) V1TaskEventListWithResponse(ctx context.Context, t return ParseV1TaskEventListResponse(rsp) } +// V1TaskGetTraceWithResponse request returning *V1TaskGetTraceResponse +func (c *ClientWithResponses) V1TaskGetTraceWithResponse(ctx context.Context, task openapi_types.UUID, params *V1TaskGetTraceParams, reqEditors ...RequestEditorFn) (*V1TaskGetTraceResponse, error) { + rsp, err := c.V1TaskGetTrace(ctx, task, params, reqEditors...) + if err != nil { + return nil, err + } + return ParseV1TaskGetTraceResponse(rsp) +} + // V1CelDebugWithBodyWithResponse request with arbitrary body returning *V1CelDebugResponse func (c *ClientWithResponses) V1CelDebugWithBodyWithResponse(ctx context.Context, tenant openapi_types.UUID, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*V1CelDebugResponse, error) { rsp, err := c.V1CelDebugWithBody(ctx, tenant, contentType, body, reqEditors...) @@ -19142,6 +19322,53 @@ func ParseV1TaskEventListResponse(rsp *http.Response) (*V1TaskEventListResponse, return response, nil } +// ParseV1TaskGetTraceResponse parses an HTTP response from a V1TaskGetTraceWithResponse call +func ParseV1TaskGetTraceResponse(rsp *http.Response) (*V1TaskGetTraceResponse, error) { + bodyBytes, err := io.ReadAll(rsp.Body) + defer func() { _ = rsp.Body.Close() }() + if err != nil { + return nil, err + } + + response := &V1TaskGetTraceResponse{ + Body: bodyBytes, + HTTPResponse: rsp, + } + + switch { + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 200: + var dest OtelSpanList + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON200 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 400: + var dest APIErrors + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON400 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 403: + var dest APIErrors + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON403 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 404: + var dest APIErrors + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON404 = &dest + + } + + return response, nil +} + // ParseV1CelDebugResponse parses an HTTP response from a V1CelDebugWithResponse call func ParseV1CelDebugResponse(rsp *http.Response) (*V1CelDebugResponse, error) { bodyBytes, err := io.ReadAll(rsp.Body) diff --git a/pkg/repository/otelcol.go b/pkg/repository/otelcol.go index 1f6c761425..fc7b8a066f 100644 --- a/pkg/repository/otelcol.go +++ b/pkg/repository/otelcol.go @@ -2,28 +2,37 @@ package repository import ( "context" + "encoding/hex" + "encoding/json" + "fmt" + "time" "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgtype" + tracev1 "go.opentelemetry.io/proto/otlp/trace/v1" + + "github.com/hatchet-dev/hatchet/pkg/repository/sqlchelpers" + "github.com/hatchet-dev/hatchet/pkg/repository/sqlcv1" ) type SpanData struct { + TenantID uuid.UUID WorkflowRunID *uuid.UUID TaskRunExternalID *uuid.UUID StatusMessage string InstrumentationScope string Name string - ResourceAttributes []byte - Attributes []byte - Events []byte - Links []byte + ResourceAttributes json.RawMessage + Attributes json.RawMessage + Events json.RawMessage + Links json.RawMessage TraceID []byte ParentSpanID []byte SpanID []byte EndTimeUnixNano uint64 StartTimeUnixNano uint64 - StatusCode int32 - Kind int32 - TenantID uuid.UUID + StatusCode tracev1.Status_StatusCode + Kind tracev1.Span_SpanKind } type CreateSpansOpts struct { @@ -31,8 +40,14 @@ type CreateSpansOpts struct { Spans []*SpanData } +type ListSpansResult struct { + Rows []*sqlcv1.ListSpansByTaskExternalIDRow + Total int64 +} + type OTelCollectorRepository interface { CreateSpans(ctx context.Context, tenantId uuid.UUID, opts *CreateSpansOpts) error + ListSpansByTaskExternalID(ctx context.Context, tenantId, taskExternalID uuid.UUID, offset, limit int64) (*ListSpansResult, error) } type otelCollectorRepositoryImpl struct { @@ -46,6 +61,150 @@ func newOTelCollectorRepository(s *sharedRepository) OTelCollectorRepository { } func (o *otelCollectorRepositoryImpl) CreateSpans(ctx context.Context, tenantId uuid.UUID, opts *CreateSpansOpts) error { - // intentional no-op, intended to be overridden + if opts == nil { + return fmt.Errorf("opts cannot be nil") + } + + if err := o.v.Validate(opts); err != nil { + return fmt.Errorf("validation error: %w", err) + } + + if len(opts.Spans) == 0 { + return nil + } + + params := make([]sqlcv1.InsertOtelSpansParams, len(opts.Spans)) + for i, sd := range opts.Spans { + var parentSpanID string + if len(sd.ParentSpanID) > 0 { + parentSpanID = hex.EncodeToString(sd.ParentSpanID) + } + + resourceAttrs := []byte(sd.ResourceAttributes) + if len(resourceAttrs) == 0 { + resourceAttrs = []byte("{}") + } + + spanAttrs := []byte(sd.Attributes) + if len(spanAttrs) == 0 { + spanAttrs = []byte("{}") + } + + var taskRunExternalID *uuid.UUID + if sd.TaskRunExternalID != nil && *sd.TaskRunExternalID != uuid.Nil { + id := *sd.TaskRunExternalID + taskRunExternalID = &id + } + + var workflowRunExternalID *uuid.UUID + if sd.WorkflowRunID != nil && *sd.WorkflowRunID != uuid.Nil { + id := *sd.WorkflowRunID + workflowRunExternalID = &id + } + + startTime := time.Unix(0, int64(sd.StartTimeUnixNano)) //nolint:gosec + + params[i] = sqlcv1.InsertOtelSpansParams{ + TenantID: tenantId, + TraceID: hex.EncodeToString(sd.TraceID), + SpanID: hex.EncodeToString(sd.SpanID), + ParentSpanID: parentSpanID, + SpanName: sd.Name, + SpanKind: protoSpanKindToDB(sd.Kind), + ServiceName: extractServiceName(sd.ResourceAttributes), + StatusCode: protoStatusCodeToDB(sd.StatusCode), + StatusMessage: sd.StatusMessage, + DurationNs: int64(sd.EndTimeUnixNano - sd.StartTimeUnixNano), //nolint:gosec + ResourceAttributes: resourceAttrs, + SpanAttributes: spanAttrs, + ScopeName: sd.InstrumentationScope, + TaskRunExternalID: taskRunExternalID, + WorkflowRunExternalID: workflowRunExternalID, + StartTime: pgtype.Timestamptz{Time: startTime, Valid: true}, + } + } + + _, err := o.queries.InsertOtelSpans(ctx, o.pool, params) + if err != nil { + return fmt.Errorf("error inserting otel spans: %w", err) + } + return nil } + +func (o *otelCollectorRepositoryImpl) ListSpansByTaskExternalID(ctx context.Context, tenantId, taskExternalID uuid.UUID, offset, limit int64) (*ListSpansResult, error) { + tx, commit, rollback, err := sqlchelpers.PrepareTx(ctx, o.pool, o.l) + if err != nil { + return nil, fmt.Errorf("error starting transaction: %w", err) + } + defer rollback() + + total, err := o.queries.CountSpansByTaskExternalID(ctx, tx, sqlcv1.CountSpansByTaskExternalIDParams{ + Tenantid: tenantId, + Taskexternalid: taskExternalID, + }) + if err != nil { + return nil, fmt.Errorf("error counting otel spans: %w", err) + } + + rows, err := o.queries.ListSpansByTaskExternalID(ctx, tx, sqlcv1.ListSpansByTaskExternalIDParams{ + Tenantid: tenantId, + Taskexternalid: taskExternalID, + Spanoffset: offset, + Spanlimit: limit, + }) + if err != nil { + return nil, fmt.Errorf("error listing otel spans: %w", err) + } + + if err := commit(ctx); err != nil { + return nil, fmt.Errorf("error committing transaction: %w", err) + } + + return &ListSpansResult{Rows: rows, Total: total}, nil +} + +func extractServiceName(resourceAttrsJSON json.RawMessage) string { + if len(resourceAttrsJSON) == 0 { + return "unknown" + } + + var attrs map[string]interface{} + if err := json.Unmarshal(resourceAttrsJSON, &attrs); err != nil { + return "unknown" + } + + if serviceName, ok := attrs["service.name"].(string); ok { + return serviceName + } + + return "unknown" +} + +func protoSpanKindToDB(kind tracev1.Span_SpanKind) sqlcv1.V1OtelSpanKind { + switch kind { + case tracev1.Span_SPAN_KIND_INTERNAL: + return sqlcv1.V1OtelSpanKindINTERNAL + case tracev1.Span_SPAN_KIND_SERVER: + return sqlcv1.V1OtelSpanKindSERVER + case tracev1.Span_SPAN_KIND_CLIENT: + return sqlcv1.V1OtelSpanKindCLIENT + case tracev1.Span_SPAN_KIND_PRODUCER: + return sqlcv1.V1OtelSpanKindPRODUCER + case tracev1.Span_SPAN_KIND_CONSUMER: + return sqlcv1.V1OtelSpanKindCONSUMER + default: + return sqlcv1.V1OtelSpanKindUNSPECIFIED + } +} + +func protoStatusCodeToDB(code tracev1.Status_StatusCode) sqlcv1.V1OtelStatusCode { + switch code { + case tracev1.Status_STATUS_CODE_OK: + return sqlcv1.V1OtelStatusCodeOK + case tracev1.Status_STATUS_CODE_ERROR: + return sqlcv1.V1OtelStatusCodeERROR + default: + return sqlcv1.V1OtelStatusCodeUNSET + } +} diff --git a/pkg/repository/sqlcv1/copyfrom.go b/pkg/repository/sqlcv1/copyfrom.go index f277818094..c0dce3b85a 100644 --- a/pkg/repository/sqlcv1/copyfrom.go +++ b/pkg/repository/sqlcv1/copyfrom.go @@ -371,3 +371,51 @@ func (r iteratorForInsertLogLine) Err() error { func (q *Queries) InsertLogLine(ctx context.Context, db DBTX, arg []InsertLogLineParams) (int64, error) { return db.CopyFrom(ctx, []string{"v1_log_line"}, []string{"tenant_id", "task_id", "task_inserted_at", "message", "metadata", "retry_count", "level"}, &iteratorForInsertLogLine{rows: arg}) } + +// iteratorForInsertOtelSpans implements pgx.CopyFromSource. +type iteratorForInsertOtelSpans struct { + rows []InsertOtelSpansParams + skippedFirstNextCall bool +} + +func (r *iteratorForInsertOtelSpans) Next() bool { + if len(r.rows) == 0 { + return false + } + if !r.skippedFirstNextCall { + r.skippedFirstNextCall = true + return true + } + r.rows = r.rows[1:] + return len(r.rows) > 0 +} + +func (r iteratorForInsertOtelSpans) Values() ([]interface{}, error) { + return []interface{}{ + r.rows[0].TenantID, + r.rows[0].TraceID, + r.rows[0].SpanID, + r.rows[0].ParentSpanID, + r.rows[0].SpanName, + r.rows[0].SpanKind, + r.rows[0].ServiceName, + r.rows[0].StatusCode, + r.rows[0].StatusMessage, + r.rows[0].DurationNs, + r.rows[0].ResourceAttributes, + r.rows[0].SpanAttributes, + r.rows[0].ScopeName, + r.rows[0].ScopeVersion, + r.rows[0].TaskRunExternalID, + r.rows[0].WorkflowRunExternalID, + r.rows[0].StartTime, + }, nil +} + +func (r iteratorForInsertOtelSpans) Err() error { + return nil +} + +func (q *Queries) InsertOtelSpans(ctx context.Context, db DBTX, arg []InsertOtelSpansParams) (int64, error) { + return db.CopyFrom(ctx, []string{"v1_otel_traces"}, []string{"tenant_id", "trace_id", "span_id", "parent_span_id", "span_name", "span_kind", "service_name", "status_code", "status_message", "duration_ns", "resource_attributes", "span_attributes", "scope_name", "scope_version", "task_run_external_id", "workflow_run_external_id", "start_time"}, &iteratorForInsertOtelSpans{rows: arg}) +} diff --git a/pkg/repository/sqlcv1/models.go b/pkg/repository/sqlcv1/models.go index bc6d5c5583..b26b90095b 100644 --- a/pkg/repository/sqlcv1/models.go +++ b/pkg/repository/sqlcv1/models.go @@ -1397,6 +1397,95 @@ func (ns NullV1MatchKind) Value() (driver.Value, error) { return string(ns.V1MatchKind), nil } +type V1OtelSpanKind string + +const ( + V1OtelSpanKindUNSPECIFIED V1OtelSpanKind = "UNSPECIFIED" + V1OtelSpanKindINTERNAL V1OtelSpanKind = "INTERNAL" + V1OtelSpanKindSERVER V1OtelSpanKind = "SERVER" + V1OtelSpanKindCLIENT V1OtelSpanKind = "CLIENT" + V1OtelSpanKindPRODUCER V1OtelSpanKind = "PRODUCER" + V1OtelSpanKindCONSUMER V1OtelSpanKind = "CONSUMER" +) + +func (e *V1OtelSpanKind) Scan(src interface{}) error { + switch s := src.(type) { + case []byte: + *e = V1OtelSpanKind(s) + case string: + *e = V1OtelSpanKind(s) + default: + return fmt.Errorf("unsupported scan type for V1OtelSpanKind: %T", src) + } + return nil +} + +type NullV1OtelSpanKind struct { + V1OtelSpanKind V1OtelSpanKind `json:"v1_otel_span_kind"` + Valid bool `json:"valid"` // Valid is true if V1OtelSpanKind is not NULL +} + +// Scan implements the Scanner interface. +func (ns *NullV1OtelSpanKind) Scan(value interface{}) error { + if value == nil { + ns.V1OtelSpanKind, ns.Valid = "", false + return nil + } + ns.Valid = true + return ns.V1OtelSpanKind.Scan(value) +} + +// Value implements the driver Valuer interface. +func (ns NullV1OtelSpanKind) Value() (driver.Value, error) { + if !ns.Valid { + return nil, nil + } + return string(ns.V1OtelSpanKind), nil +} + +type V1OtelStatusCode string + +const ( + V1OtelStatusCodeUNSET V1OtelStatusCode = "UNSET" + V1OtelStatusCodeOK V1OtelStatusCode = "OK" + V1OtelStatusCodeERROR V1OtelStatusCode = "ERROR" +) + +func (e *V1OtelStatusCode) Scan(src interface{}) error { + switch s := src.(type) { + case []byte: + *e = V1OtelStatusCode(s) + case string: + *e = V1OtelStatusCode(s) + default: + return fmt.Errorf("unsupported scan type for V1OtelStatusCode: %T", src) + } + return nil +} + +type NullV1OtelStatusCode struct { + V1OtelStatusCode V1OtelStatusCode `json:"v1_otel_status_code"` + Valid bool `json:"valid"` // Valid is true if V1OtelStatusCode is not NULL +} + +// Scan implements the Scanner interface. +func (ns *NullV1OtelStatusCode) Scan(value interface{}) error { + if value == nil { + ns.V1OtelStatusCode, ns.Valid = "", false + return nil + } + ns.Valid = true + return ns.V1OtelStatusCode.Scan(value) +} + +// Value implements the driver Valuer interface. +func (ns NullV1OtelStatusCode) Value() (driver.Value, error) { + if !ns.Valid { + return nil, nil + } + return string(ns.V1OtelStatusCode), nil +} + type V1PayloadLocation string const ( @@ -3203,6 +3292,27 @@ type V1OperationIntervalSettings struct { IntervalNanoseconds int64 `json:"interval_nanoseconds"` } +type V1OtelTraces struct { + ID int64 `json:"id"` + TenantID uuid.UUID `json:"tenant_id"` + TraceID string `json:"trace_id"` + SpanID string `json:"span_id"` + ParentSpanID string `json:"parent_span_id"` + SpanName string `json:"span_name"` + SpanKind V1OtelSpanKind `json:"span_kind"` + ServiceName string `json:"service_name"` + StatusCode V1OtelStatusCode `json:"status_code"` + StatusMessage string `json:"status_message"` + DurationNs int64 `json:"duration_ns"` + ResourceAttributes []byte `json:"resource_attributes"` + SpanAttributes []byte `json:"span_attributes"` + ScopeName string `json:"scope_name"` + ScopeVersion string `json:"scope_version"` + TaskRunExternalID *uuid.UUID `json:"task_run_external_id"` + WorkflowRunExternalID *uuid.UUID `json:"workflow_run_external_id"` + StartTime pgtype.Timestamptz `json:"start_time"` +} + type V1Payload struct { TenantID uuid.UUID `json:"tenant_id"` ID int64 `json:"id"` diff --git a/pkg/repository/sqlcv1/otel.sql b/pkg/repository/sqlcv1/otel.sql new file mode 100644 index 0000000000..a8b6c9ccd1 --- /dev/null +++ b/pkg/repository/sqlcv1/otel.sql @@ -0,0 +1,43 @@ +-- name: InsertOtelSpans :copyfrom +INSERT INTO v1_otel_traces ( + tenant_id, + trace_id, + span_id, + parent_span_id, + span_name, + span_kind, + service_name, + status_code, + status_message, + duration_ns, + resource_attributes, + span_attributes, + scope_name, + scope_version, + task_run_external_id, + workflow_run_external_id, + start_time +) VALUES ( + $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17 +); + +-- name: CountSpansByTaskExternalID :one +SELECT COUNT(*) FROM v1_otel_traces +WHERE tenant_id = @tenantId::UUID AND trace_id IN ( + SELECT DISTINCT trace_id FROM v1_otel_traces + WHERE tenant_id = @tenantId::UUID AND task_run_external_id = @taskExternalId::UUID +); + +-- name: ListSpansByTaskExternalID :many +SELECT + trace_id, span_id, parent_span_id, span_name, span_kind, + service_name, status_code, status_message, duration_ns, start_time, + resource_attributes, span_attributes, scope_name, scope_version +FROM v1_otel_traces +WHERE tenant_id = @tenantId::UUID AND trace_id IN ( + SELECT DISTINCT trace_id FROM v1_otel_traces + WHERE tenant_id = @tenantId::UUID AND task_run_external_id = @taskExternalId::UUID +) +ORDER BY start_time ASC +OFFSET COALESCE(@spanOffset::BIGINT, 0) +LIMIT COALESCE(@spanLimit::BIGINT, 1000); diff --git a/pkg/repository/sqlcv1/otel.sql.go b/pkg/repository/sqlcv1/otel.sql.go new file mode 100644 index 0000000000..de79ad0552 --- /dev/null +++ b/pkg/repository/sqlcv1/otel.sql.go @@ -0,0 +1,132 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.29.0 +// source: otel.sql + +package sqlcv1 + +import ( + "context" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgtype" +) + +const countSpansByTaskExternalID = `-- name: CountSpansByTaskExternalID :one +SELECT COUNT(*) FROM v1_otel_traces +WHERE tenant_id = $1::UUID AND trace_id IN ( + SELECT DISTINCT trace_id FROM v1_otel_traces + WHERE tenant_id = $1::UUID AND task_run_external_id = $2::UUID +) +` + +type CountSpansByTaskExternalIDParams struct { + Tenantid uuid.UUID `json:"tenantid"` + Taskexternalid uuid.UUID `json:"taskexternalid"` +} + +func (q *Queries) CountSpansByTaskExternalID(ctx context.Context, db DBTX, arg CountSpansByTaskExternalIDParams) (int64, error) { + row := db.QueryRow(ctx, countSpansByTaskExternalID, arg.Tenantid, arg.Taskexternalid) + var count int64 + err := row.Scan(&count) + return count, err +} + +type InsertOtelSpansParams struct { + TenantID uuid.UUID `json:"tenant_id"` + TraceID string `json:"trace_id"` + SpanID string `json:"span_id"` + ParentSpanID string `json:"parent_span_id"` + SpanName string `json:"span_name"` + SpanKind V1OtelSpanKind `json:"span_kind"` + ServiceName string `json:"service_name"` + StatusCode V1OtelStatusCode `json:"status_code"` + StatusMessage string `json:"status_message"` + DurationNs int64 `json:"duration_ns"` + ResourceAttributes []byte `json:"resource_attributes"` + SpanAttributes []byte `json:"span_attributes"` + ScopeName string `json:"scope_name"` + ScopeVersion string `json:"scope_version"` + TaskRunExternalID *uuid.UUID `json:"task_run_external_id"` + WorkflowRunExternalID *uuid.UUID `json:"workflow_run_external_id"` + StartTime pgtype.Timestamptz `json:"start_time"` +} + +const listSpansByTaskExternalID = `-- name: ListSpansByTaskExternalID :many +SELECT + trace_id, span_id, parent_span_id, span_name, span_kind, + service_name, status_code, status_message, duration_ns, start_time, + resource_attributes, span_attributes, scope_name, scope_version +FROM v1_otel_traces +WHERE tenant_id = $1::UUID AND trace_id IN ( + SELECT DISTINCT trace_id FROM v1_otel_traces + WHERE tenant_id = $1::UUID AND task_run_external_id = $2::UUID +) +ORDER BY start_time ASC +OFFSET COALESCE($3::BIGINT, 0) +LIMIT COALESCE($4::BIGINT, 1000) +` + +type ListSpansByTaskExternalIDParams struct { + Tenantid uuid.UUID `json:"tenantid"` + Taskexternalid uuid.UUID `json:"taskexternalid"` + Spanoffset int64 `json:"spanoffset"` + Spanlimit int64 `json:"spanlimit"` +} + +type ListSpansByTaskExternalIDRow struct { + TraceID string `json:"trace_id"` + SpanID string `json:"span_id"` + ParentSpanID string `json:"parent_span_id"` + SpanName string `json:"span_name"` + SpanKind V1OtelSpanKind `json:"span_kind"` + ServiceName string `json:"service_name"` + StatusCode V1OtelStatusCode `json:"status_code"` + StatusMessage string `json:"status_message"` + DurationNs int64 `json:"duration_ns"` + StartTime pgtype.Timestamptz `json:"start_time"` + ResourceAttributes []byte `json:"resource_attributes"` + SpanAttributes []byte `json:"span_attributes"` + ScopeName string `json:"scope_name"` + ScopeVersion string `json:"scope_version"` +} + +func (q *Queries) ListSpansByTaskExternalID(ctx context.Context, db DBTX, arg ListSpansByTaskExternalIDParams) ([]*ListSpansByTaskExternalIDRow, error) { + rows, err := db.Query(ctx, listSpansByTaskExternalID, + arg.Tenantid, + arg.Taskexternalid, + arg.Spanoffset, + arg.Spanlimit, + ) + if err != nil { + return nil, err + } + defer rows.Close() + var items []*ListSpansByTaskExternalIDRow + for rows.Next() { + var i ListSpansByTaskExternalIDRow + if err := rows.Scan( + &i.TraceID, + &i.SpanID, + &i.ParentSpanID, + &i.SpanName, + &i.SpanKind, + &i.ServiceName, + &i.StatusCode, + &i.StatusMessage, + &i.DurationNs, + &i.StartTime, + &i.ResourceAttributes, + &i.SpanAttributes, + &i.ScopeName, + &i.ScopeVersion, + ); err != nil { + return nil, err + } + items = append(items, &i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} diff --git a/pkg/repository/sqlcv1/sqlc.yaml b/pkg/repository/sqlcv1/sqlc.yaml index 966444d156..9417ee6c57 100644 --- a/pkg/repository/sqlcv1/sqlc.yaml +++ b/pkg/repository/sqlcv1/sqlc.yaml @@ -20,6 +20,7 @@ sql: - olap.sql - rate_limits.sql - log_line.sql + - otel.sql - security_check.sql - slack.sql - sleep.sql diff --git a/pkg/repository/sqlcv1/tasks.sql b/pkg/repository/sqlcv1/tasks.sql index 4639bfcb18..6a8b35935f 100644 --- a/pkg/repository/sqlcv1/tasks.sql +++ b/pkg/repository/sqlcv1/tasks.sql @@ -7,7 +7,8 @@ SELECT create_v1_range_partition('v1_payload', @date::date), create_v1_range_partition('v1_event', @date::date), create_v1_weekly_range_partition('v1_event_lookup_table', @date::date), - create_v1_range_partition('v1_event_to_run', @date::date); + create_v1_range_partition('v1_event_to_run', @date::date), + create_v1_range_partition('v1_otel_traces', @date::date); -- name: EnsureTablePartitionsExist :one WITH tomorrow_date AS ( @@ -25,6 +26,8 @@ WITH tomorrow_date AS ( SELECT 'v1_payload_' || to_char((SELECT date FROM tomorrow_date), 'YYYYMMDD') UNION ALL SELECT 'v1_event_' || to_char((SELECT date FROM tomorrow_date), 'YYYYMMDD') + UNION ALL + SELECT 'v1_otel_traces_' || to_char((SELECT date FROM tomorrow_date), 'YYYYMMDD') ), partition_check AS ( SELECT COUNT(*) AS total_tables, @@ -56,6 +59,8 @@ WITH task_partitions AS ( SELECT 'v1_event_lookup_table' AS parent_table, p::text as partition_name FROM get_v1_weekly_partitions_before_date('v1_event_lookup_table', @date::date) AS p ), event_to_run_partitions AS ( SELECT 'v1_event_to_run' AS parent_table, p::text as partition_name FROM get_v1_partitions_before_date('v1_event_to_run', @date::date) AS p +), otel_traces_partitions AS ( + SELECT 'v1_otel_traces' AS parent_table, p::text as partition_name FROM get_v1_partitions_before_date('v1_otel_traces', @date::date) AS p ) SELECT @@ -111,6 +116,13 @@ SELECT * FROM event_to_run_partitions + +UNION ALL + +SELECT + * +FROM + otel_traces_partitions ; -- name: DefaultTaskActivityGauge :one diff --git a/pkg/repository/sqlcv1/tasks.sql.go b/pkg/repository/sqlcv1/tasks.sql.go index 673c2b6243..ce43061fef 100644 --- a/pkg/repository/sqlcv1/tasks.sql.go +++ b/pkg/repository/sqlcv1/tasks.sql.go @@ -233,7 +233,8 @@ SELECT create_v1_range_partition('v1_payload', $1::date), create_v1_range_partition('v1_event', $1::date), create_v1_weekly_range_partition('v1_event_lookup_table', $1::date), - create_v1_range_partition('v1_event_to_run', $1::date) + create_v1_range_partition('v1_event_to_run', $1::date), + create_v1_range_partition('v1_otel_traces', $1::date) ` func (q *Queries) CreatePartitions(ctx context.Context, db DBTX, date pgtype.Date) error { @@ -329,6 +330,8 @@ WITH tomorrow_date AS ( SELECT 'v1_payload_' || to_char((SELECT date FROM tomorrow_date), 'YYYYMMDD') UNION ALL SELECT 'v1_event_' || to_char((SELECT date FROM tomorrow_date), 'YYYYMMDD') + UNION ALL + SELECT 'v1_otel_traces_' || to_char((SELECT date FROM tomorrow_date), 'YYYYMMDD') ), partition_check AS ( SELECT COUNT(*) AS total_tables, @@ -1242,6 +1245,8 @@ WITH task_partitions AS ( SELECT 'v1_event_lookup_table' AS parent_table, p::text as partition_name FROM get_v1_weekly_partitions_before_date('v1_event_lookup_table', $1::date) AS p ), event_to_run_partitions AS ( SELECT 'v1_event_to_run' AS parent_table, p::text as partition_name FROM get_v1_partitions_before_date('v1_event_to_run', $1::date) AS p +), otel_traces_partitions AS ( + SELECT 'v1_otel_traces' AS parent_table, p::text as partition_name FROM get_v1_partitions_before_date('v1_otel_traces', $1::date) AS p ) SELECT @@ -1297,6 +1302,13 @@ SELECT parent_table, partition_name FROM event_to_run_partitions + +UNION ALL + +SELECT + parent_table, partition_name +FROM + otel_traces_partitions ` type ListPartitionsBeforeDateRow struct { diff --git a/pkg/v1/features/cel.go b/pkg/v1/features/cel.go index 93bba02396..b8b71eeeba 100644 --- a/pkg/v1/features/cel.go +++ b/pkg/v1/features/cel.go @@ -64,15 +64,15 @@ func (c *celClientImpl) Debug(ctx context.Context, expression string, input map[ return nil, err } - if resp.JSON200.Status == rest.V1CELDebugResponseStatus(gen.V1CELDebugResponseStatusERROR) { + if resp.JSON200.Status == rest.V1CELDebugResponseStatus(gen.ERROR) { return &CELEvaluationResult{ - status: gen.V1CELDebugResponseStatusERROR, + status: gen.ERROR, err: resp.JSON200.Error, }, nil } return &CELEvaluationResult{ - status: gen.V1CELDebugResponseStatusSUCCESS, + status: gen.SUCCESS, output: resp.JSON200.Output, }, nil } diff --git a/pkg/worker/context.go b/pkg/worker/context.go index 3c9eec5925..367bbbdc1c 100644 --- a/pkg/worker/context.go +++ b/pkg/worker/context.go @@ -9,6 +9,7 @@ import ( "time" "github.com/rs/zerolog" + "go.opentelemetry.io/otel/propagation" "google.golang.org/protobuf/types/known/timestamppb" v1 "github.com/hatchet-dev/hatchet/internal/services/shared/proto/v1" @@ -91,6 +92,12 @@ type HatchetContext interface { WasSkipped(parent create.NamedTask) bool + TenantId() string + + WorkerId() string + + ActionId() string + client() client.Client action() *client.Action @@ -208,6 +215,18 @@ func (h *hatchetContext) action() *client.Action { return h.a } +func (h *hatchetContext) TenantId() string { + return h.a.TenantId +} + +func (h *hatchetContext) WorkerId() string { + return h.a.WorkerId +} + +func (h *hatchetContext) ActionId() string { + return h.a.ActionId +} + func (h *hatchetContext) Worker() HatchetWorkerContext { return h.w } @@ -481,6 +500,27 @@ func (h *hatchetContext) saveOrLoadListener() (*client.WorkflowRunsListener, err return h.client().Subscribe().SubscribeToWorkflowRunEvents(h) } +// injectTraceparent serializes the current span's W3C traceparent from ctx +// into the AdditionalMetadata map so child workflows inherit the trace. +func injectTraceparent(ctx context.Context, meta *map[string]string) *map[string]string { + propagator := propagation.TraceContext{} + carrier := propagation.MapCarrier{} + propagator.Inject(ctx, carrier) + + tp, ok := carrier["traceparent"] + if !ok || tp == "" { + return meta + } + + if meta == nil { + m := map[string]string{"traceparent": tp} + return &m + } + + (*meta)["traceparent"] = tp + return meta +} + // Deprecated: SpawnWorkflow is an internal method used by the new Go SDK. // Use the new Go SDK at github.com/hatchet-dev/hatchet/sdks/go instead of using this directly. Migration guide: https://docs.hatchet.run/home/migration-guide-go func (h *hatchetContext) SpawnWorkflow(workflowName string, input any, opts *SpawnWorkflowOpts) (*client.Workflow, error) { @@ -488,6 +528,8 @@ func (h *hatchetContext) SpawnWorkflow(workflowName string, input any, opts *Spa opts = &SpawnWorkflowOpts{} } + opts.AdditionalMetadata = injectTraceparent(h.GetContext(), opts.AdditionalMetadata) + var desiredWorker *string if opts.Sticky != nil { @@ -554,6 +596,7 @@ func (h *hatchetContext) SpawnWorkflows(childWorkflows []*SpawnWorkflowsOpts) ([ listener, err := h.saveOrLoadListener() for i, c := range childWorkflows { + c.AdditionalMetadata = injectTraceparent(h.GetContext(), c.AdditionalMetadata) var desiredWorker *string diff --git a/pkg/worker/middleware_test.go b/pkg/worker/middleware_test.go index 88ff3066ce..0dfb6eb6fa 100644 --- a/pkg/worker/middleware_test.go +++ b/pkg/worker/middleware_test.go @@ -146,6 +146,18 @@ func (c *testHatchetContext) FilterPayload() map[string]interface{} { panic("not implemented") } +func (c *testHatchetContext) ActionId() string { + panic("not implemented") +} + +func (c *testHatchetContext) TenantId() string { + panic("not implemented") +} + +func (c *testHatchetContext) WorkerId() string { + panic("not implemented") +} + func TestAddMiddleware(t *testing.T) { m := middlewares{} middlewareFunc := func(ctx HatchetContext, next func(HatchetContext) error) error { diff --git a/sdks/go/client.go b/sdks/go/client.go index 10827c00ac..57b51c03b0 100644 --- a/sdks/go/client.go +++ b/sdks/go/client.go @@ -7,6 +7,10 @@ import ( "fmt" "sync" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/trace" "golang.org/x/sync/errgroup" v1 "github.com/hatchet-dev/hatchet/internal/services/shared/proto/v1" @@ -238,6 +242,19 @@ func resolveWorkerSlotConfig( return slotConfig } +// Use registers middleware functions on the worker. +// Middleware functions are called in order for each step run execution. +// +//nolint:staticcheck // SA1019: worker.MiddlewareFunc is deprecated but still used internally +func (w *Worker) Use(mws ...worker.MiddlewareFunc) { + if w.worker != nil { + w.worker.Use(mws...) //nolint:staticcheck // SA1019 + } + if w.legacyDurable != nil { + w.legacyDurable.Use(mws...) //nolint:staticcheck // SA1019 + } +} + // Starts the worker instance and returns a cleanup function. func (w *Worker) Start() (func() error, error) { var workers []*worker.Worker @@ -602,6 +619,27 @@ func (c *Client) Run(ctx context.Context, workflowName string, input any, opts . // RunNoWait executes a workflow with the provided input without waiting for completion. // Returns a workflow run reference that can be used to track the run status. func (c *Client) RunNoWait(ctx context.Context, workflowName string, input any, opts ...RunOptFunc) (*WorkflowRunRef, error) { + // Start OTel span using the underlying context (not the HatchetContext itself) + // so that the HatchetContext type assertion still works downstream. + tracer := otel.Tracer("github.com/hatchet-dev/hatchet/sdks/go") + otelCtx := ctx + if hCtx, ok := ctx.(Context); ok { + otelCtx = hCtx.GetContext() + } + otelCtx, span := tracer.Start(otelCtx, fmt.Sprintf("hatchet trigger task %s", workflowName), + trace.WithSpanKind(trace.SpanKindClient), + trace.WithAttributes(attribute.String("hatchet.task_name", workflowName)), + ) + defer span.End() + + // Update the HatchetContext's inner context with the OTel span context, + // or use the OTel context directly for plain context.Context callers. + if hCtx, ok := ctx.(Context); ok { + hCtx.SetContext(otelCtx) + } else { + ctx = otelCtx + } + runOpts := &runOpts{} for _, opt := range opts { opt(runOpts) @@ -620,6 +658,9 @@ func (c *Client) RunNoWait(ctx context.Context, workflowName string, input any, } } + // Inject traceparent for cross-workflow trace propagation + additionalMetadata = injectTraceparentToMap(otelCtx, additionalMetadata) + var v0Opts []v0Client.RunOptFunc if additionalMetadata != nil { v0Opts = append(v0Opts, v0Client.WithRunMetadata(*additionalMetadata)) @@ -644,9 +685,14 @@ func (c *Client) RunNoWait(ctx context.Context, workflowName string, input any, } if err != nil { + span.SetStatus(codes.Error, err.Error()) + span.RecordError(err) return nil, err } + span.SetAttributes(attribute.String("hatchet.workflow_run_id", v0Workflow.RunId())) + span.SetStatus(codes.Ok, "") + return &WorkflowRunRef{RunId: v0Workflow.RunId(), v0Workflow: v0Workflow}, nil } diff --git a/sdks/go/examples/opentelemetry/main.go b/sdks/go/examples/opentelemetry/main.go new file mode 100644 index 0000000000..4a9920aaf4 --- /dev/null +++ b/sdks/go/examples/opentelemetry/main.go @@ -0,0 +1,133 @@ +package main + +import ( + "context" + "fmt" + "log" + "math/rand" + "time" + + "go.opentelemetry.io/otel" + + "github.com/hatchet-dev/hatchet/pkg/cmdutils" + hatchet "github.com/hatchet-dev/hatchet/sdks/go" + hatchetotel "github.com/hatchet-dev/hatchet/sdks/go/opentelemetry" +) + +type ParentInput struct { + Name string `json:"name"` +} + +type ParentOutput struct { + ChildResult string `json:"child_result"` +} + +type ChildInput struct { + Greeting string `json:"greeting"` +} + +type ChildOutput struct { + Message string `json:"message"` +} + +func main() { + client, err := hatchet.NewClient() + if err != nil { + log.Fatalf("failed to create client: %v", err) + } + + instrumentor, err := hatchetotel.NewInstrumentor( + hatchetotel.EnableHatchetCollector(), + ) + if err != nil { + log.Fatalf("failed to create instrumentor: %v", err) + } + + tracer := otel.Tracer("otel-example") + + var generateSpanTree func(ctx context.Context, count *int, limit int, depth int, prefix string) + generateSpanTree = func(ctx context.Context, count *int, limit int, depth int, prefix string) { + numChildren := 1 + rand.Intn(5) + for i := range numChildren { + if *count >= limit { + return + } + name := fmt.Sprintf("%s.%d", prefix, i) + childCtx, span := tracer.Start(ctx, name) + time.Sleep(time.Duration(1+rand.Intn(3)) * time.Millisecond) + *count++ + + if *count < limit && depth < 8 && rand.Float64() > float64(depth)*0.12 { + generateSpanTree(childCtx, count, limit, depth+1, name) + } + + span.End() + } + } + + childTask := client.NewStandaloneTask( + "otel-child-task", + func(ctx hatchet.Context, input ChildInput) (ChildOutput, error) { + target := 200 + rand.Intn(101) + count := 0 + round := 0 + for count < target { + generateSpanTree(ctx.GetContext(), &count, target, 0, fmt.Sprintf("child.r%d", round)) + round++ + } + + return ChildOutput{ + Message: fmt.Sprintf("Hello from child: %s (generated %d spans)", input.Greeting, count), + }, nil + }, + ) + + parentTask := client.NewStandaloneTask( + "otel-parent-task", + func(ctx hatchet.Context, input ParentInput) (ParentOutput, error) { + _, span := tracer.Start(ctx.GetContext(), "parent.prepare") + time.Sleep(30 * time.Millisecond) + span.End() + + result, err := childTask.Run(ctx, ChildInput{ + Greeting: fmt.Sprintf("greetings from %s", input.Name), + }) + if err != nil { + return ParentOutput{}, fmt.Errorf("child task failed: %w", err) + } + + var childOutput ChildOutput + if err := result.Into(&childOutput); err != nil { + return ParentOutput{}, fmt.Errorf("failed to parse child output: %w", err) + } + + return ParentOutput{ + ChildResult: childOutput.Message, + }, nil + }, + ) + + worker, err := client.NewWorker( + "otel-worker", + hatchet.WithWorkflows(parentTask, childTask), + ) + if err != nil { + log.Fatalf("failed to create worker: %v", err) + } + + worker.Use(instrumentor.Middleware()) + + interruptCtx, cancel := cmdutils.NewInterruptContext() + defer cancel() + + go func() { + <-interruptCtx.Done() + if shutdownErr := instrumentor.Shutdown(context.Background()); shutdownErr != nil { + log.Printf("failed to shutdown instrumentor: %v", shutdownErr) + } + }() + + if startErr := worker.StartBlocking(interruptCtx); startErr != nil { + log.Printf("worker error: %v", startErr) + } +} diff --git a/sdks/go/opentelemetry/attributes.go b/sdks/go/opentelemetry/attributes.go new file mode 100644 index 0000000000..7518a0d5f4 --- /dev/null +++ b/sdks/go/opentelemetry/attributes.go @@ -0,0 +1,45 @@ +package opentelemetry + +import ( + "context" + + "go.opentelemetry.io/otel/attribute" + + "github.com/hatchet-dev/hatchet/pkg/worker" +) + +type hatchetAttrsKeyType struct{} + +var hatchetAttrsKey = hatchetAttrsKeyType{} + +// hatchetAttributes builds the set of hatchet.* span attributes from a HatchetContext. +func hatchetAttributes(ctx worker.HatchetContext) []attribute.KeyValue { + attrs := []attribute.KeyValue{ + attribute.String("hatchet.tenant_id", ctx.TenantId()), + attribute.String("hatchet.worker_id", ctx.WorkerId()), + attribute.String("hatchet.workflow_run_id", ctx.WorkflowRunId()), + attribute.String("hatchet.step_run_id", ctx.StepRunId()), + attribute.String("hatchet.step_id", ctx.StepId()), + attribute.String("hatchet.action_id", ctx.ActionId()), + attribute.String("hatchet.step_name", ctx.StepName()), + attribute.Int("hatchet.retry_count", ctx.RetryCount()), + } + + if wfID := ctx.WorkflowId(); wfID != nil { + attrs = append(attrs, attribute.String("hatchet.workflow_id", *wfID)) + } + + return attrs +} + +// withHatchetAttributes stores hatchet attributes in the context so the +// SpanProcessor can inject them into child spans. +func withHatchetAttributes(ctx context.Context, attrs []attribute.KeyValue) context.Context { + return context.WithValue(ctx, hatchetAttrsKey, attrs) +} + +// getHatchetAttributes retrieves hatchet attributes from the context. +func getHatchetAttributes(ctx context.Context) []attribute.KeyValue { + attrs, _ := ctx.Value(hatchetAttrsKey).([]attribute.KeyValue) + return attrs +} diff --git a/sdks/go/opentelemetry/exporter.go b/sdks/go/opentelemetry/exporter.go new file mode 100644 index 0000000000..477d7c3011 --- /dev/null +++ b/sdks/go/opentelemetry/exporter.go @@ -0,0 +1,30 @@ +package opentelemetry + +import ( + "context" + "crypto/tls" + + "google.golang.org/grpc/credentials" + + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc" + sdktrace "go.opentelemetry.io/otel/sdk/trace" +) + +// newHatchetExporter creates a gRPC OTLP trace exporter that sends spans +// to the Hatchet engine's collector endpoint. +func newHatchetExporter(endpoint, token string, insecureConn bool) (sdktrace.SpanExporter, error) { + opts := []otlptracegrpc.Option{ + otlptracegrpc.WithEndpoint(endpoint), + otlptracegrpc.WithHeaders(map[string]string{ + "authorization": "Bearer " + token, + }), + } + + if insecureConn { + opts = append(opts, otlptracegrpc.WithInsecure()) + } else { + opts = append(opts, otlptracegrpc.WithTLSCredentials(credentials.NewTLS(&tls.Config{MinVersion: tls.VersionTLS12}))) //nolint:gosec // G402 + } + + return otlptracegrpc.New(context.Background(), opts...) +} diff --git a/sdks/go/opentelemetry/instrumentor.go b/sdks/go/opentelemetry/instrumentor.go new file mode 100644 index 0000000000..51e3537cd1 --- /dev/null +++ b/sdks/go/opentelemetry/instrumentor.go @@ -0,0 +1,139 @@ +// Package opentelemetry provides OpenTelemetry instrumentation for the Hatchet Go SDK. +// +// It automatically creates spans for each step run, propagates hatchet.* attributes +// to all child spans, supports W3C traceparent propagation, and optionally sends +// traces to the Hatchet engine's OTLP collector. +// +// Basic usage: +// +// instrumentor := opentelemetry.NewInstrumentor() +// worker.Use(instrumentor.Middleware()) +// +// Sending traces to Hatchet (auto-configured from environment): +// +// instrumentor := opentelemetry.NewInstrumentor( +// opentelemetry.EnableHatchetCollector(), +// ) +// worker.Use(instrumentor.Middleware()) +package opentelemetry + +import ( + "context" + "fmt" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/sdk/resource" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + semconv "go.opentelemetry.io/otel/semconv/v1.26.0" + "go.opentelemetry.io/otel/trace" + + "github.com/hatchet-dev/hatchet/pkg/client/loader" + "github.com/hatchet-dev/hatchet/pkg/worker" +) + +// Instrumentor sets up OpenTelemetry tracing for Hatchet workers. +type Instrumentor struct { + tracerProvider trace.TracerProvider + tracer trace.Tracer + opts instrumentorOptions +} + +type instrumentorOptions struct { + tracerProvider *sdktrace.TracerProvider + + enableCollector bool +} + +// InstrumentorOption configures the Instrumentor. +type InstrumentorOption func(*instrumentorOptions) + +// WithTracerProvider sets a custom TracerProvider. If not set, a new one is created. +// The provider must be an SDK TracerProvider to support adding span processors. +func WithTracerProvider(tp *sdktrace.TracerProvider) InstrumentorOption { + return func(o *instrumentorOptions) { + o.tracerProvider = tp + } +} + +// EnableHatchetCollector enables sending traces to the Hatchet engine's OTLP collector. +// Connection settings (endpoint, token, TLS) are automatically loaded from the same +// environment variables used by the Hatchet client (HATCHET_CLIENT_HOST_PORT, +// HATCHET_CLIENT_TOKEN, HATCHET_CLIENT_TLS_STRATEGY). +func EnableHatchetCollector() InstrumentorOption { + return func(o *instrumentorOptions) { + o.enableCollector = true + } +} + +// NewInstrumentor creates a new HatchetInstrumentor. +func NewInstrumentor(opts ...InstrumentorOption) (*Instrumentor, error) { + o := &instrumentorOptions{} + for _, opt := range opts { + opt(o) + } + + // Set up TracerProvider + tp := o.tracerProvider + if tp == nil { + tp = sdktrace.NewTracerProvider( + sdktrace.WithResource(resource.NewWithAttributes( + semconv.SchemaURL, + semconv.ServiceNameKey.String("hatchet-worker"), + )), + ) + } + + // Add Hatchet collector exporter if enabled + if o.enableCollector { + cfgFile, err := loader.LoadClientConfigFile() + if err != nil { + return nil, fmt.Errorf("failed to load client config for OTel collector: %w", err) + } + + clientCfg, err := loader.GetClientConfigFromConfigFile(nil, cfgFile) + if err != nil { + return nil, fmt.Errorf("failed to resolve client config for OTel collector: %w", err) + } + + insecure := clientCfg.TLSConfig == nil + + exporter, err := newHatchetExporter(clientCfg.GRPCBroadcastAddress, clientCfg.Token, insecure) + if err != nil { + return nil, fmt.Errorf("failed to create Hatchet OTLP exporter: %w", err) + } + bsp := sdktrace.NewBatchSpanProcessor(exporter) + tp.RegisterSpanProcessor(NewHatchetAttributeSpanProcessor(bsp)) + } + + // Set as global provider + otel.SetTracerProvider(tp) + + tracer := tp.Tracer("github.com/hatchet-dev/hatchet/sdks/go/opentelemetry") + + return &Instrumentor{ + tracerProvider: tp, + tracer: tracer, + opts: *o, + }, nil +} + +// Middleware returns the OTel middleware that should be registered on the worker. +// +//nolint:staticcheck // SA1019: worker.MiddlewareFunc is deprecated but still used internally +func (i *Instrumentor) Middleware() worker.MiddlewareFunc { + return NewMiddleware(i.tracer) +} + +// TracerProvider returns the TracerProvider used by the instrumentor. +func (i *Instrumentor) TracerProvider() trace.TracerProvider { + return i.tracerProvider +} + +// Shutdown flushes any remaining spans and shuts down the TracerProvider. +// Call this before your application exits to ensure all spans are exported. +func (i *Instrumentor) Shutdown(ctx context.Context) error { + if tp, ok := i.tracerProvider.(*sdktrace.TracerProvider); ok { + return tp.Shutdown(ctx) + } + return nil +} diff --git a/sdks/go/opentelemetry/middleware.go b/sdks/go/opentelemetry/middleware.go new file mode 100644 index 0000000000..e3cd932572 --- /dev/null +++ b/sdks/go/opentelemetry/middleware.go @@ -0,0 +1,61 @@ +package opentelemetry + +import ( + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/propagation" + "go.opentelemetry.io/otel/trace" + + "github.com/hatchet-dev/hatchet/pkg/worker" +) + +// NewMiddleware creates a Hatchet middleware that wraps each step run execution +// with an OpenTelemetry span. It: +// - Extracts W3C traceparent from AdditionalMetadata for distributed trace propagation +// - Creates a "hatchet.start_step_run" span with hatchet.* attributes +// - Stores attributes in context so HatchetAttributeSpanProcessor can inject +// them into all child spans +// +//nolint:staticcheck // SA1019: worker.MiddlewareFunc is deprecated but still used internally +func NewMiddleware(tracer trace.Tracer) worker.MiddlewareFunc { + propagator := propagation.TraceContext{} + + return func(ctx worker.HatchetContext, next func(worker.HatchetContext) error) error { + // Build hatchet attributes from context + attrs := hatchetAttributes(ctx) + + // Extract traceparent from additional metadata if present + parentCtx := ctx.GetContext() + if meta := ctx.AdditionalMetadata(); meta != nil { + if tp, ok := meta["traceparent"]; ok && tp != "" { + carrier := propagation.MapCarrier(map[string]string{ + "traceparent": tp, + }) + parentCtx = propagator.Extract(parentCtx, carrier) + } + } + + // Store hatchet attributes in context for the SpanProcessor + parentCtx = withHatchetAttributes(parentCtx, attrs) + + // Start span + spanCtx, span := tracer.Start(parentCtx, "hatchet.start_step_run", + trace.WithSpanKind(trace.SpanKindConsumer), + trace.WithAttributes(attrs...), + ) + defer span.End() + + // Update the HatchetContext with the OTel-enriched context + ctx.SetContext(spanCtx) + + // Execute the next middleware/action + err := next(ctx) + if err != nil { + span.SetStatus(codes.Error, err.Error()) + span.RecordError(err) + } else { + span.SetStatus(codes.Ok, "") + } + + return err + } +} diff --git a/sdks/go/opentelemetry/span_processor.go b/sdks/go/opentelemetry/span_processor.go new file mode 100644 index 0000000000..6fe72169d5 --- /dev/null +++ b/sdks/go/opentelemetry/span_processor.go @@ -0,0 +1,41 @@ +package opentelemetry + +import ( + "context" + + sdktrace "go.opentelemetry.io/otel/sdk/trace" +) + +// HatchetAttributeSpanProcessor wraps an inner SpanProcessor and injects +// hatchet.* attributes into every span created within a step run context. +// This ensures child spans are queryable by the same attributes (e.g. +// hatchet.step_run_id) as the parent span. +type HatchetAttributeSpanProcessor struct { + inner sdktrace.SpanProcessor +} + +// NewHatchetAttributeSpanProcessor creates a new HatchetAttributeSpanProcessor +// that wraps the given inner processor. +func NewHatchetAttributeSpanProcessor(inner sdktrace.SpanProcessor) *HatchetAttributeSpanProcessor { + return &HatchetAttributeSpanProcessor{inner: inner} +} + +func (p *HatchetAttributeSpanProcessor) OnStart(ctx context.Context, span sdktrace.ReadWriteSpan) { + attrs := getHatchetAttributes(ctx) + if len(attrs) > 0 { + span.SetAttributes(attrs...) + } + p.inner.OnStart(ctx, span) +} + +func (p *HatchetAttributeSpanProcessor) OnEnd(span sdktrace.ReadOnlySpan) { + p.inner.OnEnd(span) +} + +func (p *HatchetAttributeSpanProcessor) Shutdown(ctx context.Context) error { + return p.inner.Shutdown(ctx) +} + +func (p *HatchetAttributeSpanProcessor) ForceFlush(ctx context.Context) error { + return p.inner.ForceFlush(ctx) +} diff --git a/sdks/go/workflow.go b/sdks/go/workflow.go index 03a8ffe023..1603771d0c 100644 --- a/sdks/go/workflow.go +++ b/sdks/go/workflow.go @@ -9,6 +9,12 @@ import ( "sync" "time" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/propagation" + "go.opentelemetry.io/otel/trace" + v1 "github.com/hatchet-dev/hatchet/internal/services/shared/proto/v1" v0Client "github.com/hatchet-dev/hatchet/pkg/client" "github.com/hatchet-dev/hatchet/pkg/client/create" @@ -19,6 +25,27 @@ import ( "github.com/hatchet-dev/hatchet/sdks/go/internal" ) +// injectTraceparentToMap serializes the current span's W3C traceparent from ctx +// into the metadata map so child workflows inherit the trace. +func injectTraceparentToMap(ctx context.Context, meta *map[string]string) *map[string]string { + propagator := propagation.TraceContext{} + carrier := propagation.MapCarrier{} + propagator.Inject(ctx, carrier) + + tp, ok := carrier["traceparent"] + if !ok || tp == "" { + return meta + } + + if meta == nil { + m := map[string]string{"traceparent": tp} + return &m + } + + (*meta)["traceparent"] = tp + return meta +} + type RunPriority = features.RunPriority type DesiredWorkerLabel = types.DesiredWorkerLabel @@ -594,6 +621,27 @@ func (w *Workflow) Run(ctx context.Context, input any, opts ...RunOptFunc) (*Wor // RunNoWait executes the workflow with the provided input without waiting for completion. // Returns a workflow run reference that can be used to track the run status. func (w *Workflow) RunNoWait(ctx context.Context, input any, opts ...RunOptFunc) (*WorkflowRunRef, error) { + // Start OTel span using the underlying context (not the HatchetContext itself) + // so that the HatchetContext type assertion still works downstream. + tracer := otel.Tracer("github.com/hatchet-dev/hatchet/sdks/go") + otelCtx := ctx + if hCtx, ok := ctx.(Context); ok { + otelCtx = hCtx.GetContext() + } + otelCtx, span := tracer.Start(otelCtx, fmt.Sprintf("hatchet trigger task %s", w.declaration.Name()), + trace.WithSpanKind(trace.SpanKindClient), + trace.WithAttributes(attribute.String("hatchet.task_name", w.declaration.Name())), + ) + defer span.End() + + // Update the HatchetContext's inner context with the OTel span context, + // or use the OTel context directly for plain context.Context callers. + if hCtx, ok := ctx.(Context); ok { + hCtx.SetContext(otelCtx) + } else { + ctx = otelCtx + } + runOpts := &runOpts{} for _, opt := range opts { opt(runOpts) @@ -604,6 +652,9 @@ func (w *Workflow) RunNoWait(ctx context.Context, input any, opts ...RunOptFunc) priority = &[]int32{int32(*runOpts.Priority)}[0] } + // Inject traceparent for cross-workflow trace propagation + runOpts.AdditionalMetadata = injectTraceparentToMap(otelCtx, runOpts.AdditionalMetadata) + var v0Opts []v0Client.RunOptFunc if runOpts.AdditionalMetadata != nil { @@ -635,9 +686,14 @@ func (w *Workflow) RunNoWait(ctx context.Context, input any, opts ...RunOptFunc) } if err != nil { + span.SetStatus(codes.Error, err.Error()) + span.RecordError(err) return nil, err } + span.SetAttributes(attribute.String("hatchet.workflow_run_id", v0Workflow.RunId())) + span.SetStatus(codes.Ok, "") + return &WorkflowRunRef{RunId: v0Workflow.RunId(), v0Workflow: v0Workflow}, nil } diff --git a/sdks/python/examples/opentelemetry_instrumentation/hatchet/trigger.py b/sdks/python/examples/opentelemetry_instrumentation/hatchet/trigger.py new file mode 100644 index 0000000000..b1282b3bd1 --- /dev/null +++ b/sdks/python/examples/opentelemetry_instrumentation/hatchet/trigger.py @@ -0,0 +1,49 @@ +""" +Trigger the OTelDataPipeline workflow. + +Make sure the worker is already running: + poetry run python -m examples.opentelemetry_instrumentation.hatchet.worker + +Then run this: + poetry run python -m examples.opentelemetry_instrumentation.hatchet.trigger +""" + +from opentelemetry.sdk.resources import SERVICE_NAME, Resource +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import ConsoleSpanExporter, SimpleSpanProcessor + +from examples.opentelemetry_instrumentation.hatchet.worker import otel_workflow +from hatchet_sdk.clients.admin import TriggerWorkflowOptions +from hatchet_sdk.opentelemetry.instrumentor import HatchetInstrumentor + +# Use the same console exporter so you can see trigger-side spans too +resource = Resource(attributes={SERVICE_NAME: "hatchet-otel-pipeline-trigger"}) +provider = TracerProvider(resource=resource) +provider.add_span_processor(SimpleSpanProcessor(ConsoleSpanExporter())) + +HatchetInstrumentor( + tracer_provider=provider, + enable_hatchet_otel_collector=True, +).instrument() + +tracer = provider.get_tracer(__name__) + + +def main() -> None: + # The run_workflow call is auto-traced with a "hatchet.run_workflow" span. + # The traceparent is automatically injected into additional_metadata, + # so the worker-side spans become children of this trigger span. + with tracer.start_as_current_span("trigger_otel_data_pipeline"): + result = otel_workflow.run( + options=TriggerWorkflowOptions( + additional_metadata={ + "source": "otel-example", + "pipeline": "data-ingest", + }, + ), + ) + print(f"Workflow result: {result}") + + +if __name__ == "__main__": + main() diff --git a/sdks/python/examples/opentelemetry_instrumentation/hatchet/worker.py b/sdks/python/examples/opentelemetry_instrumentation/hatchet/worker.py new file mode 100644 index 0000000000..af26c16ee3 --- /dev/null +++ b/sdks/python/examples/opentelemetry_instrumentation/hatchet/worker.py @@ -0,0 +1,144 @@ +""" +HatchetInstrumentor example with rich traces. + +Run the worker: + poetry run python -m examples.opentelemetry_instrumentation.hatchet.worker + +Then trigger it from another terminal: + poetry run python -m examples.opentelemetry_instrumentation.hatchet.trigger +""" + +import time + +from opentelemetry.sdk.resources import SERVICE_NAME, Resource +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import ConsoleSpanExporter, SimpleSpanProcessor +from opentelemetry.trace import StatusCode, Tracer + +from hatchet_sdk import Context, EmptyModel, Hatchet +from hatchet_sdk.opentelemetry.instrumentor import HatchetInstrumentor + +hatchet = Hatchet() + +otel_workflow = hatchet.workflow(name="OTelDataPipeline") + +# Module-level tracer — will be set in main() before the worker starts. +# Tasks use this to create custom child spans inside the auto-instrumented +# hatchet task run parent span. +_tracer: Tracer | None = None + + +def _get_tracer() -> Tracer: + global _tracer + if _tracer is None: + from opentelemetry.trace import get_tracer + + _tracer = get_tracer(__name__) + return _tracer + + +@otel_workflow.task() +def fetch_data(input: EmptyModel, ctx: Context) -> dict[str, str]: + tracer = _get_tracer() + + with tracer.start_as_current_span( + "http.request", + attributes={"http.method": "GET", "http.url": "https://api.example.com/data"}, + ) as span: + time.sleep(0.05) + span.set_attribute("http.status_code", 200) + span.set_attribute("http.response_content_length", 4096) + + with tracer.start_as_current_span("json.parse") as span: + time.sleep(0.01) + span.set_attribute("json.record_count", 42) + + return {"records_fetched": "42"} + + +@otel_workflow.task() +def validate_data(input: EmptyModel, ctx: Context) -> dict[str, str]: + tracer = _get_tracer() + + with tracer.start_as_current_span("schema.validate") as span: + time.sleep(0.02) + span.set_attribute("validation.schema", "v2.1") + span.set_attribute("validation.records_checked", 42) + span.set_attribute("validation.errors", 2) + span.set_status(StatusCode.OK, "2 records failed validation") + + with tracer.start_as_current_span("data.clean") as span: + time.sleep(0.01) + span.set_attribute("clean.records_dropped", 2) + span.set_attribute("clean.records_remaining", 40) + + return {"valid_records": "40", "dropped": "2"} + + +@otel_workflow.task() +def process_data(input: EmptyModel, ctx: Context) -> dict[str, str]: + tracer = _get_tracer() + + with tracer.start_as_current_span("transform.pipeline") as pipeline_span: + pipeline_span.set_attribute("pipeline.stages", 3) + + with tracer.start_as_current_span("transform.normalize"): + time.sleep(0.015) + + with tracer.start_as_current_span("transform.enrich") as enrich_span: + time.sleep(0.02) + enrich_span.set_attribute("enrich.source", "geocoding-api") + + with tracer.start_as_current_span("transform.aggregate") as agg_span: + time.sleep(0.03) + agg_span.set_attribute("aggregate.groups", 8) + agg_span.set_attribute("aggregate.method", "sum") + + return {"processed_groups": "8"} + + +@otel_workflow.task() +def save_results(input: EmptyModel, ctx: Context) -> dict[str, str]: + tracer = _get_tracer() + + with tracer.start_as_current_span( + "db.query", + attributes={"db.system": "postgresql", "db.operation": "INSERT"}, + ) as span: + time.sleep(0.04) + span.set_attribute("db.rows_affected", 8) + + with tracer.start_as_current_span("cache.invalidate") as span: + time.sleep(0.005) + span.set_attribute("cache.keys_invalidated", 3) + + with tracer.start_as_current_span("notification.send") as span: + time.sleep(0.01) + span.set_attribute("notification.channel", "webhook") + span.set_attribute("notification.status", "delivered") + + return {"saved": "true"} + + +def main() -> None: + resource = Resource(attributes={SERVICE_NAME: "hatchet-otel-pipeline-example"}) + provider = TracerProvider(resource=resource) + provider.add_span_processor(SimpleSpanProcessor(ConsoleSpanExporter())) + + HatchetInstrumentor( + tracer_provider=provider, + enable_hatchet_otel_collector=True, + ).instrument() + + global _tracer + _tracer = provider.get_tracer(__name__) + + worker = hatchet.worker( + "otel-pipeline-worker", + workflows=[otel_workflow], + ) + worker.start() + + +if __name__ == "__main__": + main() diff --git a/sdks/python/examples/run_details/test_run_detail_getter.py b/sdks/python/examples/run_details/test_run_detail_getter.py index cb598d1da2..655afd753a 100644 --- a/sdks/python/examples/run_details/test_run_detail_getter.py +++ b/sdks/python/examples/run_details/test_run_detail_getter.py @@ -23,7 +23,8 @@ async def test_run(hatchet: Hatchet) -> None: assert details.status == RunStatus.RUNNING assert details.input == mock_input.model_dump() - assert details.additional_metadata == meta + assert details.additional_metadata is not None + assert meta.items() <= details.additional_metadata.items() assert len(details.task_runs) == 4 assert all( r.status in [V1TaskStatus.RUNNING, V1TaskStatus.QUEUED] @@ -42,7 +43,8 @@ async def test_run(hatchet: Hatchet) -> None: assert details.status == RunStatus.FAILED assert details.input == mock_input.model_dump() - assert details.additional_metadata == meta + assert details.additional_metadata is not None + assert meta.items() <= details.additional_metadata.items() assert len(details.task_runs) == 6 assert details.task_runs["step1"].status == V1TaskStatus.COMPLETED diff --git a/sdks/python/hatchet_sdk/hatchet.py b/sdks/python/hatchet_sdk/hatchet.py index eba5a15286..ea20446aa4 100644 --- a/sdks/python/hatchet_sdk/hatchet.py +++ b/sdks/python/hatchet_sdk/hatchet.py @@ -58,7 +58,9 @@ class Hatchet: def __init__( self, + # todo[next major version]: add debug to the client config instead of being a top-level parameter debug: bool = False, + # todo[next major version]: remove this parameter client: Client | None = None, config: ClientConfig | None = None, ): diff --git a/sdks/python/hatchet_sdk/opentelemetry/instrumentor.py b/sdks/python/hatchet_sdk/opentelemetry/instrumentor.py index 75a9237132..94620a5165 100644 --- a/sdks/python/hatchet_sdk/opentelemetry/instrumentor.py +++ b/sdks/python/hatchet_sdk/opentelemetry/instrumentor.py @@ -1,3 +1,4 @@ +import contextvars import json from collections.abc import Callable, Collection, Coroutine from importlib.metadata import version @@ -8,11 +9,15 @@ try: from opentelemetry.context import Context + from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter from opentelemetry.instrumentation.instrumentor import ( # type: ignore[attr-defined] BaseInstrumentor, ) from opentelemetry.instrumentation.utils import unwrap from opentelemetry.metrics import MeterProvider, NoOpMeterProvider, get_meter + from opentelemetry.sdk.trace import ReadableSpan, Span + from opentelemetry.sdk.trace import TracerProvider as SDKTracerProvider + from opentelemetry.sdk.trace.export import BatchSpanProcessor, SpanExporter from opentelemetry.trace import ( NoOpTracerProvider, SpanKind, @@ -30,6 +35,29 @@ "To use the HatchetInstrumentor, you must install Hatchet's `otel` extra using (e.g.) `pip install hatchet-sdk[otel]`" ) from e +# ContextVar that holds the hatchet.* attributes from the active +# hatchet task run span so they can be injected into child spans. +_hatchet_span_attributes: contextvars.ContextVar[dict[str, str | int] | None] = ( + contextvars.ContextVar("_hatchet_span_attributes", default=None) +) + + +class _HatchetAttributeSpanProcessor(BatchSpanProcessor): + """SpanProcessor that injects hatchet.* attributes into every span + created within a step run context, so that child spans are queryable + by the same attributes (e.g. hatchet.step_run_id) as the parent.""" + + def __init__(self, span_exporter: SpanExporter) -> None: + super().__init__(span_exporter) + + def on_start(self, span: Span, parent_context: Context | None = None) -> None: + attrs = _hatchet_span_attributes.get() + if attrs and span.is_recording(): + for key, value in attrs.items(): + span.set_attribute(key, value) + super().on_start(span, parent_context) + + import inspect from datetime import datetime @@ -185,11 +213,36 @@ class HatchetInstrumentor(BaseInstrumentor): # type: ignore[misc] tracing and metrics collection. :param tracer_provider: TracerProvider | None: The OpenTelemetry TracerProvider to use. - If not provided, the global tracer provider will be used. + If not provided and `enable_hatchet_otel_collector` is True, a new SDKTracerProvider + will be created. Otherwise, the global tracer provider will be used. :param meter_provider: MeterProvider | None: The OpenTelemetry MeterProvider to use. If not provided, a no-op meter provider will be used. :param config: ClientConfig | None: The configuration for the Hatchet client. If not provided, a default configuration will be used. + :param enable_hatchet_otel_collector: bool: If True, adds an OTLP exporter to send traces to the + Hatchet engine. Uses the same connection settings (host, TLS, token) as the Hatchet + client. This can be combined with your own tracer_provider to send traces to multiple + destinations (e.g., both Hatchet and Jaeger/Datadog). Default is False. + + Example usage:: + + # Send traces only to Hatchet + instrumentor = HatchetInstrumentor( + enable_hatchet_otel_collector=True, + ) + + # Send traces to both Hatchet and your own collector + from opentelemetry.sdk.trace import TracerProvider + from opentelemetry.sdk.trace.export import BatchSpanProcessor + from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter + + provider = TracerProvider() + provider.add_span_processor(BatchSpanProcessor(OTLPSpanExporter(endpoint="your-collector:4317"))) + + instrumentor = HatchetInstrumentor( + tracer_provider=provider, + enable_hatchet_otel_collector=True, # Also sends to Hatchet + ) """ def __init__( @@ -197,14 +250,47 @@ def __init__( tracer_provider: TracerProvider | None = None, meter_provider: MeterProvider | None = None, config: ClientConfig | None = None, + enable_hatchet_otel_collector: bool = False, ): self.config = config or ClientConfig() - self.tracer_provider = tracer_provider or get_tracer_provider() + if tracer_provider is not None: + self.tracer_provider = tracer_provider + elif enable_hatchet_otel_collector: + self.tracer_provider = SDKTracerProvider() + else: + self.tracer_provider = get_tracer_provider() + + if enable_hatchet_otel_collector: + self._add_hatchet_exporter() + self.meter_provider = meter_provider or NoOpMeterProvider() super().__init__() + def _add_hatchet_exporter(self) -> None: + if not isinstance(self.tracer_provider, SDKTracerProvider): + logger.warning( + "enable_hatchet_otel_collector requires an opentelemetry.sdk.trace.TracerProvider. " + "The provided tracer_provider does not support adding span processors. " + "Traces will not be forwarded to Hatchet." + ) + return + + endpoint = self.config.host_port + insecure = self.config.tls_config.strategy == "none" + headers = (("authorization", f"Bearer {self.config.token}"),) + + span_exporter = OTLPSpanExporter( + endpoint=endpoint, + headers=headers, + insecure=insecure, + ) + + self.tracer_provider.add_span_processor( + _HatchetAttributeSpanProcessor(span_exporter) + ) + def instrumentation_dependencies(self) -> Collection[str]: return () @@ -310,18 +396,24 @@ async def _wrap_handle_start_step_run( if self.config.otel.include_task_name_in_start_step_run_span_name: span_name += f".{action.action_id}" - with self._tracer.start_as_current_span( - span_name, - attributes=action.get_otel_attributes(self.config), - context=traceparent, - kind=SpanKind.CONSUMER, - ) as span: - result = await wrapped(*args, **kwargs) - - if isinstance(result, Exception): - span.set_status(StatusCode.ERROR, str(result)) + hatchet_attrs = action.get_otel_attributes(self.config) + token = _hatchet_span_attributes.set(hatchet_attrs) - return result + try: + with self._tracer.start_as_current_span( + span_name, + attributes=hatchet_attrs, + context=traceparent, + kind=SpanKind.CONSUMER, + ) as span: + result = await wrapped(*args, **kwargs) + + if isinstance(result, Exception): + span.set_status(StatusCode.ERROR, str(result)) + + return result + finally: + _hatchet_span_attributes.reset(token) ## IMPORTANT: Keep these types in sync with the wrapped method's signature async def _wrap_handle_cancel_action( diff --git a/sdks/python/hatchet_sdk/worker/runner/runner.py b/sdks/python/hatchet_sdk/worker/runner/runner.py index 7813a48b8d..739c99f736 100644 --- a/sdks/python/hatchet_sdk/worker/runner/runner.py +++ b/sdks/python/hatchet_sdk/worker/runner/runner.py @@ -1,4 +1,5 @@ import asyncio +import contextvars import ctypes import functools from collections.abc import Callable @@ -312,8 +313,17 @@ async def async_wrapped_action_func( dependencies, ) + # Copy the full contextvars snapshot (including OpenTelemetry span + # context) so that child spans created in the thread inherit the + # correct trace_id and parent_span_id from the active + # hatchet task run span. + ctx_snapshot = contextvars.copy_context() loop = asyncio.get_event_loop() - return await loop.run_in_executor(self.thread_pool, pfunc) + return await loop.run_in_executor( + self.thread_pool, + ctx_snapshot.run, + pfunc, + ) finally: self.cleanup_run_id(action.key) diff --git a/sdks/python/tests/otel_traces/__init__.py b/sdks/python/tests/otel_traces/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/sdks/python/tests/otel_traces/test_otel_traces.py b/sdks/python/tests/otel_traces/test_otel_traces.py new file mode 100644 index 0000000000..e75baae8a7 --- /dev/null +++ b/sdks/python/tests/otel_traces/test_otel_traces.py @@ -0,0 +1,173 @@ +import asyncio +from subprocess import Popen +from typing import Any +from uuid import uuid4 + +import pytest +import requests + +from hatchet_sdk import Hatchet, TriggerWorkflowOptions +from hatchet_sdk.clients.rest.models.v1_task_status import V1TaskStatus +from tests.otel_traces.worker import otel_retry_task, otel_simple_task + +SPANS_PORT = 8020 +WORKER_HEALTHCHECK_PORT = 8010 + +ON_DEMAND_WORKER_PARAMS = [ + ( + [ + "poetry", + "run", + "python", + "tests/otel_traces/worker.py", + "--spans-port", + str(SPANS_PORT), + ], + WORKER_HEALTHCHECK_PORT, + ) +] + + +def _get_spans() -> list[dict[str, Any]]: + """Fetch captured spans from the worker's span HTTP endpoint.""" + resp = requests.get(f"http://localhost:{SPANS_PORT}/spans", timeout=5) + resp.raise_for_status() + result: list[dict[str, Any]] = resp.json() + return result + + +def _clear_spans() -> None: + """Clear captured spans on the worker.""" + resp = requests.delete(f"http://localhost:{SPANS_PORT}/spans", timeout=5) + resp.raise_for_status() + + +@pytest.mark.parametrize("on_demand_worker", ON_DEMAND_WORKER_PARAMS, indirect=True) +@pytest.mark.asyncio(loop_scope="session") +async def test_otel_spans_created_on_task_run( + hatchet: Hatchet, on_demand_worker: Popen[Any] +) -> None: + """Verify that running a task produces correct OTel spans with hatchet.* attributes.""" + _clear_spans() + + test_run_id = str(uuid4()) + + await otel_simple_task.aio_run( + options=TriggerWorkflowOptions( + additional_metadata={"test_run_id": test_run_id}, + ), + ) + + # Give the span processor a moment to flush + await asyncio.sleep(1) + + spans = _get_spans() + + # Find the hatchet task run span + step_run_spans = [s for s in spans if s["name"] == "hatchet task run"] + assert len(step_run_spans) >= 1, ( + f"Expected at least one hatchet task run span, got {len(step_run_spans)}. " + f"All spans: {[s['name'] for s in spans]}" + ) + + step_span = step_run_spans[-1] # Use the most recent one + + # Verify hatchet attributes exist + attrs = step_span["attributes"] + assert "hatchet.step_run_id" in attrs, f"Missing hatchet.step_run_id in {attrs}" + assert ( + "hatchet.workflow_run_id" in attrs + ), f"Missing hatchet.workflow_run_id in {attrs}" + assert "hatchet.tenant_id" in attrs, f"Missing hatchet.tenant_id in {attrs}" + + # Verify span kind is CONSUMER (value=4 in OTel Python SDK) + assert step_span["kind"] == 4, f"Expected CONSUMER (4), got {step_span['kind']}" + + # Find the custom child span + child_spans = [s for s in spans if s["name"] == "custom.child.span"] + assert len(child_spans) >= 1, ( + f"Expected at least one custom.child.span, got {len(child_spans)}. " + f"All spans: {[s['name'] for s in spans]}" + ) + + child_span = child_spans[-1] + + # Child span should share the same trace_id as the step run span + assert ( + child_span["trace_id"] == step_span["trace_id"] + ), f"Child trace_id {child_span['trace_id']} != step run trace_id {step_span['trace_id']}" + + # Child span should have hatchet.* attributes injected by _HatchetAttributeSpanProcessor + child_attrs = child_span["attributes"] + assert ( + "hatchet.step_run_id" in child_attrs + ), f"Child span missing hatchet.step_run_id (attribute propagation failed). Attrs: {child_attrs}" + assert ( + child_attrs["hatchet.step_run_id"] == attrs["hatchet.step_run_id"] + ), "Child span hatchet.step_run_id doesn't match parent" + + # Verify the custom attribute is present + assert ( + child_attrs.get("test.marker") == "hello" + ), f"Missing test.marker attribute on child span. Attrs: {child_attrs}" + + +@pytest.mark.parametrize("on_demand_worker", ON_DEMAND_WORKER_PARAMS, indirect=True) +@pytest.mark.asyncio(loop_scope="session") +async def test_otel_traces_on_retry( + hatchet: Hatchet, on_demand_worker: Popen[Any] +) -> None: + """Verify that traces are produced for both the failed attempt and the retry. + + Uses a task that fails on the first attempt (raises an exception) and + succeeds on the second attempt (retries=1). Both attempts should produce + valid hatchet task run spans with correct attributes, and the retry + count attribute should differ between them. + """ + _clear_spans() + + test_run_id = str(uuid4()) + + await otel_retry_task.aio_run( + options=TriggerWorkflowOptions( + additional_metadata={"test_run_id": test_run_id}, + ), + ) + + # Give the span processor a moment to flush + await asyncio.sleep(1) + + spans = _get_spans() + + # Both the failed first attempt and the successful retry should have spans + step_run_spans = [s for s in spans if s["name"] == "hatchet task run"] + assert len(step_run_spans) >= 2, ( + f"Expected at least 2 hatchet task run spans (initial + retry), " + f"got {len(step_run_spans)}. All spans: {[s['name'] for s in spans]}" + ) + + # The first attempt should have errored + error_spans = [s for s in step_run_spans if s["status_code"] == "ERROR"] + assert len(error_spans) >= 1, ( + f"Expected at least one ERROR span from the failed first attempt. " + f"Statuses: {[s['status_code'] for s in step_run_spans]}" + ) + + # All step run spans should have valid hatchet.* attributes + for span in step_run_spans: + attrs = span["attributes"] + assert ( + "hatchet.step_run_id" in attrs + ), f"Step run span missing hatchet.step_run_id. Attrs: {attrs}" + assert ( + "hatchet.workflow_run_id" in attrs + ), f"Step run span missing hatchet.workflow_run_id. Attrs: {attrs}" + assert ( + "hatchet.tenant_id" in attrs + ), f"Step run span missing hatchet.tenant_id. Attrs: {attrs}" + + # Verify retry count differs between attempts + retry_counts = [s["attributes"].get("hatchet.retry_count") for s in step_run_spans] + assert ( + len(set(retry_counts)) >= 2 + ), f"Expected different retry_count values across attempts, got {retry_counts}" diff --git a/sdks/python/tests/otel_traces/worker.py b/sdks/python/tests/otel_traces/worker.py new file mode 100644 index 0000000000..e881efa445 --- /dev/null +++ b/sdks/python/tests/otel_traces/worker.py @@ -0,0 +1,138 @@ +""" +OTel test worker — runs with HatchetInstrumentor and exposes captured spans via HTTP. + +Usage: + poetry run python tests/otel_traces/worker.py [--spans-port 8020] +""" + +import argparse +import json +import time +from http.server import BaseHTTPRequestHandler, HTTPServer +from threading import Thread +from typing import Any, cast + +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from opentelemetry.trace import get_tracer, set_tracer_provider + +from hatchet_sdk import Context, EmptyModel, Hatchet +from hatchet_sdk.opentelemetry.instrumentor import HatchetInstrumentor + +# -- OTel setup ---------------------------------------------------------------- + +span_exporter = InMemorySpanExporter() +provider = TracerProvider() +provider.add_span_processor(SimpleSpanProcessor(span_exporter)) +set_tracer_provider(provider) + +HatchetInstrumentor( + tracer_provider=provider, + enable_hatchet_otel_collector=True, +).instrument() + +# -- Hatchet workflows --------------------------------------------------------- + +hatchet = Hatchet(debug=True) + + +@hatchet.task() +def otel_simple_task(input: EmptyModel, ctx: Context) -> dict[str, str]: + """Simple task that creates a custom child span.""" + tracer = get_tracer("otel-test") + with tracer.start_as_current_span("custom.child.span") as span: + span.set_attribute("test.marker", "hello") + time.sleep(0.01) + return {"status": "ok"} + + +@hatchet.task(retries=1) +def otel_retry_task(input: EmptyModel, ctx: Context) -> dict[str, str]: + """Task that fails on first attempt and succeeds on retry.""" + retry_count = ctx.retry_count + if retry_count == 0: + raise RuntimeError("intentional failure on first attempt") + return {"status": "ok", "retry_count": str(retry_count)} + + +# -- Span HTTP server ---------------------------------------------------------- + + +def _serialize_spans() -> list[dict[str, Any]]: + spans = span_exporter.get_finished_spans() + result: list[dict[str, Any]] = [] + for s in spans: + attrs: dict[str, Any] = {} + if s.attributes is not None: + for k, v in s.attributes.items(): + attrs[k] = str(v) if not isinstance(v, (str, int, float, bool)) else v + + result.append( + { + "name": s.name, + "trace_id": format(s.context.trace_id, "032x"), + "span_id": format(s.context.span_id, "016x"), + "parent_span_id": ( + format(s.parent.span_id, "016x") if s.parent else None + ), + "attributes": attrs, + "kind": s.kind.value, + "status_code": s.status.status_code.name, + } + ) + return result + + +class SpanHandler(BaseHTTPRequestHandler): + def do_GET(self) -> None: + if self.path == "/spans": + body = json.dumps(_serialize_spans()).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.end_headers() + self.wfile.write(body) + else: + self.send_response(404) + self.end_headers() + + def do_DELETE(self) -> None: + if self.path == "/spans": + span_exporter.clear() + self.send_response(200) + self.end_headers() + else: + self.send_response(404) + self.end_headers() + + def log_message(self, format: str, *args: Any) -> None: + pass # suppress request logs + + +def _start_span_server(port: int) -> None: + server = HTTPServer(("0.0.0.0", port), SpanHandler) + server.serve_forever() + + +# -- Main ---------------------------------------------------------------------- + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--spans-port", type=int, default=8020) + args = parser.parse_args() + + spans_port = cast(int, args.spans_port) + + # Start span server in background thread + Thread(target=_start_span_server, args=(spans_port,), daemon=True).start() + + worker = hatchet.worker( + "otel-e2e-test-worker", + workflows=[otel_simple_task, otel_retry_task], + ) + worker.start() + + +if __name__ == "__main__": + main() diff --git a/sdks/typescript/CHANGELOG.md b/sdks/typescript/CHANGELOG.md index 2d64a71ee2..6b4a839459 100644 --- a/sdks/typescript/CHANGELOG.md +++ b/sdks/typescript/CHANGELOG.md @@ -1,10 +1,16 @@ # Changelog - All notable changes to Hatchet's TypeScript SDK will be documented in this changelog. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [1.16.0] - 2026-03-09 + +### Added + +- OpenTelemetry instrumentation via `HatchetInstrumentor` with automatic tracing for workflow runs, event pushes, and step executions +- OpenTelemetry example demonstrating automatic and custom span instrumentation (`examples/opentelemetry_instrumentation`) + ## [1.15.2] - 2026-03-06 ### Fixed diff --git a/sdks/typescript/package.json b/sdks/typescript/package.json index 358209ee3b..7941fdda50 100644 --- a/sdks/typescript/package.json +++ b/sdks/typescript/package.json @@ -1,6 +1,6 @@ { "name": "@hatchet-dev/typescript-sdk", - "version": "1.15.2", + "version": "1.16.0", "description": "Background task orchestration & visibility for developers", "types": "dist/index.d.ts", "files": [ @@ -85,6 +85,10 @@ "zod-to-json-schema": "^3.24.1" }, "optionalDependencies": { + "@opentelemetry/api": "^1.9.0", + "@opentelemetry/exporter-trace-otlp-grpc": "^0.208.0", + "@opentelemetry/instrumentation": "^0.208.0", + "@opentelemetry/sdk-trace-base": "^1.30.1", "prom-client": "^15.1.3" }, "packageManager": "pnpm@10.16.1", diff --git a/sdks/typescript/pnpm-lock.yaml b/sdks/typescript/pnpm-lock.yaml index 2f513c6c79..3835c0a981 100644 --- a/sdks/typescript/pnpm-lock.yaml +++ b/sdks/typescript/pnpm-lock.yaml @@ -131,6 +131,18 @@ importers: specifier: ^8.56.1 version: 8.56.1(eslint@10.0.3)(typescript@5.9.2) optionalDependencies: + '@opentelemetry/api': + specifier: ^1.9.0 + version: 1.9.0 + '@opentelemetry/exporter-trace-otlp-grpc': + specifier: ^0.208.0 + version: 0.208.0(@opentelemetry/api@1.9.0) + '@opentelemetry/instrumentation': + specifier: ^0.208.0 + version: 0.208.0(@opentelemetry/api@1.9.0) + '@opentelemetry/sdk-trace-base': + specifier: ^1.30.1 + version: 1.30.1(@opentelemetry/api@1.9.0) prom-client: specifier: ^15.1.3 version: 15.1.3 @@ -510,10 +522,100 @@ packages: resolution: {integrity: sha512-nn5ozdjYQpUCZlWGuxcJY/KpxkWQs4DcbMCmKojjyrYDEAGy4Ce19NN4v5MduafTwJlbKc99UA8YhSVqq9yPZA==} engines: {node: '>=12.4.0'} + '@opentelemetry/api-logs@0.208.0': + resolution: {integrity: sha512-CjruKY9V6NMssL/T1kAFgzosF1v9o6oeN+aX5JB/C/xPNtmgIJqcXHG7fA82Ou1zCpWGl4lROQUKwUNE1pMCyg==} + engines: {node: '>=8.0.0'} + '@opentelemetry/api@1.9.0': resolution: {integrity: sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==} engines: {node: '>=8.0.0'} + '@opentelemetry/core@1.30.1': + resolution: {integrity: sha512-OOCM2C/QIURhJMuKaekP3TRBxBKxG/TWWA0TL2J6nXUtDnuCtccy49LUJF8xPFXMX+0LMcxFpCo8M9cGY1W6rQ==} + engines: {node: '>=14'} + peerDependencies: + '@opentelemetry/api': '>=1.0.0 <1.10.0' + + '@opentelemetry/core@2.2.0': + resolution: {integrity: sha512-FuabnnUm8LflnieVxs6eP7Z383hgQU4W1e3KJS6aOG3RxWxcHyBxH8fDMHNgu/gFx/M2jvTOW/4/PHhLz6bjWw==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': '>=1.0.0 <1.10.0' + + '@opentelemetry/exporter-trace-otlp-grpc@0.208.0': + resolution: {integrity: sha512-E/eNdcqVUTAT7BC+e8VOw/krqb+5rjzYkztMZ/o+eyJl+iEY6PfczPXpwWuICwvsm0SIhBoh9hmYED5Vh5RwIw==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + + '@opentelemetry/instrumentation@0.208.0': + resolution: {integrity: sha512-Eju0L4qWcQS+oXxi6pgh7zvE2byogAkcsVv0OjHF/97iOz1N/aKE6etSGowYkie+YA1uo6DNwdSxaaNnLvcRlA==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + + '@opentelemetry/otlp-exporter-base@0.208.0': + resolution: {integrity: sha512-gMd39gIfVb2OgxldxUtOwGJYSH8P1kVFFlJLuut32L6KgUC4gl1dMhn+YC2mGn0bDOiQYSk/uHOdSjuKp58vvA==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + + '@opentelemetry/otlp-grpc-exporter-base@0.208.0': + resolution: {integrity: sha512-fGvAg3zb8fC0oJAzfz7PQppADI2HYB7TSt/XoCaBJFi1mSquNUjtHXEoviMgObLAa1NRIgOC1lsV1OUKi+9+lQ==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + + '@opentelemetry/otlp-transformer@0.208.0': + resolution: {integrity: sha512-DCFPY8C6lAQHUNkzcNT9R+qYExvsk6C5Bto2pbNxgicpcSWbe2WHShLxkOxIdNcBiYPdVHv/e7vH7K6TI+C+fQ==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + + '@opentelemetry/resources@1.30.1': + resolution: {integrity: sha512-5UxZqiAgLYGFjS4s9qm5mBVo433u+dSPUFWVWXmLAD4wB65oMCoXaJP1KJa9DIYYMeHu3z4BZcStG3LC593cWA==} + engines: {node: '>=14'} + peerDependencies: + '@opentelemetry/api': '>=1.0.0 <1.10.0' + + '@opentelemetry/resources@2.2.0': + resolution: {integrity: sha512-1pNQf/JazQTMA0BiO5NINUzH0cbLbbl7mntLa4aJNmCCXSj0q03T5ZXXL0zw4G55TjdL9Tz32cznGClf+8zr5A==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': '>=1.3.0 <1.10.0' + + '@opentelemetry/sdk-logs@0.208.0': + resolution: {integrity: sha512-QlAyL1jRpOeaqx7/leG1vJMp84g0xKP6gJmfELBpnI4O/9xPX+Hu5m1POk9Kl+veNkyth5t19hRlN6tNY1sjbA==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': '>=1.4.0 <1.10.0' + + '@opentelemetry/sdk-metrics@2.2.0': + resolution: {integrity: sha512-G5KYP6+VJMZzpGipQw7Giif48h6SGQ2PFKEYCybeXJsOCB4fp8azqMAAzE5lnnHK3ZVwYQrgmFbsUJO/zOnwGw==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': '>=1.9.0 <1.10.0' + + '@opentelemetry/sdk-trace-base@1.30.1': + resolution: {integrity: sha512-jVPgBbH1gCy2Lb7X0AVQ8XAfgg0pJ4nvl8/IiQA6nxOsPvS+0zMJaFSs2ltXe0J6C8dqjcnpyqINDJmU30+uOg==} + engines: {node: '>=14'} + peerDependencies: + '@opentelemetry/api': '>=1.0.0 <1.10.0' + + '@opentelemetry/sdk-trace-base@2.2.0': + resolution: {integrity: sha512-xWQgL0Bmctsalg6PaXExmzdedSp3gyKV8mQBwK/j9VGdCDu2fmXIb2gAehBKbkXCpJ4HPkgv3QfoJWRT4dHWbw==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': '>=1.3.0 <1.10.0' + + '@opentelemetry/semantic-conventions@1.28.0': + resolution: {integrity: sha512-lp4qAiMTD4sNWW4DbKLBkfiMZ4jbAboJIGOQr5DvciMRI494OapieI9qiODpOt0XBr1LjIDy1xAGAnVs5supTA==} + engines: {node: '>=14'} + + '@opentelemetry/semantic-conventions@1.40.0': + resolution: {integrity: sha512-cifvXDhcqMwwTlTK04GBNeIe7yyo28Mfby85QXFe1Yk8nmi36Ab/5UQwptOx84SsoGNRg+EVSjwzfSZMy6pmlw==} + engines: {node: '>=14'} + '@pkgr/core@0.2.9': resolution: {integrity: sha512-QNqXyfVS2wm9hweSYD2O7F0G06uurj9kZ96TRQE5Y9hU7+tgdZwIkbAKc5Ocy1HxEY2kuDQa6cQ1WRs/O5LFKA==} engines: {node: ^12.20.0 || ^14.18.0 || >=16.0.0} @@ -823,6 +925,11 @@ packages: abort-controller-x@0.4.3: resolution: {integrity: sha512-VtUwTNU8fpMwvWGn4xE93ywbogTYsuT+AUxAXOeelbXuQVIwNmC5YLeho9sH4vZ4ITW8414TTAOG1nW6uIVHCA==} + acorn-import-attributes@1.9.5: + resolution: {integrity: sha512-n02Vykv5uA3eHGM/Z2dQrcD56kL8TyDb2p1+0P83PClMnC/nc+anbQRhIOWnSq4Ke/KvDPrY3C9hDtC/A3eHnQ==} + peerDependencies: + acorn: ^8 + acorn-jsx@5.3.2: resolution: {integrity: sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==} peerDependencies: @@ -1046,6 +1153,9 @@ packages: cjs-module-lexer@1.4.3: resolution: {integrity: sha512-9z8TZaGM1pfswYeXrUpzPrkx8UnWYdhJclsiYMm6x/w5+nN+8Tf/LnAgfLGQCm59qAOxU8WwHEq2vNwF6i4j+Q==} + cjs-module-lexer@2.2.0: + resolution: {integrity: sha512-4bHTS2YuzUvtoLjdy+98ykbNB5jS0+07EvFNXerqZQJ89F7DI6ET7OQo/HJuW6K0aVsKA9hj9/RVb2kQVOrPDQ==} + cliui@8.0.1: resolution: {integrity: sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==} engines: {node: '>=12'} @@ -1625,6 +1735,9 @@ packages: resolution: {integrity: sha512-Hs59xBNfUIunMFgWAbGX5cq6893IbWg4KnrjbYwX3tx0ztorVgTDA6B2sxf8ejHJ4wz8BqGUMYlnzNBer5NvGg==} engines: {node: '>= 4'} + import-in-the-middle@2.0.4: + resolution: {integrity: sha512-Al0kMpa0BqfvDnxjxGlab9vdQ0vTDs82TBKrD59X9jReUoPAzSGBb6vGDzMUMFBGyyDF03RpLT4oxGn6BpASzQ==} + import-local@3.2.0: resolution: {integrity: sha512-2SPlun1JUPWoM6t3F0dw0FkCF/jWY8kttcY4f599GLTSjh2OCuuhdTkJQsEcZzBqbXZGKMK2OqW1oZsjtf/gQA==} engines: {node: '>=8'} @@ -2063,6 +2176,9 @@ packages: resolution: {integrity: sha512-KZxYo1BUkWD2TVFLr0MQoM8vUUigWD3LlD83a/75BqC+4qE0Hb1Vo5v1FgcfaNXvfXzr+5EhQ6ing/CaBijTlw==} engines: {node: '>= 18'} + module-details-from-path@1.0.4: + resolution: {integrity: sha512-EGWKgxALGMgzvxYF1UyGTy0HXX/2vHLkw6+NvDKW2jypWbHpjQuj4UMcqQWXHERJhVGKikolT06G3bcKe4fi7w==} + ms@2.1.3: resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} @@ -2316,6 +2432,10 @@ packages: resolution: {integrity: sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==} engines: {node: '>=0.10.0'} + require-in-the-middle@8.0.1: + resolution: {integrity: sha512-QT7FVMXfWOYFbeRBF6nu+I6tr2Tf3u0q8RIEjNob/heKY/nh7drD/k7eeMFmSQgnTtCzLDcCu/XEnpW2wk4xCQ==} + engines: {node: '>=9.3.0 || >=8.10.0 <9.0.0'} + resolve-cwd@3.0.0: resolution: {integrity: sha512-OrZaX2Mb+rJCpH/6CpSqt9xFVpN++x01XnN2ie9g6P5/3xelLAkXWVADpdz1IHD/KFfEXyE6V0U01OQ3UO2rEg==} engines: {node: '>=8'} @@ -3321,9 +3441,127 @@ snapshots: '@nolyfill/is-core-module@1.0.39': {} + '@opentelemetry/api-logs@0.208.0': + dependencies: + '@opentelemetry/api': 1.9.0 + optional: true + '@opentelemetry/api@1.9.0': optional: true + '@opentelemetry/core@1.30.1(@opentelemetry/api@1.9.0)': + dependencies: + '@opentelemetry/api': 1.9.0 + '@opentelemetry/semantic-conventions': 1.28.0 + optional: true + + '@opentelemetry/core@2.2.0(@opentelemetry/api@1.9.0)': + dependencies: + '@opentelemetry/api': 1.9.0 + '@opentelemetry/semantic-conventions': 1.40.0 + optional: true + + '@opentelemetry/exporter-trace-otlp-grpc@0.208.0(@opentelemetry/api@1.9.0)': + dependencies: + '@grpc/grpc-js': 1.14.3 + '@opentelemetry/api': 1.9.0 + '@opentelemetry/core': 2.2.0(@opentelemetry/api@1.9.0) + '@opentelemetry/otlp-exporter-base': 0.208.0(@opentelemetry/api@1.9.0) + '@opentelemetry/otlp-grpc-exporter-base': 0.208.0(@opentelemetry/api@1.9.0) + '@opentelemetry/otlp-transformer': 0.208.0(@opentelemetry/api@1.9.0) + '@opentelemetry/resources': 2.2.0(@opentelemetry/api@1.9.0) + '@opentelemetry/sdk-trace-base': 2.2.0(@opentelemetry/api@1.9.0) + optional: true + + '@opentelemetry/instrumentation@0.208.0(@opentelemetry/api@1.9.0)': + dependencies: + '@opentelemetry/api': 1.9.0 + '@opentelemetry/api-logs': 0.208.0 + import-in-the-middle: 2.0.4 + require-in-the-middle: 8.0.1 + transitivePeerDependencies: + - supports-color + optional: true + + '@opentelemetry/otlp-exporter-base@0.208.0(@opentelemetry/api@1.9.0)': + dependencies: + '@opentelemetry/api': 1.9.0 + '@opentelemetry/core': 2.2.0(@opentelemetry/api@1.9.0) + '@opentelemetry/otlp-transformer': 0.208.0(@opentelemetry/api@1.9.0) + optional: true + + '@opentelemetry/otlp-grpc-exporter-base@0.208.0(@opentelemetry/api@1.9.0)': + dependencies: + '@grpc/grpc-js': 1.14.3 + '@opentelemetry/api': 1.9.0 + '@opentelemetry/core': 2.2.0(@opentelemetry/api@1.9.0) + '@opentelemetry/otlp-exporter-base': 0.208.0(@opentelemetry/api@1.9.0) + '@opentelemetry/otlp-transformer': 0.208.0(@opentelemetry/api@1.9.0) + optional: true + + '@opentelemetry/otlp-transformer@0.208.0(@opentelemetry/api@1.9.0)': + dependencies: + '@opentelemetry/api': 1.9.0 + '@opentelemetry/api-logs': 0.208.0 + '@opentelemetry/core': 2.2.0(@opentelemetry/api@1.9.0) + '@opentelemetry/resources': 2.2.0(@opentelemetry/api@1.9.0) + '@opentelemetry/sdk-logs': 0.208.0(@opentelemetry/api@1.9.0) + '@opentelemetry/sdk-metrics': 2.2.0(@opentelemetry/api@1.9.0) + '@opentelemetry/sdk-trace-base': 2.2.0(@opentelemetry/api@1.9.0) + protobufjs: 7.5.3 + optional: true + + '@opentelemetry/resources@1.30.1(@opentelemetry/api@1.9.0)': + dependencies: + '@opentelemetry/api': 1.9.0 + '@opentelemetry/core': 1.30.1(@opentelemetry/api@1.9.0) + '@opentelemetry/semantic-conventions': 1.28.0 + optional: true + + '@opentelemetry/resources@2.2.0(@opentelemetry/api@1.9.0)': + dependencies: + '@opentelemetry/api': 1.9.0 + '@opentelemetry/core': 2.2.0(@opentelemetry/api@1.9.0) + '@opentelemetry/semantic-conventions': 1.40.0 + optional: true + + '@opentelemetry/sdk-logs@0.208.0(@opentelemetry/api@1.9.0)': + dependencies: + '@opentelemetry/api': 1.9.0 + '@opentelemetry/api-logs': 0.208.0 + '@opentelemetry/core': 2.2.0(@opentelemetry/api@1.9.0) + '@opentelemetry/resources': 2.2.0(@opentelemetry/api@1.9.0) + optional: true + + '@opentelemetry/sdk-metrics@2.2.0(@opentelemetry/api@1.9.0)': + dependencies: + '@opentelemetry/api': 1.9.0 + '@opentelemetry/core': 2.2.0(@opentelemetry/api@1.9.0) + '@opentelemetry/resources': 2.2.0(@opentelemetry/api@1.9.0) + optional: true + + '@opentelemetry/sdk-trace-base@1.30.1(@opentelemetry/api@1.9.0)': + dependencies: + '@opentelemetry/api': 1.9.0 + '@opentelemetry/core': 1.30.1(@opentelemetry/api@1.9.0) + '@opentelemetry/resources': 1.30.1(@opentelemetry/api@1.9.0) + '@opentelemetry/semantic-conventions': 1.28.0 + optional: true + + '@opentelemetry/sdk-trace-base@2.2.0(@opentelemetry/api@1.9.0)': + dependencies: + '@opentelemetry/api': 1.9.0 + '@opentelemetry/core': 2.2.0(@opentelemetry/api@1.9.0) + '@opentelemetry/resources': 2.2.0(@opentelemetry/api@1.9.0) + '@opentelemetry/semantic-conventions': 1.40.0 + optional: true + + '@opentelemetry/semantic-conventions@1.28.0': + optional: true + + '@opentelemetry/semantic-conventions@1.40.0': + optional: true + '@pkgr/core@0.2.9': {} '@protobufjs/aspromise@1.1.2': {} @@ -3620,6 +3858,11 @@ snapshots: abort-controller-x@0.4.3: {} + acorn-import-attributes@1.9.5(acorn@8.15.0): + dependencies: + acorn: 8.15.0 + optional: true + acorn-jsx@5.3.2(acorn@8.16.0): dependencies: acorn: 8.16.0 @@ -3887,6 +4130,9 @@ snapshots: cjs-module-lexer@1.4.3: {} + cjs-module-lexer@2.2.0: + optional: true + cliui@8.0.1: dependencies: string-width: 4.2.3 @@ -4549,6 +4795,14 @@ snapshots: ignore@7.0.5: {} + import-in-the-middle@2.0.4: + dependencies: + acorn: 8.15.0 + acorn-import-attributes: 1.9.5(acorn@8.15.0) + cjs-module-lexer: 2.2.0 + module-details-from-path: 1.0.4 + optional: true + import-local@3.2.0: dependencies: pkg-dir: 4.2.0 @@ -5186,6 +5440,9 @@ snapshots: dependencies: minipass: 7.1.3 + module-details-from-path@1.0.4: + optional: true + ms@2.1.3: {} nanoid@3.3.11: {} @@ -5452,6 +5709,14 @@ snapshots: require-directory@2.1.1: {} + require-in-the-middle@8.0.1: + dependencies: + debug: 4.4.1 + module-details-from-path: 1.0.4 + transitivePeerDependencies: + - supports-color + optional: true + resolve-cwd@3.0.0: dependencies: resolve-from: 5.0.0 diff --git a/sdks/typescript/src/clients/event/event-client.ts b/sdks/typescript/src/clients/event/event-client.ts index 4c2fcf6f89..0b8c55cae8 100644 --- a/sdks/typescript/src/clients/event/event-client.ts +++ b/sdks/typescript/src/clients/event/event-client.ts @@ -55,6 +55,10 @@ export class EventClient { this.tenantId = config.tenant_id; } + /** + * @important This method is instrumented by HatchetInstrumentor._patchPushEvent. + * Keep the signature in sync with the instrumentor wrapper. + */ push(type: string, input: T, options: PushEventOptions = {}) { const namespacedType = applyNamespace(type, this.config.namespace); @@ -78,6 +82,10 @@ export class EventClient { } } + /** + * @important This method is instrumented by HatchetInstrumentor._patchBulkPushEvent. + * Keep the signature in sync with the instrumentor wrapper. + */ bulkPush(type: string, inputs: EventWithMetadata[], options: PushEventOptions = {}) { const namespacedType = applyNamespace(type, this.config.namespace); diff --git a/sdks/typescript/src/clients/hatchet-client/client-config.ts b/sdks/typescript/src/clients/hatchet-client/client-config.ts index 0739e21bc2..26456cdf95 100644 --- a/sdks/typescript/src/clients/hatchet-client/client-config.ts +++ b/sdks/typescript/src/clients/hatchet-client/client-config.ts @@ -16,6 +16,22 @@ const HealthcheckConfigSchema = z.object({ port: z.number().optional().default(8001), }); +export const OpenTelemetryConfigSchema = z.object({ + /** + * List of attribute keys to exclude from spans. + * Useful for filtering sensitive or verbose data like payloads. + */ + excludedAttributes: z.array(z.string()).optional().default([]), + + /** + * If true, includes the task name in the span name for task run spans. + * e.g., "hatchet task run my_task" instead of "hatchet task run" + */ + includeTaskNameInSpanName: z.boolean().optional().default(false), +}); + +export type OpenTelemetryConfig = z.infer; + const TaskMiddlewareSchema = z .object({ before: z.any().optional(), @@ -34,6 +50,7 @@ export const ClientConfigSchema = z.object({ log_level: z.enum(['OFF', 'DEBUG', 'INFO', 'WARN', 'ERROR']).optional(), tenant_id: z.string(), namespace: z.string().optional(), + otel: OpenTelemetryConfigSchema.optional(), middleware: TaskMiddlewareSchema, cancellation_grace_period: DurationMsSchema.optional().default(1000), cancellation_warning_threshold: DurationMsSchema.optional().default(300), diff --git a/sdks/typescript/src/clients/hatchet-client/fixtures/.hatchet.yaml b/sdks/typescript/src/clients/hatchet-client/fixtures/.hatchet.yaml index 62bc0f7cf0..2d33294f12 100644 --- a/sdks/typescript/src/clients/hatchet-client/fixtures/.hatchet.yaml +++ b/sdks/typescript/src/clients/hatchet-client/fixtures/.hatchet.yaml @@ -9,3 +9,7 @@ tls_config: healthcheck: enabled: true port: 8005 +otel: + excludedAttributes: + - additional_metadata + includeTaskNameInSpanName: true diff --git a/sdks/typescript/src/legacy/legacy-client.test.ts b/sdks/typescript/src/legacy/legacy-client.test.ts index 4af8e618bb..ab12112449 100644 --- a/sdks/typescript/src/legacy/legacy-client.test.ts +++ b/sdks/typescript/src/legacy/legacy-client.test.ts @@ -29,6 +29,10 @@ describe('Client', () => { enabled: true, port: 8002, }, + otel: { + excludedAttributes: ['tenant_id', 'workflow_id'], + includeTaskNameInSpanName: true, + }, }, { credentials: ChannelCredentials.createInsecure(), @@ -55,6 +59,10 @@ describe('Client', () => { enabled: true, port: 8002, }, + otel: { + excludedAttributes: ['tenant_id', 'workflow_id'], + includeTaskNameInSpanName: true, + }, }) ); }); @@ -89,6 +97,10 @@ describe('Client', () => { enabled: false, port: 8003, }, + otel: { + excludedAttributes: ['tenant_id'], + includeTaskNameInSpanName: false, + }, }, { config_path: './fixtures/.hatchet.yaml', @@ -116,6 +128,10 @@ describe('Client', () => { enabled: false, port: 8003, }, + otel: { + excludedAttributes: ['tenant_id'], + includeTaskNameInSpanName: false, + }, }) ); }); diff --git a/sdks/typescript/src/opentelemetry/hatchet-exporter.ts b/sdks/typescript/src/opentelemetry/hatchet-exporter.ts new file mode 100644 index 0000000000..f0d8d2e8f7 --- /dev/null +++ b/sdks/typescript/src/opentelemetry/hatchet-exporter.ts @@ -0,0 +1,71 @@ +/** + * Hatchet OTLP Exporter + * + * Creates an OTLP gRPC trace exporter that sends spans to the Hatchet engine's + * collector endpoint, and a SpanProcessor that injects hatchet.* attributes + * into all child spans so they are queryable by the same attributes as the parent. + * + * This mirrors the Go SDK's EnableHatchetCollector() and the Python SDK's + * enable_hatchet_otel_collector option. + */ + +import type { ClientConfig } from '@hatchet/clients/hatchet-client/client-config'; + +try { + require.resolve('@opentelemetry/exporter-trace-otlp-grpc'); + require.resolve('@opentelemetry/sdk-trace-base'); +} catch { + throw new Error( + 'To use HatchetInstrumentor with enableHatchetCollector, you must install: ' + + 'npm install @opentelemetry/exporter-trace-otlp-grpc @opentelemetry/sdk-trace-base' + ); +} + +/* eslint-disable @typescript-eslint/no-require-imports */ +// prettier-ignore +const { OTLPTraceExporter } = require('@opentelemetry/exporter-trace-otlp-grpc') as typeof import('@opentelemetry/exporter-trace-otlp-grpc'); +// prettier-ignore +const sdkTraceBase = require('@opentelemetry/sdk-trace-base') as typeof import('@opentelemetry/sdk-trace-base'); +/* eslint-enable @typescript-eslint/no-require-imports */ + +const { BatchSpanProcessor } = sdkTraceBase; + +type SdkTracerProvider = import('@opentelemetry/sdk-trace-base').BasicTracerProvider; +type ReadableSpan = import('@opentelemetry/sdk-trace-base').ReadableSpan; + +/** + * HatchetAttributeSpanProcessor wraps a BatchSpanProcessor. + * The hatchet.* attributes are already injected by the instrumentor's + * startActiveSpan call, making them available to all child spans in context. + */ +class HatchetAttributeSpanProcessor extends BatchSpanProcessor { + onEnd(span: ReadableSpan): void { + super.onEnd(span); + } +} + +/** + * Creates an OTLP gRPC trace exporter pointing at the Hatchet engine. + */ +function createHatchetExporter(config: ClientConfig): InstanceType { + const insecure = config.tls_config.tls_strategy === 'none'; + + return new OTLPTraceExporter({ + url: `${insecure ? 'http' : 'https'}://${config.host_port}`, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + metadata: { authorization: `Bearer ${config.token}` } as any, + }); +} + +/** + * Adds the Hatchet OTLP exporter to the given TracerProvider. + * The exporter sends spans to the Hatchet engine's collector endpoint + * using the same connection settings as the Hatchet client. + */ +export function addHatchetExporter(tracerProvider: SdkTracerProvider, config: ClientConfig): void { + const exporter = createHatchetExporter(config); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const processor = new HatchetAttributeSpanProcessor(exporter as any); + + tracerProvider.addSpanProcessor(processor); +} diff --git a/sdks/typescript/src/opentelemetry/index.ts b/sdks/typescript/src/opentelemetry/index.ts new file mode 100644 index 0000000000..b55f9030fb --- /dev/null +++ b/sdks/typescript/src/opentelemetry/index.ts @@ -0,0 +1,3 @@ +export { HatchetInstrumentor } from './instrumentor'; +export type { OpenTelemetryConfig } from './types'; +export { OTelAttribute, OTelAttributeType } from '../util/opentelemetry'; diff --git a/sdks/typescript/src/opentelemetry/instrumentor.ts b/sdks/typescript/src/opentelemetry/instrumentor.ts new file mode 100644 index 0000000000..48a4140e0a --- /dev/null +++ b/sdks/typescript/src/opentelemetry/instrumentor.ts @@ -0,0 +1,770 @@ +/** + * Hatchet OpenTelemetry Instrumentor + * + * This module provides automatic instrumentation for Hatchet SDK operations + * including workflow runs, event pushes, and step executions. + * + * The instrumentor follows the OpenTelemetry instrumentation pattern, + * patching module prototypes to automatically instrument all instances. + */ + +import type { Context as OtelContext, Span, Attributes } from '@opentelemetry/api'; + +import type { InstrumentationConfig } from '@opentelemetry/instrumentation'; + +import { HATCHET_VERSION } from '@hatchet/version'; +import { Action } from '@hatchet/clients/dispatcher/action-listener'; +import type { + EventClient, + PushEventOptions, + EventWithMetadata, +} from '@hatchet/clients/event/event-client'; +import type { AdminClient } from '@hatchet/v1/client/admin'; +import type { InternalWorker } from '@hatchet/v1/client/worker/worker-internal'; +import type { ClientConfig } from '@hatchet/clients/hatchet-client/client-config'; +import { OTelAttribute, type ActionOTelAttributeValue } from '../util/opentelemetry'; +import { parseJSON } from '../util/parse'; +import { OpenTelemetryConfig, DEFAULT_CONFIG } from './types'; +import { ScheduledWorkflows } from '../clients/rest/generated/data-contracts'; +import { ScheduleClient, CreateScheduledRunInput } from '../v1/client/features/schedules'; + +try { + require.resolve('@opentelemetry/api'); + require.resolve('@opentelemetry/instrumentation'); +} catch { + throw new Error( + 'To use HatchetInstrumentor, you must install OpenTelemetry packages: npm install @opentelemetry/api @opentelemetry/instrumentation' + ); +} + +/* eslint-disable @typescript-eslint/no-require-imports */ +const otelApi = require('@opentelemetry/api') as typeof import('@opentelemetry/api'); +const otelInstrumentation = + require('@opentelemetry/instrumentation') as typeof import('@opentelemetry/instrumentation'); +/* eslint-enable @typescript-eslint/no-require-imports */ + +const { context, propagation, SpanKind, SpanStatusCode, diag } = otelApi; + +const { + InstrumentationBase, + InstrumentationNodeModuleDefinition, + InstrumentationNodeModuleFile, + isWrapped, +} = otelInstrumentation; + +type HatchetInstrumentationConfig = OpenTelemetryConfig & + InstrumentationConfig & { + /** + * Enable sending traces to the Hatchet engine's OTLP collector. + * Requires @opentelemetry/exporter-trace-otlp-grpc and @opentelemetry/sdk-trace-base. + * Connection settings (endpoint, token, TLS) are read from the provided clientConfig + * or from the same environment variables used by the Hatchet client. + */ + enableHatchetCollector?: boolean; + + /** + * The Hatchet ClientConfig to use for the collector connection. + * If not provided and enableHatchetCollector is true, config will be loaded + * from environment variables / .hatchet.yaml. + */ + clientConfig?: ClientConfig; + }; +type Carrier = Record; + +const INSTRUMENTOR_NAME = '@hatchet-dev/typescript-sdk'; +// FIXME: refactor version check to use the new pattern introduced in #2954 +const SUPPORTED_VERSIONS = ['>=1.16.0']; + +function extractContext(carrier: Carrier | undefined | null): OtelContext { + return propagation.extract(context.active(), carrier ?? {}); +} + +function injectContext(carrier: Carrier): void { + propagation.inject(context.active(), carrier); +} + +function getActionOtelAttributes( + action: Action, + excludedAttributes: string[] = [], + workerId?: string +): Attributes { + const attributes = { + [OTelAttribute.TENANT_ID]: action.tenantId, + [OTelAttribute.WORKER_ID]: workerId, + [OTelAttribute.WORKFLOW_RUN_ID]: action.workflowRunId, + [OTelAttribute.STEP_ID]: action.stepId, + [OTelAttribute.STEP_RUN_ID]: action.stepRunId, + [OTelAttribute.RETRY_COUNT]: action.retryCount, + [OTelAttribute.PARENT_WORKFLOW_RUN_ID]: action.parentWorkflowRunId, + [OTelAttribute.CHILD_WORKFLOW_INDEX]: action.childWorkflowIndex, + [OTelAttribute.CHILD_WORKFLOW_KEY]: action.childWorkflowKey, + [OTelAttribute.ACTION_PAYLOAD]: action.actionPayload, + [OTelAttribute.WORKFLOW_NAME]: action.jobName, + [OTelAttribute.ACTION_NAME]: action.actionId, + [OTelAttribute.WORKFLOW_ID]: action.workflowId, + [OTelAttribute.WORKFLOW_VERSION_ID]: action.workflowVersionId, + } satisfies Record; + + const filtered: Attributes = {}; + for (const [key, value] of Object.entries(attributes)) { + if (!excludedAttributes.includes(key) && value !== undefined && value !== '') { + filtered[key] = value; + } + } + + return filtered; +} + +function filterAttributes( + attributes: Record, + excludedAttributes: string[] = [] +): Attributes { + const filtered: Attributes = {}; + for (const [key, value] of Object.entries(attributes)) { + if ( + !excludedAttributes.includes(key) && + value !== undefined && + value !== null && + value !== '' && + value !== '{}' && + value !== '[]' + ) { + filtered[`hatchet.${key}`] = + typeof value === 'object' ? JSON.stringify(value) : (value as string | number | boolean); + } + } + return filtered; +} + +/** + * HatchetInstrumentor provides OpenTelemetry instrumentation for Hatchet SDK v1. + * + * It automatically instruments: + * - Workflow runs (runWorkflow, runWorkflows) + * - Scheduled workflow runs (schedules.create) + * - Event pushes (push, bulkPush) + * - Step executions (handleStartStepRun, handleCancelStepRun) + * + * Traceparent context is automatically propagated through metadata. + * + * The instrumentor uses the global tracer/meter providers by default. + * Use `setTracerProvider()` and `setMeterProvider()` to configure custom providers. + */ +export class HatchetInstrumentor extends InstrumentationBase { + constructor(config: Partial = {}) { + super(INSTRUMENTOR_NAME, HATCHET_VERSION, { ...DEFAULT_CONFIG, ...config }); + + if (config.enableHatchetCollector) { + this._setupHatchetCollector(config.clientConfig); + } + } + + /** + * Sets up the Hatchet OTLP exporter on the current TracerProvider. + * Loads client config from environment if not provided. + */ + private _setupHatchetCollector(clientConfig?: ClientConfig): void { + try { + /* eslint-disable @typescript-eslint/no-require-imports */ + const { addHatchetExporter } = + require('./hatchet-exporter') as typeof import('./hatchet-exporter'); + + let config = clientConfig; + if (!config) { + // Load config from environment (same as HatchetClient would) + const { ConfigLoader } = + require('@hatchet/util/config-loader/config-loader') as typeof import('@hatchet/util/config-loader/config-loader'); + config = ConfigLoader.loadClientConfig() as ClientConfig; + } + + // Get the SDK TracerProvider - either from the global provider or create one + // eslint-disable-next-line @typescript-eslint/no-explicit-any + let sdkTracerProvider: any; + try { + const sdkTrace = + require('@opentelemetry/sdk-trace-base') as typeof import('@opentelemetry/sdk-trace-base'); + /* eslint-enable @typescript-eslint/no-require-imports */ + + // Check if the global tracer provider is an SDK TracerProvider + const globalProvider = otelApi.trace.getTracerProvider(); + if (globalProvider instanceof sdkTrace.BasicTracerProvider) { + sdkTracerProvider = globalProvider; + } else { + // Create a new SDK TracerProvider and set it as global + sdkTracerProvider = new sdkTrace.BasicTracerProvider(); + sdkTracerProvider.register(); + } + } catch { + diag.warn( + 'hatchet instrumentation: @opentelemetry/sdk-trace-base is required for enableHatchetCollector' + ); + return; + } + + addHatchetExporter(sdkTracerProvider, config); + diag.info('hatchet instrumentation: Hatchet OTLP collector enabled'); + } catch (e) { + diag.warn(`hatchet instrumentation: Failed to set up Hatchet collector: ${e}`); + } + } + + override setConfig(config: Partial = {}): void { + super.setConfig({ ...DEFAULT_CONFIG, ...config }); + } + + private readonly _getConfig = (): HatchetInstrumentationConfig => this.getConfig(); + + protected init(): InstanceType[] { + const eventClientModuleFile = new InstrumentationNodeModuleFile( + '@hatchet-dev/typescript-sdk/clients/event/event-client.js', + SUPPORTED_VERSIONS, + this.patchEventClient.bind(this), + this.unpatchEventClient.bind(this) + ); + + const adminClientModuleFile = new InstrumentationNodeModuleFile( + '@hatchet-dev/typescript-sdk/v1/client/admin.js', + SUPPORTED_VERSIONS, + this.patchAdminClient.bind(this), + this.unpatchAdminClient.bind(this) + ); + + const scheduleClientModuleFile = new InstrumentationNodeModuleFile( + '@hatchet-dev/typescript-sdk/v1/client/features/schedules.js', + SUPPORTED_VERSIONS, + this.patchScheduleClient.bind(this), + this.unpatchScheduleClient.bind(this) + ); + + const workerModuleFile = new InstrumentationNodeModuleFile( + '@hatchet-dev/typescript-sdk/v1/client/worker/worker-internal.js', + SUPPORTED_VERSIONS, + this.patchWorker.bind(this), + this.unpatchWorker.bind(this) + ); + + const moduleDefinition = new InstrumentationNodeModuleDefinition( + INSTRUMENTOR_NAME, + SUPPORTED_VERSIONS, + undefined, + undefined, + [eventClientModuleFile, adminClientModuleFile, workerModuleFile, scheduleClientModuleFile] + ); + + return [moduleDefinition]; + } + + private patchEventClient(moduleExports: unknown, _moduleVersion?: string): unknown { + const exports = moduleExports as { EventClient?: { prototype: EventClient } }; + if (!exports?.EventClient?.prototype) { + diag.debug('hatchet instrumentation: EventClient not found in module exports'); + return moduleExports; + } + this._patchPushEvent(exports.EventClient.prototype); + this._patchBulkPushEvent(exports.EventClient.prototype); + + return moduleExports; + } + + private unpatchEventClient(moduleExports: unknown, _moduleVersion?: string): unknown { + const exports = moduleExports as { EventClient?: { prototype: EventClient } }; + if (!exports?.EventClient?.prototype) { + return moduleExports; + } + + if (isWrapped(exports.EventClient.prototype.push)) { + this._unwrap(exports.EventClient.prototype, 'push'); + } + if (isWrapped(exports.EventClient.prototype.bulkPush)) { + this._unwrap(exports.EventClient.prototype, 'bulkPush'); + } + + return moduleExports; + } + + private _patchPushEvent(prototype: EventClient): void { + if (isWrapped(prototype.push)) { + this._unwrap(prototype, 'push'); + } + const { tracer, _getConfig: getConfig } = this; + + this._wrap(prototype, 'push', (original: EventClient['push']) => { + return function wrappedPush( + this: EventClient, + type: string, + input: T, + options: PushEventOptions = {} + ) { + const attributes = filterAttributes( + { + [OTelAttribute.EVENT_KEY]: type, + [OTelAttribute.ACTION_PAYLOAD]: JSON.stringify(input), + [OTelAttribute.ADDITIONAL_METADATA]: options.additionalMetadata + ? JSON.stringify(options.additionalMetadata) + : undefined, + [OTelAttribute.PRIORITY]: options.priority, + [OTelAttribute.FILTER_SCOPE]: options.scope, + }, + getConfig().excludedAttributes + ); + + return tracer.startActiveSpan( + 'hatchet.push_event', + { + kind: SpanKind.PRODUCER, + attributes, + }, + (span: Span) => { + const enhancedMetadata: Carrier = { ...(options.additionalMetadata ?? {}) }; + injectContext(enhancedMetadata); + + const enhancedOptions: PushEventOptions = { + ...options, + additionalMetadata: enhancedMetadata, + }; + + const result = original.call(this, type, input, enhancedOptions); + + return result.finally(() => { + span.end(); + }); + } + ); + }; + }); + } + + private _patchBulkPushEvent(prototype: EventClient): void { + if (isWrapped(prototype.bulkPush)) { + this._unwrap(prototype, 'bulkPush'); + } + const { tracer, _getConfig: getConfig } = this; + + this._wrap(prototype, 'bulkPush', (original: EventClient['bulkPush']) => { + return function wrappedBulkPush( + this: EventClient, + type: string, + inputs: EventWithMetadata[], + options: PushEventOptions = {} + ) { + const attributes = filterAttributes( + { + [OTelAttribute.EVENT_KEY]: type, + [OTelAttribute.ACTION_PAYLOAD]: JSON.stringify(inputs), + [OTelAttribute.ADDITIONAL_METADATA]: options.additionalMetadata + ? JSON.stringify(options.additionalMetadata) + : undefined, + [OTelAttribute.PRIORITY]: options.priority, + }, + getConfig().excludedAttributes + ); + + return tracer.startActiveSpan( + 'hatchet.bulk_push_event', + { + kind: SpanKind.PRODUCER, + attributes, + }, + (span: Span) => { + const enhancedInputs = inputs.map((input) => { + const enhancedMetadata: Carrier = { + ...((input.additionalMetadata as Carrier) ?? {}), + }; + injectContext(enhancedMetadata); + return { + ...input, + additionalMetadata: enhancedMetadata, + }; + }); + + const result = original.call(this, type, enhancedInputs, options); + + return result.finally(() => { + span.end(); + }); + } + ); + }; + }); + } + + private patchAdminClient(moduleExports: unknown, _moduleVersion?: string): unknown { + const exports = moduleExports as { AdminClient?: { prototype: AdminClient } }; + if (!exports?.AdminClient?.prototype) { + diag.debug('hatchet instrumentation: AdminClient not found in module exports'); + return moduleExports; + } + + this._patchRunWorkflow(exports.AdminClient.prototype); + this._patchRunWorkflows(exports.AdminClient.prototype); + + return moduleExports; + } + + private unpatchAdminClient(moduleExports: unknown, _moduleVersion?: string): unknown { + const exports = moduleExports as { AdminClient?: { prototype: AdminClient } }; + if (!exports?.AdminClient?.prototype) { + return moduleExports; + } + + if (isWrapped(exports.AdminClient.prototype.runWorkflow)) { + this._unwrap(exports.AdminClient.prototype, 'runWorkflow'); + } + if (isWrapped(exports.AdminClient.prototype.runWorkflows)) { + this._unwrap(exports.AdminClient.prototype, 'runWorkflows'); + } + + return moduleExports; + } + + private _patchRunWorkflow(prototype: AdminClient): void { + if (isWrapped(prototype.runWorkflow)) { + this._unwrap(prototype, 'runWorkflow'); + } + const { tracer, _getConfig: getConfig } = this; + + this._wrap(prototype, 'runWorkflow', (original: AdminClient['runWorkflow']) => { + return async function wrappedRunWorkflow( + this: AdminClient, + workflowName: string, + input: unknown, + options?: { + parentId?: string; + parentStepRunId?: string; + childIndex?: number; + childKey?: string; + additionalMetadata?: Record; + desiredWorkerId?: string; + priority?: number; + } + ) { + const attributes = filterAttributes( + { + [OTelAttribute.WORKFLOW_NAME]: workflowName, + [OTelAttribute.ACTION_PAYLOAD]: JSON.stringify(input), + [OTelAttribute.PARENT_ID]: options?.parentId, + [OTelAttribute.PARENT_STEP_RUN_ID]: options?.parentStepRunId, + [OTelAttribute.CHILD_INDEX]: options?.childIndex, + [OTelAttribute.CHILD_KEY]: options?.childKey, + [OTelAttribute.ADDITIONAL_METADATA]: options?.additionalMetadata + ? JSON.stringify(options.additionalMetadata) + : undefined, + [OTelAttribute.PRIORITY]: options?.priority, + [OTelAttribute.DESIRED_WORKER_ID]: options?.desiredWorkerId, + }, + getConfig().excludedAttributes + ); + + return tracer.startActiveSpan( + 'hatchet.run_workflow', + { + kind: SpanKind.PRODUCER, + attributes, + }, + (span: Span) => { + const enhancedMetadata: Carrier = { ...(options?.additionalMetadata ?? {}) }; + injectContext(enhancedMetadata); + + const enhancedOptions = { + ...options, + additionalMetadata: enhancedMetadata, + }; + + return original + .call(this, workflowName, input, enhancedOptions) + .catch((error: Error) => { + span.recordException(error); + span.setStatus({ code: SpanStatusCode.ERROR, message: error?.message }); + throw error; + }) + .finally(() => { + span.end(); + }); + } + ); + } as AdminClient['runWorkflow']; + }); + } + + private _patchRunWorkflows(prototype: AdminClient): void { + if (isWrapped(prototype.runWorkflows)) { + this._unwrap(prototype, 'runWorkflows'); + } + const { tracer, _getConfig: getConfig } = this; + + this._wrap(prototype, 'runWorkflows', (original: AdminClient['runWorkflows']) => { + return async function wrappedRunWorkflows( + this: AdminClient, + workflowRuns: Array<{ + workflowName: string; + input: unknown; + options?: { + parentId?: string; + parentStepRunId?: string; + childIndex?: number; + childKey?: string; + additionalMetadata?: Record; + desiredWorkerId?: string; + priority?: number; + }; + }>, + batchSize?: number + ) { + const attributes = filterAttributes( + { + [OTelAttribute.WORKFLOW_NAME]: JSON.stringify(workflowRuns.map((r) => r.workflowName)), + [OTelAttribute.ACTION_PAYLOAD]: JSON.stringify(workflowRuns), + }, + getConfig().excludedAttributes + ); + + return tracer.startActiveSpan( + 'hatchet.run_workflows', + { + kind: SpanKind.PRODUCER, + attributes, + }, + (span: Span) => { + const enhancedWorkflowRuns = workflowRuns.map((run) => { + const enhancedMetadata: Carrier = { ...(run.options?.additionalMetadata ?? {}) }; + injectContext(enhancedMetadata); + return { + ...run, + options: { + ...run.options, + additionalMetadata: enhancedMetadata, + }, + }; + }); + + return original + .call(this, enhancedWorkflowRuns, batchSize) + .catch((error: Error) => { + span.recordException(error); + span.setStatus({ code: SpanStatusCode.ERROR, message: error?.message }); + throw error; + }) + .finally(() => { + span.end(); + }); + } + ); + } as AdminClient['runWorkflows']; + }); + } + + private patchWorker(moduleExports: unknown, _moduleVersion?: string): unknown { + const exports = moduleExports as { InternalWorker?: { prototype: InternalWorker } }; + if (!exports?.InternalWorker?.prototype) { + diag.debug('hatchet instrumentation: InternalWorker not found in module exports'); + return moduleExports; + } + + this._patchHandleStartStepRun(exports.InternalWorker.prototype); + this._patchHandleCancelStepRun(exports.InternalWorker.prototype); + + return moduleExports; + } + + private unpatchWorker(moduleExports: unknown, _moduleVersion?: string): unknown { + const exports = moduleExports as { InternalWorker?: { prototype: InternalWorker } }; + if (!exports?.InternalWorker?.prototype) { + return moduleExports; + } + + if (isWrapped(exports.InternalWorker.prototype.handleStartStepRun)) { + this._unwrap(exports.InternalWorker.prototype, 'handleStartStepRun'); + } + if (isWrapped(exports.InternalWorker.prototype.handleCancelStepRun)) { + this._unwrap(exports.InternalWorker.prototype, 'handleCancelStepRun'); + } + + return moduleExports; + } + + // IMPORTANT: Keep this wrapper's signature in sync with InternalWorker.handleStartStepRun + private _patchHandleStartStepRun(prototype: InternalWorker): void { + if (isWrapped(prototype.handleStartStepRun)) { + this._unwrap(prototype, 'handleStartStepRun'); + } + const { tracer, _getConfig: getConfig } = this; + + this._wrap( + prototype, + 'handleStartStepRun', + (original: InternalWorker['handleStartStepRun']) => { + return async function wrappedHandleStartStepRun( + this: InternalWorker, + action: Action + ): Promise { + const additionalMetadata = action.additionalMetadata + ? parseJSON(action.additionalMetadata) + : undefined; + const parentContext = extractContext(additionalMetadata); + const attributes = getActionOtelAttributes( + action, + getConfig().excludedAttributes, + this.workerId + ); + + let spanName = 'hatchet.start_step_run'; + if (getConfig().includeTaskNameInSpanName) { + spanName += `.${action.actionId}`; + } + + return tracer.startActiveSpan( + spanName, + { + kind: SpanKind.CONSUMER, + attributes, + }, + parentContext, + (span: Span) => { + return original + .call(this, action) + .then((taskError: Error | undefined) => { + if (taskError instanceof Error) { + span.recordException(taskError); + span.setStatus({ code: SpanStatusCode.ERROR, message: taskError.message }); + } else { + span.setStatus({ code: SpanStatusCode.OK }); + } + return taskError; + }) + .finally(() => { + span.end(); + }); + } + ); + }; + } + ); + } + + private _patchHandleCancelStepRun(prototype: InternalWorker): void { + if (isWrapped(prototype.handleCancelStepRun)) { + this._unwrap(prototype, 'handleCancelStepRun'); + } + const { tracer } = this; + + this._wrap( + prototype, + 'handleCancelStepRun', + (original: InternalWorker['handleCancelStepRun']) => { + return async function wrappedHandleCancelStepRun( + this: InternalWorker, + action: Action + ): Promise { + const attributes: Attributes = { + [`hatchet.${OTelAttribute.STEP_RUN_ID}`]: action.stepRunId, + }; + + return tracer.startActiveSpan( + 'hatchet.cancel_step_run', + { + kind: SpanKind.CONSUMER, + attributes, + }, + (span: Span) => { + const result = original.call(this, action); + + return result.finally(() => { + span.end(); + }); + } + ); + }; + } + ); + } + + private patchScheduleClient(moduleExports: unknown, _moduleVersion?: string): unknown { + const exports = moduleExports as { ScheduleClient?: { prototype: ScheduleClient } }; + if (!exports?.ScheduleClient?.prototype) { + diag.debug('hatchet instrumentation: ScheduleClient not found in module exports'); + return moduleExports; + } + + this._patchScheduleCreate(exports.ScheduleClient.prototype); + + return moduleExports; + } + + private unpatchScheduleClient(moduleExports: unknown, _moduleVersion?: string): unknown { + const exports = moduleExports as { ScheduleClient?: { prototype: ScheduleClient } }; + if (!exports?.ScheduleClient?.prototype) { + return moduleExports; + } + + if (isWrapped(exports.ScheduleClient.prototype.create)) { + this._unwrap(exports.ScheduleClient.prototype, 'create'); + } + + return moduleExports; + } + + // IMPORTANT: Keep this wrapper's signature in sync with ScheduleClient.create + private _patchScheduleCreate(prototype: ScheduleClient): void { + if (isWrapped(prototype.create)) { + this._unwrap(prototype, 'create'); + } + const { tracer, _getConfig: getConfig } = this; + + this._wrap(prototype, 'create', (original: ScheduleClient['create']) => { + return async function wrappedCreate( + this: ScheduleClient, + workflow: string, + input: CreateScheduledRunInput + ): Promise { + const triggerAtIso = + input.triggerAt instanceof Date + ? input.triggerAt.toISOString() + : new Date(input.triggerAt).toISOString(); + + const attributes = filterAttributes( + { + [OTelAttribute.WORKFLOW_NAME]: workflow, + [OTelAttribute.RUN_AT_TIMESTAMPS]: JSON.stringify([triggerAtIso]), + [OTelAttribute.ACTION_PAYLOAD]: JSON.stringify(input.input), + [OTelAttribute.ADDITIONAL_METADATA]: input.additionalMetadata + ? JSON.stringify(input.additionalMetadata) + : undefined, + [OTelAttribute.PRIORITY]: input.priority, + }, + getConfig().excludedAttributes + ); + + return tracer.startActiveSpan( + 'hatchet.schedule_workflow', + { + kind: SpanKind.PRODUCER, + attributes, + }, + (span: Span) => { + // Inject traceparent into additionalMetadata for context propagation + const enhancedMetadata: Carrier = { ...(input.additionalMetadata ?? {}) }; + injectContext(enhancedMetadata); + + const enhancedInput = { + ...input, + additionalMetadata: enhancedMetadata, + }; + + return original + .call(this, workflow, enhancedInput as CreateScheduledRunInput) + .catch((error: Error) => { + span.recordException(error); + span.setStatus({ code: SpanStatusCode.ERROR, message: error?.message }); + throw error; + }) + .finally(() => { + span.end(); + }); + } + ); + }; + }); + } +} + +export default HatchetInstrumentor; diff --git a/sdks/typescript/src/opentelemetry/types.ts b/sdks/typescript/src/opentelemetry/types.ts new file mode 100644 index 0000000000..86f55768ff --- /dev/null +++ b/sdks/typescript/src/opentelemetry/types.ts @@ -0,0 +1,6 @@ +export type { OpenTelemetryConfig } from '@hatchet/clients/hatchet-client/client-config'; + +export const DEFAULT_CONFIG = { + excludedAttributes: [] as string[], + includeTaskNameInSpanName: false, +}; diff --git a/sdks/typescript/src/util/config-loader/config-loader.test.ts b/sdks/typescript/src/util/config-loader/config-loader.test.ts index 13332509b1..42b663ddb9 100644 --- a/sdks/typescript/src/util/config-loader/config-loader.test.ts +++ b/sdks/typescript/src/util/config-loader/config-loader.test.ts @@ -2,6 +2,9 @@ import { ConfigLoader } from './config-loader'; describe('ConfigLoader', () => { beforeEach(() => { + // Clear env vars that might leak from other tests + delete process.env.HATCHET_CLIENT_TLS_STRATEGY; + process.env.HATCHET_CLIENT_TOKEN = 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJncnBjX2Jyb2FkY2FzdF9hZGRyZXNzIjoiMTI3LjAuMC4xOjgwODAiLCJzZXJ2ZXJfdXJsIjoiaHR0cDovL2xvY2FsaG9zdDo4MDgwIiwic3ViIjoiNzA3ZDA4NTUtODBhYi00ZTFmLWExNTYtZjFjNDU0NmNiZjUyIn0K.abcdef'; process.env.HATCHET_CLIENT_TLS_STRATEGY = 'tls'; @@ -34,6 +37,10 @@ describe('ConfigLoader', () => { enabled: true, port: 8001, }, + otel: { + excludedAttributes: [], + includeTaskNameInSpanName: false, + }, }); }); @@ -86,6 +93,10 @@ describe('ConfigLoader', () => { enabled: true, port: 8002, }, + otel: { + excludedAttributes: ['additional_metadata'], + includeTaskNameInSpanName: true, + }, }); }); diff --git a/sdks/typescript/src/util/config-loader/config-loader.ts b/sdks/typescript/src/util/config-loader/config-loader.ts index 5e166d287e..e7b97cfe31 100644 --- a/sdks/typescript/src/util/config-loader/config-loader.ts +++ b/sdks/typescript/src/util/config-loader/config-loader.ts @@ -19,7 +19,9 @@ type EnvVars = | 'HATCHET_CLIENT_LOG_LEVEL' | 'HATCHET_CLIENT_NAMESPACE' | 'HATCHET_CLIENT_WORKER_HEALTHCHECK_ENABLED' - | 'HATCHET_CLIENT_WORKER_HEALTHCHECK_PORT'; + | 'HATCHET_CLIENT_WORKER_HEALTHCHECK_PORT' + | 'HATCHET_CLIENT_OPENTELEMETRY_EXCLUDED_ATTRIBUTES' + | 'HATCHET_CLIENT_OPENTELEMETRY_INCLUDE_TASK_NAME_IN_SPAN_NAME'; type TLSStrategy = 'tls' | 'mtls'; @@ -94,6 +96,15 @@ export class ConfigLoader { namespace = `${namespace}_`; } + const otelConfig = override?.otel ?? + yaml?.otel ?? { + excludedAttributes: this.parseJsonArray( + this.env('HATCHET_CLIENT_OPENTELEMETRY_EXCLUDED_ATTRIBUTES') || '[]' + ), + includeTaskNameInSpanName: + this.env('HATCHET_CLIENT_OPENTELEMETRY_INCLUDE_TASK_NAME_IN_SPAN_NAME') === 'true', + }; + return { token: override?.token ?? yaml?.token ?? this.env('HATCHET_CLIENT_TOKEN'), host_port: grpcBroadcastAddress, @@ -107,9 +118,19 @@ export class ConfigLoader { 'INFO', tenant_id: tenantId, namespace: namespace ? `${namespace}`.toLowerCase() : '', + otel: otelConfig, }; } + private static parseJsonArray(value: string): string[] { + try { + const parsed = JSON.parse(value); + return Array.isArray(parsed) ? parsed : []; + } catch { + return []; + } + } + static get default_yaml_config_path() { return p.join(process.cwd(), DEFAULT_CONFIG_FILE); } diff --git a/sdks/typescript/src/util/config-loader/fixtures/.hatchet.yaml b/sdks/typescript/src/util/config-loader/fixtures/.hatchet.yaml index 2886047832..b267f34d46 100644 --- a/sdks/typescript/src/util/config-loader/fixtures/.hatchet.yaml +++ b/sdks/typescript/src/util/config-loader/fixtures/.hatchet.yaml @@ -9,3 +9,7 @@ tls_config: healthcheck: enabled: true port: 8002 +otel: + excludedAttributes: + - additional_metadata + includeTaskNameInSpanName: true diff --git a/sdks/typescript/src/util/opentelemetry.ts b/sdks/typescript/src/util/opentelemetry.ts new file mode 100644 index 0000000000..b738a0e3c3 --- /dev/null +++ b/sdks/typescript/src/util/opentelemetry.ts @@ -0,0 +1,59 @@ +export const OTelAttribute = { + // Shared + NAMESPACE: 'namespace', + ADDITIONAL_METADATA: 'additional_metadata', + WORKFLOW_NAME: 'workflow_name', + + PRIORITY: 'priority', + ACTION_PAYLOAD: 'payload', + + // Action + ACTION_NAME: 'action_name', + CHILD_WORKFLOW_INDEX: 'child_workflow_index', + CHILD_WORKFLOW_KEY: 'child_workflow_key', + PARENT_WORKFLOW_RUN_ID: 'parent_workflow_run_id', + RETRY_COUNT: 'retry_count', + STEP_ID: 'step_id', + STEP_RUN_ID: 'step_run_id', + TENANT_ID: 'tenant_id', + WORKER_ID: 'worker_id', + WORKFLOW_ID: 'workflow_id', + WORKFLOW_RUN_ID: 'workflow_run_id', + WORKFLOW_VERSION_ID: 'workflow_version_id', + + // Push Event + EVENT_KEY: 'event_key', + FILTER_SCOPE: 'scope', + + // Trigger Workflow + PARENT_ID: 'parent_id', + PARENT_STEP_RUN_ID: 'parent_step_run_id', + CHILD_INDEX: 'child_index', + CHILD_KEY: 'child_key', + DESIRED_WORKER_ID: 'desired_worker_id', + STICKY: 'sticky', + KEY: 'key', + + // Schedule Workflow + RUN_AT_TIMESTAMPS: 'run_at_timestamps', +} as const; + +export type OTelAttributeType = (typeof OTelAttribute)[keyof typeof OTelAttribute]; + +type ActionOTelAttributeKey = + | 'TENANT_ID' + | 'WORKER_ID' + | 'WORKFLOW_RUN_ID' + | 'STEP_ID' + | 'STEP_RUN_ID' + | 'RETRY_COUNT' + | 'PARENT_WORKFLOW_RUN_ID' + | 'CHILD_WORKFLOW_INDEX' + | 'CHILD_WORKFLOW_KEY' + | 'ACTION_PAYLOAD' + | 'WORKFLOW_NAME' + | 'ACTION_NAME' + | 'WORKFLOW_ID' + | 'WORKFLOW_VERSION_ID'; + +export type ActionOTelAttributeValue = (typeof OTelAttribute)[ActionOTelAttributeKey]; diff --git a/sdks/typescript/src/v1/client/admin.ts b/sdks/typescript/src/v1/client/admin.ts index 1c6ffc1d9b..c120179bd0 100644 --- a/sdks/typescript/src/v1/client/admin.ts +++ b/sdks/typescript/src/v1/client/admin.ts @@ -107,6 +107,9 @@ export class AdminClient { * @param input an object containing the input to the workflow * @param options an object containing the options to run the workflow * @returns the ID of the new workflow run + * + * @important This method is instrumented by HatchetInstrumentor._patchRunWorkflow. + * Keep the signature in sync with the instrumentor wrapper. */ async runWorkflow( workflowName: string, @@ -187,6 +190,9 @@ export class AdminClient { * Order is preserved in the response. * @param workflowRuns an array of objects containing the workflow name, input, and options for each workflow run * @returns an array of workflow run references + * + * @important This method is instrumented by HatchetInstrumentor._patchRunWorkflows. + * Keep the signature in sync with the instrumentor wrapper. */ async runWorkflows( workflowRuns: Array<{ diff --git a/sdks/typescript/src/v1/client/features/schedules.ts b/sdks/typescript/src/v1/client/features/schedules.ts index 6742d6b9ce..2baa971678 100644 --- a/sdks/typescript/src/v1/client/features/schedules.ts +++ b/sdks/typescript/src/v1/client/features/schedules.ts @@ -77,6 +77,9 @@ export class ScheduleClient { * @param scheduledRun - The input data for creating the Scheduled Run. * @returns A promise that resolves to the created ScheduledWorkflows object. * @throws Will throw an error if the input is invalid or the API call fails. + * + * @important This method is instrumented by HatchetInstrumentor._patchScheduleCreate. + * Keep the signature in sync with the instrumentor wrapper. */ async create( workflow: string | Workflow, diff --git a/sdks/typescript/src/v1/client/worker/worker-internal.ts b/sdks/typescript/src/v1/client/worker/worker-internal.ts index 985c17b725..da12ec6798 100644 --- a/sdks/typescript/src/v1/client/worker/worker-internal.ts +++ b/sdks/typescript/src/v1/client/worker/worker-internal.ts @@ -416,7 +416,11 @@ export class InternalWorker { this.registerActions(workflow); } - async handleStartStepRun(action: Action) { + /** + * @important This method is instrumented by HatchetInstrumentor._patchHandleStartStepRun. + * Keep the signature in sync with the instrumentor wrapper. + */ + async handleStartStepRun(action: Action): Promise { const { actionId, taskRunExternalId, taskName } = action; try { @@ -429,7 +433,7 @@ export class InternalWorker { if (!step) { this.logger.error(`Registered actions: '${Object.keys(this.action_registry).join(', ')}'`); this.logger.error(`Could not find step '${actionId}'`); - return; + return undefined; } const run = async () => { @@ -561,14 +565,15 @@ export class InternalWorker { } }; - const future = new HatchetPromise( + const future = new HatchetPromise( (async () => { let result: any; try { result = await run(); } catch (e: any) { await failure(e); - return; + // Return error for OTel instrumentor to capture + return e; } // Postcheck: user code may swallow AbortError; don't report completion after cancellation. @@ -584,6 +589,7 @@ export class InternalWorker { throwIfAborted(context.abortController.signal); await success(result); + return undefined; })() ); this.futures[taskRunExternalId] = future; @@ -601,7 +607,7 @@ export class InternalWorker { }); try { - await future.promise; + return await future.promise; } catch (e: any) { const message = e?.message || String(e); // TODO is this cased correctly... @@ -612,9 +618,11 @@ export class InternalWorker { e ); } + return e instanceof Error ? e : new Error(String(e)); } } catch (e: any) { this.logger.error('Could not send action event (outer): ', e); + return e instanceof Error ? e : new Error(String(e)); } } @@ -753,6 +761,10 @@ export class InternalWorker { }; } + /** + * @important This method is instrumented by HatchetInstrumentor._patchHandleCancelStepRun. + * Keep the signature in sync with the instrumentor wrapper. + */ async handleCancelStepRun(action: Action) { const { taskRunExternalId, taskName } = action; diff --git a/sdks/typescript/src/v1/examples/opentelemetry/run.ts b/sdks/typescript/src/v1/examples/opentelemetry/run.ts new file mode 100644 index 0000000000..420a15d30c --- /dev/null +++ b/sdks/typescript/src/v1/examples/opentelemetry/run.ts @@ -0,0 +1,43 @@ +import { getTracer } from './tracer'; + +import { SpanStatusCode, type Span } from '@opentelemetry/api'; +import { hatchet } from '../hatchet-client'; +import { otelWorkflow } from './worker'; + +const tracer = getTracer('opentelemetry-triggers'); + +async function runWorkflow() { + return tracer.startActiveSpan('run_workflow', async (span: Span) => { + try { + const workflowRun = await hatchet.admin.runWorkflow(otelWorkflow.name, {}); + const runId = await workflowRun.getWorkflowRunId(); + console.log(`Started workflow run: ${runId}`); + + const result = await workflowRun.output; + span.setStatus({ code: SpanStatusCode.OK, message: 'Workflow completed' }); + console.log(`Workflow completed with result:`, result); + } catch (error: any) { + const errorMessage = Array.isArray(error) + ? error.join(', ') + : error?.message || String(error); + console.error('Workflow failed:', errorMessage); + span.recordException(error); + span.setStatus({ code: SpanStatusCode.ERROR, message: errorMessage }); + } finally { + span.end(); + } + }); +} + +async function main() { + await runWorkflow(); + + console.log('Waiting for spans to be exported...'); + await new Promise((resolve) => setTimeout(resolve, 5000)); + + process.exit(0); +} + +if (require.main === module) { + main().catch(console.error); +} diff --git a/sdks/typescript/src/v1/examples/opentelemetry/tracer.ts b/sdks/typescript/src/v1/examples/opentelemetry/tracer.ts new file mode 100644 index 0000000000..10892b23a8 --- /dev/null +++ b/sdks/typescript/src/v1/examples/opentelemetry/tracer.ts @@ -0,0 +1,64 @@ +/* eslint-disable @typescript-eslint/no-require-imports */ +const { NodeTracerProvider } = require('@opentelemetry/sdk-trace-node'); +const { OTLPTraceExporter } = require('@opentelemetry/exporter-trace-otlp-http'); +const { resourceFromAttributes } = require('@opentelemetry/resources'); +const { SEMRESATTRS_SERVICE_NAME } = require('@opentelemetry/semantic-conventions'); +const { BatchSpanProcessor } = require('@opentelemetry/sdk-trace-base'); +const { registerInstrumentations } = require('@opentelemetry/instrumentation'); +const { trace } = require('@opentelemetry/api'); +/* eslint-enable @typescript-eslint/no-require-imports */ + +import type { TracerProvider, Tracer } from '@opentelemetry/api'; +import { HatchetInstrumentor } from '@hatchet/opentelemetry'; + +const isCI = process.env.CI === 'true'; + +let traceProvider: TracerProvider; + +if (isCI) { + traceProvider = trace.getTracerProvider(); + registerInstrumentations({ + tracerProvider: traceProvider, + instrumentations: [new HatchetInstrumentor()], + }); +} else { + const resource = resourceFromAttributes({ + [SEMRESATTRS_SERVICE_NAME]: + process.env.HATCHET_CLIENT_OTEL_SERVICE_NAME || 'hatchet-typescript-example', + }); + + const headersEnv = process.env.HATCHET_CLIENT_OTEL_EXPORTER_OTLP_HEADERS; + const headers: Record | undefined = headersEnv + ? { [headersEnv.split('=')[0]]: headersEnv.split('=')[1] } + : undefined; + + const exporter = new OTLPTraceExporter({ + url: + process.env.HATCHET_CLIENT_OTEL_EXPORTER_OTLP_ENDPOINT || 'http://localhost:4318/v1/traces', + headers, + }); + + const provider = new NodeTracerProvider({ + resource, + spanProcessors: [new BatchSpanProcessor(exporter)], + }); + + provider.register(); + + traceProvider = provider; + + registerInstrumentations({ + tracerProvider: traceProvider, + instrumentations: [ + new HatchetInstrumentor({ + includeTaskNameInSpanName: true, + }), + ], + }); +} + +function getTracer(name: string): Tracer { + return trace.getTracer(name); +} + +export { traceProvider, getTracer }; diff --git a/sdks/typescript/src/v1/examples/opentelemetry/worker.ts b/sdks/typescript/src/v1/examples/opentelemetry/worker.ts new file mode 100644 index 0000000000..443bc9b556 --- /dev/null +++ b/sdks/typescript/src/v1/examples/opentelemetry/worker.ts @@ -0,0 +1,70 @@ +import { hatchet } from '../hatchet-client'; +import { getTracer } from './tracer'; +import { SpanStatusCode } from '@opentelemetry/api'; + +const tracer = getTracer('opentelemetry-worker'); + +export const otelWorkflow = hatchet.workflow({ + name: 'otelworkflowtypescript', +}); + +otelWorkflow.task({ + name: 'step-with-custom-spans', + fn: async () => { + return tracer.startActiveSpan('custom-business-logic', async (span) => { + try { + console.log('Executing step with custom tracing...'); + await new Promise((resolve) => setTimeout(resolve, 100)); + + return { result: 'success' }; + } finally { + span.end(); + } + }); + }, +}); + +otelWorkflow.task({ + name: 'step-with-error', + fn: async () => { + return tracer.startActiveSpan('custom-span-with-error', async (span) => { + try { + throw new Error('Intentional error for demonstration'); + } catch (error: any) { + span.recordException(error); + span.setStatus({ code: SpanStatusCode.ERROR, message: error?.message }); + throw error; + } finally { + span.end(); + } + }); + }, +}); + +otelWorkflow.task({ + name: 'auto-instrumented-step', + fn: async () => { + console.log('This step is automatically traced without manual span code'); + return { automatically: 'instrumented' }; + }, +}); + +otelWorkflow.task({ + name: 'auto-instrumented-step-with-error', + fn: async () => { + throw new Error('Auto-instrumented step error'); + }, +}); + +async function main() { + const worker = await hatchet.worker('otel-example-worker-ts', { + slots: 1, + workflows: [otelWorkflow], + }); + + await worker.start(); +} + +if (require.main === module) { + main().catch(console.error); +} diff --git a/sql/schema/v1-core.sql b/sql/schema/v1-core.sql index e2c1cd3abe..e975dc2185 100644 --- a/sql/schema/v1-core.sql +++ b/sql/schema/v1-core.sql @@ -2274,3 +2274,36 @@ CREATE TABLE v1_event_to_run ( PRIMARY KEY (event_id, event_seen_at, run_external_id) ) PARTITION BY RANGE(event_seen_at); + +CREATE TYPE v1_otel_span_kind AS ENUM ('UNSPECIFIED', 'INTERNAL', 'SERVER', 'CLIENT', 'PRODUCER', 'CONSUMER'); + +CREATE TYPE v1_otel_status_code AS ENUM ('UNSET', 'OK', 'ERROR'); + +CREATE TABLE v1_otel_traces ( + id BIGINT GENERATED ALWAYS AS IDENTITY, + tenant_id UUID NOT NULL, + trace_id TEXT NOT NULL, + span_id TEXT NOT NULL, + parent_span_id TEXT NOT NULL DEFAULT '', + span_name TEXT NOT NULL, + span_kind v1_otel_span_kind NOT NULL DEFAULT 'INTERNAL', + service_name TEXT NOT NULL DEFAULT 'unknown', + status_code v1_otel_status_code NOT NULL DEFAULT 'UNSET', + status_message TEXT NOT NULL DEFAULT '', + duration_ns BIGINT NOT NULL DEFAULT 0, + resource_attributes JSONB NOT NULL DEFAULT '{}', + span_attributes JSONB NOT NULL DEFAULT '{}', + scope_name TEXT NOT NULL DEFAULT '', + scope_version TEXT NOT NULL DEFAULT '', + task_run_external_id UUID, + workflow_run_external_id UUID, + start_time TIMESTAMPTZ NOT NULL, + PRIMARY KEY (id, start_time) +) PARTITION BY RANGE (start_time); + +CREATE INDEX idx_v1_otel_traces_task_lookup + ON v1_otel_traces (tenant_id, task_run_external_id) + WHERE task_run_external_id IS NOT NULL; + +CREATE INDEX idx_v1_otel_traces_trace + ON v1_otel_traces (tenant_id, trace_id, start_time);