From a461d8c98166da375fff20d2c6169e82d66d519d Mon Sep 17 00:00:00 2001 From: Kunal Kushwaha Date: Tue, 27 Jan 2026 23:44:57 +0900 Subject: [PATCH 1/2] audit support added --- README.md | 373 +++++------------- cmd/trace.go | 142 +++++++ internal/audit/collector.go | 204 ++++++++++ internal/audit/mermaid.go | 177 +++++++++ internal/audit/types.go | 68 ++++ internal/tui/trace_viewer.go | 68 ++++ .../templates/mcp-tools/README.md.tmpl | 53 +++ pkg/scaffold/templates/mcp-tools/go.mod.tmpl | 5 + pkg/scaffold/templates/mcp-tools/main.go.tmpl | 115 ++++++ .../templates/workflow/README.md.tmpl | 84 ++++ pkg/scaffold/templates/workflow/go.mod.tmpl | 5 + pkg/scaffold/templates/workflow/main.go.tmpl | 131 ++++++ 12 files changed, 1144 insertions(+), 281 deletions(-) create mode 100644 internal/audit/collector.go create mode 100644 internal/audit/mermaid.go create mode 100644 internal/audit/types.go create mode 100644 pkg/scaffold/templates/mcp-tools/README.md.tmpl create mode 100644 pkg/scaffold/templates/mcp-tools/go.mod.tmpl create mode 100644 pkg/scaffold/templates/mcp-tools/main.go.tmpl create mode 100644 pkg/scaffold/templates/workflow/README.md.tmpl create mode 100644 pkg/scaffold/templates/workflow/go.mod.tmpl create mode 100644 pkg/scaffold/templates/workflow/main.go.tmpl diff --git a/README.md b/README.md index 9f1fa19..7ccfa9d 100644 --- a/README.md +++ b/README.md @@ -1,352 +1,163 @@ -# AGK - AgenticGoKit Command Line Interface +# AGK - AgenticGoKit CLI -A powerful, production-ready CLI for scaffolding, managing, and deploying agentic AI systems built with **AgenticGoKit**. Generate complete project structures in seconds with multiple template options, from simple quickstart projects to enterprise-grade multi-agent workflows. +> **Build production-ready agentic AI systems in seconds.** + +AGK is the official CLI for **AgenticGoKit**, designed to accelerate your development workflow. From scaffolding simple agents to deploying enterprise-grade multi-agent swarms, AGK handles the boilerplate so you can focus on intelligence. ![Version](https://img.shields.io/badge/version-0.1.0-blue) ![License](https://img.shields.io/badge/license-Apache%202.0-green) ![Status](https://img.shields.io/badge/status-Active%20Development-yellow) -## What is AGK? +--- -AGK is a CLI tool that accelerates the development of agentic AI systems by providing: +## πŸš€ Why AGK? -- **πŸ“¦ Project Scaffolding** - Generate complete, production-ready projects with a single command -- **🎯 Multiple Templates** - Choose from 5 progressive templates based on your project complexity -- **⚑ Smart Defaults** - Automatic LLM configuration detection for OpenAI, Anthropic, and Ollama -- **πŸ”§ Developer-Friendly** - Clean, well-structured generated code with streaming enabled by default -- **πŸš€ Framework Integration** - Full integration with AgenticGoKit v0.5.1+ ecosystem +- **⚑ Instant Scaffolding**: Generate complete, compilable projects with one command. +- **🧠 Smart Defaults**: Auto-detects configuration for OpenAI, Anthropic, and Ollama. +- **οΏ½ Trace Auditor**: Built-in observability to debug agent thoughts and prompts. +- **πŸ“¦ Production Ready**: Docker support, idiomatic Go code, and established patterns. +- **🌊 Streaming Native**: All templates support real-time token streaming out of the box. -## Quick Start +--- -### Installation +## 🏁 Quick Start + +### 1. Installation ```bash # Build from source cd agk-new go build -o agk main.go - -# Verify installation -./agk version ``` -### Create Your First Project +### 2. Create Your First Agent ```bash -# List available templates -./agk init --list - -# Generate a quickstart project -./agk init my-agent --template quickstart --llm openai - -# Generate a single-agent with tools and memory -./agk init researcher --template single-agent --llm ollama - -# Non-interactive with all options -./agk init my-project \ - --template single-agent \ - --llm anthropic \ - --description "AI Research Assistant" \ - --force -``` - -## Available Templates +# Initialize a new project with the single-agent template +./agk init my-agent --template single-agent --llm openai -### 1. **Quickstart** ⭐ -Minimal setup - perfect for learning and experimentation. -- Simple hardcoded configuration -- Single agent setup -- Ideal for: Getting started, prototyping -- **Files:** 2 (main.go, go.mod) +# Navigate to the project +cd my-agent -**Use when:** You want the absolute simplest setup to understand how AgenticGoKit works. - -```bash -./agk init my-project --template quickstart --llm openai +# Install dependencies +go mod tidy ``` -### 2. **Single-Agent** ⭐⭐ -Single agent with tools and memory capabilities. -- Full agent configuration with tools support -- Memory management -- Environment-based configuration (.env) -- Streaming enabled by default -- **Files:** 5 (main.go, go.mod, .env, workflow/*, agk.toml) - -**Use when:** Building a focused agent system with specific capabilities (research, analysis, content generation). +### 3. Run It ```bash -./agk init researcher --template single-agent --llm ollama -``` - -### 3. **Multi-Agent** ⭐⭐⭐ -Multiple agents with workflow pipeline. -- Sequential workflow orchestration -- Multiple specialized agents -- Workflow factory pattern -- Advanced configuration -- **Files:** 8 (workflow pipeline with agent coordination) +# Set your API key +export OPENAI_API_KEY=sk-... -**Use when:** You need multiple agents working together in a defined sequence (e.g., planning β†’ execution β†’ review). - -```bash -./agk init workflow --template multi-agent --llm openai -``` - -### 4. **Config-Driven** ⭐⭐⭐⭐ -Enterprise setup with TOML configuration. -- Factory pattern for agents -- TOML-based configuration management -- Shared memory across agents -- Advanced error handling -- **Files:** 12 (fully configured enterprise setup) - -**Use when:** Building scalable systems that need centralized configuration and factory-based agent creation. - -```bash -./agk init enterprise-system --template config-driven --llm anthropic +# Run the agent +go run main.go ``` -### 5. **Advanced** ⭐⭐⭐⭐⭐ -Full-stack with server, frontend, and Docker. -- REST API server -- Web frontend integration -- Docker containerization -- WebSocket support for real-time updates -- Complete DevOps setup -- **Files:** 20+ (complete production system) - -**Use when:** Building production systems with web interfaces, APIs, and containerization. - -```bash -./agk init production-agent --template advanced --llm openai -``` +--- -## Supported LLM Providers +## πŸ“¦ Templates -### OpenAI -```bash -./agk init my-project --template single-agent --llm openai -# Default model: gpt-4-turbo -``` +Choose the right foundation for your project: -### Anthropic -```bash -./agk init my-project --template single-agent --llm anthropic -# Default model: claude-3-sonnet-20240229 -``` +| Template | Complexity | Best For | Description | +|----------|------------|----------|-------------| +| **Quickstart** | ⭐ | Learning | Minimal setup. Single file. Hardcoded config. Perfect for understanding the basics. | +| **Single-Agent** | ⭐⭐ | Prototypes | Adds tools, memory, and environment config. The "standard" starting point. | +| **Multi-Agent** | ⭐⭐⭐ | Workflows | Sequential pipeline of specialized agents (e.g., Researcher β†’ Writer). | +| **Config-Driven** | ⭐⭐⭐⭐ | Enterprise | Factory patterns, TOML config, shared memory. Built for scale. | +| **Advanced** | ⭐⭐⭐⭐⭐ | Production | Full-stack: REST API, WebSocket, Docker, Frontend integration. | -### Ollama (Local) +**Example usage:** ```bash -./agk init my-project --template single-agent --llm ollama -# Default model: llama3.2 -``` - -## Features - -### ✨ Smart Code Generation -- **Streaming by default** - All templates use streaming for real-time response feedback -- **System prompts included** - Customizable AI behavior prompts built-in -- **Error handling** - Production-ready error handling patterns -- **Clean code** - Idiomatic Go with proper error management - -### πŸ› οΈ Developer Experience -- **Clear examples** - Generated code includes helpful comments -- **Configuration templates** - .env and TOML configuration examples -- **Modular structure** - Workflow-based architecture for scalability -- **Version pinned** - AgenticGoKit v0.5.1 pinned for stability - -### πŸ“‹ Project Structure -Each generated project includes: -``` -my-project/ -β”œβ”€β”€ main.go # Entry point with streaming -β”œβ”€β”€ go.mod # Go module configuration -β”œβ”€β”€ .env # Environment variables template -β”œβ”€β”€ agk.toml # Project configuration -└── workflow/ # Workflow logic (if applicable) - β”œβ”€β”€ workflow.go - β”œβ”€β”€ agents.go - └── factory.go +./agk init enterprise-bot --template config-driven --llm anthropic ``` -## Commands +--- -### Initialize a Project -```bash -./agk init [project-name] [flags] -``` +## πŸ” Trace Auditor -**Flags:** -- `--template, -t` - Template type (quickstart, single-agent, multi-agent, config-driven, advanced) -- `--llm` - LLM provider (openai, anthropic, ollama) -- `--description` - Project description -- `--output, -o` - Output directory (default: current directory) -- `--force, -f` - Overwrite existing files -- `--interactive, -i` - Enable interactive prompts (coming soon) +AGK includes a powerful **Trace Auditor** to help you understand exactly what your agents are thinking. -### List Templates -```bash -./agk init --list -``` +### 1. Capture Traces +Control data granularity with `AGK_TRACE_LEVEL`: -Shows all available templates with complexity, features, and usage examples. +| Level | Data Captured | Use Case | +|-------|---------------|----------| +| `minimal` | Timing, status | Production monitoring | +| `standard` | + Tokens, latency | General debugging | +| `detailed` | + Prompts, responses, tool args | **Deep evaluation & auditing** | -### Get Help ```bash -./agk init --help -./agk --help +# Enable detailed tracing to see prompts and thoughts +$env:AGK_TRACE="true" +$env:AGK_TRACE_LEVEL="detailed" +go run main.go ``` -## Development +### 2. Analyze Traces -### Building +**Interactive Viewer (TUI)** +Browse traces, explore spans, and view content details. ```bash -# Build binary -go build -o agk main.go - -# Build with version info -go build -ldflags "-X main.Version=0.1.0" -o agk main.go +agk trace view +# Tip: Press 'd' on a span to see the full Prompt & Response content! ``` -### Testing +**Audit Report (JSON)** +Export structured data for automated evaluation pipelines. ```bash -# Run tests -make test - -# Run with coverage -make test-coverage -``` - -### Project Structure -``` -agk-new/ -β”œβ”€β”€ cmd/ # CLI commands -β”‚ β”œβ”€β”€ init.go # Init command implementation -β”‚ └── root.go # Root CLI setup -β”œβ”€β”€ pkg/ -β”‚ └── scaffold/ # Project scaffolding -β”‚ β”œβ”€β”€ template.go # Template interfaces and types -β”‚ β”œβ”€β”€ template_registry.go # Template generators -β”‚ β”œβ”€β”€ template_loader.go # Embedded template loading -β”‚ └── templates/ # Embedded template files -β”‚ β”œβ”€β”€ quickstart/ -β”‚ β”œβ”€β”€ single-agent/ -β”‚ β”œβ”€β”€ multi-agent/ -β”‚ β”œβ”€β”€ config-driven/ -β”‚ └── advanced/ -β”œβ”€β”€ main.go # Entry point -β”œβ”€β”€ go.mod # Go dependencies -β”œβ”€β”€ go.sum # Dependency checksums -β”œβ”€β”€ LICENSE # Apache 2.0 License -└── README.md # This file +agk trace audit > evaluation_dataset.json ``` -## Examples - -### Create and Run a Quickstart Project +**Visual Flowchart (Mermaid)** +Generate a diagram of the agent's execution path. ```bash -# Create project -./agk init hello-agent --template quickstart --llm openai - -# Navigate to project -cd hello-agent - -# Install dependencies -go mod tidy - -# Run -go run main.go +agk trace mermaid > trace_flow.md ``` -### Create a Research Assistant -```bash -# Create project -./agk init researcher --template single-agent --llm anthropic --description "AI Research Assistant" - -# Edit SystemPrompt in main.go to customize behavior -# Add tools in workflow/agents.go -# Run the project -cd researcher && go run main.go -``` - -## Dependencies - -- **Go 1.21+** - Required for `embed` package -- **AgenticGoKit v0.5.1** - Core framework -- **Cobra** - CLI framework (automatically included) -- **Fatih/Color** - Terminal colors (automatically included) - -## Requirements for Generated Projects - -Each generated project requires: -- **Go 1.21 or higher** -- **API Keys** (depending on LLM provider): - - OpenAI: `OPENAI_API_KEY` environment variable - - Anthropic: `ANTHROPIC_API_KEY` environment variable - - Ollama: Local instance running on port 11434 (no API key needed) +--- -## Configuration +## πŸ› οΈ Commands -### Generated Project Config -Each project includes `agk.toml` for configuration: -```toml -[project] -name = "my-agent" -description = "AI Assistant" +| Command | Description | +|---------|-------------| +| `init` | Create a new project from a template. | +| `init --list` | Show details of all available templates. | +| `trace list` | List all captured trace runs. | +| `trace show` | Display summary of a specific run. | +| `trace view` | Open the interactive TUI trace explorer. | +| `trace audit` | Analyze a trace for reasoning quality. | +| `trace export` | Export trace data (OTEL, Jaeger, JSON). | -[llm] -provider = "openai" -model = "gpt-4-turbo" -temperature = 0.7 -max_tokens = 2000 -``` - -### Environment Variables -Generated projects use `.env` for sensitive data: -```env -OPENAI_API_KEY=sk-... -ANTHROPIC_API_KEY=sk-ant-... -``` +--- -## Status & Roadmap +## πŸ—ΊοΈ Roadmap ### βœ… Completed -- Template infrastructure and routing -- Embedded template system -- 2 production templates (Quickstart, Single-Agent) -- Multi-provider LLM detection -- Streaming-enabled generation -- System prompt support -- Template listing command +- Template system (Quickstart, Single-Agent) +- Smart LLM Provider detection +- Streaming support +- **Trace Auditor** (Audit & Mermaid commands) +- **Interactive Trace Viewer** (with content inspection) ### 🚧 In Progress -- Template refinement and testing -- Multi-Agent template implementation -- Config-Driven template implementation -- Advanced template implementation +- Multi-Agent & Enterprise templates +- Advanced full-stack template ### πŸ“… Planned -- Interactive project creation (`--interactive` flag) -- Project upgrade commands -- Workflow testing utilities -- Trace visualization tools -- MCP server management -- Template customization options - -## License +- Interactive init wizard (`agk init -i`) +- MCP Server management +- Project upgrade tools -Apache License 2.0 - See [LICENSE](./LICENSE) file for details +--- -## Contributing +## 🀝 Contributing -Contributions are welcome! Please see [CONTRIBUTING.md](./CONTRIBUTING.md) for guidelines. +We love contributions! Please read [CONTRIBUTING.md](./CONTRIBUTING.md) for guidelines. -## Support +## οΏ½ License -- πŸ“– [AgenticGoKit Documentation](https://github.com/agenticgokit/agenticgokit) -- πŸ’¬ [GitHub Discussions](https://github.com/agenticgokit/agenticgokit/discussions) -- πŸ› [Report Issues](https://github.com/agenticgokit/agenticgokit/issues) +Apache 2.0 - See [LICENSE](./LICENSE). --- - **Built with ❀️ for the AgenticGoKit community** diff --git a/cmd/trace.go b/cmd/trace.go index 2749bea..5e3cddd 100644 --- a/cmd/trace.go +++ b/cmd/trace.go @@ -12,6 +12,7 @@ import ( "strings" "time" + "github.com/agenticgokit/agk/internal/audit" "github.com/agenticgokit/agk/internal/tui" tea "github.com/charmbracelet/bubbletea" "github.com/spf13/cobra" @@ -94,12 +95,57 @@ var exportCmd = &cobra.Command{ }, } +// auditCmd analyzes trace for reasoning patterns +var auditCmd = &cobra.Command{ + Use: "audit [run-id]", + Short: "Analyze trace for reasoning patterns", + Long: `Analyze a trace to extract reasoning events for evaluation. + +Outputs a TraceObject with events categorized as: + - thought: Internal reasoning/decisions + - tool_call: Tool invocations with arguments + - observation: Tool outputs/results + - llm_call: LLM API calls + +Use AGK_TRACE_LEVEL=detailed when running your agent to capture +full content (prompts, responses, tool args/outputs).`, + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + runID := "" + if len(args) > 0 { + runID = args[0] + } + return auditTrace(runID) + }, +} + +// mermaidCmd generates Mermaid diagram from trace +var mermaidCmd = &cobra.Command{ + Use: "mermaid [run-id]", + Short: "Generate Mermaid diagram from trace", + Long: `Generate a Mermaid flowchart visualizing the agent's execution path. + +The diagram shows the sequence of thoughts, tool calls, and decisions +made by the agent. Output is Markdown with embedded Mermaid code.`, + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + runID := "" + if len(args) > 0 { + runID = args[0] + } + output, _ := cmd.Flags().GetString("output") + return generateMermaid(runID, output) + }, +} + func init() { rootCmd.AddCommand(traceCmd) traceCmd.AddCommand(listCmd) traceCmd.AddCommand(showCmd) traceCmd.AddCommand(viewCmd) traceCmd.AddCommand(exportCmd) + traceCmd.AddCommand(auditCmd) + traceCmd.AddCommand(mermaidCmd) // Export flags exportCmd.Flags().String("format", "json", "Export format: json, jaeger, otel") @@ -732,3 +778,99 @@ type Span struct { ChildSpanCount int `json:"ChildSpanCount"` InstrumentationScope map[string]interface{} `json:"InstrumentationScope"` } + +// auditTrace analyzes a trace and outputs a TraceObject for evaluation +func auditTrace(runID string) error { + runsDir := runsDirName + + // If no run ID provided, use latest + if runID == "" { + runID = getLatestRunID(runsDir) + if runID == "" { + fmt.Println("No traces found. Run with AGK_TRACE=true to generate traces.") + return nil + } + } + + runPath := filepath.Join(runsDir, runID) + + // Check if run exists + if _, err := os.Stat(runPath); os.IsNotExist(err) { + return fmt.Errorf("trace not found: %s", runID) + } + + // Use the audit package to collect events + collector, err := audit.NewCollector(runPath) + if err != nil { + return fmt.Errorf("failed to create collector: %w", err) + } + + traceObj, err := collector.Collect() + if err != nil { + return fmt.Errorf("failed to collect trace: %w", err) + } + + // Output as JSON + output, err := json.MarshalIndent(traceObj, "", " ") + if err != nil { + return fmt.Errorf("failed to marshal trace object: %w", err) + } + + fmt.Println(string(output)) + return nil +} + +// generateMermaid creates a Mermaid flowchart from trace data +func generateMermaid(runID, output string) error { + runsDir := runsDirName + + // If no run ID provided, use latest + if runID == "" { + runID = getLatestRunID(runsDir) + if runID == "" { + fmt.Println("No traces found. Run with AGK_TRACE=true to generate traces.") + return nil + } + } + + runPath := filepath.Join(runsDir, runID) + + // Check if run exists + if _, err := os.Stat(runPath); os.IsNotExist(err) { + return fmt.Errorf("trace not found: %s", runID) + } + + // Use the audit package to collect events + collector, err := audit.NewCollector(runPath) + if err != nil { + return fmt.Errorf("failed to create collector: %w", err) + } + + traceObj, err := collector.Collect() + if err != nil { + return fmt.Errorf("failed to collect trace: %w", err) + } + + // Generate Mermaid diagram + mermaid := audit.GenerateMermaidWithHierarchy(traceObj) + + // Build output content + var content strings.Builder + content.WriteString(fmt.Sprintf("# Agent Trace: %s\n\n", runID)) + content.WriteString(fmt.Sprintf("**Events:** %d | **Duration:** %dms\n\n", + traceObj.Summary.TotalEvents, traceObj.Summary.TotalDurationMs)) + content.WriteString("## Execution Flow\n\n") + content.WriteString(mermaid) + + // Write to file or stdout + if output != "" { + if err := os.WriteFile(output, []byte(content.String()), 0600); err != nil { + return fmt.Errorf("failed to write file: %w", err) + } + fmt.Printf("βœ… Generated Mermaid diagram: %s\n", output) + } else { + fmt.Println(content.String()) + } + + return nil +} diff --git a/internal/audit/collector.go b/internal/audit/collector.go new file mode 100644 index 0000000..d0094a4 --- /dev/null +++ b/internal/audit/collector.go @@ -0,0 +1,204 @@ +package audit + +import ( + "encoding/json" + "os" + "path/filepath" + "sort" + "strings" + "time" +) + +// Collector extracts trace events from stored span data +type Collector struct { + runPath string + spans []RawSpan +} + +// RawSpan represents a parsed span from trace.jsonl +type RawSpan struct { + Name string `json:"Name"` + SpanContext SpanContext `json:"SpanContext"` + Parent SpanContext `json:"Parent"` + StartTime string `json:"StartTime"` + EndTime string `json:"EndTime"` + Attributes []map[string]interface{} `json:"Attributes"` + Status SpanStatus `json:"Status"` +} + +// SpanContext contains span identification +type SpanContext struct { + TraceID string `json:"TraceID"` + SpanID string `json:"SpanID"` +} + +// SpanStatus contains span status +type SpanStatus struct { + Code string `json:"Code"` + Description string `json:"Description,omitempty"` +} + +// NewCollector creates a collector from a run path +func NewCollector(runPath string) (*Collector, error) { + tracePath := filepath.Join(runPath, "trace.jsonl") + data, err := os.ReadFile(tracePath) + if err != nil { + return nil, err + } + + var spans []RawSpan + lines := strings.Split(string(data), "\n") + for _, line := range lines { + if line == "" { + continue + } + var span RawSpan + if err := json.Unmarshal([]byte(line), &span); err != nil { + continue + } + spans = append(spans, span) + } + + return &Collector{ + runPath: runPath, + spans: spans, + }, nil +} + +// Collect extracts TraceObject from the spans +func (c *Collector) Collect() (*TraceObject, error) { + runID := filepath.Base(c.runPath) + + obj := &TraceObject{ + RunID: runID, + Events: make([]TraceEvent, 0), + Summary: TraceSummary{ + HasDetailedData: false, + }, + } + + for _, span := range c.spans { + event := c.spanToEvent(span) + obj.Events = append(obj.Events, event) + + // Update summary counts + switch event.Type { + case EventTypeThought: + obj.Summary.ThoughtCount++ + case EventTypeToolCall: + obj.Summary.ToolCallCount++ + case EventTypeLLMCall: + obj.Summary.LLMCallCount++ + } + + // Check for detailed data + if event.Content != "" { + obj.Summary.HasDetailedData = true + } + + // Update timing + if obj.StartTime.IsZero() || event.Timestamp.Before(obj.StartTime) { + obj.StartTime = event.Timestamp + } + if event.Timestamp.After(obj.EndTime) { + obj.EndTime = event.Timestamp + } + } + + obj.Summary.TotalEvents = len(obj.Events) + obj.Summary.TotalDurationMs = obj.EndTime.Sub(obj.StartTime).Milliseconds() + + // Sort events by timestamp + sort.Slice(obj.Events, func(i, j int) bool { + return obj.Events[i].Timestamp.Before(obj.Events[j].Timestamp) + }) + + return obj, nil +} + +// spanToEvent converts a raw span to a TraceEvent +func (c *Collector) spanToEvent(span RawSpan) TraceEvent { + event := TraceEvent{ + SpanID: span.SpanContext.SpanID, + SpanName: span.Name, + ParentID: span.Parent.SpanID, + Metadata: make(map[string]any), + } + + // Parse timestamp + if t, err := time.Parse(time.RFC3339, span.StartTime); err == nil { + event.Timestamp = t + } + + // Calculate duration + if start, err := time.Parse(time.RFC3339, span.StartTime); err == nil { + if end, err := time.Parse(time.RFC3339, span.EndTime); err == nil { + event.DurationMs = end.Sub(start).Milliseconds() + } + } + + // Determine event type and extract attributes + event.Type = c.classifySpan(span.Name) + + // Extract attributes + for _, attr := range span.Attributes { + key, ok := attr["Key"].(string) + if !ok { + continue + } + value, ok := attr["Value"].(map[string]interface{}) + if !ok { + continue + } + val, ok := value["Value"] + if !ok { + continue + } + + // Store in metadata + event.Metadata[key] = val + + // Check for content fields (detailed trace level) + switch key { + case "agk.prompt.user", "agk.llm.response": + event.Content = val.(string) + case "agk.tool.arguments": + if event.Type == EventTypeToolCall { + event.Content = val.(string) + } + case "agk.tool.result": + if event.Type == EventTypeObservation { + event.Content = val.(string) + } + } + } + + return event +} + +// classifySpan determines the event type based on span name +func (c *Collector) classifySpan(name string) EventType { + nameLower := strings.ToLower(name) + + switch { + case strings.Contains(nameLower, "tool"): + return EventTypeToolCall + case strings.Contains(nameLower, "llm"): + return EventTypeLLMCall + case strings.Contains(nameLower, "agent"): + return EventTypeThought + case strings.Contains(nameLower, "workflow"): + return EventTypeDecision + default: + return EventTypeThought + } +} + +// GetReasoningPath extracts the sequence of event types +func (c *Collector) GetReasoningPath(obj *TraceObject) []EventType { + path := make([]EventType, len(obj.Events)) + for i, event := range obj.Events { + path[i] = event.Type + } + return path +} diff --git a/internal/audit/mermaid.go b/internal/audit/mermaid.go new file mode 100644 index 0000000..5659a5d --- /dev/null +++ b/internal/audit/mermaid.go @@ -0,0 +1,177 @@ +package audit + +import ( + "fmt" + "strings" +) + +// GenerateMermaid creates a Mermaid flowchart from a TraceObject +func GenerateMermaid(obj *TraceObject) string { + var b strings.Builder + + b.WriteString("```mermaid\n") + b.WriteString("flowchart TD\n") + + // Create nodes for each event + for i, event := range obj.Events { + nodeID := fmt.Sprintf("N%d", i) + label := formatNodeLabel(event) + shape := getNodeShape(event.Type) + + b.WriteString(fmt.Sprintf(" %s%s\n", nodeID, shape(label))) + } + + b.WriteString("\n") + + // Create edges between consecutive events + for i := 0; i < len(obj.Events)-1; i++ { + b.WriteString(fmt.Sprintf(" N%d --> N%d\n", i, i+1)) + } + + // Add styling + b.WriteString("\n") + b.WriteString(" %% Styling\n") + for i, event := range obj.Events { + style := getNodeStyle(event.Type) + if style != "" { + b.WriteString(fmt.Sprintf(" style N%d %s\n", i, style)) + } + } + + b.WriteString("```\n") + + return b.String() +} + +// GenerateMermaidWithHierarchy creates a Mermaid diagram respecting parent-child relationships +func GenerateMermaidWithHierarchy(obj *TraceObject) string { + var b strings.Builder + + // Build parent map + parentMap := make(map[string][]int) + spanIDToIndex := make(map[string]int) + + for i, event := range obj.Events { + spanIDToIndex[event.SpanID] = i + if event.ParentID != "" && event.ParentID != "0000000000000000" { + parentMap[event.ParentID] = append(parentMap[event.ParentID], i) + } + } + + b.WriteString("```mermaid\n") + b.WriteString("flowchart TD\n") + + // Create nodes + for i, event := range obj.Events { + nodeID := fmt.Sprintf("N%d", i) + label := formatNodeLabel(event) + shape := getNodeShape(event.Type) + b.WriteString(fmt.Sprintf(" %s%s\n", nodeID, shape(label))) + } + + b.WriteString("\n") + + // Create edges based on parent-child relationships + for parentSpanID, children := range parentMap { + if parentIdx, ok := spanIDToIndex[parentSpanID]; ok { + for _, childIdx := range children { + b.WriteString(fmt.Sprintf(" N%d --> N%d\n", parentIdx, childIdx)) + } + } + } + + // For orphan nodes (no parent in trace), connect sequentially + hasParent := make(map[int]bool) + for _, children := range parentMap { + for _, idx := range children { + hasParent[idx] = true + } + } + + // Add styling + b.WriteString("\n") + for i, event := range obj.Events { + style := getNodeStyle(event.Type) + if style != "" { + b.WriteString(fmt.Sprintf(" style N%d %s\n", i, style)) + } + } + + b.WriteString("```\n") + + return b.String() +} + +// formatNodeLabel creates a concise label for the node +func formatNodeLabel(event TraceEvent) string { + // Start with event type icon + icon := getEventIcon(event.Type) + + // Get a short description + desc := event.SpanName + if len(desc) > 25 { + desc = desc[:22] + "..." + } + + // Add duration if significant + duration := "" + if event.DurationMs > 100 { + duration = fmt.Sprintf(" (%dms)", event.DurationMs) + } + + return fmt.Sprintf("%s %s%s", icon, desc, duration) +} + +// getEventIcon returns an emoji for the event type +func getEventIcon(eventType EventType) string { + switch eventType { + case EventTypeThought: + return "πŸ’­" + case EventTypeToolCall: + return "πŸ”§" + case EventTypeObservation: + return "πŸ‘" + case EventTypeLLMCall: + return "πŸ€–" + case EventTypeDecision: + return "⚑" + default: + return "β—‹" + } +} + +// getNodeShape returns a function that wraps the label in the appropriate shape +func getNodeShape(eventType EventType) func(string) string { + switch eventType { + case EventTypeThought: + return func(label string) string { return fmt.Sprintf("([%s])", label) } // Stadium + case EventTypeToolCall: + return func(label string) string { return fmt.Sprintf("[[%s]]", label) } // Subroutine + case EventTypeObservation: + return func(label string) string { return fmt.Sprintf("[/%s/]", label) } // Parallelogram + case EventTypeLLMCall: + return func(label string) string { return fmt.Sprintf("{%s}", label) } // Rhombus + case EventTypeDecision: + return func(label string) string { return fmt.Sprintf("{{%s}}", label) } // Hexagon + default: + return func(label string) string { return fmt.Sprintf("[%s]", label) } + } +} + +// getNodeStyle returns Mermaid styling for the event type +func getNodeStyle(eventType EventType) string { + switch eventType { + case EventTypeThought: + return "fill:#e1f5fe,stroke:#01579b" + case EventTypeToolCall: + return "fill:#e8f5e9,stroke:#1b5e20" + case EventTypeObservation: + return "fill:#fff3e0,stroke:#e65100" + case EventTypeLLMCall: + return "fill:#f3e5f5,stroke:#4a148c" + case EventTypeDecision: + return "fill:#fce4ec,stroke:#880e4f" + default: + return "" + } +} diff --git a/internal/audit/types.go b/internal/audit/types.go new file mode 100644 index 0000000..bcc41e4 --- /dev/null +++ b/internal/audit/types.go @@ -0,0 +1,68 @@ +package audit + +import ( + "time" +) + +// EventType categorizes trace events for evaluation +type EventType string + +const ( + // EventTypeThought represents internal reasoning/decision + EventTypeThought EventType = "thought" + // EventTypeToolCall represents a tool invocation + EventTypeToolCall EventType = "tool_call" + // EventTypeObservation represents tool output/result + EventTypeObservation EventType = "observation" + // EventTypeLLMCall represents an LLM API call + EventTypeLLMCall EventType = "llm_call" + // EventTypeDecision represents a decision point + EventTypeDecision EventType = "decision" +) + +// TraceEvent represents a single step in agent execution +type TraceEvent struct { + Timestamp time.Time `json:"timestamp"` + Type EventType `json:"type"` + SpanID string `json:"span_id"` + SpanName string `json:"span_name"` + Content string `json:"content,omitempty"` // Main content (prompt, response, etc) + Metadata map[string]any `json:"metadata,omitempty"` // Additional context + DurationMs int64 `json:"duration_ms,omitempty"` // Duration in milliseconds + ParentID string `json:"parent_id,omitempty"` // Parent span for hierarchy +} + +// TraceObject is the complete trace for evaluation +type TraceObject struct { + RunID string `json:"run_id"` + Command string `json:"command,omitempty"` + StartTime time.Time `json:"start_time"` + EndTime time.Time `json:"end_time"` + Events []TraceEvent `json:"events"` + FinalOutput string `json:"final_output,omitempty"` + Summary TraceSummary `json:"summary"` +} + +// TraceSummary provides aggregate metrics for the trace +type TraceSummary struct { + TotalEvents int `json:"total_events"` + ThoughtCount int `json:"thought_count"` + ToolCallCount int `json:"tool_call_count"` + LLMCallCount int `json:"llm_call_count"` + TotalDurationMs int64 `json:"total_duration_ms"` + TokensUsed int `json:"tokens_used,omitempty"` + EstimatedCost float64 `json:"estimated_cost,omitempty"` + HasDetailedData bool `json:"has_detailed_data"` // True if content captured +} + +// ReasoningAnalysis provides evaluation-focused analysis of the trace +type ReasoningAnalysis struct { + // Path shows the sequence of event types taken + Path []EventType `json:"path"` + // DecisionPoints where agent made choices + DecisionPoints []TraceEvent `json:"decision_points,omitempty"` + // ToolUsageCorrect indicates if tool calls were appropriate + ToolUsageCorrect *bool `json:"tool_usage_correct,omitempty"` + // ReasoningQuality is a 0-1 score for reasoning quality (set by judge) + ReasoningQuality *float64 `json:"reasoning_quality,omitempty"` +} diff --git a/internal/tui/trace_viewer.go b/internal/tui/trace_viewer.go index 3b18c93..ebd76bf 100644 --- a/internal/tui/trace_viewer.go +++ b/internal/tui/trace_viewer.go @@ -736,6 +736,9 @@ func (m Model) renderDetailContent(node *SpanNode) string { // --- Hero Section (Overview) --- b.WriteString(m.renderOverviewSection(node)) + // --- Content Section (Audit Data - prompts, responses) --- + b.WriteString(m.renderContentSection(node)) + // --- Attributes Grouping --- b.WriteString(m.renderAttributeSection(node)) @@ -774,6 +777,71 @@ func (m Model) renderOverviewSection(node *SpanNode) string { return b.String() } +// renderContentSection displays audit content (prompts, responses, tool args) +// Only shown when detailed trace data is available (AGK_TRACE_LEVEL=detailed) +func (m Model) renderContentSection(node *SpanNode) string { + var b strings.Builder + attrs := node.Span.GetAllAttributes() + + // Content keys to look for + contentKeys := []struct { + Key string + Icon string + Label string + }{ + {"agk.prompt.user", "πŸ“", "User Prompt"}, + {"agk.prompt.system", "πŸ–₯️", "System Prompt"}, + {"agk.llm.response", "πŸ€–", "LLM Response"}, + {"agk.tool.arguments", "πŸ“₯", "Tool Arguments"}, + {"agk.tool.result", "πŸ“€", "Tool Result"}, + } + + // Check if any content is available + hasContent := false + for _, ck := range contentKeys { + if _, ok := attrs[ck.Key]; ok { + hasContent = true + break + } + } + + if !hasContent { + return "" + } + + b.WriteString("\n") + b.WriteString(SectionHeaderStyle.Render("Content (Detailed Trace)")) + b.WriteString("\n") + + for _, ck := range contentKeys { + if val, ok := attrs[ck.Key]; ok { + content := fmt.Sprintf("%v", val) + + // Header with icon + b.WriteString(fmt.Sprintf("\n%s ", ck.Icon)) + b.WriteString(AttributeKeyStyle.Render(ck.Label)) + b.WriteString("\n") + b.WriteString(MutedStyle.Render(strings.Repeat("─", 40))) + b.WriteString("\n") + + // Content (truncate if too long) + maxLen := 500 + if len(content) > maxLen { + content = content[:maxLen-3] + "..." + b.WriteString(content) + b.WriteString("\n") + b.WriteString(MutedStyle.Render("[truncated]")) + } else { + b.WriteString(content) + } + b.WriteString("\n") + } + } + + b.WriteString("\n") + return b.String() +} + func (m Model) renderAttributeSection(node *SpanNode) string { var b strings.Builder attrs := node.Span.GetAllAttributes() diff --git a/pkg/scaffold/templates/mcp-tools/README.md.tmpl b/pkg/scaffold/templates/mcp-tools/README.md.tmpl new file mode 100644 index 0000000..1759478 --- /dev/null +++ b/pkg/scaffold/templates/mcp-tools/README.md.tmpl @@ -0,0 +1,53 @@ +# {{.ProjectName}} - MCP Tools Agent + +An AgenticGoKit agent with MCP (Model Context Protocol) tool integration. + +## Features + +- **MCP Tool Support**: Access external tools via MCP servers +- **Streaming**: Real-time response streaming +- **Observability**: Built-in tracing with `AGK_TRACE=true` + +## Quick Start + +```bash +# Install dependencies +go mod tidy + +# Set your API key +export OPENAI_API_KEY=your-key-here +# OR for Ollama (no API key needed) +# export LLM_PROVIDER=ollama + +# Run with tracing +AGK_TRACE=true go run main.go +``` + +## Available MCP Servers + +1. **filesystem**: Read/write files and directories +2. **fetch**: Make HTTP requests +3. **brave-search**: Web search (requires BRAVE_API_KEY) +4. **memory**: Persistent key-value storage + +## Adding More Tools + +Edit `main.go` and add more MCP servers to the `Servers` slice: + +```go +{ + Name: "your-server", + Command: "npx", + Args: []string{"-y", "@your-org/mcp-server"}, +} +``` + +## Viewing Traces + +```bash +# Install agk CLI +go install github.com/agenticgokit/agk@latest + +# View traces +agk trace +``` diff --git a/pkg/scaffold/templates/mcp-tools/go.mod.tmpl b/pkg/scaffold/templates/mcp-tools/go.mod.tmpl new file mode 100644 index 0000000..8efb679 --- /dev/null +++ b/pkg/scaffold/templates/mcp-tools/go.mod.tmpl @@ -0,0 +1,5 @@ +module {{.ProjectName}} + +go 1.24.1 + +require github.com/agenticgokit/agenticgokit v0.5.2 diff --git a/pkg/scaffold/templates/mcp-tools/main.go.tmpl b/pkg/scaffold/templates/mcp-tools/main.go.tmpl new file mode 100644 index 0000000..2a5cb5d --- /dev/null +++ b/pkg/scaffold/templates/mcp-tools/main.go.tmpl @@ -0,0 +1,115 @@ +package main + +import ( + "context" + "fmt" + "log" + "os" + "time" + + // Import MCP plugins to enable MCP functionality + _ "github.com/agenticgokit/agenticgokit/plugins/mcp/default" + _ "github.com/agenticgokit/agenticgokit/plugins/mcp/unified" + + agk "github.com/agenticgokit/agenticgokit/v1beta" +) + +func main() { + ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second) + defer cancel() + + // Get service version from environment or use default + serviceVersion := os.Getenv("SERVICE_VERSION") + if serviceVersion == "" { + serviceVersion = "0.1.0" + } + + fmt.Println("Initializing Agent with MCP Tools...") + + // Create agent with MCP tools and observability + // Tracing is enabled if AGK_TRACE=true environment variable is set + agent, err := agk.NewBuilder("{{.ProjectName}}"). + + WithObservability("{{.ProjectName}}", serviceVersion). + WithConfig(&agk.Config{ + Name: "{{.ProjectName}}", + SystemPrompt: "You are a helpful assistant with access to tools. Use the available MCP tools to help the user accomplish their tasks. Always explain what tools you're using and why.", + Timeout: 120 * time.Second, + LLM: agk.LLMConfig{ + Provider: "{{.LLMProvider}}", + Model: "{{.LLMModel}}", + Temperature: 0.7, + MaxTokens: 4000, + }, + Tools: &agk.ToolsConfig{ + Enabled: true, + // Configure MCP servers for tool access + // MCP (Model Context Protocol) enables agents to use external tools + MCP: &agk.MCPConfig{ + Enabled: true, + Discovery: false, + ConnectionTimeout: 10 * time.Second, + Servers: []agk.MCPServer{ + { + Name: "filesystem", + Type: "stdio", + // Note: The current stdio implementation does not support arguments. + // You must point to a specific executable or a wrapper script. + // Example: "./mcp-server-filesystem.exe" or "./run-filesystem.bat" + Command: "npx", // Placeholder - likely needs a wrapper script + Enabled: true, + }, + // Example of HTTP SSE configuration (alternative to stdio) + // { + // Name: "weather-server", + // Type: "http_sse", + // Address: "http://localhost:8080", + // Enabled: true, + // }, + }, + }, + }, + }). + Build() + if err != nil { + log.Fatalf("Failed to create agent: %v", err) + } + defer agent.Cleanup(ctx) + + // Wait for MCP servers to initialize + fmt.Println("Initializing MCP tools...") + time.Sleep(2 * time.Second) + + // Example: Use tools to help with a task + userMessage := "List the files in the current directory and tell me what this project is about." + + fmt.Printf("\nUser: %s\n\n", userMessage) + fmt.Println("Assistant:") + + // Use streaming for real-time response + stream, err := agent.RunStream(ctx, userMessage) + if err != nil { + log.Fatalf("Failed to start streaming: %v", err) + } + + printStreamingResponse(stream) +} + +// printStreamingResponse prints the streaming response as tokens arrive +func printStreamingResponse(stream agk.Stream) { + for chunk := range stream.Chunks() { + if chunk.Error != nil { + fmt.Printf("\nError: %v\n", chunk.Error) + break + } + + switch chunk.Type { + case agk.ChunkTypeDelta: + fmt.Print(chunk.Delta) + case agk.ChunkTypeToolCall: + fmt.Printf("\n[Tool Call: %s]\n", chunk.ToolName) + case agk.ChunkTypeDone: + fmt.Println("\n\nCompleted") + } + } +} diff --git a/pkg/scaffold/templates/workflow/README.md.tmpl b/pkg/scaffold/templates/workflow/README.md.tmpl new file mode 100644 index 0000000..93d55b3 --- /dev/null +++ b/pkg/scaffold/templates/workflow/README.md.tmpl @@ -0,0 +1,84 @@ +# {{.ProjectName}} - Streaming Workflow + +An AgenticGoKit streaming workflow with multiple sequential steps. + +## Features + +- **Multi-Step Workflow**: Chain multiple agents in sequence +- **Real-Time Streaming**: See output as each step processes +- **Step Tracking**: Monitor progress through workflow stages +- **Observability**: Built-in tracing with `AGK_TRACE=true` + +## Workflow Pipeline + +``` +[Input] β†’ [Research] β†’ [Summarize] β†’ [Format] β†’ [Output] +``` + +1. **Research**: Gathers detailed information on the topic +2. **Summarize**: Condenses research into key points +3. **Format**: Creates a professional formatted report + +## Quick Start + +```bash +# Install dependencies +go mod tidy + +# Set your API key +export OPENAI_API_KEY=your-key-here +# OR for Ollama (no API key needed) +# Just make sure Ollama is running + +# Run with tracing +AGK_TRACE=true go run main.go +``` + +## Customizing the Workflow + +### Add a new step + +```go +.AddStep("analyze", agk.NewBuilder("analyzer"). + WithLLM("{{.LLMProvider}}", "{{.LLMModel}}"). + WithConfig(&agk.Config{ + SystemPrompt: "Your analysis prompt here...", + }).MustBuild()) +``` + +### Change step order + +Reorder the `.AddStep()` calls in the workflow builder chain. + +### Parallel execution + +Use `.AddParallelSteps()` for concurrent processing: + +```go +.AddParallelSteps( + agk.Step{Name: "step1", Agent: agent1}, + agk.Step{Name: "step2", Agent: agent2}, +) +``` + +## Viewing Traces + +```bash +# Install agk CLI +go install github.com/agenticgokit/agk@latest + +# View traces interactively +agk trace + +# View specific run +agk trace show +``` + +## Environment Variables + +| Variable | Description | Default | +|----------|-------------|---------| +| `AGK_TRACE` | Enable tracing | `false` | +| `OPENAI_API_KEY` | OpenAI API key | - | +| `ANTHROPIC_API_KEY` | Anthropic API key | - | +| `SERVICE_VERSION` | Service version for traces | `0.1.0` | diff --git a/pkg/scaffold/templates/workflow/go.mod.tmpl b/pkg/scaffold/templates/workflow/go.mod.tmpl new file mode 100644 index 0000000..8efb679 --- /dev/null +++ b/pkg/scaffold/templates/workflow/go.mod.tmpl @@ -0,0 +1,5 @@ +module {{.ProjectName}} + +go 1.24.1 + +require github.com/agenticgokit/agenticgokit v0.5.2 diff --git a/pkg/scaffold/templates/workflow/main.go.tmpl b/pkg/scaffold/templates/workflow/main.go.tmpl new file mode 100644 index 0000000..68bf114 --- /dev/null +++ b/pkg/scaffold/templates/workflow/main.go.tmpl @@ -0,0 +1,131 @@ +package main + +import ( + "context" + "fmt" + "log" + "os" + "time" + + agk "github.com/agenticgokit/agenticgokit/v1beta" +) + +func main() { + ctx, cancel := context.WithTimeout(context.Background(), 180*time.Second) + defer cancel() + + // Get service version from environment or use default + serviceVersion := os.Getenv("SERVICE_VERSION") + if serviceVersion == "" { + serviceVersion = "0.1.0" + } + + // Create a streaming workflow with multiple steps + // Each step is an agent that processes and transforms the input + // Steps run sequentially, with each step's output feeding into the next + + fmt.Println("Creating Streaming Workflow...") + fmt.Println("================================") + + // Create workflow using the builder pattern + workflow, err := agk.NewWorkflow("{{.ProjectName}}"). + WithObservability("{{.ProjectName}}", serviceVersion). + // Step 1: Research - gather information + AddStep("research", agk.NewBuilder("researcher"). + + WithConfig(&agk.Config{ + SystemPrompt: "You are a research assistant. When given a topic, provide detailed, factual information about it. Include key concepts, important details, and relevant context.", + LLM: agk.LLMConfig{ + Provider: "{{.LLMProvider}}", + Model: "{{.LLMModel}}", + Temperature: 0.7, + MaxTokens: 2000, + }, + }).MustBuild()). + // Step 2: Summarize - condense the research + AddStep("summarize", agk.NewBuilder("summarizer"). + + WithConfig(&agk.Config{ + SystemPrompt: "You are a summarization expert. Take the provided content and create a clear, concise summary that captures the key points. Use bullet points for clarity.", + LLM: agk.LLMConfig{ + Provider: "{{.LLMProvider}}", + Model: "{{.LLMModel}}", + Temperature: 0.5, + MaxTokens: 1000, + }, + }).MustBuild()). + // Step 3: Format - create final output + AddStep("format", agk.NewBuilder("formatter"). + + WithConfig(&agk.Config{ + SystemPrompt: "You are a content formatter. Take the provided summary and format it as a professional report with sections, headers, and clear structure.", + LLM: agk.LLMConfig{ + Provider: "{{.LLMProvider}}", + Model: "{{.LLMModel}}", + Temperature: 0.3, + MaxTokens: 1500, + }, + }).MustBuild()). + Build() + + if err != nil { + log.Fatalf("Failed to create workflow: %v", err) + } + defer workflow.Cleanup(ctx) + + // Run the workflow with streaming + topic := "artificial intelligence in healthcare" + fmt.Printf("\nTopic: %s\n\n", topic) + + // Stream the workflow execution + stream, err := workflow.RunStream(ctx, topic) + if err != nil { + log.Fatalf("Failed to start workflow: %v", err) + } + + // Track progress through steps + currentStep := "" + totalChunks := 0 + stepOutputs := make(map[string]int) + + for chunk := range stream.Chunks() { + if chunk.Error != nil { + fmt.Printf("\nError: %v\n", chunk.Error) + break + } + + switch chunk.Type { + case agk.ChunkTypeStepStart: + currentStep = chunk.StepName + fmt.Printf("\n[Step: %s]\n", currentStep) + fmt.Println(string(make([]byte, 40))) + + case agk.ChunkTypeDelta: + fmt.Print(chunk.Delta) + totalChunks++ + stepOutputs[currentStep]++ + + case agk.ChunkTypeStepComplete: + fmt.Printf("\n\nStep '%s' completed\n", chunk.StepName) + + case agk.ChunkTypeDone: + fmt.Println("\n") + fmt.Println("================================") + fmt.Println("WORKFLOW COMPLETED") + fmt.Println("================================") + fmt.Printf("Total Chunks: %d\n", totalChunks) + fmt.Println("\nChunks per Step:") + for step, count := range stepOutputs { + fmt.Printf(" - %s: %d\n", step, count) + } + } + } + + // Get final result + result := stream.Result() + if result.Error != nil { + log.Fatalf("Workflow failed: %v", result.Error) + } + + fmt.Printf("\nFinal Output Length: %d characters\n", len(result.Output)) +} From 6dbe1995345ddbae8fc9691e66a0b8d2567f9673 Mon Sep 17 00:00:00 2001 From: Kunal Kushwaha Date: Wed, 28 Jan 2026 00:38:59 +0900 Subject: [PATCH 2/2] Add Audit Support + Hot Reload Trace Viewer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Audit APIs - TraceObject and Event types - Support for thought, tool_call, observation, llm_call events - Enables evaluation frameworks to analyze agent reasoning **Hot Reload Trace Viewer** (new) - `agk trace show` now watches trace file for updates - New spans appear automatically during workflow execution - Header shows πŸ”΄ LIVE indicator when watching --- CONTRIBUTING.md | 141 +++++++++++++++++++++++++++++++++++ cmd/trace.go | 18 ++--- internal/audit/collector.go | 5 ++ internal/tui/trace_viewer.go | 130 +++++++++++++++++++++++++++++++- 4 files changed, 284 insertions(+), 10 deletions(-) create mode 100644 CONTRIBUTING.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..a440918 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,141 @@ +# Contributing to AGK + +Thank you for your interest in contributing to AGK! This document provides guidelines and instructions for contributing. + +## Code of Conduct + +Please be respectful and constructive in all interactions. We're building something together. + +## Getting Started + +### Prerequisites + +- Go 1.21 or later +- Git + +### Setup + +```bash +# Clone the repository +git clone https://github.com/agenticgokit/agk.git +cd agk + +# Install dependencies +go mod download + +# Build +go build ./... + +# Run tests +go test ./... +``` + +## How to Contribute + +### Reporting Issues + +- Check existing issues before creating a new one +- Use a clear, descriptive title +- Include steps to reproduce, expected vs actual behavior +- Include Go version and OS + +### Submitting Pull Requests + +1. **Fork** the repository +2. **Create a branch**: `git checkout -b feature/your-feature` +3. **Make changes** following our coding standards +4. **Test**: `go test ./...` +5. **Lint**: `golangci-lint run` +6. **Commit** with clear messages +7. **Push** and create a Pull Request + +### Commit Messages + +Follow conventional commits: + +``` +type(scope): description + +feat(trace): add hot reload support +fix(tui): resolve cursor navigation issue +docs(readme): update installation instructions +``` + +Types: `feat`, `fix`, `docs`, `style`, `refactor`, `test`, `chore` + +## Coding Standards + +### Go Style + +- Follow [Effective Go](https://golang.org/doc/effective_go) +- Run `gofmt` on all code +- Run `golangci-lint run` before committing +- Keep functions focused and small +- Add comments for exported functions + +### Testing + +- Write tests for new features +- Maintain or improve code coverage +- Use table-driven tests where appropriate + +```go +func TestExample(t *testing.T) { + tests := []struct { + name string + input string + expected string + }{ + {"basic", "input", "expected"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // test logic + }) + } +} +``` + +### Documentation + +- Update README.md for user-facing changes +- Add godoc comments to exported functions +- Include examples where helpful + +## Project Structure + +``` +agk/ +β”œβ”€β”€ cmd/ # CLI commands (cobra) +β”œβ”€β”€ internal/ # Private packages +β”‚ β”œβ”€β”€ audit/ # Audit/evaluation types +β”‚ └── tui/ # Terminal UI components +β”œβ”€β”€ go.mod +└── README.md +``` + +## Development Workflow + +```bash +# Build and install locally +go install ./... + +# Run specific command +go run main.go trace show + +# Run tests with coverage +go test -cover ./... + +# Lint +golangci-lint run +``` + +## Questions? + +- Open a [Discussion](https://github.com/agenticgokit/agk/discussions) +- Check existing [Issues](https://github.com/agenticgokit/agk/issues) + +## License + +By contributing, you agree that your contributions will be licensed under the Apache 2.0 License. diff --git a/cmd/trace.go b/cmd/trace.go index 5e3cddd..b9e4450 100644 --- a/cmd/trace.go +++ b/cmd/trace.go @@ -314,7 +314,7 @@ func showTrace(runID string) error { // If no run ID provided, use latest if runID == "" { - runID = getLatestRunID(runsDir) + runID = getLatestRunID() if runID == "" { fmt.Println("No traces found. Run with AGK_TRACE=true to generate traces.") return nil @@ -351,8 +351,8 @@ func showTrace(runID string) error { EstimatedCost: manifest.EstimatedCost, } - // Create and run TUI - model := tui.NewTraceViewer(runID, tuiManifest, spans) + // Create and run TUI with hot reload support + model := tui.NewTraceViewerWithPath(runID, tuiManifest, spans, tracePath) p := tea.NewProgram(model, tea.WithAltScreen()) if _, err := p.Run(); err != nil { return fmt.Errorf("failed to run TUI: %w", err) @@ -366,7 +366,7 @@ func viewRun(runID string) error { // If no run ID provided, use latest if runID == "" { - runID = getLatestRunID(runsDir) + runID = getLatestRunID() if runID == "" { fmt.Println("No traces found. Run with AGK_TRACE=true to generate traces.") return nil @@ -418,7 +418,7 @@ func exportTraceInternal(runID, format, output string) error { // If no run ID provided, use latest if runID == "" { - runID = getLatestRunID(runsDir) + runID = getLatestRunID() if runID == "" { fmt.Println("No traces found. Run with AGK_TRACE=true to generate traces.") return nil @@ -741,8 +741,8 @@ func toInt64(v interface{}) (int64, error) { } } -func getLatestRunID(runsDir string) string { - entries, err := os.ReadDir(runsDir) +func getLatestRunID() string { + entries, err := os.ReadDir(runsDirName) if err != nil { return "" } @@ -785,7 +785,7 @@ func auditTrace(runID string) error { // If no run ID provided, use latest if runID == "" { - runID = getLatestRunID(runsDir) + runID = getLatestRunID() if runID == "" { fmt.Println("No traces found. Run with AGK_TRACE=true to generate traces.") return nil @@ -826,7 +826,7 @@ func generateMermaid(runID, output string) error { // If no run ID provided, use latest if runID == "" { - runID = getLatestRunID(runsDir) + runID = getLatestRunID() if runID == "" { fmt.Println("No traces found. Run with AGK_TRACE=true to generate traces.") return nil diff --git a/internal/audit/collector.go b/internal/audit/collector.go index d0094a4..9071fb8 100644 --- a/internal/audit/collector.go +++ b/internal/audit/collector.go @@ -1,3 +1,4 @@ +// Package audit provides types and utilities for extracting evaluation data from traces. package audit import ( @@ -89,6 +90,10 @@ func (c *Collector) Collect() (*TraceObject, error) { obj.Summary.ToolCallCount++ case EventTypeLLMCall: obj.Summary.LLMCallCount++ + case EventTypeObservation: + // Observations are counted as part of tool calls + case EventTypeDecision: + // Decisions are workflow-level events } // Check for detailed data diff --git a/internal/tui/trace_viewer.go b/internal/tui/trace_viewer.go index ebd76bf..830fb84 100644 --- a/internal/tui/trace_viewer.go +++ b/internal/tui/trace_viewer.go @@ -1,13 +1,19 @@ package tui import ( + "bufio" "fmt" + "os" "strings" + "time" "github.com/charmbracelet/bubbles/viewport" tea "github.com/charmbracelet/bubbletea" ) +// tickMsg is sent periodically to check for file updates +type tickMsg time.Time + const ( StatusUnset = "Unset" CtrlC = "ctrl+c" @@ -66,6 +72,11 @@ type Model struct { errorCount int slowestSpan *SpanNode top3Slowest []*SpanNode + // Hot reload / file watching + tracePath string // Path to trace file being watched + lastOffset int64 // Bytes read so far + isLive bool // Whether we're watching for updates + lastUpdate time.Time // Last time file was updated } func calculateMetrics(nodes []*SpanNode) (totalTokens int, errorCount int, slowest *SpanNode, top3 []*SpanNode) { @@ -136,12 +147,25 @@ func (mc *MetricsCalculator) updateTop3(node *SpanNode) { // NewTraceViewer creates a new trace viewer model func NewTraceViewer(runID string, manifest TraceRun, spans []Span) Model { + return NewTraceViewerWithPath(runID, manifest, spans, "") +} + +// NewTraceViewerWithPath creates a trace viewer with hot reload support +func NewTraceViewerWithPath(runID string, manifest TraceRun, spans []Span, tracePath string) Model { roots := BuildSpanTree(spans) visible := FlattenTree(roots) totalTokens, errorCount, slowest, top3 := calculateMetrics(visible) estimatedCost := float64(totalTokens) * 0.000002 + // Calculate initial file offset if path provided + var lastOffset int64 + if tracePath != "" { + if info, err := os.Stat(tracePath); err == nil { + lastOffset = info.Size() + } + } + return Model{ runID: runID, manifest: manifest, @@ -154,6 +178,10 @@ func NewTraceViewer(runID string, manifest TraceRun, spans []Span) Model { errorCount: errorCount, slowestSpan: slowest, top3Slowest: top3, + tracePath: tracePath, + lastOffset: lastOffset, + isLive: tracePath != "", + lastUpdate: time.Now(), } } @@ -199,14 +227,36 @@ func (m *Model) computeMetrics() { // Init initializes the model func (m Model) Init() tea.Cmd { + if m.isLive && m.tracePath != "" { + return m.tickCmd() + } return nil } +// tickCmd returns a command that sends a tick after 500ms +func (m Model) tickCmd() tea.Cmd { + return tea.Tick(500*time.Millisecond, func(t time.Time) tea.Msg { + return tickMsg(t) + }) +} + // Update handles messages func (m Model) Update(msg tea.Msg) (tea.Model, tea.Cmd) { var cmd tea.Cmd switch msg := msg.(type) { + case tickMsg: + // Check for file updates + if m.isLive && m.tracePath != "" { + if newSpans := m.checkFileUpdates(); len(newSpans) > 0 { + // Add new spans and rebuild tree + m = m.addNewSpans(newSpans) + m.lastUpdate = time.Now() + } + return m, m.tickCmd() + } + return m, nil + case tea.KeyMsg: switch m.viewMode { case RunListView: @@ -235,6 +285,80 @@ func (m Model) Update(msg tea.Msg) (tea.Model, tea.Cmd) { return m, cmd } +// checkFileUpdates reads new lines from the trace file +func (m *Model) checkFileUpdates() []Span { + info, err := os.Stat(m.tracePath) + if err != nil { + return nil + } + + // No new data + if info.Size() <= m.lastOffset { + return nil + } + + // Open file and seek to last position + file, err := os.Open(m.tracePath) + if err != nil { + return nil + } + defer func() { _ = file.Close() }() + + if _, err := file.Seek(m.lastOffset, 0); err != nil { + return nil + } + + // Read new lines + var newLines []string + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := scanner.Text() + if line != "" { + newLines = append(newLines, line) + } + } + + // Update offset + m.lastOffset = info.Size() + + // Parse new spans + if len(newLines) == 0 { + return nil + } + + return ParseSpans(strings.Join(newLines, "\n")) +} + +// addNewSpans adds new spans to the existing tree +func (m Model) addNewSpans(newSpans []Span) Model { + // Get all existing spans + existingSpans := m.collectAllSpans() + + // Add new spans + allSpans := append(existingSpans, newSpans...) + + // Rebuild tree + m.roots = BuildSpanTree(allSpans) + m.visibleNodes = FlattenTree(m.roots) + + // Update metrics + m.computeMetrics() + + // Update manifest span count + m.manifest.SpanCount = len(allSpans) + + return m +} + +// collectAllSpans extracts all spans from the tree +func (m Model) collectAllSpans() []Span { + var spans []Span + for _, node := range m.visibleNodes { + spans = append(spans, node.Span) + } + return spans +} + // updateRunListView handles input in run list view func (m Model) updateRunListView(msg tea.KeyMsg) (tea.Model, tea.Cmd) { switch msg.String() { @@ -442,7 +566,11 @@ func (m Model) renderGlobalHeader() string { var b strings.Builder // Main Title - b.WriteString(TitleStyle.Render("AgenticGoKit Trace Explorer")) + title := "AgenticGoKit Trace Explorer" + if m.isLive { + title = "πŸ”΄ LIVE " + title + } + b.WriteString(TitleStyle.Render(title)) // If a run is selected, show its context in the header too? // Or keeps it simple. User said "fixed header".