diff --git a/.gitignore b/.gitignore index 5906c43..1527e52 100644 --- a/.gitignore +++ b/.gitignore @@ -5,5 +5,8 @@ dist/ agent main +# Plugin binaries +plugins/http-collector/http-collector + cover.out config.yml \ No newline at end of file diff --git a/plugins/http-collector/main.go b/plugins/http-collector/main.go new file mode 100644 index 0000000..4d0d835 --- /dev/null +++ b/plugins/http-collector/main.go @@ -0,0 +1,354 @@ +package main + +import ( + "context" + "crypto/tls" + "encoding/json" + "fmt" + "io" + "net/http" + "regexp" + "strconv" + "strings" + "time" + + "github.com/compliance-framework/agent/runner" + "github.com/compliance-framework/agent/runner/proto" + "github.com/google/uuid" + "github.com/hashicorp/go-hclog" + "github.com/hashicorp/go-plugin" + "google.golang.org/protobuf/types/known/timestamppb" +) + +// HttpCollectorConfig holds the configuration for the HTTP collector plugin +// This matches the config sample structure from the requirements +type HttpCollectorConfig struct { + URL string `json:"url"` + Method string `json:"method"` + Timeout int `json:"timeout"` + BasicAuth bool `json:"basic_auth"` + BasicAuthUsername string `json:"basic_auth_username"` + BasicAuthPassword string `json:"basic_auth_password"` + AdditionalHeaders string `json:"additional_headers"` + CheckCertificate bool `json:"check_certificate"` + BodyRegexPattern string `json:"body_regex_pattern"` +} + +// HttpResponseData represents the structured response data +// This will be converted to JSON and included in evidence +type HttpResponseData struct { + StatusCode int `json:"status_code"` + Status string `json:"status"` + Headers map[string][]string `json:"headers"` + Body string `json:"body"` + ResponseTime int64 `json:"response_time_ms"` + Success bool `json:"success"` // true if 200 <= status < 300 + Error string `json:"error,omitempty"` // only if request failed + MatchedRegex bool `json:"matched_regex,omitempty"` // only if regex pattern provided + BodyRegexPattern string `json:"body_regex_pattern,omitempty"` // echo back the pattern used +} + +// HttpCollectorPlugin implements the Runner interface +type HttpCollectorPlugin struct { + logger hclog.Logger + config *HttpCollectorConfig +} + +// Configure implements runner.Runner +// This is called by the agent to provide configuration to the plugin +func (p *HttpCollectorPlugin) Configure(req *proto.ConfigureRequest) (*proto.ConfigureResponse, error) { + p.logger.Debug("Configuring HTTP collector plugin") + + // Initialize with defaults + config := &HttpCollectorConfig{ + Method: "GET", + Timeout: 5000, + CheckCertificate: true, // default to secure + } + + // Parse configuration from the agent + for key, value := range req.Config { + switch key { + case "url": + config.URL = value + case "method": + config.Method = strings.ToUpper(value) + case "timeout": + if timeout, err := strconv.Atoi(value); err == nil { + config.Timeout = timeout + } + case "basic_auth": + // Handle various boolean representations + lowerValue := strings.ToLower(value) + config.BasicAuth = lowerValue == "true" || lowerValue == "1" || lowerValue == "yes" + case "basic_auth_username": + config.BasicAuthUsername = value + case "basic_auth_password": + config.BasicAuthPassword = value + case "additional_headers": + config.AdditionalHeaders = value + case "check_certificate": + config.CheckCertificate = strings.ToLower(value) != "false" + case "body_regex_pattern": + config.BodyRegexPattern = value + } + } + + // Validate required configuration + if config.URL == "" { + return nil, fmt.Errorf("url is required in configuration") + } + + p.config = config + p.logger.Info("HTTP collector configured successfully", + "url", config.URL, + "method", config.Method, + "timeout", config.Timeout) + + return &proto.ConfigureResponse{}, nil +} + +// makeHttpRequest performs the HTTP request and returns structured response data +func (p *HttpCollectorPlugin) makeHttpRequest() (*HttpResponseData, error) { + startTime := time.Now() + + // Create HTTP client with timeout and TLS settings + client := &http.Client{ + Timeout: time.Duration(p.config.Timeout) * time.Millisecond, + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: !p.config.CheckCertificate, + }, + }, + } + + // Create HTTP request + req, err := http.NewRequest(p.config.Method, p.config.URL, nil) + if err != nil { + return &HttpResponseData{ + Success: false, + Error: fmt.Sprintf("failed to create request: %v", err), + }, nil + } + + // Add basic authentication if configured + if p.config.BasicAuth && p.config.BasicAuthUsername != "" { + req.SetBasicAuth(p.config.BasicAuthUsername, p.config.BasicAuthPassword) + p.logger.Debug("Added basic authentication") + } + + // Parse and add additional headers + if p.config.AdditionalHeaders != "" { + headers := strings.Split(p.config.AdditionalHeaders, ";") + for _, header := range headers { + parts := strings.SplitN(header, ":", 2) + if len(parts) == 2 { + req.Header.Set(strings.TrimSpace(parts[0]), strings.TrimSpace(parts[1])) + } + } + p.logger.Debug("Added additional headers", "count", len(headers)) + } + + // Execute the HTTP request + resp, err := client.Do(req) + if err != nil { + return &HttpResponseData{ + Success: false, + Error: fmt.Sprintf("HTTP request failed: %v", err), + ResponseTime: time.Since(startTime).Milliseconds(), + }, nil + } + defer resp.Body.Close() + + // Read response body + body, err := io.ReadAll(resp.Body) + if err != nil { + return &HttpResponseData{ + Success: false, + Error: fmt.Sprintf("failed to read response body: %v", err), + StatusCode: resp.StatusCode, + Status: resp.Status, + Headers: resp.Header, + ResponseTime: time.Since(startTime).Milliseconds(), + }, nil + } + + // Check if status code indicates success (200 <= x < 300) + success := resp.StatusCode >= 200 && resp.StatusCode < 300 + + responseData := &HttpResponseData{ + StatusCode: resp.StatusCode, + Status: resp.Status, + Headers: resp.Header, + Body: string(body), + Success: success, + ResponseTime: time.Since(startTime).Milliseconds(), + } + + // Check regex pattern if configured + if p.config.BodyRegexPattern != "" { + matched, err := regexp.MatchString(p.config.BodyRegexPattern, string(body)) + if err != nil { + p.logger.Warn("Invalid regex pattern", "pattern", p.config.BodyRegexPattern, "error", err) + } else { + responseData.MatchedRegex = matched + responseData.BodyRegexPattern = p.config.BodyRegexPattern + p.logger.Debug("Regex pattern check", "pattern", p.config.BodyRegexPattern, "matched", matched) + } + } + + p.logger.Info("HTTP request completed", + "status", resp.StatusCode, + "success", success, + "response_time_ms", responseData.ResponseTime) + + return responseData, nil +} + +// createEvidence converts HTTP response data into Evidence protobuf for compliance reporting +func (p *HttpCollectorPlugin) createEvidence(responseData *HttpResponseData, jsonData string) (*proto.Evidence, error) { + startTime := time.Now().Add(-time.Duration(responseData.ResponseTime) * time.Millisecond) + endTime := time.Now() + + // Determine evidence status based on HTTP success and regex matching + evidenceState := proto.EvidenceStatusState_EVIDENCE_STATUS_STATE_SATISFIED + statusReason := "HTTP request successful" + + if !responseData.Success { + evidenceState = proto.EvidenceStatusState_EVIDENCE_STATUS_STATE_NOT_SATISFIED + if responseData.Error != "" { + statusReason = fmt.Sprintf("HTTP request failed: %s", responseData.Error) + } else { + statusReason = fmt.Sprintf("HTTP request returned non-success status: %d %s", responseData.StatusCode, responseData.Status) + } + } else if responseData.BodyRegexPattern != "" && !responseData.MatchedRegex { + evidenceState = proto.EvidenceStatusState_EVIDENCE_STATUS_STATE_NOT_SATISFIED + statusReason = fmt.Sprintf("Response body did not match required pattern: %s", responseData.BodyRegexPattern) + } + + // Create evidence with comprehensive metadata + description := fmt.Sprintf("HTTP %s request to %s", p.config.Method, p.config.URL) + evidence := &proto.Evidence{ + UUID: uuid.New().String(), + Title: "HTTP Endpoint Health Check", + Description: &description, + Start: timestamppb.New(startTime), + End: timestamppb.New(endTime), + Status: &proto.EvidenceStatus{ + State: evidenceState, + Reason: statusReason, + Remarks: jsonData, // Full JSON response data + }, + Props: []*proto.Property{ + {Name: "http_url", Value: p.config.URL}, + {Name: "http_method", Value: p.config.Method}, + {Name: "status_code", Value: strconv.Itoa(responseData.StatusCode)}, + {Name: "response_time_ms", Value: strconv.FormatInt(responseData.ResponseTime, 10)}, + {Name: "success", Value: strconv.FormatBool(responseData.Success)}, + }, + Activities: []*proto.Activity{ + { + Title: "HTTP Health Check", + Description: fmt.Sprintf("Performed %s request to %s for health monitoring", p.config.Method, p.config.URL), + Steps: []*proto.Step{ + { + Title: "Configure HTTP Client", + Description: fmt.Sprintf("Set timeout: %dms, certificate check: %t, basic auth: %t", + p.config.Timeout, p.config.CheckCertificate, p.config.BasicAuth), + }, + { + Title: "Execute HTTP Request", + Description: fmt.Sprintf("Made %s request to %s", p.config.Method, p.config.URL), + }, + { + Title: "Process Response", + Description: fmt.Sprintf("Received status %d, processed %d bytes in %dms", + responseData.StatusCode, len(responseData.Body), responseData.ResponseTime), + }, + }, + }, + }, + Subjects: []*proto.Subject{ + { + Identifier: p.config.URL, + Type: proto.SubjectType_SUBJECT_TYPE_COMPONENT, + Description: fmt.Sprintf("HTTP endpoint at %s", p.config.URL), + }, + }, + } + + // Add regex-specific properties if configured + if responseData.BodyRegexPattern != "" { + evidence.Props = append(evidence.Props, &proto.Property{ + Name: "regex_pattern", + Value: responseData.BodyRegexPattern, + }) + evidence.Props = append(evidence.Props, &proto.Property{ + Name: "regex_matched", + Value: strconv.FormatBool(responseData.MatchedRegex), + }) + } + + return evidence, nil +} + +// Eval implements runner.Runner +// This is the main execution function where we make HTTP requests and create evidence +func (p *HttpCollectorPlugin) Eval(req *proto.EvalRequest, helper runner.ApiHelper) (*proto.EvalResponse, error) { + p.logger.Debug("Starting HTTP evaluation") + + // Make HTTP request + responseData, err := p.makeHttpRequest() + if err != nil { + p.logger.Error("HTTP request failed", "error", err) + return &proto.EvalResponse{Status: proto.ExecutionStatus_FAILURE}, err + } + + // Convert response data to JSON for evidence + jsonData, err := json.MarshalIndent(responseData, "", " ") + if err != nil { + p.logger.Error("Failed to marshal response data", "error", err) + return &proto.EvalResponse{Status: proto.ExecutionStatus_FAILURE}, err + } + + // Create evidence from HTTP response + evidence, err := p.createEvidence(responseData, string(jsonData)) + if err != nil { + p.logger.Error("Failed to create evidence", "error", err) + return &proto.EvalResponse{Status: proto.ExecutionStatus_FAILURE}, err + } + + // Send evidence to the compliance API via helper + err = helper.CreateEvidence(context.Background(), []*proto.Evidence{evidence}) + if err != nil { + p.logger.Error("Failed to send evidence", "error", err) + return &proto.EvalResponse{Status: proto.ExecutionStatus_FAILURE}, err + } + + p.logger.Info("HTTP evaluation completed successfully", "success", responseData.Success) + p.logger.Debug("Response data", "json", string(jsonData)) + + return &proto.EvalResponse{Status: proto.ExecutionStatus_SUCCESS}, nil +} + +func main() { + // Create logger for the plugin + logger := hclog.New(&hclog.LoggerOptions{ + Name: "http-collector-plugin", + Output: hclog.DefaultOutput, + Level: hclog.Debug, + }) + + // Serve the plugin using HashiCorp's plugin framework + plugin.Serve(&plugin.ServeConfig{ + HandshakeConfig: runner.HandshakeConfig, + Plugins: map[string]plugin.Plugin{ + "runner": &runner.RunnerGRPCPlugin{ + Impl: &HttpCollectorPlugin{ + logger: logger, + }, + }, + }, + GRPCServer: plugin.DefaultGRPCServer, + }) +} diff --git a/plugins/http-collector/test-config.yaml b/plugins/http-collector/test-config.yaml new file mode 100644 index 0000000..2032e13 --- /dev/null +++ b/plugins/http-collector/test-config.yaml @@ -0,0 +1,53 @@ +# Test configuration for HTTP collector plugin +# This matches the config sample from the requirements + +api: + url: "http://localhost:8080" # Required by agent (can be mock API) + +plugins: + health-checker: + source: "./plugins/http-collector/http-collector" # Path to our compiled plugin + policies: [] # No policies needed for basic test + labels: + type: "http-health-check" + environment: "test" + config: + url: "https://httpbin.org/json" # Test endpoint that returns JSON + method: "GET" + timeout: 5000 + basic_auth: false + additional_headers: "Content-Type: application/json;User-Agent: ccf-http-collector" + check_certificate: true + body_regex_pattern: "slideshow" # Simple pattern - should match the JSON response + + metrics-checker: + source: "./plugins/http-collector/http-collector" # Same plugin, different config + policies: [] + labels: + type: "metrics-check" + environment: "test" + config: + url: "https://httpbin.org/status/200" # Always returns 200 OK + method: "GET" + timeout: 3000 + basic_auth: false + check_certificate: true + # No regex pattern - should pass with just 200 status + + auth-test: + source: "./plugins/http-collector/http-collector" # Test basic auth + policies: [] + labels: + type: "auth-test" + environment: "test" + config: + url: "https://httpbin.org/basic-auth/testuser/testpass" + method: "GET" + timeout: 5000 + basic_auth: true + basic_auth_username: "testuser" + basic_auth_password: "testpass" + check_certificate: true + body_regex_pattern: "authenticated" # Should match the auth response + +verbose: 1 # Enable debug logging to see what's happening \ No newline at end of file