diff --git a/pkg/jobrunaggregator/jobrunaggregatorapi/types_row_test_summary.go b/pkg/jobrunaggregator/jobrunaggregatorapi/types_row_test_summary.go new file mode 100644 index 00000000000..cb569e33ada --- /dev/null +++ b/pkg/jobrunaggregator/jobrunaggregatorapi/types_row_test_summary.go @@ -0,0 +1,20 @@ +package jobrunaggregatorapi + +import ( + "cloud.google.com/go/civil" +) + +// TestSummaryByPeriodRow represents aggregated test results for a specific suite and release over a time period. +// This data structure corresponds to the suite_summary_by_period.sql query results. +type TestSummaryByPeriodRow struct { + Release string `bigquery:"release"` + TestName string `bigquery:"test_name"` + TotalTestCount int64 `bigquery:"total_test_count"` + TotalFailureCount int64 `bigquery:"total_failure_count"` + TotalFlakeCount int64 `bigquery:"total_flake_count"` + FailureRate float64 `bigquery:"failure_rate"` + AvgDurationMs float64 `bigquery:"avg_duration_ms"` + PeriodStart civil.Date `bigquery:"period_start"` + PeriodEnd civil.Date `bigquery:"period_end"` + DaysWithData int64 `bigquery:"days_with_data"` +} diff --git a/pkg/jobrunaggregator/jobrunaggregatorlib/ci_data_client.go b/pkg/jobrunaggregator/jobrunaggregatorlib/ci_data_client.go index 992bdce5b1b..625c3b64838 100644 --- a/pkg/jobrunaggregator/jobrunaggregatorlib/ci_data_client.go +++ b/pkg/jobrunaggregator/jobrunaggregatorlib/ci_data_client.go @@ -66,6 +66,14 @@ type CIDataClient interface { // ListReleases lists all releases from the new release table ListReleases(ctx context.Context) ([]jobrunaggregatorapi.ReleaseRow, error) + + // ListTestSummaryByPeriod retrieves aggregated test results for a specific suite and release over a time period. + // Parameters: + // - suiteName: The test suite to query (e.g., 'conformance') + // - releaseName: The release version (e.g., '4.15') + // - daysBack: Number of days to look back from current date + // - minTestCount: Minimum number of test executions required to include a test in results + ListTestSummaryByPeriod(ctx context.Context, suiteName, releaseName string, daysBack, minTestCount int) ([]jobrunaggregatorapi.TestSummaryByPeriodRow, error) } type ciDataClient struct { @@ -1095,3 +1103,71 @@ func (c *ciDataClient) ListAllKnownAlerts(ctx context.Context) ([]*jobrunaggrega return allKnownAlerts, nil } + +func (c *ciDataClient) ListTestSummaryByPeriod(ctx context.Context, suiteName, releaseName string, daysBack, minTestCount int) ([]jobrunaggregatorapi.TestSummaryByPeriodRow, error) { + // Query to summarize test results for a specific suite and release over a time period + // Groups by release and test_name only (no infrastructure dimensions) + // Calculates total test_count, failure_count, flake_count, failure_rate, and avg_duration_ms + // Filters results to only include tests with sufficient test runs + queryString := c.dataCoordinates.SubstituteDataSetLocation(` +SELECT + release, + test_name, + SUM(test_count) AS total_test_count, + SUM(failure_count) AS total_failure_count, + SUM(flake_count) AS total_flake_count, + SAFE_DIVIDE(SUM(failure_count), SUM(test_count)) AS failure_rate, + AVG(avg_duration_ms) AS avg_duration_ms, + MIN(date) AS period_start, + MAX(date) AS period_end, + COUNT(DISTINCT date) AS days_with_data +FROM + DATA_SET_LOCATION.TestsSummaryByDate +WHERE + suite = @suite_name + AND release = @release_name + AND date >= DATE_SUB(CURRENT_DATE(), INTERVAL @days_back DAY) + AND date <= CURRENT_DATE() +GROUP BY + release, + test_name +HAVING + SUM(test_count) > @min_test_count +ORDER BY + release, + total_failure_count DESC, + test_name +`) + + query := c.client.Query(queryString) + query.Labels = map[string]string{ + bigQueryLabelKeyApp: bigQueryLabelValueApp, + bigQueryLabelKeyQuery: bigQueryLabelValueTestSummaryByPeriod, + } + query.QueryConfig.Parameters = []bigquery.QueryParameter{ + {Name: "suite_name", Value: suiteName}, + {Name: "release_name", Value: releaseName}, + {Name: "days_back", Value: daysBack}, + {Name: "min_test_count", Value: minTestCount}, + } + + rows, err := query.Read(ctx) + if err != nil { + return nil, fmt.Errorf("failed to query generic test summary by period with %q: %w", queryString, err) + } + + results := []jobrunaggregatorapi.TestSummaryByPeriodRow{} + for { + row := &jobrunaggregatorapi.TestSummaryByPeriodRow{} + err = rows.Next(row) + if err == iterator.Done { + break + } + if err != nil { + return nil, err + } + results = append(results, *row) + } + + return results, nil +} diff --git a/pkg/jobrunaggregator/jobrunaggregatorlib/ci_data_client_mock.go b/pkg/jobrunaggregator/jobrunaggregatorlib/ci_data_client_mock.go index 70d491dda7e..9fd06d7e369 100644 --- a/pkg/jobrunaggregator/jobrunaggregatorlib/ci_data_client_mock.go +++ b/pkg/jobrunaggregator/jobrunaggregatorlib/ci_data_client_mock.go @@ -268,6 +268,21 @@ func (mr *MockCIDataClientMockRecorder) ListReleases(ctx any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListReleases", reflect.TypeOf((*MockCIDataClient)(nil).ListReleases), ctx) } +// ListTestSummaryByPeriod mocks base method. +func (m *MockCIDataClient) ListTestSummaryByPeriod(ctx context.Context, suiteName, releaseName string, daysBack, minTestCount int) ([]jobrunaggregatorapi.TestSummaryByPeriodRow, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListTestSummaryByPeriod", ctx, suiteName, releaseName, daysBack, minTestCount) + ret0, _ := ret[0].([]jobrunaggregatorapi.TestSummaryByPeriodRow) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListTestSummaryByPeriod indicates an expected call of ListTestSummaryByPeriod. +func (mr *MockCIDataClientMockRecorder) ListTestSummaryByPeriod(ctx, suiteName, releaseName, daysBack, minTestCount any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListTestSummaryByPeriod", reflect.TypeOf((*MockCIDataClient)(nil).ListTestSummaryByPeriod), ctx, suiteName, releaseName, daysBack, minTestCount) +} + // ListUploadedJobRunIDsSinceFromTable mocks base method. func (m *MockCIDataClient) ListUploadedJobRunIDsSinceFromTable(ctx context.Context, table string, since *time.Time) (map[string]bool, error) { m.ctrl.T.Helper() diff --git a/pkg/jobrunaggregator/jobrunaggregatorlib/retrying_ci_data_client.go b/pkg/jobrunaggregator/jobrunaggregatorlib/retrying_ci_data_client.go index 43096a4cb08..a06339b5afc 100644 --- a/pkg/jobrunaggregator/jobrunaggregatorlib/retrying_ci_data_client.go +++ b/pkg/jobrunaggregator/jobrunaggregatorlib/retrying_ci_data_client.go @@ -185,6 +185,16 @@ func (c *retryingCIDataClient) ListAllKnownAlerts(ctx context.Context) ([]*jobru return ret, err } +func (c *retryingCIDataClient) ListTestSummaryByPeriod(ctx context.Context, suiteName, releaseName string, daysBack, minTestCount int) ([]jobrunaggregatorapi.TestSummaryByPeriodRow, error) { + var ret []jobrunaggregatorapi.TestSummaryByPeriodRow + err := retry.OnError(slowBackoff, isReadQuotaError, func() error { + var innerErr error + ret, innerErr = c.delegate.ListTestSummaryByPeriod(ctx, suiteName, releaseName, daysBack, minTestCount) + return innerErr + }) + return ret, err +} + var slowBackoff = wait.Backoff{ Steps: 4, Duration: 10 * time.Second, diff --git a/pkg/jobrunaggregator/jobrunaggregatorlib/util.go b/pkg/jobrunaggregator/jobrunaggregatorlib/util.go index 07f7343adce..c44f239b046 100644 --- a/pkg/jobrunaggregator/jobrunaggregatorlib/util.go +++ b/pkg/jobrunaggregator/jobrunaggregatorlib/util.go @@ -54,6 +54,7 @@ const ( bigQueryLabelValueAllReleases = "aggregator-all-releases" bigQueryLabelValueReleaseTags = "aggregator-release-tags" bigQueryLabelValueJobRunIDsSinceTime = "aggregator-job-run-ids-since-time" + bigQueryLabelValueTestSummaryByPeriod = "aggregator-test-summary-by-period" ) var ( diff --git a/pkg/jobrunaggregator/jobrunhistoricaldataanalyzer/analyzer.go b/pkg/jobrunaggregator/jobrunhistoricaldataanalyzer/analyzer.go index 01db44fa24a..cbca849d34c 100644 --- a/pkg/jobrunaggregator/jobrunhistoricaldataanalyzer/analyzer.go +++ b/pkg/jobrunaggregator/jobrunhistoricaldataanalyzer/analyzer.go @@ -24,8 +24,6 @@ type JobRunHistoricalDataAnalyzerOptions struct { func (o *JobRunHistoricalDataAnalyzerOptions) Run(ctx context.Context) error { - var newHistoricalData []jobrunaggregatorapi.HistoricalData - // targetRelease will either be what the caller specified on the CLI, or the most recent release. // previousRelease will be the one prior to targetRelease. var targetRelease, previousRelease string @@ -44,6 +42,14 @@ func (o *JobRunHistoricalDataAnalyzerOptions) Run(ctx context.Context) error { } fmt.Printf("Using target release: %s, previous release: %s\n", targetRelease, previousRelease) + // For tests data type, we don't do comparison - just fetch and write directly + if o.dataType == "tests" { + return o.runTestsDataType(ctx, targetRelease) + } + + // For other data types (alerts, disruptions), continue with comparison logic + var newHistoricalData []jobrunaggregatorapi.HistoricalData + currentHistoricalData, err := readHistoricalDataFile(o.currentFile, o.dataType) if err != nil { return err @@ -91,6 +97,80 @@ func (o *JobRunHistoricalDataAnalyzerOptions) Run(ctx context.Context) error { return nil } +func (o *JobRunHistoricalDataAnalyzerOptions) runTestsDataType(ctx context.Context, release string) error { + // Hardcoded parameters for test summary query + const ( + suiteName = "openshift-tests" + daysBack = 30 + minTestCount = 100 + minDaysRequired = 10 + ) + + fmt.Printf("Fetching test data for release %s, suite %s, last %d days, min %d test runs\n", + release, suiteName, daysBack, minTestCount) + + // Try to read existing data if the output file exists + var existingTestSummaries []jobrunaggregatorapi.TestSummaryByPeriodRow + if _, err := os.Stat(o.outputFile); err == nil { + existingTestSummaries, err = readTestSummaryFile(o.outputFile) + if err != nil { + fmt.Printf("Warning: failed to read existing test summary file: %v\n", err) + existingTestSummaries = nil + } else { + fmt.Printf("Found existing test summary data with %d test results\n", len(existingTestSummaries)) + } + } + + // Fetch new test data from BigQuery + testSummaries, err := o.ciDataClient.ListTestSummaryByPeriod(ctx, suiteName, release, daysBack, minTestCount) + if err != nil { + return fmt.Errorf("failed to list test summary by period: %w", err) + } + + // Validate the new data has sufficient days of data + hasSufficientData := hasSufficientDaysOfData(testSummaries, minDaysRequired) + + // Determine which data to use + var finalTestSummaries []jobrunaggregatorapi.TestSummaryByPeriodRow + if len(testSummaries) == 0 { + // No new data available + if len(existingTestSummaries) > 0 { + fmt.Printf("Warning: no new test data found, keeping existing data with %d test results\n", len(existingTestSummaries)) + finalTestSummaries = existingTestSummaries + } else { + return fmt.Errorf("no test data found for suite %s, release %s", suiteName, release) + } + } else if !hasSufficientData { + // New data exists but doesn't have enough days + if len(existingTestSummaries) > 0 { + fmt.Printf("Warning: new test data has insufficient days (< %d), keeping existing data with %d test results\n", + minDaysRequired, len(existingTestSummaries)) + finalTestSummaries = existingTestSummaries + } else { + fmt.Printf("Warning: new test data has insufficient days (< %d) and no existing data available, using new data with %d test results\n", + minDaysRequired, len(testSummaries)) + finalTestSummaries = testSummaries + } + } else { + // New data is sufficient + fmt.Printf("Using new test data with %d test results (sufficient days of data)\n", len(testSummaries)) + finalTestSummaries = testSummaries + } + + // Write the final test summaries to the output file as JSON + out, err := formatTestOutput(finalTestSummaries) + if err != nil { + return fmt.Errorf("error formatting test output: %w", err) + } + + if err := os.WriteFile(o.outputFile, out, 0644); err != nil { + return fmt.Errorf("failed to write output file: %w", err) + } + + fmt.Printf("Successfully wrote %d test results to %s\n", len(finalTestSummaries), o.outputFile) + return nil +} + func (o *JobRunHistoricalDataAnalyzerOptions) getAlertData(ctx context.Context) ([]jobrunaggregatorapi.HistoricalData, error) { var allKnownAlerts []*jobrunaggregatorapi.KnownAlertRow var newHistoricalData []*jobrunaggregatorapi.AlertHistoricalDataRow diff --git a/pkg/jobrunaggregator/jobrunhistoricaldataanalyzer/cmd.go b/pkg/jobrunaggregator/jobrunhistoricaldataanalyzer/cmd.go index 17934ba69c8..c6b530393c7 100644 --- a/pkg/jobrunaggregator/jobrunhistoricaldataanalyzer/cmd.go +++ b/pkg/jobrunaggregator/jobrunhistoricaldataanalyzer/cmd.go @@ -26,7 +26,7 @@ type JobRunHistoricalDataAnalyzerFlags struct { PreviousRelease string } -var supportedDataTypes = sets.New[string]("alerts", "disruptions") +var supportedDataTypes = sets.New[string]("alerts", "disruptions", "tests") func NewJobRunHistoricalDataAnalyzerFlags() *JobRunHistoricalDataAnalyzerFlags { return &JobRunHistoricalDataAnalyzerFlags{ @@ -60,7 +60,8 @@ func (f *JobRunHistoricalDataAnalyzerFlags) Validate() error { return fmt.Errorf("must provide supported datatype %v", sets.List(supportedDataTypes)) } - if f.CurrentFile == "" { + // For tests data type, we don't need --current since we don't do comparison + if f.DataType != "tests" && f.CurrentFile == "" { return fmt.Errorf("must provide --current [file_path] flag to compare against") } @@ -68,7 +69,7 @@ func (f *JobRunHistoricalDataAnalyzerFlags) Validate() error { return fmt.Errorf("leeway percent must be above 0") } - if f.TargetRelease != "" && f.PreviousRelease == "" { + if f.TargetRelease != "" && f.PreviousRelease == "" && f.DataType != "tests" { return fmt.Errorf("must specify --previous-release with --target-release") } diff --git a/pkg/jobrunaggregator/jobrunhistoricaldataanalyzer/util.go b/pkg/jobrunaggregator/jobrunhistoricaldataanalyzer/util.go index 36e21cb8e87..0e7cbe2d31e 100644 --- a/pkg/jobrunaggregator/jobrunhistoricaldataanalyzer/util.go +++ b/pkg/jobrunaggregator/jobrunhistoricaldataanalyzer/util.go @@ -166,3 +166,52 @@ func formatOutput(data []parsedJobData, format string) ([]byte, error) { return nil, fmt.Errorf("invalid output format (%s)", format) } } + +func formatTestOutput(data []jobrunaggregatorapi.TestSummaryByPeriodRow) ([]byte, error) { + if len(data) == 0 { + return nil, nil + } + // Sort by release, failure count desc, test name + sort.SliceStable(data, func(i, j int) bool { + if data[i].Release != data[j].Release { + return data[i].Release < data[j].Release + } + if data[i].TotalFailureCount != data[j].TotalFailureCount { + return data[i].TotalFailureCount > data[j].TotalFailureCount + } + return data[i].TestName < data[j].TestName + }) + return json.MarshalIndent(data, "", " ") +} + +// readTestSummaryFile reads test summary data from a JSON file +func readTestSummaryFile(filePath string) ([]jobrunaggregatorapi.TestSummaryByPeriodRow, error) { + data, err := os.ReadFile(filePath) + if err != nil { + return nil, fmt.Errorf("failed to read file at path (%s): %w", filePath, err) + } + + var testSummaries []jobrunaggregatorapi.TestSummaryByPeriodRow + if err := json.Unmarshal(data, &testSummaries); err != nil { + return nil, fmt.Errorf("failed to unmarshal test summary data: %w", err) + } + + return testSummaries, nil +} + +// hasSufficientDaysOfData checks if test summaries have at least minDays of data +// Returns true if any row has DaysWithData >= minDays +func hasSufficientDaysOfData(testSummaries []jobrunaggregatorapi.TestSummaryByPeriodRow, minDays int64) bool { + if len(testSummaries) == 0 { + return false + } + + // Check if any test has sufficient days of data + for _, summary := range testSummaries { + if summary.DaysWithData >= minDays { + return true + } + } + + return false +}