From 25d1194c9685156794018c82534e3f809a6f403e Mon Sep 17 00:00:00 2001 From: Gergely Sallai <1516628+gergely-sallai@users.noreply.github.com> Date: Mon, 9 Mar 2026 15:10:00 +0100 Subject: [PATCH 01/11] refact: STEP-2230 Add xcresult converter from deploy step - Moved over XCResult converters from steps-deploy-to-bitrise - Bumped dependency versions --- go.mod | 6 +- go.sum | 8 +- testresult/xcresult/testsummariesplist.go | 124 +++++ testresult/xcresult/xcresult.go | 121 +++++ testresult/xcresult/xcresult_test.go | 15 + .../xcresult3/action_invocation_record.go | 98 ++++ .../action_invocation_record_test.go | 124 +++++ .../xcresult3/action_test_plan_summary.go | 100 ++++ .../action_test_plan_summary_test.go | 198 +++++++ testresult/xcresult3/action_test_summary.go | 78 +++ .../xcresult3/action_test_summary_group.go | 122 +++++ .../action_test_summary_group_test.go | 85 +++ testresult/xcresult3/converter.go | 503 ++++++++++++++++++ testresult/xcresult3/converter_test.go | 263 +++++++++ testresult/xcresult3/logging.go | 10 + testresult/xcresult3/model3/conversion.go | 217 ++++++++ .../xcresult3/model3/conversion_test.go | 225 ++++++++ testresult/xcresult3/model3/data.go | 35 ++ testresult/xcresult3/model3/export.go | 38 ++ testresult/xcresult3/model3/testresults.go | 63 +++ testresult/xcresult3/xcresult3.go | 31 ++ testresult/xcresult3/xcresulttool.go | 182 +++++++ 22 files changed, 2639 insertions(+), 7 deletions(-) create mode 100644 testresult/xcresult/testsummariesplist.go create mode 100644 testresult/xcresult/xcresult.go create mode 100644 testresult/xcresult/xcresult_test.go create mode 100644 testresult/xcresult3/action_invocation_record.go create mode 100644 testresult/xcresult3/action_invocation_record_test.go create mode 100644 testresult/xcresult3/action_test_plan_summary.go create mode 100644 testresult/xcresult3/action_test_plan_summary_test.go create mode 100644 testresult/xcresult3/action_test_summary.go create mode 100644 testresult/xcresult3/action_test_summary_group.go create mode 100644 testresult/xcresult3/action_test_summary_group_test.go create mode 100644 testresult/xcresult3/converter.go create mode 100644 testresult/xcresult3/converter_test.go create mode 100644 testresult/xcresult3/logging.go create mode 100644 testresult/xcresult3/model3/conversion.go create mode 100644 testresult/xcresult3/model3/conversion_test.go create mode 100644 testresult/xcresult3/model3/data.go create mode 100644 testresult/xcresult3/model3/export.go create mode 100644 testresult/xcresult3/model3/testresults.go create mode 100644 testresult/xcresult3/xcresult3.go create mode 100644 testresult/xcresult3/xcresulttool.go diff --git a/go.mod b/go.mod index 333a1299..780b3ffa 100644 --- a/go.mod +++ b/go.mod @@ -9,9 +9,9 @@ require ( cloud.google.com/go/storage v1.50.0 github.com/bitrise-io/go-plist v0.0.0-20210301100253-4b1a112ccd10 github.com/bitrise-io/go-steputils v1.0.5 - github.com/bitrise-io/go-steputils/v2 v2.0.0-alpha.43 + github.com/bitrise-io/go-steputils/v2 v2.0.0-alpha.48.0.20260309144725-a75605d075c2 github.com/bitrise-io/go-utils v1.0.13 - github.com/bitrise-io/go-utils/v2 v2.0.0-alpha.31 + github.com/bitrise-io/go-utils/v2 v2.0.0-alpha.33 github.com/bitrise-io/go-xcode v1.3.1 github.com/globocom/go-buffer/v2 v2.0.0 github.com/golang-jwt/jwt/v4 v4.5.0 @@ -23,6 +23,7 @@ require ( golang.org/x/oauth2 v0.24.0 golang.org/x/text v0.21.0 google.golang.org/api v0.214.0 + howett.net/plist v1.0.0 ) require ( @@ -79,5 +80,4 @@ require ( google.golang.org/grpc v1.67.3 // indirect google.golang.org/protobuf v1.35.2 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - howett.net/plist v1.0.0 // indirect ) diff --git a/go.sum b/go.sum index e1f7e7bc..a1d21ee7 100644 --- a/go.sum +++ b/go.sum @@ -38,13 +38,13 @@ github.com/bitrise-io/go-plist v0.0.0-20210301100253-4b1a112ccd10 h1:/2OyBFI7GjY github.com/bitrise-io/go-plist v0.0.0-20210301100253-4b1a112ccd10/go.mod h1:pARutiL3kEuRLV3JvswidvfCj+9Y3qMZtji2BDqLFsA= github.com/bitrise-io/go-steputils v1.0.5 h1:OBH7CPXeqIWFWJw6BOUMQnUb8guspwKr2RhYBhM9tfc= github.com/bitrise-io/go-steputils v1.0.5/go.mod h1:YIUaQnIAyK4pCvQG0hYHVkSzKNT9uL2FWmkFNW4mfNI= -github.com/bitrise-io/go-steputils/v2 v2.0.0-alpha.43 h1:oahoCL46PPywHRBin54zrwDOhXlMPIXx6zdo1Hgkz1g= -github.com/bitrise-io/go-steputils/v2 v2.0.0-alpha.43/go.mod h1:SjWTgoD5wDyyIa+xPrA+U2UgL9K8Lx/xuLaK5LBCjXE= +github.com/bitrise-io/go-steputils/v2 v2.0.0-alpha.48.0.20260309144725-a75605d075c2 h1:EgMa37uxwYA+4hpex+FyQVxJPWM1FSWYXl5sxuLh8h8= +github.com/bitrise-io/go-steputils/v2 v2.0.0-alpha.48.0.20260309144725-a75605d075c2/go.mod h1:CL1sOqz4+q4XK/OCjB8YNV27Xmz/Fo7v/QKxobmGIx4= github.com/bitrise-io/go-utils v1.0.1/go.mod h1:ZY1DI+fEpZuFpO9szgDeICM4QbqoWVt0RSY3tRI1heY= github.com/bitrise-io/go-utils v1.0.13 h1:1QENhTS/JlKH9F7+/nB+TtbTcor6jGrE6cQ4CJWfp5U= github.com/bitrise-io/go-utils v1.0.13/go.mod h1:ZY1DI+fEpZuFpO9szgDeICM4QbqoWVt0RSY3tRI1heY= -github.com/bitrise-io/go-utils/v2 v2.0.0-alpha.31 h1:Lay9mco4/T88oYY+kqZlpdWeU9aj32/qWMRwcTg812o= -github.com/bitrise-io/go-utils/v2 v2.0.0-alpha.31/go.mod h1:3XUplo0dOWc3DqT2XA2SeHToDSg7+j1y1HTHibT2H68= +github.com/bitrise-io/go-utils/v2 v2.0.0-alpha.33 h1:2Skyp4yg8aNKLr5GB5amM9UK9n1yzIMT88Rb/ZBz8m4= +github.com/bitrise-io/go-utils/v2 v2.0.0-alpha.33/go.mod h1:3XUplo0dOWc3DqT2XA2SeHToDSg7+j1y1HTHibT2H68= github.com/bitrise-io/go-xcode v1.3.1 h1:ioLPHQ+XnSafCpnFJl+d9+qdfIr0Z55yQRlEA66/XxI= github.com/bitrise-io/go-xcode v1.3.1/go.mod h1:9OwsvrhZ4A2JxHVoEY7CPcABAKA+OE7FQqFfBfvbFuY= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= diff --git a/testresult/xcresult/testsummariesplist.go b/testresult/xcresult/testsummariesplist.go new file mode 100644 index 00000000..c258fc58 --- /dev/null +++ b/testresult/xcresult/testsummariesplist.go @@ -0,0 +1,124 @@ +package xcresult + +import ( + "fmt" + "strings" +) + +// TestSummaryPlist ... +type TestSummaryPlist struct { + FormatVersion string + TestableSummaries []TestableSummary +} + +func collapseSubtestTree(data Subtests) (tests Subtests) { + for _, test := range data { + if len(test.Subtests) > 0 { + tests = append(tests, collapseSubtestTree(test.Subtests)...) + } + if test.TestStatus != "" { + tests = append(tests, test) + } + } + return +} + +// Tests returns the collapsed tree of tests +func (summaryPlist TestSummaryPlist) Tests() ([]string, map[string]Subtests) { + var keyOrder []string + tests := map[string]Subtests{} + var subTests Subtests + for _, testableSummary := range summaryPlist.TestableSummaries { + for _, test := range testableSummary.Tests { + subTests = append(subTests, collapseSubtestTree(test.Subtests)...) + } + } + for _, test := range subTests { + // TestIdentifier is in a format of testID/testCase + testID := strings.Split(test.TestIdentifier, "/")[0] + if _, found := tests[testID]; !found { + keyOrder = append(keyOrder, testID) + } + tests[testID] = append(tests[testID], test) + } + return keyOrder, tests +} + +// Test ... +type Test struct { + Subtests Subtests +} + +// TestableSummary ... +type TestableSummary struct { + TargetName string + TestKind string + TestName string + TestObjectClass string + Tests []Test +} + +// FailureSummary ... +type FailureSummary struct { + FileName string + LineNumber int + Message string + PerformanceFailure bool +} + +// Subtest ... +type Subtest struct { + Duration float64 + TestStatus string + TestIdentifier string + TestName string + TestObjectClass string + Subtests Subtests + FailureSummaries []FailureSummary +} + +// Failure ... +func (st Subtest) Failure() (message string) { + prefix := "" + for _, failure := range st.FailureSummaries { + message += fmt.Sprintf("%s%s:%d - %s", prefix, failure.FileName, failure.LineNumber, failure.Message) + prefix = "\n" + } + return +} + +// Skipped ... +func (st Subtest) Skipped() bool { + return st.TestStatus == "Skipped" +} + +// Subtests ... +type Subtests []Subtest + +// FailuresCount ... +func (sts Subtests) FailuresCount() (count int) { + for _, test := range sts { + if len(test.FailureSummaries) > 0 { + count++ + } + } + return count +} + +// SkippedCount ... +func (sts Subtests) SkippedCount() (count int) { + for _, test := range sts { + if test.Skipped() { + count++ + } + } + return count +} + +// TotalTime ... +func (sts Subtests) TotalTime() (time float64) { + for _, test := range sts { + time += test.Duration + } + return time +} diff --git a/testresult/xcresult/xcresult.go b/testresult/xcresult/xcresult.go new file mode 100644 index 00000000..8c15ecae --- /dev/null +++ b/testresult/xcresult/xcresult.go @@ -0,0 +1,121 @@ +package xcresult + +import ( + "path/filepath" + "strings" + "unicode" + + "howett.net/plist" + + "github.com/bitrise-io/go-utils/fileutil" + "github.com/bitrise-io/go-utils/pathutil" + "github.com/bitrise-io/go-steputils/v2/testreport" +) + +// Converter ... +type Converter struct { + files []string + testSummariesPlistPath string +} + +func (c *Converter) Setup(_ bool) {} + +// Detect ... +func (c *Converter) Detect(files []string) bool { + c.files = files + for _, file := range c.files { + if filepath.Ext(file) == ".xcresult" { + testSummariesPlistPath := filepath.Join(file, "TestSummaries.plist") + if exist, err := pathutil.IsPathExists(testSummariesPlistPath); err != nil || !exist { + continue + } + + c.testSummariesPlistPath = testSummariesPlistPath + return true + } + } + return false +} + +// by one of our issue reports, need to replace backspace char (U+0008) as it is an invalid character for xml unmarshaller +// the legal character ranges are here: https://www.w3.org/TR/REC-xml/#charsets +// so the exclusion will be: +/* + \u0000 - \u0008 + \u000B + \u000C + \u000E - \u001F + \u007F - \u0084 + \u0086 - \u009F + \uD800 - \uDFFF + + Unicode range D800–DFFF is used as surrogate pair. Unicode and ISO/IEC 10646 do not assign characters to any of the code points in the D800–DFFF range, so an individual code value from a surrogate pair does not represent a character. (A couple of code points — the first from the high surrogate area (D800–DBFF), and the second from the low surrogate area (DC00–DFFF) — are used in UTF-16 to represent a character in supplementary planes) + \uFDD0 - \uFDEF; \uFFFE; \uFFFF +*/ +// These are non-characters in the standard, not assigned to anything; and have no meaning. +func filterIllegalChars(data []byte) (filtered []byte) { + illegalCharFilter := func(r rune) rune { + if unicode.IsPrint(r) { + return r + } + return -1 + } + filtered = []byte(strings.Map(illegalCharFilter, string(data))) + return +} + +// XML ... +func (c *Converter) Convert() (testreport.TestReport, error) { + data, err := fileutil.ReadBytesFromFile(c.testSummariesPlistPath) + if err != nil { + return testreport.TestReport{}, err + } + + data = filterIllegalChars(data) + + var plistData TestSummaryPlist + if _, err := plist.Unmarshal(data, &plistData); err != nil { + return testreport.TestReport{}, err + } + + var xmlData testreport.TestReport + keyOrder, tests := plistData.Tests() + for _, testID := range keyOrder { + tests := tests[testID] + testSuite := testreport.TestSuite{ + Name: testID, + Tests: len(tests), + Failures: tests.FailuresCount(), + Skipped: tests.SkippedCount(), + Time: tests.TotalTime(), + } + + for _, test := range tests { + failureMessage := test.Failure() + + var failure *testreport.Failure + if len(failureMessage) > 0 { + failure = &testreport.Failure{ + Value: failureMessage, + } + } + + var skipped *testreport.Skipped + if test.Skipped() { + skipped = &testreport.Skipped{} + } + + testSuite.TestCases = append(testSuite.TestCases, testreport.TestCase{ + Name: test.TestName, + ClassName: testID, + Failure: failure, + Skipped: skipped, + Time: test.Duration, + }) + } + + xmlData.TestSuites = append(xmlData.TestSuites, testSuite) + } + + return xmlData, nil +} diff --git a/testresult/xcresult/xcresult_test.go b/testresult/xcresult/xcresult_test.go new file mode 100644 index 00000000..a36158b1 --- /dev/null +++ b/testresult/xcresult/xcresult_test.go @@ -0,0 +1,15 @@ +package xcresult + +import ( + "reflect" + "testing" +) + +func Test_filterIllegalChars(t *testing.T) { + // \b == /u0008 -> backspace + content := []byte("test\b text") + + if !reflect.DeepEqual(filterIllegalChars(content), []byte("test text")) { + t.Fatal("illegal character is not removed") + } +} diff --git a/testresult/xcresult3/action_invocation_record.go b/testresult/xcresult3/action_invocation_record.go new file mode 100644 index 00000000..7497a67a --- /dev/null +++ b/testresult/xcresult3/action_invocation_record.go @@ -0,0 +1,98 @@ +package xcresult3 + +import ( + "fmt" + "strings" + + "github.com/bitrise-io/go-steputils/v2/testreport" +) + +// ActionsInvocationRecord ... +type ActionsInvocationRecord struct { + Actions struct { + Values []struct { + ActionResult struct { + TestsRef struct { + ID struct { + Value string `json:"_value"` + } `json:"id"` + } `json:"testsRef"` + } `json:"actionResult"` + } `json:"_values"` + } `json:"actions"` + + Issues Issues `json:"issues"` +} + +// Issues ... +type Issues struct { + TestFailureSummaries TestFailureSummaries `json:"testFailureSummaries"` +} + +// TestFailureSummaries ... +type TestFailureSummaries struct { + Values []TestFailureSummary `json:"_values"` +} + +// TestFailureSummary ... +type TestFailureSummary struct { + DocumentLocationInCreatingWorkspace DocumentLocationInCreatingWorkspace `json:"documentLocationInCreatingWorkspace"` + Message Message `json:"message"` + ProducingTarget ProducingTarget `json:"producingTarget"` + TestCaseName TestCaseName `json:"testCaseName"` +} + +// URL ... +type URL struct { + Value string `json:"_value"` +} + +// DocumentLocationInCreatingWorkspace ... +type DocumentLocationInCreatingWorkspace struct { + URL URL `json:"url"` +} + +// ProducingTarget ... +type ProducingTarget struct { + Value string `json:"_value"` +} + +// TestCaseName ... +type TestCaseName struct { + Value string `json:"_value"` +} + +// Message ... +type Message struct { + Value string `json:"_value"` +} + +func testCaseMatching(test ActionTestSummaryGroup, testCaseName string) bool { + class, method := test.references() + + return testCaseName == class+"."+method || + testCaseName == fmt.Sprintf("-[%s %s]", class, method) +} + +// failure returns the ActionTestSummaryGroup's failure reason from the ActionsInvocationRecord. +func (r ActionsInvocationRecord) failure(test ActionTestSummaryGroup, testSuite testreport.TestSuite) string { + for _, failureSummary := range r.Issues.TestFailureSummaries.Values { + if failureSummary.ProducingTarget.Value == testSuite.Name && testCaseMatching(test, failureSummary.TestCaseName.Value) { + file, line := failureSummary.fileAndLineNumber() + return fmt.Sprintf("%s:%s - %s", file, line, failureSummary.Message.Value) + } + } + return "" +} + +// fileAndLineNumber unwraps the file path and line number descriptor from a given ActionTestSummaryGroup's. +func (s TestFailureSummary) fileAndLineNumber() (file string, line string) { + // file:\/\/\/Users\/bitrisedeveloper\/Develop\/ios\/Xcode11Test\/Xcode11TestUITests\/Xcode11TestUITests.swift#CharacterRangeLen=0&EndingLineNumber=42&StartingLineNumber=42 + if s.DocumentLocationInCreatingWorkspace.URL.Value != "" { + i := strings.LastIndex(s.DocumentLocationInCreatingWorkspace.URL.Value, "#") + if i > -1 && i+1 < len(s.DocumentLocationInCreatingWorkspace.URL.Value) { + return s.DocumentLocationInCreatingWorkspace.URL.Value[:i], s.DocumentLocationInCreatingWorkspace.URL.Value[i+1:] + } + } + return +} diff --git a/testresult/xcresult3/action_invocation_record_test.go b/testresult/xcresult3/action_invocation_record_test.go new file mode 100644 index 00000000..a9ac4509 --- /dev/null +++ b/testresult/xcresult3/action_invocation_record_test.go @@ -0,0 +1,124 @@ +package xcresult3 + +import ( + "testing" + + "github.com/bitrise-io/go-steputils/v2/testreport" +) + +func TestTestFailureSummary_fileAndLineNumber(t *testing.T) { + tests := []struct { + name string + summary TestFailureSummary + wantFile string + wantLine string + }{ + { + name: "", + summary: TestFailureSummary{ + DocumentLocationInCreatingWorkspace: DocumentLocationInCreatingWorkspace{ + URL: URL{Value: "file:/Xcode11TestUITests2.swift#CharacterRangeLen=0&EndingLineNumber=33&StartingLineNumber=33"}, + }, + }, + wantFile: "file:/Xcode11TestUITests2.swift", + wantLine: "CharacterRangeLen=0&EndingLineNumber=33&StartingLineNumber=33", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotFile, gotLine := tt.summary.fileAndLineNumber() + if gotFile != tt.wantFile { + t.Errorf("TestFailureSummary.fileAndLineNumber() gotFile = %v, want %v", gotFile, tt.wantFile) + } + if gotLine != tt.wantLine { + t.Errorf("TestFailureSummary.fileAndLineNumber() gotLine = %v, want %v", gotLine, tt.wantLine) + } + }) + } +} + +func TestActionsInvocationRecord_failure(t *testing.T) { + tests := []struct { + name string + record ActionsInvocationRecord + test ActionTestSummaryGroup + want string + }{ + { + name: "Simple test", + record: ActionsInvocationRecord{ + Issues: Issues{ + TestFailureSummaries: TestFailureSummaries{ + Values: []TestFailureSummary{ + TestFailureSummary{ + ProducingTarget: ProducingTarget{Value: "Xcode11TestUITests2"}, + TestCaseName: TestCaseName{Value: "Xcode11TestUITests2.testFail()"}, + Message: Message{Value: "XCTAssertEqual failed: (\"1\") is not equal to (\"0\")"}, + DocumentLocationInCreatingWorkspace: DocumentLocationInCreatingWorkspace{ + URL: URL{Value: "file:/Xcode11TestUITests2.swift#CharacterRangeLen=0&EndingLineNumber=33&StartingLineNumber=33"}, + }, + }, + }, + }, + }, + }, + test: ActionTestSummaryGroup{ + Identifier: Identifier{Value: "Xcode11TestUITests2/testFail()"}, + }, + want: `file:/Xcode11TestUITests2.swift:CharacterRangeLen=0&EndingLineNumber=33&StartingLineNumber=33 - XCTAssertEqual failed: ("1") is not equal to ("0")`, + }, + { + name: "class inherited test", + record: ActionsInvocationRecord{ + Issues: Issues{ + TestFailureSummaries: TestFailureSummaries{ + Values: []TestFailureSummary{ + TestFailureSummary{ + ProducingTarget: ProducingTarget{Value: "Xcode11TestUITests2"}, + TestCaseName: TestCaseName{Value: "SomethingDifferentClass.testFail()"}, + Message: Message{Value: "XCTAssertEqual failed: (\"1\") is not equal to (\"0\")"}, + DocumentLocationInCreatingWorkspace: DocumentLocationInCreatingWorkspace{ + URL: URL{Value: "file:/Xcode11TestUITests2.swift#CharacterRangeLen=0&EndingLineNumber=33&StartingLineNumber=33"}, + }, + }, + }, + }, + }, + }, + test: ActionTestSummaryGroup{ + Identifier: Identifier{Value: "SomethingDifferentClass/testFail()"}, + }, + want: `file:/Xcode11TestUITests2.swift:CharacterRangeLen=0&EndingLineNumber=33&StartingLineNumber=33 - XCTAssertEqual failed: ("1") is not equal to ("0")`, + }, + { + name: "inner class test", + record: ActionsInvocationRecord{ + Issues: Issues{ + TestFailureSummaries: TestFailureSummaries{ + Values: []TestFailureSummary{ + TestFailureSummary{ + ProducingTarget: ProducingTarget{Value: "Xcode11TestUITests2"}, + TestCaseName: TestCaseName{Value: "-[SomethingDifferentClass testFail]"}, + Message: Message{Value: "XCTAssertEqual failed: (\"1\") is not equal to (\"0\")"}, + DocumentLocationInCreatingWorkspace: DocumentLocationInCreatingWorkspace{ + URL: URL{Value: "file:/Xcode11TestUITests2.swift#CharacterRangeLen=0&EndingLineNumber=33&StartingLineNumber=33"}, + }, + }, + }, + }, + }, + }, + test: ActionTestSummaryGroup{ + Identifier: Identifier{Value: "SomethingDifferentClass/testFail"}, + }, + want: `file:/Xcode11TestUITests2.swift:CharacterRangeLen=0&EndingLineNumber=33&StartingLineNumber=33 - XCTAssertEqual failed: ("1") is not equal to ("0")`, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := tt.record.failure(tt.test, testreport.TestSuite{Name: "Xcode11TestUITests2"}); got != tt.want { + t.Errorf("ActionsInvocationRecord.failure() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/testresult/xcresult3/action_test_plan_summary.go b/testresult/xcresult3/action_test_plan_summary.go new file mode 100644 index 00000000..8f1808c0 --- /dev/null +++ b/testresult/xcresult3/action_test_plan_summary.go @@ -0,0 +1,100 @@ +package xcresult3 + +import "strconv" + +// ActionTestPlanRunSummaries ... +type ActionTestPlanRunSummaries struct { + Summaries Summaries `json:"summaries"` +} + +// Summaries ... +type Summaries struct { + Values []Summary `json:"_values"` +} + +// Summary ... +type Summary struct { + TestableSummaries TestableSummaries `json:"testableSummaries"` +} + +// TestableSummaries ... +type TestableSummaries struct { + Values []ActionTestableSummary `json:"_values"` +} + +// ActionTestableSummary ... +type ActionTestableSummary struct { + Name Name `json:"name"` + Tests Tests `json:"tests"` +} + +// Tests ... +type Tests struct { + Values []ActionTestSummaryGroup `json:"_values"` +} + +// Name ... +type Name struct { + Value string `json:"_value"` +} + +// tests returns ActionTestSummaryGroup mapped by the container TestableSummary name. +func (s ActionTestPlanRunSummaries) tests() ([]string, map[string][]ActionTestSummaryGroup) { + summaryGroupsByName := map[string][]ActionTestSummaryGroup{} + + var testSuiteOrder []string + for _, summary := range s.Summaries.Values { + for _, testableSummary := range summary.TestableSummaries.Values { + // test suite + name := testableSummary.Name.Value + if _, found := summaryGroupsByName[name]; !found { + testSuiteOrder = append(testSuiteOrder, name) + } + + var tests []ActionTestSummaryGroup + for _, test := range testableSummary.Tests.Values { + tests = append(tests, test.testsWithStatus()...) + } + + summaryGroupsByName[name] = tests + } + } + + return testSuiteOrder, summaryGroupsByName +} + +func (s ActionTestPlanRunSummaries) failuresCount(testableSummaryName string) (failure int) { + _, testsByCase := s.tests() + tests := testsByCase[testableSummaryName] + for _, test := range tests { + if test.TestStatus.Value == "Failure" { + failure++ + } + } + return +} + +func (s ActionTestPlanRunSummaries) skippedCount(testableSummaryName string) (skipped int) { + _, testsByCase := s.tests() + tests := testsByCase[testableSummaryName] + for _, test := range tests { + if test.TestStatus.Value == "Skipped" { + skipped++ + } + } + return +} + +func (s ActionTestPlanRunSummaries) totalTime(testableSummaryName string) (time float64) { + _, testsByCase := s.tests() + tests := testsByCase[testableSummaryName] + for _, test := range tests { + if test.Duration.Value != "" { + d, err := strconv.ParseFloat(test.Duration.Value, 64) + if err == nil { + time += d + } + } + } + return +} diff --git a/testresult/xcresult3/action_test_plan_summary_test.go b/testresult/xcresult3/action_test_plan_summary_test.go new file mode 100644 index 00000000..fecd0fb5 --- /dev/null +++ b/testresult/xcresult3/action_test_plan_summary_test.go @@ -0,0 +1,198 @@ +package xcresult3 + +import ( + "fmt" + "reflect" + "testing" + + "github.com/bitrise-io/go-utils/pretty" +) + +func TestActionTestPlanRunSummaries_tests(t *testing.T) { + tests := []struct { + name string + summaries ActionTestPlanRunSummaries + want map[string][]ActionTestSummaryGroup + }{ + { + name: "single test with status", + summaries: ActionTestPlanRunSummaries{ + Summaries: Summaries{ + Values: []Summary{ + Summary{ + TestableSummaries: TestableSummaries{ + Values: []ActionTestableSummary{ + ActionTestableSummary{ + Name: Name{Value: "test case 1"}, + Tests: Tests{ + Values: []ActionTestSummaryGroup{ + ActionTestSummaryGroup{TestStatus: TestStatus{Value: "success"}}, + }, + }, + }, + }, + }, + }, + }, + }, + }, + want: map[string][]ActionTestSummaryGroup{ + "test case 1": []ActionTestSummaryGroup{ + ActionTestSummaryGroup{TestStatus: TestStatus{Value: "success"}}, + }, + }, + }, + { + name: "single test with status + subtests with status", + summaries: ActionTestPlanRunSummaries{ + Summaries: Summaries{ + Values: []Summary{ + Summary{ + TestableSummaries: TestableSummaries{ + Values: []ActionTestableSummary{ + ActionTestableSummary{ + Name: Name{Value: "test case 1"}, + Tests: Tests{ + Values: []ActionTestSummaryGroup{ + ActionTestSummaryGroup{TestStatus: TestStatus{Value: "success"}}, + }, + }, + }, + ActionTestableSummary{ + Name: Name{Value: "test case 2"}, + Tests: Tests{ + Values: []ActionTestSummaryGroup{ + ActionTestSummaryGroup{ + Subtests: Subtests{ + Values: []ActionTestSummaryGroup{ + ActionTestSummaryGroup{ + TestStatus: TestStatus{Value: "success"}, + }, + ActionTestSummaryGroup{ + TestStatus: TestStatus{Value: "success"}, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + want: map[string][]ActionTestSummaryGroup{ + "test case 1": []ActionTestSummaryGroup{ + ActionTestSummaryGroup{TestStatus: TestStatus{Value: "success"}}, + }, + "test case 2": []ActionTestSummaryGroup{ + ActionTestSummaryGroup{TestStatus: TestStatus{Value: "success"}}, + ActionTestSummaryGroup{TestStatus: TestStatus{Value: "success"}}, + }, + }, + }, + { + name: "no test with status", + summaries: ActionTestPlanRunSummaries{}, + want: map[string][]ActionTestSummaryGroup{}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if _, got := tt.summaries.tests(); !reflect.DeepEqual(got, tt.want) { + fmt.Println("want: ", pretty.Object(tt.want)) + fmt.Println("got: ", pretty.Object(got)) + t.Errorf("ActionTestPlanRunSummaries.tests() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestActionTestPlanRunSummaries_failuresCount(t *testing.T) { + tests := []struct { + name string + summaries ActionTestPlanRunSummaries + testableSummaryName string + wantFailure int + }{ + { + name: "single failure", + summaries: ActionTestPlanRunSummaries{ + Summaries: Summaries{ + Values: []Summary{ + Summary{ + TestableSummaries: TestableSummaries{ + Values: []ActionTestableSummary{ + ActionTestableSummary{ + Name: Name{Value: "test case"}, + Tests: Tests{ + Values: []ActionTestSummaryGroup{ + ActionTestSummaryGroup{TestStatus: TestStatus{Value: "Failure"}}, + }, + }, + }, + }, + }, + }, + }, + }, + }, + testableSummaryName: "test case", + wantFailure: 1, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if gotFailure := tt.summaries.failuresCount(tt.testableSummaryName); gotFailure != tt.wantFailure { + t.Errorf("ActionTestPlanRunSummaries.failuresCount() = %v, want %v", gotFailure, tt.wantFailure) + } + }) + } +} + +func TestActionTestPlanRunSummaries_totalTime(t *testing.T) { + tests := []struct { + name string + summaries ActionTestPlanRunSummaries + testableSummaryName string + wantTime float64 + }{ + { + name: "single test", + summaries: ActionTestPlanRunSummaries{ + Summaries: Summaries{ + Values: []Summary{ + Summary{ + TestableSummaries: TestableSummaries{ + Values: []ActionTestableSummary{ + ActionTestableSummary{ + Name: Name{Value: "test case"}, + Tests: Tests{ + Values: []ActionTestSummaryGroup{ + ActionTestSummaryGroup{ + Duration: Duration{Value: "10"}, + TestStatus: TestStatus{Value: "Failure"}, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + testableSummaryName: "test case", + wantTime: 10, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if gotTime := tt.summaries.totalTime(tt.testableSummaryName); gotTime != tt.wantTime { + t.Errorf("ActionTestPlanRunSummaries.totalTime() = %v, want %v", gotTime, tt.wantTime) + } + }) + } +} diff --git a/testresult/xcresult3/action_test_summary.go b/testresult/xcresult3/action_test_summary.go new file mode 100644 index 00000000..8745d56d --- /dev/null +++ b/testresult/xcresult3/action_test_summary.go @@ -0,0 +1,78 @@ +package xcresult3 + +import ( + "crypto/md5" + "encoding/hex" +) + +// Attachment ... +type Attachment struct { + Filename struct { + Value string `json:"_value"` + } `json:"filename"` + + PayloadRef struct { + ID struct { + Value string `json:"_value"` + } + } `json:"payloadRef"` +} + +// Attachments ... +type Attachments struct { + Values []Attachment `json:"_values"` +} + +// ActionTestActivitySummary ... +type ActionTestActivitySummary struct { + Attachments Attachments `json:"attachments"` +} + +// ActivitySummaries ... +type ActivitySummaries struct { + Values []ActionTestActivitySummary `json:"_values"` +} + +// ActionTestFailureSummary ... +type ActionTestFailureSummary struct { + Message struct { + Value string `json:"_value"` + } `json:"message"` + + FileName struct { + Value string `json:"_value"` + } `json:"fileName"` + + LineNumber struct { + Value string `json:"_value"` + } `json:"lineNumber"` +} + +// FailureSummaries ... +type FailureSummaries struct { + Values []ActionTestFailureSummary `json:"_values"` +} + +// Configuration ... +type Configuration struct { + Hash string +} + +// UnmarshalJSON ... +func (c *Configuration) UnmarshalJSON(data []byte) error { + if string(data) == "null" || string(data) == `""` { + return nil + } + + hash := md5.Sum(data) + c.Hash = hex.EncodeToString(hash[:]) + + return nil +} + +// ActionTestSummary ... +type ActionTestSummary struct { + ActivitySummaries ActivitySummaries `json:"activitySummaries"` + FailureSummaries FailureSummaries `json:"failureSummaries"` + Configuration Configuration `json:"configuration"` +} diff --git a/testresult/xcresult3/action_test_summary_group.go b/testresult/xcresult3/action_test_summary_group.go new file mode 100644 index 00000000..e2900c08 --- /dev/null +++ b/testresult/xcresult3/action_test_summary_group.go @@ -0,0 +1,122 @@ +package xcresult3 + +import ( + "errors" + "fmt" + "path/filepath" + "strings" +) + +// ErrSummaryNotFound ... +var ErrSummaryNotFound = errors.New("no summaryRef.ID.Value found for test case") + +// ActionTestSummaryGroup ... +type ActionTestSummaryGroup struct { + Name Name `json:"name"` + Identifier Identifier `json:"identifier"` + Duration Duration `json:"duration"` + TestStatus TestStatus `json:"testStatus"` // only the inner-most tests will have a status, the ones which don't have "subtests" + SummaryRef SummaryRef `json:"summaryRef"` // only the inner-most tests will have a summaryRef, the ones which don't have "subtests" + Subtests Subtests `json:"subtests"` +} + +// Subtests ... +type Subtests struct { + Values []ActionTestSummaryGroup `json:"_values"` +} + +// ID ... +type ID struct { + Value string `json:"_value"` +} + +// SummaryRef ... +type SummaryRef struct { + ID ID `json:"id"` +} + +// TestStatus ... +type TestStatus struct { + Value string `json:"_value"` +} + +// Duration ... +type Duration struct { + Value string `json:"_value"` +} + +// Identifier ... +type Identifier struct { + Value string `json:"_value"` +} + +func (g ActionTestSummaryGroup) references() (class, method string) { + // Xcode11TestUITests2/testFail() + if g.Identifier.Value != "" { + s := strings.Split(g.Identifier.Value, "/") + if len(s) == 2 { + return s[0], s[1] + } + } + return +} + +// testsWithStatus returns ActionTestSummaryGroup with TestStatus defined. +func (g ActionTestSummaryGroup) testsWithStatus() (tests []ActionTestSummaryGroup) { + if g.TestStatus.Value != "" { + tests = append(tests, g) + } + + for _, subtest := range g.Subtests.Values { + tests = append(tests, subtest.testsWithStatus()...) + } + return +} + +// loadActionTestSummary ... +func (g ActionTestSummaryGroup) loadActionTestSummary(xcresultPath string, useLegacyFlag bool) (ActionTestSummary, error) { + if g.SummaryRef.ID.Value == "" { + return ActionTestSummary{}, ErrSummaryNotFound + } + + var summary ActionTestSummary + if err := xcresulttoolGet(xcresultPath, g.SummaryRef.ID.Value, useLegacyFlag, &summary); err != nil { + return ActionTestSummary{}, fmt.Errorf("failed to load ActionTestSummary: %w", err) + } + return summary, nil +} + +// exportScreenshots ... +func (g ActionTestSummaryGroup) exportScreenshots(resultPth, outputDir string, useLegacyFlag bool) error { + if g.TestStatus.Value == "" { + return nil + } + + if g.SummaryRef.ID.Value == "" { + return nil + } + + var summary ActionTestSummary + if err := xcresulttoolGet(resultPth, g.SummaryRef.ID.Value, useLegacyFlag, &summary); err != nil { + return err + } + + exported := map[string]bool{} + for _, summary := range summary.ActivitySummaries.Values { + for _, value := range summary.Attachments.Values { + if value.Filename.Value != "" && value.PayloadRef.ID.Value != "" { + if exported[value.PayloadRef.ID.Value] { + continue + } + + pth := filepath.Join(outputDir, value.Filename.Value) + if err := xcresulttoolExport(resultPth, value.PayloadRef.ID.Value, pth, useLegacyFlag); err != nil { + return err + } + exported[value.PayloadRef.ID.Value] = true + } + } + } + + return nil +} diff --git a/testresult/xcresult3/action_test_summary_group_test.go b/testresult/xcresult3/action_test_summary_group_test.go new file mode 100644 index 00000000..e11687b7 --- /dev/null +++ b/testresult/xcresult3/action_test_summary_group_test.go @@ -0,0 +1,85 @@ +package xcresult3 + +import ( + "reflect" + "testing" +) + +func TestActionTestSummaryGroup_references(t *testing.T) { + tests := []struct { + name string + identifier string + wantClass string + wantMethod string + }{ + { + name: "simple test", + identifier: "Xcode11TestUITests2/testFail()", + wantClass: "Xcode11TestUITests2", + wantMethod: "testFail()", + }, + { + name: "invalid format", + identifier: "Xcode11TestUITests2testFail()", + wantMethod: "", + wantClass: "", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := ActionTestSummaryGroup{} + g.Identifier.Value = tt.identifier + + gotClass, gotMethod := g.references() + if gotClass != tt.wantClass { + t.Errorf("ActionTestSummaryGroup.references() gotClass = %v, want %v", gotClass, tt.wantClass) + } + if gotMethod != tt.wantMethod { + t.Errorf("ActionTestSummaryGroup.references() gotMethod = %v, want %v", gotMethod, tt.wantMethod) + } + }) + } +} + +func TestActionTestSummaryGroup_testsWithStatus(t *testing.T) { + + tests := []struct { + name string + group ActionTestSummaryGroup + subtests []ActionTestSummaryGroup + wantGroups []ActionTestSummaryGroup + }{ + { + name: "status in the root ActionTestSummaryGroup", + group: ActionTestSummaryGroup{ + TestStatus: TestStatus{Value: "success"}, + }, + wantGroups: []ActionTestSummaryGroup{ActionTestSummaryGroup{TestStatus: TestStatus{Value: "success"}}}, + }, + { + name: "status in a sub ActionTestSummaryGroup", + group: ActionTestSummaryGroup{ + Subtests: Subtests{ + Values: []ActionTestSummaryGroup{ + ActionTestSummaryGroup{TestStatus: TestStatus{Value: "success"}}, + }, + }, + }, + wantGroups: []ActionTestSummaryGroup{ActionTestSummaryGroup{TestStatus: TestStatus{Value: "success"}}}, + }, + { + name: "no status", + group: ActionTestSummaryGroup{}, + wantGroups: nil, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotGroups := tt.group.testsWithStatus() + if !reflect.DeepEqual(gotGroups, tt.wantGroups) { + t.Errorf("ActionTestSummaryGroup.testsWithStatus() gotTarget = %v, want %v", gotGroups, tt.wantGroups) + } + }) + } +} diff --git a/testresult/xcresult3/converter.go b/testresult/xcresult3/converter.go new file mode 100644 index 00000000..3d81668b --- /dev/null +++ b/testresult/xcresult3/converter.go @@ -0,0 +1,503 @@ +package xcresult3 + +import ( + "encoding/json" + "errors" + "fmt" + "math" + "os" + "path/filepath" + "runtime" + "sort" + "strconv" + "strings" + "sync" + "time" + + "howett.net/plist" + + "github.com/bitrise-io/go-utils/fileutil" + "github.com/bitrise-io/go-utils/log" + "github.com/bitrise-io/go-utils/pathutil" + "github.com/bitrise-io/go-xcode/xcodeproject/serialized" + "github.com/bitrise-io/go-steputils/v2/testasset" + "github.com/bitrise-io/go-steputils/v2/testreport" + "github.com/bitrise-io/go-xcode/v2/testresult/xcresult3/model3" +) + +// Converter ... +type Converter struct { + xcresultPth string + useLegacyExtractionMethod bool +} + +func majorVersion(document serialized.Object) (int, error) { + version, err := document.Object("version") + if err != nil { + return -1, err + } + + major, err := version.Value("major") + if err != nil { + return -1, err + } + return int(major.(uint64)), nil +} + +func documentMajorVersion(pth string) (int, error) { + content, err := fileutil.ReadBytesFromFile(pth) + if err != nil { + return -1, err + } + + var info serialized.Object + if _, err := plist.Unmarshal(content, &info); err != nil { + return -1, err + } + + return majorVersion(info) +} + +func (c *Converter) Setup(useOldXCResultExtractionMethod bool) { + c.useLegacyExtractionMethod = useOldXCResultExtractionMethod +} + +// Detect ... +func (c *Converter) Detect(files []string) bool { + if !isXcresulttoolAvailable() { + log.Debugf("xcresult tool is not available") + return false + } + + for _, file := range files { + if filepath.Ext(file) != ".xcresult" { + continue + } + + infoPth := filepath.Join(file, "Info.plist") + if exist, err := pathutil.IsPathExists(infoPth); err != nil { + log.Debugf("Failed to find Info.plist at %s: %s", infoPth, err) + continue + } else if !exist { + log.Debugf("No Info.plist found at %s", infoPth) + continue + } + + version, err := documentMajorVersion(infoPth) + if err != nil { + log.Debugf("failed to get document version: %s", err) + continue + } + + if version < 3 { + log.Debugf("version < 3: %d", version) + continue + } + + c.xcresultPth = file + return true + } + return false +} + +// XML ... +func (c *Converter) Convert() (testreport.TestReport, error) { + supportsNewMethod, err := supportsNewExtractionMethods() + if err != nil { + return testreport.TestReport{}, err + } + + useLegacyFlag := c.useLegacyExtractionMethod + + if supportsNewMethod && !useLegacyFlag { + log.Infof("Using new extraction method") + + junitXml, err := parse(c.xcresultPth) + if err == nil { + return junitXml, nil + } + + log.Warnf(fmt.Sprintf("Failed to parse extraction method: %s", err)) + log.Warnf("Falling back to legacy extraction method") + + sendRemoteWarning("xcresult3-parsing", "error: %s", err) + + useLegacyFlag = true + } + + log.Infof("Using legacy extraction method") + + return legacyParse(c.xcresultPth, useLegacyFlag) +} + +func legacyParse(path string, useLegacyFlag bool) (testreport.TestReport, error) { + var ( + testResultDir = filepath.Dir(path) + maxParallel = runtime.NumCPU() * 2 + ) + + log.Debugf("Maximum parallelism: %d.", maxParallel) + + _, summaries, err := Parse(path, useLegacyFlag) + if err != nil { + return testreport.TestReport{}, err + } + + var xmlData testreport.TestReport + { + testSuiteCount := testSuiteCountInSummaries(summaries) + xmlData.TestSuites = make([]testreport.TestSuite, 0, testSuiteCount) + } + + summariesCount := len(summaries) + log.Debugf("Summaries Count: %d", summariesCount) + + for _, summary := range summaries { + testSuiteOrder, testsByName := summary.tests() + + for _, name := range testSuiteOrder { + tests := testsByName[name] + + testSuite, err := genTestSuite(name, summary, tests, testResultDir, path, maxParallel, useLegacyFlag) + if err != nil { + return testreport.TestReport{}, err + } + + xmlData.TestSuites = append(xmlData.TestSuites, testSuite) + } + } + + return xmlData, nil +} + +func parse(path string) (testreport.TestReport, error) { + results, err := ParseTestResults(path) + if err != nil { + return testreport.TestReport{}, err + } + + testSummary, warnings, err := model3.Convert(results) + if err != nil { + return testreport.TestReport{}, err + } + + if len(warnings) > 0 { + sendRemoteWarning("xcresults3-data", "warnings: %s", warnings) + } + + var xml testreport.TestReport + + for _, plan := range testSummary.TestPlans { + for _, testBundle := range plan.TestBundles { + xml.TestSuites = append(xml.TestSuites, parseTestBundle(testBundle)) + } + } + + outputPath := filepath.Dir(path) + + attachmentsMap, err := extractAttachments(path, outputPath) + if err != nil { + return testreport.TestReport{}, err + } + + xml, err = connectAttachmentsToTestCases(xml, attachmentsMap) + if err != nil { + return testreport.TestReport{}, err + } + + return xml, nil +} + +func parseTestBundle(testBundle model3.TestBundle) testreport.TestSuite { + failedCount := 0 + skippedCount := 0 + var totalDuration time.Duration + var tests []testreport.TestCase + + for _, testSuite := range testBundle.TestSuites { + for _, testCase := range testSuite.TestCases { + var testCasesToConvert []model3.TestCase + if len(testCase.Retries) > 0 { + testCasesToConvert = testCase.Retries + } else { + testCasesToConvert = []model3.TestCase{testCase.TestCase} + } + + for _, testCaseToConvert := range testCasesToConvert { + test := parseTestCase(testCaseToConvert) + + if test.Failure != nil { + failedCount++ + } else if test.Skipped != nil { + skippedCount++ + } + totalDuration += testCaseToConvert.Time + + tests = append(tests, test) + } + } + } + + return testreport.TestSuite{ + Name: testBundle.Name, + Tests: len(tests), + Failures: failedCount, + Skipped: skippedCount, + Time: totalDuration.Seconds(), + TestCases: tests, + } +} + +func parseTestCase(testCase model3.TestCase) testreport.TestCase { + test := testreport.TestCase{ + Name: testCase.Name, + ClassName: testCase.ClassName, + Time: testCase.Time.Seconds(), + } + + if testCase.Result == model3.TestResultFailed { + test.Failure = &testreport.Failure{Value: testCase.Message} + } else if testCase.Result == model3.TestResultSkipped { + test.Skipped = &testreport.Skipped{} + } + + return test +} + +func extractAttachments(xcresultPath, outputPath string) (map[string][]string, error) { + var attachmentsMap = make(map[string][]string) + + if err := xcresulttoolExport(xcresultPath, "", outputPath, false); err != nil { + return nil, err + } + + manifestPath := filepath.Join(outputPath, "manifest.json") + bytes, err := os.ReadFile(manifestPath) + if err != nil { + return nil, err + } + + var manifest []model3.TestAttachmentDetails + if err := json.Unmarshal(bytes, &manifest); err != nil { + return nil, err + } + + for _, attachmentDetail := range manifest { + attachments := attachmentDetail.Attachments + + sort.Slice(attachments, func(i, j int) bool { + return time.Time(attachments[i].Timestamp).Before(time.Time(attachments[j].Timestamp)) + }) + + for _, attachment := range attachments { + oldPath := filepath.Join(outputPath, attachment.ExportedFileName) + newFilename := createUniqueFilename(attachment) + newPath := filepath.Join(outputPath, newFilename) + + if err := os.Rename(oldPath, newPath); err != nil { + // It is not a critical error if the rename fails because the file will be still exported just by its + // unique ID. + log.Warnf("Failed to rename %s to %s", oldPath, newPath) + } + + if !testasset.IsSupportedAssetType(newPath) { + continue + } + + testIdentifier := appendRepetitionToTestIdentifier(attachmentDetail.TestIdentifier, attachment.RepetitionNumber) + attachmentsMap[testIdentifier] = append(attachmentsMap[testIdentifier], filepath.Base(newPath)) + } + } + + if err := os.Remove(manifestPath); err != nil { + log.Warnf("Failed to remove manifest file %s: %s", manifestPath, err) + } + + return attachmentsMap, nil +} + +// Create unique filename using timestamp as suffix +func createUniqueFilename(attachment model3.Attachment) string { + timestamp := time.Time(attachment.Timestamp).UnixNano() + + originalName := attachment.SuggestedHumanReadableName + ext := filepath.Ext(originalName) + nameWithoutExt := strings.TrimSuffix(originalName, ext) + + // Format: originalname_timestamp.ext + return fmt.Sprintf("%s_%d%s", nameWithoutExt, timestamp, ext) +} + +func stripTrailingParentheses(s string) string { + return strings.TrimSuffix(s, "()") +} + +func buildTestIdentifier(className, testName string) string { + return className + "/" + testName +} + +func appendRepetitionToTestIdentifier(testIdentifier string, repetition int) string { + // Non-retried tests have an empty repetition, but later we treat them as a test with a repetition of 1. + // So we need to ensure that the repetition is at least 1. + value := int(math.Max(1, float64(repetition))) + return fmt.Sprintf("%s (%d)", stripTrailingParentheses(testIdentifier), value) +} + +func connectAttachmentsToTestCases(xml testreport.TestReport, attachmentsMap map[string][]string) (testreport.TestReport, error) { + for i := range xml.TestSuites { + var testRepetitionMap = make(map[string]int) + + for j := range xml.TestSuites[i].TestCases { + testCase := &xml.TestSuites[i].TestCases[j] + testIdentifier := buildTestIdentifier(testCase.ClassName, testCase.Name) + + // If the test case has a repetition, we need to append it to the test identifier + // and keep track of how many times we have seen this test identifier. + if count, exists := testRepetitionMap[testIdentifier]; exists { + testRepetitionMap[testIdentifier] = count + 1 + } else { + testRepetitionMap[testIdentifier] = 1 + } + + testIdentifier = appendRepetitionToTestIdentifier(testIdentifier, testRepetitionMap[testIdentifier]) + + // Add attachments if any exist for this test and repetition + if attachments, exists := attachmentsMap[testIdentifier]; exists { + if testCase.Properties == nil { + testCase.Properties = &testreport.Properties{ + Property: []testreport.Property{}, + } + } + + // Add each attachment as a property + for i, fileName := range attachments { + testCase.Properties.Property = append( + testCase.Properties.Property, + testreport.Property{Name: fmt.Sprintf("attachment_%d", i), Value: fileName}, + ) + } + } + } + } + + return xml, nil +} + +func testSuiteCountInSummaries(summaries []ActionTestPlanRunSummaries) int { + testSuiteCount := 0 + for _, summary := range summaries { + testSuiteOrder, _ := summary.tests() + testSuiteCount += len(testSuiteOrder) + } + return testSuiteCount +} + +func genTestSuite(name string, + summary ActionTestPlanRunSummaries, + tests []ActionTestSummaryGroup, + testResultDir string, + xcresultPath string, + maxParallel int, + useLegacyFlag bool, +) (testreport.TestSuite, error) { + var ( + start = time.Now() + genTestSuiteErr error + wg sync.WaitGroup + mtx sync.RWMutex + ) + + testSuite := testreport.TestSuite{ + Name: name, + Tests: len(tests), + Failures: summary.failuresCount(name), + Skipped: summary.skippedCount(name), + Time: summary.totalTime(name), + TestCases: make([]testreport.TestCase, len(tests)), + } + + testIdx := 0 + for testIdx < len(tests) { + for i := 0; i < maxParallel && testIdx < len(tests); i++ { + test := tests[testIdx] + wg.Add(1) + + go func(test ActionTestSummaryGroup, testIdx int) { + defer wg.Done() + + testCase, err := genTestCase(test, xcresultPath, testResultDir, useLegacyFlag) + if err != nil { + mtx.Lock() + genTestSuiteErr = err + mtx.Unlock() + } + + testSuite.TestCases[testIdx] = testCase + }(test, testIdx) + + testIdx++ + } + + wg.Wait() + } + + log.Debugf("Generating test suite [%s] (%d tests) - DONE %v", name, len(tests), time.Since(start)) + + return testSuite, genTestSuiteErr +} + +func genTestCase(test ActionTestSummaryGroup, xcresultPath, testResultDir string, useLegacyFlag bool) (testreport.TestCase, error) { + var duartion float64 + if test.Duration.Value != "" { + var err error + duartion, err = strconv.ParseFloat(test.Duration.Value, 64) + if err != nil { + return testreport.TestCase{}, err + } + } + + testSummary, err := test.loadActionTestSummary(xcresultPath, useLegacyFlag) + // Ignoring the SummaryNotFoundError error is on purpose because not having an action summary is a valid use case. + // For example, failed tests will always have a summary, but successful ones might have it or might not. + // If they do not have it, then that means that they did not log anything to the console, + // and they were not executed as device configuration tests. + if err != nil && !errors.Is(err, ErrSummaryNotFound) { + return testreport.TestCase{}, err + } + + var failure *testreport.Failure + var skipped *testreport.Skipped + switch test.TestStatus.Value { + case "Failure": + failureMessage := "" + for _, aTestFailureSummary := range testSummary.FailureSummaries.Values { + file := aTestFailureSummary.FileName.Value + line := aTestFailureSummary.LineNumber.Value + message := aTestFailureSummary.Message.Value + + if len(failureMessage) > 0 { + failureMessage += "\n" + } + failureMessage += fmt.Sprintf("%s:%s - %s", file, line, message) + } + + failure = &testreport.Failure{ + Value: failureMessage, + } + case "Skipped": + skipped = &testreport.Skipped{} + } + + if err := test.exportScreenshots(xcresultPath, testResultDir, useLegacyFlag); err != nil { + return testreport.TestCase{}, err + } + + return testreport.TestCase{ + Name: test.Name.Value, + ConfigurationHash: testSummary.Configuration.Hash, + ClassName: strings.Split(test.Identifier.Value, "/")[0], + Failure: failure, + Skipped: skipped, + Time: duartion, + }, nil +} diff --git a/testresult/xcresult3/converter_test.go b/testresult/xcresult3/converter_test.go new file mode 100644 index 00000000..0b5faabc --- /dev/null +++ b/testresult/xcresult3/converter_test.go @@ -0,0 +1,263 @@ +package xcresult3 + +import ( + "fmt" + "os" + "path/filepath" + "testing" + + "github.com/bitrise-io/go-steputils/v2/testreport" + "github.com/stretchr/testify/require" + + "github.com/bitrise-io/go-utils/command" + "github.com/bitrise-io/go-utils/pathutil" +) + +// copyTestdataToDir ... +// To populate the _tmp dir with sample data +// run `bitrise run download_sample_artifacts` before running tests here, +// which will download https://github.com/bitrise-io/sample-artifacts +// into the _tmp dir. +func copyTestdataToDir(pathInTestdataDir, dirPathToCopyInto string) (string, error) { + err := command.CopyDir( + filepath.Join("../../../_tmp/", pathInTestdataDir), + dirPathToCopyInto, + true, + ) + return dirPathToCopyInto, err +} + +func TestConverter_XML(t *testing.T) { + t.Run("xcresult3-flaky-with-rerun.xcresult", func(t *testing.T) { + fileName := "xcresult3-flaky-with-rerun.xcresult" + rootDir, xcresultPath, err := setupTestData(fileName) + require.NoError(t, err) + + defer func() { + require.NoError(t, os.RemoveAll(rootDir)) + }() + + t.Log("tempTestdataDir: ", rootDir) + + c := Converter{ + xcresultPth: xcresultPath, + } + junitXML, err := c.Convert() + require.NoError(t, err) + require.Equal(t, []testreport.TestSuite{ + { + Name: "BullsEyeTests", Tests: 5, Failures: 0, Skipped: 0, Time: 0.9774, + TestCases: []testreport.TestCase{ + { + Name: "testStartNewRoundUsesRandomValueFromApiRequest()", ClassName: "BullsEyeFakeTests", + Time: 0.014, + }, + { + Name: "testGameStyleCanBeChanged()", ClassName: "BullsEyeMockTests", + Time: 0.0093, + }, + { + Name: "testScoreIsComputedPerformance()", ClassName: "BullsEyeTests", + Time: 0.74, + }, + { + Name: "testScoreIsComputedWhenGuessIsHigherThanTarget()", ClassName: "BullsEyeTests", + Time: 0.0041, + }, + { + Name: "testScoreIsComputedWhenGuessIsLowerThanTarget()", ClassName: "BullsEyeTests", + Time: 0.21, + }, + }, + }, + { + Name: "BullsEyeSlowTests", Tests: 2, Failures: 0, Skipped: 0, Time: 0.53, + TestCases: []testreport.TestCase{ + { + Name: "testApiCallCompletes()", ClassName: "BullsEyeSlowTests", + Time: 0.28, + }, + { + Name: "testValidApiCallGetsHTTPStatusCode200()", ClassName: "BullsEyeSlowTests", + Time: 0.25, + }, + }, + }, + { + Name: "BullsEyeUITests", Tests: 1, Failures: 0, Skipped: 0, Time: 9, + TestCases: []testreport.TestCase{ + { + Name: "testGameStyleSwitch()", ClassName: "BullsEyeUITests", + Time: 9, + Properties: &testreport.Properties{ + Property: []testreport.Property{ + { + Name: "attachment_0", + Value: "Screenshot 2022-02-10 at 02.57.39 PM_1644505059194999933.jpeg", + }, + { + Name: "attachment_1", + Value: "Screenshot 2022-02-10 at 02.57.39 PM_1644505059388999938.jpeg", + }, + { + Name: "attachment_2", + Value: "Screenshot 2022-02-10 at 02.57.44 PM_1644505064670000076.jpeg", + }, + { + Name: "attachment_3", + Value: "Screenshot 2022-02-10 at 02.57.47 PM_1644505067144000053.jpeg", + }, + { + Name: "attachment_4", + Value: "Screenshot 2022-02-10 at 02.57.47 PM_1644505067476000070.jpeg", + }, + { + Name: "attachment_5", + Value: "Screenshot 2022-02-10 at 02.57.47 PM_1644505067992000102.jpeg", + }, + }, + }, + }, + }, + }, + { + Name: "BullsEyeFlakyTests", Tests: 3, Failures: 1, Skipped: 1, Time: 0.226, + TestCases: []testreport.TestCase{ + { + Name: "testFlakyFeature()", ClassName: "BullsEyeFlakyTests", Time: 0.2, + Failure: &testreport.Failure{ + Value: `BullsEyeFlakyTests.swift:43: XCTAssertEqual failed: ("1") is not equal to ("0") - Number is not even`, + }, + }, + { + Name: "testFlakyFeature()", ClassName: "BullsEyeFlakyTests", Time: 0.006, + }, + { + Name: "testFlakySkip()", ClassName: "BullsEyeSkippedTests", Time: 0.02, + Skipped: &testreport.Skipped{}, + }, + }, + }, + }, junitXML.TestSuites) + }) + + t.Run("xcresults3 success-failed-skipped-tests.xcresult", func(t *testing.T) { + fileName := "xcresult3-success-failed-skipped-tests.xcresult" + rootDir, xcresultPath, err := setupTestData(fileName) + require.NoError(t, err) + + defer func() { + require.NoError(t, os.RemoveAll(rootDir)) + }() + + t.Log("tempTestdataDir: ", rootDir) + + c := Converter{ + xcresultPth: xcresultPath, + } + junitXML, err := c.Convert() + require.NoError(t, err) + require.Equal(t, []testreport.TestSuite{ + { + Name: "testProjectUITests", + Tests: 3, + Failures: 1, + Skipped: 1, + Time: 0.435, + TestCases: []testreport.TestCase{ + { + Name: "testFailure()", + ClassName: "testProjectUITests", + Time: 0.26, + Failure: &testreport.Failure{ + Value: "testProjectUITests.swift:30: XCTAssertTrue failed", + }, + Properties: &testreport.Properties{ + Property: []testreport.Property{ + { + Name: "attachment_0", + Value: "Screenshot 2021-02-09 at 08.35.51 AM_1612859751989000082.jpeg", + }, + { + Name: "attachment_1", + Value: "Screenshot 2021-02-09 at 08.35.52 AM_1612859752052999973.jpeg", + }, + { + Name: "attachment_2", + Value: "Screenshot 2021-02-09 at 08.35.52 AM_1612859752052999973.jpeg", + }, + }, + }, + }, + { + Name: "testSkip()", + ClassName: "testProjectUITests", + Time: 0.086, + Skipped: &testreport.Skipped{}, + }, + { + Name: "testSuccess()", + ClassName: "testProjectUITests", + Time: 0.089, + }, + }, + }, + }, junitXML.TestSuites) + }) + + t.Run("xcresult3-multiple-test-plan-configurations.xcresult", func(t *testing.T) { + fileName := "xcresult3-multiple-test-plan-configurations.xcresult" + rootDir, xcresultPath, err := setupTestData(fileName) + require.NoError(t, err) + require.NotEmpty(t, rootDir) + require.NotEmpty(t, xcresultPath) + + t.Log("tempTestdataDir: ", rootDir) + + c := Converter{xcresultPth: xcresultPath} + junitXML, err := c.Convert() + require.NoError(t, err) + require.NotNil(t, junitXML) + + require.EqualValues( + t, + junitXML.TestSuites[0].TestCases[0].Failure.Value, + `English: swift_testingTests.swift:20: Expectation failed: true == false - // This test is intended to fail to demonstrate test failure reporting. +German: swift_testingTests.swift:20: Expectation failed: true == false - // This test is intended to fail to demonstrate test failure reporting. +`, + ) + + }) +} + +func BenchmarkConverter_XML(b *testing.B) { + fileName := "xcresult3-flaky-with-rerun.xcresult" + rootDir, xcresultPath, err := setupTestData(fileName) + require.NoError(b, err) + + defer func() { + require.NoError(b, os.RemoveAll(rootDir)) + }() + + b.Log("tempTestdataDir: ", rootDir) + + c := Converter{ + xcresultPth: xcresultPath, + } + _, err = c.Convert() + require.NoError(b, err) +} + +func setupTestData(fileName string) (string, string, error) { + tempTestdataDir, err := pathutil.NormalizedOSTempDirPath("test") + if err != nil { + return "", "", fmt.Errorf("failed to create temp dir: %w", err) + } + + tempXCResultPath, err := copyTestdataToDir(fmt.Sprintf("./xcresults/%s", fileName), filepath.Join(tempTestdataDir, fileName)) + if err != nil { + return "", "", err + } + + return tempTestdataDir, tempXCResultPath, nil +} diff --git a/testresult/xcresult3/logging.go b/testresult/xcresult3/logging.go new file mode 100644 index 00000000..c373dad2 --- /dev/null +++ b/testresult/xcresult3/logging.go @@ -0,0 +1,10 @@ +package xcresult3 + +import "github.com/bitrise-io/go-utils/log" + +func sendRemoteWarning(tag string, format string, v ...interface{}) { + data := map[string]interface{}{} + data["source"] = "deploy-to-bitrise-io" + + log.RWarnf("deploy-to-bitrise-io", tag, data, format, v...) +} diff --git a/testresult/xcresult3/model3/conversion.go b/testresult/xcresult3/model3/conversion.go new file mode 100644 index 00000000..67389f59 --- /dev/null +++ b/testresult/xcresult3/model3/conversion.go @@ -0,0 +1,217 @@ +package model3 + +import ( + "fmt" + "strings" + "time" +) + +func Convert(data *TestData) (*TestSummary, []string, error) { + var warnings []string + summary := TestSummary{} + + for _, testPlanNode := range data.TestNodes { + if testPlanNode.Type != TestNodeTypeTestPlan { + return nil, warnings, fmt.Errorf("test plan expected but got: %s", testPlanNode.Type) + } + + testPlan := TestPlan{Name: testPlanNode.Name} + + for _, testBundleNode := range testPlanNode.Children { + if testBundleNode.Type != TestNodeTypeUnitTestBundle && testBundleNode.Type != TestNodeTypeUITestBundle { + return nil, warnings, fmt.Errorf("test bundle expected but got: %s", testBundleNode.Type) + } + + testBundle := TestBundle{Name: testBundleNode.Name} + + for _, testSuiteNode := range testBundleNode.Children { + var name string + var testNodes []TestNode + + if testSuiteNode.Type == TestNodeTypeTestCase { + name = testBundleNode.Name + testNodes = []TestNode{testSuiteNode} + } else if testSuiteNode.Type == TestNodeTypeTestSuite { + name = testSuiteNode.Name + testNodes = testSuiteNode.Children + } else { + return nil, warnings, fmt.Errorf("test suite or test case expected but got: %s", testSuiteNode.Type) + } + + testSuite := TestSuite{Name: name} + + testCases, testCaseWarnings, err := extractTestCases(testNodes, name) + warnings = append(warnings, testCaseWarnings...) + + if err != nil { + return nil, warnings, err + } + + testSuite.TestCases = testCases + testBundle.TestSuites = append(testBundle.TestSuites, testSuite) + } + + testPlan.TestBundles = append(testPlan.TestBundles, testBundle) + } + + summary.TestPlans = append(summary.TestPlans, testPlan) + } + + return &summary, warnings, nil +} + +func extractTestCases(nodes []TestNode, fallbackName string) ([]TestCaseWithRetries, []string, error) { + var testCases []TestCaseWithRetries + var warnings []string + + for _, testCaseNode := range nodes { + // A customer's xcresult file contained this use case where a test suite is a child of a test suite. + if testCaseNode.Type == TestNodeTypeTestSuite { + nestedTestCases, nestedWarnings, err := extractTestCases(testCaseNode.Children, fallbackName) + warnings = append(warnings, nestedWarnings...) + + if err != nil { + return nil, warnings, err + } + + testCases = append(testCases, nestedTestCases...) + + continue + } + + if testCaseNode.Type != TestNodeTypeTestCase { + return nil, warnings, fmt.Errorf("test case expected but got: %s", testCaseNode.Type) + } + + testCase, testCaseWarnings := extractTestCase(testCaseNode, "", "", fallbackName) + warnings = append(warnings, testCaseWarnings...) + + retries, retryWarnings, err := extractRetries(testCaseNode, fallbackName) + if err != nil { + return nil, warnings, err + } + warnings = append(warnings, retryWarnings...) + + testCases = append(testCases, TestCaseWithRetries{ + TestCase: testCase, + Retries: retries, + }) + } + + return testCases, warnings, nil +} + +func extractDuration(text string) time.Duration { + // Duration is in the format "123.456789s", "123,456789s" or "4m 34s", so we need to do some normalization. We + // need to replace the comma with a dot and remove the space to be able to parse it with time.ParseDuration. + normalized := strings.ReplaceAll(text, ",", ".") + normalized = strings.ReplaceAll(normalized, " ", "") + + duration, err := time.ParseDuration(normalized) + if err != nil { + return 0 + } + + return duration +} + +func extractFailureMessageFromTestPlanConfigs(testPlanConfigNodes []TestNode) (string, []string) { + var warnings []string + failureMessage := "" + + for _, testPlanConfigNode := range testPlanConfigNodes { + msg, wrns := extractFailureMessage(testPlanConfigNode) + if msg != "" { + failureMessage += fmt.Sprintf("%s: %s\n", testPlanConfigNode.Name, msg) + } + if wrns != nil { + warnings = append(warnings, wrns...) + } + } + + return failureMessage, warnings +} + +func extractFailureMessage(testNode TestNode) (string, []string) { + childrenCount := len(testNode.Children) + if childrenCount == 0 { + return "", nil + } + + lastChild := testNode.Children[childrenCount-1] + if lastChild.Type == TestNodeTypeRepetition { + return extractFailureMessage(lastChild) + } + if lastChild.Type == TestNodeTypeTestPlanConfig { + return extractFailureMessageFromTestPlanConfigs(testNode.Children) + } + + var warnings []string + failureMessage := "" + + for _, child := range testNode.Children { + if child.Type == TestNodeTypeFailureMessage { + // The failure message appears in the Name field and not in the Details field. + if child.Name == "" { + warnings = append(warnings, fmt.Sprintf("'%s' type has empty name field", child.Type)) + } + if child.Details != "" { + warnings = append(warnings, fmt.Sprintf("'%s' type has unexpected details field", child.Type)) + } + + failureMessage += child.Name + } + } + + return failureMessage, warnings +} + +func extractRetries(testNode TestNode, fallbackName string) ([]TestCase, []string, error) { + var retries []TestCase + var warnings []string + + for _, child := range testNode.Children { + if child.Type == TestNodeTypeRepetition { + // Use the parent test node's identifier, instead of the repetition's identifier (1, 2, ...). + retry, testCaseWarnings := extractTestCase(child, testNode.Identifier, testNode.Name, fallbackName) + warnings = append(warnings, testCaseWarnings...) + retries = append(retries, retry) + } + } + + return retries, warnings, nil +} + +func extractTestCase(testNode TestNode, customNodeIdentifier, customName, fallbackClassName string) (TestCase, []string) { + var warnings []string + + nodeIdentifier := testNode.Identifier + if customNodeIdentifier != "" { + nodeIdentifier = customNodeIdentifier + } + + name := testNode.Name + if customName != "" { + name = customName + } + + className := strings.Split(nodeIdentifier, "/")[0] + if className == "" { + // In rare cases the identifier is an empty string, so we need to use the test suite name which is the + // same as the first part of the identifier in normal cases. + className = fallbackClassName + } + + message, failureMessageWarnings := extractFailureMessage(testNode) + if len(failureMessageWarnings) > 0 { + warnings = append(warnings, failureMessageWarnings...) + } + + return TestCase{ + Name: name, + ClassName: className, + Time: extractDuration(testNode.Duration), + Result: testNode.Result, + Message: message, + }, warnings +} diff --git a/testresult/xcresult3/model3/conversion_test.go b/testresult/xcresult3/model3/conversion_test.go new file mode 100644 index 00000000..6dd54d9f --- /dev/null +++ b/testresult/xcresult3/model3/conversion_test.go @@ -0,0 +1,225 @@ +package model3 + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestConversion(t *testing.T) { + tests := []struct { + name string + data *TestData + want *TestSummary + }{ + { + name: "Simple case with multiple test bundles, suites, and cases", + data: &TestData{ + TestNodes: []TestNode{ + { + Type: TestNodeTypeTestPlan, + Name: "TP1", + Children: []TestNode{ + { + Type: TestNodeTypeUnitTestBundle, + Name: "TB1", + Children: []TestNode{ + { + Type: TestNodeTypeTestSuite, + Name: "TS1", + Children: []TestNode{ + { + Type: TestNodeTypeTestCase, + Name: "TC1", + Result: TestResultPassed, + Duration: "0.5s", + }, + { + Type: TestNodeTypeTestCase, + Name: "TC2", + Result: TestResultFailed, + Duration: "1m 11s", + }, + }, + }, + { + Type: TestNodeTypeTestSuite, + Name: "TS2", + Children: []TestNode{ + { + Type: TestNodeTypeTestCase, + Name: "TC3", + Result: TestResultSkipped, + Duration: "66s", + }, + { + Type: TestNodeTypeTestCase, + Name: "TC4", + Result: TestResultPassed, + Duration: "0.03s", + }, + }, + }, + }, + }, + { + Type: TestNodeTypeUITestBundle, + Name: "TB2", + Children: []TestNode{ + { + Type: TestNodeTypeTestSuite, + Name: "TS3", + Children: []TestNode{ + { + Type: TestNodeTypeTestCase, + Name: "TC5", + Result: TestResultPassed, + Duration: "15s", + }, + }, + }, + }, + }, + }, + }, + }, + }, + want: &TestSummary{ + TestPlans: []TestPlan{ + { + Name: "TP1", + TestBundles: []TestBundle{ + { + Name: "TB1", + TestSuites: []TestSuite{ + { + Name: "TS1", + TestCases: []TestCaseWithRetries{ + { + TestCase: TestCase{ + Name: "TC1", + ClassName: "TS1", + Time: 500 * time.Millisecond, + Result: "Passed", + }, + }, + { + TestCase: TestCase{ + Name: "TC2", + ClassName: "TS1", + Time: 71 * time.Second, + Result: "Failed", + }, + }, + }, + }, + { + Name: "TS2", + TestCases: []TestCaseWithRetries{ + { + TestCase: TestCase{ + Name: "TC3", + ClassName: "TS2", + Time: 66 * time.Second, + Result: "Skipped", + }, + }, + { + TestCase: TestCase{ + Name: "TC4", + ClassName: "TS2", + Time: 30 * time.Millisecond, + Result: "Passed", + }, + }, + }, + }, + }, + }, + { + Name: "TB2", + TestSuites: []TestSuite{ + { + Name: "TS3", + TestCases: []TestCaseWithRetries{ + { + TestCase: TestCase{ + Name: "TC5", + ClassName: "TS3", + Time: 15 * time.Second, + Result: "Passed", + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + { + name: "Test bundle without test suites", + data: &TestData{ + TestNodes: []TestNode{ + { + Type: TestNodeTypeTestPlan, + Name: "TP1", + Children: []TestNode{ + { + Type: TestNodeTypeUnitTestBundle, + Name: "TB1", + Children: []TestNode{ + { + Type: TestNodeTypeTestCase, + Name: "TC1", + Result: TestResultPassed, + Duration: "0.5s", + }, + }, + }, + }, + }, + }, + }, + want: &TestSummary{ + TestPlans: []TestPlan{ + { + Name: "TP1", + TestBundles: []TestBundle{ + { + Name: "TB1", + TestSuites: []TestSuite{ + { + Name: "TB1", + TestCases: []TestCaseWithRetries{ + { + TestCase: TestCase{ + Name: "TC1", + ClassName: "TB1", + Time: 500 * time.Millisecond, + Result: "Passed", + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, _, err := Convert(test.data) + require.NoError(t, err) + + require.Equal(t, test.want, got) + }) + } +} diff --git a/testresult/xcresult3/model3/data.go b/testresult/xcresult3/model3/data.go new file mode 100644 index 00000000..58a852ff --- /dev/null +++ b/testresult/xcresult3/model3/data.go @@ -0,0 +1,35 @@ +package model3 + +import "time" + +type TestSummary struct { + TestPlans []TestPlan +} + +type TestPlan struct { + Name string + TestBundles []TestBundle +} + +type TestBundle struct { + Name string + TestSuites []TestSuite +} + +type TestSuite struct { + Name string + TestCases []TestCaseWithRetries +} + +type TestCaseWithRetries struct { + TestCase + Retries []TestCase +} + +type TestCase struct { + Name string + ClassName string + Time time.Duration + Result TestResult + Message string +} diff --git a/testresult/xcresult3/model3/export.go b/testresult/xcresult3/model3/export.go new file mode 100644 index 00000000..b18d805b --- /dev/null +++ b/testresult/xcresult3/model3/export.go @@ -0,0 +1,38 @@ +package model3 + +import ( + "encoding/json" + "time" +) + +type TestAttachmentDetails struct { + TestIdentifier string `json:"testIdentifier"` + Attachments []Attachment `json:"attachments"` +} + +type Timestamp time.Time + +type Attachment struct { + ExportedFileName string `json:"exportedFileName"` + SuggestedHumanReadableName string `json:"suggestedHumanReadableName"` + IsAssociatedWithFailure bool `json:"isAssociatedWithFailure"` + Timestamp Timestamp `json:"timestamp"` + ConfigurationName string `json:"configurationName"` + DeviceName string `json:"deviceName"` + DeviceID string `json:"deviceId"` + RepetitionNumber int `json:"repetitionNumber"` +} + +func (t *Timestamp) UnmarshalJSON(b []byte) error { + var timestamp float64 + if err := json.Unmarshal(b, ×tamp); err != nil { + return err + } + + // Extract seconds and nanoseconds separately to preserve fractional part + seconds := int64(timestamp) + nanoseconds := int64((timestamp - float64(seconds)) * 1e9) + + *t = Timestamp(time.Unix(seconds, nanoseconds)) + return nil +} diff --git a/testresult/xcresult3/model3/testresults.go b/testresult/xcresult3/model3/testresults.go new file mode 100644 index 00000000..65eaad4f --- /dev/null +++ b/testresult/xcresult3/model3/testresults.go @@ -0,0 +1,63 @@ +package model3 + +type TestNodeType string + +// These are all the types the xcresulttool (version 23500, format version 3.53) supports. +const ( + TestNodeTypeTestPlan TestNodeType = "Test Plan" + TestNodeTypeUnitTestBundle TestNodeType = "Unit test bundle" + TestNodeTypeUITestBundle TestNodeType = "UI test bundle" + TestNodeTypeTestSuite TestNodeType = "Test Suite" + TestNodeTypeTestCase TestNodeType = "Test Case" + TestNodeTypeDevice TestNodeType = "Device" + TestNodeTypeTestPlanConfig TestNodeType = "Test Plan Configuration" + TestNodeTypeArguments TestNodeType = "Arguments" + TestNodeTypeRepetition TestNodeType = "Repetition" + TestNodeTypeTestCaseRun TestNodeType = "Test Case Run" + TestNodeTypeFailureMessage TestNodeType = "Failure Message" + TestNodeTypeSourceCodeRef TestNodeType = "Source Code Reference" + TestNodeTypeAttachment TestNodeType = "Attachment" + TestNodeTypeExpression TestNodeType = "Expression" + TestNodeTypeTestValue TestNodeType = "Test Value" +) + +type TestResult string + +const ( + TestResultPassed TestResult = "Passed" + TestResultFailed TestResult = "Failed" + TestResultSkipped TestResult = "Skipped" + TestResultExpectedFailure TestResult = "Expected Failure" + TestResultUnknown TestResult = "unknown" +) + +type TestData struct { + Devices []Devices `json:"devices"` + TestNodes []TestNode `json:"testNodes"` + TestPlanConfigurations []Configuration `json:"testPlanConfigurations"` +} + +type Devices struct { + Identifier string `json:"deviceId"` + Name string `json:"deviceName"` + Architecture string `json:"architecture"` + ModelName string `json:"modelName"` + Platform string `json:"platform"` + OS string `json:"osVersion"` +} + +type TestNode struct { + Identifier string `json:"nodeIdentifier"` + Type TestNodeType `json:"nodeType"` + Name string `json:"name"` + Details string `json:"details"` + Duration string `json:"duration"` + Result TestResult `json:"result"` + Tags []string `json:"tags"` + Children []TestNode `json:"children"` +} + +type Configuration struct { + Identifier string `json:"configurationId"` + Name string `json:"configurationName"` +} diff --git a/testresult/xcresult3/xcresult3.go b/testresult/xcresult3/xcresult3.go new file mode 100644 index 00000000..2ee78456 --- /dev/null +++ b/testresult/xcresult3/xcresult3.go @@ -0,0 +1,31 @@ +package xcresult3 + +import "github.com/bitrise-io/go-xcode/v2/testresult/xcresult3/model3" + +// Parse parses the given xcresult file's ActionsInvocationRecord and the list of ActionTestPlanRunSummaries. +func Parse(pth string, useLegacyFlag bool) (*ActionsInvocationRecord, []ActionTestPlanRunSummaries, error) { + var r ActionsInvocationRecord + if err := xcresulttoolGet(pth, "", useLegacyFlag, &r); err != nil { + return nil, nil, err + } + + var summaries []ActionTestPlanRunSummaries + for _, action := range r.Actions.Values { + refID := action.ActionResult.TestsRef.ID.Value + var s ActionTestPlanRunSummaries + if err := xcresulttoolGet(pth, refID, useLegacyFlag, &s); err != nil { + return nil, nil, err + } + summaries = append(summaries, s) + } + return &r, summaries, nil +} + +func ParseTestResults(pth string) (*model3.TestData, error) { + var data model3.TestData + if err := xcresulttoolGet(pth, "", false, &data); err != nil { + return nil, err + } + + return &data, nil +} diff --git a/testresult/xcresult3/xcresulttool.go b/testresult/xcresult3/xcresulttool.go new file mode 100644 index 00000000..2b3ac6e2 --- /dev/null +++ b/testresult/xcresult3/xcresulttool.go @@ -0,0 +1,182 @@ +package xcresult3 + +import ( + "bufio" + "bytes" + "encoding/json" + "fmt" + "io" + "os" + "os/exec" + "regexp" + "strconv" + "strings" + + "github.com/bitrise-io/go-utils/command" + "github.com/bitrise-io/go-utils/errorutil" + command2 "github.com/bitrise-io/go-utils/v2/command" + "github.com/bitrise-io/go-utils/v2/env" + "github.com/bitrise-io/go-utils/v2/log" +) + +func isXcresulttoolAvailable() bool { + if _, err := exec.LookPath("xcrun"); err != nil { + return false + } + return command.New("xcrun", "--find", "xcresulttool").Run() == nil +} + +func xcresulttoolVersion() (int, error) { + args := []string{"xcresulttool", "version"} + cmd := command.New("xcrun", args...) + out, err := cmd.RunAndReturnTrimmedCombinedOutput() + if err != nil { + if errorutil.IsExitStatusError(err) { + return 0, fmt.Errorf("%s failed: %s", cmd.PrintableCommandArgs(), out) + } + return 0, fmt.Errorf("%s failed: %s", cmd.PrintableCommandArgs(), err) + } + // xcresulttool version 23025, format version 3.53 (current) + versionRegexp := regexp.MustCompile("xcresulttool version ([0-9]+)") + + matches := versionRegexp.FindStringSubmatch(out) + if len(matches) < 2 { + return 0, fmt.Errorf("no version matches found in output: %s", out) + } + + version, err := strconv.Atoi(matches[1]) + if err != nil { + return 0, fmt.Errorf("failed to convert version: %s", matches[1]) + } + + return version, nil +} + +func isXcode16OrNewer() bool { + version, err := xcresulttoolVersion() + if err != nil { + return false + } + + return version >= 23_000 // Xcode 16 beta1 has version 23000 +} + +func supportsNewExtractionMethods() (bool, error) { + version, err := xcresulttoolVersion() + if err != nil { + return false, err + } + + return version >= 23_021, nil // Xcode 16 beta3 has version 23021 +} + +// xcresulttoolGet performs xcrun xcresulttool get with --id flag defined if id provided and marshals the output into v. +func xcresulttoolGet(xcresultPth, id string, useLegacyFlag bool, v interface{}) error { + commandFactory := command2.NewFactory(env.NewRepository()) + logger := log.NewLogger() + + args := []string{"xcresulttool", "get"} + + supportsNewMethod, err := supportsNewExtractionMethods() + if err != nil { + return err + } + + if supportsNewMethod && !useLegacyFlag { + args = append(args, "test-results", "tests") + } else { + args = append(args, "--format", "json") + + if isXcode16OrNewer() && useLegacyFlag { + args = append(args, "--legacy") + } + } + + args = append(args, "--path", xcresultPth) + + if id != "" { + args = append(args, "--id", id) + } + + var outBuffer, errBuffer, combinedBuffer bytes.Buffer + outWriter := io.MultiWriter(&outBuffer, &combinedBuffer) + errWriter := io.MultiWriter(&errBuffer, &combinedBuffer) + + cmd := commandFactory.Create("xcrun", args, &command2.Opts{ + Stdout: outWriter, + Stderr: errWriter, + Stdin: nil, + Env: os.Environ(), + Dir: "", + ErrorFinder: nil, + }) + if err := cmd.Run(); err != nil { + if errorutil.IsExitStatusError(err) { + return fmt.Errorf("%s failed: %s", cmd.PrintableCommandArgs(), combinedBuffer.String()) + } + return fmt.Errorf("%s failed: %s", cmd.PrintableCommandArgs(), err) + } + if stdErr := errBuffer.String(); stdErr != "" { + logger.Warnf("%s: %s", cmd.PrintableCommandArgs(), stdErr) + } + + stdout := outBuffer.Bytes() + if err := json.Unmarshal(stdout, v); err != nil { + logger.Warnf("Failed to parse %s command output, first lines:\n%s", cmd.PrintableCommandArgs(), firstLines(string(stdout), 10)) + return err + } + return nil +} + +func firstLines(out string, count int) string { + if count < 1 { + return "" + } + + var lines []string + scanner := bufio.NewScanner(strings.NewReader(out)) + for scanner.Scan() { + lines = append(lines, scanner.Text()) + if len(lines) >= count { + break + } + } + return strings.Join(lines, "\n") +} + +// xcresulttoolExport exports a file with the given id at the given output path. +func xcresulttoolExport(xcresultPth, id, outputPth string, useLegacyFlag bool) error { + args := []string{"xcresulttool", "export"} + + supportsNewMethod, err := supportsNewExtractionMethods() + if err != nil { + return err + } + + if supportsNewMethod && !useLegacyFlag { + args = append(args, "attachments") + } else { + args = append(args, "--type", "file") + + if isXcode16OrNewer() && useLegacyFlag { + args = append(args, "--legacy") + } + } + + args = append(args, "--path", xcresultPth) + args = append(args, "--output-path", outputPth) + + if id != "" { + args = append(args, "--id", id) + } + + cmd := command.New("xcrun", args...) + out, err := cmd.RunAndReturnTrimmedCombinedOutput() + if err != nil { + if errorutil.IsExitStatusError(err) { + return fmt.Errorf("%s failed: %s", cmd.PrintableCommandArgs(), out) + } + return fmt.Errorf("%s failed: %s", cmd.PrintableCommandArgs(), err) + } + return nil +} From e8906cedfe37c0221a4f8aa7fb0bd07b01e559ce Mon Sep 17 00:00:00 2001 From: Gergely Sallai <1516628+gergely-sallai@users.noreply.github.com> Date: Mon, 9 Mar 2026 17:23:34 +0100 Subject: [PATCH 02/11] style: STEP-2230 Fix lint Added mandatory docs for exported functions and variables. Did not check for the validity of exports, though. --- testresult/xcresult/xcresult.go | 3 ++- testresult/xcresult3/converter.go | 3 ++- testresult/xcresult3/model3/conversion.go | 1 + testresult/xcresult3/model3/data.go | 6 ++++++ testresult/xcresult3/model3/export.go | 4 ++++ testresult/xcresult3/model3/testresults.go | 7 +++++++ testresult/xcresult3/xcresult3.go | 1 + 7 files changed, 23 insertions(+), 2 deletions(-) diff --git a/testresult/xcresult/xcresult.go b/testresult/xcresult/xcresult.go index 8c15ecae..9bcabec2 100644 --- a/testresult/xcresult/xcresult.go +++ b/testresult/xcresult/xcresult.go @@ -18,6 +18,7 @@ type Converter struct { testSummariesPlistPath string } +// Setup configures the converter. func (c *Converter) Setup(_ bool) {} // Detect ... @@ -64,7 +65,7 @@ func filterIllegalChars(data []byte) (filtered []byte) { return } -// XML ... +// Convert returns the test report parsed from the xcresult file. func (c *Converter) Convert() (testreport.TestReport, error) { data, err := fileutil.ReadBytesFromFile(c.testSummariesPlistPath) if err != nil { diff --git a/testresult/xcresult3/converter.go b/testresult/xcresult3/converter.go index 3d81668b..53ae2c89 100644 --- a/testresult/xcresult3/converter.go +++ b/testresult/xcresult3/converter.go @@ -58,6 +58,7 @@ func documentMajorVersion(pth string) (int, error) { return majorVersion(info) } +// Setup configures the converter with the given extraction method preference. func (c *Converter) Setup(useOldXCResultExtractionMethod bool) { c.useLegacyExtractionMethod = useOldXCResultExtractionMethod } @@ -100,7 +101,7 @@ func (c *Converter) Detect(files []string) bool { return false } -// XML ... +// Convert returns the test report parsed from the xcresult file. func (c *Converter) Convert() (testreport.TestReport, error) { supportsNewMethod, err := supportsNewExtractionMethods() if err != nil { diff --git a/testresult/xcresult3/model3/conversion.go b/testresult/xcresult3/model3/conversion.go index 67389f59..2aaeb007 100644 --- a/testresult/xcresult3/model3/conversion.go +++ b/testresult/xcresult3/model3/conversion.go @@ -6,6 +6,7 @@ import ( "time" ) +// Convert converts TestData into a TestSummary and a list of warnings. func Convert(data *TestData) (*TestSummary, []string, error) { var warnings []string summary := TestSummary{} diff --git a/testresult/xcresult3/model3/data.go b/testresult/xcresult3/model3/data.go index 58a852ff..fa183b1d 100644 --- a/testresult/xcresult3/model3/data.go +++ b/testresult/xcresult3/model3/data.go @@ -2,30 +2,36 @@ package model3 import "time" +// TestSummary holds the top-level test results grouped by test plans. type TestSummary struct { TestPlans []TestPlan } +// TestPlan represents a test plan and its test bundles. type TestPlan struct { Name string TestBundles []TestBundle } +// TestBundle represents a test bundle and its test suites. type TestBundle struct { Name string TestSuites []TestSuite } +// TestSuite represents a test suite and its test cases. type TestSuite struct { Name string TestCases []TestCaseWithRetries } +// TestCaseWithRetries holds a test case along with any retry attempts. type TestCaseWithRetries struct { TestCase Retries []TestCase } +// TestCase represents a single test case execution result. type TestCase struct { Name string ClassName string diff --git a/testresult/xcresult3/model3/export.go b/testresult/xcresult3/model3/export.go index b18d805b..f14c1e7f 100644 --- a/testresult/xcresult3/model3/export.go +++ b/testresult/xcresult3/model3/export.go @@ -5,13 +5,16 @@ import ( "time" ) +// TestAttachmentDetails contains the test identifier and its list of attachments. type TestAttachmentDetails struct { TestIdentifier string `json:"testIdentifier"` Attachments []Attachment `json:"attachments"` } +// Timestamp is a time.Time that unmarshals from a Unix epoch float. type Timestamp time.Time +// Attachment describes a single exported test attachment file. type Attachment struct { ExportedFileName string `json:"exportedFileName"` SuggestedHumanReadableName string `json:"suggestedHumanReadableName"` @@ -23,6 +26,7 @@ type Attachment struct { RepetitionNumber int `json:"repetitionNumber"` } +// UnmarshalJSON decodes a Unix epoch float into a Timestamp. func (t *Timestamp) UnmarshalJSON(b []byte) error { var timestamp float64 if err := json.Unmarshal(b, ×tamp); err != nil { diff --git a/testresult/xcresult3/model3/testresults.go b/testresult/xcresult3/model3/testresults.go index 65eaad4f..37aab474 100644 --- a/testresult/xcresult3/model3/testresults.go +++ b/testresult/xcresult3/model3/testresults.go @@ -1,5 +1,6 @@ package model3 +// TestNodeType identifies the kind of a node in the xcresulttool test tree. type TestNodeType string // These are all the types the xcresulttool (version 23500, format version 3.53) supports. @@ -21,8 +22,10 @@ const ( TestNodeTypeTestValue TestNodeType = "Test Value" ) +// TestResult represents the outcome of a test case. type TestResult string +// TestResult values reported by xcresulttool. const ( TestResultPassed TestResult = "Passed" TestResultFailed TestResult = "Failed" @@ -31,12 +34,14 @@ const ( TestResultUnknown TestResult = "unknown" ) +// TestData is the top-level structure returned by xcresulttool for a test run. type TestData struct { Devices []Devices `json:"devices"` TestNodes []TestNode `json:"testNodes"` TestPlanConfigurations []Configuration `json:"testPlanConfigurations"` } +// Devices describes a device that participated in the test run. type Devices struct { Identifier string `json:"deviceId"` Name string `json:"deviceName"` @@ -46,6 +51,7 @@ type Devices struct { OS string `json:"osVersion"` } +// TestNode is a node in the xcresulttool test tree (plan, bundle, suite, case, etc.). type TestNode struct { Identifier string `json:"nodeIdentifier"` Type TestNodeType `json:"nodeType"` @@ -57,6 +63,7 @@ type TestNode struct { Children []TestNode `json:"children"` } +// Configuration describes a test plan configuration used during the test run. type Configuration struct { Identifier string `json:"configurationId"` Name string `json:"configurationName"` diff --git a/testresult/xcresult3/xcresult3.go b/testresult/xcresult3/xcresult3.go index 2ee78456..59f700a5 100644 --- a/testresult/xcresult3/xcresult3.go +++ b/testresult/xcresult3/xcresult3.go @@ -21,6 +21,7 @@ func Parse(pth string, useLegacyFlag bool) (*ActionsInvocationRecord, []ActionTe return &r, summaries, nil } +// ParseTestResults parses the test results from the given xcresult file. func ParseTestResults(pth string) (*model3.TestData, error) { var data model3.TestData if err := xcresulttoolGet(pth, "", false, &data); err != nil { From e3453b922e10c5a7a270398735af3b29f1a3c5d3 Mon Sep 17 00:00:00 2001 From: Gergely Sallai <1516628+gergely-sallai@users.noreply.github.com> Date: Tue, 10 Mar 2026 13:38:01 +0100 Subject: [PATCH 03/11] build: STEP-2230 Fix failing tests Missing test artifacts. --- bitrise.yml | 20 ++++++++++++++++++++ testresult/xcresult3/converter_test.go | 2 +- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/bitrise.yml b/bitrise.yml index 491f081d..a0015dd9 100755 --- a/bitrise.yml +++ b/bitrise.yml @@ -1,9 +1,14 @@ format_version: "11" default_step_lib_source: https://github.com/bitrise-io/bitrise-steplib.git +app: + envs: + - SAMPLE_ARTIFACTS_GIT_CLONE_URL: https://github.com/bitrise-io/sample-artifacts.git + workflows: check: before_run: + - _download_sample_artifacts - test - integration_test @@ -29,3 +34,18 @@ workflows: - change-workdir: inputs: - path: .. + + _download_sample_artifacts: + steps: + - script: + inputs: + - content: |- + #!/bin/bash + set -ex + rm -rf ./_tmp + - script: + inputs: + - content: | + #!/bin/bash + set -ex + git clone --depth 1 $SAMPLE_ARTIFACTS_GIT_CLONE_URL ./_tmp diff --git a/testresult/xcresult3/converter_test.go b/testresult/xcresult3/converter_test.go index 0b5faabc..15664280 100644 --- a/testresult/xcresult3/converter_test.go +++ b/testresult/xcresult3/converter_test.go @@ -20,7 +20,7 @@ import ( // into the _tmp dir. func copyTestdataToDir(pathInTestdataDir, dirPathToCopyInto string) (string, error) { err := command.CopyDir( - filepath.Join("../../../_tmp/", pathInTestdataDir), + filepath.Join("../../_tmp/", pathInTestdataDir), dirPathToCopyInto, true, ) From 695cc299bda79a261b35b0a53a8e21af0f1340f3 Mon Sep 17 00:00:00 2001 From: Gergely Sallai <1516628+gergely-sallai@users.noreply.github.com> Date: Tue, 10 Mar 2026 15:25:57 +0100 Subject: [PATCH 04/11] refact: STEP-2230 Move test logic entirely to go test - Removed the added bitrise.yml config that prepared the test environment, because that way `go test` would heavily depend on bitrise ci config. - Moved test data preparation logic to test_converter --- bitrise.yml | 20 ---- testresult/xcresult3/converter_test.go | 129 +++++++++++-------------- 2 files changed, 55 insertions(+), 94 deletions(-) diff --git a/bitrise.yml b/bitrise.yml index a0015dd9..491f081d 100755 --- a/bitrise.yml +++ b/bitrise.yml @@ -1,14 +1,9 @@ format_version: "11" default_step_lib_source: https://github.com/bitrise-io/bitrise-steplib.git -app: - envs: - - SAMPLE_ARTIFACTS_GIT_CLONE_URL: https://github.com/bitrise-io/sample-artifacts.git - workflows: check: before_run: - - _download_sample_artifacts - test - integration_test @@ -34,18 +29,3 @@ workflows: - change-workdir: inputs: - path: .. - - _download_sample_artifacts: - steps: - - script: - inputs: - - content: |- - #!/bin/bash - set -ex - rm -rf ./_tmp - - script: - inputs: - - content: | - #!/bin/bash - set -ex - git clone --depth 1 $SAMPLE_ARTIFACTS_GIT_CLONE_URL ./_tmp diff --git a/testresult/xcresult3/converter_test.go b/testresult/xcresult3/converter_test.go index 15664280..9a06019c 100644 --- a/testresult/xcresult3/converter_test.go +++ b/testresult/xcresult3/converter_test.go @@ -7,41 +7,60 @@ import ( "testing" "github.com/bitrise-io/go-steputils/v2/testreport" + "github.com/bitrise-io/go-utils/v2/command" + "github.com/bitrise-io/go-utils/v2/env" "github.com/stretchr/testify/require" - - "github.com/bitrise-io/go-utils/command" - "github.com/bitrise-io/go-utils/pathutil" ) -// copyTestdataToDir ... -// To populate the _tmp dir with sample data -// run `bitrise run download_sample_artifacts` before running tests here, -// which will download https://github.com/bitrise-io/sample-artifacts -// into the _tmp dir. -func copyTestdataToDir(pathInTestdataDir, dirPathToCopyInto string) (string, error) { - err := command.CopyDir( - filepath.Join("../../_tmp/", pathInTestdataDir), - dirPathToCopyInto, - true, - ) - return dirPathToCopyInto, err +const sampleArtifactsGitURI = "https://github.com/bitrise-io/sample-artifacts.git" + +var sampleArtifactsDir string + +func TestMain(m *testing.M) { + os.Exit(runTests(m)) } -func TestConverter_XML(t *testing.T) { - t.Run("xcresult3-flaky-with-rerun.xcresult", func(t *testing.T) { - fileName := "xcresult3-flaky-with-rerun.xcresult" - rootDir, xcresultPath, err := setupTestData(fileName) - require.NoError(t, err) +func runTests(m *testing.M) int { + dir, err := os.MkdirTemp("", "sample-artifacts-*") + if err != nil { + fmt.Printf("failed to create temp dir: %s\n", err) + return 1 + } + defer os.RemoveAll(dir) - defer func() { - require.NoError(t, os.RemoveAll(rootDir)) - }() + cmdFactory := command.NewFactory(env.NewRepository()) + cmd := cmdFactory.Create("git", []string{"clone", "--depth=1", sampleArtifactsGitURI, dir}, nil) + if out, err := cmd.RunAndReturnTrimmedCombinedOutput(); err != nil { + fmt.Printf("git clone failed: %s\n%s\n", err, out) + return 1 + } - t.Log("tempTestdataDir: ", rootDir) + sampleArtifactsDir = dir + return m.Run() +} - c := Converter{ - xcresultPth: xcresultPath, - } +// setupTestData copies an xcresult bundle from the cloned sample-artifacts repo +// into a per-test temp directory and returns the path to the copied bundle. +func setupTestData(t testing.TB, fileName string) string { + t.Helper() + + srcPath := filepath.Join(sampleArtifactsDir, "xcresults", fileName) + dstPath := filepath.Join(t.TempDir(), fileName) + + cmdFactory := command.NewFactory(env.NewRepository()) + cmd := cmdFactory.Create("cp", []string{"-r", srcPath, dstPath}, nil) + out, err := cmd.RunAndReturnTrimmedCombinedOutput() + require.NoError(t, err, "failed to copy test data: %s", out) + + return dstPath +} + +func TestConverter_XML(t *testing.T) { + t.Run("xcresult3-flaky-with-rerun.xcresult", func(t *testing.T) { + xcresultPath := setupTestData(t, "xcresult3-flaky-with-rerun.xcresult") + t.Log("xcresultPath: ", xcresultPath) + + c := Converter{xcresultPth: xcresultPath} junitXML, err := c.Convert() require.NoError(t, err) require.Equal(t, []testreport.TestSuite{ @@ -142,19 +161,10 @@ func TestConverter_XML(t *testing.T) { }) t.Run("xcresults3 success-failed-skipped-tests.xcresult", func(t *testing.T) { - fileName := "xcresult3-success-failed-skipped-tests.xcresult" - rootDir, xcresultPath, err := setupTestData(fileName) - require.NoError(t, err) - - defer func() { - require.NoError(t, os.RemoveAll(rootDir)) - }() - - t.Log("tempTestdataDir: ", rootDir) + xcresultPath := setupTestData(t, "xcresult3-success-failed-skipped-tests.xcresult") + t.Log("xcresultPath: ", xcresultPath) - c := Converter{ - xcresultPth: xcresultPath, - } + c := Converter{xcresultPth: xcresultPath} junitXML, err := c.Convert() require.NoError(t, err) require.Equal(t, []testreport.TestSuite{ @@ -206,13 +216,8 @@ func TestConverter_XML(t *testing.T) { }) t.Run("xcresult3-multiple-test-plan-configurations.xcresult", func(t *testing.T) { - fileName := "xcresult3-multiple-test-plan-configurations.xcresult" - rootDir, xcresultPath, err := setupTestData(fileName) - require.NoError(t, err) - require.NotEmpty(t, rootDir) - require.NotEmpty(t, xcresultPath) - - t.Log("tempTestdataDir: ", rootDir) + xcresultPath := setupTestData(t, "xcresult3-multiple-test-plan-configurations.xcresult") + t.Log("xcresultPath: ", xcresultPath) c := Converter{xcresultPth: xcresultPath} junitXML, err := c.Convert() @@ -226,38 +231,14 @@ func TestConverter_XML(t *testing.T) { German: swift_testingTests.swift:20: Expectation failed: true == false - // This test is intended to fail to demonstrate test failure reporting. `, ) - }) } func BenchmarkConverter_XML(b *testing.B) { - fileName := "xcresult3-flaky-with-rerun.xcresult" - rootDir, xcresultPath, err := setupTestData(fileName) - require.NoError(b, err) - - defer func() { - require.NoError(b, os.RemoveAll(rootDir)) - }() - - b.Log("tempTestdataDir: ", rootDir) + xcresultPath := setupTestData(b, "xcresult3-flaky-with-rerun.xcresult") + b.Log("xcresultPath: ", xcresultPath) - c := Converter{ - xcresultPth: xcresultPath, - } - _, err = c.Convert() + c := Converter{xcresultPth: xcresultPath} + _, err := c.Convert() require.NoError(b, err) } - -func setupTestData(fileName string) (string, string, error) { - tempTestdataDir, err := pathutil.NormalizedOSTempDirPath("test") - if err != nil { - return "", "", fmt.Errorf("failed to create temp dir: %w", err) - } - - tempXCResultPath, err := copyTestdataToDir(fmt.Sprintf("./xcresults/%s", fileName), filepath.Join(tempTestdataDir, fileName)) - if err != nil { - return "", "", err - } - - return tempTestdataDir, tempXCResultPath, nil -} From e5f1a14b6c4b28a3fe4d688b7c12854655028b8f Mon Sep 17 00:00:00 2001 From: Gergely Sallai <1516628+gergely-sallai@users.noreply.github.com> Date: Tue, 10 Mar 2026 16:22:19 +0100 Subject: [PATCH 05/11] fix: STEP-2230 Decouple test from local timezone This test was failing when it is run on a machine that has different timezone than UTC. The reason is that we were using xcrun xcresulttool to extract attachments from xcresult and that assembles filenames on the fly using local TZ, this made the test prone to failures (false negative) because it explicitly verified for UTC times. I don't think we need to test xcresult, so refactored this part. --- testresult/xcresult3/converter_test.go | 88 ++++++++++++-------------- 1 file changed, 39 insertions(+), 49 deletions(-) diff --git a/testresult/xcresult3/converter_test.go b/testresult/xcresult3/converter_test.go index 9a06019c..030b0a09 100644 --- a/testresult/xcresult3/converter_test.go +++ b/testresult/xcresult3/converter_test.go @@ -26,7 +26,12 @@ func runTests(m *testing.M) int { fmt.Printf("failed to create temp dir: %s\n", err) return 1 } - defer os.RemoveAll(dir) + defer func(path string) { + err := os.RemoveAll(path) + if err != nil { + fmt.Printf("failed to remove temp dir: %s\n", err) + } + }(dir) cmdFactory := command.NewFactory(env.NewRepository()) cmd := cmdFactory.Create("git", []string{"clone", "--depth=1", sampleArtifactsGitURI, dir}, nil) @@ -55,6 +60,22 @@ func setupTestData(t testing.TB, fileName string) string { return dstPath } +// assertAttachmentProperties checks that props contain exactly expectedCount attachments, +// each with a sequentially numbered Name ("attachment_0", "attachment_1", ...) and a Value +// matching the pattern "_.". +// The human-readable part of the filename is generated by xcresulttool using the local +// timezone, and I don't see any guarantee that it is always assembled like this, +// so only the overall pattern is verified rather than the exact string. +func assertAttachmentProperties(t *testing.T, props *testreport.Properties, expectedCount int) { + t.Helper() + require.NotNil(t, props) + require.Len(t, props.Property, expectedCount) + for i, p := range props.Property { + require.Equal(t, fmt.Sprintf("attachment_%d", i), p.Name) + require.Regexp(t, `^.+_\d+\.\w+$`, p.Value) + } +} + func TestConverter_XML(t *testing.T) { t.Run("xcresult3-flaky-with-rerun.xcresult", func(t *testing.T) { xcresultPath := setupTestData(t, "xcresult3-flaky-with-rerun.xcresult") @@ -63,6 +84,12 @@ func TestConverter_XML(t *testing.T) { c := Converter{xcresultPth: xcresultPath} junitXML, err := c.Convert() require.NoError(t, err) + + // Save and nil out the timezone-dependent attachment properties before the + // structural comparison; they are verified separately with assertAttachmentProperties. + uiTestProps := junitXML.TestSuites[2].TestCases[0].Properties + junitXML.TestSuites[2].TestCases[0].Properties = nil + require.Equal(t, []testreport.TestSuite{ { Name: "BullsEyeTests", Tests: 5, Failures: 0, Skipped: 0, Time: 0.9774, @@ -105,38 +132,7 @@ func TestConverter_XML(t *testing.T) { { Name: "BullsEyeUITests", Tests: 1, Failures: 0, Skipped: 0, Time: 9, TestCases: []testreport.TestCase{ - { - Name: "testGameStyleSwitch()", ClassName: "BullsEyeUITests", - Time: 9, - Properties: &testreport.Properties{ - Property: []testreport.Property{ - { - Name: "attachment_0", - Value: "Screenshot 2022-02-10 at 02.57.39 PM_1644505059194999933.jpeg", - }, - { - Name: "attachment_1", - Value: "Screenshot 2022-02-10 at 02.57.39 PM_1644505059388999938.jpeg", - }, - { - Name: "attachment_2", - Value: "Screenshot 2022-02-10 at 02.57.44 PM_1644505064670000076.jpeg", - }, - { - Name: "attachment_3", - Value: "Screenshot 2022-02-10 at 02.57.47 PM_1644505067144000053.jpeg", - }, - { - Name: "attachment_4", - Value: "Screenshot 2022-02-10 at 02.57.47 PM_1644505067476000070.jpeg", - }, - { - Name: "attachment_5", - Value: "Screenshot 2022-02-10 at 02.57.47 PM_1644505067992000102.jpeg", - }, - }, - }, - }, + {Name: "testGameStyleSwitch()", ClassName: "BullsEyeUITests", Time: 9}, }, }, { @@ -158,6 +154,8 @@ func TestConverter_XML(t *testing.T) { }, }, }, junitXML.TestSuites) + + assertAttachmentProperties(t, uiTestProps, 6) }) t.Run("xcresults3 success-failed-skipped-tests.xcresult", func(t *testing.T) { @@ -167,6 +165,12 @@ func TestConverter_XML(t *testing.T) { c := Converter{xcresultPth: xcresultPath} junitXML, err := c.Convert() require.NoError(t, err) + + // Save and nil out the timezone-dependent attachment properties before the + // structural comparison; they are verified separately with assertAttachmentProperties. + failureProps := junitXML.TestSuites[0].TestCases[0].Properties + junitXML.TestSuites[0].TestCases[0].Properties = nil + require.Equal(t, []testreport.TestSuite{ { Name: "testProjectUITests", @@ -182,22 +186,6 @@ func TestConverter_XML(t *testing.T) { Failure: &testreport.Failure{ Value: "testProjectUITests.swift:30: XCTAssertTrue failed", }, - Properties: &testreport.Properties{ - Property: []testreport.Property{ - { - Name: "attachment_0", - Value: "Screenshot 2021-02-09 at 08.35.51 AM_1612859751989000082.jpeg", - }, - { - Name: "attachment_1", - Value: "Screenshot 2021-02-09 at 08.35.52 AM_1612859752052999973.jpeg", - }, - { - Name: "attachment_2", - Value: "Screenshot 2021-02-09 at 08.35.52 AM_1612859752052999973.jpeg", - }, - }, - }, }, { Name: "testSkip()", @@ -213,6 +201,8 @@ func TestConverter_XML(t *testing.T) { }, }, }, junitXML.TestSuites) + + assertAttachmentProperties(t, failureProps, 3) }) t.Run("xcresult3-multiple-test-plan-configurations.xcresult", func(t *testing.T) { From a5027024ed4408f6014d558a5211570c437b07fb Mon Sep 17 00:00:00 2001 From: Gergely Sallai <1516628+gergely-sallai@users.noreply.github.com> Date: Wed, 11 Mar 2026 14:28:41 +0100 Subject: [PATCH 06/11] refact: STEP-2230 private visibility in xcresult --- testresult/xcresult/testsummariesplist.go | 58 +++++++++-------------- testresult/xcresult/xcresult.go | 14 +++--- 2 files changed, 30 insertions(+), 42 deletions(-) diff --git a/testresult/xcresult/testsummariesplist.go b/testresult/xcresult/testsummariesplist.go index c258fc58..886db80c 100644 --- a/testresult/xcresult/testsummariesplist.go +++ b/testresult/xcresult/testsummariesplist.go @@ -5,16 +5,15 @@ import ( "strings" ) -// TestSummaryPlist ... -type TestSummaryPlist struct { +type testSummaryPlist struct { FormatVersion string - TestableSummaries []TestableSummary + TestableSummaries []testableSummary } -func collapseSubtestTree(data Subtests) (tests Subtests) { +func collapsesubtestTree(data subtests) (tests subtests) { for _, test := range data { if len(test.Subtests) > 0 { - tests = append(tests, collapseSubtestTree(test.Subtests)...) + tests = append(tests, collapsesubtestTree(test.Subtests)...) } if test.TestStatus != "" { tests = append(tests, test) @@ -23,14 +22,13 @@ func collapseSubtestTree(data Subtests) (tests Subtests) { return } -// Tests returns the collapsed tree of tests -func (summaryPlist TestSummaryPlist) Tests() ([]string, map[string]Subtests) { +func (summaryPlist testSummaryPlist) tests() ([]string, map[string]subtests) { var keyOrder []string - tests := map[string]Subtests{} - var subTests Subtests + tests := map[string]subtests{} + var subTests subtests for _, testableSummary := range summaryPlist.TestableSummaries { for _, test := range testableSummary.Tests { - subTests = append(subTests, collapseSubtestTree(test.Subtests)...) + subTests = append(subTests, collapsesubtestTree(test.subtests)...) } } for _, test := range subTests { @@ -44,41 +42,36 @@ func (summaryPlist TestSummaryPlist) Tests() ([]string, map[string]Subtests) { return keyOrder, tests } -// Test ... -type Test struct { - Subtests Subtests +type test struct { + subtests subtests } -// TestableSummary ... -type TestableSummary struct { +type testableSummary struct { TargetName string TestKind string TestName string TestObjectClass string - Tests []Test + Tests []test } -// FailureSummary ... -type FailureSummary struct { +type failureSummary struct { FileName string LineNumber int Message string PerformanceFailure bool } -// Subtest ... -type Subtest struct { +type subtest struct { Duration float64 TestStatus string TestIdentifier string TestName string TestObjectClass string - Subtests Subtests - FailureSummaries []FailureSummary + Subtests subtests + FailureSummaries []failureSummary } -// Failure ... -func (st Subtest) Failure() (message string) { +func (st subtest) failure() (message string) { prefix := "" for _, failure := range st.FailureSummaries { message += fmt.Sprintf("%s%s:%d - %s", prefix, failure.FileName, failure.LineNumber, failure.Message) @@ -87,16 +80,13 @@ func (st Subtest) Failure() (message string) { return } -// Skipped ... -func (st Subtest) Skipped() bool { +func (st subtest) skipped() bool { return st.TestStatus == "Skipped" } -// Subtests ... -type Subtests []Subtest +type subtests []subtest -// FailuresCount ... -func (sts Subtests) FailuresCount() (count int) { +func (sts subtests) failuresCount() (count int) { for _, test := range sts { if len(test.FailureSummaries) > 0 { count++ @@ -105,18 +95,16 @@ func (sts Subtests) FailuresCount() (count int) { return count } -// SkippedCount ... -func (sts Subtests) SkippedCount() (count int) { +func (sts subtests) skippedCount() (count int) { for _, test := range sts { - if test.Skipped() { + if test.skipped() { count++ } } return count } -// TotalTime ... -func (sts Subtests) TotalTime() (time float64) { +func (sts subtests) totalTime() (time float64) { for _, test := range sts { time += test.Duration } diff --git a/testresult/xcresult/xcresult.go b/testresult/xcresult/xcresult.go index 9bcabec2..afac2843 100644 --- a/testresult/xcresult/xcresult.go +++ b/testresult/xcresult/xcresult.go @@ -74,25 +74,25 @@ func (c *Converter) Convert() (testreport.TestReport, error) { data = filterIllegalChars(data) - var plistData TestSummaryPlist + var plistData testSummaryPlist if _, err := plist.Unmarshal(data, &plistData); err != nil { return testreport.TestReport{}, err } var xmlData testreport.TestReport - keyOrder, tests := plistData.Tests() + keyOrder, tests := plistData.tests() for _, testID := range keyOrder { tests := tests[testID] testSuite := testreport.TestSuite{ Name: testID, Tests: len(tests), - Failures: tests.FailuresCount(), - Skipped: tests.SkippedCount(), - Time: tests.TotalTime(), + Failures: tests.failuresCount(), + Skipped: tests.skippedCount(), + Time: tests.totalTime(), } for _, test := range tests { - failureMessage := test.Failure() + failureMessage := test.failure() var failure *testreport.Failure if len(failureMessage) > 0 { @@ -102,7 +102,7 @@ func (c *Converter) Convert() (testreport.TestReport, error) { } var skipped *testreport.Skipped - if test.Skipped() { + if test.skipped() { skipped = &testreport.Skipped{} } From 0a19d98959452e92c9850f87c49b30e98fbbd264 Mon Sep 17 00:00:00 2001 From: Gergely Sallai <1516628+gergely-sallai@users.noreply.github.com> Date: Wed, 11 Mar 2026 14:35:19 +0100 Subject: [PATCH 07/11] refact: STEP-2230 private visibility in xcresult3 Note: had to rename some variable to not be confused with private types now. This also true for `func Parse(....)` -> `func loadXCResultData(....)` it was clashing with converter.go's `parse()` --- .../xcresult3/action_invocation_record.go | 57 +++--- .../action_invocation_record_test.go | 98 +++++------ .../xcresult3/action_test_plan_summary.go | 75 ++++---- .../action_test_plan_summary_test.go | 162 +++++++++--------- testresult/xcresult3/action_test_summary.go | 41 ++--- .../xcresult3/action_test_summary_group.go | 66 +++---- .../action_test_summary_group_test.go | 44 +++-- testresult/xcresult3/converter.go | 14 +- testresult/xcresult3/xcresult3.go | 10 +- 9 files changed, 263 insertions(+), 304 deletions(-) diff --git a/testresult/xcresult3/action_invocation_record.go b/testresult/xcresult3/action_invocation_record.go index 7497a67a..2aa929b8 100644 --- a/testresult/xcresult3/action_invocation_record.go +++ b/testresult/xcresult3/action_invocation_record.go @@ -7,8 +7,7 @@ import ( "github.com/bitrise-io/go-steputils/v2/testreport" ) -// ActionsInvocationRecord ... -type ActionsInvocationRecord struct { +type actionsInvocationRecord struct { Actions struct { Values []struct { ActionResult struct { @@ -21,61 +20,53 @@ type ActionsInvocationRecord struct { } `json:"_values"` } `json:"actions"` - Issues Issues `json:"issues"` + Issues issues `json:"issues"` } -// Issues ... -type Issues struct { - TestFailureSummaries TestFailureSummaries `json:"testFailureSummaries"` +type issues struct { + TestFailureSummaries testFailureSummaries `json:"testFailureSummaries"` } -// TestFailureSummaries ... -type TestFailureSummaries struct { - Values []TestFailureSummary `json:"_values"` +type testFailureSummaries struct { + Values []testFailureSummary `json:"_values"` } -// TestFailureSummary ... -type TestFailureSummary struct { - DocumentLocationInCreatingWorkspace DocumentLocationInCreatingWorkspace `json:"documentLocationInCreatingWorkspace"` - Message Message `json:"message"` - ProducingTarget ProducingTarget `json:"producingTarget"` - TestCaseName TestCaseName `json:"testCaseName"` +type testFailureSummary struct { + DocumentLocationInCreatingWorkspace documentLocationInCreatingWorkspace `json:"documentLocationInCreatingWorkspace"` + Message message `json:"message"` + ProducingTarget producingTarget `json:"producingTarget"` + TestCaseName testCaseName `json:"testCaseName"` } -// URL ... -type URL struct { +type url struct { Value string `json:"_value"` } -// DocumentLocationInCreatingWorkspace ... -type DocumentLocationInCreatingWorkspace struct { - URL URL `json:"url"` +type documentLocationInCreatingWorkspace struct { + URL url `json:"url"` } -// ProducingTarget ... -type ProducingTarget struct { +type producingTarget struct { Value string `json:"_value"` } -// TestCaseName ... -type TestCaseName struct { +type testCaseName struct { Value string `json:"_value"` } -// Message ... -type Message struct { +type message struct { Value string `json:"_value"` } -func testCaseMatching(test ActionTestSummaryGroup, testCaseName string) bool { +func testCaseMatching(test actionTestSummaryGroup, tcName string) bool { class, method := test.references() - return testCaseName == class+"."+method || - testCaseName == fmt.Sprintf("-[%s %s]", class, method) + return tcName == class+"."+method || + tcName == fmt.Sprintf("-[%s %s]", class, method) } -// failure returns the ActionTestSummaryGroup's failure reason from the ActionsInvocationRecord. -func (r ActionsInvocationRecord) failure(test ActionTestSummaryGroup, testSuite testreport.TestSuite) string { +// failure returns the failure reason for the given test from the invocation record. +func (r actionsInvocationRecord) failure(test actionTestSummaryGroup, testSuite testreport.TestSuite) string { for _, failureSummary := range r.Issues.TestFailureSummaries.Values { if failureSummary.ProducingTarget.Value == testSuite.Name && testCaseMatching(test, failureSummary.TestCaseName.Value) { file, line := failureSummary.fileAndLineNumber() @@ -85,8 +76,8 @@ func (r ActionsInvocationRecord) failure(test ActionTestSummaryGroup, testSuite return "" } -// fileAndLineNumber unwraps the file path and line number descriptor from a given ActionTestSummaryGroup's. -func (s TestFailureSummary) fileAndLineNumber() (file string, line string) { +// fileAndLineNumber unwraps the file path and line number from the document location URL. +func (s testFailureSummary) fileAndLineNumber() (file string, line string) { // file:\/\/\/Users\/bitrisedeveloper\/Develop\/ios\/Xcode11Test\/Xcode11TestUITests\/Xcode11TestUITests.swift#CharacterRangeLen=0&EndingLineNumber=42&StartingLineNumber=42 if s.DocumentLocationInCreatingWorkspace.URL.Value != "" { i := strings.LastIndex(s.DocumentLocationInCreatingWorkspace.URL.Value, "#") diff --git a/testresult/xcresult3/action_invocation_record_test.go b/testresult/xcresult3/action_invocation_record_test.go index a9ac4509..918c7e13 100644 --- a/testresult/xcresult3/action_invocation_record_test.go +++ b/testresult/xcresult3/action_invocation_record_test.go @@ -7,117 +7,117 @@ import ( ) func TestTestFailureSummary_fileAndLineNumber(t *testing.T) { - tests := []struct { + testCases := []struct { name string - summary TestFailureSummary + summary testFailureSummary wantFile string wantLine string }{ { name: "", - summary: TestFailureSummary{ - DocumentLocationInCreatingWorkspace: DocumentLocationInCreatingWorkspace{ - URL: URL{Value: "file:/Xcode11TestUITests2.swift#CharacterRangeLen=0&EndingLineNumber=33&StartingLineNumber=33"}, + summary: testFailureSummary{ + DocumentLocationInCreatingWorkspace: documentLocationInCreatingWorkspace{ + URL: url{Value: "file:/Xcode11TestUITests2.swift#CharacterRangeLen=0&EndingLineNumber=33&StartingLineNumber=33"}, }, }, wantFile: "file:/Xcode11TestUITests2.swift", wantLine: "CharacterRangeLen=0&EndingLineNumber=33&StartingLineNumber=33", }, } - for _, tt := range tests { + for _, tt := range testCases { t.Run(tt.name, func(t *testing.T) { gotFile, gotLine := tt.summary.fileAndLineNumber() if gotFile != tt.wantFile { - t.Errorf("TestFailureSummary.fileAndLineNumber() gotFile = %v, want %v", gotFile, tt.wantFile) + t.Errorf("testFailureSummary.fileAndLineNumber() gotFile = %v, want %v", gotFile, tt.wantFile) } if gotLine != tt.wantLine { - t.Errorf("TestFailureSummary.fileAndLineNumber() gotLine = %v, want %v", gotLine, tt.wantLine) + t.Errorf("testFailureSummary.fileAndLineNumber() gotLine = %v, want %v", gotLine, tt.wantLine) } }) } } func TestActionsInvocationRecord_failure(t *testing.T) { - tests := []struct { + testCases := []struct { name string - record ActionsInvocationRecord - test ActionTestSummaryGroup + record actionsInvocationRecord + test actionTestSummaryGroup want string }{ { name: "Simple test", - record: ActionsInvocationRecord{ - Issues: Issues{ - TestFailureSummaries: TestFailureSummaries{ - Values: []TestFailureSummary{ - TestFailureSummary{ - ProducingTarget: ProducingTarget{Value: "Xcode11TestUITests2"}, - TestCaseName: TestCaseName{Value: "Xcode11TestUITests2.testFail()"}, - Message: Message{Value: "XCTAssertEqual failed: (\"1\") is not equal to (\"0\")"}, - DocumentLocationInCreatingWorkspace: DocumentLocationInCreatingWorkspace{ - URL: URL{Value: "file:/Xcode11TestUITests2.swift#CharacterRangeLen=0&EndingLineNumber=33&StartingLineNumber=33"}, + record: actionsInvocationRecord{ + Issues: issues{ + TestFailureSummaries: testFailureSummaries{ + Values: []testFailureSummary{ + { + ProducingTarget: producingTarget{Value: "Xcode11TestUITests2"}, + TestCaseName: testCaseName{Value: "Xcode11TestUITests2.testFail()"}, + Message: message{Value: "XCTAssertEqual failed: (\"1\") is not equal to (\"0\")"}, + DocumentLocationInCreatingWorkspace: documentLocationInCreatingWorkspace{ + URL: url{Value: "file:/Xcode11TestUITests2.swift#CharacterRangeLen=0&EndingLineNumber=33&StartingLineNumber=33"}, }, }, }, }, }, }, - test: ActionTestSummaryGroup{ - Identifier: Identifier{Value: "Xcode11TestUITests2/testFail()"}, + test: actionTestSummaryGroup{ + Identifier: identifier{Value: "Xcode11TestUITests2/testFail()"}, }, want: `file:/Xcode11TestUITests2.swift:CharacterRangeLen=0&EndingLineNumber=33&StartingLineNumber=33 - XCTAssertEqual failed: ("1") is not equal to ("0")`, }, { name: "class inherited test", - record: ActionsInvocationRecord{ - Issues: Issues{ - TestFailureSummaries: TestFailureSummaries{ - Values: []TestFailureSummary{ - TestFailureSummary{ - ProducingTarget: ProducingTarget{Value: "Xcode11TestUITests2"}, - TestCaseName: TestCaseName{Value: "SomethingDifferentClass.testFail()"}, - Message: Message{Value: "XCTAssertEqual failed: (\"1\") is not equal to (\"0\")"}, - DocumentLocationInCreatingWorkspace: DocumentLocationInCreatingWorkspace{ - URL: URL{Value: "file:/Xcode11TestUITests2.swift#CharacterRangeLen=0&EndingLineNumber=33&StartingLineNumber=33"}, + record: actionsInvocationRecord{ + Issues: issues{ + TestFailureSummaries: testFailureSummaries{ + Values: []testFailureSummary{ + { + ProducingTarget: producingTarget{Value: "Xcode11TestUITests2"}, + TestCaseName: testCaseName{Value: "SomethingDifferentClass.testFail()"}, + Message: message{Value: "XCTAssertEqual failed: (\"1\") is not equal to (\"0\")"}, + DocumentLocationInCreatingWorkspace: documentLocationInCreatingWorkspace{ + URL: url{Value: "file:/Xcode11TestUITests2.swift#CharacterRangeLen=0&EndingLineNumber=33&StartingLineNumber=33"}, }, }, }, }, }, }, - test: ActionTestSummaryGroup{ - Identifier: Identifier{Value: "SomethingDifferentClass/testFail()"}, + test: actionTestSummaryGroup{ + Identifier: identifier{Value: "SomethingDifferentClass/testFail()"}, }, want: `file:/Xcode11TestUITests2.swift:CharacterRangeLen=0&EndingLineNumber=33&StartingLineNumber=33 - XCTAssertEqual failed: ("1") is not equal to ("0")`, }, { name: "inner class test", - record: ActionsInvocationRecord{ - Issues: Issues{ - TestFailureSummaries: TestFailureSummaries{ - Values: []TestFailureSummary{ - TestFailureSummary{ - ProducingTarget: ProducingTarget{Value: "Xcode11TestUITests2"}, - TestCaseName: TestCaseName{Value: "-[SomethingDifferentClass testFail]"}, - Message: Message{Value: "XCTAssertEqual failed: (\"1\") is not equal to (\"0\")"}, - DocumentLocationInCreatingWorkspace: DocumentLocationInCreatingWorkspace{ - URL: URL{Value: "file:/Xcode11TestUITests2.swift#CharacterRangeLen=0&EndingLineNumber=33&StartingLineNumber=33"}, + record: actionsInvocationRecord{ + Issues: issues{ + TestFailureSummaries: testFailureSummaries{ + Values: []testFailureSummary{ + { + ProducingTarget: producingTarget{Value: "Xcode11TestUITests2"}, + TestCaseName: testCaseName{Value: "-[SomethingDifferentClass testFail]"}, + Message: message{Value: "XCTAssertEqual failed: (\"1\") is not equal to (\"0\")"}, + DocumentLocationInCreatingWorkspace: documentLocationInCreatingWorkspace{ + URL: url{Value: "file:/Xcode11TestUITests2.swift#CharacterRangeLen=0&EndingLineNumber=33&StartingLineNumber=33"}, }, }, }, }, }, }, - test: ActionTestSummaryGroup{ - Identifier: Identifier{Value: "SomethingDifferentClass/testFail"}, + test: actionTestSummaryGroup{ + Identifier: identifier{Value: "SomethingDifferentClass/testFail"}, }, want: `file:/Xcode11TestUITests2.swift:CharacterRangeLen=0&EndingLineNumber=33&StartingLineNumber=33 - XCTAssertEqual failed: ("1") is not equal to ("0")`, }, } - for _, tt := range tests { + for _, tt := range testCases { t.Run(tt.name, func(t *testing.T) { if got := tt.record.failure(tt.test, testreport.TestSuite{Name: "Xcode11TestUITests2"}); got != tt.want { - t.Errorf("ActionsInvocationRecord.failure() = %v, want %v", got, tt.want) + t.Errorf("actionsInvocationRecord.failure() = %v, want %v", got, tt.want) } }) } diff --git a/testresult/xcresult3/action_test_plan_summary.go b/testresult/xcresult3/action_test_plan_summary.go index 8f1808c0..6605da4a 100644 --- a/testresult/xcresult3/action_test_plan_summary.go +++ b/testresult/xcresult3/action_test_plan_summary.go @@ -2,71 +2,64 @@ package xcresult3 import "strconv" -// ActionTestPlanRunSummaries ... -type ActionTestPlanRunSummaries struct { - Summaries Summaries `json:"summaries"` +type actionTestPlanRunSummaries struct { + Summaries summaries `json:"summaries"` } -// Summaries ... -type Summaries struct { - Values []Summary `json:"_values"` +type summaries struct { + Values []summary `json:"_values"` } -// Summary ... -type Summary struct { - TestableSummaries TestableSummaries `json:"testableSummaries"` +type summary struct { + TestableSummaries testableSummaries `json:"testableSummaries"` } -// TestableSummaries ... -type TestableSummaries struct { - Values []ActionTestableSummary `json:"_values"` +type testableSummaries struct { + Values []actionTestableSummary `json:"_values"` } -// ActionTestableSummary ... -type ActionTestableSummary struct { - Name Name `json:"name"` - Tests Tests `json:"tests"` +type actionTestableSummary struct { + Name name `json:"name"` + Tests tests `json:"tests"` } -// Tests ... -type Tests struct { - Values []ActionTestSummaryGroup `json:"_values"` +type tests struct { + Values []actionTestSummaryGroup `json:"_values"` } -// Name ... -type Name struct { +type name struct { Value string `json:"_value"` } -// tests returns ActionTestSummaryGroup mapped by the container TestableSummary name. -func (s ActionTestPlanRunSummaries) tests() ([]string, map[string][]ActionTestSummaryGroup) { - summaryGroupsByName := map[string][]ActionTestSummaryGroup{} +// tests returns actionTestSummaryGroup mapped by the container testableSummary name. +func (s actionTestPlanRunSummaries) tests() ([]string, map[string][]actionTestSummaryGroup) { + summaryGroupsByName := map[string][]actionTestSummaryGroup{} var testSuiteOrder []string - for _, summary := range s.Summaries.Values { - for _, testableSummary := range summary.TestableSummaries.Values { + for _, smry := range s.Summaries.Values { + for _, testableSummary := range smry.TestableSummaries.Values { // test suite - name := testableSummary.Name.Value - if _, found := summaryGroupsByName[name]; !found { - testSuiteOrder = append(testSuiteOrder, name) + n := testableSummary.Name.Value + if _, found := summaryGroupsByName[n]; !found { + testSuiteOrder = append(testSuiteOrder, n) } - var tests []ActionTestSummaryGroup + var ts []actionTestSummaryGroup for _, test := range testableSummary.Tests.Values { - tests = append(tests, test.testsWithStatus()...) + ts = append(ts, test.testsWithStatus()...) } - summaryGroupsByName[name] = tests + summaryGroupsByName[n] = ts } } return testSuiteOrder, summaryGroupsByName } -func (s ActionTestPlanRunSummaries) failuresCount(testableSummaryName string) (failure int) { +func (s actionTestPlanRunSummaries) failuresCount(testableSummaryName string) (failure int) { _, testsByCase := s.tests() - tests := testsByCase[testableSummaryName] - for _, test := range tests { + ts := testsByCase[testableSummaryName] + for _, test := range ts { if test.TestStatus.Value == "Failure" { failure++ } @@ -74,10 +67,10 @@ func (s ActionTestPlanRunSummaries) failuresCount(testableSummaryName string) (f return } -func (s ActionTestPlanRunSummaries) skippedCount(testableSummaryName string) (skipped int) { +func (s actionTestPlanRunSummaries) skippedCount(testableSummaryName string) (skipped int) { _, testsByCase := s.tests() - tests := testsByCase[testableSummaryName] - for _, test := range tests { + ts := testsByCase[testableSummaryName] + for _, test := range ts { if test.TestStatus.Value == "Skipped" { skipped++ } @@ -85,10 +78,10 @@ func (s ActionTestPlanRunSummaries) skippedCount(testableSummaryName string) (sk return } -func (s ActionTestPlanRunSummaries) totalTime(testableSummaryName string) (time float64) { +func (s actionTestPlanRunSummaries) totalTime(testableSummaryName string) (time float64) { _, testsByCase := s.tests() - tests := testsByCase[testableSummaryName] - for _, test := range tests { + ts := testsByCase[testableSummaryName] + for _, test := range ts { if test.Duration.Value != "" { d, err := strconv.ParseFloat(test.Duration.Value, 64) if err == nil { diff --git a/testresult/xcresult3/action_test_plan_summary_test.go b/testresult/xcresult3/action_test_plan_summary_test.go index fecd0fb5..bf504316 100644 --- a/testresult/xcresult3/action_test_plan_summary_test.go +++ b/testresult/xcresult3/action_test_plan_summary_test.go @@ -9,24 +9,24 @@ import ( ) func TestActionTestPlanRunSummaries_tests(t *testing.T) { - tests := []struct { + testCases := []struct { name string - summaries ActionTestPlanRunSummaries - want map[string][]ActionTestSummaryGroup + summaries actionTestPlanRunSummaries + want map[string][]actionTestSummaryGroup }{ { name: "single test with status", - summaries: ActionTestPlanRunSummaries{ - Summaries: Summaries{ - Values: []Summary{ - Summary{ - TestableSummaries: TestableSummaries{ - Values: []ActionTestableSummary{ - ActionTestableSummary{ - Name: Name{Value: "test case 1"}, - Tests: Tests{ - Values: []ActionTestSummaryGroup{ - ActionTestSummaryGroup{TestStatus: TestStatus{Value: "success"}}, + summaries: actionTestPlanRunSummaries{ + Summaries: summaries{ + Values: []summary{ + { + TestableSummaries: testableSummaries{ + Values: []actionTestableSummary{ + { + Name: name{Value: "test case 1"}, + Tests: tests{ + Values: []actionTestSummaryGroup{ + {TestStatus: testStatus{Value: "success"}}, }, }, }, @@ -36,41 +36,37 @@ func TestActionTestPlanRunSummaries_tests(t *testing.T) { }, }, }, - want: map[string][]ActionTestSummaryGroup{ - "test case 1": []ActionTestSummaryGroup{ - ActionTestSummaryGroup{TestStatus: TestStatus{Value: "success"}}, + want: map[string][]actionTestSummaryGroup{ + "test case 1": { + {TestStatus: testStatus{Value: "success"}}, }, }, }, { name: "single test with status + subtests with status", - summaries: ActionTestPlanRunSummaries{ - Summaries: Summaries{ - Values: []Summary{ - Summary{ - TestableSummaries: TestableSummaries{ - Values: []ActionTestableSummary{ - ActionTestableSummary{ - Name: Name{Value: "test case 1"}, - Tests: Tests{ - Values: []ActionTestSummaryGroup{ - ActionTestSummaryGroup{TestStatus: TestStatus{Value: "success"}}, + summaries: actionTestPlanRunSummaries{ + Summaries: summaries{ + Values: []summary{ + { + TestableSummaries: testableSummaries{ + Values: []actionTestableSummary{ + { + Name: name{Value: "test case 1"}, + Tests: tests{ + Values: []actionTestSummaryGroup{ + {TestStatus: testStatus{Value: "success"}}, }, }, }, - ActionTestableSummary{ - Name: Name{Value: "test case 2"}, - Tests: Tests{ - Values: []ActionTestSummaryGroup{ - ActionTestSummaryGroup{ - Subtests: Subtests{ - Values: []ActionTestSummaryGroup{ - ActionTestSummaryGroup{ - TestStatus: TestStatus{Value: "success"}, - }, - ActionTestSummaryGroup{ - TestStatus: TestStatus{Value: "success"}, - }, + { + Name: name{Value: "test case 2"}, + Tests: tests{ + Values: []actionTestSummaryGroup{ + { + Subtests: subtests{ + Values: []actionTestSummaryGroup{ + {TestStatus: testStatus{Value: "success"}}, + {TestStatus: testStatus{Value: "success"}}, }, }, }, @@ -83,53 +79,53 @@ func TestActionTestPlanRunSummaries_tests(t *testing.T) { }, }, }, - want: map[string][]ActionTestSummaryGroup{ - "test case 1": []ActionTestSummaryGroup{ - ActionTestSummaryGroup{TestStatus: TestStatus{Value: "success"}}, + want: map[string][]actionTestSummaryGroup{ + "test case 1": { + {TestStatus: testStatus{Value: "success"}}, }, - "test case 2": []ActionTestSummaryGroup{ - ActionTestSummaryGroup{TestStatus: TestStatus{Value: "success"}}, - ActionTestSummaryGroup{TestStatus: TestStatus{Value: "success"}}, + "test case 2": { + {TestStatus: testStatus{Value: "success"}}, + {TestStatus: testStatus{Value: "success"}}, }, }, }, { name: "no test with status", - summaries: ActionTestPlanRunSummaries{}, - want: map[string][]ActionTestSummaryGroup{}, + summaries: actionTestPlanRunSummaries{}, + want: map[string][]actionTestSummaryGroup{}, }, } - for _, tt := range tests { + for _, tt := range testCases { t.Run(tt.name, func(t *testing.T) { if _, got := tt.summaries.tests(); !reflect.DeepEqual(got, tt.want) { fmt.Println("want: ", pretty.Object(tt.want)) fmt.Println("got: ", pretty.Object(got)) - t.Errorf("ActionTestPlanRunSummaries.tests() = %v, want %v", got, tt.want) + t.Errorf("actionTestPlanRunSummaries.tests() = %v, want %v", got, tt.want) } }) } } func TestActionTestPlanRunSummaries_failuresCount(t *testing.T) { - tests := []struct { + testCases := []struct { name string - summaries ActionTestPlanRunSummaries + summaries actionTestPlanRunSummaries testableSummaryName string wantFailure int }{ { name: "single failure", - summaries: ActionTestPlanRunSummaries{ - Summaries: Summaries{ - Values: []Summary{ - Summary{ - TestableSummaries: TestableSummaries{ - Values: []ActionTestableSummary{ - ActionTestableSummary{ - Name: Name{Value: "test case"}, - Tests: Tests{ - Values: []ActionTestSummaryGroup{ - ActionTestSummaryGroup{TestStatus: TestStatus{Value: "Failure"}}, + summaries: actionTestPlanRunSummaries{ + Summaries: summaries{ + Values: []summary{ + { + TestableSummaries: testableSummaries{ + Values: []actionTestableSummary{ + { + Name: name{Value: "test case"}, + Tests: tests{ + Values: []actionTestSummaryGroup{ + {TestStatus: testStatus{Value: "Failure"}}, }, }, }, @@ -143,37 +139,37 @@ func TestActionTestPlanRunSummaries_failuresCount(t *testing.T) { wantFailure: 1, }, } - for _, tt := range tests { + for _, tt := range testCases { t.Run(tt.name, func(t *testing.T) { if gotFailure := tt.summaries.failuresCount(tt.testableSummaryName); gotFailure != tt.wantFailure { - t.Errorf("ActionTestPlanRunSummaries.failuresCount() = %v, want %v", gotFailure, tt.wantFailure) + t.Errorf("actionTestPlanRunSummaries.failuresCount() = %v, want %v", gotFailure, tt.wantFailure) } }) } } func TestActionTestPlanRunSummaries_totalTime(t *testing.T) { - tests := []struct { + testCases := []struct { name string - summaries ActionTestPlanRunSummaries + summaries actionTestPlanRunSummaries testableSummaryName string wantTime float64 }{ { name: "single test", - summaries: ActionTestPlanRunSummaries{ - Summaries: Summaries{ - Values: []Summary{ - Summary{ - TestableSummaries: TestableSummaries{ - Values: []ActionTestableSummary{ - ActionTestableSummary{ - Name: Name{Value: "test case"}, - Tests: Tests{ - Values: []ActionTestSummaryGroup{ - ActionTestSummaryGroup{ - Duration: Duration{Value: "10"}, - TestStatus: TestStatus{Value: "Failure"}, + summaries: actionTestPlanRunSummaries{ + Summaries: summaries{ + Values: []summary{ + { + TestableSummaries: testableSummaries{ + Values: []actionTestableSummary{ + { + Name: name{Value: "test case"}, + Tests: tests{ + Values: []actionTestSummaryGroup{ + { + Duration: duration{Value: "10"}, + TestStatus: testStatus{Value: "Failure"}, }, }, }, @@ -188,10 +184,10 @@ func TestActionTestPlanRunSummaries_totalTime(t *testing.T) { wantTime: 10, }, } - for _, tt := range tests { + for _, tt := range testCases { t.Run(tt.name, func(t *testing.T) { if gotTime := tt.summaries.totalTime(tt.testableSummaryName); gotTime != tt.wantTime { - t.Errorf("ActionTestPlanRunSummaries.totalTime() = %v, want %v", gotTime, tt.wantTime) + t.Errorf("actionTestPlanRunSummaries.totalTime() = %v, want %v", gotTime, tt.wantTime) } }) } diff --git a/testresult/xcresult3/action_test_summary.go b/testresult/xcresult3/action_test_summary.go index 8745d56d..cb3a0f48 100644 --- a/testresult/xcresult3/action_test_summary.go +++ b/testresult/xcresult3/action_test_summary.go @@ -5,8 +5,7 @@ import ( "encoding/hex" ) -// Attachment ... -type Attachment struct { +type attachment struct { Filename struct { Value string `json:"_value"` } `json:"filename"` @@ -18,23 +17,19 @@ type Attachment struct { } `json:"payloadRef"` } -// Attachments ... -type Attachments struct { - Values []Attachment `json:"_values"` +type attachments struct { + Values []attachment `json:"_values"` } -// ActionTestActivitySummary ... -type ActionTestActivitySummary struct { - Attachments Attachments `json:"attachments"` +type actionTestActivitySummary struct { + Attachments attachments `json:"attachments"` } -// ActivitySummaries ... -type ActivitySummaries struct { - Values []ActionTestActivitySummary `json:"_values"` +type activitySummaries struct { + Values []actionTestActivitySummary `json:"_values"` } -// ActionTestFailureSummary ... -type ActionTestFailureSummary struct { +type actionTestFailureSummary struct { Message struct { Value string `json:"_value"` } `json:"message"` @@ -48,18 +43,15 @@ type ActionTestFailureSummary struct { } `json:"lineNumber"` } -// FailureSummaries ... -type FailureSummaries struct { - Values []ActionTestFailureSummary `json:"_values"` +type failureSummaries struct { + Values []actionTestFailureSummary `json:"_values"` } -// Configuration ... -type Configuration struct { +type configuration struct { Hash string } -// UnmarshalJSON ... -func (c *Configuration) UnmarshalJSON(data []byte) error { +func (c *configuration) UnmarshalJSON(data []byte) error { if string(data) == "null" || string(data) == `""` { return nil } @@ -70,9 +62,8 @@ func (c *Configuration) UnmarshalJSON(data []byte) error { return nil } -// ActionTestSummary ... -type ActionTestSummary struct { - ActivitySummaries ActivitySummaries `json:"activitySummaries"` - FailureSummaries FailureSummaries `json:"failureSummaries"` - Configuration Configuration `json:"configuration"` +type actionTestSummary struct { + ActivitySummaries activitySummaries `json:"activitySummaries"` + FailureSummaries failureSummaries `json:"failureSummaries"` + Configuration configuration `json:"configuration"` } diff --git a/testresult/xcresult3/action_test_summary_group.go b/testresult/xcresult3/action_test_summary_group.go index e2900c08..9f1065c2 100644 --- a/testresult/xcresult3/action_test_summary_group.go +++ b/testresult/xcresult3/action_test_summary_group.go @@ -7,50 +7,42 @@ import ( "strings" ) -// ErrSummaryNotFound ... -var ErrSummaryNotFound = errors.New("no summaryRef.ID.Value found for test case") - -// ActionTestSummaryGroup ... -type ActionTestSummaryGroup struct { - Name Name `json:"name"` - Identifier Identifier `json:"identifier"` - Duration Duration `json:"duration"` - TestStatus TestStatus `json:"testStatus"` // only the inner-most tests will have a status, the ones which don't have "subtests" - SummaryRef SummaryRef `json:"summaryRef"` // only the inner-most tests will have a summaryRef, the ones which don't have "subtests" - Subtests Subtests `json:"subtests"` +var errSummaryNotFound = errors.New("no summaryRef.ID.Value found for test case") + +type actionTestSummaryGroup struct { + Name name `json:"name"` + Identifier identifier `json:"identifier"` + Duration duration `json:"duration"` + TestStatus testStatus `json:"testStatus"` // only the inner-most tests will have a status, the ones which don't have "subtests" + SummaryRef summaryRef `json:"summaryRef"` // only the inner-most tests will have a summaryRef, the ones which don't have "subtests" + Subtests subtests `json:"subtests"` } -// Subtests ... -type Subtests struct { - Values []ActionTestSummaryGroup `json:"_values"` +type subtests struct { + Values []actionTestSummaryGroup `json:"_values"` } -// ID ... -type ID struct { +type id struct { Value string `json:"_value"` } -// SummaryRef ... -type SummaryRef struct { - ID ID `json:"id"` +type summaryRef struct { + ID id `json:"id"` } -// TestStatus ... -type TestStatus struct { +type testStatus struct { Value string `json:"_value"` } -// Duration ... -type Duration struct { +type duration struct { Value string `json:"_value"` } -// Identifier ... -type Identifier struct { +type identifier struct { Value string `json:"_value"` } -func (g ActionTestSummaryGroup) references() (class, method string) { +func (g actionTestSummaryGroup) references() (class, method string) { // Xcode11TestUITests2/testFail() if g.Identifier.Value != "" { s := strings.Split(g.Identifier.Value, "/") @@ -61,33 +53,31 @@ func (g ActionTestSummaryGroup) references() (class, method string) { return } -// testsWithStatus returns ActionTestSummaryGroup with TestStatus defined. -func (g ActionTestSummaryGroup) testsWithStatus() (tests []ActionTestSummaryGroup) { +// testsWithStatus returns actionTestSummaryGroup entries with TestStatus set. +func (g actionTestSummaryGroup) testsWithStatus() (result []actionTestSummaryGroup) { if g.TestStatus.Value != "" { - tests = append(tests, g) + result = append(result, g) } for _, subtest := range g.Subtests.Values { - tests = append(tests, subtest.testsWithStatus()...) + result = append(result, subtest.testsWithStatus()...) } return } -// loadActionTestSummary ... -func (g ActionTestSummaryGroup) loadActionTestSummary(xcresultPath string, useLegacyFlag bool) (ActionTestSummary, error) { +func (g actionTestSummaryGroup) loadActionTestSummary(xcresultPath string, useLegacyFlag bool) (actionTestSummary, error) { if g.SummaryRef.ID.Value == "" { - return ActionTestSummary{}, ErrSummaryNotFound + return actionTestSummary{}, errSummaryNotFound } - var summary ActionTestSummary + var summary actionTestSummary if err := xcresulttoolGet(xcresultPath, g.SummaryRef.ID.Value, useLegacyFlag, &summary); err != nil { - return ActionTestSummary{}, fmt.Errorf("failed to load ActionTestSummary: %w", err) + return actionTestSummary{}, fmt.Errorf("failed to load action test summary: %w", err) } return summary, nil } -// exportScreenshots ... -func (g ActionTestSummaryGroup) exportScreenshots(resultPth, outputDir string, useLegacyFlag bool) error { +func (g actionTestSummaryGroup) exportScreenshots(resultPth, outputDir string, useLegacyFlag bool) error { if g.TestStatus.Value == "" { return nil } @@ -96,7 +86,7 @@ func (g ActionTestSummaryGroup) exportScreenshots(resultPth, outputDir string, u return nil } - var summary ActionTestSummary + var summary actionTestSummary if err := xcresulttoolGet(resultPth, g.SummaryRef.ID.Value, useLegacyFlag, &summary); err != nil { return err } diff --git a/testresult/xcresult3/action_test_summary_group_test.go b/testresult/xcresult3/action_test_summary_group_test.go index e11687b7..557bb88a 100644 --- a/testresult/xcresult3/action_test_summary_group_test.go +++ b/testresult/xcresult3/action_test_summary_group_test.go @@ -6,7 +6,7 @@ import ( ) func TestActionTestSummaryGroup_references(t *testing.T) { - tests := []struct { + testCases := []struct { name string identifier string wantClass string @@ -25,60 +25,58 @@ func TestActionTestSummaryGroup_references(t *testing.T) { wantClass: "", }, } - for _, tt := range tests { + for _, tt := range testCases { t.Run(tt.name, func(t *testing.T) { - g := ActionTestSummaryGroup{} + g := actionTestSummaryGroup{} g.Identifier.Value = tt.identifier gotClass, gotMethod := g.references() if gotClass != tt.wantClass { - t.Errorf("ActionTestSummaryGroup.references() gotClass = %v, want %v", gotClass, tt.wantClass) + t.Errorf("actionTestSummaryGroup.references() gotClass = %v, want %v", gotClass, tt.wantClass) } if gotMethod != tt.wantMethod { - t.Errorf("ActionTestSummaryGroup.references() gotMethod = %v, want %v", gotMethod, tt.wantMethod) + t.Errorf("actionTestSummaryGroup.references() gotMethod = %v, want %v", gotMethod, tt.wantMethod) } }) } } func TestActionTestSummaryGroup_testsWithStatus(t *testing.T) { - - tests := []struct { + testCases := []struct { name string - group ActionTestSummaryGroup - subtests []ActionTestSummaryGroup - wantGroups []ActionTestSummaryGroup + group actionTestSummaryGroup + wantGroups []actionTestSummaryGroup }{ { - name: "status in the root ActionTestSummaryGroup", - group: ActionTestSummaryGroup{ - TestStatus: TestStatus{Value: "success"}, + name: "status in the root actionTestSummaryGroup", + group: actionTestSummaryGroup{ + TestStatus: testStatus{Value: "success"}, }, - wantGroups: []ActionTestSummaryGroup{ActionTestSummaryGroup{TestStatus: TestStatus{Value: "success"}}}, + wantGroups: []actionTestSummaryGroup{{TestStatus: testStatus{Value: "success"}}}, }, { - name: "status in a sub ActionTestSummaryGroup", - group: ActionTestSummaryGroup{ - Subtests: Subtests{ - Values: []ActionTestSummaryGroup{ - ActionTestSummaryGroup{TestStatus: TestStatus{Value: "success"}}, + name: "status in a sub actionTestSummaryGroup", + group: actionTestSummaryGroup{ + Subtests: subtests{ + Values: []actionTestSummaryGroup{ + {TestStatus: testStatus{Value: "success"}}, }, }, }, - wantGroups: []ActionTestSummaryGroup{ActionTestSummaryGroup{TestStatus: TestStatus{Value: "success"}}}, + wantGroups: []actionTestSummaryGroup{{TestStatus: testStatus{Value: "success"}}}, }, { name: "no status", - group: ActionTestSummaryGroup{}, + group: actionTestSummaryGroup{}, wantGroups: nil, }, } - for _, tt := range tests { + for _, tt := range testCases { t.Run(tt.name, func(t *testing.T) { gotGroups := tt.group.testsWithStatus() if !reflect.DeepEqual(gotGroups, tt.wantGroups) { - t.Errorf("ActionTestSummaryGroup.testsWithStatus() gotTarget = %v, want %v", gotGroups, tt.wantGroups) + t.Errorf("actionTestSummaryGroup.testsWithStatus() gotTarget = %v, want %v", gotGroups, tt.wantGroups) } }) } diff --git a/testresult/xcresult3/converter.go b/testresult/xcresult3/converter.go index 53ae2c89..0b635b10 100644 --- a/testresult/xcresult3/converter.go +++ b/testresult/xcresult3/converter.go @@ -139,7 +139,7 @@ func legacyParse(path string, useLegacyFlag bool) (testreport.TestReport, error) log.Debugf("Maximum parallelism: %d.", maxParallel) - _, summaries, err := Parse(path, useLegacyFlag) + _, summaries, err := loadXCResultData(path, useLegacyFlag) if err != nil { return testreport.TestReport{}, err } @@ -384,7 +384,7 @@ func connectAttachmentsToTestCases(xml testreport.TestReport, attachmentsMap map return xml, nil } -func testSuiteCountInSummaries(summaries []ActionTestPlanRunSummaries) int { +func testSuiteCountInSummaries(summaries []actionTestPlanRunSummaries) int { testSuiteCount := 0 for _, summary := range summaries { testSuiteOrder, _ := summary.tests() @@ -394,8 +394,8 @@ func testSuiteCountInSummaries(summaries []ActionTestPlanRunSummaries) int { } func genTestSuite(name string, - summary ActionTestPlanRunSummaries, - tests []ActionTestSummaryGroup, + summary actionTestPlanRunSummaries, + tests []actionTestSummaryGroup, testResultDir string, xcresultPath string, maxParallel int, @@ -423,7 +423,7 @@ func genTestSuite(name string, test := tests[testIdx] wg.Add(1) - go func(test ActionTestSummaryGroup, testIdx int) { + go func(test actionTestSummaryGroup, testIdx int) { defer wg.Done() testCase, err := genTestCase(test, xcresultPath, testResultDir, useLegacyFlag) @@ -447,7 +447,7 @@ func genTestSuite(name string, return testSuite, genTestSuiteErr } -func genTestCase(test ActionTestSummaryGroup, xcresultPath, testResultDir string, useLegacyFlag bool) (testreport.TestCase, error) { +func genTestCase(test actionTestSummaryGroup, xcresultPath, testResultDir string, useLegacyFlag bool) (testreport.TestCase, error) { var duartion float64 if test.Duration.Value != "" { var err error @@ -462,7 +462,7 @@ func genTestCase(test ActionTestSummaryGroup, xcresultPath, testResultDir string // For example, failed tests will always have a summary, but successful ones might have it or might not. // If they do not have it, then that means that they did not log anything to the console, // and they were not executed as device configuration tests. - if err != nil && !errors.Is(err, ErrSummaryNotFound) { + if err != nil && !errors.Is(err, errSummaryNotFound) { return testreport.TestCase{}, err } diff --git a/testresult/xcresult3/xcresult3.go b/testresult/xcresult3/xcresult3.go index 59f700a5..0484aaf5 100644 --- a/testresult/xcresult3/xcresult3.go +++ b/testresult/xcresult3/xcresult3.go @@ -2,17 +2,17 @@ package xcresult3 import "github.com/bitrise-io/go-xcode/v2/testresult/xcresult3/model3" -// Parse parses the given xcresult file's ActionsInvocationRecord and the list of ActionTestPlanRunSummaries. -func Parse(pth string, useLegacyFlag bool) (*ActionsInvocationRecord, []ActionTestPlanRunSummaries, error) { - var r ActionsInvocationRecord +// loadXCResultData loads the actions invocation record and test plan run summaries from an xcresult file. +func loadXCResultData(pth string, useLegacyFlag bool) (*actionsInvocationRecord, []actionTestPlanRunSummaries, error) { + var r actionsInvocationRecord if err := xcresulttoolGet(pth, "", useLegacyFlag, &r); err != nil { return nil, nil, err } - var summaries []ActionTestPlanRunSummaries + var summaries []actionTestPlanRunSummaries for _, action := range r.Actions.Values { refID := action.ActionResult.TestsRef.ID.Value - var s ActionTestPlanRunSummaries + var s actionTestPlanRunSummaries if err := xcresulttoolGet(pth, refID, useLegacyFlag, &s); err != nil { return nil, nil, err } From 9b7ed0334e62c803cc89ceafec1e102283b5d36a Mon Sep 17 00:00:00 2001 From: Gergely Sallai <1516628+gergely-sallai@users.noreply.github.com> Date: Wed, 11 Mar 2026 14:55:53 +0100 Subject: [PATCH 08/11] style: STEP-2230 Adding missing docs. This has to stay exported, was too eager to remove its doc. --- testresult/xcresult3/action_test_summary.go | 1 + 1 file changed, 1 insertion(+) diff --git a/testresult/xcresult3/action_test_summary.go b/testresult/xcresult3/action_test_summary.go index cb3a0f48..0b84ff73 100644 --- a/testresult/xcresult3/action_test_summary.go +++ b/testresult/xcresult3/action_test_summary.go @@ -51,6 +51,7 @@ type configuration struct { Hash string } +// UnmarshalJSON implements json.Unmarshaler by hashing the raw JSON bytes into an MD5 fingerprint. func (c *configuration) UnmarshalJSON(data []byte) error { if string(data) == "null" || string(data) == `""` { return nil From 5213353decdc3957f79ead1417ff736b65b4f75e Mon Sep 17 00:00:00 2001 From: Gergely Sallai <1516628+gergely-sallai@users.noreply.github.com> Date: Wed, 11 Mar 2026 15:54:28 +0100 Subject: [PATCH 09/11] chore: update to new go-xcode v1 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 064bf4cb..168d9828 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/bitrise-io/go-steputils/v2 v2.0.0-alpha.48.0.20260311110650-2026f65db4da github.com/bitrise-io/go-utils v1.0.13 github.com/bitrise-io/go-utils/v2 v2.0.0-alpha.33 - github.com/bitrise-io/go-xcode v1.3.3-0.20260309170849-9b6f618441f7 + github.com/bitrise-io/go-xcode v1.3.3 github.com/globocom/go-buffer/v2 v2.0.0 github.com/golang-jwt/jwt/v4 v4.5.0 github.com/google/go-querystring v1.1.0 diff --git a/go.sum b/go.sum index 1f938730..f2e5e4ab 100644 --- a/go.sum +++ b/go.sum @@ -45,8 +45,8 @@ github.com/bitrise-io/go-utils v1.0.13 h1:1QENhTS/JlKH9F7+/nB+TtbTcor6jGrE6cQ4CJ github.com/bitrise-io/go-utils v1.0.13/go.mod h1:ZY1DI+fEpZuFpO9szgDeICM4QbqoWVt0RSY3tRI1heY= github.com/bitrise-io/go-utils/v2 v2.0.0-alpha.33 h1:2Skyp4yg8aNKLr5GB5amM9UK9n1yzIMT88Rb/ZBz8m4= github.com/bitrise-io/go-utils/v2 v2.0.0-alpha.33/go.mod h1:3XUplo0dOWc3DqT2XA2SeHToDSg7+j1y1HTHibT2H68= -github.com/bitrise-io/go-xcode v1.3.3-0.20260309170849-9b6f618441f7 h1:hTsfgezmMkCGeHV/+KPXxT8A4R3o4UGzrooJnfmNPtA= -github.com/bitrise-io/go-xcode v1.3.3-0.20260309170849-9b6f618441f7/go.mod h1:9OwsvrhZ4A2JxHVoEY7CPcABAKA+OE7FQqFfBfvbFuY= +github.com/bitrise-io/go-xcode v1.3.3 h1:aYkSMWP+1/n2ZabRy3OMfeaWmE4l1gAPq63azx06LIw= +github.com/bitrise-io/go-xcode v1.3.3/go.mod h1:9OwsvrhZ4A2JxHVoEY7CPcABAKA+OE7FQqFfBfvbFuY= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= From 3291dafef993a3ba555099393862a6f68dbb875a Mon Sep 17 00:00:00 2001 From: Gergely Sallai <1516628+gergely-sallai@users.noreply.github.com> Date: Thu, 12 Mar 2026 10:14:24 +0100 Subject: [PATCH 10/11] feat!: STEP-2230 Remove Setup from Converter interface Reasons: - This was no-op for most of our converters - The parameter (the reason the func was introduced in the first place I assume) is closely tied to xcresult3 conversion, which is a close implementation detail, not something that belongs in a general interface. --- go.mod | 2 +- go.sum | 2 ++ testresult/xcresult/xcresult.go | 3 --- testresult/xcresult3/converter.go | 6 +++--- 4 files changed, 6 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index 168d9828..2aa8b017 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( cloud.google.com/go/storage v1.50.0 github.com/bitrise-io/go-plist v0.0.0-20210301100253-4b1a112ccd10 github.com/bitrise-io/go-steputils v1.0.5 - github.com/bitrise-io/go-steputils/v2 v2.0.0-alpha.48.0.20260311110650-2026f65db4da + github.com/bitrise-io/go-steputils/v2 v2.0.0-alpha.48.0.20260312091018-7447bc60506b github.com/bitrise-io/go-utils v1.0.13 github.com/bitrise-io/go-utils/v2 v2.0.0-alpha.33 github.com/bitrise-io/go-xcode v1.3.3 diff --git a/go.sum b/go.sum index f2e5e4ab..8b766151 100644 --- a/go.sum +++ b/go.sum @@ -40,6 +40,8 @@ github.com/bitrise-io/go-steputils v1.0.5 h1:OBH7CPXeqIWFWJw6BOUMQnUb8guspwKr2Rh github.com/bitrise-io/go-steputils v1.0.5/go.mod h1:YIUaQnIAyK4pCvQG0hYHVkSzKNT9uL2FWmkFNW4mfNI= github.com/bitrise-io/go-steputils/v2 v2.0.0-alpha.48.0.20260311110650-2026f65db4da h1:d05p0A4po0o5nytc1GaRNsjq1ktqjmNa7FeGp58DvmY= github.com/bitrise-io/go-steputils/v2 v2.0.0-alpha.48.0.20260311110650-2026f65db4da/go.mod h1:CL1sOqz4+q4XK/OCjB8YNV27Xmz/Fo7v/QKxobmGIx4= +github.com/bitrise-io/go-steputils/v2 v2.0.0-alpha.48.0.20260312091018-7447bc60506b h1:GIRdLWgdpa+qImAJlfuHAQppI7EFieCyNceVcUwNbo4= +github.com/bitrise-io/go-steputils/v2 v2.0.0-alpha.48.0.20260312091018-7447bc60506b/go.mod h1:CL1sOqz4+q4XK/OCjB8YNV27Xmz/Fo7v/QKxobmGIx4= github.com/bitrise-io/go-utils v1.0.1/go.mod h1:ZY1DI+fEpZuFpO9szgDeICM4QbqoWVt0RSY3tRI1heY= github.com/bitrise-io/go-utils v1.0.13 h1:1QENhTS/JlKH9F7+/nB+TtbTcor6jGrE6cQ4CJWfp5U= github.com/bitrise-io/go-utils v1.0.13/go.mod h1:ZY1DI+fEpZuFpO9szgDeICM4QbqoWVt0RSY3tRI1heY= diff --git a/testresult/xcresult/xcresult.go b/testresult/xcresult/xcresult.go index afac2843..604eadf9 100644 --- a/testresult/xcresult/xcresult.go +++ b/testresult/xcresult/xcresult.go @@ -18,9 +18,6 @@ type Converter struct { testSummariesPlistPath string } -// Setup configures the converter. -func (c *Converter) Setup(_ bool) {} - // Detect ... func (c *Converter) Detect(files []string) bool { c.files = files diff --git a/testresult/xcresult3/converter.go b/testresult/xcresult3/converter.go index 0b635b10..9535542e 100644 --- a/testresult/xcresult3/converter.go +++ b/testresult/xcresult3/converter.go @@ -58,9 +58,9 @@ func documentMajorVersion(pth string) (int, error) { return majorVersion(info) } -// Setup configures the converter with the given extraction method preference. -func (c *Converter) Setup(useOldXCResultExtractionMethod bool) { - c.useLegacyExtractionMethod = useOldXCResultExtractionMethod +// NewConverter creates a Converter with the given extraction method preference. +func NewConverter(useLegacy bool) *Converter { + return &Converter{useLegacyExtractionMethod: useLegacy} } // Detect ... From 0c898dd1f2a32f66dc4fc7a960ce0d5383aab489 Mon Sep 17 00:00:00 2001 From: Gergely Sallai <1516628+gergely-sallai@users.noreply.github.com> Date: Thu, 12 Mar 2026 11:22:56 +0100 Subject: [PATCH 11/11] fix: STEP-2230 Revert making test.Subtest private It was causing test failures in deploy-to-bitrise-io step during plist parsing --- testresult/xcresult/testsummariesplist.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/testresult/xcresult/testsummariesplist.go b/testresult/xcresult/testsummariesplist.go index 886db80c..97d2069a 100644 --- a/testresult/xcresult/testsummariesplist.go +++ b/testresult/xcresult/testsummariesplist.go @@ -28,7 +28,7 @@ func (summaryPlist testSummaryPlist) tests() ([]string, map[string]subtests) { var subTests subtests for _, testableSummary := range summaryPlist.TestableSummaries { for _, test := range testableSummary.Tests { - subTests = append(subTests, collapsesubtestTree(test.subtests)...) + subTests = append(subTests, collapsesubtestTree(test.Subtests)...) } } for _, test := range subTests { @@ -43,7 +43,7 @@ func (summaryPlist testSummaryPlist) tests() ([]string, map[string]subtests) { } type test struct { - subtests subtests + Subtests subtests } type testableSummary struct {