diff --git a/go.mod b/go.mod index 56c4262b..2aa8b017 100644 --- a/go.mod +++ b/go.mod @@ -9,10 +9,10 @@ require ( cloud.google.com/go/storage v1.50.0 github.com/bitrise-io/go-plist v0.0.0-20210301100253-4b1a112ccd10 github.com/bitrise-io/go-steputils v1.0.5 - github.com/bitrise-io/go-steputils/v2 v2.0.0-alpha.43 + github.com/bitrise-io/go-steputils/v2 v2.0.0-alpha.48.0.20260312091018-7447bc60506b github.com/bitrise-io/go-utils v1.0.13 - github.com/bitrise-io/go-utils/v2 v2.0.0-alpha.31 - github.com/bitrise-io/go-xcode v1.3.3-0.20260309170849-9b6f618441f7 + github.com/bitrise-io/go-utils/v2 v2.0.0-alpha.33 + github.com/bitrise-io/go-xcode v1.3.3 github.com/globocom/go-buffer/v2 v2.0.0 github.com/golang-jwt/jwt/v4 v4.5.0 github.com/google/go-querystring v1.1.0 @@ -23,6 +23,7 @@ require ( golang.org/x/oauth2 v0.24.0 golang.org/x/text v0.21.0 google.golang.org/api v0.214.0 + howett.net/plist v1.0.0 ) require ( @@ -79,5 +80,4 @@ require ( google.golang.org/grpc v1.67.3 // indirect google.golang.org/protobuf v1.35.2 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - howett.net/plist v1.0.0 // indirect ) diff --git a/go.sum b/go.sum index 9490b2cd..8b766151 100644 --- a/go.sum +++ b/go.sum @@ -38,15 +38,17 @@ github.com/bitrise-io/go-plist v0.0.0-20210301100253-4b1a112ccd10 h1:/2OyBFI7GjY github.com/bitrise-io/go-plist v0.0.0-20210301100253-4b1a112ccd10/go.mod h1:pARutiL3kEuRLV3JvswidvfCj+9Y3qMZtji2BDqLFsA= github.com/bitrise-io/go-steputils v1.0.5 h1:OBH7CPXeqIWFWJw6BOUMQnUb8guspwKr2RhYBhM9tfc= github.com/bitrise-io/go-steputils v1.0.5/go.mod h1:YIUaQnIAyK4pCvQG0hYHVkSzKNT9uL2FWmkFNW4mfNI= -github.com/bitrise-io/go-steputils/v2 v2.0.0-alpha.43 h1:oahoCL46PPywHRBin54zrwDOhXlMPIXx6zdo1Hgkz1g= -github.com/bitrise-io/go-steputils/v2 v2.0.0-alpha.43/go.mod h1:SjWTgoD5wDyyIa+xPrA+U2UgL9K8Lx/xuLaK5LBCjXE= +github.com/bitrise-io/go-steputils/v2 v2.0.0-alpha.48.0.20260311110650-2026f65db4da h1:d05p0A4po0o5nytc1GaRNsjq1ktqjmNa7FeGp58DvmY= +github.com/bitrise-io/go-steputils/v2 v2.0.0-alpha.48.0.20260311110650-2026f65db4da/go.mod h1:CL1sOqz4+q4XK/OCjB8YNV27Xmz/Fo7v/QKxobmGIx4= +github.com/bitrise-io/go-steputils/v2 v2.0.0-alpha.48.0.20260312091018-7447bc60506b h1:GIRdLWgdpa+qImAJlfuHAQppI7EFieCyNceVcUwNbo4= +github.com/bitrise-io/go-steputils/v2 v2.0.0-alpha.48.0.20260312091018-7447bc60506b/go.mod h1:CL1sOqz4+q4XK/OCjB8YNV27Xmz/Fo7v/QKxobmGIx4= github.com/bitrise-io/go-utils v1.0.1/go.mod h1:ZY1DI+fEpZuFpO9szgDeICM4QbqoWVt0RSY3tRI1heY= github.com/bitrise-io/go-utils v1.0.13 h1:1QENhTS/JlKH9F7+/nB+TtbTcor6jGrE6cQ4CJWfp5U= github.com/bitrise-io/go-utils v1.0.13/go.mod h1:ZY1DI+fEpZuFpO9szgDeICM4QbqoWVt0RSY3tRI1heY= -github.com/bitrise-io/go-utils/v2 v2.0.0-alpha.31 h1:Lay9mco4/T88oYY+kqZlpdWeU9aj32/qWMRwcTg812o= -github.com/bitrise-io/go-utils/v2 v2.0.0-alpha.31/go.mod h1:3XUplo0dOWc3DqT2XA2SeHToDSg7+j1y1HTHibT2H68= -github.com/bitrise-io/go-xcode v1.3.3-0.20260309170849-9b6f618441f7 h1:hTsfgezmMkCGeHV/+KPXxT8A4R3o4UGzrooJnfmNPtA= -github.com/bitrise-io/go-xcode v1.3.3-0.20260309170849-9b6f618441f7/go.mod h1:9OwsvrhZ4A2JxHVoEY7CPcABAKA+OE7FQqFfBfvbFuY= +github.com/bitrise-io/go-utils/v2 v2.0.0-alpha.33 h1:2Skyp4yg8aNKLr5GB5amM9UK9n1yzIMT88Rb/ZBz8m4= +github.com/bitrise-io/go-utils/v2 v2.0.0-alpha.33/go.mod h1:3XUplo0dOWc3DqT2XA2SeHToDSg7+j1y1HTHibT2H68= +github.com/bitrise-io/go-xcode v1.3.3 h1:aYkSMWP+1/n2ZabRy3OMfeaWmE4l1gAPq63azx06LIw= +github.com/bitrise-io/go-xcode v1.3.3/go.mod h1:9OwsvrhZ4A2JxHVoEY7CPcABAKA+OE7FQqFfBfvbFuY= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= diff --git a/testresult/xcresult/testsummariesplist.go b/testresult/xcresult/testsummariesplist.go new file mode 100644 index 00000000..97d2069a --- /dev/null +++ b/testresult/xcresult/testsummariesplist.go @@ -0,0 +1,112 @@ +package xcresult + +import ( + "fmt" + "strings" +) + +type testSummaryPlist struct { + FormatVersion string + TestableSummaries []testableSummary +} + +func collapsesubtestTree(data subtests) (tests subtests) { + for _, test := range data { + if len(test.Subtests) > 0 { + tests = append(tests, collapsesubtestTree(test.Subtests)...) + } + if test.TestStatus != "" { + tests = append(tests, test) + } + } + return +} + +func (summaryPlist testSummaryPlist) tests() ([]string, map[string]subtests) { + var keyOrder []string + tests := map[string]subtests{} + var subTests subtests + for _, testableSummary := range summaryPlist.TestableSummaries { + for _, test := range testableSummary.Tests { + subTests = append(subTests, collapsesubtestTree(test.Subtests)...) + } + } + for _, test := range subTests { + // TestIdentifier is in a format of testID/testCase + testID := strings.Split(test.TestIdentifier, "/")[0] + if _, found := tests[testID]; !found { + keyOrder = append(keyOrder, testID) + } + tests[testID] = append(tests[testID], test) + } + return keyOrder, tests +} + +type test struct { + Subtests subtests +} + +type testableSummary struct { + TargetName string + TestKind string + TestName string + TestObjectClass string + Tests []test +} + +type failureSummary struct { + FileName string + LineNumber int + Message string + PerformanceFailure bool +} + +type subtest struct { + Duration float64 + TestStatus string + TestIdentifier string + TestName string + TestObjectClass string + Subtests subtests + FailureSummaries []failureSummary +} + +func (st subtest) failure() (message string) { + prefix := "" + for _, failure := range st.FailureSummaries { + message += fmt.Sprintf("%s%s:%d - %s", prefix, failure.FileName, failure.LineNumber, failure.Message) + prefix = "\n" + } + return +} + +func (st subtest) skipped() bool { + return st.TestStatus == "Skipped" +} + +type subtests []subtest + +func (sts subtests) failuresCount() (count int) { + for _, test := range sts { + if len(test.FailureSummaries) > 0 { + count++ + } + } + return count +} + +func (sts subtests) skippedCount() (count int) { + for _, test := range sts { + if test.skipped() { + count++ + } + } + return count +} + +func (sts subtests) totalTime() (time float64) { + for _, test := range sts { + time += test.Duration + } + return time +} diff --git a/testresult/xcresult/xcresult.go b/testresult/xcresult/xcresult.go new file mode 100644 index 00000000..604eadf9 --- /dev/null +++ b/testresult/xcresult/xcresult.go @@ -0,0 +1,119 @@ +package xcresult + +import ( + "path/filepath" + "strings" + "unicode" + + "howett.net/plist" + + "github.com/bitrise-io/go-utils/fileutil" + "github.com/bitrise-io/go-utils/pathutil" + "github.com/bitrise-io/go-steputils/v2/testreport" +) + +// Converter ... +type Converter struct { + files []string + testSummariesPlistPath string +} + +// Detect ... +func (c *Converter) Detect(files []string) bool { + c.files = files + for _, file := range c.files { + if filepath.Ext(file) == ".xcresult" { + testSummariesPlistPath := filepath.Join(file, "TestSummaries.plist") + if exist, err := pathutil.IsPathExists(testSummariesPlistPath); err != nil || !exist { + continue + } + + c.testSummariesPlistPath = testSummariesPlistPath + return true + } + } + return false +} + +// by one of our issue reports, need to replace backspace char (U+0008) as it is an invalid character for xml unmarshaller +// the legal character ranges are here: https://www.w3.org/TR/REC-xml/#charsets +// so the exclusion will be: +/* + \u0000 - \u0008 + \u000B + \u000C + \u000E - \u001F + \u007F - \u0084 + \u0086 - \u009F + \uD800 - \uDFFF + + Unicode range D800–DFFF is used as surrogate pair. Unicode and ISO/IEC 10646 do not assign characters to any of the code points in the D800–DFFF range, so an individual code value from a surrogate pair does not represent a character. (A couple of code points — the first from the high surrogate area (D800–DBFF), and the second from the low surrogate area (DC00–DFFF) — are used in UTF-16 to represent a character in supplementary planes) + \uFDD0 - \uFDEF; \uFFFE; \uFFFF +*/ +// These are non-characters in the standard, not assigned to anything; and have no meaning. +func filterIllegalChars(data []byte) (filtered []byte) { + illegalCharFilter := func(r rune) rune { + if unicode.IsPrint(r) { + return r + } + return -1 + } + filtered = []byte(strings.Map(illegalCharFilter, string(data))) + return +} + +// Convert returns the test report parsed from the xcresult file. +func (c *Converter) Convert() (testreport.TestReport, error) { + data, err := fileutil.ReadBytesFromFile(c.testSummariesPlistPath) + if err != nil { + return testreport.TestReport{}, err + } + + data = filterIllegalChars(data) + + var plistData testSummaryPlist + if _, err := plist.Unmarshal(data, &plistData); err != nil { + return testreport.TestReport{}, err + } + + var xmlData testreport.TestReport + keyOrder, tests := plistData.tests() + for _, testID := range keyOrder { + tests := tests[testID] + testSuite := testreport.TestSuite{ + Name: testID, + Tests: len(tests), + Failures: tests.failuresCount(), + Skipped: tests.skippedCount(), + Time: tests.totalTime(), + } + + for _, test := range tests { + failureMessage := test.failure() + + var failure *testreport.Failure + if len(failureMessage) > 0 { + failure = &testreport.Failure{ + Value: failureMessage, + } + } + + var skipped *testreport.Skipped + if test.skipped() { + skipped = &testreport.Skipped{} + } + + testSuite.TestCases = append(testSuite.TestCases, testreport.TestCase{ + Name: test.TestName, + ClassName: testID, + Failure: failure, + Skipped: skipped, + Time: test.Duration, + }) + } + + xmlData.TestSuites = append(xmlData.TestSuites, testSuite) + } + + return xmlData, nil +} diff --git a/testresult/xcresult/xcresult_test.go b/testresult/xcresult/xcresult_test.go new file mode 100644 index 00000000..a36158b1 --- /dev/null +++ b/testresult/xcresult/xcresult_test.go @@ -0,0 +1,15 @@ +package xcresult + +import ( + "reflect" + "testing" +) + +func Test_filterIllegalChars(t *testing.T) { + // \b == /u0008 -> backspace + content := []byte("test\b text") + + if !reflect.DeepEqual(filterIllegalChars(content), []byte("test text")) { + t.Fatal("illegal character is not removed") + } +} diff --git a/testresult/xcresult3/action_invocation_record.go b/testresult/xcresult3/action_invocation_record.go new file mode 100644 index 00000000..2aa929b8 --- /dev/null +++ b/testresult/xcresult3/action_invocation_record.go @@ -0,0 +1,89 @@ +package xcresult3 + +import ( + "fmt" + "strings" + + "github.com/bitrise-io/go-steputils/v2/testreport" +) + +type actionsInvocationRecord struct { + Actions struct { + Values []struct { + ActionResult struct { + TestsRef struct { + ID struct { + Value string `json:"_value"` + } `json:"id"` + } `json:"testsRef"` + } `json:"actionResult"` + } `json:"_values"` + } `json:"actions"` + + Issues issues `json:"issues"` +} + +type issues struct { + TestFailureSummaries testFailureSummaries `json:"testFailureSummaries"` +} + +type testFailureSummaries struct { + Values []testFailureSummary `json:"_values"` +} + +type testFailureSummary struct { + DocumentLocationInCreatingWorkspace documentLocationInCreatingWorkspace `json:"documentLocationInCreatingWorkspace"` + Message message `json:"message"` + ProducingTarget producingTarget `json:"producingTarget"` + TestCaseName testCaseName `json:"testCaseName"` +} + +type url struct { + Value string `json:"_value"` +} + +type documentLocationInCreatingWorkspace struct { + URL url `json:"url"` +} + +type producingTarget struct { + Value string `json:"_value"` +} + +type testCaseName struct { + Value string `json:"_value"` +} + +type message struct { + Value string `json:"_value"` +} + +func testCaseMatching(test actionTestSummaryGroup, tcName string) bool { + class, method := test.references() + + return tcName == class+"."+method || + tcName == fmt.Sprintf("-[%s %s]", class, method) +} + +// failure returns the failure reason for the given test from the invocation record. +func (r actionsInvocationRecord) failure(test actionTestSummaryGroup, testSuite testreport.TestSuite) string { + for _, failureSummary := range r.Issues.TestFailureSummaries.Values { + if failureSummary.ProducingTarget.Value == testSuite.Name && testCaseMatching(test, failureSummary.TestCaseName.Value) { + file, line := failureSummary.fileAndLineNumber() + return fmt.Sprintf("%s:%s - %s", file, line, failureSummary.Message.Value) + } + } + return "" +} + +// fileAndLineNumber unwraps the file path and line number from the document location URL. +func (s testFailureSummary) fileAndLineNumber() (file string, line string) { + // file:\/\/\/Users\/bitrisedeveloper\/Develop\/ios\/Xcode11Test\/Xcode11TestUITests\/Xcode11TestUITests.swift#CharacterRangeLen=0&EndingLineNumber=42&StartingLineNumber=42 + if s.DocumentLocationInCreatingWorkspace.URL.Value != "" { + i := strings.LastIndex(s.DocumentLocationInCreatingWorkspace.URL.Value, "#") + if i > -1 && i+1 < len(s.DocumentLocationInCreatingWorkspace.URL.Value) { + return s.DocumentLocationInCreatingWorkspace.URL.Value[:i], s.DocumentLocationInCreatingWorkspace.URL.Value[i+1:] + } + } + return +} diff --git a/testresult/xcresult3/action_invocation_record_test.go b/testresult/xcresult3/action_invocation_record_test.go new file mode 100644 index 00000000..918c7e13 --- /dev/null +++ b/testresult/xcresult3/action_invocation_record_test.go @@ -0,0 +1,124 @@ +package xcresult3 + +import ( + "testing" + + "github.com/bitrise-io/go-steputils/v2/testreport" +) + +func TestTestFailureSummary_fileAndLineNumber(t *testing.T) { + testCases := []struct { + name string + summary testFailureSummary + wantFile string + wantLine string + }{ + { + name: "", + summary: testFailureSummary{ + DocumentLocationInCreatingWorkspace: documentLocationInCreatingWorkspace{ + URL: url{Value: "file:/Xcode11TestUITests2.swift#CharacterRangeLen=0&EndingLineNumber=33&StartingLineNumber=33"}, + }, + }, + wantFile: "file:/Xcode11TestUITests2.swift", + wantLine: "CharacterRangeLen=0&EndingLineNumber=33&StartingLineNumber=33", + }, + } + for _, tt := range testCases { + t.Run(tt.name, func(t *testing.T) { + gotFile, gotLine := tt.summary.fileAndLineNumber() + if gotFile != tt.wantFile { + t.Errorf("testFailureSummary.fileAndLineNumber() gotFile = %v, want %v", gotFile, tt.wantFile) + } + if gotLine != tt.wantLine { + t.Errorf("testFailureSummary.fileAndLineNumber() gotLine = %v, want %v", gotLine, tt.wantLine) + } + }) + } +} + +func TestActionsInvocationRecord_failure(t *testing.T) { + testCases := []struct { + name string + record actionsInvocationRecord + test actionTestSummaryGroup + want string + }{ + { + name: "Simple test", + record: actionsInvocationRecord{ + Issues: issues{ + TestFailureSummaries: testFailureSummaries{ + Values: []testFailureSummary{ + { + ProducingTarget: producingTarget{Value: "Xcode11TestUITests2"}, + TestCaseName: testCaseName{Value: "Xcode11TestUITests2.testFail()"}, + Message: message{Value: "XCTAssertEqual failed: (\"1\") is not equal to (\"0\")"}, + DocumentLocationInCreatingWorkspace: documentLocationInCreatingWorkspace{ + URL: url{Value: "file:/Xcode11TestUITests2.swift#CharacterRangeLen=0&EndingLineNumber=33&StartingLineNumber=33"}, + }, + }, + }, + }, + }, + }, + test: actionTestSummaryGroup{ + Identifier: identifier{Value: "Xcode11TestUITests2/testFail()"}, + }, + want: `file:/Xcode11TestUITests2.swift:CharacterRangeLen=0&EndingLineNumber=33&StartingLineNumber=33 - XCTAssertEqual failed: ("1") is not equal to ("0")`, + }, + { + name: "class inherited test", + record: actionsInvocationRecord{ + Issues: issues{ + TestFailureSummaries: testFailureSummaries{ + Values: []testFailureSummary{ + { + ProducingTarget: producingTarget{Value: "Xcode11TestUITests2"}, + TestCaseName: testCaseName{Value: "SomethingDifferentClass.testFail()"}, + Message: message{Value: "XCTAssertEqual failed: (\"1\") is not equal to (\"0\")"}, + DocumentLocationInCreatingWorkspace: documentLocationInCreatingWorkspace{ + URL: url{Value: "file:/Xcode11TestUITests2.swift#CharacterRangeLen=0&EndingLineNumber=33&StartingLineNumber=33"}, + }, + }, + }, + }, + }, + }, + test: actionTestSummaryGroup{ + Identifier: identifier{Value: "SomethingDifferentClass/testFail()"}, + }, + want: `file:/Xcode11TestUITests2.swift:CharacterRangeLen=0&EndingLineNumber=33&StartingLineNumber=33 - XCTAssertEqual failed: ("1") is not equal to ("0")`, + }, + { + name: "inner class test", + record: actionsInvocationRecord{ + Issues: issues{ + TestFailureSummaries: testFailureSummaries{ + Values: []testFailureSummary{ + { + ProducingTarget: producingTarget{Value: "Xcode11TestUITests2"}, + TestCaseName: testCaseName{Value: "-[SomethingDifferentClass testFail]"}, + Message: message{Value: "XCTAssertEqual failed: (\"1\") is not equal to (\"0\")"}, + DocumentLocationInCreatingWorkspace: documentLocationInCreatingWorkspace{ + URL: url{Value: "file:/Xcode11TestUITests2.swift#CharacterRangeLen=0&EndingLineNumber=33&StartingLineNumber=33"}, + }, + }, + }, + }, + }, + }, + test: actionTestSummaryGroup{ + Identifier: identifier{Value: "SomethingDifferentClass/testFail"}, + }, + want: `file:/Xcode11TestUITests2.swift:CharacterRangeLen=0&EndingLineNumber=33&StartingLineNumber=33 - XCTAssertEqual failed: ("1") is not equal to ("0")`, + }, + } + for _, tt := range testCases { + t.Run(tt.name, func(t *testing.T) { + if got := tt.record.failure(tt.test, testreport.TestSuite{Name: "Xcode11TestUITests2"}); got != tt.want { + t.Errorf("actionsInvocationRecord.failure() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/testresult/xcresult3/action_test_plan_summary.go b/testresult/xcresult3/action_test_plan_summary.go new file mode 100644 index 00000000..6605da4a --- /dev/null +++ b/testresult/xcresult3/action_test_plan_summary.go @@ -0,0 +1,93 @@ +package xcresult3 + +import "strconv" + +type actionTestPlanRunSummaries struct { + Summaries summaries `json:"summaries"` +} + +type summaries struct { + Values []summary `json:"_values"` +} + +type summary struct { + TestableSummaries testableSummaries `json:"testableSummaries"` +} + +type testableSummaries struct { + Values []actionTestableSummary `json:"_values"` +} + +type actionTestableSummary struct { + Name name `json:"name"` + Tests tests `json:"tests"` +} + +type tests struct { + Values []actionTestSummaryGroup `json:"_values"` +} + +type name struct { + Value string `json:"_value"` +} + +// tests returns actionTestSummaryGroup mapped by the container testableSummary name. +func (s actionTestPlanRunSummaries) tests() ([]string, map[string][]actionTestSummaryGroup) { + summaryGroupsByName := map[string][]actionTestSummaryGroup{} + + var testSuiteOrder []string + for _, smry := range s.Summaries.Values { + for _, testableSummary := range smry.TestableSummaries.Values { + // test suite + n := testableSummary.Name.Value + if _, found := summaryGroupsByName[n]; !found { + testSuiteOrder = append(testSuiteOrder, n) + } + + var ts []actionTestSummaryGroup + for _, test := range testableSummary.Tests.Values { + ts = append(ts, test.testsWithStatus()...) + } + + summaryGroupsByName[n] = ts + } + } + + return testSuiteOrder, summaryGroupsByName +} + +func (s actionTestPlanRunSummaries) failuresCount(testableSummaryName string) (failure int) { + _, testsByCase := s.tests() + ts := testsByCase[testableSummaryName] + for _, test := range ts { + if test.TestStatus.Value == "Failure" { + failure++ + } + } + return +} + +func (s actionTestPlanRunSummaries) skippedCount(testableSummaryName string) (skipped int) { + _, testsByCase := s.tests() + ts := testsByCase[testableSummaryName] + for _, test := range ts { + if test.TestStatus.Value == "Skipped" { + skipped++ + } + } + return +} + +func (s actionTestPlanRunSummaries) totalTime(testableSummaryName string) (time float64) { + _, testsByCase := s.tests() + ts := testsByCase[testableSummaryName] + for _, test := range ts { + if test.Duration.Value != "" { + d, err := strconv.ParseFloat(test.Duration.Value, 64) + if err == nil { + time += d + } + } + } + return +} diff --git a/testresult/xcresult3/action_test_plan_summary_test.go b/testresult/xcresult3/action_test_plan_summary_test.go new file mode 100644 index 00000000..bf504316 --- /dev/null +++ b/testresult/xcresult3/action_test_plan_summary_test.go @@ -0,0 +1,194 @@ +package xcresult3 + +import ( + "fmt" + "reflect" + "testing" + + "github.com/bitrise-io/go-utils/pretty" +) + +func TestActionTestPlanRunSummaries_tests(t *testing.T) { + testCases := []struct { + name string + summaries actionTestPlanRunSummaries + want map[string][]actionTestSummaryGroup + }{ + { + name: "single test with status", + summaries: actionTestPlanRunSummaries{ + Summaries: summaries{ + Values: []summary{ + { + TestableSummaries: testableSummaries{ + Values: []actionTestableSummary{ + { + Name: name{Value: "test case 1"}, + Tests: tests{ + Values: []actionTestSummaryGroup{ + {TestStatus: testStatus{Value: "success"}}, + }, + }, + }, + }, + }, + }, + }, + }, + }, + want: map[string][]actionTestSummaryGroup{ + "test case 1": { + {TestStatus: testStatus{Value: "success"}}, + }, + }, + }, + { + name: "single test with status + subtests with status", + summaries: actionTestPlanRunSummaries{ + Summaries: summaries{ + Values: []summary{ + { + TestableSummaries: testableSummaries{ + Values: []actionTestableSummary{ + { + Name: name{Value: "test case 1"}, + Tests: tests{ + Values: []actionTestSummaryGroup{ + {TestStatus: testStatus{Value: "success"}}, + }, + }, + }, + { + Name: name{Value: "test case 2"}, + Tests: tests{ + Values: []actionTestSummaryGroup{ + { + Subtests: subtests{ + Values: []actionTestSummaryGroup{ + {TestStatus: testStatus{Value: "success"}}, + {TestStatus: testStatus{Value: "success"}}, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + want: map[string][]actionTestSummaryGroup{ + "test case 1": { + {TestStatus: testStatus{Value: "success"}}, + }, + "test case 2": { + {TestStatus: testStatus{Value: "success"}}, + {TestStatus: testStatus{Value: "success"}}, + }, + }, + }, + { + name: "no test with status", + summaries: actionTestPlanRunSummaries{}, + want: map[string][]actionTestSummaryGroup{}, + }, + } + for _, tt := range testCases { + t.Run(tt.name, func(t *testing.T) { + if _, got := tt.summaries.tests(); !reflect.DeepEqual(got, tt.want) { + fmt.Println("want: ", pretty.Object(tt.want)) + fmt.Println("got: ", pretty.Object(got)) + t.Errorf("actionTestPlanRunSummaries.tests() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestActionTestPlanRunSummaries_failuresCount(t *testing.T) { + testCases := []struct { + name string + summaries actionTestPlanRunSummaries + testableSummaryName string + wantFailure int + }{ + { + name: "single failure", + summaries: actionTestPlanRunSummaries{ + Summaries: summaries{ + Values: []summary{ + { + TestableSummaries: testableSummaries{ + Values: []actionTestableSummary{ + { + Name: name{Value: "test case"}, + Tests: tests{ + Values: []actionTestSummaryGroup{ + {TestStatus: testStatus{Value: "Failure"}}, + }, + }, + }, + }, + }, + }, + }, + }, + }, + testableSummaryName: "test case", + wantFailure: 1, + }, + } + for _, tt := range testCases { + t.Run(tt.name, func(t *testing.T) { + if gotFailure := tt.summaries.failuresCount(tt.testableSummaryName); gotFailure != tt.wantFailure { + t.Errorf("actionTestPlanRunSummaries.failuresCount() = %v, want %v", gotFailure, tt.wantFailure) + } + }) + } +} + +func TestActionTestPlanRunSummaries_totalTime(t *testing.T) { + testCases := []struct { + name string + summaries actionTestPlanRunSummaries + testableSummaryName string + wantTime float64 + }{ + { + name: "single test", + summaries: actionTestPlanRunSummaries{ + Summaries: summaries{ + Values: []summary{ + { + TestableSummaries: testableSummaries{ + Values: []actionTestableSummary{ + { + Name: name{Value: "test case"}, + Tests: tests{ + Values: []actionTestSummaryGroup{ + { + Duration: duration{Value: "10"}, + TestStatus: testStatus{Value: "Failure"}, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + testableSummaryName: "test case", + wantTime: 10, + }, + } + for _, tt := range testCases { + t.Run(tt.name, func(t *testing.T) { + if gotTime := tt.summaries.totalTime(tt.testableSummaryName); gotTime != tt.wantTime { + t.Errorf("actionTestPlanRunSummaries.totalTime() = %v, want %v", gotTime, tt.wantTime) + } + }) + } +} diff --git a/testresult/xcresult3/action_test_summary.go b/testresult/xcresult3/action_test_summary.go new file mode 100644 index 00000000..0b84ff73 --- /dev/null +++ b/testresult/xcresult3/action_test_summary.go @@ -0,0 +1,70 @@ +package xcresult3 + +import ( + "crypto/md5" + "encoding/hex" +) + +type attachment struct { + Filename struct { + Value string `json:"_value"` + } `json:"filename"` + + PayloadRef struct { + ID struct { + Value string `json:"_value"` + } + } `json:"payloadRef"` +} + +type attachments struct { + Values []attachment `json:"_values"` +} + +type actionTestActivitySummary struct { + Attachments attachments `json:"attachments"` +} + +type activitySummaries struct { + Values []actionTestActivitySummary `json:"_values"` +} + +type actionTestFailureSummary struct { + Message struct { + Value string `json:"_value"` + } `json:"message"` + + FileName struct { + Value string `json:"_value"` + } `json:"fileName"` + + LineNumber struct { + Value string `json:"_value"` + } `json:"lineNumber"` +} + +type failureSummaries struct { + Values []actionTestFailureSummary `json:"_values"` +} + +type configuration struct { + Hash string +} + +// UnmarshalJSON implements json.Unmarshaler by hashing the raw JSON bytes into an MD5 fingerprint. +func (c *configuration) UnmarshalJSON(data []byte) error { + if string(data) == "null" || string(data) == `""` { + return nil + } + + hash := md5.Sum(data) + c.Hash = hex.EncodeToString(hash[:]) + + return nil +} + +type actionTestSummary struct { + ActivitySummaries activitySummaries `json:"activitySummaries"` + FailureSummaries failureSummaries `json:"failureSummaries"` + Configuration configuration `json:"configuration"` +} diff --git a/testresult/xcresult3/action_test_summary_group.go b/testresult/xcresult3/action_test_summary_group.go new file mode 100644 index 00000000..9f1065c2 --- /dev/null +++ b/testresult/xcresult3/action_test_summary_group.go @@ -0,0 +1,112 @@ +package xcresult3 + +import ( + "errors" + "fmt" + "path/filepath" + "strings" +) + +var errSummaryNotFound = errors.New("no summaryRef.ID.Value found for test case") + +type actionTestSummaryGroup struct { + Name name `json:"name"` + Identifier identifier `json:"identifier"` + Duration duration `json:"duration"` + TestStatus testStatus `json:"testStatus"` // only the inner-most tests will have a status, the ones which don't have "subtests" + SummaryRef summaryRef `json:"summaryRef"` // only the inner-most tests will have a summaryRef, the ones which don't have "subtests" + Subtests subtests `json:"subtests"` +} + +type subtests struct { + Values []actionTestSummaryGroup `json:"_values"` +} + +type id struct { + Value string `json:"_value"` +} + +type summaryRef struct { + ID id `json:"id"` +} + +type testStatus struct { + Value string `json:"_value"` +} + +type duration struct { + Value string `json:"_value"` +} + +type identifier struct { + Value string `json:"_value"` +} + +func (g actionTestSummaryGroup) references() (class, method string) { + // Xcode11TestUITests2/testFail() + if g.Identifier.Value != "" { + s := strings.Split(g.Identifier.Value, "/") + if len(s) == 2 { + return s[0], s[1] + } + } + return +} + +// testsWithStatus returns actionTestSummaryGroup entries with TestStatus set. +func (g actionTestSummaryGroup) testsWithStatus() (result []actionTestSummaryGroup) { + if g.TestStatus.Value != "" { + result = append(result, g) + } + + for _, subtest := range g.Subtests.Values { + result = append(result, subtest.testsWithStatus()...) + } + return +} + +func (g actionTestSummaryGroup) loadActionTestSummary(xcresultPath string, useLegacyFlag bool) (actionTestSummary, error) { + if g.SummaryRef.ID.Value == "" { + return actionTestSummary{}, errSummaryNotFound + } + + var summary actionTestSummary + if err := xcresulttoolGet(xcresultPath, g.SummaryRef.ID.Value, useLegacyFlag, &summary); err != nil { + return actionTestSummary{}, fmt.Errorf("failed to load action test summary: %w", err) + } + return summary, nil +} + +func (g actionTestSummaryGroup) exportScreenshots(resultPth, outputDir string, useLegacyFlag bool) error { + if g.TestStatus.Value == "" { + return nil + } + + if g.SummaryRef.ID.Value == "" { + return nil + } + + var summary actionTestSummary + if err := xcresulttoolGet(resultPth, g.SummaryRef.ID.Value, useLegacyFlag, &summary); err != nil { + return err + } + + exported := map[string]bool{} + for _, summary := range summary.ActivitySummaries.Values { + for _, value := range summary.Attachments.Values { + if value.Filename.Value != "" && value.PayloadRef.ID.Value != "" { + if exported[value.PayloadRef.ID.Value] { + continue + } + + pth := filepath.Join(outputDir, value.Filename.Value) + if err := xcresulttoolExport(resultPth, value.PayloadRef.ID.Value, pth, useLegacyFlag); err != nil { + return err + } + exported[value.PayloadRef.ID.Value] = true + } + } + } + + return nil +} diff --git a/testresult/xcresult3/action_test_summary_group_test.go b/testresult/xcresult3/action_test_summary_group_test.go new file mode 100644 index 00000000..557bb88a --- /dev/null +++ b/testresult/xcresult3/action_test_summary_group_test.go @@ -0,0 +1,83 @@ +package xcresult3 + +import ( + "reflect" + "testing" +) + +func TestActionTestSummaryGroup_references(t *testing.T) { + testCases := []struct { + name string + identifier string + wantClass string + wantMethod string + }{ + { + name: "simple test", + identifier: "Xcode11TestUITests2/testFail()", + wantClass: "Xcode11TestUITests2", + wantMethod: "testFail()", + }, + { + name: "invalid format", + identifier: "Xcode11TestUITests2testFail()", + wantMethod: "", + wantClass: "", + }, + } + for _, tt := range testCases { + t.Run(tt.name, func(t *testing.T) { + g := actionTestSummaryGroup{} + g.Identifier.Value = tt.identifier + + gotClass, gotMethod := g.references() + if gotClass != tt.wantClass { + t.Errorf("actionTestSummaryGroup.references() gotClass = %v, want %v", gotClass, tt.wantClass) + } + if gotMethod != tt.wantMethod { + t.Errorf("actionTestSummaryGroup.references() gotMethod = %v, want %v", gotMethod, tt.wantMethod) + } + }) + } +} + +func TestActionTestSummaryGroup_testsWithStatus(t *testing.T) { + testCases := []struct { + name string + group actionTestSummaryGroup + wantGroups []actionTestSummaryGroup + }{ + { + name: "status in the root actionTestSummaryGroup", + group: actionTestSummaryGroup{ + TestStatus: testStatus{Value: "success"}, + }, + wantGroups: []actionTestSummaryGroup{{TestStatus: testStatus{Value: "success"}}}, + }, + { + name: "status in a sub actionTestSummaryGroup", + group: actionTestSummaryGroup{ + Subtests: subtests{ + Values: []actionTestSummaryGroup{ + {TestStatus: testStatus{Value: "success"}}, + }, + }, + }, + wantGroups: []actionTestSummaryGroup{{TestStatus: testStatus{Value: "success"}}}, + }, + { + name: "no status", + group: actionTestSummaryGroup{}, + wantGroups: nil, + }, + } + + for _, tt := range testCases { + t.Run(tt.name, func(t *testing.T) { + gotGroups := tt.group.testsWithStatus() + if !reflect.DeepEqual(gotGroups, tt.wantGroups) { + t.Errorf("actionTestSummaryGroup.testsWithStatus() gotTarget = %v, want %v", gotGroups, tt.wantGroups) + } + }) + } +} diff --git a/testresult/xcresult3/converter.go b/testresult/xcresult3/converter.go new file mode 100644 index 00000000..9535542e --- /dev/null +++ b/testresult/xcresult3/converter.go @@ -0,0 +1,504 @@ +package xcresult3 + +import ( + "encoding/json" + "errors" + "fmt" + "math" + "os" + "path/filepath" + "runtime" + "sort" + "strconv" + "strings" + "sync" + "time" + + "howett.net/plist" + + "github.com/bitrise-io/go-utils/fileutil" + "github.com/bitrise-io/go-utils/log" + "github.com/bitrise-io/go-utils/pathutil" + "github.com/bitrise-io/go-xcode/xcodeproject/serialized" + "github.com/bitrise-io/go-steputils/v2/testasset" + "github.com/bitrise-io/go-steputils/v2/testreport" + "github.com/bitrise-io/go-xcode/v2/testresult/xcresult3/model3" +) + +// Converter ... +type Converter struct { + xcresultPth string + useLegacyExtractionMethod bool +} + +func majorVersion(document serialized.Object) (int, error) { + version, err := document.Object("version") + if err != nil { + return -1, err + } + + major, err := version.Value("major") + if err != nil { + return -1, err + } + return int(major.(uint64)), nil +} + +func documentMajorVersion(pth string) (int, error) { + content, err := fileutil.ReadBytesFromFile(pth) + if err != nil { + return -1, err + } + + var info serialized.Object + if _, err := plist.Unmarshal(content, &info); err != nil { + return -1, err + } + + return majorVersion(info) +} + +// NewConverter creates a Converter with the given extraction method preference. +func NewConverter(useLegacy bool) *Converter { + return &Converter{useLegacyExtractionMethod: useLegacy} +} + +// Detect ... +func (c *Converter) Detect(files []string) bool { + if !isXcresulttoolAvailable() { + log.Debugf("xcresult tool is not available") + return false + } + + for _, file := range files { + if filepath.Ext(file) != ".xcresult" { + continue + } + + infoPth := filepath.Join(file, "Info.plist") + if exist, err := pathutil.IsPathExists(infoPth); err != nil { + log.Debugf("Failed to find Info.plist at %s: %s", infoPth, err) + continue + } else if !exist { + log.Debugf("No Info.plist found at %s", infoPth) + continue + } + + version, err := documentMajorVersion(infoPth) + if err != nil { + log.Debugf("failed to get document version: %s", err) + continue + } + + if version < 3 { + log.Debugf("version < 3: %d", version) + continue + } + + c.xcresultPth = file + return true + } + return false +} + +// Convert returns the test report parsed from the xcresult file. +func (c *Converter) Convert() (testreport.TestReport, error) { + supportsNewMethod, err := supportsNewExtractionMethods() + if err != nil { + return testreport.TestReport{}, err + } + + useLegacyFlag := c.useLegacyExtractionMethod + + if supportsNewMethod && !useLegacyFlag { + log.Infof("Using new extraction method") + + junitXml, err := parse(c.xcresultPth) + if err == nil { + return junitXml, nil + } + + log.Warnf(fmt.Sprintf("Failed to parse extraction method: %s", err)) + log.Warnf("Falling back to legacy extraction method") + + sendRemoteWarning("xcresult3-parsing", "error: %s", err) + + useLegacyFlag = true + } + + log.Infof("Using legacy extraction method") + + return legacyParse(c.xcresultPth, useLegacyFlag) +} + +func legacyParse(path string, useLegacyFlag bool) (testreport.TestReport, error) { + var ( + testResultDir = filepath.Dir(path) + maxParallel = runtime.NumCPU() * 2 + ) + + log.Debugf("Maximum parallelism: %d.", maxParallel) + + _, summaries, err := loadXCResultData(path, useLegacyFlag) + if err != nil { + return testreport.TestReport{}, err + } + + var xmlData testreport.TestReport + { + testSuiteCount := testSuiteCountInSummaries(summaries) + xmlData.TestSuites = make([]testreport.TestSuite, 0, testSuiteCount) + } + + summariesCount := len(summaries) + log.Debugf("Summaries Count: %d", summariesCount) + + for _, summary := range summaries { + testSuiteOrder, testsByName := summary.tests() + + for _, name := range testSuiteOrder { + tests := testsByName[name] + + testSuite, err := genTestSuite(name, summary, tests, testResultDir, path, maxParallel, useLegacyFlag) + if err != nil { + return testreport.TestReport{}, err + } + + xmlData.TestSuites = append(xmlData.TestSuites, testSuite) + } + } + + return xmlData, nil +} + +func parse(path string) (testreport.TestReport, error) { + results, err := ParseTestResults(path) + if err != nil { + return testreport.TestReport{}, err + } + + testSummary, warnings, err := model3.Convert(results) + if err != nil { + return testreport.TestReport{}, err + } + + if len(warnings) > 0 { + sendRemoteWarning("xcresults3-data", "warnings: %s", warnings) + } + + var xml testreport.TestReport + + for _, plan := range testSummary.TestPlans { + for _, testBundle := range plan.TestBundles { + xml.TestSuites = append(xml.TestSuites, parseTestBundle(testBundle)) + } + } + + outputPath := filepath.Dir(path) + + attachmentsMap, err := extractAttachments(path, outputPath) + if err != nil { + return testreport.TestReport{}, err + } + + xml, err = connectAttachmentsToTestCases(xml, attachmentsMap) + if err != nil { + return testreport.TestReport{}, err + } + + return xml, nil +} + +func parseTestBundle(testBundle model3.TestBundle) testreport.TestSuite { + failedCount := 0 + skippedCount := 0 + var totalDuration time.Duration + var tests []testreport.TestCase + + for _, testSuite := range testBundle.TestSuites { + for _, testCase := range testSuite.TestCases { + var testCasesToConvert []model3.TestCase + if len(testCase.Retries) > 0 { + testCasesToConvert = testCase.Retries + } else { + testCasesToConvert = []model3.TestCase{testCase.TestCase} + } + + for _, testCaseToConvert := range testCasesToConvert { + test := parseTestCase(testCaseToConvert) + + if test.Failure != nil { + failedCount++ + } else if test.Skipped != nil { + skippedCount++ + } + totalDuration += testCaseToConvert.Time + + tests = append(tests, test) + } + } + } + + return testreport.TestSuite{ + Name: testBundle.Name, + Tests: len(tests), + Failures: failedCount, + Skipped: skippedCount, + Time: totalDuration.Seconds(), + TestCases: tests, + } +} + +func parseTestCase(testCase model3.TestCase) testreport.TestCase { + test := testreport.TestCase{ + Name: testCase.Name, + ClassName: testCase.ClassName, + Time: testCase.Time.Seconds(), + } + + if testCase.Result == model3.TestResultFailed { + test.Failure = &testreport.Failure{Value: testCase.Message} + } else if testCase.Result == model3.TestResultSkipped { + test.Skipped = &testreport.Skipped{} + } + + return test +} + +func extractAttachments(xcresultPath, outputPath string) (map[string][]string, error) { + var attachmentsMap = make(map[string][]string) + + if err := xcresulttoolExport(xcresultPath, "", outputPath, false); err != nil { + return nil, err + } + + manifestPath := filepath.Join(outputPath, "manifest.json") + bytes, err := os.ReadFile(manifestPath) + if err != nil { + return nil, err + } + + var manifest []model3.TestAttachmentDetails + if err := json.Unmarshal(bytes, &manifest); err != nil { + return nil, err + } + + for _, attachmentDetail := range manifest { + attachments := attachmentDetail.Attachments + + sort.Slice(attachments, func(i, j int) bool { + return time.Time(attachments[i].Timestamp).Before(time.Time(attachments[j].Timestamp)) + }) + + for _, attachment := range attachments { + oldPath := filepath.Join(outputPath, attachment.ExportedFileName) + newFilename := createUniqueFilename(attachment) + newPath := filepath.Join(outputPath, newFilename) + + if err := os.Rename(oldPath, newPath); err != nil { + // It is not a critical error if the rename fails because the file will be still exported just by its + // unique ID. + log.Warnf("Failed to rename %s to %s", oldPath, newPath) + } + + if !testasset.IsSupportedAssetType(newPath) { + continue + } + + testIdentifier := appendRepetitionToTestIdentifier(attachmentDetail.TestIdentifier, attachment.RepetitionNumber) + attachmentsMap[testIdentifier] = append(attachmentsMap[testIdentifier], filepath.Base(newPath)) + } + } + + if err := os.Remove(manifestPath); err != nil { + log.Warnf("Failed to remove manifest file %s: %s", manifestPath, err) + } + + return attachmentsMap, nil +} + +// Create unique filename using timestamp as suffix +func createUniqueFilename(attachment model3.Attachment) string { + timestamp := time.Time(attachment.Timestamp).UnixNano() + + originalName := attachment.SuggestedHumanReadableName + ext := filepath.Ext(originalName) + nameWithoutExt := strings.TrimSuffix(originalName, ext) + + // Format: originalname_timestamp.ext + return fmt.Sprintf("%s_%d%s", nameWithoutExt, timestamp, ext) +} + +func stripTrailingParentheses(s string) string { + return strings.TrimSuffix(s, "()") +} + +func buildTestIdentifier(className, testName string) string { + return className + "/" + testName +} + +func appendRepetitionToTestIdentifier(testIdentifier string, repetition int) string { + // Non-retried tests have an empty repetition, but later we treat them as a test with a repetition of 1. + // So we need to ensure that the repetition is at least 1. + value := int(math.Max(1, float64(repetition))) + return fmt.Sprintf("%s (%d)", stripTrailingParentheses(testIdentifier), value) +} + +func connectAttachmentsToTestCases(xml testreport.TestReport, attachmentsMap map[string][]string) (testreport.TestReport, error) { + for i := range xml.TestSuites { + var testRepetitionMap = make(map[string]int) + + for j := range xml.TestSuites[i].TestCases { + testCase := &xml.TestSuites[i].TestCases[j] + testIdentifier := buildTestIdentifier(testCase.ClassName, testCase.Name) + + // If the test case has a repetition, we need to append it to the test identifier + // and keep track of how many times we have seen this test identifier. + if count, exists := testRepetitionMap[testIdentifier]; exists { + testRepetitionMap[testIdentifier] = count + 1 + } else { + testRepetitionMap[testIdentifier] = 1 + } + + testIdentifier = appendRepetitionToTestIdentifier(testIdentifier, testRepetitionMap[testIdentifier]) + + // Add attachments if any exist for this test and repetition + if attachments, exists := attachmentsMap[testIdentifier]; exists { + if testCase.Properties == nil { + testCase.Properties = &testreport.Properties{ + Property: []testreport.Property{}, + } + } + + // Add each attachment as a property + for i, fileName := range attachments { + testCase.Properties.Property = append( + testCase.Properties.Property, + testreport.Property{Name: fmt.Sprintf("attachment_%d", i), Value: fileName}, + ) + } + } + } + } + + return xml, nil +} + +func testSuiteCountInSummaries(summaries []actionTestPlanRunSummaries) int { + testSuiteCount := 0 + for _, summary := range summaries { + testSuiteOrder, _ := summary.tests() + testSuiteCount += len(testSuiteOrder) + } + return testSuiteCount +} + +func genTestSuite(name string, + summary actionTestPlanRunSummaries, + tests []actionTestSummaryGroup, + testResultDir string, + xcresultPath string, + maxParallel int, + useLegacyFlag bool, +) (testreport.TestSuite, error) { + var ( + start = time.Now() + genTestSuiteErr error + wg sync.WaitGroup + mtx sync.RWMutex + ) + + testSuite := testreport.TestSuite{ + Name: name, + Tests: len(tests), + Failures: summary.failuresCount(name), + Skipped: summary.skippedCount(name), + Time: summary.totalTime(name), + TestCases: make([]testreport.TestCase, len(tests)), + } + + testIdx := 0 + for testIdx < len(tests) { + for i := 0; i < maxParallel && testIdx < len(tests); i++ { + test := tests[testIdx] + wg.Add(1) + + go func(test actionTestSummaryGroup, testIdx int) { + defer wg.Done() + + testCase, err := genTestCase(test, xcresultPath, testResultDir, useLegacyFlag) + if err != nil { + mtx.Lock() + genTestSuiteErr = err + mtx.Unlock() + } + + testSuite.TestCases[testIdx] = testCase + }(test, testIdx) + + testIdx++ + } + + wg.Wait() + } + + log.Debugf("Generating test suite [%s] (%d tests) - DONE %v", name, len(tests), time.Since(start)) + + return testSuite, genTestSuiteErr +} + +func genTestCase(test actionTestSummaryGroup, xcresultPath, testResultDir string, useLegacyFlag bool) (testreport.TestCase, error) { + var duartion float64 + if test.Duration.Value != "" { + var err error + duartion, err = strconv.ParseFloat(test.Duration.Value, 64) + if err != nil { + return testreport.TestCase{}, err + } + } + + testSummary, err := test.loadActionTestSummary(xcresultPath, useLegacyFlag) + // Ignoring the SummaryNotFoundError error is on purpose because not having an action summary is a valid use case. + // For example, failed tests will always have a summary, but successful ones might have it or might not. + // If they do not have it, then that means that they did not log anything to the console, + // and they were not executed as device configuration tests. + if err != nil && !errors.Is(err, errSummaryNotFound) { + return testreport.TestCase{}, err + } + + var failure *testreport.Failure + var skipped *testreport.Skipped + switch test.TestStatus.Value { + case "Failure": + failureMessage := "" + for _, aTestFailureSummary := range testSummary.FailureSummaries.Values { + file := aTestFailureSummary.FileName.Value + line := aTestFailureSummary.LineNumber.Value + message := aTestFailureSummary.Message.Value + + if len(failureMessage) > 0 { + failureMessage += "\n" + } + failureMessage += fmt.Sprintf("%s:%s - %s", file, line, message) + } + + failure = &testreport.Failure{ + Value: failureMessage, + } + case "Skipped": + skipped = &testreport.Skipped{} + } + + if err := test.exportScreenshots(xcresultPath, testResultDir, useLegacyFlag); err != nil { + return testreport.TestCase{}, err + } + + return testreport.TestCase{ + Name: test.Name.Value, + ConfigurationHash: testSummary.Configuration.Hash, + ClassName: strings.Split(test.Identifier.Value, "/")[0], + Failure: failure, + Skipped: skipped, + Time: duartion, + }, nil +} diff --git a/testresult/xcresult3/converter_test.go b/testresult/xcresult3/converter_test.go new file mode 100644 index 00000000..030b0a09 --- /dev/null +++ b/testresult/xcresult3/converter_test.go @@ -0,0 +1,234 @@ +package xcresult3 + +import ( + "fmt" + "os" + "path/filepath" + "testing" + + "github.com/bitrise-io/go-steputils/v2/testreport" + "github.com/bitrise-io/go-utils/v2/command" + "github.com/bitrise-io/go-utils/v2/env" + "github.com/stretchr/testify/require" +) + +const sampleArtifactsGitURI = "https://github.com/bitrise-io/sample-artifacts.git" + +var sampleArtifactsDir string + +func TestMain(m *testing.M) { + os.Exit(runTests(m)) +} + +func runTests(m *testing.M) int { + dir, err := os.MkdirTemp("", "sample-artifacts-*") + if err != nil { + fmt.Printf("failed to create temp dir: %s\n", err) + return 1 + } + defer func(path string) { + err := os.RemoveAll(path) + if err != nil { + fmt.Printf("failed to remove temp dir: %s\n", err) + } + }(dir) + + cmdFactory := command.NewFactory(env.NewRepository()) + cmd := cmdFactory.Create("git", []string{"clone", "--depth=1", sampleArtifactsGitURI, dir}, nil) + if out, err := cmd.RunAndReturnTrimmedCombinedOutput(); err != nil { + fmt.Printf("git clone failed: %s\n%s\n", err, out) + return 1 + } + + sampleArtifactsDir = dir + return m.Run() +} + +// setupTestData copies an xcresult bundle from the cloned sample-artifacts repo +// into a per-test temp directory and returns the path to the copied bundle. +func setupTestData(t testing.TB, fileName string) string { + t.Helper() + + srcPath := filepath.Join(sampleArtifactsDir, "xcresults", fileName) + dstPath := filepath.Join(t.TempDir(), fileName) + + cmdFactory := command.NewFactory(env.NewRepository()) + cmd := cmdFactory.Create("cp", []string{"-r", srcPath, dstPath}, nil) + out, err := cmd.RunAndReturnTrimmedCombinedOutput() + require.NoError(t, err, "failed to copy test data: %s", out) + + return dstPath +} + +// assertAttachmentProperties checks that props contain exactly expectedCount attachments, +// each with a sequentially numbered Name ("attachment_0", "attachment_1", ...) and a Value +// matching the pattern "_.". +// The human-readable part of the filename is generated by xcresulttool using the local +// timezone, and I don't see any guarantee that it is always assembled like this, +// so only the overall pattern is verified rather than the exact string. +func assertAttachmentProperties(t *testing.T, props *testreport.Properties, expectedCount int) { + t.Helper() + require.NotNil(t, props) + require.Len(t, props.Property, expectedCount) + for i, p := range props.Property { + require.Equal(t, fmt.Sprintf("attachment_%d", i), p.Name) + require.Regexp(t, `^.+_\d+\.\w+$`, p.Value) + } +} + +func TestConverter_XML(t *testing.T) { + t.Run("xcresult3-flaky-with-rerun.xcresult", func(t *testing.T) { + xcresultPath := setupTestData(t, "xcresult3-flaky-with-rerun.xcresult") + t.Log("xcresultPath: ", xcresultPath) + + c := Converter{xcresultPth: xcresultPath} + junitXML, err := c.Convert() + require.NoError(t, err) + + // Save and nil out the timezone-dependent attachment properties before the + // structural comparison; they are verified separately with assertAttachmentProperties. + uiTestProps := junitXML.TestSuites[2].TestCases[0].Properties + junitXML.TestSuites[2].TestCases[0].Properties = nil + + require.Equal(t, []testreport.TestSuite{ + { + Name: "BullsEyeTests", Tests: 5, Failures: 0, Skipped: 0, Time: 0.9774, + TestCases: []testreport.TestCase{ + { + Name: "testStartNewRoundUsesRandomValueFromApiRequest()", ClassName: "BullsEyeFakeTests", + Time: 0.014, + }, + { + Name: "testGameStyleCanBeChanged()", ClassName: "BullsEyeMockTests", + Time: 0.0093, + }, + { + Name: "testScoreIsComputedPerformance()", ClassName: "BullsEyeTests", + Time: 0.74, + }, + { + Name: "testScoreIsComputedWhenGuessIsHigherThanTarget()", ClassName: "BullsEyeTests", + Time: 0.0041, + }, + { + Name: "testScoreIsComputedWhenGuessIsLowerThanTarget()", ClassName: "BullsEyeTests", + Time: 0.21, + }, + }, + }, + { + Name: "BullsEyeSlowTests", Tests: 2, Failures: 0, Skipped: 0, Time: 0.53, + TestCases: []testreport.TestCase{ + { + Name: "testApiCallCompletes()", ClassName: "BullsEyeSlowTests", + Time: 0.28, + }, + { + Name: "testValidApiCallGetsHTTPStatusCode200()", ClassName: "BullsEyeSlowTests", + Time: 0.25, + }, + }, + }, + { + Name: "BullsEyeUITests", Tests: 1, Failures: 0, Skipped: 0, Time: 9, + TestCases: []testreport.TestCase{ + {Name: "testGameStyleSwitch()", ClassName: "BullsEyeUITests", Time: 9}, + }, + }, + { + Name: "BullsEyeFlakyTests", Tests: 3, Failures: 1, Skipped: 1, Time: 0.226, + TestCases: []testreport.TestCase{ + { + Name: "testFlakyFeature()", ClassName: "BullsEyeFlakyTests", Time: 0.2, + Failure: &testreport.Failure{ + Value: `BullsEyeFlakyTests.swift:43: XCTAssertEqual failed: ("1") is not equal to ("0") - Number is not even`, + }, + }, + { + Name: "testFlakyFeature()", ClassName: "BullsEyeFlakyTests", Time: 0.006, + }, + { + Name: "testFlakySkip()", ClassName: "BullsEyeSkippedTests", Time: 0.02, + Skipped: &testreport.Skipped{}, + }, + }, + }, + }, junitXML.TestSuites) + + assertAttachmentProperties(t, uiTestProps, 6) + }) + + t.Run("xcresults3 success-failed-skipped-tests.xcresult", func(t *testing.T) { + xcresultPath := setupTestData(t, "xcresult3-success-failed-skipped-tests.xcresult") + t.Log("xcresultPath: ", xcresultPath) + + c := Converter{xcresultPth: xcresultPath} + junitXML, err := c.Convert() + require.NoError(t, err) + + // Save and nil out the timezone-dependent attachment properties before the + // structural comparison; they are verified separately with assertAttachmentProperties. + failureProps := junitXML.TestSuites[0].TestCases[0].Properties + junitXML.TestSuites[0].TestCases[0].Properties = nil + + require.Equal(t, []testreport.TestSuite{ + { + Name: "testProjectUITests", + Tests: 3, + Failures: 1, + Skipped: 1, + Time: 0.435, + TestCases: []testreport.TestCase{ + { + Name: "testFailure()", + ClassName: "testProjectUITests", + Time: 0.26, + Failure: &testreport.Failure{ + Value: "testProjectUITests.swift:30: XCTAssertTrue failed", + }, + }, + { + Name: "testSkip()", + ClassName: "testProjectUITests", + Time: 0.086, + Skipped: &testreport.Skipped{}, + }, + { + Name: "testSuccess()", + ClassName: "testProjectUITests", + Time: 0.089, + }, + }, + }, + }, junitXML.TestSuites) + + assertAttachmentProperties(t, failureProps, 3) + }) + + t.Run("xcresult3-multiple-test-plan-configurations.xcresult", func(t *testing.T) { + xcresultPath := setupTestData(t, "xcresult3-multiple-test-plan-configurations.xcresult") + t.Log("xcresultPath: ", xcresultPath) + + c := Converter{xcresultPth: xcresultPath} + junitXML, err := c.Convert() + require.NoError(t, err) + require.NotNil(t, junitXML) + + require.EqualValues( + t, + junitXML.TestSuites[0].TestCases[0].Failure.Value, + `English: swift_testingTests.swift:20: Expectation failed: true == false - // This test is intended to fail to demonstrate test failure reporting. +German: swift_testingTests.swift:20: Expectation failed: true == false - // This test is intended to fail to demonstrate test failure reporting. +`, + ) + }) +} + +func BenchmarkConverter_XML(b *testing.B) { + xcresultPath := setupTestData(b, "xcresult3-flaky-with-rerun.xcresult") + b.Log("xcresultPath: ", xcresultPath) + + c := Converter{xcresultPth: xcresultPath} + _, err := c.Convert() + require.NoError(b, err) +} diff --git a/testresult/xcresult3/logging.go b/testresult/xcresult3/logging.go new file mode 100644 index 00000000..c373dad2 --- /dev/null +++ b/testresult/xcresult3/logging.go @@ -0,0 +1,10 @@ +package xcresult3 + +import "github.com/bitrise-io/go-utils/log" + +func sendRemoteWarning(tag string, format string, v ...interface{}) { + data := map[string]interface{}{} + data["source"] = "deploy-to-bitrise-io" + + log.RWarnf("deploy-to-bitrise-io", tag, data, format, v...) +} diff --git a/testresult/xcresult3/model3/conversion.go b/testresult/xcresult3/model3/conversion.go new file mode 100644 index 00000000..2aaeb007 --- /dev/null +++ b/testresult/xcresult3/model3/conversion.go @@ -0,0 +1,218 @@ +package model3 + +import ( + "fmt" + "strings" + "time" +) + +// Convert converts TestData into a TestSummary and a list of warnings. +func Convert(data *TestData) (*TestSummary, []string, error) { + var warnings []string + summary := TestSummary{} + + for _, testPlanNode := range data.TestNodes { + if testPlanNode.Type != TestNodeTypeTestPlan { + return nil, warnings, fmt.Errorf("test plan expected but got: %s", testPlanNode.Type) + } + + testPlan := TestPlan{Name: testPlanNode.Name} + + for _, testBundleNode := range testPlanNode.Children { + if testBundleNode.Type != TestNodeTypeUnitTestBundle && testBundleNode.Type != TestNodeTypeUITestBundle { + return nil, warnings, fmt.Errorf("test bundle expected but got: %s", testBundleNode.Type) + } + + testBundle := TestBundle{Name: testBundleNode.Name} + + for _, testSuiteNode := range testBundleNode.Children { + var name string + var testNodes []TestNode + + if testSuiteNode.Type == TestNodeTypeTestCase { + name = testBundleNode.Name + testNodes = []TestNode{testSuiteNode} + } else if testSuiteNode.Type == TestNodeTypeTestSuite { + name = testSuiteNode.Name + testNodes = testSuiteNode.Children + } else { + return nil, warnings, fmt.Errorf("test suite or test case expected but got: %s", testSuiteNode.Type) + } + + testSuite := TestSuite{Name: name} + + testCases, testCaseWarnings, err := extractTestCases(testNodes, name) + warnings = append(warnings, testCaseWarnings...) + + if err != nil { + return nil, warnings, err + } + + testSuite.TestCases = testCases + testBundle.TestSuites = append(testBundle.TestSuites, testSuite) + } + + testPlan.TestBundles = append(testPlan.TestBundles, testBundle) + } + + summary.TestPlans = append(summary.TestPlans, testPlan) + } + + return &summary, warnings, nil +} + +func extractTestCases(nodes []TestNode, fallbackName string) ([]TestCaseWithRetries, []string, error) { + var testCases []TestCaseWithRetries + var warnings []string + + for _, testCaseNode := range nodes { + // A customer's xcresult file contained this use case where a test suite is a child of a test suite. + if testCaseNode.Type == TestNodeTypeTestSuite { + nestedTestCases, nestedWarnings, err := extractTestCases(testCaseNode.Children, fallbackName) + warnings = append(warnings, nestedWarnings...) + + if err != nil { + return nil, warnings, err + } + + testCases = append(testCases, nestedTestCases...) + + continue + } + + if testCaseNode.Type != TestNodeTypeTestCase { + return nil, warnings, fmt.Errorf("test case expected but got: %s", testCaseNode.Type) + } + + testCase, testCaseWarnings := extractTestCase(testCaseNode, "", "", fallbackName) + warnings = append(warnings, testCaseWarnings...) + + retries, retryWarnings, err := extractRetries(testCaseNode, fallbackName) + if err != nil { + return nil, warnings, err + } + warnings = append(warnings, retryWarnings...) + + testCases = append(testCases, TestCaseWithRetries{ + TestCase: testCase, + Retries: retries, + }) + } + + return testCases, warnings, nil +} + +func extractDuration(text string) time.Duration { + // Duration is in the format "123.456789s", "123,456789s" or "4m 34s", so we need to do some normalization. We + // need to replace the comma with a dot and remove the space to be able to parse it with time.ParseDuration. + normalized := strings.ReplaceAll(text, ",", ".") + normalized = strings.ReplaceAll(normalized, " ", "") + + duration, err := time.ParseDuration(normalized) + if err != nil { + return 0 + } + + return duration +} + +func extractFailureMessageFromTestPlanConfigs(testPlanConfigNodes []TestNode) (string, []string) { + var warnings []string + failureMessage := "" + + for _, testPlanConfigNode := range testPlanConfigNodes { + msg, wrns := extractFailureMessage(testPlanConfigNode) + if msg != "" { + failureMessage += fmt.Sprintf("%s: %s\n", testPlanConfigNode.Name, msg) + } + if wrns != nil { + warnings = append(warnings, wrns...) + } + } + + return failureMessage, warnings +} + +func extractFailureMessage(testNode TestNode) (string, []string) { + childrenCount := len(testNode.Children) + if childrenCount == 0 { + return "", nil + } + + lastChild := testNode.Children[childrenCount-1] + if lastChild.Type == TestNodeTypeRepetition { + return extractFailureMessage(lastChild) + } + if lastChild.Type == TestNodeTypeTestPlanConfig { + return extractFailureMessageFromTestPlanConfigs(testNode.Children) + } + + var warnings []string + failureMessage := "" + + for _, child := range testNode.Children { + if child.Type == TestNodeTypeFailureMessage { + // The failure message appears in the Name field and not in the Details field. + if child.Name == "" { + warnings = append(warnings, fmt.Sprintf("'%s' type has empty name field", child.Type)) + } + if child.Details != "" { + warnings = append(warnings, fmt.Sprintf("'%s' type has unexpected details field", child.Type)) + } + + failureMessage += child.Name + } + } + + return failureMessage, warnings +} + +func extractRetries(testNode TestNode, fallbackName string) ([]TestCase, []string, error) { + var retries []TestCase + var warnings []string + + for _, child := range testNode.Children { + if child.Type == TestNodeTypeRepetition { + // Use the parent test node's identifier, instead of the repetition's identifier (1, 2, ...). + retry, testCaseWarnings := extractTestCase(child, testNode.Identifier, testNode.Name, fallbackName) + warnings = append(warnings, testCaseWarnings...) + retries = append(retries, retry) + } + } + + return retries, warnings, nil +} + +func extractTestCase(testNode TestNode, customNodeIdentifier, customName, fallbackClassName string) (TestCase, []string) { + var warnings []string + + nodeIdentifier := testNode.Identifier + if customNodeIdentifier != "" { + nodeIdentifier = customNodeIdentifier + } + + name := testNode.Name + if customName != "" { + name = customName + } + + className := strings.Split(nodeIdentifier, "/")[0] + if className == "" { + // In rare cases the identifier is an empty string, so we need to use the test suite name which is the + // same as the first part of the identifier in normal cases. + className = fallbackClassName + } + + message, failureMessageWarnings := extractFailureMessage(testNode) + if len(failureMessageWarnings) > 0 { + warnings = append(warnings, failureMessageWarnings...) + } + + return TestCase{ + Name: name, + ClassName: className, + Time: extractDuration(testNode.Duration), + Result: testNode.Result, + Message: message, + }, warnings +} diff --git a/testresult/xcresult3/model3/conversion_test.go b/testresult/xcresult3/model3/conversion_test.go new file mode 100644 index 00000000..6dd54d9f --- /dev/null +++ b/testresult/xcresult3/model3/conversion_test.go @@ -0,0 +1,225 @@ +package model3 + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestConversion(t *testing.T) { + tests := []struct { + name string + data *TestData + want *TestSummary + }{ + { + name: "Simple case with multiple test bundles, suites, and cases", + data: &TestData{ + TestNodes: []TestNode{ + { + Type: TestNodeTypeTestPlan, + Name: "TP1", + Children: []TestNode{ + { + Type: TestNodeTypeUnitTestBundle, + Name: "TB1", + Children: []TestNode{ + { + Type: TestNodeTypeTestSuite, + Name: "TS1", + Children: []TestNode{ + { + Type: TestNodeTypeTestCase, + Name: "TC1", + Result: TestResultPassed, + Duration: "0.5s", + }, + { + Type: TestNodeTypeTestCase, + Name: "TC2", + Result: TestResultFailed, + Duration: "1m 11s", + }, + }, + }, + { + Type: TestNodeTypeTestSuite, + Name: "TS2", + Children: []TestNode{ + { + Type: TestNodeTypeTestCase, + Name: "TC3", + Result: TestResultSkipped, + Duration: "66s", + }, + { + Type: TestNodeTypeTestCase, + Name: "TC4", + Result: TestResultPassed, + Duration: "0.03s", + }, + }, + }, + }, + }, + { + Type: TestNodeTypeUITestBundle, + Name: "TB2", + Children: []TestNode{ + { + Type: TestNodeTypeTestSuite, + Name: "TS3", + Children: []TestNode{ + { + Type: TestNodeTypeTestCase, + Name: "TC5", + Result: TestResultPassed, + Duration: "15s", + }, + }, + }, + }, + }, + }, + }, + }, + }, + want: &TestSummary{ + TestPlans: []TestPlan{ + { + Name: "TP1", + TestBundles: []TestBundle{ + { + Name: "TB1", + TestSuites: []TestSuite{ + { + Name: "TS1", + TestCases: []TestCaseWithRetries{ + { + TestCase: TestCase{ + Name: "TC1", + ClassName: "TS1", + Time: 500 * time.Millisecond, + Result: "Passed", + }, + }, + { + TestCase: TestCase{ + Name: "TC2", + ClassName: "TS1", + Time: 71 * time.Second, + Result: "Failed", + }, + }, + }, + }, + { + Name: "TS2", + TestCases: []TestCaseWithRetries{ + { + TestCase: TestCase{ + Name: "TC3", + ClassName: "TS2", + Time: 66 * time.Second, + Result: "Skipped", + }, + }, + { + TestCase: TestCase{ + Name: "TC4", + ClassName: "TS2", + Time: 30 * time.Millisecond, + Result: "Passed", + }, + }, + }, + }, + }, + }, + { + Name: "TB2", + TestSuites: []TestSuite{ + { + Name: "TS3", + TestCases: []TestCaseWithRetries{ + { + TestCase: TestCase{ + Name: "TC5", + ClassName: "TS3", + Time: 15 * time.Second, + Result: "Passed", + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + { + name: "Test bundle without test suites", + data: &TestData{ + TestNodes: []TestNode{ + { + Type: TestNodeTypeTestPlan, + Name: "TP1", + Children: []TestNode{ + { + Type: TestNodeTypeUnitTestBundle, + Name: "TB1", + Children: []TestNode{ + { + Type: TestNodeTypeTestCase, + Name: "TC1", + Result: TestResultPassed, + Duration: "0.5s", + }, + }, + }, + }, + }, + }, + }, + want: &TestSummary{ + TestPlans: []TestPlan{ + { + Name: "TP1", + TestBundles: []TestBundle{ + { + Name: "TB1", + TestSuites: []TestSuite{ + { + Name: "TB1", + TestCases: []TestCaseWithRetries{ + { + TestCase: TestCase{ + Name: "TC1", + ClassName: "TB1", + Time: 500 * time.Millisecond, + Result: "Passed", + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, _, err := Convert(test.data) + require.NoError(t, err) + + require.Equal(t, test.want, got) + }) + } +} diff --git a/testresult/xcresult3/model3/data.go b/testresult/xcresult3/model3/data.go new file mode 100644 index 00000000..fa183b1d --- /dev/null +++ b/testresult/xcresult3/model3/data.go @@ -0,0 +1,41 @@ +package model3 + +import "time" + +// TestSummary holds the top-level test results grouped by test plans. +type TestSummary struct { + TestPlans []TestPlan +} + +// TestPlan represents a test plan and its test bundles. +type TestPlan struct { + Name string + TestBundles []TestBundle +} + +// TestBundle represents a test bundle and its test suites. +type TestBundle struct { + Name string + TestSuites []TestSuite +} + +// TestSuite represents a test suite and its test cases. +type TestSuite struct { + Name string + TestCases []TestCaseWithRetries +} + +// TestCaseWithRetries holds a test case along with any retry attempts. +type TestCaseWithRetries struct { + TestCase + Retries []TestCase +} + +// TestCase represents a single test case execution result. +type TestCase struct { + Name string + ClassName string + Time time.Duration + Result TestResult + Message string +} diff --git a/testresult/xcresult3/model3/export.go b/testresult/xcresult3/model3/export.go new file mode 100644 index 00000000..f14c1e7f --- /dev/null +++ b/testresult/xcresult3/model3/export.go @@ -0,0 +1,42 @@ +package model3 + +import ( + "encoding/json" + "time" +) + +// TestAttachmentDetails contains the test identifier and its list of attachments. +type TestAttachmentDetails struct { + TestIdentifier string `json:"testIdentifier"` + Attachments []Attachment `json:"attachments"` +} + +// Timestamp is a time.Time that unmarshals from a Unix epoch float. +type Timestamp time.Time + +// Attachment describes a single exported test attachment file. +type Attachment struct { + ExportedFileName string `json:"exportedFileName"` + SuggestedHumanReadableName string `json:"suggestedHumanReadableName"` + IsAssociatedWithFailure bool `json:"isAssociatedWithFailure"` + Timestamp Timestamp `json:"timestamp"` + ConfigurationName string `json:"configurationName"` + DeviceName string `json:"deviceName"` + DeviceID string `json:"deviceId"` + RepetitionNumber int `json:"repetitionNumber"` +} + +// UnmarshalJSON decodes a Unix epoch float into a Timestamp. +func (t *Timestamp) UnmarshalJSON(b []byte) error { + var timestamp float64 + if err := json.Unmarshal(b, ×tamp); err != nil { + return err + } + + // Extract seconds and nanoseconds separately to preserve fractional part + seconds := int64(timestamp) + nanoseconds := int64((timestamp - float64(seconds)) * 1e9) + + *t = Timestamp(time.Unix(seconds, nanoseconds)) + return nil +} diff --git a/testresult/xcresult3/model3/testresults.go b/testresult/xcresult3/model3/testresults.go new file mode 100644 index 00000000..37aab474 --- /dev/null +++ b/testresult/xcresult3/model3/testresults.go @@ -0,0 +1,70 @@ +package model3 + +// TestNodeType identifies the kind of a node in the xcresulttool test tree. +type TestNodeType string + +// These are all the types the xcresulttool (version 23500, format version 3.53) supports. +const ( + TestNodeTypeTestPlan TestNodeType = "Test Plan" + TestNodeTypeUnitTestBundle TestNodeType = "Unit test bundle" + TestNodeTypeUITestBundle TestNodeType = "UI test bundle" + TestNodeTypeTestSuite TestNodeType = "Test Suite" + TestNodeTypeTestCase TestNodeType = "Test Case" + TestNodeTypeDevice TestNodeType = "Device" + TestNodeTypeTestPlanConfig TestNodeType = "Test Plan Configuration" + TestNodeTypeArguments TestNodeType = "Arguments" + TestNodeTypeRepetition TestNodeType = "Repetition" + TestNodeTypeTestCaseRun TestNodeType = "Test Case Run" + TestNodeTypeFailureMessage TestNodeType = "Failure Message" + TestNodeTypeSourceCodeRef TestNodeType = "Source Code Reference" + TestNodeTypeAttachment TestNodeType = "Attachment" + TestNodeTypeExpression TestNodeType = "Expression" + TestNodeTypeTestValue TestNodeType = "Test Value" +) + +// TestResult represents the outcome of a test case. +type TestResult string + +// TestResult values reported by xcresulttool. +const ( + TestResultPassed TestResult = "Passed" + TestResultFailed TestResult = "Failed" + TestResultSkipped TestResult = "Skipped" + TestResultExpectedFailure TestResult = "Expected Failure" + TestResultUnknown TestResult = "unknown" +) + +// TestData is the top-level structure returned by xcresulttool for a test run. +type TestData struct { + Devices []Devices `json:"devices"` + TestNodes []TestNode `json:"testNodes"` + TestPlanConfigurations []Configuration `json:"testPlanConfigurations"` +} + +// Devices describes a device that participated in the test run. +type Devices struct { + Identifier string `json:"deviceId"` + Name string `json:"deviceName"` + Architecture string `json:"architecture"` + ModelName string `json:"modelName"` + Platform string `json:"platform"` + OS string `json:"osVersion"` +} + +// TestNode is a node in the xcresulttool test tree (plan, bundle, suite, case, etc.). +type TestNode struct { + Identifier string `json:"nodeIdentifier"` + Type TestNodeType `json:"nodeType"` + Name string `json:"name"` + Details string `json:"details"` + Duration string `json:"duration"` + Result TestResult `json:"result"` + Tags []string `json:"tags"` + Children []TestNode `json:"children"` +} + +// Configuration describes a test plan configuration used during the test run. +type Configuration struct { + Identifier string `json:"configurationId"` + Name string `json:"configurationName"` +} diff --git a/testresult/xcresult3/xcresult3.go b/testresult/xcresult3/xcresult3.go new file mode 100644 index 00000000..0484aaf5 --- /dev/null +++ b/testresult/xcresult3/xcresult3.go @@ -0,0 +1,32 @@ +package xcresult3 + +import "github.com/bitrise-io/go-xcode/v2/testresult/xcresult3/model3" + +// loadXCResultData loads the actions invocation record and test plan run summaries from an xcresult file. +func loadXCResultData(pth string, useLegacyFlag bool) (*actionsInvocationRecord, []actionTestPlanRunSummaries, error) { + var r actionsInvocationRecord + if err := xcresulttoolGet(pth, "", useLegacyFlag, &r); err != nil { + return nil, nil, err + } + + var summaries []actionTestPlanRunSummaries + for _, action := range r.Actions.Values { + refID := action.ActionResult.TestsRef.ID.Value + var s actionTestPlanRunSummaries + if err := xcresulttoolGet(pth, refID, useLegacyFlag, &s); err != nil { + return nil, nil, err + } + summaries = append(summaries, s) + } + return &r, summaries, nil +} + +// ParseTestResults parses the test results from the given xcresult file. +func ParseTestResults(pth string) (*model3.TestData, error) { + var data model3.TestData + if err := xcresulttoolGet(pth, "", false, &data); err != nil { + return nil, err + } + + return &data, nil +} diff --git a/testresult/xcresult3/xcresulttool.go b/testresult/xcresult3/xcresulttool.go new file mode 100644 index 00000000..2b3ac6e2 --- /dev/null +++ b/testresult/xcresult3/xcresulttool.go @@ -0,0 +1,182 @@ +package xcresult3 + +import ( + "bufio" + "bytes" + "encoding/json" + "fmt" + "io" + "os" + "os/exec" + "regexp" + "strconv" + "strings" + + "github.com/bitrise-io/go-utils/command" + "github.com/bitrise-io/go-utils/errorutil" + command2 "github.com/bitrise-io/go-utils/v2/command" + "github.com/bitrise-io/go-utils/v2/env" + "github.com/bitrise-io/go-utils/v2/log" +) + +func isXcresulttoolAvailable() bool { + if _, err := exec.LookPath("xcrun"); err != nil { + return false + } + return command.New("xcrun", "--find", "xcresulttool").Run() == nil +} + +func xcresulttoolVersion() (int, error) { + args := []string{"xcresulttool", "version"} + cmd := command.New("xcrun", args...) + out, err := cmd.RunAndReturnTrimmedCombinedOutput() + if err != nil { + if errorutil.IsExitStatusError(err) { + return 0, fmt.Errorf("%s failed: %s", cmd.PrintableCommandArgs(), out) + } + return 0, fmt.Errorf("%s failed: %s", cmd.PrintableCommandArgs(), err) + } + // xcresulttool version 23025, format version 3.53 (current) + versionRegexp := regexp.MustCompile("xcresulttool version ([0-9]+)") + + matches := versionRegexp.FindStringSubmatch(out) + if len(matches) < 2 { + return 0, fmt.Errorf("no version matches found in output: %s", out) + } + + version, err := strconv.Atoi(matches[1]) + if err != nil { + return 0, fmt.Errorf("failed to convert version: %s", matches[1]) + } + + return version, nil +} + +func isXcode16OrNewer() bool { + version, err := xcresulttoolVersion() + if err != nil { + return false + } + + return version >= 23_000 // Xcode 16 beta1 has version 23000 +} + +func supportsNewExtractionMethods() (bool, error) { + version, err := xcresulttoolVersion() + if err != nil { + return false, err + } + + return version >= 23_021, nil // Xcode 16 beta3 has version 23021 +} + +// xcresulttoolGet performs xcrun xcresulttool get with --id flag defined if id provided and marshals the output into v. +func xcresulttoolGet(xcresultPth, id string, useLegacyFlag bool, v interface{}) error { + commandFactory := command2.NewFactory(env.NewRepository()) + logger := log.NewLogger() + + args := []string{"xcresulttool", "get"} + + supportsNewMethod, err := supportsNewExtractionMethods() + if err != nil { + return err + } + + if supportsNewMethod && !useLegacyFlag { + args = append(args, "test-results", "tests") + } else { + args = append(args, "--format", "json") + + if isXcode16OrNewer() && useLegacyFlag { + args = append(args, "--legacy") + } + } + + args = append(args, "--path", xcresultPth) + + if id != "" { + args = append(args, "--id", id) + } + + var outBuffer, errBuffer, combinedBuffer bytes.Buffer + outWriter := io.MultiWriter(&outBuffer, &combinedBuffer) + errWriter := io.MultiWriter(&errBuffer, &combinedBuffer) + + cmd := commandFactory.Create("xcrun", args, &command2.Opts{ + Stdout: outWriter, + Stderr: errWriter, + Stdin: nil, + Env: os.Environ(), + Dir: "", + ErrorFinder: nil, + }) + if err := cmd.Run(); err != nil { + if errorutil.IsExitStatusError(err) { + return fmt.Errorf("%s failed: %s", cmd.PrintableCommandArgs(), combinedBuffer.String()) + } + return fmt.Errorf("%s failed: %s", cmd.PrintableCommandArgs(), err) + } + if stdErr := errBuffer.String(); stdErr != "" { + logger.Warnf("%s: %s", cmd.PrintableCommandArgs(), stdErr) + } + + stdout := outBuffer.Bytes() + if err := json.Unmarshal(stdout, v); err != nil { + logger.Warnf("Failed to parse %s command output, first lines:\n%s", cmd.PrintableCommandArgs(), firstLines(string(stdout), 10)) + return err + } + return nil +} + +func firstLines(out string, count int) string { + if count < 1 { + return "" + } + + var lines []string + scanner := bufio.NewScanner(strings.NewReader(out)) + for scanner.Scan() { + lines = append(lines, scanner.Text()) + if len(lines) >= count { + break + } + } + return strings.Join(lines, "\n") +} + +// xcresulttoolExport exports a file with the given id at the given output path. +func xcresulttoolExport(xcresultPth, id, outputPth string, useLegacyFlag bool) error { + args := []string{"xcresulttool", "export"} + + supportsNewMethod, err := supportsNewExtractionMethods() + if err != nil { + return err + } + + if supportsNewMethod && !useLegacyFlag { + args = append(args, "attachments") + } else { + args = append(args, "--type", "file") + + if isXcode16OrNewer() && useLegacyFlag { + args = append(args, "--legacy") + } + } + + args = append(args, "--path", xcresultPth) + args = append(args, "--output-path", outputPth) + + if id != "" { + args = append(args, "--id", id) + } + + cmd := command.New("xcrun", args...) + out, err := cmd.RunAndReturnTrimmedCombinedOutput() + if err != nil { + if errorutil.IsExitStatusError(err) { + return fmt.Errorf("%s failed: %s", cmd.PrintableCommandArgs(), out) + } + return fmt.Errorf("%s failed: %s", cmd.PrintableCommandArgs(), err) + } + return nil +}