diff --git a/.harness/blah.yaml b/.harness/blah.yaml new file mode 100644 index 0000000000000..2263497a31e0e --- /dev/null +++ b/.harness/blah.yaml @@ -0,0 +1,14 @@ +inputSet: + identifier: "blah" + name: "blah" + orgIdentifier: "default" + projectIdentifier: "local_project" + pipeline: + identifier: "local_pipe_march" + properties: + ci: + codebase: + build: + type: "branch" + spec: + branch: "blah" diff --git a/.harness/connectors/CI_3574.yaml b/.harness/connectors/CI_3574.yaml new file mode 100644 index 0000000000000..56b0a6bcfa93c --- /dev/null +++ b/.harness/connectors/CI_3574.yaml @@ -0,0 +1,19 @@ +connector: + name: CI_3574 + identifier: CI_3574 + description: + orgIdentifier: default + projectIdentifier: local_project + type: Github + spec: + url: https://github.com/devkimittal/ + validationRepo: harness-core.git + authentication: + type: Http + spec: + type: UsernameToken + spec: + username: devkimittal + tokenRef: devki_mittal + executeOnDelegate: false + type: Account diff --git a/.harness/connectors/github_connect.yaml b/.harness/connectors/github_connect.yaml new file mode 100644 index 0000000000000..9af82277f015d --- /dev/null +++ b/.harness/connectors/github_connect.yaml @@ -0,0 +1,23 @@ +connector: + name: github_connect1 + identifier: github_connect + description: + orgIdentifier: default + projectIdentifier: local_project + type: Github + spec: + url: https://github.com/devkimittal/harness-core.git + validationRepo: harness-core.git + authentication: + type: Http + spec: + type: UsernameToken + spec: + username: devkimittal + tokenRef: git_connector_token + apiAccess: + type: Token + spec: + tokenRef: git_connector_token + executeOnDelegate: true + type: Repo diff --git a/.harness/connectors/kube_cluster.yaml b/.harness/connectors/kube_cluster.yaml new file mode 100644 index 0000000000000..d0c6db86ede8b --- /dev/null +++ b/.harness/connectors/kube_cluster.yaml @@ -0,0 +1,19 @@ +connector: + name: kube_cluster + identifier: kube_cluster + description: + orgIdentifier: default + projectIdentifier: local_project + type: K8sCluster + spec: + credential: + type: ManualConfig + spec: + masterUrl: https://35.184.86.207 + auth: + type: ServiceAccount + spec: + serviceAccountTokenRef: ciplay_secret + caCertRef: null + delegateSelectors: + - acctgroup diff --git a/.harness/connectors/local_dock.yaml b/.harness/connectors/local_dock.yaml new file mode 100644 index 0000000000000..8aac92e1d7d49 --- /dev/null +++ b/.harness/connectors/local_dock.yaml @@ -0,0 +1,14 @@ +connector: + name: local_dock + identifier: local_dock + description: + orgIdentifier: default + projectIdentifier: local_project + type: DockerRegistry + spec: + dockerRegistryUrl: https://index.docker.io/v2/ + providerType: DockerHub + auth: + type: Anonymous + delegateSelectors: + - acctgroup diff --git a/.harness/overlayff.yaml b/.harness/overlayff.yaml new file mode 100644 index 0000000000000..7f53606a16d57 --- /dev/null +++ b/.harness/overlayff.yaml @@ -0,0 +1,14 @@ +inputSet: + identifier: "overlayff" + name: "overlayff" + orgIdentifier: "default" + projectIdentifier: "local_project" + pipeline: + identifier: "local_pipe_march" + properties: + ci: + codebase: + build: + type: "branch" + spec: + branch: "develop" diff --git a/.harness/pipe234545.yaml b/.harness/pipe234545.yaml new file mode 100644 index 0000000000000..bea9847e73859 --- /dev/null +++ b/.harness/pipe234545.yaml @@ -0,0 +1,30 @@ +pipeline: + name: pipe2 + identifier: pipe2 + allowStageExecutions: false + projectIdentifier: local_project + orgIdentifier: default + tags: {} + stages: + - stage: + name: buidl + identifier: buidl + type: CI + spec: + cloneCodebase: false + infrastructure: + type: KubernetesDirect + spec: + connectorRef: kube + namespace: harness-qa-delegate + execution: + steps: + - step: + type: Run + name: run + identifier: run + spec: + connectorRef: account.harnessImage + image: alpine + shell: Sh + command: echo "new pipe" diff --git a/.harness/pipelines/CI_3745.yaml b/.harness/pipelines/CI_3745.yaml new file mode 100644 index 0000000000000..1fd92a7acafa6 --- /dev/null +++ b/.harness/pipelines/CI_3745.yaml @@ -0,0 +1,35 @@ +pipeline: + name: CI_3745 + identifier: CI_3745 + allowStageExecutions: false + projectIdentifier: local_project + orgIdentifier: default + tags: {} + properties: + ci: + codebase: + connectorRef: github_connect + build: <+input> + stages: + - stage: + name: build + identifier: build + type: CI + spec: + cloneCodebase: false + infrastructure: + type: KubernetesDirect + spec: + connectorRef: kube_cluster + namespace: harness-qa-delegate + execution: + steps: + - step: + type: Run + name: docker + identifier: docker + spec: + connectorRef: local_dock + image: alpine + shell: Sh + command: echo "build it" diff --git a/.harness/pipelines/local_pipe_march.yaml b/.harness/pipelines/local_pipe_march.yaml new file mode 100644 index 0000000000000..614fdc9d3ee48 --- /dev/null +++ b/.harness/pipelines/local_pipe_march.yaml @@ -0,0 +1,47 @@ +pipeline: + name: local_pipe_march + identifier: local_pipe_march + allowStageExecutions: false + projectIdentifier: local_project + orgIdentifier: default + tags: {} + properties: + ci: + codebase: + connectorRef: github_connect + build: <+input> + stages: + - stage: + name: build + identifier: build + type: CI + spec: + cloneCodebase: true + infrastructure: + type: KubernetesDirect + spec: + connectorRef: kube_cluster + namespace: harness-qa-delegate + execution: + steps: + - step: + type: Run + name: run + identifier: run + spec: + connectorRef: local_dock + image: alpine + shell: Sh + command: |- + echo "Hello_trigegr \n" + echo "<+trigger.branch>" + - step: + type: Run + name: docker 3 + identifier: docker_3 + spec: + connectorRef: local_dock + image: alpine + shell: Sh + command: echo "run2" + variables: [] diff --git a/.harness/test_git.yaml b/.harness/test_git.yaml new file mode 100644 index 0000000000000..6bb0dd71ee476 --- /dev/null +++ b/.harness/test_git.yaml @@ -0,0 +1,18 @@ +connector: + name: test_git_ + identifier: test_git + description: + orgIdentifier: default + projectIdentifier: local_project + type: Github + spec: + url: https://github.com/devkimittal/harness-core.git + authentication: + type: Http + spec: + type: UsernameToken + spec: + username: devkimittal + tokenRef: git_connector_token + executeOnDelegate: true + type: Repo diff --git a/889-yaml-commons/src/main/java/io/harness/pms/merger/helpers/MergeHelper.java b/889-yaml-commons/src/main/java/io/harness/pms/merger/helpers/MergeHelper.java index a21f67b147f2e..963eaa9e57cf4 100644 --- a/889-yaml-commons/src/main/java/io/harness/pms/merger/helpers/MergeHelper.java +++ b/889-yaml-commons/src/main/java/io/harness/pms/merger/helpers/MergeHelper.java @@ -121,16 +121,23 @@ public String mergeUpdatesIntoJson(String pipelineJson, Map fqnT return JsonUtils.asJson(pipelineNode.getCurrJsonNode()); } fqnToJsonMap.keySet().forEach(fqn -> { + String content = fqnToJsonMap.get(fqn); + content = removeNonASCII(content); try { - pipelineNode.replacePath(fqn, YamlUtils.readTree(fqnToJsonMap.get(fqn)).getNode().getCurrJsonNode()); + pipelineNode.replacePath(fqn, YamlUtils.readTree(content).getNode().getCurrJsonNode()); } catch (IOException e) { - log.error("Could not read json provided for the fqn: " + fqn + ". Json:\n" + fqnToJsonMap.get(fqn), e); + log.error("Could not read json provided for the fqn: " + fqn + ". Json:\n" + content, e); throw new YamlException("Could not read json provided for the fqn: " + fqn); } }); return JsonUtils.asJson(pipelineNode.getCurrJsonNode()); } + // Yaml Object Mapper can't handle emojis and non ascii characters + public String removeNonASCII(String content) { + return content.replaceAll("[^\\x00-\\x7F]", ""); + } + public String removeFQNs(String json, List toBeRemovedFQNs) { if (EmptyPredicate.isEmpty(toBeRemovedFQNs)) { return json; diff --git a/889-yaml-commons/src/test/java/io/harness/pms/merger/helpers/MergeHelperTest.java b/889-yaml-commons/src/test/java/io/harness/pms/merger/helpers/MergeHelperTest.java index 60dfae53e4e36..bd7c18da014f0 100644 --- a/889-yaml-commons/src/test/java/io/harness/pms/merger/helpers/MergeHelperTest.java +++ b/889-yaml-commons/src/test/java/io/harness/pms/merger/helpers/MergeHelperTest.java @@ -9,6 +9,7 @@ import static io.harness.annotations.dev.HarnessTeam.PIPELINE; import static io.harness.pms.merger.helpers.MergeHelper.mergeRuntimeInputValuesIntoOriginalYaml; +import static io.harness.rule.OwnerRule.DEV_MITTAL; import static io.harness.rule.OwnerRule.NAMAN; import static org.assertj.core.api.Assertions.assertThat; @@ -120,6 +121,53 @@ public void testMergeYamlUpdates() throws IOException { assertThat(noUpdatesOnYaml).isEqualTo(readFile("opa-pipeline.json")); } + @Test + @Owner(developers = DEV_MITTAL) + @Category(UnitTests.class) + public void testMergeYamlUpdatesEmojis() throws IOException { + String filename = "opa-pipeline.yaml"; + String pipeline = readFile(filename); + + String fqn1 = "pipeline/stages/[0]/stage/spec/execution/steps/[0]/step/spec/connector"; + String exp1 = readFile("emojiExp1.txt"); + + String fqn2 = "pipeline/stages/[1]/stage/spec/infrastructure/environment"; + String exp2 = readFile("emojiExp2.txt"); + + Map fqnToJsonMap = new HashMap<>(); + fqnToJsonMap.put(fqn1, exp1); + fqnToJsonMap.put(fqn2, exp2); + String expandedPipeline = MergeHelper.mergeUpdatesIntoJson(pipeline, fqnToJsonMap); + assertThat(expandedPipeline).isNotNull(); + String expandedPipelineExpected = readFile("opa-pipeline-with-expansions-no-removals.json"); + assertThat(expandedPipeline).isEqualTo(expandedPipelineExpected); + YamlField yamlField = YamlUtils.readTree(expandedPipeline); + YamlNode firstExp = + yamlField.getNode().gotoPath("pipeline/stages/[0]/stage/spec/execution/steps/[0]/step/spec/connector"); + YamlNode secondExp = yamlField.getNode().gotoPath("pipeline/stages/[1]/stage/spec/infrastructure/environment"); + assertThat(firstExp).isNotNull(); + assertThat(secondExp).isNotNull(); + + String noUpdates = MergeHelper.mergeUpdatesIntoJson(expandedPipeline, null); + assertThat(noUpdates).isEqualTo(expandedPipeline); + + String noUpdatesOnYaml = MergeHelper.mergeUpdatesIntoJson(pipeline, null); + assertThat(noUpdatesOnYaml).isEqualTo(readFile("opa-pipeline.json")); + } + + @Test + @Owner(developers = DEV_MITTAL) + @Category(UnitTests.class) + public void testRemoveNonASCII() { + String asciiString = ""; + for (int i = 0; i < 128; i++) { + asciiString += (char) i; + } + assertThat(MergeHelper.removeNonASCII(asciiString)).isEqualTo(asciiString); + assertThat(MergeHelper.removeNonASCII(asciiString + "\ud330\ud803\ud823")).isEqualTo(asciiString); + assertThat(MergeHelper.removeNonASCII("\ud230\ud803\ud123" + asciiString)).isEqualTo(asciiString); + } + @Test @Owner(developers = NAMAN) @Category(UnitTests.class) diff --git a/889-yaml-commons/src/test/resources/emojiExp1.txt b/889-yaml-commons/src/test/resources/emojiExp1.txt new file mode 100644 index 0000000000000..086ccb697b267 --- /dev/null +++ b/889-yaml-commons/src/test/resources/emojiExp1.txt @@ -0,0 +1,15 @@ +{ + "name": "jira ๐Ÿคฃ๐Ÿ˜„๐˜…โ…๐Ÿ˜ดโŽbasic", + "identifier": "jira_b๐Ÿ˜จโ„ฌ๐Ÿ˜asic", + "description": "", + "orgIdentifier": "defโ˜บ๏ธ๐Ÿ˜Šault", + "projectIdentifier": "Local๐Ÿ˜๐Ÿ˜Š_Dev", + "tags": {}, + "type": "Ji๐Ÿง‘โ€๐Ÿญ๐Ÿ‘ฉ๐Ÿฝโ€๐Ÿญ๐Ÿ‘จ๐Ÿผโ€๐Ÿ”งra", + "spec": { + "jiraUrl": "๐Ÿ‘จ๐Ÿปโ€๐Ÿ’ผ๐Ÿ™‡๐Ÿปโ€โ™‚๏ธ๐Ÿ™‡๐Ÿปhttps://harness.atlassian.net/", + "username": "naman๐Ÿ™‡๐Ÿปvermah", + "passwordRef": "nvh_do๐Ÿ๐˜…๐Ÿ€cker_pass", + "delegateSelectors": [] + } +} \ No newline at end of file diff --git a/889-yaml-commons/src/test/resources/emojiExp2.txt b/889-yaml-commons/src/test/resources/emojiExp2.txt new file mode 100644 index 0000000000000..a8a70a50612f0 --- /dev/null +++ b/889-yaml-commons/src/test/resources/emojiExp2.txt @@ -0,0 +1,10 @@ +{ + "identifier": "PR_ENV", + "name": "PR ENV๐˜…๐Ÿ˜ด๐Ÿ˜โšก๏ธŽ", + "description": "", + "type": "PreP๐Ÿ’๐Ÿผโ€โ™€๏ธ๐Ÿ๐Ÿขroduction", + "accountIdentifier": "kmpySmUISimoRrJL6NL73w", + "orgIdentifier": "d๐Ÿ๐Ÿชฐ๐Ÿฆ„efault", + "projectIdentifier": "Loc๐Ÿฟ๐Ÿ€๐Ÿ€al_Dev", + "tags": {} +} \ No newline at end of file