Ci 4254

.harness/blah.yaml

@@ -0,0 +1,14 @@
+inputSet:
+  identifier: "blah"
+  name: "blah"
+  orgIdentifier: "default"
+  projectIdentifier: "local_project"
+  pipeline:
+    identifier: "local_pipe_march"
+    properties:
+      ci:
+        codebase:
+          build:
+            type: "branch"
+            spec:
+              branch: "blah"

.harness/connectors/CI_3574.yaml

@@ -0,0 +1,19 @@
+connector:
+  name: CI_3574
+  identifier: CI_3574
+  description:
+  orgIdentifier: default
+  projectIdentifier: local_project
+  type: Github
+  spec:
+    url: https://github.com/devkimittal/
+    validationRepo: harness-core.git
+    authentication:
+      type: Http
+      spec:
+        type: UsernameToken
+        spec:
+          username: devkimittal
+          tokenRef: devki_mittal
+    executeOnDelegate: false
+    type: Account

.harness/connectors/github_connect.yaml

@@ -0,0 +1,23 @@
+connector:
+  name: github_connect1
+  identifier: github_connect
+  description:
+  orgIdentifier: default
+  projectIdentifier: local_project
+  type: Github
+  spec:
+    url: https://github.com/devkimittal/harness-core.git
+    validationRepo: harness-core.git
+    authentication:
+      type: Http
+      spec:
+        type: UsernameToken
+        spec:
+          username: devkimittal
+          tokenRef: git_connector_token
+    apiAccess:
+      type: Token
+      spec:
+        tokenRef: git_connector_token
+    executeOnDelegate: true
+    type: Repo

.harness/connectors/kube_cluster.yaml

@@ -0,0 +1,19 @@
+connector:
+  name: kube_cluster
+  identifier: kube_cluster
+  description:
+  orgIdentifier: default
+  projectIdentifier: local_project
+  type: K8sCluster
+  spec:
+    credential:
+      type: ManualConfig
+      spec:
+        masterUrl: https://35.184.86.207
+        auth:
+          type: ServiceAccount
+          spec:
+            serviceAccountTokenRef: ciplay_secret
+            caCertRef: null
+    delegateSelectors:
+    - acctgroup

.harness/connectors/local_dock.yaml

@@ -0,0 +1,14 @@
+connector:
+  name: local_dock
+  identifier: local_dock
+  description:
+  orgIdentifier: default
+  projectIdentifier: local_project
+  type: DockerRegistry
+  spec:
+    dockerRegistryUrl: https://index.docker.io/v2/
+    providerType: DockerHub
+    auth:
+      type: Anonymous
+    delegateSelectors:
+    - acctgroup

.harness/overlayff.yaml

@@ -0,0 +1,14 @@
+inputSet:
+  identifier: "overlayff"
+  name: "overlayff"
+  orgIdentifier: "default"
+  projectIdentifier: "local_project"
+  pipeline:
+    identifier: "local_pipe_march"
+    properties:
+      ci:
+        codebase:
+          build:
+            type: "branch"
+            spec:
+              branch: "develop"

.harness/pipe234545.yaml

@@ -0,0 +1,30 @@
+pipeline:
+    name: pipe2
+    identifier: pipe2
+    allowStageExecutions: false
+    projectIdentifier: local_project
+    orgIdentifier: default
+    tags: {}
+    stages:
+        - stage:
+              name: buidl
+              identifier: buidl
+              type: CI
+              spec:
+                  cloneCodebase: false
+                  infrastructure:
+                      type: KubernetesDirect
+                      spec:
+                          connectorRef: kube
+                          namespace: harness-qa-delegate
+                  execution:
+                      steps:
+                          - step:
+                                type: Run
+                                name: run
+                                identifier: run
+                                spec:
+                                    connectorRef: account.harnessImage
+                                    image: alpine
+                                    shell: Sh
+                                    command: echo "new pipe"

.harness/pipelines/CI_3745.yaml

@@ -0,0 +1,35 @@
+pipeline:
+    name: CI_3745
+    identifier: CI_3745
+    allowStageExecutions: false
+    projectIdentifier: local_project
+    orgIdentifier: default
+    tags: {}
+    properties:
+        ci:
+            codebase:
+                connectorRef: github_connect
+                build: <+input>
+    stages:
+        - stage:
+              name: build
+              identifier: build
+              type: CI
+              spec:
+                  cloneCodebase: false
+                  infrastructure:
+                      type: KubernetesDirect
+                      spec:
+                          connectorRef: kube_cluster
+                          namespace: harness-qa-delegate
+                  execution:
+                      steps:
+                          - step:
+                                type: Run
+                                name: docker
+                                identifier: docker
+                                spec:
+                                    connectorRef: local_dock
+                                    image: alpine
+                                    shell: Sh
+                                    command: echo "build it"

.harness/pipelines/local_pipe_march.yaml

@@ -0,0 +1,47 @@
+pipeline:
+    name: local_pipe_march
+    identifier: local_pipe_march
+    allowStageExecutions: false
+    projectIdentifier: local_project
+    orgIdentifier: default
+    tags: {}
+    properties:
+        ci:
+            codebase:
+                connectorRef: github_connect
+                build: <+input>
+    stages:
+        - stage:
+              name: build
+              identifier: build
+              type: CI
+              spec:
+                  cloneCodebase: true
+                  infrastructure:
+                      type: KubernetesDirect
+                      spec:
+                          connectorRef: kube_cluster
+                          namespace: harness-qa-delegate
+                  execution:
+                      steps:
+                          - step:
+                                type: Run
+                                name: run
+                                identifier: run
+                                spec:
+                                    connectorRef: local_dock
+                                    image: alpine
+                                    shell: Sh
+                                    command: |-
+                                        echo "Hello_trigegr \n"
+                                        echo "<+trigger.branch>"
+                          - step:
+                                type: Run
+                                name: docker 3
+                                identifier: docker_3
+                                spec:
+                                    connectorRef: local_dock
+                                    image: alpine
+                                    shell: Sh
+                                    command: echo "run2"
+    variables: []

.harness/test_git.yaml

@@ -0,0 +1,18 @@
+connector:
+  name: test_git_
+  identifier: test_git
+  description:
+  orgIdentifier: default
+  projectIdentifier: local_project
+  type: Github
+  spec:
+    url: https://github.com/devkimittal/harness-core.git
+    authentication:
+      type: Http
+      spec:
+        type: UsernameToken
+        spec:
+          username: devkimittal
+          tokenRef: git_connector_token
+    executeOnDelegate: true
+    type: Repo

889-yaml-commons/src/main/java/io/harness/pms/merger/helpers/MergeHelper.java

@@ -121,16 +121,23 @@ public String mergeUpdatesIntoJson(String pipelineJson, Map<String, String> fqnT
       return JsonUtils.asJson(pipelineNode.getCurrJsonNode());
     }
     fqnToJsonMap.keySet().forEach(fqn -> {
+      String content = fqnToJsonMap.get(fqn);
+      content = removeNonASCII(content);
       try {
-        pipelineNode.replacePath(fqn, YamlUtils.readTree(fqnToJsonMap.get(fqn)).getNode().getCurrJsonNode());
+        pipelineNode.replacePath(fqn, YamlUtils.readTree(content).getNode().getCurrJsonNode());
       } catch (IOException e) {
-        log.error("Could not read json provided for the fqn: " + fqn + ". Json:\n" + fqnToJsonMap.get(fqn), e);
+        log.error("Could not read json provided for the fqn: " + fqn + ". Json:\n" + content, e);
         throw new YamlException("Could not read json provided for the fqn: " + fqn);
       }
     });
     return JsonUtils.asJson(pipelineNode.getCurrJsonNode());
   }
 
+  // Yaml Object Mapper can't handle emojis and non ascii characters
+  public String removeNonASCII(String content) {
+    return content.replaceAll("[^\\x00-\\x7F]", "");
+  }
+
   public String removeFQNs(String json, List<String> toBeRemovedFQNs) {
     if (EmptyPredicate.isEmpty(toBeRemovedFQNs)) {
       return json;

889-yaml-commons/src/test/java/io/harness/pms/merger/helpers/MergeHelperTest.java

@@ -9,6 +9,7 @@
 
 import static io.harness.annotations.dev.HarnessTeam.PIPELINE;
 import static io.harness.pms.merger.helpers.MergeHelper.mergeRuntimeInputValuesIntoOriginalYaml;
+import static io.harness.rule.OwnerRule.DEV_MITTAL;
 import static io.harness.rule.OwnerRule.NAMAN;
 
 import static org.assertj.core.api.Assertions.assertThat;
@@ -120,6 +121,53 @@ public void testMergeYamlUpdates() throws IOException {
     assertThat(noUpdatesOnYaml).isEqualTo(readFile("opa-pipeline.json"));
   }
 
+  @Test
+  @Owner(developers = DEV_MITTAL)
+  @Category(UnitTests.class)
+  public void testMergeYamlUpdatesEmojis() throws IOException {
+    String filename = "opa-pipeline.yaml";
+    String pipeline = readFile(filename);
+
+    String fqn1 = "pipeline/stages/[0]/stage/spec/execution/steps/[0]/step/spec/connector";
+    String exp1 = readFile("emojiExp1.txt");
+
+    String fqn2 = "pipeline/stages/[1]/stage/spec/infrastructure/environment";
+    String exp2 = readFile("emojiExp2.txt");
+
+    Map<String, String> fqnToJsonMap = new HashMap<>();
+    fqnToJsonMap.put(fqn1, exp1);
+    fqnToJsonMap.put(fqn2, exp2);
+    String expandedPipeline = MergeHelper.mergeUpdatesIntoJson(pipeline, fqnToJsonMap);
+    assertThat(expandedPipeline).isNotNull();
+    String expandedPipelineExpected = readFile("opa-pipeline-with-expansions-no-removals.json");
+    assertThat(expandedPipeline).isEqualTo(expandedPipelineExpected);
+    YamlField yamlField = YamlUtils.readTree(expandedPipeline);
+    YamlNode firstExp =
+        yamlField.getNode().gotoPath("pipeline/stages/[0]/stage/spec/execution/steps/[0]/step/spec/connector");
+    YamlNode secondExp = yamlField.getNode().gotoPath("pipeline/stages/[1]/stage/spec/infrastructure/environment");
+    assertThat(firstExp).isNotNull();
+    assertThat(secondExp).isNotNull();
+
+    String noUpdates = MergeHelper.mergeUpdatesIntoJson(expandedPipeline, null);
+    assertThat(noUpdates).isEqualTo(expandedPipeline);
+
+    String noUpdatesOnYaml = MergeHelper.mergeUpdatesIntoJson(pipeline, null);
+    assertThat(noUpdatesOnYaml).isEqualTo(readFile("opa-pipeline.json"));
+  }
+
+  @Test
+  @Owner(developers = DEV_MITTAL)
+  @Category(UnitTests.class)
+  public void testRemoveNonASCII() {
+    String asciiString = "";
+    for (int i = 0; i < 128; i++) {
+      asciiString += (char) i;
+    }
+    assertThat(MergeHelper.removeNonASCII(asciiString)).isEqualTo(asciiString);
+    assertThat(MergeHelper.removeNonASCII(asciiString + "\ud330\ud803\ud823")).isEqualTo(asciiString);
+    assertThat(MergeHelper.removeNonASCII("\ud230\ud803\ud123" + asciiString)).isEqualTo(asciiString);
+  }
+
   @Test
   @Owner(developers = NAMAN)
   @Category(UnitTests.class)

889-yaml-commons/src/test/resources/emojiExp1.txt

@@ -0,0 +1,15 @@
+{
+  "name": "jira 🤣😄𝘅␅😴␎basic",
+  "identifier": "jira_b😨ℬ😁asic",
+  "description": "",
+  "orgIdentifier": "def☺️😊ault",
+  "projectIdentifier": "Local😁😊_Dev",
+  "tags": {},
+  "type": "Ji🧑‍🏭👩🏽‍🏭👨🏼‍🔧ra",
+  "spec": {
+    "jiraUrl": "👨🏻‍💼🙇🏻‍♂️🙇🏻https://harness.atlassian.net/",
+    "username": "naman🙇🏻vermah",
+    "passwordRef": "nvh_do🐁𝘅🍀cker_pass",
+    "delegateSelectors": []
+  }
+}

889-yaml-commons/src/test/resources/emojiExp2.txt

@@ -0,0 +1,10 @@
+{
+  "identifier": "PR_ENV",
+  "name": "PR ENV𝘅😴😁⚡︎",
+  "description": "",
+  "type": "PreP💁🏼‍♀️🐍🐢roduction",
+  "accountIdentifier": "kmpySmUISimoRrJL6NL73w",
+  "orgIdentifier": "d🐍🪰🦄efault",
+  "projectIdentifier": "Loc🐿🐀🍀al_Dev",
+  "tags": {}
+}