From 7a763cf54b2fd2540b7c5d8ddd25238019069f96 Mon Sep 17 00:00:00 2001 From: Yevhenii Chekanskyi Date: Thu, 31 Oct 2019 13:39:16 +0200 Subject: [PATCH 01/16] PLUGIN-75 Marketo Plugin - initial skeleton for entity plugin. --- .gitignore | 3 + checkstyle.xml | 406 ++++++++++++++++++ pom.xml | 297 +++++++++++++ .../cdap/plugin/marketo/common/Marketo.java | 151 +++++++ .../plugin/marketo/common/MarketoEntity.java | 56 +++ .../marketo/common/MarketoSchemaReader.java | 60 +++ .../plugin/marketo/common/MarketoToken.java | 55 +++ .../marketo/common/response/MarketoPage.java | 80 ++++ .../response/describe/DescribeResponse.java | 45 ++ .../describe/DescribeResponseLeads.java | 35 ++ .../common/response/describe/Field.java | 51 +++ .../source/batch/MarketoBatchSource.java | 90 ++++ .../batch/MarketoBatchSourceConfig.java | 153 +++++++ .../source/batch/MarketoInputFormat.java | 41 ++ .../batch/MarketoInputFormatProvider.java | 51 +++ .../source/batch/MarketoRecordReader.java | 75 ++++ .../source/batch/NoOpMarketoSplit.java | 49 +++ suppressions.xml | 33 ++ 18 files changed, 1731 insertions(+) create mode 100644 .gitignore create mode 100644 checkstyle.xml create mode 100644 pom.xml create mode 100644 src/main/java/io/cdap/plugin/marketo/common/Marketo.java create mode 100644 src/main/java/io/cdap/plugin/marketo/common/MarketoEntity.java create mode 100644 src/main/java/io/cdap/plugin/marketo/common/MarketoSchemaReader.java create mode 100644 src/main/java/io/cdap/plugin/marketo/common/MarketoToken.java create mode 100644 src/main/java/io/cdap/plugin/marketo/common/response/MarketoPage.java create mode 100644 src/main/java/io/cdap/plugin/marketo/common/response/describe/DescribeResponse.java create mode 100644 src/main/java/io/cdap/plugin/marketo/common/response/describe/DescribeResponseLeads.java create mode 100644 src/main/java/io/cdap/plugin/marketo/common/response/describe/Field.java create mode 100644 src/main/java/io/cdap/plugin/marketo/source/batch/MarketoBatchSource.java create mode 100644 src/main/java/io/cdap/plugin/marketo/source/batch/MarketoBatchSourceConfig.java create mode 100644 src/main/java/io/cdap/plugin/marketo/source/batch/MarketoInputFormat.java create mode 100644 src/main/java/io/cdap/plugin/marketo/source/batch/MarketoInputFormatProvider.java create mode 100644 src/main/java/io/cdap/plugin/marketo/source/batch/MarketoRecordReader.java create mode 100644 src/main/java/io/cdap/plugin/marketo/source/batch/NoOpMarketoSplit.java create mode 100644 suppressions.xml diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..34e1547 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +target +.idea +*.iml \ No newline at end of file diff --git a/checkstyle.xml b/checkstyle.xml new file mode 100644 index 0000000..fb35f2d --- /dev/null +++ b/checkstyle.xml @@ -0,0 +1,406 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/pom.xml b/pom.xml new file mode 100644 index 0000000..60f2184 --- /dev/null +++ b/pom.xml @@ -0,0 +1,297 @@ + + + 4.0.0 + + io.cdap + marketo-entity-plugin + 1.0-SNAPSHOT + + + + sonatype + https://oss.sonatype.org/content/groups/public + + + sonatype-snapshots + https://oss.sonatype.org/content/repositories/snapshots + + + + + 6.1.0-SNAPSHOT + 2.3.0 + 2.3.0-SNAPSHOT + 2.1.3 + + + + + io.cdap.cdap + cdap-api + ${cdap.version} + provided + + + io.cdap.cdap + cdap-etl-api + ${cdap.version} + provided + + + io.cdap.cdap + cdap-formats + ${cdap.version} + + + org.apache.avro + avro + + + io.thekraken + grok + + + + + io.cdap.plugin + hydrator-common + ${hydrator.version} + + + org.apache.hadoop + hadoop-common + ${hadoop.version} + provided + + + commons-logging + commons-logging + + + log4j + log4j + + + org.slf4j + slf4j-log4j12 + + + org.apache.avro + avro + + + org.apache.zookeeper + zookeeper + + + com.google.guava + guava + + + jersey-core + com.sun.jersey + + + jersey-json + com.sun.jersey + + + jersey-server + com.sun.jersey + + + servlet-api + javax.servlet + + + org.mortbay.jetty + jetty + + + org.mortbay.jetty + jetty-util + + + jasper-compiler + tomcat + + + jasper-runtime + tomcat + + + jsp-api + javax.servlet.jsp + + + slf4j-api + org.slf4j + + + + + org.apache.hadoop + hadoop-mapreduce-client-core + ${hadoop.version} + provided + + + org.slf4j + slf4j-log4j12 + + + com.google.inject.extensions + guice-servlet + + + com.sun.jersey + jersey-core + + + com.sun.jersey + jersey-server + + + com.sun.jersey + jersey-json + + + com.sun.jersey.contribs + jersey-guice + + + javax.servlet + servlet-api + + + com.google.guava + guava + + + + + + io.cdap.cdap + cdap-etl-api-spark + ${cdap.version} + provided + + + org.apache.spark + spark-streaming_2.11 + ${spark2.version} + provided + + + org.apache.spark + spark-core_2.11 + ${spark2.version} + provided + + + org.slf4j + slf4j-log4j12 + + + log4j + log4j + + + org.apache.hadoop + hadoop-client + + + com.esotericsoftware.reflectasm + reflectasm + + + org.apache.curator + curator-recipes + + + + org.scala-lang + scala-compiler + + + org.scala-lang + scala-reflect + + + org.eclipse.jetty.orbit + javax.servlet + + + + net.java.dev.jets3t + jets3t + + + asm + asm + + + + + + com.google.code.gson + gson + 2.8.6 + + + + junit + junit + 4.12 + test + + + + + + + org.apache.maven.plugins + maven-checkstyle-plugin + 2.17 + + + validate + process-test-classes + + checkstyle.xml + suppressions.xml + UTF-8 + true + true + true + **/org/apache/cassandra/**,**/org/apache/hadoop/** + + + check + + + + + + com.puppycrawl.tools + checkstyle + 6.19 + + + + + org.apache.maven.plugins + maven-compiler-plugin + + 8 + 8 + + + + + \ No newline at end of file diff --git a/src/main/java/io/cdap/plugin/marketo/common/Marketo.java b/src/main/java/io/cdap/plugin/marketo/common/Marketo.java new file mode 100644 index 0000000..2c5fd18 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/Marketo.java @@ -0,0 +1,151 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common; + +import com.google.common.base.Strings; +import com.google.gson.Gson; +import io.cdap.plugin.marketo.common.response.MarketoPage; + +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.Reader; +import java.net.URL; +import java.util.Iterator; +import java.util.Map; +import java.util.NoSuchElementException; +import javax.net.ssl.HttpsURLConnection; + +/** + * Helper class to perform Marketo Api operations. + */ +public class Marketo { + /** + * Marketo page iterator. + */ + public static class MarketoPageIterator implements Iterator> { + private MarketoPage currentPage; + private Marketo marketo; + private String queryUrl; + private Iterator> currentPageResultIterator; + + private MarketoPageIterator(MarketoPage page, Marketo marketo, String queryUrl) { + this.currentPage = page; + this.marketo = marketo; + this.queryUrl = queryUrl; + currentPageResultIterator = currentPage.getResults().iterator(); + } + + @Override + public boolean hasNext() { + if (currentPageResultIterator.hasNext()) { + return true; + } else { + MarketoPage nextPage = marketo.getNextPage(currentPage, queryUrl); + if (nextPage != null) { + currentPage = nextPage; + currentPage.getResults().iterator(); + return hasNext(); + } else { + return false; + } + } + } + + @Override + public Map next() { + if (hasNext()) { + return currentPageResultIterator.next(); + } else { + throw new NoSuchElementException(); + } + } + } + + private static final Gson GSON = new Gson(); + + private String marketoEndpoint; + private MarketoToken token; + + public Marketo(String marketoEndpoint, String clientId, String clientSecret) { + this.marketoEndpoint = marketoEndpoint; + token = getToken(marketoEndpoint, clientId, clientSecret); + } + + public Marketo(String marketoEndpoint, MarketoToken token) { + this.marketoEndpoint = marketoEndpoint; + this.token = token; + } + + public MarketoPage getPage(String queryUrl) { + return get(queryUrl, MarketoPage.class); + } + + public MarketoPage getNextPage(MarketoPage page, String queryUrl) { + if (!Strings.isNullOrEmpty(page.getNextPageToken())) { + return getPage(queryUrl + "&nextPageToken=" + page.getNextPageToken()); + } + return null; + } + + public MarketoPageIterator iteratePage(String queryUrl) { + return new MarketoPageIterator(getPage(queryUrl), this, queryUrl); + } + + private String appendToken(String requestUrl) { + if (!requestUrl.contains("access_token")) { + if (requestUrl.contains("?")) { + return requestUrl + "&access_token=" + token.getAccessToken(); + } else { + return requestUrl + "?access_token=" + token.getAccessToken(); + } + } + return requestUrl; + } + + public T get(String queryUrl, Class cls) { + return doGet(appendToken(marketoEndpoint + queryUrl), cls); + } + + public static MarketoToken getToken(String marketoEndpoint, String clientId, String clientSecret) { + String marketoIdURL = marketoEndpoint + "/identity"; + String idEndpoint = marketoIdURL + "/oauth/token?grant_type=client_credentials&client_id=" + + clientId + "&client_secret=" + clientSecret; + + return doGet(idEndpoint, MarketoToken.class); + } + + + private static T doGet(String queryUrl, Class cls) { + try { + URL url = new URL(queryUrl); + HttpsURLConnection urlConn = (HttpsURLConnection) url.openConnection(); + urlConn.setRequestMethod("GET"); + urlConn.setRequestProperty("accept", "application/json"); + int responseCode = urlConn.getResponseCode(); + if (responseCode == 200) { + InputStream inStream = urlConn.getInputStream(); + Reader reader = new InputStreamReader(inStream); + return GSON.fromJson(reader, cls); + } else { + throw new IOException("Status: " + responseCode); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/MarketoEntity.java b/src/main/java/io/cdap/plugin/marketo/common/MarketoEntity.java new file mode 100644 index 0000000..111f239 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/MarketoEntity.java @@ -0,0 +1,56 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common; + +import java.util.Arrays; + +/** + * Marketo Entity types. + */ +public enum MarketoEntity { + Leads("Leads", "GET /rest/v1/leads.json", "/rest/v1/leads/describe.json"), + Campaigns("Campaigns", "/rest/v1/campaigns.json", null), + Companies("Companies", "/rest/v1/companies.json", "/rest/v1/companies/describe.json"); + + private String name; + private String getEndpoint; + private String describeEndpoint; + + MarketoEntity(String name, String getEndpoint, String describeEndpoint) { + this.name = name; + this.getEndpoint = getEndpoint; + this.describeEndpoint = describeEndpoint; + } + + public String getName() { + return name; + } + + public String getGetEndpoint() { + return getEndpoint; + } + + public String getDescribeEndpoint() { + return describeEndpoint; + } + + public static MarketoEntity fromString(String marketoEntity) { + return Arrays.stream(MarketoEntity.values()) + .filter(entity -> entity.getName().equals(marketoEntity)).findFirst() + .orElseThrow(() -> new RuntimeException(String.format("'%s' is not a valid Marketo entity", marketoEntity))); + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/MarketoSchemaReader.java b/src/main/java/io/cdap/plugin/marketo/common/MarketoSchemaReader.java new file mode 100644 index 0000000..6d4996b --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/MarketoSchemaReader.java @@ -0,0 +1,60 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common; + +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.plugin.marketo.common.response.describe.DescribeResponse; +import io.cdap.plugin.marketo.common.response.describe.DescribeResponseLeads; +import io.cdap.plugin.marketo.common.response.describe.Field; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * Generates schema for given entity. + */ +public class MarketoSchemaReader { + // TODO fill predefined schemas + private static final Map PREDEFINED_SCHEMAS = new HashMap<>(); + + public static Schema getSchemaForEntity(MarketoEntity entity, Marketo marketo) { + if (entity.getDescribeEndpoint() != null) { + List entityFields; + + if (entity == MarketoEntity.Leads) { + DescribeResponseLeads leadsDescribe = marketo.get(entity.getDescribeEndpoint(), + DescribeResponseLeads.class); + entityFields = leadsDescribe.getFields(); + } else { + DescribeResponse describe = marketo.get(entity.getDescribeEndpoint(), DescribeResponse.class); + entityFields = describe.getFields(); + } + + List schemaFields = entityFields.stream() + //TODO map types here + .map(field -> Schema.Field.of(field.getName(), Schema.nullableOf(Schema.of(Schema.Type.STRING)))) + .collect(Collectors.toList()); + return Schema.recordOf(entity.getName(), schemaFields); + } else if (PREDEFINED_SCHEMAS.containsKey(entity)) { + return PREDEFINED_SCHEMAS.get(entity); + } + throw new RuntimeException(String.format("Unable to get schema for entity '%s'", entity.getName())); + } + +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/MarketoToken.java b/src/main/java/io/cdap/plugin/marketo/common/MarketoToken.java new file mode 100644 index 0000000..6f7c235 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/MarketoToken.java @@ -0,0 +1,55 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common; + +import com.google.gson.annotations.SerializedName; + +/** + * Marketo Token holder. + */ +public class MarketoToken { + @SerializedName("access_token") + private String accessToken; + private String scope; + @SerializedName("expires_in") + private String expiresIn; + @SerializedName("token_type") + private String tokenType; + + public MarketoToken(String accessToken, String scope, String expiresIn, String tokenType) { + this.accessToken = accessToken; + this.scope = scope; + this.expiresIn = expiresIn; + this.tokenType = tokenType; + } + + public String getAccessToken() { + return accessToken; + } + + public String getScope() { + return scope; + } + + public String getExpiresIn() { + return expiresIn; + } + + public String getTokenType() { + return tokenType; + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/response/MarketoPage.java b/src/main/java/io/cdap/plugin/marketo/common/response/MarketoPage.java new file mode 100644 index 0000000..5311e7f --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/response/MarketoPage.java @@ -0,0 +1,80 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.response; + +import java.util.Collections; +import java.util.List; +import java.util.Map; + +/** + * Marketo page response. + */ +public class MarketoPage { + /** + * Error or warn message description. + */ + public static class Description { + private String code; + private String message; + + public String getCode() { + return code; + } + + public String getMessage() { + return message; + } + } + + private Boolean moreResult = null; + private String nextPageToken = null; + private String requestId = null; + private Boolean success = null; + + private List errors = Collections.emptyList(); + private List warnings = Collections.emptyList(); + + private List> result = Collections.emptyList(); + + public Boolean hasMoreResults() { + return moreResult; + } + + public String getNextPageToken() { + return nextPageToken; + } + + public String getRequestId() { + return requestId; + } + + public Boolean isSuccess() { + return success; + } + + public List getErrors() { + return errors; + } + + public List getWarnings() { + return warnings; + } + + public List> getResults() { + return result; + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/response/describe/DescribeResponse.java b/src/main/java/io/cdap/plugin/marketo/common/response/describe/DescribeResponse.java new file mode 100644 index 0000000..23cc88f --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/response/describe/DescribeResponse.java @@ -0,0 +1,45 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.response.describe; + +import java.util.Collections; +import java.util.List; + +/** + * Regular describe response. + * Fields located in first object in 'result' field. + */ +public class DescribeResponse { + /** + * Result holder. + */ + public static class Result { + List fields = Collections.emptyList(); + } + + private String requestId = null; + private Boolean success = null; + + private List result = Collections.emptyList(); + + public List getFields() { + if (result.size() != 1) { + throw new RuntimeException("Expected to have one result."); + } + return result.get(0).fields; + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/response/describe/DescribeResponseLeads.java b/src/main/java/io/cdap/plugin/marketo/common/response/describe/DescribeResponseLeads.java new file mode 100644 index 0000000..e5eefea --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/response/describe/DescribeResponseLeads.java @@ -0,0 +1,35 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.response.describe; + +import java.util.Collections; +import java.util.List; + +/** + * Describe response for Leads-Leads entity. + * Fields here is directly located in 'result' field of response. + */ +public class DescribeResponseLeads { + private String requestId = null; + private Boolean success = null; + + List result = Collections.emptyList(); + + public List getFields() { + return result; + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/response/describe/Field.java b/src/main/java/io/cdap/plugin/marketo/common/response/describe/Field.java new file mode 100644 index 0000000..7086298 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/response/describe/Field.java @@ -0,0 +1,51 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.response.describe; + +/** + * Marketo field description. + */ +public class Field { + /** + * Soap or rest descriptor. + */ + public static class FieldDescription { + String name; + Boolean readOnly; + } + + String id = null; + private String name = null; + String displayName = null; + String dataType = null; + int length = -1; + Boolean updateable = null; + FieldDescription rest = null; + FieldDescription soap = null; + + public String getName() { + if (name == null) { + if (rest == null) { + throw new RuntimeException("Failed to get name for field."); + } else { + return rest.name; + } + } else { + return name; + } + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoBatchSource.java b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoBatchSource.java new file mode 100644 index 0000000..ad6545e --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoBatchSource.java @@ -0,0 +1,90 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.source.batch; + +import io.cdap.cdap.api.annotation.Description; +import io.cdap.cdap.api.annotation.Name; +import io.cdap.cdap.api.annotation.Plugin; +import io.cdap.cdap.api.data.batch.Input; +import io.cdap.cdap.api.data.format.StructuredRecord; +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.cdap.api.dataset.lib.KeyValue; +import io.cdap.cdap.etl.api.Emitter; +import io.cdap.cdap.etl.api.FailureCollector; +import io.cdap.cdap.etl.api.PipelineConfigurer; +import io.cdap.cdap.etl.api.batch.BatchSource; +import io.cdap.cdap.etl.api.batch.BatchSourceContext; +import io.cdap.plugin.common.LineageRecorder; +import org.apache.hadoop.io.NullWritable; + +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; + +/** + * Plugin that reads entities from Marketo api. + */ +@Plugin(type = BatchSource.PLUGIN_TYPE) +@Name(MarketoBatchSource.NAME) +@Description("Reads entities from Marketo.") +public class MarketoBatchSource extends BatchSource, StructuredRecord> { + public static final String NAME = "MarketoEntityPlugin"; + + private final MarketoBatchSourceConfig config; + + public MarketoBatchSource(MarketoBatchSourceConfig config) { + this.config = config; + + } + + @Override + public void configurePipeline(PipelineConfigurer pipelineConfigurer) { + validateConfiguration(pipelineConfigurer.getStageConfigurer().getFailureCollector()); + pipelineConfigurer.getStageConfigurer().setOutputSchema(config.getSchema()); + } + + @Override + public void prepareRun(BatchSourceContext batchSourceContext) { + validateConfiguration(batchSourceContext.getFailureCollector()); + LineageRecorder lineageRecorder = new LineageRecorder(batchSourceContext, config.referenceName); + lineageRecorder.createExternalDataset(config.getSchema()); + lineageRecorder.recordRead("Read", "Reading Marketo entities", + Objects.requireNonNull(config.getSchema().getFields()).stream() + .map(Schema.Field::getName) + .collect(Collectors.toList())); + + batchSourceContext.setInput(Input.of(config.referenceName, new MarketoInputFormatProvider(config))); + } + + @Override + public void transform(KeyValue> input, Emitter emitter) { + StructuredRecord.Builder builder = StructuredRecord.builder(config.getSchema()); + Map inputMap = input.getValue(); + config.getSchema().getFields().forEach( + field -> { + if (inputMap.containsKey(field.getName())) { + //TODO map fields, also refactor io.cdap.plugin.marketo.common.MarketoSchemaReader.getSchemaForEntity + builder.set(field.getName(), String.valueOf(inputMap.remove(field.getName()))); + } + } + ); + } + + private void validateConfiguration(FailureCollector failureCollector) { + failureCollector.getOrThrowException(); + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoBatchSourceConfig.java b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoBatchSourceConfig.java new file mode 100644 index 0000000..f7f16f2 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoBatchSourceConfig.java @@ -0,0 +1,153 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.source.batch; + +import io.cdap.cdap.api.annotation.Description; +import io.cdap.cdap.api.annotation.Macro; +import io.cdap.cdap.api.annotation.Name; +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.plugin.common.ReferencePluginConfig; +import io.cdap.plugin.marketo.common.Marketo; +import io.cdap.plugin.marketo.common.MarketoEntity; +import io.cdap.plugin.marketo.common.MarketoSchemaReader; +import io.cdap.plugin.marketo.common.MarketoToken; + +/** + * Provides all required configuration for reading Marketo entities. + */ +public class MarketoBatchSourceConfig extends ReferencePluginConfig { + public static final String PROPERTY_ENTITY_NAME = "entityName"; + public static final String PROPERTY_CLIENT_ID = "clientId"; + public static final String PROPERTY_CLIENT_SECRET = "clientSecret"; + public static final String PROPERTY_REST_API_ENDPOINT = "restApiEndpoint"; + public static final String PROPERTY_REST_API_IDENTITY = "restApiIdentity"; + public static final String PROPERTY_DAILY_API_LIMIT = "dailyApiLimit"; + public static final String PROPERTY_REPORT_FORMAT = "reportFormat"; + public static final String PROPERTY_START_DATE = "startDate"; + public static final String PROPERTY_END_DATE = "endDate"; + + @Name(PROPERTY_ENTITY_NAME) + @Description("Marketo entity name to fetch.") + @Macro + protected String entityName; + + @Name(PROPERTY_CLIENT_ID) + @Description("Marketo Client ID.") + @Macro + protected String clientId; + + @Name(PROPERTY_CLIENT_SECRET) + @Description("Marketo Client secret.") + @Macro + protected String clientSecret; + + @Name(PROPERTY_REST_API_ENDPOINT) + @Description("REST API endpoint URL.") + @Macro + protected String restApiEndpoint; + + @Name(PROPERTY_REST_API_IDENTITY) + @Description("REST API identity.") + @Macro + protected String restApiIdentity; + + @Name(PROPERTY_DAILY_API_LIMIT) + @Description("Marketo enforced daily API limit.") + @Macro + protected String dailyApiLimit; + + @Name(PROPERTY_REPORT_FORMAT) + @Description("Report format.") + @Macro + protected String reportFormat; + + @Name(PROPERTY_START_DATE) + @Description("Start date for the report.") + @Macro + protected String startDate; + + @Name(PROPERTY_END_DATE) + @Description("End date for the report.") + @Macro + protected String endDate; + + + private transient MarketoToken token = null; + private transient Schema schema = null; + private transient Marketo marketo = null; + + public MarketoBatchSourceConfig(String referenceName) { + super(referenceName); + } + + public Schema getSchema() { + if (schema == null) { + schema = MarketoSchemaReader.getSchemaForEntity(getEntityType(), getMarketo()); + } + return schema; + } + + public Marketo getMarketo() { + if (marketo == null) { + marketo = new Marketo(getRestApiEndpoint(), getMarketoToken()); + } + return marketo; + } + + public MarketoEntity getEntityType() { + return MarketoEntity.fromString(entityName); + } + + public String getClientId() { + return clientId; + } + + public String getClientSecret() { + return clientSecret; + } + + public String getRestApiEndpoint() { + return restApiEndpoint; + } + + public String getRestApiIdentity() { + return restApiIdentity; + } + + public String getDailyApiLimit() { + return dailyApiLimit; + } + + public String getReportFormat() { + return reportFormat; + } + + public String getStartDate() { + return startDate; + } + + public String getEndDate() { + return endDate; + } + + public MarketoToken getMarketoToken() { + if (token == null) { + token = Marketo.getToken(getRestApiEndpoint(), getClientId(), getClientSecret()); + } + return token; + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoInputFormat.java b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoInputFormat.java new file mode 100644 index 0000000..4fd514e --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoInputFormat.java @@ -0,0 +1,41 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.source.batch; + +import org.apache.hadoop.mapreduce.InputFormat; +import org.apache.hadoop.mapreduce.InputSplit; +import org.apache.hadoop.mapreduce.JobContext; +import org.apache.hadoop.mapreduce.RecordReader; +import org.apache.hadoop.mapreduce.TaskAttemptContext; + +import java.util.Collections; +import java.util.List; + +/** + * InputFormat for mapreduce job, which provides a single split of data. + */ +public class MarketoInputFormat extends InputFormat { + @Override + public List getSplits(JobContext jobContext) { + return Collections.singletonList(new NoOpMarketoSplit()); + } + + @Override + public RecordReader createRecordReader(InputSplit inputSplit, TaskAttemptContext taskAttemptContext) { + return new MarketoRecordReader(); + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoInputFormatProvider.java b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoInputFormatProvider.java new file mode 100644 index 0000000..b1bebb7 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoInputFormatProvider.java @@ -0,0 +1,51 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.source.batch; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import io.cdap.cdap.api.data.batch.InputFormatProvider; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +/** + * InputFormatProvider used by cdap to provide configurations to mapreduce job + */ +public class MarketoInputFormatProvider implements InputFormatProvider { + public static final String PROPERTY_CONFIG_JSON = "cdap.marketo.entity.config"; + private static final Gson gson = new GsonBuilder().create(); + private final Map conf; + + + MarketoInputFormatProvider(MarketoBatchSourceConfig config) { + this.conf = Collections.unmodifiableMap(new HashMap() {{ + put(PROPERTY_CONFIG_JSON, gson.toJson(config)); + }}); + } + + @Override + public String getInputFormatClassName() { + return MarketoInputFormat.class.getName(); + } + + @Override + public Map getInputFormatConfiguration() { + return conf; + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoRecordReader.java b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoRecordReader.java new file mode 100644 index 0000000..9d36451 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoRecordReader.java @@ -0,0 +1,75 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.source.batch; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import io.cdap.plugin.marketo.common.Marketo; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.io.NullWritable; +import org.apache.hadoop.mapreduce.InputSplit; +import org.apache.hadoop.mapreduce.RecordReader; +import org.apache.hadoop.mapreduce.TaskAttemptContext; + +import java.io.IOException; +import java.util.Map; + +/** + * RecordReader implementation, which reads events from Marketo api. + */ +public class MarketoRecordReader extends RecordReader> { + private static final Gson GSON = new GsonBuilder().create(); + private Marketo.MarketoPageIterator iterator; + private Map current = null; + + @Override + public void initialize(InputSplit inputSplit, TaskAttemptContext taskAttemptContext) throws IOException { + Configuration conf = taskAttemptContext.getConfiguration(); + String configJson = conf.get(MarketoInputFormatProvider.PROPERTY_CONFIG_JSON); + MarketoBatchSourceConfig config = GSON.fromJson(configJson, MarketoBatchSourceConfig.class); + + iterator = config.getMarketo().iteratePage(config.getEntityType().getGetEndpoint()); + } + + @Override + public boolean nextKeyValue() { + if (iterator.hasNext()) { + current = iterator.next(); + return true; + } + return false; + } + + @Override + public NullWritable getCurrentKey() { + return null; + } + + @Override + public Map getCurrentValue() { + return current; + } + + @Override + public float getProgress() { + return 0; + } + + @Override + public void close() { + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/source/batch/NoOpMarketoSplit.java b/src/main/java/io/cdap/plugin/marketo/source/batch/NoOpMarketoSplit.java new file mode 100644 index 0000000..396ad97 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/source/batch/NoOpMarketoSplit.java @@ -0,0 +1,49 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.source.batch; + +import org.apache.hadoop.io.Writable; +import org.apache.hadoop.mapreduce.InputSplit; + +import java.io.DataInput; +import java.io.DataOutput; + +/** + * A no-op split. + */ +public class NoOpMarketoSplit extends InputSplit implements Writable { + public NoOpMarketoSplit() { + } + + @Override + public void readFields(DataInput dataInput) { + } + + @Override + public void write(DataOutput dataOutput) { + } + + @Override + public long getLength() { + return 0; + } + + @Override + public String[] getLocations() { + return new String[0]; + } +} diff --git a/suppressions.xml b/suppressions.xml new file mode 100644 index 0000000..600350b --- /dev/null +++ b/suppressions.xml @@ -0,0 +1,33 @@ + + + + + + + + + + + + + + + + + From 1cfa51feab80417d7296c4c4a31a74bd6b1f57de Mon Sep 17 00:00:00 2001 From: Yevhenii Chekanskyi Date: Mon, 25 Nov 2019 14:09:36 +0200 Subject: [PATCH 02/16] PLUGIN-75 Marketo Plugin - bulk export of leads. --- pom.xml | 51 ++++++ .../cdap/plugin/marketo/common/Marketo.java | 151 ------------------ .../plugin/marketo/common/MarketoEntity.java | 56 ------- .../marketo/common/MarketoSchemaReader.java | 60 ------- .../plugin/marketo/common/api/HttpHelper.java | 106 ++++++++++++ .../marketo/common/api/LeadsExportJob.java | 63 ++++++++ .../plugin/marketo/common/api/Marketo.java | 121 ++++++++++++++ .../common/api/MarketoPageIterator.java | 59 +++++++ .../cdap/plugin/marketo/common/api/Urls.java | 12 ++ .../common/api/entities/BaseResponse.java | 41 +++++ .../marketo/common/api/entities/Error.java | 22 +++ .../{ => api/entities}/MarketoToken.java | 4 +- .../marketo/common/api/entities/Warning.java | 22 +++ .../api/entities/leads/LeadsDescribe.java | 69 ++++++++ .../api/entities/leads/LeadsExport.java | 82 ++++++++++ .../entities/leads/LeadsExportRequest.java | 107 +++++++++++++ .../marketo/common/response/MarketoPage.java | 80 ---------- .../response/describe/DescribeResponse.java | 45 ------ .../describe/DescribeResponseLeads.java | 35 ---- .../common/response/describe/Field.java | 51 ------ .../batch/MarketoInputFormatProvider.java | 4 +- .../source/batch/MarketoRecordReader.java | 50 +++++- ...ource.java => MarketoReportingPlugin.java} | 16 +- ...java => MarketoReportingSourceConfig.java} | 29 ++-- .../MarketoReportingPlugin-batchsource.json | 28 ++++ 25 files changed, 856 insertions(+), 508 deletions(-) delete mode 100644 src/main/java/io/cdap/plugin/marketo/common/Marketo.java delete mode 100644 src/main/java/io/cdap/plugin/marketo/common/MarketoEntity.java delete mode 100644 src/main/java/io/cdap/plugin/marketo/common/MarketoSchemaReader.java create mode 100644 src/main/java/io/cdap/plugin/marketo/common/api/HttpHelper.java create mode 100644 src/main/java/io/cdap/plugin/marketo/common/api/LeadsExportJob.java create mode 100644 src/main/java/io/cdap/plugin/marketo/common/api/Marketo.java create mode 100644 src/main/java/io/cdap/plugin/marketo/common/api/MarketoPageIterator.java create mode 100644 src/main/java/io/cdap/plugin/marketo/common/api/Urls.java create mode 100644 src/main/java/io/cdap/plugin/marketo/common/api/entities/BaseResponse.java create mode 100644 src/main/java/io/cdap/plugin/marketo/common/api/entities/Error.java rename src/main/java/io/cdap/plugin/marketo/common/{ => api/entities}/MarketoToken.java (93%) create mode 100644 src/main/java/io/cdap/plugin/marketo/common/api/entities/Warning.java create mode 100644 src/main/java/io/cdap/plugin/marketo/common/api/entities/leads/LeadsDescribe.java create mode 100644 src/main/java/io/cdap/plugin/marketo/common/api/entities/leads/LeadsExport.java create mode 100644 src/main/java/io/cdap/plugin/marketo/common/api/entities/leads/LeadsExportRequest.java delete mode 100644 src/main/java/io/cdap/plugin/marketo/common/response/MarketoPage.java delete mode 100644 src/main/java/io/cdap/plugin/marketo/common/response/describe/DescribeResponse.java delete mode 100644 src/main/java/io/cdap/plugin/marketo/common/response/describe/DescribeResponseLeads.java delete mode 100644 src/main/java/io/cdap/plugin/marketo/common/response/describe/Field.java rename src/main/java/io/cdap/plugin/marketo/source/batch/{MarketoBatchSource.java => MarketoReportingPlugin.java} (83%) rename src/main/java/io/cdap/plugin/marketo/source/batch/{MarketoBatchSourceConfig.java => MarketoReportingSourceConfig.java} (82%) create mode 100644 widgets/MarketoReportingPlugin-batchsource.json diff --git a/pom.xml b/pom.xml index 60f2184..fb9a529 100644 --- a/pom.xml +++ b/pom.xml @@ -243,6 +243,16 @@ gson 2.8.6 + + org.apache.commons + commons-csv + 1.7 + + + commons-io + commons-io + 2.6 + junit @@ -292,6 +302,47 @@ 8 + + io.cdap + cdap-maven-plugin + 1.1.0 + + + system:cdap-data-pipeline[6.1.0-SNAPSHOT,7.0.0-SNAPSHOT) + + + + + create-artifact-config + prepare-package + + create-plugin-json + + + + + + org.apache.felix + maven-bundle-plugin + 3.5.0 + true + + + <_exportcontents>io.cdap.plugin.marketo.* + *;inline=false;scope=compile + true + lib + + + + + package + + bundle + + + + \ No newline at end of file diff --git a/src/main/java/io/cdap/plugin/marketo/common/Marketo.java b/src/main/java/io/cdap/plugin/marketo/common/Marketo.java deleted file mode 100644 index 2c5fd18..0000000 --- a/src/main/java/io/cdap/plugin/marketo/common/Marketo.java +++ /dev/null @@ -1,151 +0,0 @@ -/* - * Copyright © 2019 Cask Data, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package io.cdap.plugin.marketo.common; - -import com.google.common.base.Strings; -import com.google.gson.Gson; -import io.cdap.plugin.marketo.common.response.MarketoPage; - -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.Reader; -import java.net.URL; -import java.util.Iterator; -import java.util.Map; -import java.util.NoSuchElementException; -import javax.net.ssl.HttpsURLConnection; - -/** - * Helper class to perform Marketo Api operations. - */ -public class Marketo { - /** - * Marketo page iterator. - */ - public static class MarketoPageIterator implements Iterator> { - private MarketoPage currentPage; - private Marketo marketo; - private String queryUrl; - private Iterator> currentPageResultIterator; - - private MarketoPageIterator(MarketoPage page, Marketo marketo, String queryUrl) { - this.currentPage = page; - this.marketo = marketo; - this.queryUrl = queryUrl; - currentPageResultIterator = currentPage.getResults().iterator(); - } - - @Override - public boolean hasNext() { - if (currentPageResultIterator.hasNext()) { - return true; - } else { - MarketoPage nextPage = marketo.getNextPage(currentPage, queryUrl); - if (nextPage != null) { - currentPage = nextPage; - currentPage.getResults().iterator(); - return hasNext(); - } else { - return false; - } - } - } - - @Override - public Map next() { - if (hasNext()) { - return currentPageResultIterator.next(); - } else { - throw new NoSuchElementException(); - } - } - } - - private static final Gson GSON = new Gson(); - - private String marketoEndpoint; - private MarketoToken token; - - public Marketo(String marketoEndpoint, String clientId, String clientSecret) { - this.marketoEndpoint = marketoEndpoint; - token = getToken(marketoEndpoint, clientId, clientSecret); - } - - public Marketo(String marketoEndpoint, MarketoToken token) { - this.marketoEndpoint = marketoEndpoint; - this.token = token; - } - - public MarketoPage getPage(String queryUrl) { - return get(queryUrl, MarketoPage.class); - } - - public MarketoPage getNextPage(MarketoPage page, String queryUrl) { - if (!Strings.isNullOrEmpty(page.getNextPageToken())) { - return getPage(queryUrl + "&nextPageToken=" + page.getNextPageToken()); - } - return null; - } - - public MarketoPageIterator iteratePage(String queryUrl) { - return new MarketoPageIterator(getPage(queryUrl), this, queryUrl); - } - - private String appendToken(String requestUrl) { - if (!requestUrl.contains("access_token")) { - if (requestUrl.contains("?")) { - return requestUrl + "&access_token=" + token.getAccessToken(); - } else { - return requestUrl + "?access_token=" + token.getAccessToken(); - } - } - return requestUrl; - } - - public T get(String queryUrl, Class cls) { - return doGet(appendToken(marketoEndpoint + queryUrl), cls); - } - - public static MarketoToken getToken(String marketoEndpoint, String clientId, String clientSecret) { - String marketoIdURL = marketoEndpoint + "/identity"; - String idEndpoint = marketoIdURL + "/oauth/token?grant_type=client_credentials&client_id=" - + clientId + "&client_secret=" + clientSecret; - - return doGet(idEndpoint, MarketoToken.class); - } - - - private static T doGet(String queryUrl, Class cls) { - try { - URL url = new URL(queryUrl); - HttpsURLConnection urlConn = (HttpsURLConnection) url.openConnection(); - urlConn.setRequestMethod("GET"); - urlConn.setRequestProperty("accept", "application/json"); - int responseCode = urlConn.getResponseCode(); - if (responseCode == 200) { - InputStream inStream = urlConn.getInputStream(); - Reader reader = new InputStreamReader(inStream); - return GSON.fromJson(reader, cls); - } else { - throw new IOException("Status: " + responseCode); - } - } catch (IOException e) { - throw new RuntimeException(e); - } - } -} diff --git a/src/main/java/io/cdap/plugin/marketo/common/MarketoEntity.java b/src/main/java/io/cdap/plugin/marketo/common/MarketoEntity.java deleted file mode 100644 index 111f239..0000000 --- a/src/main/java/io/cdap/plugin/marketo/common/MarketoEntity.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright © 2019 Cask Data, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package io.cdap.plugin.marketo.common; - -import java.util.Arrays; - -/** - * Marketo Entity types. - */ -public enum MarketoEntity { - Leads("Leads", "GET /rest/v1/leads.json", "/rest/v1/leads/describe.json"), - Campaigns("Campaigns", "/rest/v1/campaigns.json", null), - Companies("Companies", "/rest/v1/companies.json", "/rest/v1/companies/describe.json"); - - private String name; - private String getEndpoint; - private String describeEndpoint; - - MarketoEntity(String name, String getEndpoint, String describeEndpoint) { - this.name = name; - this.getEndpoint = getEndpoint; - this.describeEndpoint = describeEndpoint; - } - - public String getName() { - return name; - } - - public String getGetEndpoint() { - return getEndpoint; - } - - public String getDescribeEndpoint() { - return describeEndpoint; - } - - public static MarketoEntity fromString(String marketoEntity) { - return Arrays.stream(MarketoEntity.values()) - .filter(entity -> entity.getName().equals(marketoEntity)).findFirst() - .orElseThrow(() -> new RuntimeException(String.format("'%s' is not a valid Marketo entity", marketoEntity))); - } -} diff --git a/src/main/java/io/cdap/plugin/marketo/common/MarketoSchemaReader.java b/src/main/java/io/cdap/plugin/marketo/common/MarketoSchemaReader.java deleted file mode 100644 index 6d4996b..0000000 --- a/src/main/java/io/cdap/plugin/marketo/common/MarketoSchemaReader.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright © 2019 Cask Data, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package io.cdap.plugin.marketo.common; - -import io.cdap.cdap.api.data.schema.Schema; -import io.cdap.plugin.marketo.common.response.describe.DescribeResponse; -import io.cdap.plugin.marketo.common.response.describe.DescribeResponseLeads; -import io.cdap.plugin.marketo.common.response.describe.Field; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -/** - * Generates schema for given entity. - */ -public class MarketoSchemaReader { - // TODO fill predefined schemas - private static final Map PREDEFINED_SCHEMAS = new HashMap<>(); - - public static Schema getSchemaForEntity(MarketoEntity entity, Marketo marketo) { - if (entity.getDescribeEndpoint() != null) { - List entityFields; - - if (entity == MarketoEntity.Leads) { - DescribeResponseLeads leadsDescribe = marketo.get(entity.getDescribeEndpoint(), - DescribeResponseLeads.class); - entityFields = leadsDescribe.getFields(); - } else { - DescribeResponse describe = marketo.get(entity.getDescribeEndpoint(), DescribeResponse.class); - entityFields = describe.getFields(); - } - - List schemaFields = entityFields.stream() - //TODO map types here - .map(field -> Schema.Field.of(field.getName(), Schema.nullableOf(Schema.of(Schema.Type.STRING)))) - .collect(Collectors.toList()); - return Schema.recordOf(entity.getName(), schemaFields); - } else if (PREDEFINED_SCHEMAS.containsKey(entity)) { - return PREDEFINED_SCHEMAS.get(entity); - } - throw new RuntimeException(String.format("Unable to get schema for entity '%s'", entity.getName())); - } - -} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/HttpHelper.java b/src/main/java/io/cdap/plugin/marketo/common/api/HttpHelper.java new file mode 100644 index 0000000..ba46efd --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/HttpHelper.java @@ -0,0 +1,106 @@ +package io.cdap.plugin.marketo.common.api; + +import com.google.gson.Gson; +import com.google.gson.reflect.TypeToken; +import io.cdap.plugin.marketo.common.api.entities.BaseResponse; +import org.apache.commons.io.Charsets; +import org.apache.commons.io.IOUtils; + +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.io.Reader; +import java.net.URL; +import java.util.function.Function; +import javax.net.ssl.HttpsURLConnection; + +/** + * Helper class with http routines. + */ +class HttpHelper { + private static final Gson GSON = new Gson(); + + static String doGet(String queryUrl) { + return HttpHelper.doGet(queryUrl, inputStream -> { + try { + return IOUtils.toString(inputStream, Charsets.UTF_8); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + } + + static T doGet(String queryUrl, Class cls) { + return doGet(queryUrl, inputStream -> { + Reader reader = new InputStreamReader(inputStream); + return GSON.fromJson(reader, cls); + }); + } + + static T doGet(String queryUrl, TypeToken pageTypeToken) { + return doGet(queryUrl, inputStream -> { + Reader reader = new InputStreamReader(inputStream); + return GSON.fromJson(reader, pageTypeToken.getType()); + }); + } + + private static T doGet(String queryUrl, Function transformer) { + try { + URL url = new URL(queryUrl); + HttpsURLConnection httpConnection = (HttpsURLConnection) url.openConnection(); + httpConnection.setRequestMethod("GET"); + httpConnection.setRequestProperty("accept", "application/json"); + int responseCode = httpConnection.getResponseCode(); + if (responseCode == 200) { + InputStream inStream = httpConnection.getInputStream(); + return transformer.apply(inStream); + } else { + throw new IOException(String.format("Failed to make http request, code: %s, message: %s", + responseCode, getConnectionError(httpConnection))); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private static String getConnectionError(HttpsURLConnection connection) throws IOException { + InputStream inStream = connection.getErrorStream(); + return IOUtils.toString(inStream, Charsets.UTF_8); + } + + static T doPost(String queryUrl, R body, Class requestCls, + Class responseCls) { + try { + URL url = new URL(queryUrl); + HttpsURLConnection urlConn = (HttpsURLConnection) url.openConnection(); + urlConn.setRequestMethod("POST"); + urlConn.setRequestProperty("accept", "application/json"); + + if (body != null && requestCls != null) { + urlConn.setDoOutput(true); + byte[] requestData = GSON.toJson(body, requestCls).getBytes(); + urlConn.setFixedLengthStreamingMode(requestData.length); + urlConn.setRequestProperty("Content-Type", "application/json"); + urlConn.connect(); + try (OutputStream os = urlConn.getOutputStream()) { + os.write(requestData); + } + } else { + urlConn.connect(); + } + + int responseCode = urlConn.getResponseCode(); + + if (responseCode == 200) { + InputStream inStream = urlConn.getInputStream(); + Reader reader = new InputStreamReader(inStream); + return GSON.fromJson(reader, responseCls); + } else { + throw new IOException("Status: " + responseCode); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/LeadsExportJob.java b/src/main/java/io/cdap/plugin/marketo/common/api/LeadsExportJob.java new file mode 100644 index 0000000..e869f76 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/LeadsExportJob.java @@ -0,0 +1,63 @@ +package io.cdap.plugin.marketo.common.api; + +import io.cdap.plugin.marketo.common.api.entities.leads.LeadsExport; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Arrays; +import java.util.List; + +/** + * Leads export job. + */ +public class LeadsExportJob { + private static final Logger LOG = LoggerFactory.getLogger(LeadsExportJob.class); + + private static final List WAITABLE_STATE = Arrays.asList("Queued", "Processing"); + private static final List COMPLETED_STATUS = Arrays.asList("Canceled", "Completed", "Failed"); + private String jobId; + private LeadsExport.ExportResponse last; + private Marketo marketo; + + public LeadsExportJob(LeadsExport lastStatus, Marketo marketo) { + this.jobId = lastStatus.singleExport().getExportId(); + this.last = lastStatus.singleExport(); + this.marketo = marketo; + LOG.info("Created bulk lead export job with id '{}'", this.jobId); + } + + public String getStatus() { + return last.getStatus(); + } + + public void waitCompletion() throws InterruptedException { + if (!WAITABLE_STATE.contains(getStatus())) { + throw new IllegalStateException("Job must be enqueued before waiting for completion."); + } + + while (!COMPLETED_STATUS.contains(getStatus())) { + LeadsExport currentResp = marketo.get(String.format(Urls.BULK_EXPORT_LEADS_STATUS, jobId), LeadsExport.class); + LeadsExport.ExportResponse current = currentResp.singleExport(); + String previousStatus = getStatus(); + String currentStatus = current.getStatus(); + if (!currentStatus.equals(previousStatus)) { + LOG.info("Bulk lead export job with id '{}' changed status from '{}' to '{}'", jobId, previousStatus, + currentStatus); + } + last = current; + Thread.sleep(30 * 1000); + } + LOG.info("Bulk lead export job with id '{}' finished with status '{}'", jobId, getStatus()); + } + + public void enqueue() { + last = marketo.post(String.format(Urls.BULK_EXPORT_LEADS_ENQUEUE, jobId), + null, null, LeadsExport.class).singleExport(); + + LOG.info("Bulk lead export job with id '{}' enqueued", jobId); + } + + public String getFile() { + return marketo.get(String.format(Urls.BULK_EXPORT_LEADS_FILE, jobId)); + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/Marketo.java b/src/main/java/io/cdap/plugin/marketo/common/api/Marketo.java new file mode 100644 index 0000000..2ef7a16 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/Marketo.java @@ -0,0 +1,121 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api; + +import com.google.common.base.Strings; +import com.google.gson.reflect.TypeToken; +import io.cdap.plugin.marketo.common.api.entities.BaseResponse; +import io.cdap.plugin.marketo.common.api.entities.MarketoToken; +import io.cdap.plugin.marketo.common.api.entities.leads.LeadsDescribe; +import io.cdap.plugin.marketo.common.api.entities.leads.LeadsExport; +import io.cdap.plugin.marketo.common.api.entities.leads.LeadsExportRequest; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.List; +import java.util.Spliterator; +import java.util.Spliterators; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; + +/** + * Helper class to perform Marketo Api operations. + */ +public class Marketo { + + private static final Logger LOG = LoggerFactory.getLogger(Marketo.class); + private static final TypeToken LEADS_DESCRIBE_TYPE_TOKEN = new TypeToken() { + }; + private String marketoEndpoint; + private MarketoToken token; + + public Marketo(String marketoEndpoint, String clientId, String clientSecret) { + this.marketoEndpoint = marketoEndpoint; + token = getToken(marketoEndpoint, clientId, clientSecret); + } + + public Marketo(String marketoEndpoint, MarketoToken token) { + this.marketoEndpoint = marketoEndpoint; + this.token = token; + } + + public List describeLeads() { + return StreamSupport.stream(Spliterators.spliteratorUnknownSize( + iteratePage(Urls.LEADS_DESCRIBE, LEADS_DESCRIBE_TYPE_TOKEN, LeadsDescribe::getResult), + Spliterator.ORDERED), false).collect(Collectors.toList()); + } + + public LeadsExportJob exportLeads(LeadsExportRequest request) { + LeadsExport export = post(Urls.BULK_EXPORT_LEADS_CREATE, request, LeadsExportRequest.class, LeadsExport.class); + return new LeadsExportJob(export, this); + } + + T getPage(String queryUrl, TypeToken pageTypeToken) { + return get(queryUrl, pageTypeToken); + } + + T getNextPage(T currentPage, String queryUrl, TypeToken pageTypeToken) { + if (!Strings.isNullOrEmpty(currentPage.getNextPageToken())) { + return getPage(queryUrl + "&nextPageToken=" + currentPage.getNextPageToken(), pageTypeToken); + } + return null; + } + + MarketoPageIterator iteratePage(String queryUrl, + TypeToken pageTypeToken, + Function> resultsGetter) { + return new MarketoPageIterator<>(getPage(queryUrl, pageTypeToken), this, queryUrl, pageTypeToken, resultsGetter); + } + + T get(String queryUrl, TypeToken pageTypeToken) { + return HttpHelper.doGet(appendToken(marketoEndpoint + queryUrl), pageTypeToken); + } + + T get(String queryUrl, Class cls) { + return HttpHelper.doGet(appendToken(marketoEndpoint + queryUrl), cls); + } + + T post(String queryUrl, R body, Class requestCls, Class responseCls) { + return HttpHelper.doPost(appendToken(marketoEndpoint + queryUrl), body, requestCls, responseCls); + } + + String get(String queryUrl) { + return HttpHelper.doGet(queryUrl); + } + + public static MarketoToken getToken(String marketoEndpoint, String clientId, String clientSecret) { + LOG.info("Requesting marketo token"); + String marketoIdURL = marketoEndpoint + "/identity"; + String idEndpoint = marketoIdURL + "/oauth/token?grant_type=client_credentials&client_id=" + + clientId + "&client_secret=" + clientSecret; + + return HttpHelper.doGet(idEndpoint, MarketoToken.class); + } + + private String appendToken(String requestUrl) { + if (!requestUrl.contains("access_token")) { + if (requestUrl.contains("?")) { + return requestUrl + "&access_token=" + token.getAccessToken(); + } else { + return requestUrl + "?access_token=" + token.getAccessToken(); + } + } + return requestUrl; + } + +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/MarketoPageIterator.java b/src/main/java/io/cdap/plugin/marketo/common/api/MarketoPageIterator.java new file mode 100644 index 0000000..fc1a0a9 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/MarketoPageIterator.java @@ -0,0 +1,59 @@ +package io.cdap.plugin.marketo.common.api; + +import com.google.gson.reflect.TypeToken; +import io.cdap.plugin.marketo.common.api.entities.BaseResponse; + +import java.util.Iterator; +import java.util.List; +import java.util.NoSuchElementException; +import java.util.function.Function; + +/** + * Marketo page iterator. + * + * @param type of page response + * @param type of page item entity + */ +public class MarketoPageIterator implements Iterator { + private T currentPage; + private Marketo marketo; + private String queryUrl; + private TypeToken pageTypeToken; + private Function> resultsGetter; + private Iterator currentPageResultIterator; + + MarketoPageIterator(T page, Marketo marketo, String queryUrl, TypeToken pageTypeToken, + Function> resultsGetter) { + this.currentPage = page; + this.marketo = marketo; + this.queryUrl = queryUrl; + this.pageTypeToken = pageTypeToken; + this.resultsGetter = resultsGetter; + currentPageResultIterator = resultsGetter.apply(this.currentPage).iterator(); + } + + @Override + public boolean hasNext() { + if (currentPageResultIterator.hasNext()) { + return true; + } else { + T nextPage = marketo.getNextPage(currentPage, queryUrl, pageTypeToken); + if (nextPage != null) { + currentPage = nextPage; + currentPageResultIterator = resultsGetter.apply(this.currentPage).iterator(); + return hasNext(); + } else { + return false; + } + } + } + + @Override + public I next() { + if (hasNext()) { + return currentPageResultIterator.next(); + } else { + throw new NoSuchElementException(); + } + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/Urls.java b/src/main/java/io/cdap/plugin/marketo/common/api/Urls.java new file mode 100644 index 0000000..dc0ae5f --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/Urls.java @@ -0,0 +1,12 @@ +package io.cdap.plugin.marketo.common.api; + +/** + * Marketo API urls. + */ +public class Urls { + public static final String BULK_EXPORT_LEADS_CREATE = "/bulk/v1/leads/export/create.json"; + public static final String BULK_EXPORT_LEADS_ENQUEUE = "/bulk/v1/leads/export/%s/enqueue.json"; + public static final String BULK_EXPORT_LEADS_STATUS = "/bulk/v1/leads/export/%s/status.json"; + public static final String BULK_EXPORT_LEADS_FILE = "/bulk/v1/leads/export/%s/file.json"; + public static final String LEADS_DESCRIBE = "/rest/v1/leads/describe.json"; +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/BaseResponse.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/BaseResponse.java new file mode 100644 index 0000000..368fe0e --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/BaseResponse.java @@ -0,0 +1,41 @@ +package io.cdap.plugin.marketo.common.api.entities; + +import java.util.Collections; +import java.util.List; + +/** + * Represents common parts for all responses. + */ +public class BaseResponse { + + private boolean success = false; + private List errors = Collections.emptyList(); + private List warnings = Collections.emptyList(); + private String requestId; + private boolean moreResult = false; + private String nextPageToken; + + public boolean isSuccess() { + return success; + } + + public List getErrors() { + return errors; + } + + public List getWarnings() { + return warnings; + } + + public String getRequestId() { + return requestId; + } + + public boolean isMoreResult() { + return moreResult; + } + + public String getNextPageToken() { + return nextPageToken; + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/Error.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/Error.java new file mode 100644 index 0000000..681577d --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/Error.java @@ -0,0 +1,22 @@ +package io.cdap.plugin.marketo.common.api.entities; + +/** + * Represents error message. + */ +public class Error { + private int code; + private String message; + + public int getCode() { + return code; + } + + public String getMessage() { + return message; + } + + @Override + public String toString() { + return String.format("code: %d, message: %s", code, message); + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/MarketoToken.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/MarketoToken.java similarity index 93% rename from src/main/java/io/cdap/plugin/marketo/common/MarketoToken.java rename to src/main/java/io/cdap/plugin/marketo/common/api/entities/MarketoToken.java index 6f7c235..258d93f 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/MarketoToken.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/MarketoToken.java @@ -14,12 +14,12 @@ * the License. */ -package io.cdap.plugin.marketo.common; +package io.cdap.plugin.marketo.common.api.entities; import com.google.gson.annotations.SerializedName; /** - * Marketo Token holder. + * Represents marketo token response. */ public class MarketoToken { @SerializedName("access_token") diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/Warning.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/Warning.java new file mode 100644 index 0000000..562ffa1 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/Warning.java @@ -0,0 +1,22 @@ +package io.cdap.plugin.marketo.common.api.entities; + +/** + * Represents warning message. + */ +public class Warning { + private int code; + private String message; + + public int getCode() { + return code; + } + + public String getMessage() { + return message; + } + + @Override + public String toString() { + return String.format("code: %d, message: %s", code, message); + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/leads/LeadsDescribe.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/leads/LeadsDescribe.java new file mode 100644 index 0000000..e1c832b --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/leads/LeadsDescribe.java @@ -0,0 +1,69 @@ +package io.cdap.plugin.marketo.common.api.entities.leads; + +import io.cdap.plugin.marketo.common.api.entities.BaseResponse; + +import java.util.Collections; +import java.util.List; + +/** + * Represents leads describe response. + */ +public class LeadsDescribe extends BaseResponse { + /** + * Represents lead field description. + */ + public static class LeadAttribute { + String dataType; + String displayName; + int id; + int length; + LeadMapAttribute rest; + LeadMapAttribute soap; + + public String getDataType() { + return dataType; + } + + public String getDisplayName() { + return displayName; + } + + public int getId() { + return id; + } + + public int getLength() { + return length; + } + + public LeadMapAttribute getRest() { + return rest; + } + + public LeadMapAttribute getSoap() { + return soap; + } + } + + /** + * Represents leads field name. + */ + public static class LeadMapAttribute { + String name; + boolean readOnly = true; + + public String getName() { + return name; + } + + public boolean isReadOnly() { + return readOnly; + } + } + + List result = Collections.emptyList(); + + public List getResult() { + return result; + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/leads/LeadsExport.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/leads/LeadsExport.java new file mode 100644 index 0000000..877630b --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/leads/LeadsExport.java @@ -0,0 +1,82 @@ +package io.cdap.plugin.marketo.common.api.entities.leads; + +import io.cdap.plugin.marketo.common.api.entities.BaseResponse; + +import java.util.Collections; +import java.util.List; + +/** + * Represents leads bulk export response. + */ +public class LeadsExport extends BaseResponse { + /** + * Represents export response item. + */ + public static class ExportResponse { + String createdAt; + String errorMsg; + String exportId; + int fileSize; + String fileChecksum; + String finishedAt; + String format; + int numberOfRecords; + String queuedAt; + String startedAt; + String status; + + public String getCreatedAt() { + return createdAt; + } + + public String getErrorMsg() { + return errorMsg; + } + + public String getExportId() { + return exportId; + } + + public int getFileSize() { + return fileSize; + } + + public String getFileChecksum() { + return fileChecksum; + } + + public String getFinishedAt() { + return finishedAt; + } + + public String getFormat() { + return format; + } + + public int getNumberOfRecords() { + return numberOfRecords; + } + + public String getQueuedAt() { + return queuedAt; + } + + public String getStartedAt() { + return startedAt; + } + + public String getStatus() { + return status; + } + } + + List result = Collections.emptyList(); + + public ExportResponse singleExport() { + if (result.size() != 1) { + throw new IllegalStateException("Expected single export job result."); + } + return result.get(0); + } + +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/leads/LeadsExportRequest.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/leads/LeadsExportRequest.java new file mode 100644 index 0000000..d46f0d4 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/leads/LeadsExportRequest.java @@ -0,0 +1,107 @@ +package io.cdap.plugin.marketo.common.api.entities.leads; + +import java.util.List; +import java.util.Map; + +/** + * Represents leads bulk export request. + */ +public class LeadsExportRequest { + /** + * Represents date range. + */ + public static class DateRange { + String endAt = null; + String startAt = null; + + public DateRange(String startAt, String endAt) { + this.endAt = endAt; + this.startAt = startAt; + } + } + + /** + * Represents request filter. + */ + public static class ExportLeadFilter { + DateRange createdAt = null; + Integer smartListId = null; + String smartListName = null; + Integer staticListId = null; + Integer staticListName = null; + DateRange updatedAt = null; + + /** + * Builder for ExportLeadFilter. + */ + public static class Builder { + private DateRange createdAt = null; + private Integer smartListId = null; + private String smartListName = null; + private Integer staticListId = null; + private Integer staticListName = null; + private DateRange updatedAt = null; + + public Builder() { + } + + public Builder createdAt(DateRange createdAt) { + this.createdAt = createdAt; + return Builder.this; + } + + public Builder smartListId(Integer smartListId) { + this.smartListId = smartListId; + return Builder.this; + } + + public Builder smartListName(String smartListName) { + this.smartListName = smartListName; + return Builder.this; + } + + public Builder staticListId(Integer staticListId) { + this.staticListId = staticListId; + return Builder.this; + } + + public Builder staticListName(Integer staticListName) { + this.staticListName = staticListName; + return Builder.this; + } + + public Builder updatedAt(DateRange updatedAt) { + this.updatedAt = updatedAt; + return Builder.this; + } + + public ExportLeadFilter build() { + + return new ExportLeadFilter(this); + } + } + + private ExportLeadFilter(Builder builder) { + this.createdAt = builder.createdAt; + this.smartListId = builder.smartListId; + this.smartListName = builder.smartListName; + this.staticListId = builder.staticListId; + this.staticListName = builder.staticListName; + this.updatedAt = builder.updatedAt; + } + + public static Builder builder() { + return new Builder(); + } + } + + Map columnHeaderNames = null; + List fields = null; + ExportLeadFilter filter = null; + String format = "CSV"; + + public LeadsExportRequest(List fields, ExportLeadFilter filter) { + this.fields = fields; + this.filter = filter; + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/response/MarketoPage.java b/src/main/java/io/cdap/plugin/marketo/common/response/MarketoPage.java deleted file mode 100644 index 5311e7f..0000000 --- a/src/main/java/io/cdap/plugin/marketo/common/response/MarketoPage.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Copyright © 2019 Cask Data, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package io.cdap.plugin.marketo.common.response; - -import java.util.Collections; -import java.util.List; -import java.util.Map; - -/** - * Marketo page response. - */ -public class MarketoPage { - /** - * Error or warn message description. - */ - public static class Description { - private String code; - private String message; - - public String getCode() { - return code; - } - - public String getMessage() { - return message; - } - } - - private Boolean moreResult = null; - private String nextPageToken = null; - private String requestId = null; - private Boolean success = null; - - private List errors = Collections.emptyList(); - private List warnings = Collections.emptyList(); - - private List> result = Collections.emptyList(); - - public Boolean hasMoreResults() { - return moreResult; - } - - public String getNextPageToken() { - return nextPageToken; - } - - public String getRequestId() { - return requestId; - } - - public Boolean isSuccess() { - return success; - } - - public List getErrors() { - return errors; - } - - public List getWarnings() { - return warnings; - } - - public List> getResults() { - return result; - } -} diff --git a/src/main/java/io/cdap/plugin/marketo/common/response/describe/DescribeResponse.java b/src/main/java/io/cdap/plugin/marketo/common/response/describe/DescribeResponse.java deleted file mode 100644 index 23cc88f..0000000 --- a/src/main/java/io/cdap/plugin/marketo/common/response/describe/DescribeResponse.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright © 2019 Cask Data, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package io.cdap.plugin.marketo.common.response.describe; - -import java.util.Collections; -import java.util.List; - -/** - * Regular describe response. - * Fields located in first object in 'result' field. - */ -public class DescribeResponse { - /** - * Result holder. - */ - public static class Result { - List fields = Collections.emptyList(); - } - - private String requestId = null; - private Boolean success = null; - - private List result = Collections.emptyList(); - - public List getFields() { - if (result.size() != 1) { - throw new RuntimeException("Expected to have one result."); - } - return result.get(0).fields; - } -} diff --git a/src/main/java/io/cdap/plugin/marketo/common/response/describe/DescribeResponseLeads.java b/src/main/java/io/cdap/plugin/marketo/common/response/describe/DescribeResponseLeads.java deleted file mode 100644 index e5eefea..0000000 --- a/src/main/java/io/cdap/plugin/marketo/common/response/describe/DescribeResponseLeads.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright © 2019 Cask Data, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package io.cdap.plugin.marketo.common.response.describe; - -import java.util.Collections; -import java.util.List; - -/** - * Describe response for Leads-Leads entity. - * Fields here is directly located in 'result' field of response. - */ -public class DescribeResponseLeads { - private String requestId = null; - private Boolean success = null; - - List result = Collections.emptyList(); - - public List getFields() { - return result; - } -} diff --git a/src/main/java/io/cdap/plugin/marketo/common/response/describe/Field.java b/src/main/java/io/cdap/plugin/marketo/common/response/describe/Field.java deleted file mode 100644 index 7086298..0000000 --- a/src/main/java/io/cdap/plugin/marketo/common/response/describe/Field.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright © 2019 Cask Data, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package io.cdap.plugin.marketo.common.response.describe; - -/** - * Marketo field description. - */ -public class Field { - /** - * Soap or rest descriptor. - */ - public static class FieldDescription { - String name; - Boolean readOnly; - } - - String id = null; - private String name = null; - String displayName = null; - String dataType = null; - int length = -1; - Boolean updateable = null; - FieldDescription rest = null; - FieldDescription soap = null; - - public String getName() { - if (name == null) { - if (rest == null) { - throw new RuntimeException("Failed to get name for field."); - } else { - return rest.name; - } - } else { - return name; - } - } -} diff --git a/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoInputFormatProvider.java b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoInputFormatProvider.java index b1bebb7..8a70ed1 100644 --- a/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoInputFormatProvider.java +++ b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoInputFormatProvider.java @@ -28,12 +28,12 @@ * InputFormatProvider used by cdap to provide configurations to mapreduce job */ public class MarketoInputFormatProvider implements InputFormatProvider { - public static final String PROPERTY_CONFIG_JSON = "cdap.marketo.entity.config"; + public static final String PROPERTY_CONFIG_JSON = "cdap.marketo.reporter.config"; private static final Gson gson = new GsonBuilder().create(); private final Map conf; - MarketoInputFormatProvider(MarketoBatchSourceConfig config) { + MarketoInputFormatProvider(MarketoReportingSourceConfig config) { this.conf = Collections.unmodifiableMap(new HashMap() {{ put(PROPERTY_CONFIG_JSON, gson.toJson(config)); }}); diff --git a/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoRecordReader.java b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoRecordReader.java index 9d36451..a3b0e70 100644 --- a/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoRecordReader.java +++ b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoRecordReader.java @@ -18,37 +18,71 @@ import com.google.gson.Gson; import com.google.gson.GsonBuilder; -import io.cdap.plugin.marketo.common.Marketo; +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.plugin.marketo.common.api.LeadsExportJob; +import io.cdap.plugin.marketo.common.api.Marketo; +import io.cdap.plugin.marketo.common.api.entities.leads.LeadsExportRequest; +import org.apache.commons.csv.CSVFormat; +import org.apache.commons.csv.CSVParser; +import org.apache.commons.csv.CSVRecord; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.mapreduce.InputSplit; import org.apache.hadoop.mapreduce.RecordReader; import org.apache.hadoop.mapreduce.TaskAttemptContext; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.IOException; +import java.io.StringReader; +import java.util.Iterator; +import java.util.List; import java.util.Map; +import java.util.stream.Collectors; /** * RecordReader implementation, which reads events from Marketo api. */ -public class MarketoRecordReader extends RecordReader> { +public class MarketoRecordReader extends RecordReader> { + private static final Logger LOG = LoggerFactory.getLogger(MarketoRecordReader.class); private static final Gson GSON = new GsonBuilder().create(); - private Marketo.MarketoPageIterator iterator; - private Map current = null; + private Map current = null; + private Iterator iterator = null; @Override public void initialize(InputSplit inputSplit, TaskAttemptContext taskAttemptContext) throws IOException { Configuration conf = taskAttemptContext.getConfiguration(); String configJson = conf.get(MarketoInputFormatProvider.PROPERTY_CONFIG_JSON); - MarketoBatchSourceConfig config = GSON.fromJson(configJson, MarketoBatchSourceConfig.class); + MarketoReportingSourceConfig config = GSON.fromJson(configJson, MarketoReportingSourceConfig.class); - iterator = config.getMarketo().iteratePage(config.getEntityType().getGetEndpoint()); + Marketo marketo = config.getMarketo(); + + List fields = config.getSchema().getFields().stream() + .map(Schema.Field::getName).collect(Collectors.toList()); + LeadsExportRequest.ExportLeadFilter filter = LeadsExportRequest.ExportLeadFilter.builder() + .createdAt(new LeadsExportRequest.DateRange(config.getStartDate(), config.getEndDate())).build(); + LeadsExportRequest request = new LeadsExportRequest(fields, filter); + + LeadsExportJob job = marketo.exportLeads(request); + job.enqueue(); + try { + job.waitCompletion(); + } catch (InterruptedException ex) { + throw new RuntimeException(ex); + } + + String data = job.getFile(); + LOG.info(data); + CSVParser parser = CSVFormat.DEFAULT.withHeader().parse(new StringReader(data)); + iterator = parser.iterator(); +// iterator = config.getMarketo().iteratePage(config.getEntityType().getGetEndpoint()); } @Override public boolean nextKeyValue() { if (iterator.hasNext()) { - current = iterator.next(); + current = iterator.next().toMap(); + LOG.debug("Got record '{}'", current.toString()); return true; } return false; @@ -60,7 +94,7 @@ public NullWritable getCurrentKey() { } @Override - public Map getCurrentValue() { + public Map getCurrentValue() { return current; } diff --git a/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoBatchSource.java b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoReportingPlugin.java similarity index 83% rename from src/main/java/io/cdap/plugin/marketo/source/batch/MarketoBatchSource.java rename to src/main/java/io/cdap/plugin/marketo/source/batch/MarketoReportingPlugin.java index ad6545e..e7949e7 100644 --- a/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoBatchSource.java +++ b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoReportingPlugin.java @@ -39,14 +39,14 @@ * Plugin that reads entities from Marketo api. */ @Plugin(type = BatchSource.PLUGIN_TYPE) -@Name(MarketoBatchSource.NAME) +@Name(MarketoReportingPlugin.NAME) @Description("Reads entities from Marketo.") -public class MarketoBatchSource extends BatchSource, StructuredRecord> { - public static final String NAME = "MarketoEntityPlugin"; +public class MarketoReportingPlugin extends BatchSource, StructuredRecord> { + public static final String NAME = "MarketoReportingPlugin"; - private final MarketoBatchSourceConfig config; + private final MarketoReportingSourceConfig config; - public MarketoBatchSource(MarketoBatchSourceConfig config) { + public MarketoReportingPlugin(MarketoReportingSourceConfig config) { this.config = config; } @@ -71,17 +71,17 @@ public void prepareRun(BatchSourceContext batchSourceContext) { } @Override - public void transform(KeyValue> input, Emitter emitter) { + public void transform(KeyValue> input, Emitter emitter) { StructuredRecord.Builder builder = StructuredRecord.builder(config.getSchema()); - Map inputMap = input.getValue(); + Map inputMap = input.getValue(); config.getSchema().getFields().forEach( field -> { if (inputMap.containsKey(field.getName())) { - //TODO map fields, also refactor io.cdap.plugin.marketo.common.MarketoSchemaReader.getSchemaForEntity builder.set(field.getName(), String.valueOf(inputMap.remove(field.getName()))); } } ); + emitter.emit(builder.build()); } private void validateConfiguration(FailureCollector failureCollector) { diff --git a/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoBatchSourceConfig.java b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoReportingSourceConfig.java similarity index 82% rename from src/main/java/io/cdap/plugin/marketo/source/batch/MarketoBatchSourceConfig.java rename to src/main/java/io/cdap/plugin/marketo/source/batch/MarketoReportingSourceConfig.java index f7f16f2..f139ddf 100644 --- a/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoBatchSourceConfig.java +++ b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoReportingSourceConfig.java @@ -21,15 +21,17 @@ import io.cdap.cdap.api.annotation.Name; import io.cdap.cdap.api.data.schema.Schema; import io.cdap.plugin.common.ReferencePluginConfig; -import io.cdap.plugin.marketo.common.Marketo; -import io.cdap.plugin.marketo.common.MarketoEntity; -import io.cdap.plugin.marketo.common.MarketoSchemaReader; -import io.cdap.plugin.marketo.common.MarketoToken; +import io.cdap.plugin.marketo.common.api.Marketo; +import io.cdap.plugin.marketo.common.api.entities.MarketoToken; + +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; /** * Provides all required configuration for reading Marketo entities. */ -public class MarketoBatchSourceConfig extends ReferencePluginConfig { +public class MarketoReportingSourceConfig extends ReferencePluginConfig { public static final String PROPERTY_ENTITY_NAME = "entityName"; public static final String PROPERTY_CLIENT_ID = "clientId"; public static final String PROPERTY_CLIENT_SECRET = "clientSecret"; @@ -90,13 +92,23 @@ public class MarketoBatchSourceConfig extends ReferencePluginConfig { private transient Schema schema = null; private transient Marketo marketo = null; - public MarketoBatchSourceConfig(String referenceName) { + public MarketoReportingSourceConfig(String referenceName) { super(referenceName); } public Schema getSchema() { if (schema == null) { - schema = MarketoSchemaReader.getSchemaForEntity(getEntityType(), getMarketo()); + List fields = getMarketo().describeLeads().stream().map( + leadAttribute -> { + if (leadAttribute.getRest() != null) { + return Schema.Field.of(leadAttribute.getRest().getName(), Schema.nullableOf(Schema.of(Schema.Type.STRING))); + } else { + return null; + } + } + ).filter(Objects::nonNull).collect(Collectors.toList()); + + schema = Schema.recordOf("LeadsRecord", fields); } return schema; } @@ -108,9 +120,6 @@ public Marketo getMarketo() { return marketo; } - public MarketoEntity getEntityType() { - return MarketoEntity.fromString(entityName); - } public String getClientId() { return clientId; diff --git a/widgets/MarketoReportingPlugin-batchsource.json b/widgets/MarketoReportingPlugin-batchsource.json new file mode 100644 index 0000000..8e4ee54 --- /dev/null +++ b/widgets/MarketoReportingPlugin-batchsource.json @@ -0,0 +1,28 @@ +{ + "metadata": { + "spec-version": "1.0" + }, + "display-name": "Marketo Reporting", + "configuration-groups": [ + { + "label": "General", + "properties": [ + { + "widget-type": "textbox", + "label": "Reference Name", + "name": "referenceName" + } + ] + }, + { + "label": "Advanced", + "properties": [ ] + } + ], + "outputs": [ + { + "widget-type": "non-editable-schema-editor", + "schema": { } + } + ] +} \ No newline at end of file From fd5b4048e1aa49832874d4acae540b383cea704b Mon Sep 17 00:00:00 2001 From: Yevhenii Chekanskyi Date: Mon, 25 Nov 2019 21:13:38 +0200 Subject: [PATCH 03/16] PLUGIN-75 Marketo Plugin - refactoring. --- pom.xml | 18 ++ .../plugin/marketo/common/api/Helpers.java | 51 +++++ .../plugin/marketo/common/api/HttpHelper.java | 106 ---------- .../marketo/common/api/LeadsExportJob.java | 14 +- .../plugin/marketo/common/api/Marketo.java | 192 ++++++++++++++---- .../common/api/MarketoPageIterator.java | 9 +- .../common/api/entities/BaseResponse.java | 28 ++- .../marketo/common/api/entities/Error.java | 8 + .../marketo/common/api/entities/Warning.java | 8 + .../batch/MarketoReportingSourceConfig.java | 9 +- .../marketo/common/api/MarketoTest.java | 187 +++++++++++++++++ 11 files changed, 463 insertions(+), 167 deletions(-) create mode 100644 src/main/java/io/cdap/plugin/marketo/common/api/Helpers.java delete mode 100644 src/main/java/io/cdap/plugin/marketo/common/api/HttpHelper.java create mode 100644 src/test/java/io/cdap/plugin/marketo/common/api/MarketoTest.java diff --git a/pom.xml b/pom.xml index fb9a529..d237f88 100644 --- a/pom.xml +++ b/pom.xml @@ -22,6 +22,7 @@ 6.1.0-SNAPSHOT 2.3.0 + 4.5.9 2.3.0-SNAPSHOT 2.1.3 @@ -238,6 +239,11 @@ + + org.apache.httpcomponents + httpclient + ${httpcomponents.version} + com.google.code.gson gson @@ -254,12 +260,24 @@ 2.6 + + org.slf4j + slf4j-simple + 1.7.29 + + junit junit 4.12 test + + com.github.tomakehurst + wiremock-jre8-standalone + 2.25.1 + test + diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/Helpers.java b/src/main/java/io/cdap/plugin/marketo/common/api/Helpers.java new file mode 100644 index 0000000..78c77ad --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/Helpers.java @@ -0,0 +1,51 @@ +package io.cdap.plugin.marketo.common.api; + +import com.google.common.base.Strings; +import org.apache.commons.io.IOUtils; +import org.apache.http.NameValuePair; +import org.apache.http.client.utils.URIBuilder; + +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.charset.StandardCharsets; +import java.util.List; + +public class Helpers { + public static String streamToString(InputStream inputStream) { + try { + return IOUtils.toString(inputStream, StandardCharsets.UTF_8); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + public static T streamToObject(InputStream inputStream, Class cls) { + return Marketo.GSON.fromJson(new InputStreamReader(inputStream), cls); + } + + public static RuntimeException failForUri(String method, URI uri, Exception ex) { + String message = ex.getMessage(); + if (Strings.isNullOrEmpty(message)) { + if (ex.getCause() != null) { + message = ex.getCause().getMessage(); + if (Strings.isNullOrEmpty(message)) { + message = "failed to make request"; + } + } + } + + URIBuilder uriBuilder = new URIBuilder(uri); + List queryParameters = uriBuilder.getQueryParams(); + queryParameters.removeIf(queryParameter -> queryParameter.getName().equals("access_token")); + uriBuilder.setParameters(queryParameters); + try { + String uriString = uriBuilder.build().toString(); + return new RuntimeException(String.format("Failed %s %s - %s", method, uriString, message)); + } catch (URISyntaxException e) { + return new RuntimeException(message); + } + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/HttpHelper.java b/src/main/java/io/cdap/plugin/marketo/common/api/HttpHelper.java deleted file mode 100644 index ba46efd..0000000 --- a/src/main/java/io/cdap/plugin/marketo/common/api/HttpHelper.java +++ /dev/null @@ -1,106 +0,0 @@ -package io.cdap.plugin.marketo.common.api; - -import com.google.gson.Gson; -import com.google.gson.reflect.TypeToken; -import io.cdap.plugin.marketo.common.api.entities.BaseResponse; -import org.apache.commons.io.Charsets; -import org.apache.commons.io.IOUtils; - -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.OutputStream; -import java.io.Reader; -import java.net.URL; -import java.util.function.Function; -import javax.net.ssl.HttpsURLConnection; - -/** - * Helper class with http routines. - */ -class HttpHelper { - private static final Gson GSON = new Gson(); - - static String doGet(String queryUrl) { - return HttpHelper.doGet(queryUrl, inputStream -> { - try { - return IOUtils.toString(inputStream, Charsets.UTF_8); - } catch (IOException e) { - throw new RuntimeException(e); - } - }); - } - - static T doGet(String queryUrl, Class cls) { - return doGet(queryUrl, inputStream -> { - Reader reader = new InputStreamReader(inputStream); - return GSON.fromJson(reader, cls); - }); - } - - static T doGet(String queryUrl, TypeToken pageTypeToken) { - return doGet(queryUrl, inputStream -> { - Reader reader = new InputStreamReader(inputStream); - return GSON.fromJson(reader, pageTypeToken.getType()); - }); - } - - private static T doGet(String queryUrl, Function transformer) { - try { - URL url = new URL(queryUrl); - HttpsURLConnection httpConnection = (HttpsURLConnection) url.openConnection(); - httpConnection.setRequestMethod("GET"); - httpConnection.setRequestProperty("accept", "application/json"); - int responseCode = httpConnection.getResponseCode(); - if (responseCode == 200) { - InputStream inStream = httpConnection.getInputStream(); - return transformer.apply(inStream); - } else { - throw new IOException(String.format("Failed to make http request, code: %s, message: %s", - responseCode, getConnectionError(httpConnection))); - } - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - private static String getConnectionError(HttpsURLConnection connection) throws IOException { - InputStream inStream = connection.getErrorStream(); - return IOUtils.toString(inStream, Charsets.UTF_8); - } - - static T doPost(String queryUrl, R body, Class requestCls, - Class responseCls) { - try { - URL url = new URL(queryUrl); - HttpsURLConnection urlConn = (HttpsURLConnection) url.openConnection(); - urlConn.setRequestMethod("POST"); - urlConn.setRequestProperty("accept", "application/json"); - - if (body != null && requestCls != null) { - urlConn.setDoOutput(true); - byte[] requestData = GSON.toJson(body, requestCls).getBytes(); - urlConn.setFixedLengthStreamingMode(requestData.length); - urlConn.setRequestProperty("Content-Type", "application/json"); - urlConn.connect(); - try (OutputStream os = urlConn.getOutputStream()) { - os.write(requestData); - } - } else { - urlConn.connect(); - } - - int responseCode = urlConn.getResponseCode(); - - if (responseCode == 200) { - InputStream inStream = urlConn.getInputStream(); - Reader reader = new InputStreamReader(inStream); - return GSON.fromJson(reader, responseCls); - } else { - throw new IOException("Status: " + responseCode); - } - } catch (IOException e) { - throw new RuntimeException(e); - } - } -} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/LeadsExportJob.java b/src/main/java/io/cdap/plugin/marketo/common/api/LeadsExportJob.java index e869f76..2feaf0f 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/LeadsExportJob.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/LeadsExportJob.java @@ -5,6 +5,7 @@ import org.slf4j.LoggerFactory; import java.util.Arrays; +import java.util.Collections; import java.util.List; /** @@ -12,9 +13,9 @@ */ public class LeadsExportJob { private static final Logger LOG = LoggerFactory.getLogger(LeadsExportJob.class); - private static final List WAITABLE_STATE = Arrays.asList("Queued", "Processing"); private static final List COMPLETED_STATUS = Arrays.asList("Canceled", "Completed", "Failed"); + private String jobId; private LeadsExport.ExportResponse last; private Marketo marketo; @@ -36,13 +37,15 @@ public void waitCompletion() throws InterruptedException { } while (!COMPLETED_STATUS.contains(getStatus())) { - LeadsExport currentResp = marketo.get(String.format(Urls.BULK_EXPORT_LEADS_STATUS, jobId), LeadsExport.class); + LeadsExport currentResp = marketo.validatedGet( + String.format(Urls.BULK_EXPORT_LEADS_STATUS, jobId), + Collections.emptyMap(), inputStream -> Helpers.streamToObject(inputStream, LeadsExport.class)); LeadsExport.ExportResponse current = currentResp.singleExport(); String previousStatus = getStatus(); String currentStatus = current.getStatus(); if (!currentStatus.equals(previousStatus)) { LOG.info("Bulk lead export job with id '{}' changed status from '{}' to '{}'", jobId, previousStatus, - currentStatus); + currentStatus); } last = current; Thread.sleep(30 * 1000); @@ -52,12 +55,13 @@ public void waitCompletion() throws InterruptedException { public void enqueue() { last = marketo.post(String.format(Urls.BULK_EXPORT_LEADS_ENQUEUE, jobId), - null, null, LeadsExport.class).singleExport(); + null, LeadsExport.class).singleExport(); LOG.info("Bulk lead export job with id '{}' enqueued", jobId); } public String getFile() { - return marketo.get(String.format(Urls.BULK_EXPORT_LEADS_FILE, jobId)); + return marketo.get(marketo.buildUri(String.format(Urls.BULK_EXPORT_LEADS_FILE, jobId), Collections.emptyMap()), + Helpers::streamToString); } } diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/Marketo.java b/src/main/java/io/cdap/plugin/marketo/common/api/Marketo.java index 2ef7a16..49e569b 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/Marketo.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/Marketo.java @@ -17,19 +17,41 @@ package io.cdap.plugin.marketo.common.api; import com.google.common.base.Strings; -import com.google.gson.reflect.TypeToken; +import com.google.common.collect.ImmutableMap; +import com.google.gson.Gson; import io.cdap.plugin.marketo.common.api.entities.BaseResponse; +import io.cdap.plugin.marketo.common.api.entities.Error; import io.cdap.plugin.marketo.common.api.entities.MarketoToken; import io.cdap.plugin.marketo.common.api.entities.leads.LeadsDescribe; import io.cdap.plugin.marketo.common.api.entities.leads.LeadsExport; import io.cdap.plugin.marketo.common.api.entities.leads.LeadsExportRequest; +import org.apache.commons.io.IOUtils; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.client.protocol.HttpClientContext; +import org.apache.http.client.utils.URIBuilder; +import org.apache.http.entity.ContentType; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.charset.StandardCharsets; +import java.util.Collections; import java.util.List; +import java.util.Map; +import java.util.Objects; import java.util.Spliterator; import java.util.Spliterators; import java.util.function.Function; +import java.util.function.Supplier; import java.util.stream.Collectors; import java.util.stream.StreamSupport; @@ -37,85 +59,173 @@ * Helper class to perform Marketo Api operations. */ public class Marketo { - private static final Logger LOG = LoggerFactory.getLogger(Marketo.class); - private static final TypeToken LEADS_DESCRIBE_TYPE_TOKEN = new TypeToken() { - }; + static final Gson GSON = new Gson(); private String marketoEndpoint; + private String clientId; + private String clientSecret; private MarketoToken token; + private HttpClientContext httpClientContext = HttpClientContext.create(); public Marketo(String marketoEndpoint, String clientId, String clientSecret) { this.marketoEndpoint = marketoEndpoint; - token = getToken(marketoEndpoint, clientId, clientSecret); - } - - public Marketo(String marketoEndpoint, MarketoToken token) { - this.marketoEndpoint = marketoEndpoint; - this.token = token; + this.clientId = clientId; + this.clientSecret = clientSecret; + token = refreshToken(); } public List describeLeads() { return StreamSupport.stream(Spliterators.spliteratorUnknownSize( - iteratePage(Urls.LEADS_DESCRIBE, LEADS_DESCRIBE_TYPE_TOKEN, LeadsDescribe::getResult), + iteratePage(Urls.LEADS_DESCRIBE, LeadsDescribe.class, LeadsDescribe::getResult), Spliterator.ORDERED), false).collect(Collectors.toList()); } public LeadsExportJob exportLeads(LeadsExportRequest request) { - LeadsExport export = post(Urls.BULK_EXPORT_LEADS_CREATE, request, LeadsExportRequest.class, LeadsExport.class); + LeadsExport export = validatedPost(Urls.BULK_EXPORT_LEADS_CREATE, Collections.emptyMap(), + inputStream -> Helpers.streamToObject(inputStream, LeadsExport.class), + request, + GSON::toJson); return new LeadsExportJob(export, this); } - T getPage(String queryUrl, TypeToken pageTypeToken) { - return get(queryUrl, pageTypeToken); + private T getPage(String queryUrl, Class pageClass) { + return validatedGet(queryUrl, Collections.emptyMap(), + inputStream -> Helpers.streamToObject(inputStream, pageClass)); } - T getNextPage(T currentPage, String queryUrl, TypeToken pageTypeToken) { + T getNextPage(T currentPage, String queryUrl, Class pageClass) { if (!Strings.isNullOrEmpty(currentPage.getNextPageToken())) { - return getPage(queryUrl + "&nextPageToken=" + currentPage.getNextPageToken(), pageTypeToken); + return validatedGet(queryUrl, + ImmutableMap.of("nextPageToken", currentPage.getNextPageToken()), + inputStream -> Helpers.streamToObject(inputStream, pageClass)); } return null; } - MarketoPageIterator iteratePage(String queryUrl, - TypeToken pageTypeToken, - Function> resultsGetter) { - return new MarketoPageIterator<>(getPage(queryUrl, pageTypeToken), this, queryUrl, pageTypeToken, resultsGetter); + private MarketoPageIterator iteratePage(String queryUrl, + Class pageClass, + Function> resultsGetter) { + return new MarketoPageIterator<>(getPage(queryUrl, pageClass), this, queryUrl, pageClass, resultsGetter); } - T get(String queryUrl, TypeToken pageTypeToken) { - return HttpHelper.doGet(appendToken(marketoEndpoint + queryUrl), pageTypeToken); + T post(String queryUrl, R body, Class responseCls) { + return validatedPost(queryUrl, Collections.emptyMap(), + inputStream -> Helpers.streamToObject(inputStream, responseCls), + body, GSON::toJson); } - T get(String queryUrl, Class cls) { - return HttpHelper.doGet(appendToken(marketoEndpoint + queryUrl), cls); + T validatedGet(String queryUrl, Map parameters, + Function deserializer) { + String logUri = "GET " + buildUri(queryUrl, parameters, false).toString(); + return retryableValidate(logUri, () -> { + URI queryUri = buildUri(queryUrl, parameters, true); + return get(queryUri, deserializer); + }); } - T post(String queryUrl, R body, Class requestCls, Class responseCls) { - return HttpHelper.doPost(appendToken(marketoEndpoint + queryUrl), body, requestCls, responseCls); + private T validatedPost(String queryUrl, Map parameters, + Function deserializer, + B body, Function qSerializer) { + String logUri = "POST " + buildUri(queryUrl, parameters, false).toString(); + return retryableValidate(logUri, () -> { + URI queryUri = buildUri(queryUrl, parameters, true); + return post(queryUri, deserializer, body, qSerializer); + }); } - String get(String queryUrl) { - return HttpHelper.doGet(queryUrl); + private T retryableValidate(String logUri, Supplier tryQuery) { + T result = tryQuery.get(); + // check for expired token + if (!result.isSuccess()) { + for (Error error : result.getErrors()) { + if (error.getCode() == 602 && error.getMessage().equals("Access token expired")) { + // refresh token and retry + token = refreshToken(); + LOG.info("Refreshed token"); + return tryQuery.get(); + } + } + } + + // log warnings if required + if (result.getWarnings().size() > 0) { + String warnings = result.getWarnings().stream() + .map(error -> String.format("code: %s, message: %s", error.getCode(), error.getMessage())) + .collect(Collectors.joining("; ")); + LOG.warn("Warnings when calling '{}' - {}", logUri, warnings); + } + + if (!result.isSuccess()) { + String msg = String.format("Errors when calling '%s'", logUri); + // log errors if required + if (result.getErrors().size() > 0) { + String errors = result.getErrors().stream() + .map(error -> String.format("code: %s, message: %s", error.getCode(), error.getMessage())) + .collect(Collectors.joining("; ")); + msg = msg + " - " + errors; + LOG.error(msg); + } + throw new RuntimeException(msg); + } + return result; } - public static MarketoToken getToken(String marketoEndpoint, String clientId, String clientSecret) { - LOG.info("Requesting marketo token"); - String marketoIdURL = marketoEndpoint + "/identity"; - String idEndpoint = marketoIdURL + "/oauth/token?grant_type=client_credentials&client_id=" - + clientId + "&client_secret=" + clientSecret; + T get(URI uri, Function deserializer) { + try (CloseableHttpClient httpClient = HttpClients.createDefault()) { + HttpGet request = new HttpGet(uri); + try (CloseableHttpResponse response = httpClient.execute(request, httpClientContext)) { + if(response.getStatusLine().getStatusCode() >= 300) { + throw new IOException(IOUtils.toString(response.getEntity().getContent(), StandardCharsets.UTF_8)); + } + return deserializer.apply(response.getEntity().getContent()); + } + } catch (Exception e) { + throw Helpers.failForUri("GET", uri, e); + } + } - return HttpHelper.doGet(idEndpoint, MarketoToken.class); + private T post(URI uri, Function respDeserializer, B body, Function qSerializer) { + try (CloseableHttpClient httpClient = HttpClients.createDefault()) { + HttpPost request = new HttpPost(uri); + if (body != null) { + Objects.requireNonNull(qSerializer, "body serializer must be specified with body"); + request.setEntity(new StringEntity(qSerializer.apply(body), ContentType.APPLICATION_JSON)); + } + try (CloseableHttpResponse response = httpClient.execute(request, httpClientContext)) { + return respDeserializer.apply(response.getEntity().getContent()); + } + } catch (Exception e) { + throw Helpers.failForUri("POST", uri, e); + } } - private String appendToken(String requestUrl) { - if (!requestUrl.contains("access_token")) { - if (requestUrl.contains("?")) { - return requestUrl + "&access_token=" + token.getAccessToken(); - } else { - return requestUrl + "?access_token=" + token.getAccessToken(); + + URI buildUri(String queryUrl, Map parameters) { + return buildUri(queryUrl, parameters, true); + } + + URI buildUri(String queryUrl, Map parameters, boolean includeToken) { + try { + URIBuilder builder = new URIBuilder(marketoEndpoint + queryUrl); + parameters.forEach(builder::setParameter); + if (includeToken) { + builder.setParameter("access_token", token.getAccessToken()); } + return builder.build(); + } catch (URISyntaxException e) { + throw new IllegalArgumentException(String.format("'%s' is invalid URI", marketoEndpoint + queryUrl)); } - return requestUrl; } + MarketoToken getCurrentToken() { + return this.token; + } + + private MarketoToken refreshToken() { + LOG.debug("Requesting marketo token"); + URI getTokenUri = buildUri("/identity/oauth/token", + ImmutableMap.of("grant_type", "client_credentials", "client_id", clientId, + "client_secret", clientSecret), false); + return get(getTokenUri, inputStream -> GSON.fromJson(new InputStreamReader(inputStream), MarketoToken.class)); + } } diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/MarketoPageIterator.java b/src/main/java/io/cdap/plugin/marketo/common/api/MarketoPageIterator.java index fc1a0a9..3a7a424 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/MarketoPageIterator.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/MarketoPageIterator.java @@ -1,6 +1,5 @@ package io.cdap.plugin.marketo.common.api; -import com.google.gson.reflect.TypeToken; import io.cdap.plugin.marketo.common.api.entities.BaseResponse; import java.util.Iterator; @@ -18,16 +17,16 @@ public class MarketoPageIterator implements Iterator< private T currentPage; private Marketo marketo; private String queryUrl; - private TypeToken pageTypeToken; + private Class pageClass; private Function> resultsGetter; private Iterator currentPageResultIterator; - MarketoPageIterator(T page, Marketo marketo, String queryUrl, TypeToken pageTypeToken, + MarketoPageIterator(T page, Marketo marketo, String queryUrl, Class pageClass, Function> resultsGetter) { this.currentPage = page; this.marketo = marketo; this.queryUrl = queryUrl; - this.pageTypeToken = pageTypeToken; + this.pageClass = pageClass; this.resultsGetter = resultsGetter; currentPageResultIterator = resultsGetter.apply(this.currentPage).iterator(); } @@ -37,7 +36,7 @@ public boolean hasNext() { if (currentPageResultIterator.hasNext()) { return true; } else { - T nextPage = marketo.getNextPage(currentPage, queryUrl, pageTypeToken); + T nextPage = marketo.getNextPage(currentPage, queryUrl, pageClass); if (nextPage != null) { currentPage = nextPage; currentPageResultIterator = resultsGetter.apply(this.currentPage).iterator(); diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/BaseResponse.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/BaseResponse.java index 368fe0e..dc3deb0 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/entities/BaseResponse.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/BaseResponse.java @@ -10,7 +10,7 @@ public class BaseResponse { private boolean success = false; private List errors = Collections.emptyList(); - private List warnings = Collections.emptyList(); + private List warnings = Collections.emptyList(); private String requestId; private boolean moreResult = false; private String nextPageToken; @@ -19,23 +19,47 @@ public boolean isSuccess() { return success; } + public void setSuccess(boolean success) { + this.success = success; + } + public List getErrors() { return errors; } - public List getWarnings() { + public void setErrors(List errors) { + this.errors = errors; + } + + public List getWarnings() { return warnings; } + public void setWarnings(List warnings) { + this.warnings = warnings; + } + public String getRequestId() { return requestId; } + public void setRequestId(String requestId) { + this.requestId = requestId; + } + public boolean isMoreResult() { return moreResult; } + public void setMoreResult(boolean moreResult) { + this.moreResult = moreResult; + } + public String getNextPageToken() { return nextPageToken; } + + public void setNextPageToken(String nextPageToken) { + this.nextPageToken = nextPageToken; + } } diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/Error.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/Error.java index 681577d..2c9a9e2 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/entities/Error.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/Error.java @@ -7,6 +7,14 @@ public class Error { private int code; private String message; + public Error(int code, String message) { + this.code = code; + this.message = message; + } + + public Error() { + } + public int getCode() { return code; } diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/Warning.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/Warning.java index 562ffa1..5722e51 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/entities/Warning.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/Warning.java @@ -7,6 +7,14 @@ public class Warning { private int code; private String message; + public Warning(int code, String message) { + this.code = code; + this.message = message; + } + + public Warning() { + } + public int getCode() { return code; } diff --git a/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoReportingSourceConfig.java b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoReportingSourceConfig.java index f139ddf..c9cc414 100644 --- a/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoReportingSourceConfig.java +++ b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoReportingSourceConfig.java @@ -115,7 +115,7 @@ public Schema getSchema() { public Marketo getMarketo() { if (marketo == null) { - marketo = new Marketo(getRestApiEndpoint(), getMarketoToken()); + marketo = new Marketo(getRestApiEndpoint(), getClientId(), getClientSecret()); } return marketo; } @@ -152,11 +152,4 @@ public String getStartDate() { public String getEndDate() { return endDate; } - - public MarketoToken getMarketoToken() { - if (token == null) { - token = Marketo.getToken(getRestApiEndpoint(), getClientId(), getClientSecret()); - } - return token; - } } diff --git a/src/test/java/io/cdap/plugin/marketo/common/api/MarketoTest.java b/src/test/java/io/cdap/plugin/marketo/common/api/MarketoTest.java new file mode 100644 index 0000000..04d2d3e --- /dev/null +++ b/src/test/java/io/cdap/plugin/marketo/common/api/MarketoTest.java @@ -0,0 +1,187 @@ +package io.cdap.plugin.marketo.common.api; + +import com.github.tomakehurst.wiremock.client.WireMock; +import com.github.tomakehurst.wiremock.core.WireMockConfiguration; +import com.github.tomakehurst.wiremock.junit.WireMockRule; +import com.github.tomakehurst.wiremock.stubbing.Scenario; +import com.google.common.collect.ImmutableMap; +import com.google.gson.Gson; +import io.cdap.plugin.marketo.common.api.entities.BaseResponse; +import io.cdap.plugin.marketo.common.api.entities.Error; +import io.cdap.plugin.marketo.common.api.entities.MarketoToken; +import io.cdap.plugin.marketo.common.api.entities.Warning; +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +public class MarketoTest { + private static Gson GSON = new Gson(); + + @Rule + public WireMockRule wireMockRule = new WireMockRule( + WireMockConfiguration.wireMockConfig().dynamicPort() + ); + + public static class StubResponse extends BaseResponse { + StubResponse(boolean success, List errors, List warnings) { + setSuccess(success); + setErrors(errors); + setWarnings(warnings); + } + } + + @Test + public void testToken() { + WireMock.stubFor( + WireMock.get(WireMock.urlPathMatching("/identity/oauth/token")) + .withQueryParam("grant_type", WireMock.equalTo("client_credentials")) + .withQueryParam("client_id", WireMock.equalTo("clientNiceId")) + .withQueryParam("client_secret", WireMock.equalTo("clientNiceSecret")) + .willReturn( + WireMock.aResponse().withBody( + GSON.toJson(new MarketoToken("niceToken", "hello@world.com", "3600", "bearer") + ) + ) + ) + ); + + Marketo m = new Marketo(getApiUrl(), "clientNiceId", "clientNiceSecret"); + Assert.assertEquals("niceToken", m.getCurrentToken().getAccessToken()); + } + + @Test + public void testTokenRefresh() { + setupToken(); + + WireMock.stubFor( + WireMock.get(WireMock.urlPathMatching("/rest/v1/stub.json")).inScenario("retry") + .whenScenarioStateIs(Scenario.STARTED) + .willReturn( + WireMock.aResponse().withBody( + GSON.toJson(new StubResponse(false, + Collections.singletonList( + new Error(602, "Access token expired")), + Collections.emptyList())) + ) + ) + .willSetStateTo("refreshed") + ); + WireMock.stubFor( + WireMock.get(WireMock.urlPathMatching("/rest/v1/stub.json")).inScenario("retry") + .whenScenarioStateIs("refreshed") + .willReturn( + WireMock.aResponse().withBody( + GSON.toJson(new StubResponse(true, Collections.emptyList(), Collections.emptyList())) + ) + ) + ); + Marketo m = new Marketo(getApiUrl(), "clientNiceId", "clientNiceSecret"); + m.validatedGet("/rest/v1/stub.json", Collections.emptyMap(), + inputStream -> Helpers.streamToObject(inputStream, StubResponse.class)); + WireMock.verify(WireMock.exactly(2), + WireMock.getRequestedFor(WireMock.urlPathEqualTo("/identity/oauth/token"))); + WireMock.verify(WireMock.exactly(2), + WireMock.getRequestedFor(WireMock.urlPathEqualTo("/rest/v1/stub.json"))); + } + + @Test + public void testMessages() { + setupToken(); + + WireMock.stubFor( + WireMock.get(WireMock.urlPathMatching("/rest/v1/justWarnings.json")) + .willReturn( + WireMock.aResponse().withBody( + GSON.toJson(new StubResponse(true, Collections.emptyList(), Arrays.asList( + new Warning(700, "Reversed agent 007"), + new Warning(777, "Result of 1000 - 333") + )))) + ) + ); + + WireMock.stubFor( + WireMock.get(WireMock.urlPathMatching("/rest/v1/errors.json")) + .willReturn( + WireMock.aResponse().withBody( + GSON.toJson(new StubResponse(false, + Collections.singletonList(new Error(123, "No way")), + Collections.emptyList()))) + ) + ); + + Marketo m = new Marketo(getApiUrl(), "clientNiceId", "clientNiceSecret"); + m.validatedGet("/rest/v1/justWarnings.json", Collections.emptyMap(), + inputStream -> Helpers.streamToObject(inputStream, StubResponse.class)); + + try { + m.validatedGet("/rest/v1/errors.json", Collections.emptyMap(), + inputStream -> Helpers.streamToObject(inputStream, StubResponse.class)); + Assert.fail("This call expected to fail."); + } catch (RuntimeException ex) { + Assert.assertTrue(ex.getMessage().contains("123")); + Assert.assertTrue(ex.getMessage().contains("No way")); + } + } + + @Test + public void testBuildUri() { + setupToken(); + Marketo m = new Marketo(getApiUrl(), "clientNiceId", "clientNiceSecret"); + String uriWithToken = m.buildUri("/hello", Collections.emptyMap()).toString(); + Assert.assertTrue(uriWithToken.contains("access_token")); + String uriWithoutToken = m.buildUri("/hello", ImmutableMap.of("param", "value"), false).toString(); + Assert.assertFalse(uriWithoutToken.contains("access_token")); + Assert.assertTrue(uriWithoutToken.contains("param=value")); + } + + @Test + public void testHttpError() { + setupToken(); + + WireMock.stubFor( + WireMock.get(WireMock.urlPathMatching("/rest/v1/fail.json")) + .willReturn( + WireMock.aResponse().withStatus(500).withBody("GJ server.") + ) + ); + + try { + Marketo m = new Marketo(getApiUrl(), "clientNiceId", "clientNiceSecret"); + m.validatedGet("/rest/v1/fail.json", Collections.emptyMap(), + inputStream -> Helpers.streamToObject(inputStream, StubResponse.class)); + Assert.fail("This call expected to fail."); + } catch (RuntimeException ex) { + Assert.assertTrue(ex.getMessage().contains("GJ server")); + } + } + + @Test + public void invalidEndpoint() { + try { + new Marketo("%^%^&%^", "clientNiceId", "clientNiceSecret"); + Assert.fail("This call expected to fail."); + } catch (IllegalArgumentException ex) { + Assert.assertEquals("'%^%^&%^/identity/oauth/token' is invalid URI", ex.getMessage()); + } + } + + void setupToken() { + WireMock.stubFor( + WireMock.get(WireMock.urlPathMatching("/identity/oauth/token")) + .willReturn( + WireMock.aResponse().withBody( + GSON.toJson(new MarketoToken("niceToken", "hello@world.com", "3600", "bearer") + ) + ) + ) + ); + } + + String getApiUrl() { + return String.format("http://localhost:%d", wireMockRule.port()); + } +} \ No newline at end of file From 2cd7bd3a6548db928a54906773cf437fcd2dfc90 Mon Sep 17 00:00:00 2001 From: Yevhenii Chekanskyi Date: Tue, 26 Nov 2019 01:37:05 +0200 Subject: [PATCH 04/16] PLUGIN-75 Marketo Plugin - add some tests. --- pom.xml | 12 +- .../plugin/marketo/common/api/Helpers.java | 3 + .../marketo/common/api/LeadsExportJob.java | 5 +- .../plugin/marketo/common/api/Marketo.java | 179 +---------------- .../marketo/common/api/MarketoHttp.java | 188 ++++++++++++++++++ .../common/api/MarketoPageIterator.java | 4 +- ...{MarketoTest.java => MarketoHttpTest.java} | 106 +++++++++- 7 files changed, 302 insertions(+), 195 deletions(-) create mode 100644 src/main/java/io/cdap/plugin/marketo/common/api/MarketoHttp.java rename src/test/java/io/cdap/plugin/marketo/common/api/{MarketoTest.java => MarketoHttpTest.java} (63%) diff --git a/pom.xml b/pom.xml index d237f88..b697a2b 100644 --- a/pom.xml +++ b/pom.xml @@ -260,12 +260,12 @@ 2.6 - - org.slf4j - slf4j-simple - 1.7.29 - - + + + + + + junit junit diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/Helpers.java b/src/main/java/io/cdap/plugin/marketo/common/api/Helpers.java index 78c77ad..7d109cd 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/Helpers.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/Helpers.java @@ -13,6 +13,9 @@ import java.nio.charset.StandardCharsets; import java.util.List; +/** + * Various helper methods. + */ public class Helpers { public static String streamToString(InputStream inputStream) { try { diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/LeadsExportJob.java b/src/main/java/io/cdap/plugin/marketo/common/api/LeadsExportJob.java index 2feaf0f..8e53e9c 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/LeadsExportJob.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/LeadsExportJob.java @@ -54,8 +54,9 @@ public void waitCompletion() throws InterruptedException { } public void enqueue() { - last = marketo.post(String.format(Urls.BULK_EXPORT_LEADS_ENQUEUE, jobId), - null, LeadsExport.class).singleExport(); + last = marketo.validatedPost(String.format(Urls.BULK_EXPORT_LEADS_ENQUEUE, jobId), Collections.emptyMap(), + inputStream -> Helpers.streamToObject(inputStream, LeadsExport.class), + null, null).singleExport(); LOG.info("Bulk lead export job with id '{}' enqueued", jobId); } diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/Marketo.java b/src/main/java/io/cdap/plugin/marketo/common/api/Marketo.java index 49e569b..8b6fda4 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/Marketo.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/Marketo.java @@ -16,62 +16,29 @@ package io.cdap.plugin.marketo.common.api; -import com.google.common.base.Strings; -import com.google.common.collect.ImmutableMap; import com.google.gson.Gson; -import io.cdap.plugin.marketo.common.api.entities.BaseResponse; -import io.cdap.plugin.marketo.common.api.entities.Error; -import io.cdap.plugin.marketo.common.api.entities.MarketoToken; import io.cdap.plugin.marketo.common.api.entities.leads.LeadsDescribe; import io.cdap.plugin.marketo.common.api.entities.leads.LeadsExport; import io.cdap.plugin.marketo.common.api.entities.leads.LeadsExportRequest; -import org.apache.commons.io.IOUtils; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.client.protocol.HttpClientContext; -import org.apache.http.client.utils.URIBuilder; -import org.apache.http.entity.ContentType; -import org.apache.http.entity.StringEntity; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClients; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.net.URI; -import java.net.URISyntaxException; -import java.nio.charset.StandardCharsets; import java.util.Collections; import java.util.List; -import java.util.Map; -import java.util.Objects; import java.util.Spliterator; import java.util.Spliterators; -import java.util.function.Function; -import java.util.function.Supplier; import java.util.stream.Collectors; import java.util.stream.StreamSupport; /** - * Helper class to perform Marketo Api operations. + * Class that expose marketo rest api endpoints. */ -public class Marketo { +public class Marketo extends MarketoHttp { private static final Logger LOG = LoggerFactory.getLogger(Marketo.class); static final Gson GSON = new Gson(); - private String marketoEndpoint; - private String clientId; - private String clientSecret; - private MarketoToken token; - private HttpClientContext httpClientContext = HttpClientContext.create(); public Marketo(String marketoEndpoint, String clientId, String clientSecret) { - this.marketoEndpoint = marketoEndpoint; - this.clientId = clientId; - this.clientSecret = clientSecret; - token = refreshToken(); + super(marketoEndpoint, clientId, clientSecret); } public List describeLeads() { @@ -88,144 +55,4 @@ public LeadsExportJob exportLeads(LeadsExportRequest request) { return new LeadsExportJob(export, this); } - private T getPage(String queryUrl, Class pageClass) { - return validatedGet(queryUrl, Collections.emptyMap(), - inputStream -> Helpers.streamToObject(inputStream, pageClass)); - } - - T getNextPage(T currentPage, String queryUrl, Class pageClass) { - if (!Strings.isNullOrEmpty(currentPage.getNextPageToken())) { - return validatedGet(queryUrl, - ImmutableMap.of("nextPageToken", currentPage.getNextPageToken()), - inputStream -> Helpers.streamToObject(inputStream, pageClass)); - } - return null; - } - - private MarketoPageIterator iteratePage(String queryUrl, - Class pageClass, - Function> resultsGetter) { - return new MarketoPageIterator<>(getPage(queryUrl, pageClass), this, queryUrl, pageClass, resultsGetter); - } - - T post(String queryUrl, R body, Class responseCls) { - return validatedPost(queryUrl, Collections.emptyMap(), - inputStream -> Helpers.streamToObject(inputStream, responseCls), - body, GSON::toJson); - } - - T validatedGet(String queryUrl, Map parameters, - Function deserializer) { - String logUri = "GET " + buildUri(queryUrl, parameters, false).toString(); - return retryableValidate(logUri, () -> { - URI queryUri = buildUri(queryUrl, parameters, true); - return get(queryUri, deserializer); - }); - } - - private T validatedPost(String queryUrl, Map parameters, - Function deserializer, - B body, Function qSerializer) { - String logUri = "POST " + buildUri(queryUrl, parameters, false).toString(); - return retryableValidate(logUri, () -> { - URI queryUri = buildUri(queryUrl, parameters, true); - return post(queryUri, deserializer, body, qSerializer); - }); - } - - private T retryableValidate(String logUri, Supplier tryQuery) { - T result = tryQuery.get(); - // check for expired token - if (!result.isSuccess()) { - for (Error error : result.getErrors()) { - if (error.getCode() == 602 && error.getMessage().equals("Access token expired")) { - // refresh token and retry - token = refreshToken(); - LOG.info("Refreshed token"); - return tryQuery.get(); - } - } - } - - // log warnings if required - if (result.getWarnings().size() > 0) { - String warnings = result.getWarnings().stream() - .map(error -> String.format("code: %s, message: %s", error.getCode(), error.getMessage())) - .collect(Collectors.joining("; ")); - LOG.warn("Warnings when calling '{}' - {}", logUri, warnings); - } - - if (!result.isSuccess()) { - String msg = String.format("Errors when calling '%s'", logUri); - // log errors if required - if (result.getErrors().size() > 0) { - String errors = result.getErrors().stream() - .map(error -> String.format("code: %s, message: %s", error.getCode(), error.getMessage())) - .collect(Collectors.joining("; ")); - msg = msg + " - " + errors; - LOG.error(msg); - } - throw new RuntimeException(msg); - } - return result; - } - - T get(URI uri, Function deserializer) { - try (CloseableHttpClient httpClient = HttpClients.createDefault()) { - HttpGet request = new HttpGet(uri); - try (CloseableHttpResponse response = httpClient.execute(request, httpClientContext)) { - if(response.getStatusLine().getStatusCode() >= 300) { - throw new IOException(IOUtils.toString(response.getEntity().getContent(), StandardCharsets.UTF_8)); - } - return deserializer.apply(response.getEntity().getContent()); - } - } catch (Exception e) { - throw Helpers.failForUri("GET", uri, e); - } - } - - private T post(URI uri, Function respDeserializer, B body, Function qSerializer) { - try (CloseableHttpClient httpClient = HttpClients.createDefault()) { - HttpPost request = new HttpPost(uri); - if (body != null) { - Objects.requireNonNull(qSerializer, "body serializer must be specified with body"); - request.setEntity(new StringEntity(qSerializer.apply(body), ContentType.APPLICATION_JSON)); - } - try (CloseableHttpResponse response = httpClient.execute(request, httpClientContext)) { - return respDeserializer.apply(response.getEntity().getContent()); - } - } catch (Exception e) { - throw Helpers.failForUri("POST", uri, e); - } - } - - - URI buildUri(String queryUrl, Map parameters) { - return buildUri(queryUrl, parameters, true); - } - - URI buildUri(String queryUrl, Map parameters, boolean includeToken) { - try { - URIBuilder builder = new URIBuilder(marketoEndpoint + queryUrl); - parameters.forEach(builder::setParameter); - if (includeToken) { - builder.setParameter("access_token", token.getAccessToken()); - } - return builder.build(); - } catch (URISyntaxException e) { - throw new IllegalArgumentException(String.format("'%s' is invalid URI", marketoEndpoint + queryUrl)); - } - } - - MarketoToken getCurrentToken() { - return this.token; - } - - private MarketoToken refreshToken() { - LOG.debug("Requesting marketo token"); - URI getTokenUri = buildUri("/identity/oauth/token", - ImmutableMap.of("grant_type", "client_credentials", "client_id", clientId, - "client_secret", clientSecret), false); - return get(getTokenUri, inputStream -> GSON.fromJson(new InputStreamReader(inputStream), MarketoToken.class)); - } } diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/MarketoHttp.java b/src/main/java/io/cdap/plugin/marketo/common/api/MarketoHttp.java new file mode 100644 index 0000000..f71cd77 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/MarketoHttp.java @@ -0,0 +1,188 @@ +package io.cdap.plugin.marketo.common.api; + +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableMap; +import com.google.gson.Gson; +import io.cdap.plugin.marketo.common.api.entities.BaseResponse; +import io.cdap.plugin.marketo.common.api.entities.Error; +import io.cdap.plugin.marketo.common.api.entities.MarketoToken; +import org.apache.commons.io.IOUtils; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.client.protocol.HttpClientContext; +import org.apache.http.client.utils.URIBuilder; +import org.apache.http.entity.ContentType; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.charset.StandardCharsets; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.function.Function; +import java.util.function.Supplier; +import java.util.stream.Collectors; + +/** + * Class that encapsulates common http functions for marketo rest api. + */ +class MarketoHttp { + private static final Logger LOG = LoggerFactory.getLogger(Marketo.class); + private static final Gson GSON = new Gson(); + private String marketoEndpoint; + private String clientId; + private String clientSecret; + private MarketoToken token; + private HttpClientContext httpClientContext = HttpClientContext.create(); + + MarketoHttp(String marketoEndpoint, String clientId, String clientSecret) { + this.marketoEndpoint = marketoEndpoint; + this.clientId = clientId; + this.clientSecret = clientSecret; + token = refreshToken(); + } + + private T getPage(String queryUrl, Class pageClass) { + return validatedGet(queryUrl, Collections.emptyMap(), + inputStream -> Helpers.streamToObject(inputStream, pageClass)); + } + + T getNextPage(T currentPage, String queryUrl, Class pageClass) { + if (!Strings.isNullOrEmpty(currentPage.getNextPageToken())) { + return validatedGet(queryUrl, + ImmutableMap.of("nextPageToken", currentPage.getNextPageToken()), + inputStream -> Helpers.streamToObject(inputStream, pageClass)); + } + return null; + } + + MarketoPageIterator iteratePage(String queryUrl, + Class pageClass, + Function> resultsGetter) { + return new MarketoPageIterator<>(getPage(queryUrl, pageClass), this, queryUrl, pageClass, resultsGetter); + } + + T validatedGet(String queryUrl, Map parameters, + Function deserializer) { + String logUri = "GET " + buildUri(queryUrl, parameters, false).toString(); + return retryableValidate(logUri, () -> { + URI queryUri = buildUri(queryUrl, parameters, true); + return get(queryUri, deserializer); + }); + } + + T validatedPost(String queryUrl, Map parameters, + Function deserializer, + B body, Function qSerializer) { + String logUri = "POST " + buildUri(queryUrl, parameters, false).toString(); + return retryableValidate(logUri, () -> { + URI queryUri = buildUri(queryUrl, parameters, true); + return post(queryUri, deserializer, body, qSerializer); + }); + } + + private T retryableValidate(String logUri, Supplier tryQuery) { + T result = tryQuery.get(); + // check for expired token + if (!result.isSuccess()) { + for (Error error : result.getErrors()) { + if (error.getCode() == 602 && error.getMessage().equals("Access token expired")) { + // refresh token and retry + token = refreshToken(); + LOG.info("Refreshed token"); + return tryQuery.get(); + } + } + } + + // log warnings if required + if (result.getWarnings().size() > 0) { + String warnings = result.getWarnings().stream() + .map(error -> String.format("code: %s, message: %s", error.getCode(), error.getMessage())) + .collect(Collectors.joining("; ")); + LOG.warn("Warnings when calling '{}' - {}", logUri, warnings); + } + + if (!result.isSuccess()) { + String msg = String.format("Errors when calling '%s'", logUri); + // log errors if required + if (result.getErrors().size() > 0) { + String errors = result.getErrors().stream() + .map(error -> String.format("code: %s, message: %s", error.getCode(), error.getMessage())) + .collect(Collectors.joining("; ")); + msg = msg + " - " + errors; + LOG.error(msg); + } + throw new RuntimeException(msg); + } + return result; + } + + T get(URI uri, Function deserializer) { + try (CloseableHttpClient httpClient = HttpClients.createDefault()) { + HttpGet request = new HttpGet(uri); + try (CloseableHttpResponse response = httpClient.execute(request, httpClientContext)) { + if (response.getStatusLine().getStatusCode() >= 300) { + throw new IOException(IOUtils.toString(response.getEntity().getContent(), StandardCharsets.UTF_8)); + } + return deserializer.apply(response.getEntity().getContent()); + } + } catch (Exception e) { + throw Helpers.failForUri("GET", uri, e); + } + } + + private T post(URI uri, Function respDeserializer, B body, Function qSerializer) { + try (CloseableHttpClient httpClient = HttpClients.createDefault()) { + HttpPost request = new HttpPost(uri); + if (body != null) { + Objects.requireNonNull(qSerializer, "body serializer must be specified with body"); + request.setEntity(new StringEntity(qSerializer.apply(body), ContentType.APPLICATION_JSON)); + } + try (CloseableHttpResponse response = httpClient.execute(request, httpClientContext)) { + return respDeserializer.apply(response.getEntity().getContent()); + } + } catch (Exception e) { + throw Helpers.failForUri("POST", uri, e); + } + } + + URI buildUri(String queryUrl, Map parameters) { + return buildUri(queryUrl, parameters, true); + } + + URI buildUri(String queryUrl, Map parameters, boolean includeToken) { + try { + URIBuilder builder = new URIBuilder(marketoEndpoint + queryUrl); + parameters.forEach(builder::setParameter); + if (includeToken) { + builder.setParameter("access_token", token.getAccessToken()); + } + return builder.build(); + } catch (URISyntaxException e) { + throw new IllegalArgumentException(String.format("'%s' is invalid URI", marketoEndpoint + queryUrl)); + } + } + + MarketoToken getCurrentToken() { + return this.token; + } + + private MarketoToken refreshToken() { + LOG.debug("Requesting marketo token"); + URI getTokenUri = buildUri("/identity/oauth/token", + ImmutableMap.of("grant_type", "client_credentials", "client_id", clientId, + "client_secret", clientSecret), false); + return get(getTokenUri, inputStream -> GSON.fromJson(new InputStreamReader(inputStream), MarketoToken.class)); + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/MarketoPageIterator.java b/src/main/java/io/cdap/plugin/marketo/common/api/MarketoPageIterator.java index 3a7a424..0c32729 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/MarketoPageIterator.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/MarketoPageIterator.java @@ -15,13 +15,13 @@ */ public class MarketoPageIterator implements Iterator { private T currentPage; - private Marketo marketo; + private MarketoHttp marketo; private String queryUrl; private Class pageClass; private Function> resultsGetter; private Iterator currentPageResultIterator; - MarketoPageIterator(T page, Marketo marketo, String queryUrl, Class pageClass, + MarketoPageIterator(T page, MarketoHttp marketo, String queryUrl, Class pageClass, Function> resultsGetter) { this.currentPage = page; this.marketo = marketo; diff --git a/src/test/java/io/cdap/plugin/marketo/common/api/MarketoTest.java b/src/test/java/io/cdap/plugin/marketo/common/api/MarketoHttpTest.java similarity index 63% rename from src/test/java/io/cdap/plugin/marketo/common/api/MarketoTest.java rename to src/test/java/io/cdap/plugin/marketo/common/api/MarketoHttpTest.java index 04d2d3e..3ad8844 100644 --- a/src/test/java/io/cdap/plugin/marketo/common/api/MarketoTest.java +++ b/src/test/java/io/cdap/plugin/marketo/common/api/MarketoHttpTest.java @@ -17,9 +17,13 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Spliterator; +import java.util.Spliterators; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; -public class MarketoTest { - private static Gson GSON = new Gson(); +public class MarketoHttpTest { + private static final Gson GSON = new Gson(); @Rule public WireMockRule wireMockRule = new WireMockRule( @@ -34,6 +38,67 @@ public static class StubResponse extends BaseResponse { } } + public static class PageResponse extends BaseResponse { + private List results; + + PageResponse(boolean moreResults, String nextPageToken, String... items) { + setSuccess(true); + setMoreResult(moreResults); + results = Arrays.asList(items); + setNextPageToken(nextPageToken); + } + + public List getResults() { + return results; + } + } + + @Test + public void testPaging() { + setupToken(); + + WireMock.stubFor( + WireMock.get(WireMock.urlPathMatching("/rest/v1/paged.json")).inScenario("page") + .whenScenarioStateIs(Scenario.STARTED) + .willReturn( + WireMock.aResponse().withBody( + GSON.toJson(new PageResponse(true, "page1", "1", "2", "3")) + ) + ) + .willSetStateTo("page1") + ); + WireMock.stubFor( + WireMock.get(WireMock.urlPathMatching("/rest/v1/paged.json")).inScenario("page") + .whenScenarioStateIs("page1") + .withQueryParam("nextPageToken", WireMock.equalTo("page1")) + .willReturn( + WireMock.aResponse().withBody( + GSON.toJson(new PageResponse(true, "page2", "4", "5", "6")) + ) + ) + .willSetStateTo("page2") + ); + WireMock.stubFor( + WireMock.get(WireMock.urlPathMatching("/rest/v1/paged.json")).inScenario("page") + .whenScenarioStateIs("page2") + .withQueryParam("nextPageToken", WireMock.equalTo("page2")) + .willReturn( + WireMock.aResponse().withBody( + GSON.toJson(new PageResponse(false, null, "7", "8", "9")) + ) + ) + .willSetStateTo("page3") + ); + + MarketoHttp m = new MarketoHttp(getApiUrl(), "clientNiceId", "clientNiceSecret"); + + List results = StreamSupport.stream(Spliterators.spliteratorUnknownSize( + m.iteratePage("/rest/v1/paged.json", PageResponse.class, PageResponse::getResults), + Spliterator.ORDERED), false).sorted().collect(Collectors.toList()); + + Assert.assertArrayEquals(new String[]{"1", "2", "3", "4", "5", "6", "7", "8", "9"}, results.toArray()); + } + @Test public void testToken() { WireMock.stubFor( @@ -49,7 +114,7 @@ public void testToken() { ) ); - Marketo m = new Marketo(getApiUrl(), "clientNiceId", "clientNiceSecret"); + MarketoHttp m = new MarketoHttp(getApiUrl(), "clientNiceId", "clientNiceSecret"); Assert.assertEquals("niceToken", m.getCurrentToken().getAccessToken()); } @@ -79,7 +144,7 @@ public void testTokenRefresh() { ) ) ); - Marketo m = new Marketo(getApiUrl(), "clientNiceId", "clientNiceSecret"); + MarketoHttp m = new MarketoHttp(getApiUrl(), "clientNiceId", "clientNiceSecret"); m.validatedGet("/rest/v1/stub.json", Collections.emptyMap(), inputStream -> Helpers.streamToObject(inputStream, StubResponse.class)); WireMock.verify(WireMock.exactly(2), @@ -88,6 +153,29 @@ public void testTokenRefresh() { WireMock.getRequestedFor(WireMock.urlPathEqualTo("/rest/v1/stub.json"))); } + @Test + public void testPost() { + setupToken(); + + WireMock.stubFor( + WireMock.post(WireMock.urlPathMatching("/rest/v1/post.json")) + .willReturn( + WireMock.aResponse().withBody( + GSON.toJson(new StubResponse(true, Collections.emptyList(), Collections.emptyList())) + ) + ) + ); + + MarketoHttp m = new MarketoHttp(getApiUrl(), "clientNiceId", "clientNiceSecret"); + m.validatedPost("/rest/v1/post.json", Collections.emptyMap(), + inputStream -> Helpers.streamToObject(inputStream, StubResponse.class), "body", String::toString); + + WireMock.verify( + WireMock.postRequestedFor(WireMock.urlPathEqualTo("/rest/v1/post.json")) + .withRequestBody(WireMock.equalTo("body")) + ); + } + @Test public void testMessages() { setupToken(); @@ -113,7 +201,7 @@ public void testMessages() { ) ); - Marketo m = new Marketo(getApiUrl(), "clientNiceId", "clientNiceSecret"); + MarketoHttp m = new MarketoHttp(getApiUrl(), "clientNiceId", "clientNiceSecret"); m.validatedGet("/rest/v1/justWarnings.json", Collections.emptyMap(), inputStream -> Helpers.streamToObject(inputStream, StubResponse.class)); @@ -130,7 +218,7 @@ public void testMessages() { @Test public void testBuildUri() { setupToken(); - Marketo m = new Marketo(getApiUrl(), "clientNiceId", "clientNiceSecret"); + MarketoHttp m = new MarketoHttp(getApiUrl(), "clientNiceId", "clientNiceSecret"); String uriWithToken = m.buildUri("/hello", Collections.emptyMap()).toString(); Assert.assertTrue(uriWithToken.contains("access_token")); String uriWithoutToken = m.buildUri("/hello", ImmutableMap.of("param", "value"), false).toString(); @@ -150,7 +238,7 @@ public void testHttpError() { ); try { - Marketo m = new Marketo(getApiUrl(), "clientNiceId", "clientNiceSecret"); + MarketoHttp m = new MarketoHttp(getApiUrl(), "clientNiceId", "clientNiceSecret"); m.validatedGet("/rest/v1/fail.json", Collections.emptyMap(), inputStream -> Helpers.streamToObject(inputStream, StubResponse.class)); Assert.fail("This call expected to fail."); @@ -162,7 +250,7 @@ public void testHttpError() { @Test public void invalidEndpoint() { try { - new Marketo("%^%^&%^", "clientNiceId", "clientNiceSecret"); + new MarketoHttp("%^%^&%^", "clientNiceId", "clientNiceSecret"); Assert.fail("This call expected to fail."); } catch (IllegalArgumentException ex) { Assert.assertEquals("'%^%^&%^/identity/oauth/token' is invalid URI", ex.getMessage()); @@ -184,4 +272,4 @@ void setupToken() { String getApiUrl() { return String.format("http://localhost:%d", wireMockRule.port()); } -} \ No newline at end of file +} From 91ac74d58aa885ee3d85d16eaf41c1e3b1b63507 Mon Sep 17 00:00:00 2001 From: Yevhenii Chekanskyi Date: Tue, 26 Nov 2019 23:00:40 +0200 Subject: [PATCH 05/16] PLUGIN-75 Marketo Plugin - support splits. --- .../plugin/marketo/common/api/Helpers.java | 47 ++++++++++++ .../marketo/common/api/LeadsExportJob.java | 76 ++++++++++++------- .../plugin/marketo/common/api/Marketo.java | 45 ++++++++++- .../cdap/plugin/marketo/common/api/Urls.java | 1 + .../common/api/entities/DateRange.java | 31 ++++++++ .../api/entities/leads/LeadsExport.java | 4 + .../entities/leads/LeadsExportRequest.java | 15 +--- .../source/batch/MarketoInputFormat.java | 18 ++++- .../source/batch/MarketoRecordReader.java | 21 ++++- ...oSplit.java => MarketoReportingSplit.java} | 29 ++++++- 10 files changed, 234 insertions(+), 53 deletions(-) create mode 100644 src/main/java/io/cdap/plugin/marketo/common/api/entities/DateRange.java rename src/main/java/io/cdap/plugin/marketo/source/batch/{NoOpMarketoSplit.java => MarketoReportingSplit.java} (58%) diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/Helpers.java b/src/main/java/io/cdap/plugin/marketo/common/api/Helpers.java index 7d109cd..7301043 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/Helpers.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/Helpers.java @@ -1,6 +1,8 @@ package io.cdap.plugin.marketo.common.api; import com.google.common.base.Strings; +import com.google.common.collect.ImmutableList; +import io.cdap.plugin.marketo.common.api.entities.DateRange; import org.apache.commons.io.IOUtils; import org.apache.http.NameValuePair; import org.apache.http.client.utils.URIBuilder; @@ -11,6 +13,8 @@ import java.net.URI; import java.net.URISyntaxException; import java.nio.charset.StandardCharsets; +import java.time.OffsetDateTime; +import java.util.ArrayList; import java.util.List; /** @@ -51,4 +55,47 @@ public static RuntimeException failForUri(String method, URI uri, Exception ex) return new RuntimeException(message); } } + + /** + * Splits date range in 30 day ranges. + * Return date range as is if difference is less or equals to 30 days. + * + * @param beginDate + * @param endDate + * @return + */ + public static List getDateRanges(String beginDate, String endDate) { + OffsetDateTime start = OffsetDateTime.parse(beginDate); + OffsetDateTime end = OffsetDateTime.parse(endDate); + + if (start.compareTo(end) > 0) { + throw new IllegalArgumentException("start date more than end date"); + } + + int compareResult = start.plusDays(30).compareTo(end); + if (compareResult >= 0) { + // we are in range of 30 days, dates are okay + return ImmutableList.of(new DateRange(start.toString(), end.toString())); + } else { + List result = new ArrayList<>(); + OffsetDateTime currentStart = start; + + while (currentStart.compareTo(end) < 0) { + OffsetDateTime nextEnd = currentStart.plusDays(30); + result.add(new DateRange(currentStart.toString(), + min(nextEnd.minusSeconds(1), end).toString())); + currentStart = nextEnd; + } + + return result; + } + } + + public static > T min(T o1, T o2) { + if (o1.compareTo(o2) < 0) { + return o1; + } else { + return o2; + } + } } diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/LeadsExportJob.java b/src/main/java/io/cdap/plugin/marketo/common/api/LeadsExportJob.java index 8e53e9c..9d2c13b 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/LeadsExportJob.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/LeadsExportJob.java @@ -7,62 +7,82 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.concurrent.TimeUnit; /** * Leads export job. */ public class LeadsExportJob { private static final Logger LOG = LoggerFactory.getLogger(LeadsExportJob.class); - private static final List WAITABLE_STATE = Arrays.asList("Queued", "Processing"); - private static final List COMPLETED_STATUS = Arrays.asList("Canceled", "Completed", "Failed"); + private static final List WAIT_ABLE_STATE = Arrays.asList("Queued", "Processing"); + private static final String ENQUEUE_ABLE_STATUS = "Created"; + private static final String COMPLETED_STATUS = "Completed"; private String jobId; - private LeadsExport.ExportResponse last; + private LeadsExport.ExportResponse lastStatus; private Marketo marketo; - public LeadsExportJob(LeadsExport lastStatus, Marketo marketo) { - this.jobId = lastStatus.singleExport().getExportId(); - this.last = lastStatus.singleExport(); + public LeadsExportJob(LeadsExport.ExportResponse lastStatus, Marketo marketo) { + this.jobId = lastStatus.getExportId(); + this.lastStatus = lastStatus; this.marketo = marketo; - LOG.info("Created bulk lead export job with id '{}'", this.jobId); + LOG.info("BULK LEADS EXPORT - created job '{}'", this.jobId); } - public String getStatus() { - return last.getStatus(); + public String getLastStatus() { + return lastStatus.getStatus(); } public void waitCompletion() throws InterruptedException { - if (!WAITABLE_STATE.contains(getStatus())) { + if (!WAIT_ABLE_STATE.contains(getLastStatus())) { throw new IllegalStateException("Job must be enqueued before waiting for completion."); } - while (!COMPLETED_STATUS.contains(getStatus())) { - LeadsExport currentResp = marketo.validatedGet( - String.format(Urls.BULK_EXPORT_LEADS_STATUS, jobId), - Collections.emptyMap(), inputStream -> Helpers.streamToObject(inputStream, LeadsExport.class)); - LeadsExport.ExportResponse current = currentResp.singleExport(); - String previousStatus = getStatus(); - String currentStatus = current.getStatus(); - if (!currentStatus.equals(previousStatus)) { - LOG.info("Bulk lead export job with id '{}' changed status from '{}' to '{}'", jobId, previousStatus, - currentStatus); - } - last = current; - Thread.sleep(30 * 1000); + do { + Thread.sleep(TimeUnit.SECONDS.toMillis(30)); + LeadsExport.ExportResponse newState = marketo.leadsExportJobStatus(jobId); + logStatusChange(getLastStatus(), newState.getStatus()); + lastStatus = newState; + } while (WAIT_ABLE_STATE.contains(getLastStatus())); + + if (!getLastStatus().equals(COMPLETED_STATUS)) { + throw new IllegalStateException("Job expected to be in Completed state, but was in " + getLastStatus()); } - LOG.info("Bulk lead export job with id '{}' finished with status '{}'", jobId, getStatus()); } public void enqueue() { - last = marketo.validatedPost(String.format(Urls.BULK_EXPORT_LEADS_ENQUEUE, jobId), Collections.emptyMap(), - inputStream -> Helpers.streamToObject(inputStream, LeadsExport.class), - null, null).singleExport(); + if (!getLastStatus().equals(ENQUEUE_ABLE_STATUS)) { + throw new IllegalStateException("Job must be in Created status before enqueuing, but was in " + getLastStatus()); + } + + LeadsExport.ExportResponse newState = marketo.validatedPost( + String.format(Urls.BULK_EXPORT_LEADS_ENQUEUE, jobId), + Collections.emptyMap(), + inputStream -> Helpers.streamToObject(inputStream, LeadsExport.class), + null, null).singleExport(); + + logStatusChange(getLastStatus(), newState.getStatus()); + + if (!(newState.getStatus().equals("Queued") || newState.getStatus().equals("Processing"))) { + throw new IllegalStateException( + String.format("Expected Queued|Processing state for job '%s' but got '%s'", jobId, newState.getStatus())); + } - LOG.info("Bulk lead export job with id '{}' enqueued", jobId); + lastStatus = newState; + } + + private void logStatusChange(String oldStatus, String newStatus) { + if (!oldStatus.equals(newStatus)) { + LOG.info("BULK LEADS EXPORT - job '{}' changed state '{}' -> '{}'", jobId, oldStatus, newStatus); + } } public String getFile() { return marketo.get(marketo.buildUri(String.format(Urls.BULK_EXPORT_LEADS_FILE, jobId), Collections.emptyMap()), Helpers::streamToString); } + + public String getJobId() { + return jobId; + } } diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/Marketo.java b/src/main/java/io/cdap/plugin/marketo/common/api/Marketo.java index 8b6fda4..d2bb9f9 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/Marketo.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/Marketo.java @@ -16,6 +16,7 @@ package io.cdap.plugin.marketo.common.api; +import com.google.common.collect.ImmutableMap; import com.google.gson.Gson; import io.cdap.plugin.marketo.common.api.entities.leads.LeadsDescribe; import io.cdap.plugin.marketo.common.api.entities.leads.LeadsExport; @@ -27,6 +28,7 @@ import java.util.List; import java.util.Spliterator; import java.util.Spliterators; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import java.util.stream.StreamSupport; @@ -52,7 +54,48 @@ public LeadsExportJob exportLeads(LeadsExportRequest request) { inputStream -> Helpers.streamToObject(inputStream, LeadsExport.class), request, GSON::toJson); - return new LeadsExportJob(export, this); + return new LeadsExportJob(export.singleExport(), this); } + public LeadsExport.ExportResponse leadsExportJobStatus(String jobId) { + LeadsExport currentResp = validatedGet( + String.format(Urls.BULK_EXPORT_LEADS_STATUS, jobId), + Collections.emptyMap(), inputStream -> Helpers.streamToObject(inputStream, LeadsExport.class)); + return currentResp.singleExport(); + } + + /** + * Waits until bulk extract queue has available slot and executes given action. + * + * @param action action to execute once slot is available + * @param timeoutSeconds timeout, in seconds + */ + public void onBulkExtractQueueAvailable(Runnable action, long timeoutSeconds) { + long timeoutMillis = TimeUnit.SECONDS.toMillis(timeoutSeconds); + long startTime = System.currentTimeMillis(); + while (System.currentTimeMillis() - startTime < timeoutMillis) { + if (canEnqueueJob()) { + action.run(); + return; + } else { + try { + Thread.sleep(TimeUnit.SECONDS.toMillis(60)); + } catch (InterruptedException e) { + throw new RuntimeException("Failed to get slot in bulk export queue - interrupted"); + } + } + } + throw new RuntimeException("Failed to get slot in bulk export queue - timeout"); + } + + private boolean canEnqueueJob() { + LeadsExport exportJobs = validatedGet(Urls.BULK_EXPORT_LEADS_LIST, + ImmutableMap.of("status", "queued,processing"), + inputStream -> Helpers.streamToObject(inputStream, LeadsExport.class) + ); + int jobsInQueue = exportJobs.getResult().size(); + LOG.debug("Jobs in queue: {}", jobsInQueue); + // TODO handle activity queue here + return jobsInQueue < 10; + } } diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/Urls.java b/src/main/java/io/cdap/plugin/marketo/common/api/Urls.java index dc0ae5f..2db9085 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/Urls.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/Urls.java @@ -4,6 +4,7 @@ * Marketo API urls. */ public class Urls { + public static final String BULK_EXPORT_LEADS_LIST = "/bulk/v1/leads/export.json"; public static final String BULK_EXPORT_LEADS_CREATE = "/bulk/v1/leads/export/create.json"; public static final String BULK_EXPORT_LEADS_ENQUEUE = "/bulk/v1/leads/export/%s/enqueue.json"; public static final String BULK_EXPORT_LEADS_STATUS = "/bulk/v1/leads/export/%s/status.json"; diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/DateRange.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/DateRange.java new file mode 100644 index 0000000..3764947 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/DateRange.java @@ -0,0 +1,31 @@ +package io.cdap.plugin.marketo.common.api.entities; + +/** + * Represents date range. + */ +public class DateRange { + String endAt; + String startAt; + + public DateRange() { + + } + + public DateRange(String startAt, String endAt) { + this.endAt = endAt; + this.startAt = startAt; + } + + public String getEndAt() { + return endAt; + } + + public String getStartAt() { + return startAt; + } + + @Override + public String toString() { + return startAt + " -- " + endAt; + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/leads/LeadsExport.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/leads/LeadsExport.java index 877630b..65f6312 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/entities/leads/LeadsExport.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/leads/LeadsExport.java @@ -79,4 +79,8 @@ public ExportResponse singleExport() { return result.get(0); } + public List getResult() { + return result; + } + } diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/leads/LeadsExportRequest.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/leads/LeadsExportRequest.java index d46f0d4..572072e 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/entities/leads/LeadsExportRequest.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/leads/LeadsExportRequest.java @@ -1,5 +1,7 @@ package io.cdap.plugin.marketo.common.api.entities.leads; +import io.cdap.plugin.marketo.common.api.entities.DateRange; + import java.util.List; import java.util.Map; @@ -7,19 +9,6 @@ * Represents leads bulk export request. */ public class LeadsExportRequest { - /** - * Represents date range. - */ - public static class DateRange { - String endAt = null; - String startAt = null; - - public DateRange(String startAt, String endAt) { - this.endAt = endAt; - this.startAt = startAt; - } - } - /** * Represents request filter. */ diff --git a/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoInputFormat.java b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoInputFormat.java index 4fd514e..096b842 100644 --- a/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoInputFormat.java +++ b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoInputFormat.java @@ -16,26 +16,38 @@ package io.cdap.plugin.marketo.source.batch; +import com.google.gson.Gson; +import io.cdap.plugin.marketo.common.api.Helpers; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.mapreduce.InputFormat; import org.apache.hadoop.mapreduce.InputSplit; import org.apache.hadoop.mapreduce.JobContext; import org.apache.hadoop.mapreduce.RecordReader; import org.apache.hadoop.mapreduce.TaskAttemptContext; -import java.util.Collections; import java.util.List; +import java.util.stream.Collectors; /** * InputFormat for mapreduce job, which provides a single split of data. */ public class MarketoInputFormat extends InputFormat { + private static final Gson GSON = new Gson(); + @Override public List getSplits(JobContext jobContext) { - return Collections.singletonList(new NoOpMarketoSplit()); + Configuration conf = jobContext.getConfiguration(); + MarketoReportingSourceConfig config = GSON.fromJson( + conf.get(MarketoInputFormatProvider.PROPERTY_CONFIG_JSON), MarketoReportingSourceConfig.class); + + return Helpers.getDateRanges(config.getStartDate(), config.getEndDate()).stream() + .map(dateRange -> new MarketoReportingSplit(dateRange.getStartAt(), dateRange.getEndAt())) + .collect(Collectors.toList()); } @Override public RecordReader createRecordReader(InputSplit inputSplit, TaskAttemptContext taskAttemptContext) { - return new MarketoRecordReader(); + MarketoReportingSplit split = (MarketoReportingSplit) inputSplit; + return new MarketoRecordReader(split.getBeginDate(), split.getEndDate()); } } diff --git a/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoRecordReader.java b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoRecordReader.java index a3b0e70..48e49e2 100644 --- a/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoRecordReader.java +++ b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoRecordReader.java @@ -21,6 +21,7 @@ import io.cdap.cdap.api.data.schema.Schema; import io.cdap.plugin.marketo.common.api.LeadsExportJob; import io.cdap.plugin.marketo.common.api.Marketo; +import io.cdap.plugin.marketo.common.api.entities.DateRange; import io.cdap.plugin.marketo.common.api.entities.leads.LeadsExportRequest; import org.apache.commons.csv.CSVFormat; import org.apache.commons.csv.CSVParser; @@ -48,6 +49,13 @@ public class MarketoRecordReader extends RecordReader current = null; private Iterator iterator = null; + private String beginDate; + private String endDate; + + public MarketoRecordReader(String beginDate, String endDate) { + this.beginDate = beginDate; + this.endDate = endDate; + } @Override public void initialize(InputSplit inputSplit, TaskAttemptContext taskAttemptContext) throws IOException { @@ -59,12 +67,18 @@ public void initialize(InputSplit inputSplit, TaskAttemptContext taskAttemptCont List fields = config.getSchema().getFields().stream() .map(Schema.Field::getName).collect(Collectors.toList()); + DateRange dateRange = new DateRange(beginDate, endDate); LeadsExportRequest.ExportLeadFilter filter = LeadsExportRequest.ExportLeadFilter.builder() - .createdAt(new LeadsExportRequest.DateRange(config.getStartDate(), config.getEndDate())).build(); + .createdAt(dateRange).build(); LeadsExportRequest request = new LeadsExportRequest(fields, filter); LeadsExportJob job = marketo.exportLeads(request); - job.enqueue(); + LOG.info("Bulk leads export job with id '{}' has date range {}", job.getJobId(), dateRange); + + // TODO handle possible concurrent issues here, another mapper can take our slot + // wait for 10 minutes for available slot and enqueue job + marketo.onBulkExtractQueueAvailable(job::enqueue, 60 * 10); + try { job.waitCompletion(); } catch (InterruptedException ex) { @@ -72,10 +86,9 @@ public void initialize(InputSplit inputSplit, TaskAttemptContext taskAttemptCont } String data = job.getFile(); - LOG.info(data); + // TODO stream here CSVParser parser = CSVFormat.DEFAULT.withHeader().parse(new StringReader(data)); iterator = parser.iterator(); -// iterator = config.getMarketo().iteratePage(config.getEntityType().getGetEndpoint()); } @Override diff --git a/src/main/java/io/cdap/plugin/marketo/source/batch/NoOpMarketoSplit.java b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoReportingSplit.java similarity index 58% rename from src/main/java/io/cdap/plugin/marketo/source/batch/NoOpMarketoSplit.java rename to src/main/java/io/cdap/plugin/marketo/source/batch/MarketoReportingSplit.java index 396ad97..7e12e27 100644 --- a/src/main/java/io/cdap/plugin/marketo/source/batch/NoOpMarketoSplit.java +++ b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoReportingSplit.java @@ -21,20 +21,33 @@ import java.io.DataInput; import java.io.DataOutput; +import java.io.IOException; /** * A no-op split. */ -public class NoOpMarketoSplit extends InputSplit implements Writable { - public NoOpMarketoSplit() { +public class MarketoReportingSplit extends InputSplit implements Writable { + private String beginDate; + private String endDate; + + public MarketoReportingSplit() { + } + + public MarketoReportingSplit(String beginDate, String endDate) { + this.beginDate = beginDate; + this.endDate = endDate; } @Override - public void readFields(DataInput dataInput) { + public void readFields(DataInput dataInput) throws IOException { + beginDate = dataInput.readUTF(); + endDate = dataInput.readUTF(); } @Override - public void write(DataOutput dataOutput) { + public void write(DataOutput dataOutput) throws IOException { + dataOutput.writeUTF(beginDate); + dataOutput.writeUTF(endDate); } @Override @@ -46,4 +59,12 @@ public long getLength() { public String[] getLocations() { return new String[0]; } + + public String getBeginDate() { + return beginDate; + } + + public String getEndDate() { + return endDate; + } } From c7a0f317e8fc003b0c70f56d773711541ad1dbed Mon Sep 17 00:00:00 2001 From: Yevhenii Chekanskyi Date: Wed, 27 Nov 2019 17:32:28 +0200 Subject: [PATCH 06/16] PLUGIN-75 Marketo Plugin - add activities export. --- pom.xml | 1 - .../plugin/marketo/common/api/Helpers.java | 2 +- .../marketo/common/api/LeadsExportJob.java | 88 ------------- .../plugin/marketo/common/api/Marketo.java | 66 ++++++++-- .../marketo/common/api/MarketoHttp.java | 6 +- .../cdap/plugin/marketo/common/api/Urls.java | 8 +- .../entities/activities/ActivitiesExport.java | 86 +++++++++++++ .../activities/ActivitiesExportRequest.java | 20 +++ .../activities/ActivityTypeResponse.java | 79 ++++++++++++ .../activities/ExportActivityFilter.java | 59 +++++++++ .../common/api/job/AbstractBulkExportJob.java | 116 ++++++++++++++++++ .../common/api/job/ActivitiesExportJob.java | 55 +++++++++ .../common/api/job/LeadsExportJob.java | 55 +++++++++ .../source/batch/MarketoRecordReader.java | 35 ++++-- .../source/batch/MarketoReportingPlugin.java | 11 +- .../batch/MarketoReportingSchemaHelper.java | 55 +++++++++ .../batch/MarketoReportingSourceConfig.java | 26 ++-- .../marketo/source/batch/ReportType.java | 24 ++++ 18 files changed, 651 insertions(+), 141 deletions(-) delete mode 100644 src/main/java/io/cdap/plugin/marketo/common/api/LeadsExportJob.java create mode 100644 src/main/java/io/cdap/plugin/marketo/common/api/entities/activities/ActivitiesExport.java create mode 100644 src/main/java/io/cdap/plugin/marketo/common/api/entities/activities/ActivitiesExportRequest.java create mode 100644 src/main/java/io/cdap/plugin/marketo/common/api/entities/activities/ActivityTypeResponse.java create mode 100644 src/main/java/io/cdap/plugin/marketo/common/api/entities/activities/ExportActivityFilter.java create mode 100644 src/main/java/io/cdap/plugin/marketo/common/api/job/AbstractBulkExportJob.java create mode 100644 src/main/java/io/cdap/plugin/marketo/common/api/job/ActivitiesExportJob.java create mode 100644 src/main/java/io/cdap/plugin/marketo/common/api/job/LeadsExportJob.java create mode 100644 src/main/java/io/cdap/plugin/marketo/source/batch/MarketoReportingSchemaHelper.java create mode 100644 src/main/java/io/cdap/plugin/marketo/source/batch/ReportType.java diff --git a/pom.xml b/pom.xml index b697a2b..e3e9248 100644 --- a/pom.xml +++ b/pom.xml @@ -264,7 +264,6 @@ - junit diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/Helpers.java b/src/main/java/io/cdap/plugin/marketo/common/api/Helpers.java index 7301043..7418bb8 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/Helpers.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/Helpers.java @@ -83,7 +83,7 @@ public static List getDateRanges(String beginDate, String endDate) { while (currentStart.compareTo(end) < 0) { OffsetDateTime nextEnd = currentStart.plusDays(30); result.add(new DateRange(currentStart.toString(), - min(nextEnd.minusSeconds(1), end).toString())); + min(nextEnd.minusSeconds(1), end).toString())); currentStart = nextEnd; } diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/LeadsExportJob.java b/src/main/java/io/cdap/plugin/marketo/common/api/LeadsExportJob.java deleted file mode 100644 index 9d2c13b..0000000 --- a/src/main/java/io/cdap/plugin/marketo/common/api/LeadsExportJob.java +++ /dev/null @@ -1,88 +0,0 @@ -package io.cdap.plugin.marketo.common.api; - -import io.cdap.plugin.marketo.common.api.entities.leads.LeadsExport; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.concurrent.TimeUnit; - -/** - * Leads export job. - */ -public class LeadsExportJob { - private static final Logger LOG = LoggerFactory.getLogger(LeadsExportJob.class); - private static final List WAIT_ABLE_STATE = Arrays.asList("Queued", "Processing"); - private static final String ENQUEUE_ABLE_STATUS = "Created"; - private static final String COMPLETED_STATUS = "Completed"; - - private String jobId; - private LeadsExport.ExportResponse lastStatus; - private Marketo marketo; - - public LeadsExportJob(LeadsExport.ExportResponse lastStatus, Marketo marketo) { - this.jobId = lastStatus.getExportId(); - this.lastStatus = lastStatus; - this.marketo = marketo; - LOG.info("BULK LEADS EXPORT - created job '{}'", this.jobId); - } - - public String getLastStatus() { - return lastStatus.getStatus(); - } - - public void waitCompletion() throws InterruptedException { - if (!WAIT_ABLE_STATE.contains(getLastStatus())) { - throw new IllegalStateException("Job must be enqueued before waiting for completion."); - } - - do { - Thread.sleep(TimeUnit.SECONDS.toMillis(30)); - LeadsExport.ExportResponse newState = marketo.leadsExportJobStatus(jobId); - logStatusChange(getLastStatus(), newState.getStatus()); - lastStatus = newState; - } while (WAIT_ABLE_STATE.contains(getLastStatus())); - - if (!getLastStatus().equals(COMPLETED_STATUS)) { - throw new IllegalStateException("Job expected to be in Completed state, but was in " + getLastStatus()); - } - } - - public void enqueue() { - if (!getLastStatus().equals(ENQUEUE_ABLE_STATUS)) { - throw new IllegalStateException("Job must be in Created status before enqueuing, but was in " + getLastStatus()); - } - - LeadsExport.ExportResponse newState = marketo.validatedPost( - String.format(Urls.BULK_EXPORT_LEADS_ENQUEUE, jobId), - Collections.emptyMap(), - inputStream -> Helpers.streamToObject(inputStream, LeadsExport.class), - null, null).singleExport(); - - logStatusChange(getLastStatus(), newState.getStatus()); - - if (!(newState.getStatus().equals("Queued") || newState.getStatus().equals("Processing"))) { - throw new IllegalStateException( - String.format("Expected Queued|Processing state for job '%s' but got '%s'", jobId, newState.getStatus())); - } - - lastStatus = newState; - } - - private void logStatusChange(String oldStatus, String newStatus) { - if (!oldStatus.equals(newStatus)) { - LOG.info("BULK LEADS EXPORT - job '{}' changed state '{}' -> '{}'", jobId, oldStatus, newStatus); - } - } - - public String getFile() { - return marketo.get(marketo.buildUri(String.format(Urls.BULK_EXPORT_LEADS_FILE, jobId), Collections.emptyMap()), - Helpers::streamToString); - } - - public String getJobId() { - return jobId; - } -} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/Marketo.java b/src/main/java/io/cdap/plugin/marketo/common/api/Marketo.java index d2bb9f9..ead5070 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/Marketo.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/Marketo.java @@ -16,14 +16,21 @@ package io.cdap.plugin.marketo.common.api; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.gson.Gson; +import io.cdap.plugin.marketo.common.api.entities.activities.ActivitiesExport; +import io.cdap.plugin.marketo.common.api.entities.activities.ActivitiesExportRequest; +import io.cdap.plugin.marketo.common.api.entities.activities.ActivityTypeResponse; import io.cdap.plugin.marketo.common.api.entities.leads.LeadsDescribe; import io.cdap.plugin.marketo.common.api.entities.leads.LeadsExport; import io.cdap.plugin.marketo.common.api.entities.leads.LeadsExportRequest; +import io.cdap.plugin.marketo.common.api.job.ActivitiesExportJob; +import io.cdap.plugin.marketo.common.api.job.LeadsExportJob; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.InputStream; import java.util.Collections; import java.util.List; import java.util.Spliterator; @@ -38,6 +45,10 @@ public class Marketo extends MarketoHttp { private static final Logger LOG = LoggerFactory.getLogger(Marketo.class); static final Gson GSON = new Gson(); + public static final List ACTIVITY_FIELDS = ImmutableList.of("marketoGUID", "leadId", "activityDate", + "activityTypeId", "campaignId", + "primaryAttributeValueId", + "primaryAttributeValue", "attributes"); public Marketo(String marketoEndpoint, String clientId, String clientSecret) { super(marketoEndpoint, clientId, clientSecret); @@ -49,9 +60,15 @@ public List describeLeads() { Spliterator.ORDERED), false).collect(Collectors.toList()); } + public List describeBuildInActivities() { + return StreamSupport.stream(Spliterators.spliteratorUnknownSize( + iteratePage(Urls.BUILD_IN_ACTIVITIES_TYPES, ActivityTypeResponse.class, ActivityTypeResponse::getResult), + Spliterator.ORDERED), false).collect(Collectors.toList()); + } + public LeadsExportJob exportLeads(LeadsExportRequest request) { LeadsExport export = validatedPost(Urls.BULK_EXPORT_LEADS_CREATE, Collections.emptyMap(), - inputStream -> Helpers.streamToObject(inputStream, LeadsExport.class), + Marketo::streamToLeadsExport, request, GSON::toJson); return new LeadsExportJob(export.singleExport(), this); @@ -60,15 +77,30 @@ public LeadsExportJob exportLeads(LeadsExportRequest request) { public LeadsExport.ExportResponse leadsExportJobStatus(String jobId) { LeadsExport currentResp = validatedGet( String.format(Urls.BULK_EXPORT_LEADS_STATUS, jobId), - Collections.emptyMap(), inputStream -> Helpers.streamToObject(inputStream, LeadsExport.class)); + Collections.emptyMap(), Marketo::streamToLeadsExport); + return currentResp.singleExport(); + } + + public ActivitiesExportJob exportActivities(ActivitiesExportRequest request) { + ActivitiesExport export = validatedPost(Urls.BULK_EXPORT_ACTIVITIES_CREATE, Collections.emptyMap(), + Marketo::streamToActivitiesExport, + request, + GSON::toJson); + return new ActivitiesExportJob(export.singleExport(), this); + } + + public ActivitiesExport.ExportResponse activitiesExportJobStatus(String jobId) { + ActivitiesExport currentResp = validatedGet( + String.format(Urls.BULK_EXPORT_ACTIVITIES_STATUS, jobId), + Collections.emptyMap(), Marketo::streamToActivitiesExport); return currentResp.singleExport(); } /** * Waits until bulk extract queue has available slot and executes given action. * - * @param action action to execute once slot is available - * @param timeoutSeconds timeout, in seconds + * @param action action to execute once slot is available + * @param timeoutSeconds timeout in seconds */ public void onBulkExtractQueueAvailable(Runnable action, long timeoutSeconds) { long timeoutMillis = TimeUnit.SECONDS.toMillis(timeoutSeconds); @@ -88,14 +120,30 @@ public void onBulkExtractQueueAvailable(Runnable action, long timeoutSeconds) { throw new RuntimeException("Failed to get slot in bulk export queue - timeout"); } + public static LeadsExport streamToLeadsExport(InputStream inputStream) { + return Helpers.streamToObject(inputStream, LeadsExport.class); + } + + public static ActivitiesExport streamToActivitiesExport(InputStream inputStream) { + return Helpers.streamToObject(inputStream, ActivitiesExport.class); + } + private boolean canEnqueueJob() { - LeadsExport exportJobs = validatedGet(Urls.BULK_EXPORT_LEADS_LIST, - ImmutableMap.of("status", "queued,processing"), - inputStream -> Helpers.streamToObject(inputStream, LeadsExport.class) + LeadsExport leadsExportJobs = validatedGet(Urls.BULK_EXPORT_LEADS_LIST, + ImmutableMap.of("status", "queued,processing"), + Marketo::streamToLeadsExport ); - int jobsInQueue = exportJobs.getResult().size(); + + int jobsInQueue = leadsExportJobs.getResult().size(); + + ActivitiesExport activitiesExportJobs = validatedGet(Urls.BULK_EXPORT_ACTIVITIES_LIST, + ImmutableMap.of("status", "queued,processing"), + Marketo::streamToActivitiesExport + ); + jobsInQueue += activitiesExportJobs.getResult().size(); + LOG.debug("Jobs in queue: {}", jobsInQueue); - // TODO handle activity queue here + return jobsInQueue < 10; } } diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/MarketoHttp.java b/src/main/java/io/cdap/plugin/marketo/common/api/MarketoHttp.java index f71cd77..bb94064 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/MarketoHttp.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/MarketoHttp.java @@ -81,7 +81,7 @@ T validatedGet(String queryUrl, Map par }); } - T validatedPost(String queryUrl, Map parameters, + public T validatedPost(String queryUrl, Map parameters, Function deserializer, B body, Function qSerializer) { String logUri = "POST " + buildUri(queryUrl, parameters, false).toString(); @@ -128,7 +128,7 @@ private T retryableValidate(String logUri, Supplier return result; } - T get(URI uri, Function deserializer) { + public T get(URI uri, Function deserializer) { try (CloseableHttpClient httpClient = HttpClients.createDefault()) { HttpGet request = new HttpGet(uri); try (CloseableHttpResponse response = httpClient.execute(request, httpClientContext)) { @@ -157,7 +157,7 @@ private T post(URI uri, Function respDeserializer, B body } } - URI buildUri(String queryUrl, Map parameters) { + public URI buildUri(String queryUrl, Map parameters) { return buildUri(queryUrl, parameters, true); } diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/Urls.java b/src/main/java/io/cdap/plugin/marketo/common/api/Urls.java index 2db9085..32478d3 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/Urls.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/Urls.java @@ -4,10 +4,16 @@ * Marketo API urls. */ public class Urls { + public static final String LEADS_DESCRIBE = "/rest/v1/leads/describe.json"; public static final String BULK_EXPORT_LEADS_LIST = "/bulk/v1/leads/export.json"; public static final String BULK_EXPORT_LEADS_CREATE = "/bulk/v1/leads/export/create.json"; public static final String BULK_EXPORT_LEADS_ENQUEUE = "/bulk/v1/leads/export/%s/enqueue.json"; public static final String BULK_EXPORT_LEADS_STATUS = "/bulk/v1/leads/export/%s/status.json"; public static final String BULK_EXPORT_LEADS_FILE = "/bulk/v1/leads/export/%s/file.json"; - public static final String LEADS_DESCRIBE = "/rest/v1/leads/describe.json"; + public static final String BULK_EXPORT_ACTIVITIES_LIST = "/bulk/v1/activities/export.json"; + public static final String BULK_EXPORT_ACTIVITIES_CREATE = "/bulk/v1/activities/export/create.json"; + public static final String BULK_EXPORT_ACTIVITIES_ENQUEUE = "/bulk/v1/activities/export/%s/enqueue.json"; + public static final String BULK_EXPORT_ACTIVITIES_STATUS = "/bulk/v1/activities/export/%s/status.json"; + public static final String BULK_EXPORT_ACTIVITIES_FILE = "/bulk/v1/activities/export/%s/file.json"; + public static final String BUILD_IN_ACTIVITIES_TYPES = "/rest/v1/activities/types.json"; } diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/activities/ActivitiesExport.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/activities/ActivitiesExport.java new file mode 100644 index 0000000..d8f92aa --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/activities/ActivitiesExport.java @@ -0,0 +1,86 @@ +package io.cdap.plugin.marketo.common.api.entities.activities; + +import io.cdap.plugin.marketo.common.api.entities.BaseResponse; + +import java.util.Collections; +import java.util.List; + +/** + * Represents activities bulk export response. + */ +public class ActivitiesExport extends BaseResponse { + /** + * Represents export response item. + */ + public static class ExportResponse { + String createdAt; + String errorMsg; + String exportId; + int fileSize; + String fileChecksum; + String finishedAt; + String format; + int numberOfRecords; + String queuedAt; + String startedAt; + String status; + + public String getCreatedAt() { + return createdAt; + } + + public String getErrorMsg() { + return errorMsg; + } + + public String getExportId() { + return exportId; + } + + public int getFileSize() { + return fileSize; + } + + public String getFileChecksum() { + return fileChecksum; + } + + public String getFinishedAt() { + return finishedAt; + } + + public String getFormat() { + return format; + } + + public int getNumberOfRecords() { + return numberOfRecords; + } + + public String getQueuedAt() { + return queuedAt; + } + + public String getStartedAt() { + return startedAt; + } + + public String getStatus() { + return status; + } + } + + List result = Collections.emptyList(); + + public ExportResponse singleExport() { + if (result.size() != 1) { + throw new IllegalStateException("Expected single export job result."); + } + return result.get(0); + } + + public List getResult() { + return result; + } + +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/activities/ActivitiesExportRequest.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/activities/ActivitiesExportRequest.java new file mode 100644 index 0000000..bb05924 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/activities/ActivitiesExportRequest.java @@ -0,0 +1,20 @@ +package io.cdap.plugin.marketo.common.api.entities.activities; + +import java.util.List; +import java.util.Map; + +/** + * Represents activities bulk export request. + */ +public class ActivitiesExportRequest { + + Map columnHeaderNames = null; + List fields = null; + ExportActivityFilter filter = null; + String format = "CSV"; + + public ActivitiesExportRequest(List fields, ExportActivityFilter filter) { + this.fields = fields; + this.filter = filter; + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/activities/ActivityTypeResponse.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/activities/ActivityTypeResponse.java new file mode 100644 index 0000000..dc73a2a --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/activities/ActivityTypeResponse.java @@ -0,0 +1,79 @@ +package io.cdap.plugin.marketo.common.api.entities.activities; + +import io.cdap.plugin.marketo.common.api.entities.BaseResponse; + +import java.util.Collections; +import java.util.List; + +/** + * Activity type response. + */ +public class ActivityTypeResponse extends BaseResponse { + /** + * Activity type attribute. + */ + public static class ActivityTypeAttribute { + private String apiName; + private String dataType; + private String name; + + public String getApiName() { + return apiName; + } + + public String getDataType() { + return dataType; + } + + public String getName() { + return name; + } + + @Override + public String toString() { + return String.format("%s(%s, %s)", getApiName(), getDataType(), getName()); + } + } + + /** + * Attribute type. + */ + public static class ActivityType { + private String apiName; + private List attributes; + private String description; + private Integer id; + private String name; + private ActivityTypeAttribute primaryAttribute; + + public String getApiName() { + return apiName; + } + + public List getAttributes() { + return attributes; + } + + public String getDescription() { + return description; + } + + public Integer getId() { + return id; + } + + public String getName() { + return name; + } + + public ActivityTypeAttribute getPrimaryAttribute() { + return primaryAttribute; + } + } + + List result = Collections.emptyList(); + + public List getResult() { + return result; + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/activities/ExportActivityFilter.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/activities/ExportActivityFilter.java new file mode 100644 index 0000000..6f1b071 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/activities/ExportActivityFilter.java @@ -0,0 +1,59 @@ +package io.cdap.plugin.marketo.common.api.entities.activities; + +import io.cdap.plugin.marketo.common.api.entities.DateRange; + +import java.util.List; + +/** + * Represents request filter. + */ +public class ExportActivityFilter { + private List activityTypeIds; + private DateRange createdAt; + + /** + * Builder. + */ + public static class Builder { + + private List activityTypeIds = null; + private DateRange createdAt = null; + + public Builder() { + } + + Builder(List activityTypeIds, DateRange createdAt) { + this.activityTypeIds = activityTypeIds; + this.createdAt = createdAt; + } + + public Builder activityTypeIds(List activityTypeIds) { + this.activityTypeIds = activityTypeIds; + return Builder.this; + } + + public Builder addActivityTypeIds(Integer activityTypeIds) { + this.activityTypeIds.add(activityTypeIds); + return Builder.this; + } + + public Builder createdAt(DateRange createdAt) { + this.createdAt = createdAt; + return Builder.this; + } + + public ExportActivityFilter build() { + + return new ExportActivityFilter(this); + } + } + + private ExportActivityFilter(Builder builder) { + this.activityTypeIds = builder.activityTypeIds; + this.createdAt = builder.createdAt; + } + + public static Builder builder() { + return new Builder(); + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/job/AbstractBulkExportJob.java b/src/main/java/io/cdap/plugin/marketo/common/api/job/AbstractBulkExportJob.java new file mode 100644 index 0000000..05e5a02 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/job/AbstractBulkExportJob.java @@ -0,0 +1,116 @@ +package io.cdap.plugin.marketo.common.api.job; + +import io.cdap.plugin.marketo.common.api.Helpers; +import io.cdap.plugin.marketo.common.api.Marketo; +import org.slf4j.Logger; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.TimeUnit; + +/** + * Base bulk export job wrapper. + * + * @param object that represents job status + */ +public abstract class AbstractBulkExportJob { + private static final List WAIT_ABLE_STATE = Arrays.asList("Queued", "Processing"); + private static final String ENQUEUE_ABLE_STATUS = "Created"; + private static final String COMPLETED_STATUS = "Completed"; + + private String jobId; + private T lastState; + private Marketo marketo; + + /** + * @param jobId job id + * @param lastState last status + * @param marketo marketo api instance + */ + public AbstractBulkExportJob(String jobId, + T lastState, + Marketo marketo) { + + this.jobId = jobId; + this.lastState = lastState; + this.marketo = marketo; + getLogger().info("{} - created job '{}'", getLogPrefix(), this.jobId); + } + + public abstract Logger getLogger(); + + public abstract T getFreshState(); + + public abstract String getStateStatus(T state); + + public abstract String getFileUrlTemplate(); + + public abstract String getLogPrefix(); + + protected abstract T enqueueImpl(); + + public T getLastState() { + return lastState; + } + + public Marketo getMarketo() { + return marketo; + } + + public void waitCompletion() { + if (!WAIT_ABLE_STATE.contains(getStateStatus(getLastState()))) { + throw new IllegalStateException("Job must be enqueued before waiting for completion."); + } + + do { + try { + Thread.sleep(TimeUnit.SECONDS.toMillis(30)); + } catch (InterruptedException e) { + throw new IllegalStateException("Failed to wait for job completion - interrupted"); + } + + T newState = getFreshState(); + logStatusChange(getStateStatus(getLastState()), getStateStatus(newState)); + lastState = newState; + } while (WAIT_ABLE_STATE.contains(getStateStatus(getLastState()))); + + if (!getStateStatus(getLastState()).equals(COMPLETED_STATUS)) { + throw new IllegalStateException("Job expected to be in Completed state, but was in " + + getStateStatus(getLastState())); + } + } + + public void enqueue() { + if (!getStateStatus(getLastState()).equals(ENQUEUE_ABLE_STATUS)) { + throw new IllegalStateException("Job must be in Created status before enqueuing, but was in " + + getStateStatus(getLastState())); + } + + T newState = enqueueImpl(); + + logStatusChange(getStateStatus(getLastState()), getStateStatus(newState)); + + if (!(getStateStatus(newState).equals("Queued") || getStateStatus(newState).equals("Processing"))) { + throw new IllegalStateException( + String.format("Expected Queued|Processing state for job '%s' but got '%s'", jobId, getStateStatus(newState))); + } + + lastState = newState; + } + + private void logStatusChange(String oldStatus, String newStatus) { + if (!oldStatus.equals(newStatus)) { + getLogger().info("{} - job '{}' changed state '{}' -> '{}'", getLogPrefix(), jobId, oldStatus, newStatus); + } + } + + public String getFile() { + return marketo.get(marketo.buildUri(String.format(getFileUrlTemplate(), jobId), Collections.emptyMap()), + Helpers::streamToString); + } + + public String getJobId() { + return jobId; + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/job/ActivitiesExportJob.java b/src/main/java/io/cdap/plugin/marketo/common/api/job/ActivitiesExportJob.java new file mode 100644 index 0000000..de26917 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/job/ActivitiesExportJob.java @@ -0,0 +1,55 @@ +package io.cdap.plugin.marketo.common.api.job; + +import io.cdap.plugin.marketo.common.api.Helpers; +import io.cdap.plugin.marketo.common.api.Marketo; +import io.cdap.plugin.marketo.common.api.Urls; +import io.cdap.plugin.marketo.common.api.entities.activities.ActivitiesExport; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Collections; + +/** + * Activities export job. + */ +public class ActivitiesExportJob extends AbstractBulkExportJob { + private static final Logger LOG = LoggerFactory.getLogger(ActivitiesExportJob.class); + + public ActivitiesExportJob(ActivitiesExport.ExportResponse lastState, Marketo marketo) { + super(lastState.getExportId(), lastState, marketo); + } + + @Override + public Logger getLogger() { + return LOG; + } + + @Override + public ActivitiesExport.ExportResponse getFreshState() { + return getMarketo().activitiesExportJobStatus(getJobId()); + } + + @Override + public String getStateStatus(ActivitiesExport.ExportResponse state) { + return state.getStatus(); + } + + @Override + public String getFileUrlTemplate() { + return Urls.BULK_EXPORT_ACTIVITIES_FILE; + } + + @Override + public String getLogPrefix() { + return "BULK ACTIVITIES EXPORT"; + } + + @Override + protected ActivitiesExport.ExportResponse enqueueImpl() { + return getMarketo().validatedPost( + String.format(Urls.BULK_EXPORT_ACTIVITIES_ENQUEUE, getJobId()), + Collections.emptyMap(), + inputStream -> Helpers.streamToObject(inputStream, ActivitiesExport.class), + null, null).singleExport(); + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/job/LeadsExportJob.java b/src/main/java/io/cdap/plugin/marketo/common/api/job/LeadsExportJob.java new file mode 100644 index 0000000..e0bf289 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/job/LeadsExportJob.java @@ -0,0 +1,55 @@ +package io.cdap.plugin.marketo.common.api.job; + +import io.cdap.plugin.marketo.common.api.Helpers; +import io.cdap.plugin.marketo.common.api.Marketo; +import io.cdap.plugin.marketo.common.api.Urls; +import io.cdap.plugin.marketo.common.api.entities.leads.LeadsExport; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Collections; + +/** + * Leads export job. + */ +public class LeadsExportJob extends AbstractBulkExportJob { + private static final Logger LOG = LoggerFactory.getLogger(LeadsExportJob.class); + + public LeadsExportJob(LeadsExport.ExportResponse lastState, Marketo marketo) { + super(lastState.getExportId(), lastState, marketo); + } + + @Override + public Logger getLogger() { + return LOG; + } + + @Override + public LeadsExport.ExportResponse getFreshState() { + return getMarketo().leadsExportJobStatus(getJobId()); + } + + @Override + public String getStateStatus(LeadsExport.ExportResponse state) { + return state.getStatus(); + } + + @Override + public String getFileUrlTemplate() { + return Urls.BULK_EXPORT_LEADS_FILE; + } + + @Override + public String getLogPrefix() { + return "BULK LEADS EXPORT"; + } + + @Override + protected LeadsExport.ExportResponse enqueueImpl() { + return getMarketo().validatedPost( + String.format(Urls.BULK_EXPORT_LEADS_ENQUEUE, getJobId()), + Collections.emptyMap(), + inputStream -> Helpers.streamToObject(inputStream, LeadsExport.class), + null, null).singleExport(); + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoRecordReader.java b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoRecordReader.java index 48e49e2..30f9b0e 100644 --- a/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoRecordReader.java +++ b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoRecordReader.java @@ -19,10 +19,12 @@ import com.google.gson.Gson; import com.google.gson.GsonBuilder; import io.cdap.cdap.api.data.schema.Schema; -import io.cdap.plugin.marketo.common.api.LeadsExportJob; import io.cdap.plugin.marketo.common.api.Marketo; import io.cdap.plugin.marketo.common.api.entities.DateRange; +import io.cdap.plugin.marketo.common.api.entities.activities.ActivitiesExportRequest; +import io.cdap.plugin.marketo.common.api.entities.activities.ExportActivityFilter; import io.cdap.plugin.marketo.common.api.entities.leads.LeadsExportRequest; +import io.cdap.plugin.marketo.common.api.job.AbstractBulkExportJob; import org.apache.commons.csv.CSVFormat; import org.apache.commons.csv.CSVParser; import org.apache.commons.csv.CSVRecord; @@ -65,25 +67,32 @@ public void initialize(InputSplit inputSplit, TaskAttemptContext taskAttemptCont Marketo marketo = config.getMarketo(); - List fields = config.getSchema().getFields().stream() - .map(Schema.Field::getName).collect(Collectors.toList()); DateRange dateRange = new DateRange(beginDate, endDate); - LeadsExportRequest.ExportLeadFilter filter = LeadsExportRequest.ExportLeadFilter.builder() - .createdAt(dateRange).build(); - LeadsExportRequest request = new LeadsExportRequest(fields, filter); + AbstractBulkExportJob job; + switch (config.getReportType()) { + case LEADS: + List leadsFields = config.getSchema().getFields().stream() + .map(Schema.Field::getName).collect(Collectors.toList()); + LeadsExportRequest.ExportLeadFilter leadsFilter = LeadsExportRequest.ExportLeadFilter.builder() + .createdAt(dateRange).build(); + job = marketo.exportLeads(new LeadsExportRequest(leadsFields, leadsFilter)); + break; + case ACTIVITIES: + ExportActivityFilter activitiesFilter = ExportActivityFilter.builder() + .createdAt(dateRange).build(); + job = marketo.exportActivities(new ActivitiesExportRequest(Marketo.ACTIVITY_FIELDS, activitiesFilter)); + break; + default: + throw new IllegalArgumentException("Invalid report type " + config.getReportType()); + } - LeadsExportJob job = marketo.exportLeads(request); - LOG.info("Bulk leads export job with id '{}' has date range {}", job.getJobId(), dateRange); + LOG.info("BULK EXPORT JOB - job '{}' has date range '{}'", job.getJobId(), dateRange); // TODO handle possible concurrent issues here, another mapper can take our slot // wait for 10 minutes for available slot and enqueue job marketo.onBulkExtractQueueAvailable(job::enqueue, 60 * 10); - try { - job.waitCompletion(); - } catch (InterruptedException ex) { - throw new RuntimeException(ex); - } + job.waitCompletion(); String data = job.getFile(); // TODO stream here diff --git a/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoReportingPlugin.java b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoReportingPlugin.java index e7949e7..875c15a 100644 --- a/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoReportingPlugin.java +++ b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoReportingPlugin.java @@ -72,16 +72,7 @@ public void prepareRun(BatchSourceContext batchSourceContext) { @Override public void transform(KeyValue> input, Emitter emitter) { - StructuredRecord.Builder builder = StructuredRecord.builder(config.getSchema()); - Map inputMap = input.getValue(); - config.getSchema().getFields().forEach( - field -> { - if (inputMap.containsKey(field.getName())) { - builder.set(field.getName(), String.valueOf(inputMap.remove(field.getName()))); - } - } - ); - emitter.emit(builder.build()); + emitter.emit(MarketoReportingSchemaHelper.getRecord(config.getSchema(), input.getValue())); } private void validateConfiguration(FailureCollector failureCollector) { diff --git a/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoReportingSchemaHelper.java b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoReportingSchemaHelper.java new file mode 100644 index 0000000..1edef60 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoReportingSchemaHelper.java @@ -0,0 +1,55 @@ +package io.cdap.plugin.marketo.source.batch; + +import io.cdap.cdap.api.data.format.StructuredRecord; +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.plugin.marketo.common.api.Marketo; + +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; + +/** + * Various methods to deal with schema and record. + */ +public class MarketoReportingSchemaHelper { + public static Schema getActivitySchema() { + List fields = Marketo.ACTIVITY_FIELDS.stream() + .map(s -> Schema.Field.of(s, Schema.nullableOf(Schema.of(Schema.Type.STRING)))) + .collect(Collectors.toList()); + return Schema.recordOf("activityRecord", fields); + } + + public static Schema getSchema(MarketoReportingSourceConfig config) { + switch (config.getReportType()) { + case LEADS: + List fields = config.getMarketo().describeLeads().stream().map( + leadAttribute -> { + if (leadAttribute.getRest() != null) { + return Schema.Field.of(leadAttribute.getRest().getName(), + Schema.nullableOf(Schema.of(Schema.Type.STRING))); + } else { + return null; + } + } + ).filter(Objects::nonNull).collect(Collectors.toList()); + + return Schema.recordOf("LeadsRecord", fields); + case ACTIVITIES: + return getActivitySchema(); + } + throw new IllegalArgumentException("Failed to get schema for type " + config.getReportType()); + } + + public static StructuredRecord getRecord(Schema schema, Map fields) { + StructuredRecord.Builder builder = StructuredRecord.builder(schema); + schema.getFields().forEach( + field -> { + if (fields.containsKey(field.getName())) { + builder.set(field.getName(), fields.get(field.getName())); + } + } + ); + return builder.build(); + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoReportingSourceConfig.java b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoReportingSourceConfig.java index c9cc414..8377cd0 100644 --- a/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoReportingSourceConfig.java +++ b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoReportingSourceConfig.java @@ -24,10 +24,6 @@ import io.cdap.plugin.marketo.common.api.Marketo; import io.cdap.plugin.marketo.common.api.entities.MarketoToken; -import java.util.List; -import java.util.Objects; -import java.util.stream.Collectors; - /** * Provides all required configuration for reading Marketo entities. */ @@ -38,6 +34,7 @@ public class MarketoReportingSourceConfig extends ReferencePluginConfig { public static final String PROPERTY_REST_API_ENDPOINT = "restApiEndpoint"; public static final String PROPERTY_REST_API_IDENTITY = "restApiIdentity"; public static final String PROPERTY_DAILY_API_LIMIT = "dailyApiLimit"; + public static final String PROPERTY_REPORT_TYPE = "reportType"; public static final String PROPERTY_REPORT_FORMAT = "reportFormat"; public static final String PROPERTY_START_DATE = "startDate"; public static final String PROPERTY_END_DATE = "endDate"; @@ -72,6 +69,11 @@ public class MarketoReportingSourceConfig extends ReferencePluginConfig { @Macro protected String dailyApiLimit; + @Name(PROPERTY_REPORT_TYPE) + @Description("Report type format, leads or activities.") + @Macro + protected String reportType; + @Name(PROPERTY_REPORT_FORMAT) @Description("Report format.") @Macro @@ -98,17 +100,7 @@ public MarketoReportingSourceConfig(String referenceName) { public Schema getSchema() { if (schema == null) { - List fields = getMarketo().describeLeads().stream().map( - leadAttribute -> { - if (leadAttribute.getRest() != null) { - return Schema.Field.of(leadAttribute.getRest().getName(), Schema.nullableOf(Schema.of(Schema.Type.STRING))); - } else { - return null; - } - } - ).filter(Objects::nonNull).collect(Collectors.toList()); - - schema = Schema.recordOf("LeadsRecord", fields); + schema = MarketoReportingSchemaHelper.getSchema(this); } return schema; } @@ -152,4 +144,8 @@ public String getStartDate() { public String getEndDate() { return endDate; } + + public ReportType getReportType() { + return ReportType.fromString(reportType); + } } diff --git a/src/main/java/io/cdap/plugin/marketo/source/batch/ReportType.java b/src/main/java/io/cdap/plugin/marketo/source/batch/ReportType.java new file mode 100644 index 0000000..59c2291 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/source/batch/ReportType.java @@ -0,0 +1,24 @@ +package io.cdap.plugin.marketo.source.batch; + +/** + * Represents report type. + */ +public enum ReportType { + LEADS("leads"), + ACTIVITIES("activities"); + + private String type; + + ReportType(String type) { + this.type = type; + } + + public static ReportType fromString(String reportType) { + for (ReportType rt : ReportType.values()) { + if (rt.type.equals(reportType)) { + return rt; + } + } + throw new IllegalArgumentException("unknown report type: " + reportType); + } +} From b7b257c4536d04c7ca0eb301925a88b0d8333384 Mon Sep 17 00:00:00 2001 From: Yevhenii Chekanskyi Date: Thu, 28 Nov 2019 12:24:57 +0200 Subject: [PATCH 07/16] PLUGIN-75 Marketo Plugin - add licence headers. --- .../plugin/marketo/common/api/Helpers.java | 16 ++++++++ .../marketo/common/api/MarketoHttp.java | 16 ++++++++ .../common/api/MarketoPageIterator.java | 16 ++++++++ .../cdap/plugin/marketo/common/api/Urls.java | 16 ++++++++ .../common/api/entities/BaseResponse.java | 16 ++++++++ .../common/api/entities/DateRange.java | 16 ++++++++ .../marketo/common/api/entities/Error.java | 16 ++++++++ .../marketo/common/api/entities/Warning.java | 16 ++++++++ .../entities/activities/ActivitiesExport.java | 16 ++++++++ .../activities/ActivitiesExportRequest.java | 16 ++++++++ .../activities/ActivityTypeResponse.java | 16 ++++++++ .../activities/ExportActivityFilter.java | 16 ++++++++ .../api/entities/leads/LeadsDescribe.java | 16 ++++++++ .../api/entities/leads/LeadsExport.java | 16 ++++++++ .../entities/leads/LeadsExportRequest.java | 16 ++++++++ .../common/api/job/AbstractBulkExportJob.java | 16 ++++++++ .../common/api/job/ActivitiesExportJob.java | 16 ++++++++ .../common/api/job/LeadsExportJob.java | 16 ++++++++ .../batch/MarketoReportingSchemaHelper.java | 16 ++++++++ .../batch/MarketoReportingSourceConfig.java | 38 ------------------- .../marketo/source/batch/ReportType.java | 16 ++++++++ .../marketo/common/api/MarketoHttpTest.java | 16 ++++++++ 22 files changed, 336 insertions(+), 38 deletions(-) diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/Helpers.java b/src/main/java/io/cdap/plugin/marketo/common/api/Helpers.java index 7418bb8..d01e0ee 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/Helpers.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/Helpers.java @@ -1,3 +1,19 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package io.cdap.plugin.marketo.common.api; import com.google.common.base.Strings; diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/MarketoHttp.java b/src/main/java/io/cdap/plugin/marketo/common/api/MarketoHttp.java index bb94064..2ac5e80 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/MarketoHttp.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/MarketoHttp.java @@ -1,3 +1,19 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package io.cdap.plugin.marketo.common.api; import com.google.common.base.Strings; diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/MarketoPageIterator.java b/src/main/java/io/cdap/plugin/marketo/common/api/MarketoPageIterator.java index 0c32729..1de07bf 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/MarketoPageIterator.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/MarketoPageIterator.java @@ -1,3 +1,19 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package io.cdap.plugin.marketo.common.api; import io.cdap.plugin.marketo.common.api.entities.BaseResponse; diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/Urls.java b/src/main/java/io/cdap/plugin/marketo/common/api/Urls.java index 32478d3..32479bb 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/Urls.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/Urls.java @@ -1,3 +1,19 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package io.cdap.plugin.marketo.common.api; /** diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/BaseResponse.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/BaseResponse.java index dc3deb0..2f679ca 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/entities/BaseResponse.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/BaseResponse.java @@ -1,3 +1,19 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package io.cdap.plugin.marketo.common.api.entities; import java.util.Collections; diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/DateRange.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/DateRange.java index 3764947..9e71503 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/entities/DateRange.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/DateRange.java @@ -1,3 +1,19 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package io.cdap.plugin.marketo.common.api.entities; /** diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/Error.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/Error.java index 2c9a9e2..594e9cb 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/entities/Error.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/Error.java @@ -1,3 +1,19 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package io.cdap.plugin.marketo.common.api.entities; /** diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/Warning.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/Warning.java index 5722e51..b1167fc 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/entities/Warning.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/Warning.java @@ -1,3 +1,19 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package io.cdap.plugin.marketo.common.api.entities; /** diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/activities/ActivitiesExport.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/activities/ActivitiesExport.java index d8f92aa..6137d52 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/entities/activities/ActivitiesExport.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/activities/ActivitiesExport.java @@ -1,3 +1,19 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package io.cdap.plugin.marketo.common.api.entities.activities; import io.cdap.plugin.marketo.common.api.entities.BaseResponse; diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/activities/ActivitiesExportRequest.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/activities/ActivitiesExportRequest.java index bb05924..31fc681 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/entities/activities/ActivitiesExportRequest.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/activities/ActivitiesExportRequest.java @@ -1,3 +1,19 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package io.cdap.plugin.marketo.common.api.entities.activities; import java.util.List; diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/activities/ActivityTypeResponse.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/activities/ActivityTypeResponse.java index dc73a2a..b853484 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/entities/activities/ActivityTypeResponse.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/activities/ActivityTypeResponse.java @@ -1,3 +1,19 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package io.cdap.plugin.marketo.common.api.entities.activities; import io.cdap.plugin.marketo.common.api.entities.BaseResponse; diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/activities/ExportActivityFilter.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/activities/ExportActivityFilter.java index 6f1b071..5e225cf 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/entities/activities/ExportActivityFilter.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/activities/ExportActivityFilter.java @@ -1,3 +1,19 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package io.cdap.plugin.marketo.common.api.entities.activities; import io.cdap.plugin.marketo.common.api.entities.DateRange; diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/leads/LeadsDescribe.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/leads/LeadsDescribe.java index e1c832b..ba445fa 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/entities/leads/LeadsDescribe.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/leads/LeadsDescribe.java @@ -1,3 +1,19 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package io.cdap.plugin.marketo.common.api.entities.leads; import io.cdap.plugin.marketo.common.api.entities.BaseResponse; diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/leads/LeadsExport.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/leads/LeadsExport.java index 65f6312..09d9cd4 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/entities/leads/LeadsExport.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/leads/LeadsExport.java @@ -1,3 +1,19 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package io.cdap.plugin.marketo.common.api.entities.leads; import io.cdap.plugin.marketo.common.api.entities.BaseResponse; diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/leads/LeadsExportRequest.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/leads/LeadsExportRequest.java index 572072e..e2035ba 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/entities/leads/LeadsExportRequest.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/leads/LeadsExportRequest.java @@ -1,3 +1,19 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package io.cdap.plugin.marketo.common.api.entities.leads; import io.cdap.plugin.marketo.common.api.entities.DateRange; diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/job/AbstractBulkExportJob.java b/src/main/java/io/cdap/plugin/marketo/common/api/job/AbstractBulkExportJob.java index 05e5a02..f58109a 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/job/AbstractBulkExportJob.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/job/AbstractBulkExportJob.java @@ -1,3 +1,19 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package io.cdap.plugin.marketo.common.api.job; import io.cdap.plugin.marketo.common.api.Helpers; diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/job/ActivitiesExportJob.java b/src/main/java/io/cdap/plugin/marketo/common/api/job/ActivitiesExportJob.java index de26917..db5cdf3 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/job/ActivitiesExportJob.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/job/ActivitiesExportJob.java @@ -1,3 +1,19 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package io.cdap.plugin.marketo.common.api.job; import io.cdap.plugin.marketo.common.api.Helpers; diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/job/LeadsExportJob.java b/src/main/java/io/cdap/plugin/marketo/common/api/job/LeadsExportJob.java index e0bf289..740a9d7 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/job/LeadsExportJob.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/job/LeadsExportJob.java @@ -1,3 +1,19 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package io.cdap.plugin.marketo.common.api.job; import io.cdap.plugin.marketo.common.api.Helpers; diff --git a/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoReportingSchemaHelper.java b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoReportingSchemaHelper.java index 1edef60..0a578fd 100644 --- a/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoReportingSchemaHelper.java +++ b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoReportingSchemaHelper.java @@ -1,3 +1,19 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package io.cdap.plugin.marketo.source.batch; import io.cdap.cdap.api.data.format.StructuredRecord; diff --git a/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoReportingSourceConfig.java b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoReportingSourceConfig.java index 8377cd0..e1460bd 100644 --- a/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoReportingSourceConfig.java +++ b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoReportingSourceConfig.java @@ -22,28 +22,18 @@ import io.cdap.cdap.api.data.schema.Schema; import io.cdap.plugin.common.ReferencePluginConfig; import io.cdap.plugin.marketo.common.api.Marketo; -import io.cdap.plugin.marketo.common.api.entities.MarketoToken; /** * Provides all required configuration for reading Marketo entities. */ public class MarketoReportingSourceConfig extends ReferencePluginConfig { - public static final String PROPERTY_ENTITY_NAME = "entityName"; public static final String PROPERTY_CLIENT_ID = "clientId"; public static final String PROPERTY_CLIENT_SECRET = "clientSecret"; public static final String PROPERTY_REST_API_ENDPOINT = "restApiEndpoint"; - public static final String PROPERTY_REST_API_IDENTITY = "restApiIdentity"; - public static final String PROPERTY_DAILY_API_LIMIT = "dailyApiLimit"; public static final String PROPERTY_REPORT_TYPE = "reportType"; - public static final String PROPERTY_REPORT_FORMAT = "reportFormat"; public static final String PROPERTY_START_DATE = "startDate"; public static final String PROPERTY_END_DATE = "endDate"; - @Name(PROPERTY_ENTITY_NAME) - @Description("Marketo entity name to fetch.") - @Macro - protected String entityName; - @Name(PROPERTY_CLIENT_ID) @Description("Marketo Client ID.") @Macro @@ -59,26 +49,11 @@ public class MarketoReportingSourceConfig extends ReferencePluginConfig { @Macro protected String restApiEndpoint; - @Name(PROPERTY_REST_API_IDENTITY) - @Description("REST API identity.") - @Macro - protected String restApiIdentity; - - @Name(PROPERTY_DAILY_API_LIMIT) - @Description("Marketo enforced daily API limit.") - @Macro - protected String dailyApiLimit; - @Name(PROPERTY_REPORT_TYPE) @Description("Report type format, leads or activities.") @Macro protected String reportType; - @Name(PROPERTY_REPORT_FORMAT) - @Description("Report format.") - @Macro - protected String reportFormat; - @Name(PROPERTY_START_DATE) @Description("Start date for the report.") @Macro @@ -90,7 +65,6 @@ public class MarketoReportingSourceConfig extends ReferencePluginConfig { protected String endDate; - private transient MarketoToken token = null; private transient Schema schema = null; private transient Marketo marketo = null; @@ -125,18 +99,6 @@ public String getRestApiEndpoint() { return restApiEndpoint; } - public String getRestApiIdentity() { - return restApiIdentity; - } - - public String getDailyApiLimit() { - return dailyApiLimit; - } - - public String getReportFormat() { - return reportFormat; - } - public String getStartDate() { return startDate; } diff --git a/src/main/java/io/cdap/plugin/marketo/source/batch/ReportType.java b/src/main/java/io/cdap/plugin/marketo/source/batch/ReportType.java index 59c2291..1f19f6a 100644 --- a/src/main/java/io/cdap/plugin/marketo/source/batch/ReportType.java +++ b/src/main/java/io/cdap/plugin/marketo/source/batch/ReportType.java @@ -1,3 +1,19 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package io.cdap.plugin.marketo.source.batch; /** diff --git a/src/test/java/io/cdap/plugin/marketo/common/api/MarketoHttpTest.java b/src/test/java/io/cdap/plugin/marketo/common/api/MarketoHttpTest.java index 3ad8844..80c7071 100644 --- a/src/test/java/io/cdap/plugin/marketo/common/api/MarketoHttpTest.java +++ b/src/test/java/io/cdap/plugin/marketo/common/api/MarketoHttpTest.java @@ -1,3 +1,19 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package io.cdap.plugin.marketo.common.api; import com.github.tomakehurst.wiremock.client.WireMock; From e90db264ed36e6bdfc59b5b8e8029794bf9fc2c5 Mon Sep 17 00:00:00 2001 From: Yevhenii Chekanskyi Date: Thu, 28 Nov 2019 12:49:59 +0200 Subject: [PATCH 08/16] PLUGIN-75 Marketo Plugin - properties and docs.. --- docs/MarketoReportingPlugin-batchsource.md | 26 ++++++++++ icons/MarketoReportingPlugin-batchsource.png | Bin 0 -> 996 bytes .../MarketoReportingPlugin-batchsource.json | 49 ++++++++++++++++-- 3 files changed, 72 insertions(+), 3 deletions(-) create mode 100644 docs/MarketoReportingPlugin-batchsource.md create mode 100644 icons/MarketoReportingPlugin-batchsource.png diff --git a/docs/MarketoReportingPlugin-batchsource.md b/docs/MarketoReportingPlugin-batchsource.md new file mode 100644 index 0000000..68bca09 --- /dev/null +++ b/docs/MarketoReportingPlugin-batchsource.md @@ -0,0 +1,26 @@ +# Marketo Reporting batch source + +Description +----------- +This plugin used to query Leads or Activities entities for specified date range from Marketo. + +Properties +---------- +### General + +**Reference Name:** Name used to uniquely identify this source for lineage, annotating metadata, etc. + +**Rest API endpoint:** Marketo rest API endpoint, unique for each client. +### Authentication + +**Client ID:** Client ID. + +**Client Secret:** Client secret. + +### Report + +**Report Type:** Type of report, leads or activities. + +**Start Date:** Start date of report, in ISO 8601 format(1997-07-16T19:20:30+01:00) + +**End Date:** End date of report, in ISO 8601 format(1997-07-16T19:20:30+01:00) \ No newline at end of file diff --git a/icons/MarketoReportingPlugin-batchsource.png b/icons/MarketoReportingPlugin-batchsource.png new file mode 100644 index 0000000000000000000000000000000000000000..4abe7e74fa2422e7bba39b77d633ece4ccce275f GIT binary patch literal 996 zcmVPx#1ZP1_K>z@;j|==^1poj532;bRa{vGi!vFvd!vV){sAK>D19?eAK~!i%?U`Rl zQ&Akp&$-)_Ec-(dC7B6E_0)srgAronLv9*^270gIw$hNEq8BNuy-4&Jh!wh_?qXya z=w=B~4l*dRhh8!a0yC_vh{`|vcYpn+C*fwB(_Qr}_QCDm^F5bwKKK02xxytpAQjp3 zq|&ynthpovc3fR36~PyLE1Db^rNXwgEL{|Qh#-`bR7!}ABn5^=lfAYk`-XvaMFlq^ zb5OdS5_)}do8gscY$;{wf@mTl6}M%Um*3vNn%Y3=UP^^7EVqHmV#h(2hDDnhQ35qd zQxoHo$?*uw;%tS{zMJ*L43=A-0OSe&c2`wV>pm9G1Fwr6(bUpFiBQj)A%Fn90YY5> zwCBUbyQdzHEx`KbAsHQwIFdTn3=0tA$8QJm`!36Ur^i@4!*Z<=Eb~xlkaL0np7hj= z&p24Q^60`P5(&KP2K?a~YDgNwQchy=3@fzQ=_VXSrJy6_EQk@hKKc8!YFAT~1vf>NmeNjEy()~l?WJJcdN@&gCoglA>m0Hnz- zOODfRsc&~%AGUhT)3X=xiAqGiS2c!%uXDQ0ckeu~jJ$65Kn*xa~6^au@bx1lE6}G%=)$K$gQ4JpI8(T?nF>v8b8&zQJk@P z3c|jd{~NAEP+&w*ltU=d%A%;@NCf(WLNK@e{>c|Ch(9D_%l`cQ$1%M6Cge97Rx1D) SQdJ880000 Date: Thu, 28 Nov 2019 13:08:33 +0200 Subject: [PATCH 09/16] PLUGIN-75 Marketo Plugin - config validation. --- .../source/batch/MarketoReportingPlugin.java | 4 +- .../batch/MarketoReportingSourceConfig.java | 74 +++++++++++++++++++ .../MarketoReportingPlugin-batchsource.json | 2 +- 3 files changed, 78 insertions(+), 2 deletions(-) diff --git a/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoReportingPlugin.java b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoReportingPlugin.java index 875c15a..82b71b8 100644 --- a/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoReportingPlugin.java +++ b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoReportingPlugin.java @@ -28,6 +28,7 @@ import io.cdap.cdap.etl.api.PipelineConfigurer; import io.cdap.cdap.etl.api.batch.BatchSource; import io.cdap.cdap.etl.api.batch.BatchSourceContext; +import io.cdap.plugin.common.IdUtils; import io.cdap.plugin.common.LineageRecorder; import org.apache.hadoop.io.NullWritable; @@ -48,7 +49,6 @@ public class MarketoReportingPlugin extends BatchSource> input, Emitter } private void validateConfiguration(FailureCollector failureCollector) { + IdUtils.validateReferenceName(config.referenceName, failureCollector); + config.validate(failureCollector); failureCollector.getOrThrowException(); } } diff --git a/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoReportingSourceConfig.java b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoReportingSourceConfig.java index e1460bd..50b98c3 100644 --- a/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoReportingSourceConfig.java +++ b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoReportingSourceConfig.java @@ -16,13 +16,20 @@ package io.cdap.plugin.marketo.source.batch; +import com.google.common.base.Strings; import io.cdap.cdap.api.annotation.Description; import io.cdap.cdap.api.annotation.Macro; import io.cdap.cdap.api.annotation.Name; import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.cdap.etl.api.FailureCollector; import io.cdap.plugin.common.ReferencePluginConfig; +import io.cdap.plugin.marketo.common.api.Helpers; import io.cdap.plugin.marketo.common.api.Marketo; +import java.net.MalformedURLException; +import java.net.URL; +import java.time.format.DateTimeParseException; + /** * Provides all required configuration for reading Marketo entities. */ @@ -110,4 +117,71 @@ public String getEndDate() { public ReportType getReportType() { return ReportType.fromString(reportType); } + + void validate(FailureCollector failureCollector) { + validateDate(failureCollector); + validateReportType(failureCollector); + validateMarketoEndpoint(failureCollector); + validateSecrets(failureCollector); + } + + void validateDate(FailureCollector failureCollector) { + if (!(containsMacro(PROPERTY_START_DATE) && containsMacro(PROPERTY_END_DATE))) { + try { + Helpers.getDateRanges(getStartDate(), getEndDate()); + } catch (IllegalArgumentException ex) { + String message = String.format("Failed to validate dates: %s.", ex.getMessage()); + String correctiveAction = null; + if (ex.getMessage().contains("start date more than end date")) { + message = "Start date more than end date."; + correctiveAction = "Swap dates."; + } + failureCollector.addFailure(message, correctiveAction) + .withConfigProperty(PROPERTY_START_DATE).withConfigProperty(PROPERTY_END_DATE); + } catch (DateTimeParseException ex) { + failureCollector.addFailure("Failed to parse one of dates.", + "Correct dates to ISO 8601 format.") + .withConfigProperty(PROPERTY_START_DATE).withConfigProperty(PROPERTY_END_DATE); + } + } + } + + void validateReportType(FailureCollector failureCollector) { + if (!containsMacro(PROPERTY_REPORT_TYPE)) { + try { + getReportType(); + } catch (IllegalArgumentException ex) { + failureCollector.addFailure(String.format("Incorrect reporting type '%s'.", reportType), + "Set reporting type to 'activities' or 'leads'.") + .withConfigProperty(PROPERTY_REPORT_TYPE); + } + } + } + + void validateSecrets(FailureCollector failureCollector) { + if (!containsMacro(PROPERTY_CLIENT_ID) && Strings.isNullOrEmpty(getClientId())) { + failureCollector.addFailure("Client ID is null or empty.", + "Set Client ID to non empty string.") + .withConfigProperty(PROPERTY_CLIENT_ID); + } + + if (!containsMacro(PROPERTY_CLIENT_SECRET) && Strings.isNullOrEmpty(getClientSecret())) { + failureCollector.addFailure("Client Secret is null or empty.", + "Set Client Secret to non empty string.") + .withConfigProperty(PROPERTY_CLIENT_SECRET); + } + } + + void validateMarketoEndpoint(FailureCollector failureCollector) { + if (!containsMacro(PROPERTY_REST_API_ENDPOINT)) { + try { + new URL(getRestApiEndpoint()); + } catch (MalformedURLException e) { + failureCollector + .addFailure(String.format("Malformed Marketo Rest API endpoint URL '%s'.", getRestApiEndpoint()), + "Change URL to valid.") + .withConfigProperty(PROPERTY_REST_API_ENDPOINT); + } + } + } } diff --git a/widgets/MarketoReportingPlugin-batchsource.json b/widgets/MarketoReportingPlugin-batchsource.json index 1910011..96bf773 100644 --- a/widgets/MarketoReportingPlugin-batchsource.json +++ b/widgets/MarketoReportingPlugin-batchsource.json @@ -38,7 +38,7 @@ "label": "Report", "properties": [ { - "widget-type": "textbox", + "widget-type": "select", "label": "Report Type", "name": "reportType", "widget-attributes": { From 578b693752157303823690dff5be2c49bc9ee1c6 Mon Sep 17 00:00:00 2001 From: Yevhenii Chekanskyi Date: Thu, 28 Nov 2019 17:03:41 +0200 Subject: [PATCH 10/16] PLUGIN-75 Marketo Plugin - address review comments. --- checkstyle.xml | 25 ++-- docs/MarketoReportingPlugin-batchsource.md | 10 +- pom.xml | 78 +++++++++++- .../plugin/marketo/common/api/Helpers.java | 13 +- .../plugin/marketo/common/api/Marketo.java | 58 ++++----- .../marketo/common/api/MarketoHttp.java | 21 ++-- .../entities/activities/ActivitiesExport.java | 112 +++++++----------- .../activities/ActivitiesExportResponse.java | 43 +++++++ ...scribe.java => LeadsDescribeResponse.java} | 2 +- .../api/entities/leads/LeadsExport.java | 112 +++++++----------- .../entities/leads/LeadsExportResponse.java | 42 +++++++ .../common/api/job/ActivitiesExportJob.java | 13 +- .../common/api/job/LeadsExportJob.java | 13 +- .../source/batch/MarketoRecordReader.java | 13 +- .../source/batch/MarketoReportingPlugin.java | 3 +- .../batch/MarketoReportingSourceConfig.java | 54 +++++---- .../marketo/source/batch/ReportType.java | 2 +- .../MarketoReportingPlugin-batchsource.json | 10 +- 18 files changed, 378 insertions(+), 246 deletions(-) create mode 100644 src/main/java/io/cdap/plugin/marketo/common/api/entities/activities/ActivitiesExportResponse.java rename src/main/java/io/cdap/plugin/marketo/common/api/entities/leads/{LeadsDescribe.java => LeadsDescribeResponse.java} (96%) create mode 100644 src/main/java/io/cdap/plugin/marketo/common/api/entities/leads/LeadsExportResponse.java diff --git a/checkstyle.xml b/checkstyle.xml index fb35f2d..04c6eda 100644 --- a/checkstyle.xml +++ b/checkstyle.xml @@ -16,8 +16,8 @@ --> + "-//Puppy Crawl//DTD Check Configuration 1.3//EN" + "http://www.puppycrawl.com/dtds/configuration_1_3.dtd"> - - - + @@ -114,8 +112,8 @@ page at http://checkstyle.sourceforge.net/config.html --> - - + + --> + value="${com.puppycrawl.tools.checkstyle.checks.sizes.LineLength.ignorePattern}" + default="^(package .*;\s*)|(import .*;\s*)|( *\* *https?://.*)$"/> @@ -298,7 +296,7 @@ page at http://checkstyle.sourceforge.net/config.html --> some other variants which we don't publicized to promote consistency). --> + value="fall through|Fall through|fallthru|Fallthru|falls through|Falls through|fallthrough|Fallthrough|No break|NO break|no break|continue on"/> @@ -396,11 +394,4 @@ page at http://checkstyle.sourceforge.net/config.html --> - - - - - - - diff --git a/docs/MarketoReportingPlugin-batchsource.md b/docs/MarketoReportingPlugin-batchsource.md index 68bca09..5451214 100644 --- a/docs/MarketoReportingPlugin-batchsource.md +++ b/docs/MarketoReportingPlugin-batchsource.md @@ -1,8 +1,8 @@ -# Marketo Reporting batch source +# Marketo Reporting Batch Source Description ----------- -This plugin used to query Leads or Activities entities for specified date range from Marketo. +This plugin is used to query Leads or Activities entities for specified date range from Marketo. Properties ---------- @@ -19,8 +19,8 @@ Properties ### Report -**Report Type:** Type of report, leads or activities. +**Report Type:** Type of report. One of 'leads' or 'activities'. -**Start Date:** Start date of report, in ISO 8601 format(1997-07-16T19:20:30+01:00) +**Start Date:** Start date of report. In ISO 8601 format(1997-07-16T19:20:30+01:00). -**End Date:** End date of report, in ISO 8601 format(1997-07-16T19:20:30+01:00) \ No newline at end of file +**End Date:** End date of report. In ISO 8601 format(1997-07-16T19:20:30+01:00). \ No newline at end of file diff --git a/pom.xml b/pom.xml index e3e9248..e8a095b 100644 --- a/pom.xml +++ b/pom.xml @@ -1,12 +1,27 @@ + 4.0.0 - io.cdap + io.cdap.plugin marketo-entity-plugin - 1.0-SNAPSHOT + 1.0.0-SNAPSHOT @@ -21,7 +36,7 @@ 6.1.0-SNAPSHOT - 2.3.0 + 2.8.0 4.5.9 2.3.0-SNAPSHOT 2.1.3 @@ -281,6 +296,61 @@ + + org.apache.rat + apache-rat-plugin + 0.13 + + + org.apache.maven.doxia + doxia-core + 1.6 + + + xerces + xercesImpl + + + + + + + rat-check + validate + + check + + + + LICENSE*.txt + + *.rst + *.md + **/*.cdap + **/*.yaml + **/*.md + logs/** + .git/** + .idea/** + **/grok/patterns/** + conf/** + data/** + plugins/** + **/*.patch + **/logrotate.d/** + **/limits.d/** + **/*.json + **/*.json.template + **/MANIFEST.MF + + **/org/apache/hadoop/** + + **/resources/** + + + + + org.apache.maven.plugins maven-checkstyle-plugin @@ -307,7 +377,7 @@ com.puppycrawl.tools checkstyle - 6.19 + 8.18 diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/Helpers.java b/src/main/java/io/cdap/plugin/marketo/common/api/Helpers.java index d01e0ee..bda2387 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/Helpers.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/Helpers.java @@ -41,7 +41,7 @@ public static String streamToString(InputStream inputStream) { try { return IOUtils.toString(inputStream, StandardCharsets.UTF_8); } catch (IOException e) { - throw new RuntimeException(e); + throw new RuntimeException(String.format("Failed to read stream completely due to '%s'", e.getMessage())); } } @@ -49,13 +49,13 @@ public static T streamToObject(InputStream inputStream, Class cls) { return Marketo.GSON.fromJson(new InputStreamReader(inputStream), cls); } - public static RuntimeException failForUri(String method, URI uri, Exception ex) { + public static RuntimeException failForMethodAndUri(String method, URI uri, Exception ex) { String message = ex.getMessage(); if (Strings.isNullOrEmpty(message)) { if (ex.getCause() != null) { message = ex.getCause().getMessage(); if (Strings.isNullOrEmpty(message)) { - message = "failed to make request"; + message = "Unknown failure"; } } } @@ -66,9 +66,10 @@ public static RuntimeException failForUri(String method, URI uri, Exception ex) uriBuilder.setParameters(queryParameters); try { String uriString = uriBuilder.build().toString(); - return new RuntimeException(String.format("Failed %s %s - %s", method, uriString, message)); + return new RuntimeException(String.format("Failed '%s' '%s' - '%s'", method, uriString, message)); } catch (URISyntaxException e) { - return new RuntimeException(message); + // this will never happen since we rebuilding already validated uri, just make compiler happy + return new RuntimeException(e); } } @@ -85,7 +86,7 @@ public static List getDateRanges(String beginDate, String endDate) { OffsetDateTime end = OffsetDateTime.parse(endDate); if (start.compareTo(end) > 0) { - throw new IllegalArgumentException("start date more than end date"); + throw new IllegalArgumentException("Start date cannot be greater than the end date."); } int compareResult = start.plusDays(30).compareTo(end); diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/Marketo.java b/src/main/java/io/cdap/plugin/marketo/common/api/Marketo.java index ead5070..809702c 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/Marketo.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/Marketo.java @@ -21,10 +21,12 @@ import com.google.gson.Gson; import io.cdap.plugin.marketo.common.api.entities.activities.ActivitiesExport; import io.cdap.plugin.marketo.common.api.entities.activities.ActivitiesExportRequest; +import io.cdap.plugin.marketo.common.api.entities.activities.ActivitiesExportResponse; import io.cdap.plugin.marketo.common.api.entities.activities.ActivityTypeResponse; -import io.cdap.plugin.marketo.common.api.entities.leads.LeadsDescribe; +import io.cdap.plugin.marketo.common.api.entities.leads.LeadsDescribeResponse; import io.cdap.plugin.marketo.common.api.entities.leads.LeadsExport; import io.cdap.plugin.marketo.common.api.entities.leads.LeadsExportRequest; +import io.cdap.plugin.marketo.common.api.entities.leads.LeadsExportResponse; import io.cdap.plugin.marketo.common.api.job.ActivitiesExportJob; import io.cdap.plugin.marketo.common.api.job.LeadsExportJob; import org.slf4j.Logger; @@ -54,9 +56,9 @@ public Marketo(String marketoEndpoint, String clientId, String clientSecret) { super(marketoEndpoint, clientId, clientSecret); } - public List describeLeads() { + public List describeLeads() { return StreamSupport.stream(Spliterators.spliteratorUnknownSize( - iteratePage(Urls.LEADS_DESCRIBE, LeadsDescribe.class, LeadsDescribe::getResult), + iteratePage(Urls.LEADS_DESCRIBE, LeadsDescribeResponse.class, LeadsDescribeResponse::getResult), Spliterator.ORDERED), false).collect(Collectors.toList()); } @@ -67,30 +69,30 @@ public List describeBuildInActivities() { } public LeadsExportJob exportLeads(LeadsExportRequest request) { - LeadsExport export = validatedPost(Urls.BULK_EXPORT_LEADS_CREATE, Collections.emptyMap(), - Marketo::streamToLeadsExport, - request, - GSON::toJson); + LeadsExportResponse export = validatedPost(Urls.BULK_EXPORT_LEADS_CREATE, Collections.emptyMap(), + Marketo::streamToLeadsExport, + request, + GSON::toJson); return new LeadsExportJob(export.singleExport(), this); } - public LeadsExport.ExportResponse leadsExportJobStatus(String jobId) { - LeadsExport currentResp = validatedGet( + public LeadsExport leadsExportJobStatus(String jobId) { + LeadsExportResponse currentResp = validatedGet( String.format(Urls.BULK_EXPORT_LEADS_STATUS, jobId), Collections.emptyMap(), Marketo::streamToLeadsExport); return currentResp.singleExport(); } public ActivitiesExportJob exportActivities(ActivitiesExportRequest request) { - ActivitiesExport export = validatedPost(Urls.BULK_EXPORT_ACTIVITIES_CREATE, Collections.emptyMap(), - Marketo::streamToActivitiesExport, - request, - GSON::toJson); + ActivitiesExportResponse export = validatedPost(Urls.BULK_EXPORT_ACTIVITIES_CREATE, Collections.emptyMap(), + Marketo::streamToActivitiesExport, + request, + GSON::toJson); return new ActivitiesExportJob(export.singleExport(), this); } - public ActivitiesExport.ExportResponse activitiesExportJobStatus(String jobId) { - ActivitiesExport currentResp = validatedGet( + public ActivitiesExport activitiesExportJobStatus(String jobId) { + ActivitiesExportResponse currentResp = validatedGet( String.format(Urls.BULK_EXPORT_ACTIVITIES_STATUS, jobId), Collections.emptyMap(), Marketo::streamToActivitiesExport); return currentResp.singleExport(); @@ -117,30 +119,30 @@ public void onBulkExtractQueueAvailable(Runnable action, long timeoutSeconds) { } } } - throw new RuntimeException("Failed to get slot in bulk export queue - timeout"); + throw new RuntimeException("Failed to get slot in bulk export queue due to timeout"); } - public static LeadsExport streamToLeadsExport(InputStream inputStream) { - return Helpers.streamToObject(inputStream, LeadsExport.class); + public static LeadsExportResponse streamToLeadsExport(InputStream inputStream) { + return Helpers.streamToObject(inputStream, LeadsExportResponse.class); } - public static ActivitiesExport streamToActivitiesExport(InputStream inputStream) { - return Helpers.streamToObject(inputStream, ActivitiesExport.class); + public static ActivitiesExportResponse streamToActivitiesExport(InputStream inputStream) { + return Helpers.streamToObject(inputStream, ActivitiesExportResponse.class); } private boolean canEnqueueJob() { - LeadsExport leadsExportJobs = validatedGet(Urls.BULK_EXPORT_LEADS_LIST, - ImmutableMap.of("status", "queued,processing"), - Marketo::streamToLeadsExport + LeadsExportResponse leadsExportResponseJobs = validatedGet(Urls.BULK_EXPORT_LEADS_LIST, + ImmutableMap.of("status", "queued,processing"), + Marketo::streamToLeadsExport ); - int jobsInQueue = leadsExportJobs.getResult().size(); + int jobsInQueue = leadsExportResponseJobs.getResult().size(); - ActivitiesExport activitiesExportJobs = validatedGet(Urls.BULK_EXPORT_ACTIVITIES_LIST, - ImmutableMap.of("status", "queued,processing"), - Marketo::streamToActivitiesExport + ActivitiesExportResponse activitiesExportResponceJobs = validatedGet(Urls.BULK_EXPORT_ACTIVITIES_LIST, + ImmutableMap.of("status", "queued,processing"), + Marketo::streamToActivitiesExport ); - jobsInQueue += activitiesExportJobs.getResult().size(); + jobsInQueue += activitiesExportResponceJobs.getResult().size(); LOG.debug("Jobs in queue: {}", jobsInQueue); diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/MarketoHttp.java b/src/main/java/io/cdap/plugin/marketo/common/api/MarketoHttp.java index 2ac5e80..c4e7cf1 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/MarketoHttp.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/MarketoHttp.java @@ -98,8 +98,8 @@ T validatedGet(String queryUrl, Map par } public T validatedPost(String queryUrl, Map parameters, - Function deserializer, - B body, Function qSerializer) { + Function deserializer, + B body, Function qSerializer) { String logUri = "POST " + buildUri(queryUrl, parameters, false).toString(); return retryableValidate(logUri, () -> { URI queryUri = buildUri(queryUrl, parameters, true); @@ -148,13 +148,11 @@ public T get(URI uri, Function deserializer) { try (CloseableHttpClient httpClient = HttpClients.createDefault()) { HttpGet request = new HttpGet(uri); try (CloseableHttpResponse response = httpClient.execute(request, httpClientContext)) { - if (response.getStatusLine().getStatusCode() >= 300) { - throw new IOException(IOUtils.toString(response.getEntity().getContent(), StandardCharsets.UTF_8)); - } + checkResponseCode(response); return deserializer.apply(response.getEntity().getContent()); } } catch (Exception e) { - throw Helpers.failForUri("GET", uri, e); + throw Helpers.failForMethodAndUri("GET", uri, e); } } @@ -166,10 +164,19 @@ private T post(URI uri, Function respDeserializer, B body request.setEntity(new StringEntity(qSerializer.apply(body), ContentType.APPLICATION_JSON)); } try (CloseableHttpResponse response = httpClient.execute(request, httpClientContext)) { + checkResponseCode(response); return respDeserializer.apply(response.getEntity().getContent()); } } catch (Exception e) { - throw Helpers.failForUri("POST", uri, e); + throw Helpers.failForMethodAndUri("POST", uri, e); + } + } + + private static void checkResponseCode(CloseableHttpResponse response) throws IOException { + int statusCode = response.getStatusLine().getStatusCode(); + if (statusCode >= 300) { + String responseBody = IOUtils.toString(response.getEntity().getContent(), StandardCharsets.UTF_8); + throw new RuntimeException(String.format("Http code '%s', response '%s'", statusCode, responseBody)); } } diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/activities/ActivitiesExport.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/activities/ActivitiesExport.java index 6137d52..2cfc72e 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/entities/activities/ActivitiesExport.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/activities/ActivitiesExport.java @@ -16,87 +16,63 @@ package io.cdap.plugin.marketo.common.api.entities.activities; -import io.cdap.plugin.marketo.common.api.entities.BaseResponse; - -import java.util.Collections; -import java.util.List; - /** - * Represents activities bulk export response. + * Represents export response item. */ -public class ActivitiesExport extends BaseResponse { - /** - * Represents export response item. - */ - public static class ExportResponse { - String createdAt; - String errorMsg; - String exportId; - int fileSize; - String fileChecksum; - String finishedAt; - String format; - int numberOfRecords; - String queuedAt; - String startedAt; - String status; - - public String getCreatedAt() { - return createdAt; - } - - public String getErrorMsg() { - return errorMsg; - } - - public String getExportId() { - return exportId; - } - - public int getFileSize() { - return fileSize; - } - - public String getFileChecksum() { - return fileChecksum; - } +public class ActivitiesExport { + String createdAt; + String errorMsg; + String exportId; + int fileSize; + String fileChecksum; + String finishedAt; + String format; + int numberOfRecords; + String queuedAt; + String startedAt; + String status; + + public String getCreatedAt() { + return createdAt; + } - public String getFinishedAt() { - return finishedAt; - } + public String getErrorMsg() { + return errorMsg; + } - public String getFormat() { - return format; - } + public String getExportId() { + return exportId; + } - public int getNumberOfRecords() { - return numberOfRecords; - } + public int getFileSize() { + return fileSize; + } - public String getQueuedAt() { - return queuedAt; - } + public String getFileChecksum() { + return fileChecksum; + } - public String getStartedAt() { - return startedAt; - } + public String getFinishedAt() { + return finishedAt; + } - public String getStatus() { - return status; - } + public String getFormat() { + return format; } - List result = Collections.emptyList(); + public int getNumberOfRecords() { + return numberOfRecords; + } - public ExportResponse singleExport() { - if (result.size() != 1) { - throw new IllegalStateException("Expected single export job result."); - } - return result.get(0); + public String getQueuedAt() { + return queuedAt; } - public List getResult() { - return result; + public String getStartedAt() { + return startedAt; } + public String getStatus() { + return status; + } } diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/activities/ActivitiesExportResponse.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/activities/ActivitiesExportResponse.java new file mode 100644 index 0000000..0b19058 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/activities/ActivitiesExportResponse.java @@ -0,0 +1,43 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api.entities.activities; + +import io.cdap.plugin.marketo.common.api.entities.BaseResponse; + +import java.util.Collections; +import java.util.List; + +/** + * Represents activities bulk export response. + */ +public class ActivitiesExportResponse extends BaseResponse { + + List result = Collections.emptyList(); + + public ActivitiesExport singleExport() { + if (result.size() != 1) { + throw new IllegalStateException( + String.format("Expected single export job result, but found '%s' results.", result.size())); + } + return result.get(0); + } + + public List getResult() { + return result; + } + +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/leads/LeadsDescribe.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/leads/LeadsDescribeResponse.java similarity index 96% rename from src/main/java/io/cdap/plugin/marketo/common/api/entities/leads/LeadsDescribe.java rename to src/main/java/io/cdap/plugin/marketo/common/api/entities/leads/LeadsDescribeResponse.java index ba445fa..afd89ec 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/entities/leads/LeadsDescribe.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/leads/LeadsDescribeResponse.java @@ -24,7 +24,7 @@ /** * Represents leads describe response. */ -public class LeadsDescribe extends BaseResponse { +public class LeadsDescribeResponse extends BaseResponse { /** * Represents lead field description. */ diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/leads/LeadsExport.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/leads/LeadsExport.java index 09d9cd4..f833207 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/entities/leads/LeadsExport.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/leads/LeadsExport.java @@ -16,87 +16,63 @@ package io.cdap.plugin.marketo.common.api.entities.leads; -import io.cdap.plugin.marketo.common.api.entities.BaseResponse; - -import java.util.Collections; -import java.util.List; - /** - * Represents leads bulk export response. + * Represents export response item. */ -public class LeadsExport extends BaseResponse { - /** - * Represents export response item. - */ - public static class ExportResponse { - String createdAt; - String errorMsg; - String exportId; - int fileSize; - String fileChecksum; - String finishedAt; - String format; - int numberOfRecords; - String queuedAt; - String startedAt; - String status; - - public String getCreatedAt() { - return createdAt; - } - - public String getErrorMsg() { - return errorMsg; - } - - public String getExportId() { - return exportId; - } - - public int getFileSize() { - return fileSize; - } - - public String getFileChecksum() { - return fileChecksum; - } +public class LeadsExport { + String createdAt; + String errorMsg; + String exportId; + int fileSize; + String fileChecksum; + String finishedAt; + String format; + int numberOfRecords; + String queuedAt; + String startedAt; + String status; + + public String getCreatedAt() { + return createdAt; + } - public String getFinishedAt() { - return finishedAt; - } + public String getErrorMsg() { + return errorMsg; + } - public String getFormat() { - return format; - } + public String getExportId() { + return exportId; + } - public int getNumberOfRecords() { - return numberOfRecords; - } + public int getFileSize() { + return fileSize; + } - public String getQueuedAt() { - return queuedAt; - } + public String getFileChecksum() { + return fileChecksum; + } - public String getStartedAt() { - return startedAt; - } + public String getFinishedAt() { + return finishedAt; + } - public String getStatus() { - return status; - } + public String getFormat() { + return format; } - List result = Collections.emptyList(); + public int getNumberOfRecords() { + return numberOfRecords; + } - public ExportResponse singleExport() { - if (result.size() != 1) { - throw new IllegalStateException("Expected single export job result."); - } - return result.get(0); + public String getQueuedAt() { + return queuedAt; } - public List getResult() { - return result; + public String getStartedAt() { + return startedAt; } + public String getStatus() { + return status; + } } diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/leads/LeadsExportResponse.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/leads/LeadsExportResponse.java new file mode 100644 index 0000000..bb58467 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/leads/LeadsExportResponse.java @@ -0,0 +1,42 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api.entities.leads; + +import io.cdap.plugin.marketo.common.api.entities.BaseResponse; + +import java.util.Collections; +import java.util.List; + +/** + * Represents leads bulk export response. + */ +public class LeadsExportResponse extends BaseResponse { + + List result = Collections.emptyList(); + + public LeadsExport singleExport() { + if (result.size() != 1) { + throw new IllegalStateException("Expected single export job result."); + } + return result.get(0); + } + + public List getResult() { + return result; + } + +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/job/ActivitiesExportJob.java b/src/main/java/io/cdap/plugin/marketo/common/api/job/ActivitiesExportJob.java index db5cdf3..7a6c35d 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/job/ActivitiesExportJob.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/job/ActivitiesExportJob.java @@ -20,6 +20,7 @@ import io.cdap.plugin.marketo.common.api.Marketo; import io.cdap.plugin.marketo.common.api.Urls; import io.cdap.plugin.marketo.common.api.entities.activities.ActivitiesExport; +import io.cdap.plugin.marketo.common.api.entities.activities.ActivitiesExportResponse; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -28,10 +29,10 @@ /** * Activities export job. */ -public class ActivitiesExportJob extends AbstractBulkExportJob { +public class ActivitiesExportJob extends AbstractBulkExportJob { private static final Logger LOG = LoggerFactory.getLogger(ActivitiesExportJob.class); - public ActivitiesExportJob(ActivitiesExport.ExportResponse lastState, Marketo marketo) { + public ActivitiesExportJob(ActivitiesExport lastState, Marketo marketo) { super(lastState.getExportId(), lastState, marketo); } @@ -41,12 +42,12 @@ public Logger getLogger() { } @Override - public ActivitiesExport.ExportResponse getFreshState() { + public ActivitiesExport getFreshState() { return getMarketo().activitiesExportJobStatus(getJobId()); } @Override - public String getStateStatus(ActivitiesExport.ExportResponse state) { + public String getStateStatus(ActivitiesExport state) { return state.getStatus(); } @@ -61,11 +62,11 @@ public String getLogPrefix() { } @Override - protected ActivitiesExport.ExportResponse enqueueImpl() { + protected ActivitiesExport enqueueImpl() { return getMarketo().validatedPost( String.format(Urls.BULK_EXPORT_ACTIVITIES_ENQUEUE, getJobId()), Collections.emptyMap(), - inputStream -> Helpers.streamToObject(inputStream, ActivitiesExport.class), + inputStream -> Helpers.streamToObject(inputStream, ActivitiesExportResponse.class), null, null).singleExport(); } } diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/job/LeadsExportJob.java b/src/main/java/io/cdap/plugin/marketo/common/api/job/LeadsExportJob.java index 740a9d7..f96587f 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/job/LeadsExportJob.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/job/LeadsExportJob.java @@ -20,6 +20,7 @@ import io.cdap.plugin.marketo.common.api.Marketo; import io.cdap.plugin.marketo.common.api.Urls; import io.cdap.plugin.marketo.common.api.entities.leads.LeadsExport; +import io.cdap.plugin.marketo.common.api.entities.leads.LeadsExportResponse; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -28,10 +29,10 @@ /** * Leads export job. */ -public class LeadsExportJob extends AbstractBulkExportJob { +public class LeadsExportJob extends AbstractBulkExportJob { private static final Logger LOG = LoggerFactory.getLogger(LeadsExportJob.class); - public LeadsExportJob(LeadsExport.ExportResponse lastState, Marketo marketo) { + public LeadsExportJob(LeadsExport lastState, Marketo marketo) { super(lastState.getExportId(), lastState, marketo); } @@ -41,12 +42,12 @@ public Logger getLogger() { } @Override - public LeadsExport.ExportResponse getFreshState() { + public LeadsExport getFreshState() { return getMarketo().leadsExportJobStatus(getJobId()); } @Override - public String getStateStatus(LeadsExport.ExportResponse state) { + public String getStateStatus(LeadsExport state) { return state.getStatus(); } @@ -61,11 +62,11 @@ public String getLogPrefix() { } @Override - protected LeadsExport.ExportResponse enqueueImpl() { + protected LeadsExport enqueueImpl() { return getMarketo().validatedPost( String.format(Urls.BULK_EXPORT_LEADS_ENQUEUE, getJobId()), Collections.emptyMap(), - inputStream -> Helpers.streamToObject(inputStream, LeadsExport.class), + inputStream -> Helpers.streamToObject(inputStream, LeadsExportResponse.class), null, null).singleExport(); } } diff --git a/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoRecordReader.java b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoRecordReader.java index 30f9b0e..f6b306b 100644 --- a/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoRecordReader.java +++ b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoRecordReader.java @@ -72,14 +72,20 @@ public void initialize(InputSplit inputSplit, TaskAttemptContext taskAttemptCont switch (config.getReportType()) { case LEADS: List leadsFields = config.getSchema().getFields().stream() - .map(Schema.Field::getName).collect(Collectors.toList()); + .map(Schema.Field::getName) + .collect(Collectors.toList()); + LeadsExportRequest.ExportLeadFilter leadsFilter = LeadsExportRequest.ExportLeadFilter.builder() - .createdAt(dateRange).build(); + .createdAt(dateRange) + .build(); + job = marketo.exportLeads(new LeadsExportRequest(leadsFields, leadsFilter)); break; case ACTIVITIES: ExportActivityFilter activitiesFilter = ExportActivityFilter.builder() - .createdAt(dateRange).build(); + .createdAt(dateRange) + .build(); + job = marketo.exportActivities(new ActivitiesExportRequest(Marketo.ACTIVITY_FIELDS, activitiesFilter)); break; default: @@ -104,7 +110,6 @@ public void initialize(InputSplit inputSplit, TaskAttemptContext taskAttemptCont public boolean nextKeyValue() { if (iterator.hasNext()) { current = iterator.next().toMap(); - LOG.debug("Got record '{}'", current.toString()); return true; } return false; diff --git a/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoReportingPlugin.java b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoReportingPlugin.java index 82b71b8..585814d 100644 --- a/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoReportingPlugin.java +++ b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoReportingPlugin.java @@ -41,7 +41,7 @@ */ @Plugin(type = BatchSource.PLUGIN_TYPE) @Name(MarketoReportingPlugin.NAME) -@Description("Reads entities from Marketo.") +@Description("Reads Leads or Activities from Marketo.") public class MarketoReportingPlugin extends BatchSource, StructuredRecord> { public static final String NAME = "MarketoReportingPlugin"; @@ -76,7 +76,6 @@ public void transform(KeyValue> input, Emitter } private void validateConfiguration(FailureCollector failureCollector) { - IdUtils.validateReferenceName(config.referenceName, failureCollector); config.validate(failureCollector); failureCollector.getOrThrowException(); } diff --git a/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoReportingSourceConfig.java b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoReportingSourceConfig.java index 50b98c3..e7282aa 100644 --- a/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoReportingSourceConfig.java +++ b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoReportingSourceConfig.java @@ -22,12 +22,13 @@ import io.cdap.cdap.api.annotation.Name; import io.cdap.cdap.api.data.schema.Schema; import io.cdap.cdap.etl.api.FailureCollector; +import io.cdap.plugin.common.IdUtils; import io.cdap.plugin.common.ReferencePluginConfig; -import io.cdap.plugin.marketo.common.api.Helpers; import io.cdap.plugin.marketo.common.api.Marketo; import java.net.MalformedURLException; import java.net.URL; +import java.time.OffsetDateTime; import java.time.format.DateTimeParseException; /** @@ -71,7 +72,6 @@ public class MarketoReportingSourceConfig extends ReferencePluginConfig { @Macro protected String endDate; - private transient Schema schema = null; private transient Marketo marketo = null; @@ -93,7 +93,6 @@ public Marketo getMarketo() { return marketo; } - public String getClientId() { return clientId; } @@ -119,6 +118,7 @@ public ReportType getReportType() { } void validate(FailureCollector failureCollector) { + IdUtils.validateReferenceName(referenceName, failureCollector); validateDate(failureCollector); validateReportType(failureCollector); validateMarketoEndpoint(failureCollector); @@ -126,22 +126,37 @@ void validate(FailureCollector failureCollector) { } void validateDate(FailureCollector failureCollector) { + if (!containsMacro(PROPERTY_START_DATE)) { + try { + OffsetDateTime.parse(getStartDate()); + } catch (DateTimeParseException ex) { + failureCollector.addFailure("Failed to parse start date.", + "Correct date to ISO 8601 format.") + .withConfigProperty(PROPERTY_START_DATE); + } + } + + if (!containsMacro(PROPERTY_END_DATE)) { + try { + OffsetDateTime.parse(getStartDate()); + } catch (DateTimeParseException ex) { + failureCollector.addFailure("Failed to parse end date.", + "Correct date to ISO 8601 format.") + .withConfigProperty(PROPERTY_END_DATE); + } + } + if (!(containsMacro(PROPERTY_START_DATE) && containsMacro(PROPERTY_END_DATE))) { try { - Helpers.getDateRanges(getStartDate(), getEndDate()); - } catch (IllegalArgumentException ex) { - String message = String.format("Failed to validate dates: %s.", ex.getMessage()); - String correctiveAction = null; - if (ex.getMessage().contains("start date more than end date")) { - message = "Start date more than end date."; - correctiveAction = "Swap dates."; + OffsetDateTime start = OffsetDateTime.parse(getStartDate()); + OffsetDateTime end = OffsetDateTime.parse(getEndDate()); + + if (start.compareTo(end) > 0) { + failureCollector.addFailure("Start date cannot be greater than the end date.", "Swap dates.") + .withConfigProperty(PROPERTY_START_DATE).withConfigProperty(PROPERTY_END_DATE); } - failureCollector.addFailure(message, correctiveAction) - .withConfigProperty(PROPERTY_START_DATE).withConfigProperty(PROPERTY_END_DATE); } catch (DateTimeParseException ex) { - failureCollector.addFailure("Failed to parse one of dates.", - "Correct dates to ISO 8601 format.") - .withConfigProperty(PROPERTY_START_DATE).withConfigProperty(PROPERTY_END_DATE); + // silently ignore parsing exceptions, we already pushed messages for malformed dates } } } @@ -160,14 +175,12 @@ void validateReportType(FailureCollector failureCollector) { void validateSecrets(FailureCollector failureCollector) { if (!containsMacro(PROPERTY_CLIENT_ID) && Strings.isNullOrEmpty(getClientId())) { - failureCollector.addFailure("Client ID is null or empty.", - "Set Client ID to non empty string.") + failureCollector.addFailure("Client ID is empty.", null) .withConfigProperty(PROPERTY_CLIENT_ID); } if (!containsMacro(PROPERTY_CLIENT_SECRET) && Strings.isNullOrEmpty(getClientSecret())) { - failureCollector.addFailure("Client Secret is null or empty.", - "Set Client Secret to non empty string.") + failureCollector.addFailure("Client Secret is empty.", null) .withConfigProperty(PROPERTY_CLIENT_SECRET); } } @@ -178,8 +191,7 @@ void validateMarketoEndpoint(FailureCollector failureCollector) { new URL(getRestApiEndpoint()); } catch (MalformedURLException e) { failureCollector - .addFailure(String.format("Malformed Marketo Rest API endpoint URL '%s'.", getRestApiEndpoint()), - "Change URL to valid.") + .addFailure(String.format("Malformed Marketo Rest API endpoint URL '%s'.", getRestApiEndpoint()), null) .withConfigProperty(PROPERTY_REST_API_ENDPOINT); } } diff --git a/src/main/java/io/cdap/plugin/marketo/source/batch/ReportType.java b/src/main/java/io/cdap/plugin/marketo/source/batch/ReportType.java index 1f19f6a..13193f0 100644 --- a/src/main/java/io/cdap/plugin/marketo/source/batch/ReportType.java +++ b/src/main/java/io/cdap/plugin/marketo/source/batch/ReportType.java @@ -35,6 +35,6 @@ public static ReportType fromString(String reportType) { return rt; } } - throw new IllegalArgumentException("unknown report type: " + reportType); + throw new IllegalArgumentException("Unknown report type: " + reportType); } } diff --git a/widgets/MarketoReportingPlugin-batchsource.json b/widgets/MarketoReportingPlugin-batchsource.json index 96bf773..e5390ba 100644 --- a/widgets/MarketoReportingPlugin-batchsource.json +++ b/widgets/MarketoReportingPlugin-batchsource.json @@ -52,12 +52,18 @@ { "widget-type": "textbox", "label": "Start Date", - "name": "startDate" + "name": "startDate", + "widget-attributes": { + "placeholder": "Start date in ISO 8601 format(1997-07-16T19:20:30+01:00)" + } }, { "widget-type": "textbox", "label": "End Date", - "name": "endDate" + "name": "endDate", + "widget-attributes": { + "placeholder": "End date in ISO 8601 format(1997-07-16T19:20:30+01:00)" + } } ] } From 701b466e1e9892dd16b03327e33de009745c57ec Mon Sep 17 00:00:00 2001 From: Yevhenii Chekanskyi Date: Thu, 28 Nov 2019 17:47:28 +0200 Subject: [PATCH 11/16] PLUGIN-75 Marketo Plugin - extend ignore. --- .gitignore | 55 ++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 53 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 34e1547..8731f78 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,54 @@ -target + +*.class +.*.swp +.beamer +# Package Files # +*.jar +*.war +*.ear +*.versionsBackup + +# Intellij Files & Dir # +*.iml +*.ipr +*.iws +atlassian-ide-plugin.xml +out/ +.DS_Store +./lib/ .idea -*.iml \ No newline at end of file + +# Gradle Files & Dir # +build/ +.gradle/ +.stickyStorage +.build/ +target/ + +# Node log +npm-*.log +logs/ + +# Singlenode and test data files. +/templates/ +/data/ +/data-fabric-tests/data/ + +# ANTLR4 +/core/gen +*.tokens +DirectivesLexer.java +DirectivesParser.java +DirectivesBaseListener.java +DirectivesBaseVisitor.java +DirectivesListener.java +DirectivesVisitor.java + +# generated by docs build +*.py + +# Remove release.properties +release.properties + +# Remove dev directory. +dev \ No newline at end of file From bae9c92af3af03152d155a74c034ac8daa61fb4f Mon Sep 17 00:00:00 2001 From: Yevhenii Chekanskyi Date: Wed, 4 Dec 2019 17:57:47 +0200 Subject: [PATCH 12/16] PLUGIN-75 Marketo Plugin - handle possible race condition. --- pom.xml | 6 ++- .../plugin/marketo/common/api/Marketo.java | 43 ++++++++++++------- .../marketo/common/api/MarketoHttp.java | 19 +++++++- .../common/api/TooManyJobsException.java | 23 ++++++++++ .../common/api/job/AbstractBulkExportJob.java | 8 +++- .../source/batch/MarketoRecordReader.java | 10 +++-- 6 files changed, 86 insertions(+), 23 deletions(-) create mode 100644 src/main/java/io/cdap/plugin/marketo/common/api/TooManyJobsException.java diff --git a/pom.xml b/pom.xml index e8a095b..f7a867b 100644 --- a/pom.xml +++ b/pom.xml @@ -274,7 +274,11 @@ commons-io 2.6 - + + org.awaitility + awaitility + 4.0.1 + diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/Marketo.java b/src/main/java/io/cdap/plugin/marketo/common/api/Marketo.java index 809702c..4ccd799 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/Marketo.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/Marketo.java @@ -29,6 +29,8 @@ import io.cdap.plugin.marketo.common.api.entities.leads.LeadsExportResponse; import io.cdap.plugin.marketo.common.api.job.ActivitiesExportJob; import io.cdap.plugin.marketo.common.api.job.LeadsExportJob; +import org.awaitility.Awaitility; +import org.awaitility.core.ConditionTimeoutException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -37,6 +39,7 @@ import java.util.List; import java.util.Spliterator; import java.util.Spliterators; +import java.util.concurrent.Callable; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import java.util.stream.StreamSupport; @@ -51,6 +54,14 @@ public class Marketo extends MarketoHttp { "activityTypeId", "campaignId", "primaryAttributeValueId", "primaryAttributeValue", "attributes"); + /** + * Job queue will be checked every 60 seconds. + */ + private static final long JOB_QUEUE_POLL_INTERVAL = 60; + /** + * Wait for 10 seconds before trying to enqueue job, this will minimize chance of race condition. + */ + private static final long JOB_QUEUE_POLL_DELAY = 10; public Marketo(String marketoEndpoint, String clientId, String clientSecret) { super(marketoEndpoint, clientId, clientSecret); @@ -104,22 +115,17 @@ public ActivitiesExport activitiesExportJobStatus(String jobId) { * @param action action to execute once slot is available * @param timeoutSeconds timeout in seconds */ - public void onBulkExtractQueueAvailable(Runnable action, long timeoutSeconds) { - long timeoutMillis = TimeUnit.SECONDS.toMillis(timeoutSeconds); - long startTime = System.currentTimeMillis(); - while (System.currentTimeMillis() - startTime < timeoutMillis) { - if (canEnqueueJob()) { - action.run(); - return; - } else { - try { - Thread.sleep(TimeUnit.SECONDS.toMillis(60)); - } catch (InterruptedException e) { - throw new RuntimeException("Failed to get slot in bulk export queue - interrupted"); - } - } + public void onBulkExtractQueueAvailable(Callable action, long timeoutSeconds) { + try { + Awaitility.given() + .ignoreException(TooManyJobsException.class) // ignore exception in case another reader took our slot + .atMost(timeoutSeconds, TimeUnit.SECONDS) + .pollInterval(JOB_QUEUE_POLL_INTERVAL, TimeUnit.SECONDS) + .pollDelay(JOB_QUEUE_POLL_DELAY, TimeUnit.SECONDS) + .until(action); + } catch (ConditionTimeoutException ex) { + throw new RuntimeException("Failed to get slot in bulk export queue due to timeout"); } - throw new RuntimeException("Failed to get slot in bulk export queue due to timeout"); } public static LeadsExportResponse streamToLeadsExport(InputStream inputStream) { @@ -130,7 +136,12 @@ public static ActivitiesExportResponse streamToActivitiesExport(InputStream inpu return Helpers.streamToObject(inputStream, ActivitiesExportResponse.class); } - private boolean canEnqueueJob() { + /** + * Check if job can be enqueued. + * + * @return true, if job can be enqueued + */ + public boolean canEnqueueJob() { LeadsExportResponse leadsExportResponseJobs = validatedGet(Urls.BULK_EXPORT_LEADS_LIST, ImmutableMap.of("status", "queued,processing"), Marketo::streamToLeadsExport diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/MarketoHttp.java b/src/main/java/io/cdap/plugin/marketo/common/api/MarketoHttp.java index c4e7cf1..44c8698 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/MarketoHttp.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/MarketoHttp.java @@ -107,6 +107,7 @@ public T validatedPost(String queryUrl, Map T retryableValidate(String logUri, Supplier tryQuery) { T result = tryQuery.get(); // check for expired token @@ -139,11 +140,27 @@ private T retryableValidate(String logUri, Supplier msg = msg + " - " + errors; LOG.error(msg); } - throw new RuntimeException(msg); + throw mapErrorsToException(result.getErrors(), msg); } return result; } + private RuntimeException mapErrorsToException(List errors, String defaultMessage) { + if (errors.size() == 1) { + Error e = errors.get(0); + String message = e.getMessage(); + if (e.getCode() == 1029 && message != null && message.contains("many jobs")) { + return new TooManyJobsException(); + } else { + // this error don't require specific handling + return new RuntimeException(defaultMessage); + } + } else { + // something outstanding happened and we have more than one error, we can't handle this in specific way + return new RuntimeException(defaultMessage); + } + } + public T get(URI uri, Function deserializer) { try (CloseableHttpClient httpClient = HttpClients.createDefault()) { HttpGet request = new HttpGet(uri); diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/TooManyJobsException.java b/src/main/java/io/cdap/plugin/marketo/common/api/TooManyJobsException.java new file mode 100644 index 0000000..ca23321 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/TooManyJobsException.java @@ -0,0 +1,23 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api; + +/** + * Exception thrown if too many jobs already queued. + */ +public class TooManyJobsException extends RuntimeException { +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/job/AbstractBulkExportJob.java b/src/main/java/io/cdap/plugin/marketo/common/api/job/AbstractBulkExportJob.java index f58109a..de21ca8 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/job/AbstractBulkExportJob.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/job/AbstractBulkExportJob.java @@ -97,12 +97,16 @@ public void waitCompletion() { } } - public void enqueue() { + public boolean enqueue() { if (!getStateStatus(getLastState()).equals(ENQUEUE_ABLE_STATUS)) { throw new IllegalStateException("Job must be in Created status before enqueuing, but was in " + getStateStatus(getLastState())); } + if (!marketo.canEnqueueJob()) { + return false; + } + T newState = enqueueImpl(); logStatusChange(getStateStatus(getLastState()), getStateStatus(newState)); @@ -113,6 +117,8 @@ public void enqueue() { } lastState = newState; + + return true; } private void logStatusChange(String oldStatus, String newStatus) { diff --git a/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoRecordReader.java b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoRecordReader.java index f6b306b..4af14f6 100644 --- a/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoRecordReader.java +++ b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoRecordReader.java @@ -49,6 +49,10 @@ public class MarketoRecordReader extends RecordReader> { private static final Logger LOG = LoggerFactory.getLogger(MarketoRecordReader.class); private static final Gson GSON = new GsonBuilder().create(); + /** + * Wait for 25 minutes for available slot in queue. + */ + private static final long JOB_ENQUEUE_TIMEOUT = 25 * 60; private Map current = null; private Iterator iterator = null; private String beginDate; @@ -94,14 +98,12 @@ public void initialize(InputSplit inputSplit, TaskAttemptContext taskAttemptCont LOG.info("BULK EXPORT JOB - job '{}' has date range '{}'", job.getJobId(), dateRange); - // TODO handle possible concurrent issues here, another mapper can take our slot - // wait for 10 minutes for available slot and enqueue job - marketo.onBulkExtractQueueAvailable(job::enqueue, 60 * 10); + // wait for 25 minutes for available slot and enqueue job + marketo.onBulkExtractQueueAvailable(job::enqueue, JOB_ENQUEUE_TIMEOUT); job.waitCompletion(); String data = job.getFile(); - // TODO stream here CSVParser parser = CSVFormat.DEFAULT.withHeader().parse(new StringReader(data)); iterator = parser.iterator(); } From 0cc2050ee8046943cf0eeb4d966f8b605c1a8637 Mon Sep 17 00:00:00 2001 From: Yevhenii Chekanskyi Date: Thu, 5 Dec 2019 15:55:50 +0200 Subject: [PATCH 13/16] PLUGIN-75 Marketo Plugin - add missing entities. --- .../plugin/marketo/common/api/Marketo.java | 7 +- .../marketo/common/api/MarketoHttp.java | 19 +- .../api/entities/WarningDeserializer.java | 54 +++++ .../common/api/entities/asset/Email.java | 131 +++++++++++++ .../api/entities/asset/EmailCCField.java | 53 +++++ .../common/api/entities/asset/EmailField.java | 33 ++++ .../api/entities/asset/EmailResponse.java | 24 +++ .../api/entities/asset/EmailTemplate.java | 73 +++++++ .../entities/asset/EmailTemplateResponse.java | 24 +++ .../common/api/entities/asset/File.java | 68 +++++++ .../common/api/entities/asset/FileFolder.java | 38 ++++ .../api/entities/asset/FileResponse.java | 24 +++ .../common/api/entities/asset/Folder.java | 93 +++++++++ .../api/entities/asset/FolderDescriptor.java | 38 ++++ .../api/entities/asset/FolderResponse.java | 24 +++ .../common/api/entities/asset/Form.java | 126 ++++++++++++ .../common/api/entities/asset/FormField.java | 185 ++++++++++++++++++ .../entities/asset/FormFieldsResponse.java | 24 +++ .../entities/asset/FormKnownVisitorDTO.java | 25 +++ .../api/entities/asset/FormResponse.java | 24 +++ .../entities/asset/FormThankYouPageDTO.java | 35 ++++ .../api/entities/asset/LandingPage.java | 113 +++++++++++ .../entities/asset/LandingPageResponse.java | 24 +++ .../entities/asset/LandingPageTemplate.java | 122 ++++++++++++ .../asset/LandingPageTemplateResponse.java | 24 +++ .../common/api/entities/asset/Program.java | 88 +++++++++ .../api/entities/asset/ProgramResponse.java | 23 +++ .../common/api/entities/asset/Recurrence.java | 71 +++++++ .../api/entities/asset/Segmentation.java | 68 +++++++ .../entities/asset/SegmentationResponse.java | 24 +++ .../entities/asset/SimpleBaseResponse.java | 34 ++++ .../api/entities/asset/SmartCampaign.java | 123 ++++++++++++ .../asset/SmartCampaignsResponse.java | 24 +++ .../common/api/entities/asset/SmartList.java | 95 +++++++++ .../entities/asset/SmartListsResponse.java | 23 +++ .../common/api/entities/asset/Snippet.java | 68 +++++++ .../api/entities/asset/SnippetsResponse.java | 23 +++ .../common/api/entities/asset/StaticList.java | 68 +++++++ .../entities/asset/StaticListsResponse.java | 23 +++ .../common/api/entities/asset/Tag.java | 38 ++++ .../api/entities/asset/TagsResponse.java | 23 +++ 41 files changed, 2217 insertions(+), 4 deletions(-) create mode 100644 src/main/java/io/cdap/plugin/marketo/common/api/entities/WarningDeserializer.java create mode 100644 src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Email.java create mode 100644 src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/EmailCCField.java create mode 100644 src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/EmailField.java create mode 100644 src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/EmailResponse.java create mode 100644 src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/EmailTemplate.java create mode 100644 src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/EmailTemplateResponse.java create mode 100644 src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/File.java create mode 100644 src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/FileFolder.java create mode 100644 src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/FileResponse.java create mode 100644 src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Folder.java create mode 100644 src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/FolderDescriptor.java create mode 100644 src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/FolderResponse.java create mode 100644 src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Form.java create mode 100644 src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/FormField.java create mode 100644 src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/FormFieldsResponse.java create mode 100644 src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/FormKnownVisitorDTO.java create mode 100644 src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/FormResponse.java create mode 100644 src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/FormThankYouPageDTO.java create mode 100644 src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/LandingPage.java create mode 100644 src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/LandingPageResponse.java create mode 100644 src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/LandingPageTemplate.java create mode 100644 src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/LandingPageTemplateResponse.java create mode 100644 src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Program.java create mode 100644 src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/ProgramResponse.java create mode 100644 src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Recurrence.java create mode 100644 src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Segmentation.java create mode 100644 src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/SegmentationResponse.java create mode 100644 src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/SimpleBaseResponse.java create mode 100644 src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/SmartCampaign.java create mode 100644 src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/SmartCampaignsResponse.java create mode 100644 src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/SmartList.java create mode 100644 src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/SmartListsResponse.java create mode 100644 src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Snippet.java create mode 100644 src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/SnippetsResponse.java create mode 100644 src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/StaticList.java create mode 100644 src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/StaticListsResponse.java create mode 100644 src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Tag.java create mode 100644 src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/TagsResponse.java diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/Marketo.java b/src/main/java/io/cdap/plugin/marketo/common/api/Marketo.java index 4ccd799..c49e389 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/Marketo.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/Marketo.java @@ -19,6 +19,9 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import io.cdap.plugin.marketo.common.api.entities.Warning; +import io.cdap.plugin.marketo.common.api.entities.WarningDeserializer; import io.cdap.plugin.marketo.common.api.entities.activities.ActivitiesExport; import io.cdap.plugin.marketo.common.api.entities.activities.ActivitiesExportRequest; import io.cdap.plugin.marketo.common.api.entities.activities.ActivitiesExportResponse; @@ -49,7 +52,9 @@ */ public class Marketo extends MarketoHttp { private static final Logger LOG = LoggerFactory.getLogger(Marketo.class); - static final Gson GSON = new Gson(); + static final Gson GSON = new GsonBuilder() + .registerTypeAdapter(Warning.class, new WarningDeserializer()) + .create(); public static final List ACTIVITY_FIELDS = ImmutableList.of("marketoGUID", "leadId", "activityDate", "activityTypeId", "campaignId", "primaryAttributeValueId", diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/MarketoHttp.java b/src/main/java/io/cdap/plugin/marketo/common/api/MarketoHttp.java index 44c8698..2b4df11 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/MarketoHttp.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/MarketoHttp.java @@ -68,6 +68,11 @@ class MarketoHttp { token = refreshToken(); } + private T getPage(String queryUrl, Class pageClass, Map parameters) { + return validatedGet(queryUrl, parameters, + inputStream -> Helpers.streamToObject(inputStream, pageClass)); + } + private T getPage(String queryUrl, Class pageClass) { return validatedGet(queryUrl, Collections.emptyMap(), inputStream -> Helpers.streamToObject(inputStream, pageClass)); @@ -82,9 +87,17 @@ T getNextPage(T currentPage, String queryUrl, Class return null; } - MarketoPageIterator iteratePage(String queryUrl, - Class pageClass, - Function> resultsGetter) { + public MarketoPageIterator iteratePage(String queryUrl, + Class pageClass, + Function> resultsGetter, + Map parameters) { + return new MarketoPageIterator<>(getPage(queryUrl, pageClass, parameters), this, queryUrl, pageClass, + resultsGetter); + } + + public MarketoPageIterator iteratePage(String queryUrl, + Class pageClass, + Function> resultsGetter) { return new MarketoPageIterator<>(getPage(queryUrl, pageClass), this, queryUrl, pageClass, resultsGetter); } diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/WarningDeserializer.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/WarningDeserializer.java new file mode 100644 index 0000000..b5b013d --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/WarningDeserializer.java @@ -0,0 +1,54 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api.entities; + +import com.google.gson.JsonDeserializationContext; +import com.google.gson.JsonDeserializer; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParseException; + +import java.lang.reflect.Type; + +/** + * Deserializer for warning messages that can handle simple string warning messages and code+message warnings. + */ +public class WarningDeserializer implements JsonDeserializer { + @Override + public Warning deserialize(JsonElement jsonElement, Type type, JsonDeserializationContext jsonDeserializationContext) + throws JsonParseException { + if (jsonElement.isJsonPrimitive()) { + return new Warning(-1, jsonElement.getAsString()); + } else if (jsonElement.isJsonObject()) { + JsonObject obj = jsonElement.getAsJsonObject(); + int code = -1; + String message = ""; + + if (obj.has("code")) { + code = obj.get("code").getAsInt(); + } + + if (obj.has("message")) { + message = obj.get("message").getAsString(); + } + + return new Warning(code, message); + } else { + throw new RuntimeException("Failed to deserialize warning message: " + jsonElement.toString()); + } + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Email.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Email.java new file mode 100644 index 0000000..ef56709 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Email.java @@ -0,0 +1,131 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api.entities.asset; + +import java.util.Collections; +import java.util.List; + +/** + * Email entity. + */ +public class Email { + String createdAt; + String description; + FolderDescriptor folder; + EmailField fromEmail; + EmailField fromName; + Integer id; + String name; + Boolean operational; + Boolean publishToMSI; + EmailField replyEmail; + String status; + EmailField subject; + Integer template; + Boolean textOnly; + String updatedAt; + String url; + Integer version; + Boolean webView; + String workspace; + Boolean autoCopyToText; + List ccFields = Collections.emptyList(); + + public String getCreatedAt() { + return createdAt; + } + + public String getDescription() { + return description; + } + + public FolderDescriptor getFolder() { + return folder; + } + + public EmailField getFromEmail() { + return fromEmail; + } + + public EmailField getFromName() { + return fromName; + } + + public Integer getId() { + return id; + } + + public String getName() { + return name; + } + + public Boolean getOperational() { + return operational; + } + + public Boolean getPublishToMSI() { + return publishToMSI; + } + + public EmailField getReplyEmail() { + return replyEmail; + } + + public String getStatus() { + return status; + } + + public EmailField getSubject() { + return subject; + } + + public Integer getTemplate() { + return template; + } + + public Boolean getTextOnly() { + return textOnly; + } + + public String getUpdatedAt() { + return updatedAt; + } + + public String getUrl() { + return url; + } + + public Integer getVersion() { + return version; + } + + public Boolean getWebView() { + return webView; + } + + public String getWorkspace() { + return workspace; + } + + public Boolean getAutoCopyToText() { + return autoCopyToText; + } + + public List getCcFields() { + return ccFields; + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/EmailCCField.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/EmailCCField.java new file mode 100644 index 0000000..a098af0 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/EmailCCField.java @@ -0,0 +1,53 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api.entities.asset; + +/** + * Email CC field. + */ +public class EmailCCField { + String attributeId; + String objectName; + String displayName; + String apiName; + + public String getAttributeId() { + return attributeId; + } + + public String getObjectName() { + return objectName; + } + + public String getDisplayName() { + return displayName; + } + + public String getApiName() { + return apiName; + } + + @Override + public String toString() { + return "EmailCCField{" + + "attributeId='" + attributeId + '\'' + + ", objectName='" + objectName + '\'' + + ", displayName='" + displayName + '\'' + + ", apiName='" + apiName + '\'' + + '}'; + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/EmailField.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/EmailField.java new file mode 100644 index 0000000..0b448c6 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/EmailField.java @@ -0,0 +1,33 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api.entities.asset; + +/** + * Email field. + */ +public class EmailField { + String type; + String value; + + public String getType() { + return type; + } + + public String getValue() { + return value; + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/EmailResponse.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/EmailResponse.java new file mode 100644 index 0000000..5cfb5a5 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/EmailResponse.java @@ -0,0 +1,24 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api.entities.asset; + +/** + * GET /rest/asset/v1/emails.json + */ +public class EmailResponse extends SimpleBaseResponse { + +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/EmailTemplate.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/EmailTemplate.java new file mode 100644 index 0000000..31648a6 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/EmailTemplate.java @@ -0,0 +1,73 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api.entities.asset; + +/** + * Email template entity. + */ +public class EmailTemplate { + String createdAt; + String description; + FolderDescriptor folder; + Integer id; + String name; + String status; + String updatedAt; + String url; + Integer version; + String workspace; + + public String getCreatedAt() { + return createdAt; + } + + public String getDescription() { + return description; + } + + public FolderDescriptor getFolder() { + return folder; + } + + public Integer getId() { + return id; + } + + public String getName() { + return name; + } + + public String getStatus() { + return status; + } + + public String getUpdatedAt() { + return updatedAt; + } + + public String getUrl() { + return url; + } + + public Integer getVersion() { + return version; + } + + public String getWorkspace() { + return workspace; + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/EmailTemplateResponse.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/EmailTemplateResponse.java new file mode 100644 index 0000000..186fa6a --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/EmailTemplateResponse.java @@ -0,0 +1,24 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api.entities.asset; + +/** + * GET /rest/asset/v1/emailTemplates.json + */ +public class EmailTemplateResponse extends SimpleBaseResponse { + +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/File.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/File.java new file mode 100644 index 0000000..61b0324 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/File.java @@ -0,0 +1,68 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api.entities.asset; + +/** + * File entity. + */ +public class File { + String createdAt; + String description; + FileFolder folder; + Integer id; + String mimeType; + String name; + Integer size; + String updatedAt; + String url; + + public String getCreatedAt() { + return createdAt; + } + + public String getDescription() { + return description; + } + + public FileFolder getFolder() { + return folder; + } + + public Integer getId() { + return id; + } + + public String getMimeType() { + return mimeType; + } + + public String getName() { + return name; + } + + public Integer getSize() { + return size; + } + + public String getUpdatedAt() { + return updatedAt; + } + + public String getUrl() { + return url; + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/FileFolder.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/FileFolder.java new file mode 100644 index 0000000..dda3ecd --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/FileFolder.java @@ -0,0 +1,38 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api.entities.asset; + +/** + * File folder descriptor. + */ +public class FileFolder { + Integer id; + String name; + String type; + + public Integer getId() { + return id; + } + + public String getName() { + return name; + } + + public String getType() { + return type; + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/FileResponse.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/FileResponse.java new file mode 100644 index 0000000..9f8cf78 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/FileResponse.java @@ -0,0 +1,24 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api.entities.asset; + +/** + * GET /rest/asset/v1/files.json + */ +public class FileResponse extends SimpleBaseResponse { + +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Folder.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Folder.java new file mode 100644 index 0000000..c246118 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Folder.java @@ -0,0 +1,93 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api.entities.asset; + +/** + * Folder entity. + */ +public class Folder { + Integer accessZoneId; + String createdAt; + String description; + FolderDescriptor folderId; + String folderType; + Integer id; + Boolean isArchive; + Boolean isSystem; + String name; + FolderDescriptor parent; + String path; + String updatedAt; + String url; + String workspace; + + public Integer getAccessZoneId() { + return accessZoneId; + } + + public String getCreatedAt() { + return createdAt; + } + + public String getDescription() { + return description; + } + + public FolderDescriptor getFolderId() { + return folderId; + } + + public String getFolderType() { + return folderType; + } + + public Integer getId() { + return id; + } + + public Boolean getArchive() { + return isArchive; + } + + public Boolean getSystem() { + return isSystem; + } + + public String getName() { + return name; + } + + public FolderDescriptor getParent() { + return parent; + } + + public String getPath() { + return path; + } + + public String getUpdatedAt() { + return updatedAt; + } + + public String getUrl() { + return url; + } + + public String getWorkspace() { + return workspace; + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/FolderDescriptor.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/FolderDescriptor.java new file mode 100644 index 0000000..45ab3fb --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/FolderDescriptor.java @@ -0,0 +1,38 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api.entities.asset; + +/** + * Folder descriptor. + */ +public class FolderDescriptor { + String id; + String type; + String folderName; + + public String getId() { + return id; + } + + public String getType() { + return type; + } + + public String getFolderName() { + return folderName; + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/FolderResponse.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/FolderResponse.java new file mode 100644 index 0000000..553538e --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/FolderResponse.java @@ -0,0 +1,24 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api.entities.asset; + +/** + * GET /rest/asset/v1/folders.json + */ +public class FolderResponse extends SimpleBaseResponse { + +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Form.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Form.java new file mode 100644 index 0000000..0254592 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Form.java @@ -0,0 +1,126 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api.entities.asset; + +import java.util.Collections; +import java.util.List; + +/** + * Form entity. + */ +public class Form { + String buttonLabel; + Integer buttonLocation; + String createdAt; + String description; + FolderDescriptor folder; + String fontFamily; + String fontSize; + Integer id; + FormKnownVisitorDTO knownVisitor; + String labelPosition; + String language; + String locale; + String name; + Boolean progressiveProfiling; + String status; + List thankYouList = Collections.emptyList(); + String theme; + String updatedAt; + String url; + String waitingLabel; + + public String getButtonLabel() { + return buttonLabel; + } + + public Integer getButtonLocation() { + return buttonLocation; + } + + public String getCreatedAt() { + return createdAt; + } + + public String getDescription() { + return description; + } + + public FolderDescriptor getFolder() { + return folder; + } + + public String getFontFamily() { + return fontFamily; + } + + public String getFontSize() { + return fontSize; + } + + public Integer getId() { + return id; + } + + public FormKnownVisitorDTO getKnownVisitor() { + return knownVisitor; + } + + public String getLabelPosition() { + return labelPosition; + } + + public String getLanguage() { + return language; + } + + public String getLocale() { + return locale; + } + + public String getName() { + return name; + } + + public Boolean getProgressiveProfiling() { + return progressiveProfiling; + } + + public String getStatus() { + return status; + } + + public List getThankYouList() { + return thankYouList; + } + + public String getTheme() { + return theme; + } + + public String getUpdatedAt() { + return updatedAt; + } + + public String getUrl() { + return url; + } + + public String getWaitingLabel() { + return waitingLabel; + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/FormField.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/FormField.java new file mode 100644 index 0000000..7425a4c --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/FormField.java @@ -0,0 +1,185 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api.entities.asset; + +/** + * Form filed entity. + */ +public class FormField { + String dataType; + String defaultValue; + String description; + String fieldMaskValues; + Integer fieldWidth; + String id; + Boolean initiallyChecked; + Boolean isLabelToRight; + Boolean isMultiselect; + Boolean isRequired; + Integer labelWidth; + Integer maxLength; + String maximumNumber; + String minimumNumber; + String picklistValues; + String placeholderText; + String validationMessage; + Integer visibleRows; + + public String getDataType() { + return dataType; + } + + public void setDataType(String dataType) { + this.dataType = dataType; + } + + public String getDefaultValue() { + return defaultValue; + } + + public void setDefaultValue(String defaultValue) { + this.defaultValue = defaultValue; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public String getFieldMaskValues() { + return fieldMaskValues; + } + + public void setFieldMaskValues(String fieldMaskValues) { + this.fieldMaskValues = fieldMaskValues; + } + + public Integer getFieldWidth() { + return fieldWidth; + } + + public void setFieldWidth(Integer fieldWidth) { + this.fieldWidth = fieldWidth; + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public Boolean getInitiallyChecked() { + return initiallyChecked; + } + + public void setInitiallyChecked(Boolean initiallyChecked) { + this.initiallyChecked = initiallyChecked; + } + + public Boolean getLabelToRight() { + return isLabelToRight; + } + + public void setLabelToRight(Boolean labelToRight) { + isLabelToRight = labelToRight; + } + + public Boolean getMultiselect() { + return isMultiselect; + } + + public void setMultiselect(Boolean multiselect) { + isMultiselect = multiselect; + } + + public Boolean getRequired() { + return isRequired; + } + + public void setRequired(Boolean required) { + isRequired = required; + } + + public Integer getLabelWidth() { + return labelWidth; + } + + public void setLabelWidth(Integer labelWidth) { + this.labelWidth = labelWidth; + } + + public Integer getMaxLength() { + return maxLength; + } + + public void setMaxLength(Integer maxLength) { + this.maxLength = maxLength; + } + + public String getMaximumNumber() { + return maximumNumber; + } + + public void setMaximumNumber(String maximumNumber) { + this.maximumNumber = maximumNumber; + } + + public String getMinimumNumber() { + return minimumNumber; + } + + public void setMinimumNumber(String minimumNumber) { + this.minimumNumber = minimumNumber; + } + + public String getPicklistValues() { + return picklistValues; + } + + public void setPicklistValues(String picklistValues) { + this.picklistValues = picklistValues; + } + + public String getPlaceholderText() { + return placeholderText; + } + + public void setPlaceholderText(String placeholderText) { + this.placeholderText = placeholderText; + } + + public String getValidationMessage() { + return validationMessage; + } + + public void setValidationMessage(String validationMessage) { + this.validationMessage = validationMessage; + } + + public Integer getVisibleRows() { + return visibleRows; + } + + public void setVisibleRows(Integer visibleRows) { + this.visibleRows = visibleRows; + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/FormFieldsResponse.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/FormFieldsResponse.java new file mode 100644 index 0000000..8e38999 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/FormFieldsResponse.java @@ -0,0 +1,24 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api.entities.asset; + +/** + * GET /rest/asset/v1/form/fields.json + */ +public class FormFieldsResponse extends SimpleBaseResponse { + +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/FormKnownVisitorDTO.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/FormKnownVisitorDTO.java new file mode 100644 index 0000000..c3e3846 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/FormKnownVisitorDTO.java @@ -0,0 +1,25 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api.entities.asset; + +/** + * Known visitor behavior for the form. + */ +public class FormKnownVisitorDTO { + String template; + String type; +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/FormResponse.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/FormResponse.java new file mode 100644 index 0000000..9cee13c --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/FormResponse.java @@ -0,0 +1,24 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api.entities.asset; + +/** + * GET /rest/asset/v1/forms.json + */ +public class FormResponse extends SimpleBaseResponse
{ + +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/FormThankYouPageDTO.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/FormThankYouPageDTO.java new file mode 100644 index 0000000..51e18fe --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/FormThankYouPageDTO.java @@ -0,0 +1,35 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api.entities.asset; + +import com.google.gson.annotations.SerializedName; + +import java.util.Collections; +import java.util.List; + +/** + * Thank you page behaviors for the form. + */ +public class FormThankYouPageDTO { + @SerializedName("default") + Boolean isDefault; + String followupType; + String followupValue; + String operator; + String subjectField; + List values = Collections.emptyList(); +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/LandingPage.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/LandingPage.java new file mode 100644 index 0000000..7bdb54c --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/LandingPage.java @@ -0,0 +1,113 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api.entities.asset; + +/** + * Landing page entity. + */ +public class LandingPage { + String url; + String computedUrl; + String createdAt; + String customHeadHTML; + String description; + String facebookOgTags; + FolderDescriptor folder; + Boolean formPrefill; + Integer id; + String keywords; + Boolean mobileEnabled; + String name; + String robots; + String status; + Integer template; + String title; + String updatedAt; + String workspace; + + public String getUrl() { + return url; + } + + public String getComputedUrl() { + return computedUrl; + } + + public String getCreatedAt() { + return createdAt; + } + + public String getCustomHeadHTML() { + return customHeadHTML; + } + + public String getDescription() { + return description; + } + + public String getFacebookOgTags() { + return facebookOgTags; + } + + public FolderDescriptor getFolder() { + return folder; + } + + public Boolean getFormPrefill() { + return formPrefill; + } + + public Integer getId() { + return id; + } + + public String getKeywords() { + return keywords; + } + + public Boolean getMobileEnabled() { + return mobileEnabled; + } + + public String getName() { + return name; + } + + public String getRobots() { + return robots; + } + + public String getStatus() { + return status; + } + + public Integer getTemplate() { + return template; + } + + public String getTitle() { + return title; + } + + public String getUpdatedAt() { + return updatedAt; + } + + public String getWorkspace() { + return workspace; + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/LandingPageResponse.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/LandingPageResponse.java new file mode 100644 index 0000000..c5f14e8 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/LandingPageResponse.java @@ -0,0 +1,24 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api.entities.asset; + +/** + * GET /rest/asset/v1/landingPages.json + */ +public class LandingPageResponse extends SimpleBaseResponse { + +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/LandingPageTemplate.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/LandingPageTemplate.java new file mode 100644 index 0000000..c4f52d3 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/LandingPageTemplate.java @@ -0,0 +1,122 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api.entities.asset; + +/** + * Landing page template entity. + */ +public class LandingPageTemplate { + String createdAt; + String description; + Boolean enableMunchkin; + FolderDescriptor folder; + Integer id; + String name; + String status; + String templateType; + String updatedAt; + String url; + String workspace; + + public String getCreatedAt() { + return createdAt; + } + + public void setCreatedAt(String createdAt) { + this.createdAt = createdAt; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public Boolean getEnableMunchkin() { + return enableMunchkin; + } + + public void setEnableMunchkin(Boolean enableMunchkin) { + this.enableMunchkin = enableMunchkin; + } + + public FolderDescriptor getFolder() { + return folder; + } + + public void setFolder(FolderDescriptor folder) { + this.folder = folder; + } + + public Integer getId() { + return id; + } + + public void setId(Integer id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getStatus() { + return status; + } + + public void setStatus(String status) { + this.status = status; + } + + public String getTemplateType() { + return templateType; + } + + public void setTemplateType(String templateType) { + this.templateType = templateType; + } + + public String getUpdatedAt() { + return updatedAt; + } + + public void setUpdatedAt(String updatedAt) { + this.updatedAt = updatedAt; + } + + public String getUrl() { + return url; + } + + public void setUrl(String url) { + this.url = url; + } + + public String getWorkspace() { + return workspace; + } + + public void setWorkspace(String workspace) { + this.workspace = workspace; + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/LandingPageTemplateResponse.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/LandingPageTemplateResponse.java new file mode 100644 index 0000000..4912f9b --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/LandingPageTemplateResponse.java @@ -0,0 +1,24 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api.entities.asset; + +/** + * GET /rest/asset/v1/landingPageTemplates.json + */ +public class LandingPageTemplateResponse extends SimpleBaseResponse { + +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Program.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Program.java new file mode 100644 index 0000000..b0b49c4 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Program.java @@ -0,0 +1,88 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api.entities.asset; + +/** + * Program entity. + */ +public class Program { + String channel; + String createdAt; + String description; + FolderDescriptor folder; + Integer id; + String name; + String sfdcId; + String sfdcName; + String status; + String type; + String updatedAt; + String url; + String workspace; + + public String getChannel() { + return channel; + } + + public String getCreatedAt() { + return createdAt; + } + + public String getDescription() { + return description; + } + + public FolderDescriptor getFolder() { + return folder; + } + + public Integer getId() { + return id; + } + + public String getName() { + return name; + } + + public String getSfdcId() { + return sfdcId; + } + + public String getSfdcName() { + return sfdcName; + } + + public String getStatus() { + return status; + } + + public String getType() { + return type; + } + + public String getUpdatedAt() { + return updatedAt; + } + + public String getUrl() { + return url; + } + + public String getWorkspace() { + return workspace; + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/ProgramResponse.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/ProgramResponse.java new file mode 100644 index 0000000..a69b142 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/ProgramResponse.java @@ -0,0 +1,23 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api.entities.asset; + +/** + * GET /rest/asset/v1/programs.json + */ +public class ProgramResponse extends SimpleBaseResponse { +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Recurrence.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Recurrence.java new file mode 100644 index 0000000..a6ad6b6 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Recurrence.java @@ -0,0 +1,71 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api.entities.asset; + +import java.util.Collections; +import java.util.List; + +/** + * Recurrence descriptor. + */ +public class Recurrence { + String startAt; + String endAt; + String intervalType; + Integer interval; + Boolean weekdayOnly; + List weekdayMask = Collections.emptyList(); + Integer dayOfMonth; + Integer dayOfWeek; + Integer weekOfMonth; + + public String getStartAt() { + return startAt; + } + + public String getEndAt() { + return endAt; + } + + public String getIntervalType() { + return intervalType; + } + + public Integer getInterval() { + return interval; + } + + public Boolean getWeekdayOnly() { + return weekdayOnly; + } + + public List getWeekdayMask() { + return weekdayMask; + } + + public Integer getDayOfMonth() { + return dayOfMonth; + } + + public Integer getDayOfWeek() { + return dayOfWeek; + } + + public Integer getWeekOfMonth() { + return weekOfMonth; + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Segmentation.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Segmentation.java new file mode 100644 index 0000000..68c0161 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Segmentation.java @@ -0,0 +1,68 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api.entities.asset; + +/** + * Segmentation entity. + */ +public class Segmentation { + String createdAt; + String description; + FolderDescriptor folder; + Integer id; + String name; + String status; + String updatedAt; + String url; + String workspace; + + public String getCreatedAt() { + return createdAt; + } + + public String getDescription() { + return description; + } + + public FolderDescriptor getFolder() { + return folder; + } + + public Integer getId() { + return id; + } + + public String getName() { + return name; + } + + public String getStatus() { + return status; + } + + public String getUpdatedAt() { + return updatedAt; + } + + public String getUrl() { + return url; + } + + public String getWorkspace() { + return workspace; + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/SegmentationResponse.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/SegmentationResponse.java new file mode 100644 index 0000000..e2c4d80 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/SegmentationResponse.java @@ -0,0 +1,24 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api.entities.asset; + +/** + * GET /rest/asset/v1/segmentation.json + * NO PAGING + */ +public class SegmentationResponse extends SimpleBaseResponse { +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/SimpleBaseResponse.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/SimpleBaseResponse.java new file mode 100644 index 0000000..e7c3138 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/SimpleBaseResponse.java @@ -0,0 +1,34 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api.entities.asset; + +import io.cdap.plugin.marketo.common.api.entities.BaseResponse; + +import java.util.Collections; +import java.util.List; + +/** + * Base response that contains only additional results in 'result' field. + * @param + */ +public class SimpleBaseResponse extends BaseResponse { + List result = Collections.emptyList(); + + public List getResult() { + return result; + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/SmartCampaign.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/SmartCampaign.java new file mode 100644 index 0000000..15d2e4e --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/SmartCampaign.java @@ -0,0 +1,123 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api.entities.asset; + +/** + * SmartCampaign entity. + */ +public class SmartCampaign { + Integer id; + String name; + String description; + String type; + Boolean isSystem; + Boolean isActive; + Boolean isRequestable; + Recurrence recurrence; + String qualificationRuleType; + Integer qualificationRuleInterval; + String qualificationRuleUnit; + Integer maxMembers; + Boolean isCommunicationLimitEnabled; + Integer smartListId; + Integer flowId; + FolderDescriptor folder; + String createdAt; + String updatedAt; + String workspace; + String status; + + public Integer getId() { + return id; + } + + public String getName() { + return name; + } + + public String getDescription() { + return description; + } + + public String getType() { + return type; + } + + public Boolean getSystem() { + return isSystem; + } + + public Boolean getActive() { + return isActive; + } + + public Boolean getRequestable() { + return isRequestable; + } + + public Recurrence getRecurrence() { + return recurrence; + } + + public String getQualificationRuleType() { + return qualificationRuleType; + } + + public Integer getQualificationRuleInterval() { + return qualificationRuleInterval; + } + + public String getQualificationRuleUnit() { + return qualificationRuleUnit; + } + + public Integer getMaxMembers() { + return maxMembers; + } + + public Boolean getCommunicationLimitEnabled() { + return isCommunicationLimitEnabled; + } + + public Integer getSmartListId() { + return smartListId; + } + + public Integer getFlowId() { + return flowId; + } + + public FolderDescriptor getFolder() { + return folder; + } + + public String getCreatedAt() { + return createdAt; + } + + public String getUpdatedAt() { + return updatedAt; + } + + public String getWorkspace() { + return workspace; + } + + public String getStatus() { + return status; + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/SmartCampaignsResponse.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/SmartCampaignsResponse.java new file mode 100644 index 0000000..bdfd0dd --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/SmartCampaignsResponse.java @@ -0,0 +1,24 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api.entities.asset; + +/** + * GET /rest/asset/v1/smartCampaigns.json + */ +public class SmartCampaignsResponse extends SimpleBaseResponse { + +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/SmartList.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/SmartList.java new file mode 100644 index 0000000..32a50af --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/SmartList.java @@ -0,0 +1,95 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api.entities.asset; + +/** + * SmartList entity. + */ +public class SmartList { + Integer id; + String name; + String description; + String createdAt; + String updatedAt; + String url; + FolderDescriptor folder; + String workspace; + + public Integer getId() { + return id; + } + + public void setId(Integer id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public String getCreatedAt() { + return createdAt; + } + + public void setCreatedAt(String createdAt) { + this.createdAt = createdAt; + } + + public String getUpdatedAt() { + return updatedAt; + } + + public void setUpdatedAt(String updatedAt) { + this.updatedAt = updatedAt; + } + + public String getUrl() { + return url; + } + + public void setUrl(String url) { + this.url = url; + } + + public FolderDescriptor getFolder() { + return folder; + } + + public void setFolder(FolderDescriptor folder) { + this.folder = folder; + } + + public String getWorkspace() { + return workspace; + } + + public void setWorkspace(String workspace) { + this.workspace = workspace; + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/SmartListsResponse.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/SmartListsResponse.java new file mode 100644 index 0000000..a2ca48a --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/SmartListsResponse.java @@ -0,0 +1,23 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api.entities.asset; + +/** + * GET /rest/asset/v1/smartLists.json + */ +public class SmartListsResponse extends SimpleBaseResponse { +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Snippet.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Snippet.java new file mode 100644 index 0000000..cfafdf6 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Snippet.java @@ -0,0 +1,68 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api.entities.asset; + +/** + * Snippet entity. + */ +public class Snippet { + String createdAt; + String description; + FolderDescriptor folder; + Integer id; + String name; + String status; + String updatedAt; + String url; + String workspace; + + public String getCreatedAt() { + return createdAt; + } + + public String getDescription() { + return description; + } + + public FolderDescriptor getFolder() { + return folder; + } + + public Integer getId() { + return id; + } + + public String getName() { + return name; + } + + public String getStatus() { + return status; + } + + public String getUpdatedAt() { + return updatedAt; + } + + public String getUrl() { + return url; + } + + public String getWorkspace() { + return workspace; + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/SnippetsResponse.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/SnippetsResponse.java new file mode 100644 index 0000000..fc8f442 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/SnippetsResponse.java @@ -0,0 +1,23 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api.entities.asset; + +/** + * GET /rest/asset/v1/snippets.json + */ +public class SnippetsResponse extends SimpleBaseResponse { +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/StaticList.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/StaticList.java new file mode 100644 index 0000000..abb0047 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/StaticList.java @@ -0,0 +1,68 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api.entities.asset; + +/** + * StaticList entity. + */ +public class StaticList { + Integer id; + String name; + String description; + String createdAt; + String updatedAt; + String url; + FolderDescriptor folder; + String workspace; + String computedUrl; + + public Integer getId() { + return id; + } + + public String getName() { + return name; + } + + public String getDescription() { + return description; + } + + public String getCreatedAt() { + return createdAt; + } + + public String getUpdatedAt() { + return updatedAt; + } + + public String getUrl() { + return url; + } + + public FolderDescriptor getFolder() { + return folder; + } + + public String getWorkspace() { + return workspace; + } + + public String getComputedUrl() { + return computedUrl; + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/StaticListsResponse.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/StaticListsResponse.java new file mode 100644 index 0000000..c5d8f99 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/StaticListsResponse.java @@ -0,0 +1,23 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api.entities.asset; + +/** + * GET /rest/asset/v1/staticLists.json + */ +public class StaticListsResponse extends SimpleBaseResponse { +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Tag.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Tag.java new file mode 100644 index 0000000..75ee37d --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Tag.java @@ -0,0 +1,38 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api.entities.asset; + +/** + * Tag entity. + */ +public class Tag { + String applicableProgramTypes; + String required; + String tagType; + + public String getApplicableProgramTypes() { + return applicableProgramTypes; + } + + public String getRequired() { + return required; + } + + public String getTagType() { + return tagType; + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/TagsResponse.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/TagsResponse.java new file mode 100644 index 0000000..bf7e8ef --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/TagsResponse.java @@ -0,0 +1,23 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api.entities.asset; + +/** + * GET /rest/asset/v1/tagTypes.json + */ +public class TagsResponse extends SimpleBaseResponse { +} From 88c0b5a24658890e9043ea7e52e118d41950dd9c Mon Sep 17 00:00:00 2001 From: Yevhenii Chekanskyi Date: Thu, 5 Dec 2019 20:27:56 +0200 Subject: [PATCH 14/16] PLUGIN-75 Marketo Plugin - add schema helper for entities. --- contrib/Generate.java | 236 +++++ pom.xml | 17 +- .../common/api/entities/asset/Email.java | 178 ++++ .../api/entities/asset/EmailCCField.java | 13 + .../common/api/entities/asset/EmailField.java | 3 + .../api/entities/asset/EmailTemplate.java | 3 + .../common/api/entities/asset/File.java | 3 + .../common/api/entities/asset/FileFolder.java | 3 + .../common/api/entities/asset/Folder.java | 3 + .../api/entities/asset/FolderDescriptor.java | 9 + .../common/api/entities/asset/Form.java | 3 + .../common/api/entities/asset/FormField.java | 3 + .../entities/asset/FormKnownVisitorDTO.java | 11 + .../entities/asset/FormThankYouPageDTO.java | 26 + .../api/entities/asset/LandingPage.java | 3 + .../entities/asset/LandingPageTemplate.java | 3 + .../common/api/entities/asset/Program.java | 119 +++ .../common/api/entities/asset/Recurrence.java | 3 + .../api/entities/asset/Segmentation.java | 3 + .../api/entities/asset/SmartCampaign.java | 3 + .../common/api/entities/asset/SmartList.java | 3 + .../common/api/entities/asset/Snippet.java | 3 + .../common/api/entities/asset/StaticList.java | 3 + .../common/api/entities/asset/Tag.java | 3 + .../common/api/entities/asset/gen/Entity.java | 27 + .../batch/entity/EntitySchemaHelper.java | 900 ++++++++++++++++++ .../batch/entity/EntitySchemaHelperTest.java | 54 ++ 27 files changed, 1633 insertions(+), 5 deletions(-) create mode 100644 contrib/Generate.java create mode 100644 src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/gen/Entity.java create mode 100644 src/main/java/io/cdap/plugin/marketo/source/batch/entity/EntitySchemaHelper.java create mode 100644 src/test/java/io/cdap/plugin/marketo/source/batch/entity/EntitySchemaHelperTest.java diff --git a/contrib/Generate.java b/contrib/Generate.java new file mode 100644 index 0000000..6b8e3b5 --- /dev/null +++ b/contrib/Generate.java @@ -0,0 +1,236 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api.entities.asset.gen; + +import com.google.common.collect.ImmutableList; +import com.google.common.reflect.ClassPath; + +import java.io.IOException; +import java.lang.annotation.Annotation; +import java.lang.reflect.Field; +import java.lang.reflect.ParameterizedType; +import java.util.List; +import java.util.stream.Collectors; + +/** + * Class that generates schema for entities. + * This class is not required anymore in near future and not intended to be used by developers... + * ...but can be used later again if some massive changes to schema will be required. + * + * Requires guava. + */ +public class Generate { + public static void main(String... args) throws IOException { + System.out.println("@SuppressWarnings(\"DuplicatedCode\")"); + System.out.println("public class EntitySchemaHelper {"); + + // entity name to schema + System.out.println(" public static Schema schemaForEntityName(String entityName) {"); + System.out.println(" switch(entityName) {"); + getEntityClasses().forEach(aClass -> { + System.out.println(" case \"" + aClass.getSimpleName() + "\":"); + System.out.println(" return " + generateSchemaGetterMethod(aClass) + "();"); + }); + System.out.println(" default:\n"); + System.out.println(" throw new IllegalArgumentException(\"Unknown entity name:\" + entityName);"); + System.out.println(" }"); + System.out.println(" }"); + + // entity object to structured record + System.out.println(" public static StructuredRecord structuredRecordFromEntity(String entityName, Object " + + "entity, Schema schema) {"); + System.out.println(" StructuredRecord.Builder builder = StructuredRecord.builder(schema);"); + System.out.println(" switch(entityName) {"); + getEntityClasses().forEach(aClass -> { + System.out.println(" case \"" + aClass.getSimpleName() + "\":"); + System.out.println(" " + aClass.getSimpleName() + " " + aClass.getSimpleName().toLowerCase() + " = (" + + aClass.getSimpleName() + ") entity;"); + System.out.println(" if (entity == null) {\n" + + " break;\n" + + " }"); + for (Field field : aClass.getDeclaredFields()) { + if (isSimpleType(field)) { + System.out.println(" builder.set(\"" + field.getName() + "\", " + aClass.getSimpleName().toLowerCase() + + "." + findGetterMethod(field, aClass) + "());"); + } else if (isComplexType(field)) { + System.out.println(" builder.set(\"" + field.getName() + "\", EntitySchemaHelper" + + ".structuredRecordFromEntity(\n" + + " \"" + field.getType().getSimpleName() + "\",\n" + + " " + aClass.getSimpleName().toLowerCase() + "." + + findGetterMethod(field, aClass) + "()" + ",\n" + + " schema.getField(\"" + field.getName() + "\").getSchema().getNonNullable())\n" + + " );"); + } else if (isListType(field)) { + ParameterizedType integerListType = (ParameterizedType) field.getGenericType(); + Class listType = (Class) integerListType.getActualTypeArguments()[0]; + if (isSimpleType(listType)) { + System.out.println(" builder.set(\"" + field.getName() + "\", " + + aClass.getSimpleName().toLowerCase() + "." + findGetterMethod(field, aClass) + + "());"); + } else { + System.out.println(" builder.set(\n" + + " \"" + field.getName() + "\",\n" + + " " + aClass.getSimpleName().toLowerCase() + "." + + findGetterMethod(field, aClass) + "().stream()\n" + + " .map(ent -> EntitySchemaHelper.structuredRecordFromEntity(\n" + + " \"" + listType.getSimpleName() + "\",\n" + + " ent,\n" + + " schema.getField(\"" + field.getName() + "\").getSchema()" + + ".getNonNullable().getComponentSchema()))\n" + + " .collect(Collectors.toList())\n" + + " );"); + } + } else { + throw new RuntimeException("Unknown field type."); + } + } + System.out.println(" break;"); + }); + System.out.println(" default:\n"); + System.out.println(" throw new IllegalArgumentException(\"Unknown entity name:\" + entityName);"); + System.out.println(" }"); + System.out.println(""); + System.out.println(" return builder.build();"); + System.out.println(" }"); + + // individual schemas + getEntityClasses().forEach(aClass -> { + System.out.println(" public static Schema " + generateSchemaGetterMethod(aClass) + "() {"); + System.out.println(" List fields = new ArrayList<>();"); + for (Field field : aClass.getDeclaredFields()) { + if (isSimpleType(field)) { + System.out.println(" fields.add(Schema.Field.of(\"" + field.getName() + "\", " + + simpleTypeToCdapType(field) + "));"); + } else if (isComplexType(field)) { + System.out.println(" fields.add(Schema.Field.of(\"" + field.getName() + "\", " + + "Schema.nullableOf(EntitySchemaHelper." + + generateSchemaGetterMethod(field.getType()) + "())));"); + } else if (isListType(field)) { + System.out.println(" fields.add(Schema.Field.of(\"" + field.getName() + "\", " + + listTypeToCdapType(field) + "));"); + } else { + throw new RuntimeException("Unknown field type."); + } + } + System.out.println(" return Schema.recordOf(UUID.randomUUID().toString().replace(\"-\", \"\"), fields);"); + System.out.println(" }"); + System.out.println(""); + }); + System.out.println("}"); + } + + private static final List SIMPLE_TYPES = ImmutableList.of(Boolean.class, Integer.class, int.class, + boolean.class, String.class); + + private static final List COLLECTION_TYPES = ImmutableList.of(List.class); + + public static boolean isSimpleType(Field field) { + return SIMPLE_TYPES.contains(field.getType()); + } + + public static boolean isSimpleType(Class cls) { + return SIMPLE_TYPES.contains(cls); + } + + public static String generateSchemaGetterMethod(Class cls) { + return "get" + capitalize(cls.getSimpleName()) + "Schema"; + } + + public static boolean isListType(Field field) { + return COLLECTION_TYPES.contains(field.getType()); + } + + public static boolean isComplexType(Field field) { + return !SIMPLE_TYPES.contains(field.getType()) && !COLLECTION_TYPES.contains(field.getType()); + } + + public static String simpleTypeToCdapType(Field field) { + if (field.getType().equals(Boolean.class) || field.getType().equals(boolean.class)) { + return "Schema.nullableOf(Schema.of(Schema.Type.BOOLEAN))"; + } else if (field.getType().equals(String.class)) { + return "Schema.nullableOf(Schema.of(Schema.Type.STRING))"; + } else if (field.getType().equals(Integer.class) || field.getType().equals(int.class)) { + return "Schema.nullableOf(Schema.of(Schema.Type.INT))"; + } else { + throw new RuntimeException("Unsupported simple type"); + } + } + + public static String listTypeToCdapType(Field field) { + ParameterizedType integerListType = (ParameterizedType) field.getGenericType(); + Class listType = (Class) integerListType.getActualTypeArguments()[0]; + if (listType.equals(Boolean.class) || listType.equals(boolean.class)) { + return "SSchema.nullableOf(Schema.arrayOf(Schema.of(Schema.Type.BOOLEAN)))"; + } else if (listType.equals(String.class)) { + return "Schema.nullableOf(Schema.arrayOf(Schema.of(Schema.Type.STRING)))"; + } else if (listType.equals(Integer.class) || listType.equals(int.class)) { + return "Schema.nullableOf(Schema.arrayOf(Schema.of(Schema.Type.INT)))"; + } else { + return "Schema.nullableOf(Schema.arrayOf(EntitySchemaHelper." + generateSchemaGetterMethod(listType) + "()))"; + } + } + + public static String findGetterMethod(Field field, Class cls) { + if (field.getType().equals(Boolean.class) || field.getType().equals(boolean.class)) { + String getterName = "get" + capitalize(field.getName()); + try { + cls.getMethod(getterName); + return getterName; + } catch (NoSuchMethodException e) { + if (field.getName().startsWith("is")) { + String alternateGetter = "get" + capitalize(field.getName().substring(2)); + try { + cls.getMethod(alternateGetter); + return alternateGetter; + } catch (NoSuchMethodException altE) { + throw new RuntimeException(altE); + } + } + } + } else { + String getterName = "get" + capitalize(field.getName()); + try { + cls.getMethod(getterName); + return getterName; + } catch (NoSuchMethodException altE) { + throw new RuntimeException(altE); + } + } + throw new RuntimeException(); + } + + public static String capitalize(String str) { + return str.substring(0, 1).toUpperCase() + str.substring(1); + } + + public static List getEntityClasses() throws IOException { + return ClassPath.from(Generate.class.getClassLoader()) + .getTopLevelClasses("io.cdap.plugin.marketo.common.api.entities.asset").stream() + .map(ClassPath.ClassInfo::load) + .filter(Generate::isEntity) + .collect(Collectors.toList()); + } + + public static boolean isEntity(Class cls) { + for (Annotation a : cls.getAnnotations()) { + if (a.annotationType().equals(Entity.class)) { + return true; + } + } + return false; + } +} diff --git a/pom.xml b/pom.xml index f7a867b..a1d3cbf 100644 --- a/pom.xml +++ b/pom.xml @@ -279,11 +279,6 @@ awaitility 4.0.1 - - - - - junit junit @@ -296,6 +291,18 @@ 2.25.1 test + + io.cdap.cdap + cdap-data-pipeline + ${cdap.version} + test + + + io.cdap.cdap + hydrator-test + ${cdap.version} + test + diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Email.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Email.java index ef56709..a1bbd91 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Email.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Email.java @@ -16,12 +16,16 @@ package io.cdap.plugin.marketo.common.api.entities.asset; +import io.cdap.plugin.marketo.common.api.entities.asset.gen.Entity; + +import java.util.ArrayList; import java.util.Collections; import java.util.List; /** * Email entity. */ +@Entity public class Email { String createdAt; String description; @@ -128,4 +132,178 @@ public Boolean getAutoCopyToText() { public List getCcFields() { return ccFields; } + + public static Builder builder() { + return new Builder(); + } + + /** + * Builder for Email entity. + */ + public static class Builder { + + private String createdAt; + private String description; + private FolderDescriptor folder; + private EmailField fromEmail; + private EmailField fromName; + private Integer id; + private String name; + private Boolean operational; + private Boolean publishToMSI; + private EmailField replyEmail; + private String status; + private EmailField subject; + private Integer template; + private Boolean textOnly; + private String updatedAt; + private String url; + private Integer version; + private Boolean webView; + private String workspace; + private Boolean autoCopyToText; + private List ccFields = new ArrayList<>(); + + public Builder() { + } + + public Builder createdAt(String createdAt) { + this.createdAt = createdAt; + return Builder.this; + } + + public Builder description(String description) { + this.description = description; + return Builder.this; + } + + public Builder folder(FolderDescriptor folder) { + this.folder = folder; + return Builder.this; + } + + public Builder fromEmail(EmailField fromEmail) { + this.fromEmail = fromEmail; + return Builder.this; + } + + public Builder fromName(EmailField fromName) { + this.fromName = fromName; + return Builder.this; + } + + public Builder id(Integer id) { + this.id = id; + return Builder.this; + } + + public Builder name(String name) { + this.name = name; + return Builder.this; + } + + public Builder operational(Boolean operational) { + this.operational = operational; + return Builder.this; + } + + public Builder publishToMSI(Boolean publishToMSI) { + this.publishToMSI = publishToMSI; + return Builder.this; + } + + public Builder replyEmail(EmailField replyEmail) { + this.replyEmail = replyEmail; + return Builder.this; + } + + public Builder status(String status) { + this.status = status; + return Builder.this; + } + + public Builder subject(EmailField subject) { + this.subject = subject; + return Builder.this; + } + + public Builder template(Integer template) { + this.template = template; + return Builder.this; + } + + public Builder textOnly(Boolean textOnly) { + this.textOnly = textOnly; + return Builder.this; + } + + public Builder updatedAt(String updatedAt) { + this.updatedAt = updatedAt; + return Builder.this; + } + + public Builder url(String url) { + this.url = url; + return Builder.this; + } + + public Builder version(Integer version) { + this.version = version; + return Builder.this; + } + + public Builder webView(Boolean webView) { + this.webView = webView; + return Builder.this; + } + + public Builder workspace(String workspace) { + this.workspace = workspace; + return Builder.this; + } + + public Builder autoCopyToText(Boolean autoCopyToText) { + this.autoCopyToText = autoCopyToText; + return Builder.this; + } + + public Builder ccFields(List ccFields) { + this.ccFields = ccFields; + return Builder.this; + } + + public Builder addCcFields(EmailCCField ccFields) { + this.ccFields.add(ccFields); + return Builder.this; + } + + public Email build() { + + return new Email(this); + } + } + + private Email(Builder builder) { + this.createdAt = builder.createdAt; + this.description = builder.description; + this.folder = builder.folder; + this.fromEmail = builder.fromEmail; + this.fromName = builder.fromName; + this.id = builder.id; + this.name = builder.name; + this.operational = builder.operational; + this.publishToMSI = builder.publishToMSI; + this.replyEmail = builder.replyEmail; + this.status = builder.status; + this.subject = builder.subject; + this.template = builder.template; + this.textOnly = builder.textOnly; + this.updatedAt = builder.updatedAt; + this.url = builder.url; + this.version = builder.version; + this.webView = builder.webView; + this.workspace = builder.workspace; + this.autoCopyToText = builder.autoCopyToText; + this.ccFields = builder.ccFields; + } } diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/EmailCCField.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/EmailCCField.java index a098af0..d59a89c 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/EmailCCField.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/EmailCCField.java @@ -16,15 +16,28 @@ package io.cdap.plugin.marketo.common.api.entities.asset; +import io.cdap.plugin.marketo.common.api.entities.asset.gen.Entity; + /** * Email CC field. */ +@Entity public class EmailCCField { String attributeId; String objectName; String displayName; String apiName; + public EmailCCField() { + } + + public EmailCCField(String attributeId, String objectName, String displayName, String apiName) { + this.attributeId = attributeId; + this.objectName = objectName; + this.displayName = displayName; + this.apiName = apiName; + } + public String getAttributeId() { return attributeId; } diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/EmailField.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/EmailField.java index 0b448c6..5cf78b7 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/EmailField.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/EmailField.java @@ -16,9 +16,12 @@ package io.cdap.plugin.marketo.common.api.entities.asset; +import io.cdap.plugin.marketo.common.api.entities.asset.gen.Entity; + /** * Email field. */ +@Entity public class EmailField { String type; String value; diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/EmailTemplate.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/EmailTemplate.java index 31648a6..9b7209b 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/EmailTemplate.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/EmailTemplate.java @@ -16,9 +16,12 @@ package io.cdap.plugin.marketo.common.api.entities.asset; +import io.cdap.plugin.marketo.common.api.entities.asset.gen.Entity; + /** * Email template entity. */ +@Entity public class EmailTemplate { String createdAt; String description; diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/File.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/File.java index 61b0324..2fcc0b2 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/File.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/File.java @@ -16,9 +16,12 @@ package io.cdap.plugin.marketo.common.api.entities.asset; +import io.cdap.plugin.marketo.common.api.entities.asset.gen.Entity; + /** * File entity. */ +@Entity public class File { String createdAt; String description; diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/FileFolder.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/FileFolder.java index dda3ecd..9dec989 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/FileFolder.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/FileFolder.java @@ -16,9 +16,12 @@ package io.cdap.plugin.marketo.common.api.entities.asset; +import io.cdap.plugin.marketo.common.api.entities.asset.gen.Entity; + /** * File folder descriptor. */ +@Entity public class FileFolder { Integer id; String name; diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Folder.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Folder.java index c246118..f07b693 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Folder.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Folder.java @@ -16,9 +16,12 @@ package io.cdap.plugin.marketo.common.api.entities.asset; +import io.cdap.plugin.marketo.common.api.entities.asset.gen.Entity; + /** * Folder entity. */ +@Entity public class Folder { Integer accessZoneId; String createdAt; diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/FolderDescriptor.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/FolderDescriptor.java index 45ab3fb..6d8d5b7 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/FolderDescriptor.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/FolderDescriptor.java @@ -16,14 +16,23 @@ package io.cdap.plugin.marketo.common.api.entities.asset; +import io.cdap.plugin.marketo.common.api.entities.asset.gen.Entity; + /** * Folder descriptor. */ +@Entity public class FolderDescriptor { String id; String type; String folderName; + public FolderDescriptor(String id, String type, String folderName) { + this.id = id; + this.type = type; + this.folderName = folderName; + } + public String getId() { return id; } diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Form.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Form.java index 0254592..74eed78 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Form.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Form.java @@ -16,12 +16,15 @@ package io.cdap.plugin.marketo.common.api.entities.asset; +import io.cdap.plugin.marketo.common.api.entities.asset.gen.Entity; + import java.util.Collections; import java.util.List; /** * Form entity. */ +@Entity public class Form { String buttonLabel; Integer buttonLocation; diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/FormField.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/FormField.java index 7425a4c..a97bdaf 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/FormField.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/FormField.java @@ -16,9 +16,12 @@ package io.cdap.plugin.marketo.common.api.entities.asset; +import io.cdap.plugin.marketo.common.api.entities.asset.gen.Entity; + /** * Form filed entity. */ +@Entity public class FormField { String dataType; String defaultValue; diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/FormKnownVisitorDTO.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/FormKnownVisitorDTO.java index c3e3846..aba9e2e 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/FormKnownVisitorDTO.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/FormKnownVisitorDTO.java @@ -16,10 +16,21 @@ package io.cdap.plugin.marketo.common.api.entities.asset; +import io.cdap.plugin.marketo.common.api.entities.asset.gen.Entity; + /** * Known visitor behavior for the form. */ +@Entity public class FormKnownVisitorDTO { String template; String type; + + public String getTemplate() { + return template; + } + + public String getType() { + return type; + } } diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/FormThankYouPageDTO.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/FormThankYouPageDTO.java index 51e18fe..8ee904f 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/FormThankYouPageDTO.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/FormThankYouPageDTO.java @@ -17,6 +17,7 @@ package io.cdap.plugin.marketo.common.api.entities.asset; import com.google.gson.annotations.SerializedName; +import io.cdap.plugin.marketo.common.api.entities.asset.gen.Entity; import java.util.Collections; import java.util.List; @@ -24,6 +25,7 @@ /** * Thank you page behaviors for the form. */ +@Entity public class FormThankYouPageDTO { @SerializedName("default") Boolean isDefault; @@ -32,4 +34,28 @@ public class FormThankYouPageDTO { String operator; String subjectField; List values = Collections.emptyList(); + + public Boolean getDefault() { + return isDefault; + } + + public String getFollowupType() { + return followupType; + } + + public String getFollowupValue() { + return followupValue; + } + + public String getOperator() { + return operator; + } + + public String getSubjectField() { + return subjectField; + } + + public List getValues() { + return values; + } } diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/LandingPage.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/LandingPage.java index 7bdb54c..959eb55 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/LandingPage.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/LandingPage.java @@ -16,9 +16,12 @@ package io.cdap.plugin.marketo.common.api.entities.asset; +import io.cdap.plugin.marketo.common.api.entities.asset.gen.Entity; + /** * Landing page entity. */ +@Entity public class LandingPage { String url; String computedUrl; diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/LandingPageTemplate.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/LandingPageTemplate.java index c4f52d3..2179236 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/LandingPageTemplate.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/LandingPageTemplate.java @@ -16,9 +16,12 @@ package io.cdap.plugin.marketo.common.api.entities.asset; +import io.cdap.plugin.marketo.common.api.entities.asset.gen.Entity; + /** * Landing page template entity. */ +@Entity public class LandingPageTemplate { String createdAt; String description; diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Program.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Program.java index b0b49c4..0f325ee 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Program.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Program.java @@ -16,9 +16,12 @@ package io.cdap.plugin.marketo.common.api.entities.asset; +import io.cdap.plugin.marketo.common.api.entities.asset.gen.Entity; + /** * Program entity. */ +@Entity public class Program { String channel; String createdAt; @@ -34,6 +37,9 @@ public class Program { String url; String workspace; + public Program() { + } + public String getChannel() { return channel; } @@ -85,4 +91,117 @@ public String getUrl() { public String getWorkspace() { return workspace; } + + public static Builder builder() { + return new Builder(); + } + + /** + * Builder for Program entity. + */ + public static class Builder { + + private String channel; + private String createdAt; + private String description; + private FolderDescriptor folder; + private Integer id; + private String name; + private String sfdcId; + private String sfdcName; + private String status; + private String type; + private String updatedAt; + private String url; + private String workspace; + + private Builder() { + } + + public Builder channel(String channel) { + this.channel = channel; + return Builder.this; + } + + public Builder createdAt(String createdAt) { + this.createdAt = createdAt; + return Builder.this; + } + + public Builder description(String description) { + this.description = description; + return Builder.this; + } + + public Builder folder(FolderDescriptor folder) { + this.folder = folder; + return Builder.this; + } + + public Builder id(Integer id) { + this.id = id; + return Builder.this; + } + + public Builder name(String name) { + this.name = name; + return Builder.this; + } + + public Builder sfdcId(String sfdcId) { + this.sfdcId = sfdcId; + return Builder.this; + } + + public Builder sfdcName(String sfdcName) { + this.sfdcName = sfdcName; + return Builder.this; + } + + public Builder status(String status) { + this.status = status; + return Builder.this; + } + + public Builder type(String type) { + this.type = type; + return Builder.this; + } + + public Builder updatedAt(String updatedAt) { + this.updatedAt = updatedAt; + return Builder.this; + } + + public Builder url(String url) { + this.url = url; + return Builder.this; + } + + public Builder workspace(String workspace) { + this.workspace = workspace; + return Builder.this; + } + + public Program build() { + + return new Program(this); + } + } + + private Program(Builder builder) { + this.channel = builder.channel; + this.createdAt = builder.createdAt; + this.description = builder.description; + this.folder = builder.folder; + this.id = builder.id; + this.name = builder.name; + this.sfdcId = builder.sfdcId; + this.sfdcName = builder.sfdcName; + this.status = builder.status; + this.type = builder.type; + this.updatedAt = builder.updatedAt; + this.url = builder.url; + this.workspace = builder.workspace; + } } diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Recurrence.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Recurrence.java index a6ad6b6..0ca6eb1 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Recurrence.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Recurrence.java @@ -16,12 +16,15 @@ package io.cdap.plugin.marketo.common.api.entities.asset; +import io.cdap.plugin.marketo.common.api.entities.asset.gen.Entity; + import java.util.Collections; import java.util.List; /** * Recurrence descriptor. */ +@Entity public class Recurrence { String startAt; String endAt; diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Segmentation.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Segmentation.java index 68c0161..a91e0a1 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Segmentation.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Segmentation.java @@ -16,9 +16,12 @@ package io.cdap.plugin.marketo.common.api.entities.asset; +import io.cdap.plugin.marketo.common.api.entities.asset.gen.Entity; + /** * Segmentation entity. */ +@Entity public class Segmentation { String createdAt; String description; diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/SmartCampaign.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/SmartCampaign.java index 15d2e4e..338d23e 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/SmartCampaign.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/SmartCampaign.java @@ -16,9 +16,12 @@ package io.cdap.plugin.marketo.common.api.entities.asset; +import io.cdap.plugin.marketo.common.api.entities.asset.gen.Entity; + /** * SmartCampaign entity. */ +@Entity public class SmartCampaign { Integer id; String name; diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/SmartList.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/SmartList.java index 32a50af..cfcddf4 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/SmartList.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/SmartList.java @@ -16,9 +16,12 @@ package io.cdap.plugin.marketo.common.api.entities.asset; +import io.cdap.plugin.marketo.common.api.entities.asset.gen.Entity; + /** * SmartList entity. */ +@Entity public class SmartList { Integer id; String name; diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Snippet.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Snippet.java index cfafdf6..d8b108e 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Snippet.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Snippet.java @@ -16,9 +16,12 @@ package io.cdap.plugin.marketo.common.api.entities.asset; +import io.cdap.plugin.marketo.common.api.entities.asset.gen.Entity; + /** * Snippet entity. */ +@Entity public class Snippet { String createdAt; String description; diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/StaticList.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/StaticList.java index abb0047..7d2b6cd 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/StaticList.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/StaticList.java @@ -16,9 +16,12 @@ package io.cdap.plugin.marketo.common.api.entities.asset; +import io.cdap.plugin.marketo.common.api.entities.asset.gen.Entity; + /** * StaticList entity. */ +@Entity public class StaticList { Integer id; String name; diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Tag.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Tag.java index 75ee37d..ce5bd70 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Tag.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Tag.java @@ -16,9 +16,12 @@ package io.cdap.plugin.marketo.common.api.entities.asset; +import io.cdap.plugin.marketo.common.api.entities.asset.gen.Entity; + /** * Tag entity. */ +@Entity public class Tag { String applicableProgramTypes; String required; diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/gen/Entity.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/gen/Entity.java new file mode 100644 index 0000000..135e87b --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/gen/Entity.java @@ -0,0 +1,27 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api.entities.asset.gen; + +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +/** + * Marks class as entity for contrib/Generate.java utility. + */ +@Retention(RetentionPolicy.RUNTIME) +public @interface Entity { +} diff --git a/src/main/java/io/cdap/plugin/marketo/source/batch/entity/EntitySchemaHelper.java b/src/main/java/io/cdap/plugin/marketo/source/batch/entity/EntitySchemaHelper.java new file mode 100644 index 0000000..72f2661 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/source/batch/entity/EntitySchemaHelper.java @@ -0,0 +1,900 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.source.batch.entity; + +import io.cdap.cdap.api.data.format.StructuredRecord; +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.plugin.marketo.common.api.entities.asset.Email; +import io.cdap.plugin.marketo.common.api.entities.asset.EmailCCField; +import io.cdap.plugin.marketo.common.api.entities.asset.EmailField; +import io.cdap.plugin.marketo.common.api.entities.asset.EmailTemplate; +import io.cdap.plugin.marketo.common.api.entities.asset.File; +import io.cdap.plugin.marketo.common.api.entities.asset.FileFolder; +import io.cdap.plugin.marketo.common.api.entities.asset.Folder; +import io.cdap.plugin.marketo.common.api.entities.asset.FolderDescriptor; +import io.cdap.plugin.marketo.common.api.entities.asset.Form; +import io.cdap.plugin.marketo.common.api.entities.asset.FormField; +import io.cdap.plugin.marketo.common.api.entities.asset.FormKnownVisitorDTO; +import io.cdap.plugin.marketo.common.api.entities.asset.FormThankYouPageDTO; +import io.cdap.plugin.marketo.common.api.entities.asset.LandingPage; +import io.cdap.plugin.marketo.common.api.entities.asset.LandingPageTemplate; +import io.cdap.plugin.marketo.common.api.entities.asset.Program; +import io.cdap.plugin.marketo.common.api.entities.asset.Recurrence; +import io.cdap.plugin.marketo.common.api.entities.asset.Segmentation; +import io.cdap.plugin.marketo.common.api.entities.asset.SmartCampaign; +import io.cdap.plugin.marketo.common.api.entities.asset.SmartList; +import io.cdap.plugin.marketo.common.api.entities.asset.Snippet; +import io.cdap.plugin.marketo.common.api.entities.asset.StaticList; +import io.cdap.plugin.marketo.common.api.entities.asset.Tag; + +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; +import java.util.stream.Collectors; + +/** + * Schema helper for entity plugin. + */ +@SuppressWarnings("DuplicatedCode") +public class EntitySchemaHelper { + public static Schema schemaForEntityName(String entityName) { + switch (entityName) { + case "Program": + return getProgramSchema(); + case "Folder": + return getFolderSchema(); + case "FileFolder": + return getFileFolderSchema(); + case "LandingPage": + return getLandingPageSchema(); + case "FormField": + return getFormFieldSchema(); + case "StaticList": + return getStaticListSchema(); + case "FormThankYouPageDTO": + return getFormThankYouPageDTOSchema(); + case "LandingPageTemplate": + return getLandingPageTemplateSchema(); + case "Email": + return getEmailSchema(); + case "SmartList": + return getSmartListSchema(); + case "Snippet": + return getSnippetSchema(); + case "FormKnownVisitorDTO": + return getFormKnownVisitorDTOSchema(); + case "EmailField": + return getEmailFieldSchema(); + case "SmartCampaign": + return getSmartCampaignSchema(); + case "Form": + return getFormSchema(); + case "FolderDescriptor": + return getFolderDescriptorSchema(); + case "Recurrence": + return getRecurrenceSchema(); + case "File": + return getFileSchema(); + case "Segmentation": + return getSegmentationSchema(); + case "Tag": + return getTagSchema(); + case "EmailTemplate": + return getEmailTemplateSchema(); + case "EmailCCField": + return getEmailCCFieldSchema(); + default: + + throw new IllegalArgumentException("Unknown entity name:" + entityName); + } + } + + public static StructuredRecord structuredRecordFromEntity(String entityName, Object entity, Schema schema) { + StructuredRecord.Builder builder = StructuredRecord.builder(schema); + switch (entityName) { + case "Program": + Program program = (Program) entity; + if (entity == null) { + break; + } + builder.set("channel", program.getChannel()); + builder.set("createdAt", program.getCreatedAt()); + builder.set("description", program.getDescription()); + builder.set("folder", EntitySchemaHelper.structuredRecordFromEntity( + "FolderDescriptor", + program.getFolder(), + schema.getField("folder").getSchema().getNonNullable()) + ); + builder.set("id", program.getId()); + builder.set("name", program.getName()); + builder.set("sfdcId", program.getSfdcId()); + builder.set("sfdcName", program.getSfdcName()); + builder.set("status", program.getStatus()); + builder.set("type", program.getType()); + builder.set("updatedAt", program.getUpdatedAt()); + builder.set("url", program.getUrl()); + builder.set("workspace", program.getWorkspace()); + break; + case "Folder": + Folder folder = (Folder) entity; + if (entity == null) { + break; + } + builder.set("accessZoneId", folder.getAccessZoneId()); + builder.set("createdAt", folder.getCreatedAt()); + builder.set("description", folder.getDescription()); + builder.set("folderId", EntitySchemaHelper.structuredRecordFromEntity( + "FolderDescriptor", + folder.getFolderId(), + schema.getField("folderId").getSchema().getNonNullable()) + ); + builder.set("folderType", folder.getFolderType()); + builder.set("id", folder.getId()); + builder.set("isArchive", folder.getArchive()); + builder.set("isSystem", folder.getSystem()); + builder.set("name", folder.getName()); + builder.set("parent", EntitySchemaHelper.structuredRecordFromEntity( + "FolderDescriptor", + folder.getParent(), + schema.getField("parent").getSchema().getNonNullable()) + ); + builder.set("path", folder.getPath()); + builder.set("updatedAt", folder.getUpdatedAt()); + builder.set("url", folder.getUrl()); + builder.set("workspace", folder.getWorkspace()); + break; + case "FileFolder": + FileFolder filefolder = (FileFolder) entity; + if (entity == null) { + break; + } + builder.set("id", filefolder.getId()); + builder.set("name", filefolder.getName()); + builder.set("type", filefolder.getType()); + break; + case "LandingPage": + LandingPage landingpage = (LandingPage) entity; + if (entity == null) { + break; + } + builder.set("url", landingpage.getUrl()); + builder.set("computedUrl", landingpage.getComputedUrl()); + builder.set("createdAt", landingpage.getCreatedAt()); + builder.set("customHeadHTML", landingpage.getCustomHeadHTML()); + builder.set("description", landingpage.getDescription()); + builder.set("facebookOgTags", landingpage.getFacebookOgTags()); + builder.set("folder", EntitySchemaHelper.structuredRecordFromEntity( + "FolderDescriptor", + landingpage.getFolder(), + schema.getField("folder").getSchema().getNonNullable()) + ); + builder.set("formPrefill", landingpage.getFormPrefill()); + builder.set("id", landingpage.getId()); + builder.set("keywords", landingpage.getKeywords()); + builder.set("mobileEnabled", landingpage.getMobileEnabled()); + builder.set("name", landingpage.getName()); + builder.set("robots", landingpage.getRobots()); + builder.set("status", landingpage.getStatus()); + builder.set("template", landingpage.getTemplate()); + builder.set("title", landingpage.getTitle()); + builder.set("updatedAt", landingpage.getUpdatedAt()); + builder.set("workspace", landingpage.getWorkspace()); + break; + case "FormField": + FormField formfield = (FormField) entity; + if (entity == null) { + break; + } + builder.set("dataType", formfield.getDataType()); + builder.set("defaultValue", formfield.getDefaultValue()); + builder.set("description", formfield.getDescription()); + builder.set("fieldMaskValues", formfield.getFieldMaskValues()); + builder.set("fieldWidth", formfield.getFieldWidth()); + builder.set("id", formfield.getId()); + builder.set("initiallyChecked", formfield.getInitiallyChecked()); + builder.set("isLabelToRight", formfield.getLabelToRight()); + builder.set("isMultiselect", formfield.getMultiselect()); + builder.set("isRequired", formfield.getRequired()); + builder.set("labelWidth", formfield.getLabelWidth()); + builder.set("maxLength", formfield.getMaxLength()); + builder.set("maximumNumber", formfield.getMaximumNumber()); + builder.set("minimumNumber", formfield.getMinimumNumber()); + builder.set("picklistValues", formfield.getPicklistValues()); + builder.set("placeholderText", formfield.getPlaceholderText()); + builder.set("validationMessage", formfield.getValidationMessage()); + builder.set("visibleRows", formfield.getVisibleRows()); + break; + case "StaticList": + StaticList staticlist = (StaticList) entity; + if (entity == null) { + break; + } + builder.set("id", staticlist.getId()); + builder.set("name", staticlist.getName()); + builder.set("description", staticlist.getDescription()); + builder.set("createdAt", staticlist.getCreatedAt()); + builder.set("updatedAt", staticlist.getUpdatedAt()); + builder.set("url", staticlist.getUrl()); + builder.set("folder", EntitySchemaHelper.structuredRecordFromEntity( + "FolderDescriptor", + staticlist.getFolder(), + schema.getField("folder").getSchema().getNonNullable()) + ); + builder.set("workspace", staticlist.getWorkspace()); + builder.set("computedUrl", staticlist.getComputedUrl()); + break; + case "FormThankYouPageDTO": + FormThankYouPageDTO formthankyoupagedto = (FormThankYouPageDTO) entity; + if (entity == null) { + break; + } + builder.set("isDefault", formthankyoupagedto.getDefault()); + builder.set("followupType", formthankyoupagedto.getFollowupType()); + builder.set("followupValue", formthankyoupagedto.getFollowupValue()); + builder.set("operator", formthankyoupagedto.getOperator()); + builder.set("subjectField", formthankyoupagedto.getSubjectField()); + builder.set("values", formthankyoupagedto.getValues()); + break; + case "LandingPageTemplate": + LandingPageTemplate landingpagetemplate = (LandingPageTemplate) entity; + if (entity == null) { + break; + } + builder.set("createdAt", landingpagetemplate.getCreatedAt()); + builder.set("description", landingpagetemplate.getDescription()); + builder.set("enableMunchkin", landingpagetemplate.getEnableMunchkin()); + builder.set("folder", EntitySchemaHelper.structuredRecordFromEntity( + "FolderDescriptor", + landingpagetemplate.getFolder(), + schema.getField("folder").getSchema().getNonNullable()) + ); + builder.set("id", landingpagetemplate.getId()); + builder.set("name", landingpagetemplate.getName()); + builder.set("status", landingpagetemplate.getStatus()); + builder.set("templateType", landingpagetemplate.getTemplateType()); + builder.set("updatedAt", landingpagetemplate.getUpdatedAt()); + builder.set("url", landingpagetemplate.getUrl()); + builder.set("workspace", landingpagetemplate.getWorkspace()); + break; + case "Email": + Email email = (Email) entity; + if (entity == null) { + break; + } + builder.set("createdAt", email.getCreatedAt()); + builder.set("description", email.getDescription()); + builder.set("folder", EntitySchemaHelper.structuredRecordFromEntity( + "FolderDescriptor", + email.getFolder(), + schema.getField("folder").getSchema().getNonNullable()) + ); + builder.set("fromEmail", EntitySchemaHelper.structuredRecordFromEntity( + "EmailField", + email.getFromEmail(), + schema.getField("fromEmail").getSchema().getNonNullable()) + ); + builder.set("fromName", EntitySchemaHelper.structuredRecordFromEntity( + "EmailField", + email.getFromName(), + schema.getField("fromName").getSchema().getNonNullable()) + ); + builder.set("id", email.getId()); + builder.set("name", email.getName()); + builder.set("operational", email.getOperational()); + builder.set("publishToMSI", email.getPublishToMSI()); + builder.set("replyEmail", EntitySchemaHelper.structuredRecordFromEntity( + "EmailField", + email.getReplyEmail(), + schema.getField("replyEmail").getSchema().getNonNullable()) + ); + builder.set("status", email.getStatus()); + builder.set("subject", EntitySchemaHelper.structuredRecordFromEntity( + "EmailField", + email.getSubject(), + schema.getField("subject").getSchema().getNonNullable()) + ); + builder.set("template", email.getTemplate()); + builder.set("textOnly", email.getTextOnly()); + builder.set("updatedAt", email.getUpdatedAt()); + builder.set("url", email.getUrl()); + builder.set("version", email.getVersion()); + builder.set("webView", email.getWebView()); + builder.set("workspace", email.getWorkspace()); + builder.set("autoCopyToText", email.getAutoCopyToText()); + builder.set( + "ccFields", + email.getCcFields().stream() + .map(ent -> EntitySchemaHelper.structuredRecordFromEntity( + "EmailCCField", + ent, + schema.getField("ccFields").getSchema().getNonNullable().getComponentSchema())) + .collect(Collectors.toList()) + ); + break; + case "SmartList": + SmartList smartlist = (SmartList) entity; + if (entity == null) { + break; + } + builder.set("id", smartlist.getId()); + builder.set("name", smartlist.getName()); + builder.set("description", smartlist.getDescription()); + builder.set("createdAt", smartlist.getCreatedAt()); + builder.set("updatedAt", smartlist.getUpdatedAt()); + builder.set("url", smartlist.getUrl()); + builder.set("folder", EntitySchemaHelper.structuredRecordFromEntity( + "FolderDescriptor", + smartlist.getFolder(), + schema.getField("folder").getSchema().getNonNullable()) + ); + builder.set("workspace", smartlist.getWorkspace()); + break; + case "Snippet": + Snippet snippet = (Snippet) entity; + if (entity == null) { + break; + } + builder.set("createdAt", snippet.getCreatedAt()); + builder.set("description", snippet.getDescription()); + builder.set("folder", EntitySchemaHelper.structuredRecordFromEntity( + "FolderDescriptor", + snippet.getFolder(), + schema.getField("folder").getSchema().getNonNullable()) + ); + builder.set("id", snippet.getId()); + builder.set("name", snippet.getName()); + builder.set("status", snippet.getStatus()); + builder.set("updatedAt", snippet.getUpdatedAt()); + builder.set("url", snippet.getUrl()); + builder.set("workspace", snippet.getWorkspace()); + break; + case "FormKnownVisitorDTO": + FormKnownVisitorDTO formknownvisitordto = (FormKnownVisitorDTO) entity; + if (entity == null) { + break; + } + builder.set("template", formknownvisitordto.getTemplate()); + builder.set("type", formknownvisitordto.getType()); + break; + case "EmailField": + EmailField emailfield = (EmailField) entity; + if (entity == null) { + break; + } + builder.set("type", emailfield.getType()); + builder.set("value", emailfield.getValue()); + break; + case "SmartCampaign": + SmartCampaign smartcampaign = (SmartCampaign) entity; + if (entity == null) { + break; + } + builder.set("id", smartcampaign.getId()); + builder.set("name", smartcampaign.getName()); + builder.set("description", smartcampaign.getDescription()); + builder.set("type", smartcampaign.getType()); + builder.set("isSystem", smartcampaign.getSystem()); + builder.set("isActive", smartcampaign.getActive()); + builder.set("isRequestable", smartcampaign.getRequestable()); + builder.set("recurrence", EntitySchemaHelper.structuredRecordFromEntity( + "Recurrence", + smartcampaign.getRecurrence(), + schema.getField("recurrence").getSchema().getNonNullable()) + ); + builder.set("qualificationRuleType", smartcampaign.getQualificationRuleType()); + builder.set("qualificationRuleInterval", smartcampaign.getQualificationRuleInterval()); + builder.set("qualificationRuleUnit", smartcampaign.getQualificationRuleUnit()); + builder.set("maxMembers", smartcampaign.getMaxMembers()); + builder.set("isCommunicationLimitEnabled", smartcampaign.getCommunicationLimitEnabled()); + builder.set("smartListId", smartcampaign.getSmartListId()); + builder.set("flowId", smartcampaign.getFlowId()); + builder.set("folder", EntitySchemaHelper.structuredRecordFromEntity( + "FolderDescriptor", + smartcampaign.getFolder(), + schema.getField("folder").getSchema().getNonNullable()) + ); + builder.set("createdAt", smartcampaign.getCreatedAt()); + builder.set("updatedAt", smartcampaign.getUpdatedAt()); + builder.set("workspace", smartcampaign.getWorkspace()); + builder.set("status", smartcampaign.getStatus()); + break; + case "Form": + Form form = (Form) entity; + if (entity == null) { + break; + } + builder.set("buttonLabel", form.getButtonLabel()); + builder.set("buttonLocation", form.getButtonLocation()); + builder.set("createdAt", form.getCreatedAt()); + builder.set("description", form.getDescription()); + builder.set("folder", EntitySchemaHelper.structuredRecordFromEntity( + "FolderDescriptor", + form.getFolder(), + schema.getField("folder").getSchema().getNonNullable()) + ); + builder.set("fontFamily", form.getFontFamily()); + builder.set("fontSize", form.getFontSize()); + builder.set("id", form.getId()); + builder.set("knownVisitor", EntitySchemaHelper.structuredRecordFromEntity( + "FormKnownVisitorDTO", + form.getKnownVisitor(), + schema.getField("knownVisitor").getSchema().getNonNullable()) + ); + builder.set("labelPosition", form.getLabelPosition()); + builder.set("language", form.getLanguage()); + builder.set("locale", form.getLocale()); + builder.set("name", form.getName()); + builder.set("progressiveProfiling", form.getProgressiveProfiling()); + builder.set("status", form.getStatus()); + builder.set( + "thankYouList", + form.getThankYouList().stream() + .map(ent -> EntitySchemaHelper.structuredRecordFromEntity( + "FormThankYouPageDTO", + ent, + schema.getField("thankYouList").getSchema().getNonNullable().getComponentSchema())) + .collect(Collectors.toList()) + ); + builder.set("theme", form.getTheme()); + builder.set("updatedAt", form.getUpdatedAt()); + builder.set("url", form.getUrl()); + builder.set("waitingLabel", form.getWaitingLabel()); + break; + case "FolderDescriptor": + FolderDescriptor folderdescriptor = (FolderDescriptor) entity; + if (entity == null) { + break; + } + builder.set("id", folderdescriptor.getId()); + builder.set("type", folderdescriptor.getType()); + builder.set("folderName", folderdescriptor.getFolderName()); + break; + case "Recurrence": + Recurrence recurrence = (Recurrence) entity; + if (entity == null) { + break; + } + builder.set("startAt", recurrence.getStartAt()); + builder.set("endAt", recurrence.getEndAt()); + builder.set("intervalType", recurrence.getIntervalType()); + builder.set("interval", recurrence.getInterval()); + builder.set("weekdayOnly", recurrence.getWeekdayOnly()); + builder.set("weekdayMask", recurrence.getWeekdayMask()); + builder.set("dayOfMonth", recurrence.getDayOfMonth()); + builder.set("dayOfWeek", recurrence.getDayOfWeek()); + builder.set("weekOfMonth", recurrence.getWeekOfMonth()); + break; + case "File": + File file = (File) entity; + if (entity == null) { + break; + } + builder.set("createdAt", file.getCreatedAt()); + builder.set("description", file.getDescription()); + builder.set("folder", EntitySchemaHelper.structuredRecordFromEntity( + "FileFolder", + file.getFolder(), + schema.getField("folder").getSchema().getNonNullable()) + ); + builder.set("id", file.getId()); + builder.set("mimeType", file.getMimeType()); + builder.set("name", file.getName()); + builder.set("size", file.getSize()); + builder.set("updatedAt", file.getUpdatedAt()); + builder.set("url", file.getUrl()); + break; + case "Segmentation": + Segmentation segmentation = (Segmentation) entity; + if (entity == null) { + break; + } + builder.set("createdAt", segmentation.getCreatedAt()); + builder.set("description", segmentation.getDescription()); + builder.set("folder", EntitySchemaHelper.structuredRecordFromEntity( + "FolderDescriptor", + segmentation.getFolder(), + schema.getField("folder").getSchema().getNonNullable()) + ); + builder.set("id", segmentation.getId()); + builder.set("name", segmentation.getName()); + builder.set("status", segmentation.getStatus()); + builder.set("updatedAt", segmentation.getUpdatedAt()); + builder.set("url", segmentation.getUrl()); + builder.set("workspace", segmentation.getWorkspace()); + break; + case "Tag": + Tag tag = (Tag) entity; + if (entity == null) { + break; + } + builder.set("applicableProgramTypes", tag.getApplicableProgramTypes()); + builder.set("required", tag.getRequired()); + builder.set("tagType", tag.getTagType()); + break; + case "EmailTemplate": + EmailTemplate emailtemplate = (EmailTemplate) entity; + if (entity == null) { + break; + } + builder.set("createdAt", emailtemplate.getCreatedAt()); + builder.set("description", emailtemplate.getDescription()); + builder.set("folder", EntitySchemaHelper.structuredRecordFromEntity( + "FolderDescriptor", + emailtemplate.getFolder(), + schema.getField("folder").getSchema().getNonNullable()) + ); + builder.set("id", emailtemplate.getId()); + builder.set("name", emailtemplate.getName()); + builder.set("status", emailtemplate.getStatus()); + builder.set("updatedAt", emailtemplate.getUpdatedAt()); + builder.set("url", emailtemplate.getUrl()); + builder.set("version", emailtemplate.getVersion()); + builder.set("workspace", emailtemplate.getWorkspace()); + break; + case "EmailCCField": + EmailCCField emailccfield = (EmailCCField) entity; + if (entity == null) { + break; + } + builder.set("attributeId", emailccfield.getAttributeId()); + builder.set("objectName", emailccfield.getObjectName()); + builder.set("displayName", emailccfield.getDisplayName()); + builder.set("apiName", emailccfield.getApiName()); + break; + default: + + throw new IllegalArgumentException("Unknown entity name:" + entityName); + } + + return builder.build(); + } + + public static Schema getProgramSchema() { + List fields = new ArrayList<>(); + fields.add(Schema.Field.of("channel", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("createdAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("description", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("folder", Schema.nullableOf(EntitySchemaHelper.getFolderDescriptorSchema()))); + fields.add(Schema.Field.of("id", Schema.nullableOf(Schema.of(Schema.Type.INT)))); + fields.add(Schema.Field.of("name", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("sfdcId", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("sfdcName", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("status", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("type", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("updatedAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("url", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("workspace", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + return Schema.recordOf(UUID.randomUUID().toString().replace("-", ""), fields); + } + + public static Schema getFolderSchema() { + List fields = new ArrayList<>(); + fields.add(Schema.Field.of("accessZoneId", Schema.nullableOf(Schema.of(Schema.Type.INT)))); + fields.add(Schema.Field.of("createdAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("description", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("folderId", Schema.nullableOf(EntitySchemaHelper.getFolderDescriptorSchema()))); + fields.add(Schema.Field.of("folderType", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("id", Schema.nullableOf(Schema.of(Schema.Type.INT)))); + fields.add(Schema.Field.of("isArchive", Schema.nullableOf(Schema.of(Schema.Type.BOOLEAN)))); + fields.add(Schema.Field.of("isSystem", Schema.nullableOf(Schema.of(Schema.Type.BOOLEAN)))); + fields.add(Schema.Field.of("name", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("parent", Schema.nullableOf(EntitySchemaHelper.getFolderDescriptorSchema()))); + fields.add(Schema.Field.of("path", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("updatedAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("url", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("workspace", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + return Schema.recordOf(UUID.randomUUID().toString().replace("-", ""), fields); + } + + public static Schema getFileFolderSchema() { + List fields = new ArrayList<>(); + fields.add(Schema.Field.of("id", Schema.nullableOf(Schema.of(Schema.Type.INT)))); + fields.add(Schema.Field.of("name", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("type", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + return Schema.recordOf(UUID.randomUUID().toString().replace("-", ""), fields); + } + + public static Schema getLandingPageSchema() { + List fields = new ArrayList<>(); + fields.add(Schema.Field.of("url", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("computedUrl", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("createdAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("customHeadHTML", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("description", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("facebookOgTags", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("folder", Schema.nullableOf(EntitySchemaHelper.getFolderDescriptorSchema()))); + fields.add(Schema.Field.of("formPrefill", Schema.nullableOf(Schema.of(Schema.Type.BOOLEAN)))); + fields.add(Schema.Field.of("id", Schema.nullableOf(Schema.of(Schema.Type.INT)))); + fields.add(Schema.Field.of("keywords", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("mobileEnabled", Schema.nullableOf(Schema.of(Schema.Type.BOOLEAN)))); + fields.add(Schema.Field.of("name", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("robots", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("status", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("template", Schema.nullableOf(Schema.of(Schema.Type.INT)))); + fields.add(Schema.Field.of("title", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("updatedAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("workspace", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + return Schema.recordOf(UUID.randomUUID().toString().replace("-", ""), fields); + } + + public static Schema getFormFieldSchema() { + List fields = new ArrayList<>(); + fields.add(Schema.Field.of("dataType", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("defaultValue", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("description", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("fieldMaskValues", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("fieldWidth", Schema.nullableOf(Schema.of(Schema.Type.INT)))); + fields.add(Schema.Field.of("id", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("initiallyChecked", Schema.nullableOf(Schema.of(Schema.Type.BOOLEAN)))); + fields.add(Schema.Field.of("isLabelToRight", Schema.nullableOf(Schema.of(Schema.Type.BOOLEAN)))); + fields.add(Schema.Field.of("isMultiselect", Schema.nullableOf(Schema.of(Schema.Type.BOOLEAN)))); + fields.add(Schema.Field.of("isRequired", Schema.nullableOf(Schema.of(Schema.Type.BOOLEAN)))); + fields.add(Schema.Field.of("labelWidth", Schema.nullableOf(Schema.of(Schema.Type.INT)))); + fields.add(Schema.Field.of("maxLength", Schema.nullableOf(Schema.of(Schema.Type.INT)))); + fields.add(Schema.Field.of("maximumNumber", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("minimumNumber", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("picklistValues", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("placeholderText", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("validationMessage", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("visibleRows", Schema.nullableOf(Schema.of(Schema.Type.INT)))); + return Schema.recordOf(UUID.randomUUID().toString().replace("-", ""), fields); + } + + public static Schema getStaticListSchema() { + List fields = new ArrayList<>(); + fields.add(Schema.Field.of("id", Schema.nullableOf(Schema.of(Schema.Type.INT)))); + fields.add(Schema.Field.of("name", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("description", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("createdAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("updatedAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("url", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("folder", Schema.nullableOf(EntitySchemaHelper.getFolderDescriptorSchema()))); + fields.add(Schema.Field.of("workspace", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("computedUrl", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + return Schema.recordOf(UUID.randomUUID().toString().replace("-", ""), fields); + } + + public static Schema getFormThankYouPageDTOSchema() { + List fields = new ArrayList<>(); + fields.add(Schema.Field.of("isDefault", Schema.nullableOf(Schema.of(Schema.Type.BOOLEAN)))); + fields.add(Schema.Field.of("followupType", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("followupValue", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("operator", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("subjectField", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("values", Schema.nullableOf(Schema.arrayOf(Schema.of(Schema.Type.STRING))))); + return Schema.recordOf(UUID.randomUUID().toString().replace("-", ""), fields); + } + + public static Schema getLandingPageTemplateSchema() { + List fields = new ArrayList<>(); + fields.add(Schema.Field.of("createdAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("description", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("enableMunchkin", Schema.nullableOf(Schema.of(Schema.Type.BOOLEAN)))); + fields.add(Schema.Field.of("folder", Schema.nullableOf(EntitySchemaHelper.getFolderDescriptorSchema()))); + fields.add(Schema.Field.of("id", Schema.nullableOf(Schema.of(Schema.Type.INT)))); + fields.add(Schema.Field.of("name", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("status", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("templateType", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("updatedAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("url", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("workspace", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + return Schema.recordOf(UUID.randomUUID().toString().replace("-", ""), fields); + } + + public static Schema getEmailSchema() { + List fields = new ArrayList<>(); + fields.add(Schema.Field.of("createdAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("description", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("folder", Schema.nullableOf(EntitySchemaHelper.getFolderDescriptorSchema()))); + fields.add(Schema.Field.of("fromEmail", Schema.nullableOf(EntitySchemaHelper.getEmailFieldSchema()))); + fields.add(Schema.Field.of("fromName", Schema.nullableOf(EntitySchemaHelper.getEmailFieldSchema()))); + fields.add(Schema.Field.of("id", Schema.nullableOf(Schema.of(Schema.Type.INT)))); + fields.add(Schema.Field.of("name", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("operational", Schema.nullableOf(Schema.of(Schema.Type.BOOLEAN)))); + fields.add(Schema.Field.of("publishToMSI", Schema.nullableOf(Schema.of(Schema.Type.BOOLEAN)))); + fields.add(Schema.Field.of("replyEmail", Schema.nullableOf(EntitySchemaHelper.getEmailFieldSchema()))); + fields.add(Schema.Field.of("status", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("subject", Schema.nullableOf(EntitySchemaHelper.getEmailFieldSchema()))); + fields.add(Schema.Field.of("template", Schema.nullableOf(Schema.of(Schema.Type.INT)))); + fields.add(Schema.Field.of("textOnly", Schema.nullableOf(Schema.of(Schema.Type.BOOLEAN)))); + fields.add(Schema.Field.of("updatedAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("url", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("version", Schema.nullableOf(Schema.of(Schema.Type.INT)))); + fields.add(Schema.Field.of("webView", Schema.nullableOf(Schema.of(Schema.Type.BOOLEAN)))); + fields.add(Schema.Field.of("workspace", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("autoCopyToText", Schema.nullableOf(Schema.of(Schema.Type.BOOLEAN)))); + fields.add(Schema.Field.of("ccFields", Schema.nullableOf( + Schema.arrayOf(EntitySchemaHelper.getEmailCCFieldSchema())))); + return Schema.recordOf(UUID.randomUUID().toString().replace("-", ""), fields); + } + + public static Schema getSmartListSchema() { + List fields = new ArrayList<>(); + fields.add(Schema.Field.of("id", Schema.nullableOf(Schema.of(Schema.Type.INT)))); + fields.add(Schema.Field.of("name", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("description", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("createdAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("updatedAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("url", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("folder", Schema.nullableOf(EntitySchemaHelper.getFolderDescriptorSchema()))); + fields.add(Schema.Field.of("workspace", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + return Schema.recordOf(UUID.randomUUID().toString().replace("-", ""), fields); + } + + public static Schema getSnippetSchema() { + List fields = new ArrayList<>(); + fields.add(Schema.Field.of("createdAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("description", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("folder", Schema.nullableOf(EntitySchemaHelper.getFolderDescriptorSchema()))); + fields.add(Schema.Field.of("id", Schema.nullableOf(Schema.of(Schema.Type.INT)))); + fields.add(Schema.Field.of("name", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("status", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("updatedAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("url", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("workspace", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + return Schema.recordOf(UUID.randomUUID().toString().replace("-", ""), fields); + } + + public static Schema getFormKnownVisitorDTOSchema() { + List fields = new ArrayList<>(); + fields.add(Schema.Field.of("template", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("type", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + return Schema.recordOf(UUID.randomUUID().toString().replace("-", ""), fields); + } + + public static Schema getEmailFieldSchema() { + List fields = new ArrayList<>(); + fields.add(Schema.Field.of("type", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("value", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + return Schema.recordOf(UUID.randomUUID().toString().replace("-", ""), fields); + } + + public static Schema getSmartCampaignSchema() { + List fields = new ArrayList<>(); + fields.add(Schema.Field.of("id", Schema.nullableOf(Schema.of(Schema.Type.INT)))); + fields.add(Schema.Field.of("name", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("description", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("type", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("isSystem", Schema.nullableOf(Schema.of(Schema.Type.BOOLEAN)))); + fields.add(Schema.Field.of("isActive", Schema.nullableOf(Schema.of(Schema.Type.BOOLEAN)))); + fields.add(Schema.Field.of("isRequestable", Schema.nullableOf(Schema.of(Schema.Type.BOOLEAN)))); + fields.add(Schema.Field.of("recurrence", Schema.nullableOf(EntitySchemaHelper.getRecurrenceSchema()))); + fields.add(Schema.Field.of("qualificationRuleType", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("qualificationRuleInterval", Schema.nullableOf(Schema.of(Schema.Type.INT)))); + fields.add(Schema.Field.of("qualificationRuleUnit", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("maxMembers", Schema.nullableOf(Schema.of(Schema.Type.INT)))); + fields.add(Schema.Field.of("isCommunicationLimitEnabled", + Schema.nullableOf(Schema.of(Schema.Type.BOOLEAN)))); + fields.add(Schema.Field.of("smartListId", Schema.nullableOf(Schema.of(Schema.Type.INT)))); + fields.add(Schema.Field.of("flowId", Schema.nullableOf(Schema.of(Schema.Type.INT)))); + fields.add(Schema.Field.of("folder", Schema.nullableOf(EntitySchemaHelper.getFolderDescriptorSchema()))); + fields.add(Schema.Field.of("createdAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("updatedAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("workspace", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("status", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + return Schema.recordOf(UUID.randomUUID().toString().replace("-", ""), fields); + } + + public static Schema getFormSchema() { + List fields = new ArrayList<>(); + fields.add(Schema.Field.of("buttonLabel", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("buttonLocation", Schema.nullableOf(Schema.of(Schema.Type.INT)))); + fields.add(Schema.Field.of("createdAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("description", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("folder", Schema.nullableOf(EntitySchemaHelper.getFolderDescriptorSchema()))); + fields.add(Schema.Field.of("fontFamily", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("fontSize", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("id", Schema.nullableOf(Schema.of(Schema.Type.INT)))); + fields.add(Schema.Field.of("knownVisitor", Schema.nullableOf(EntitySchemaHelper.getFormKnownVisitorDTOSchema()))); + fields.add(Schema.Field.of("labelPosition", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("language", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("locale", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("name", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("progressiveProfiling", Schema.nullableOf(Schema.of(Schema.Type.BOOLEAN)))); + fields.add(Schema.Field.of("status", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("thankYouList", Schema.nullableOf( + Schema.arrayOf(EntitySchemaHelper.getFormThankYouPageDTOSchema())))); + fields.add(Schema.Field.of("theme", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("updatedAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("url", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("waitingLabel", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + return Schema.recordOf(UUID.randomUUID().toString().replace("-", ""), fields); + } + + public static Schema getFolderDescriptorSchema() { + List fields = new ArrayList<>(); + fields.add(Schema.Field.of("id", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("type", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("folderName", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + return Schema.recordOf(UUID.randomUUID().toString().replace("-", ""), fields); + } + + public static Schema getRecurrenceSchema() { + List fields = new ArrayList<>(); + fields.add(Schema.Field.of("startAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("endAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("intervalType", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("interval", Schema.nullableOf(Schema.of(Schema.Type.INT)))); + fields.add(Schema.Field.of("weekdayOnly", Schema.nullableOf(Schema.of(Schema.Type.BOOLEAN)))); + fields.add(Schema.Field.of("weekdayMask", Schema.nullableOf(Schema.arrayOf(Schema.of(Schema.Type.STRING))))); + fields.add(Schema.Field.of("dayOfMonth", Schema.nullableOf(Schema.of(Schema.Type.INT)))); + fields.add(Schema.Field.of("dayOfWeek", Schema.nullableOf(Schema.of(Schema.Type.INT)))); + fields.add(Schema.Field.of("weekOfMonth", Schema.nullableOf(Schema.of(Schema.Type.INT)))); + return Schema.recordOf(UUID.randomUUID().toString().replace("-", ""), fields); + } + + public static Schema getFileSchema() { + List fields = new ArrayList<>(); + fields.add(Schema.Field.of("createdAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("description", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("folder", Schema.nullableOf(EntitySchemaHelper.getFileFolderSchema()))); + fields.add(Schema.Field.of("id", Schema.nullableOf(Schema.of(Schema.Type.INT)))); + fields.add(Schema.Field.of("mimeType", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("name", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("size", Schema.nullableOf(Schema.of(Schema.Type.INT)))); + fields.add(Schema.Field.of("updatedAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("url", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + return Schema.recordOf(UUID.randomUUID().toString().replace("-", ""), fields); + } + + public static Schema getSegmentationSchema() { + List fields = new ArrayList<>(); + fields.add(Schema.Field.of("createdAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("description", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("folder", Schema.nullableOf(EntitySchemaHelper.getFolderDescriptorSchema()))); + fields.add(Schema.Field.of("id", Schema.nullableOf(Schema.of(Schema.Type.INT)))); + fields.add(Schema.Field.of("name", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("status", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("updatedAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("url", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("workspace", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + return Schema.recordOf(UUID.randomUUID().toString().replace("-", ""), fields); + } + + public static Schema getTagSchema() { + List fields = new ArrayList<>(); + fields.add(Schema.Field.of("applicableProgramTypes", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("required", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("tagType", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + return Schema.recordOf(UUID.randomUUID().toString().replace("-", ""), fields); + } + + public static Schema getEmailTemplateSchema() { + List fields = new ArrayList<>(); + fields.add(Schema.Field.of("createdAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("description", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("folder", Schema.nullableOf(EntitySchemaHelper.getFolderDescriptorSchema()))); + fields.add(Schema.Field.of("id", Schema.nullableOf(Schema.of(Schema.Type.INT)))); + fields.add(Schema.Field.of("name", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("status", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("updatedAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("url", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("version", Schema.nullableOf(Schema.of(Schema.Type.INT)))); + fields.add(Schema.Field.of("workspace", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + return Schema.recordOf(UUID.randomUUID().toString().replace("-", ""), fields); + } + + public static Schema getEmailCCFieldSchema() { + List fields = new ArrayList<>(); + fields.add(Schema.Field.of("attributeId", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("objectName", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("displayName", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("apiName", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + return Schema.recordOf(UUID.randomUUID().toString().replace("-", ""), fields); + } + +} diff --git a/src/test/java/io/cdap/plugin/marketo/source/batch/entity/EntitySchemaHelperTest.java b/src/test/java/io/cdap/plugin/marketo/source/batch/entity/EntitySchemaHelperTest.java new file mode 100644 index 0000000..cc29097 --- /dev/null +++ b/src/test/java/io/cdap/plugin/marketo/source/batch/entity/EntitySchemaHelperTest.java @@ -0,0 +1,54 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.source.batch.entity; + +import com.google.common.collect.ImmutableList; +import io.cdap.cdap.api.data.format.StructuredRecord; +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.plugin.marketo.common.api.entities.asset.Email; +import io.cdap.plugin.marketo.common.api.entities.asset.EmailCCField; +import io.cdap.plugin.marketo.common.api.entities.asset.FolderDescriptor; +import io.cdap.plugin.marketo.common.api.entities.asset.Program; +import org.junit.Assert; +import org.junit.Test; + +import java.util.List; + +public class EntitySchemaHelperTest { + @Test + public void testNestedRecords() { + Schema programSchema = EntitySchemaHelper.getProgramSchema(); + Program program = Program.builder().folder(new FolderDescriptor("1234", "Hello", null)) + .build(); + StructuredRecord programRecord = EntitySchemaHelper.structuredRecordFromEntity("Program", program, + programSchema); + + Assert.assertEquals(programRecord.get("folder").get("id"), "1234"); + + Schema emailSchema = EntitySchemaHelper.getEmailSchema(); + Email email = Email.builder().ccFields(ImmutableList.of( + new EmailCCField("attr1", "cc1", "cc 1", "cc1"), + new EmailCCField("attr2", "cc2", "cc 2", "cc2") + )).build(); + StructuredRecord emailRecord = EntitySchemaHelper.structuredRecordFromEntity("Email", email, + emailSchema); + + List ccRecords = emailRecord.>get("ccFields"); + Assert.assertEquals(2, ccRecords.size()); + Assert.assertEquals(ccRecords.get(0).get("attributeId"), "attr1"); + } +} From ba5ea6fc6ee10fce80410b7dcbb2487e03b90dd8 Mon Sep 17 00:00:00 2001 From: Yevhenii Chekanskyi Date: Fri, 6 Dec 2019 13:25:43 +0200 Subject: [PATCH 15/16] PLUGIN-75 Marketo Plugin - refactor generator. --- .gitignore | 2 +- contrib/Generate.java | 236 --- contrib/pom.xml | 66 + .../io/cdap/plugin/generate/Generate.java | 335 ++++ .../common/api/entities/asset/Email.java | 2 +- .../api/entities/asset/EmailTemplate.java | 2 +- .../common/api/entities/asset/File.java | 2 +- .../common/api/entities/asset/Folder.java | 2 +- .../common/api/entities/asset/Form.java | 2 +- .../common/api/entities/asset/FormField.java | 2 +- .../api/entities/asset/LandingPage.java | 2 +- .../entities/asset/LandingPageTemplate.java | 2 +- .../common/api/entities/asset/Program.java | 2 +- .../api/entities/asset/Segmentation.java | 2 +- .../api/entities/asset/SmartCampaign.java | 2 +- .../common/api/entities/asset/SmartList.java | 2 +- .../common/api/entities/asset/Snippet.java | 2 +- .../common/api/entities/asset/StaticList.java | 2 +- .../common/api/entities/asset/Tag.java | 2 +- .../common/api/entities/asset/gen/Entity.java | 1 + .../batch/entity/EntitySchemaHelper.java | 1347 +++++++++-------- 21 files changed, 1122 insertions(+), 895 deletions(-) delete mode 100644 contrib/Generate.java create mode 100644 contrib/pom.xml create mode 100644 contrib/src/main/java/io/cdap/plugin/generate/Generate.java diff --git a/.gitignore b/.gitignore index 8731f78..cd62d2e 100644 --- a/.gitignore +++ b/.gitignore @@ -9,7 +9,7 @@ *.versionsBackup # Intellij Files & Dir # -*.iml +**/*.iml *.ipr *.iws atlassian-ide-plugin.xml diff --git a/contrib/Generate.java b/contrib/Generate.java deleted file mode 100644 index 6b8e3b5..0000000 --- a/contrib/Generate.java +++ /dev/null @@ -1,236 +0,0 @@ -/* - * Copyright © 2019 Cask Data, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package io.cdap.plugin.marketo.common.api.entities.asset.gen; - -import com.google.common.collect.ImmutableList; -import com.google.common.reflect.ClassPath; - -import java.io.IOException; -import java.lang.annotation.Annotation; -import java.lang.reflect.Field; -import java.lang.reflect.ParameterizedType; -import java.util.List; -import java.util.stream.Collectors; - -/** - * Class that generates schema for entities. - * This class is not required anymore in near future and not intended to be used by developers... - * ...but can be used later again if some massive changes to schema will be required. - * - * Requires guava. - */ -public class Generate { - public static void main(String... args) throws IOException { - System.out.println("@SuppressWarnings(\"DuplicatedCode\")"); - System.out.println("public class EntitySchemaHelper {"); - - // entity name to schema - System.out.println(" public static Schema schemaForEntityName(String entityName) {"); - System.out.println(" switch(entityName) {"); - getEntityClasses().forEach(aClass -> { - System.out.println(" case \"" + aClass.getSimpleName() + "\":"); - System.out.println(" return " + generateSchemaGetterMethod(aClass) + "();"); - }); - System.out.println(" default:\n"); - System.out.println(" throw new IllegalArgumentException(\"Unknown entity name:\" + entityName);"); - System.out.println(" }"); - System.out.println(" }"); - - // entity object to structured record - System.out.println(" public static StructuredRecord structuredRecordFromEntity(String entityName, Object " + - "entity, Schema schema) {"); - System.out.println(" StructuredRecord.Builder builder = StructuredRecord.builder(schema);"); - System.out.println(" switch(entityName) {"); - getEntityClasses().forEach(aClass -> { - System.out.println(" case \"" + aClass.getSimpleName() + "\":"); - System.out.println(" " + aClass.getSimpleName() + " " + aClass.getSimpleName().toLowerCase() + " = (" - + aClass.getSimpleName() + ") entity;"); - System.out.println(" if (entity == null) {\n" + - " break;\n" + - " }"); - for (Field field : aClass.getDeclaredFields()) { - if (isSimpleType(field)) { - System.out.println(" builder.set(\"" + field.getName() + "\", " + aClass.getSimpleName().toLowerCase() - + "." + findGetterMethod(field, aClass) + "());"); - } else if (isComplexType(field)) { - System.out.println(" builder.set(\"" + field.getName() + "\", EntitySchemaHelper" + - ".structuredRecordFromEntity(\n" + - " \"" + field.getType().getSimpleName() + "\",\n" + - " " + aClass.getSimpleName().toLowerCase() + "." - + findGetterMethod(field, aClass) + "()" + ",\n" + - " schema.getField(\"" + field.getName() + "\").getSchema().getNonNullable())\n" - + " );"); - } else if (isListType(field)) { - ParameterizedType integerListType = (ParameterizedType) field.getGenericType(); - Class listType = (Class) integerListType.getActualTypeArguments()[0]; - if (isSimpleType(listType)) { - System.out.println(" builder.set(\"" + field.getName() + "\", " - + aClass.getSimpleName().toLowerCase() + "." + findGetterMethod(field, aClass) - + "());"); - } else { - System.out.println(" builder.set(\n" + - " \"" + field.getName() + "\",\n" + - " " + aClass.getSimpleName().toLowerCase() + "." - + findGetterMethod(field, aClass) + "().stream()\n" + - " .map(ent -> EntitySchemaHelper.structuredRecordFromEntity(\n" + - " \"" + listType.getSimpleName() + "\",\n" + - " ent,\n" + - " schema.getField(\"" + field.getName() + "\").getSchema()" + - ".getNonNullable().getComponentSchema()))\n" + - " .collect(Collectors.toList())\n" + - " );"); - } - } else { - throw new RuntimeException("Unknown field type."); - } - } - System.out.println(" break;"); - }); - System.out.println(" default:\n"); - System.out.println(" throw new IllegalArgumentException(\"Unknown entity name:\" + entityName);"); - System.out.println(" }"); - System.out.println(""); - System.out.println(" return builder.build();"); - System.out.println(" }"); - - // individual schemas - getEntityClasses().forEach(aClass -> { - System.out.println(" public static Schema " + generateSchemaGetterMethod(aClass) + "() {"); - System.out.println(" List fields = new ArrayList<>();"); - for (Field field : aClass.getDeclaredFields()) { - if (isSimpleType(field)) { - System.out.println(" fields.add(Schema.Field.of(\"" + field.getName() + "\", " - + simpleTypeToCdapType(field) + "));"); - } else if (isComplexType(field)) { - System.out.println(" fields.add(Schema.Field.of(\"" + field.getName() + "\", " + - "Schema.nullableOf(EntitySchemaHelper." - + generateSchemaGetterMethod(field.getType()) + "())));"); - } else if (isListType(field)) { - System.out.println(" fields.add(Schema.Field.of(\"" + field.getName() + "\", " - + listTypeToCdapType(field) + "));"); - } else { - throw new RuntimeException("Unknown field type."); - } - } - System.out.println(" return Schema.recordOf(UUID.randomUUID().toString().replace(\"-\", \"\"), fields);"); - System.out.println(" }"); - System.out.println(""); - }); - System.out.println("}"); - } - - private static final List SIMPLE_TYPES = ImmutableList.of(Boolean.class, Integer.class, int.class, - boolean.class, String.class); - - private static final List COLLECTION_TYPES = ImmutableList.of(List.class); - - public static boolean isSimpleType(Field field) { - return SIMPLE_TYPES.contains(field.getType()); - } - - public static boolean isSimpleType(Class cls) { - return SIMPLE_TYPES.contains(cls); - } - - public static String generateSchemaGetterMethod(Class cls) { - return "get" + capitalize(cls.getSimpleName()) + "Schema"; - } - - public static boolean isListType(Field field) { - return COLLECTION_TYPES.contains(field.getType()); - } - - public static boolean isComplexType(Field field) { - return !SIMPLE_TYPES.contains(field.getType()) && !COLLECTION_TYPES.contains(field.getType()); - } - - public static String simpleTypeToCdapType(Field field) { - if (field.getType().equals(Boolean.class) || field.getType().equals(boolean.class)) { - return "Schema.nullableOf(Schema.of(Schema.Type.BOOLEAN))"; - } else if (field.getType().equals(String.class)) { - return "Schema.nullableOf(Schema.of(Schema.Type.STRING))"; - } else if (field.getType().equals(Integer.class) || field.getType().equals(int.class)) { - return "Schema.nullableOf(Schema.of(Schema.Type.INT))"; - } else { - throw new RuntimeException("Unsupported simple type"); - } - } - - public static String listTypeToCdapType(Field field) { - ParameterizedType integerListType = (ParameterizedType) field.getGenericType(); - Class listType = (Class) integerListType.getActualTypeArguments()[0]; - if (listType.equals(Boolean.class) || listType.equals(boolean.class)) { - return "SSchema.nullableOf(Schema.arrayOf(Schema.of(Schema.Type.BOOLEAN)))"; - } else if (listType.equals(String.class)) { - return "Schema.nullableOf(Schema.arrayOf(Schema.of(Schema.Type.STRING)))"; - } else if (listType.equals(Integer.class) || listType.equals(int.class)) { - return "Schema.nullableOf(Schema.arrayOf(Schema.of(Schema.Type.INT)))"; - } else { - return "Schema.nullableOf(Schema.arrayOf(EntitySchemaHelper." + generateSchemaGetterMethod(listType) + "()))"; - } - } - - public static String findGetterMethod(Field field, Class cls) { - if (field.getType().equals(Boolean.class) || field.getType().equals(boolean.class)) { - String getterName = "get" + capitalize(field.getName()); - try { - cls.getMethod(getterName); - return getterName; - } catch (NoSuchMethodException e) { - if (field.getName().startsWith("is")) { - String alternateGetter = "get" + capitalize(field.getName().substring(2)); - try { - cls.getMethod(alternateGetter); - return alternateGetter; - } catch (NoSuchMethodException altE) { - throw new RuntimeException(altE); - } - } - } - } else { - String getterName = "get" + capitalize(field.getName()); - try { - cls.getMethod(getterName); - return getterName; - } catch (NoSuchMethodException altE) { - throw new RuntimeException(altE); - } - } - throw new RuntimeException(); - } - - public static String capitalize(String str) { - return str.substring(0, 1).toUpperCase() + str.substring(1); - } - - public static List getEntityClasses() throws IOException { - return ClassPath.from(Generate.class.getClassLoader()) - .getTopLevelClasses("io.cdap.plugin.marketo.common.api.entities.asset").stream() - .map(ClassPath.ClassInfo::load) - .filter(Generate::isEntity) - .collect(Collectors.toList()); - } - - public static boolean isEntity(Class cls) { - for (Annotation a : cls.getAnnotations()) { - if (a.annotationType().equals(Entity.class)) { - return true; - } - } - return false; - } -} diff --git a/contrib/pom.xml b/contrib/pom.xml new file mode 100644 index 0000000..1c8fba3 --- /dev/null +++ b/contrib/pom.xml @@ -0,0 +1,66 @@ + + + + 4.0.0 + + io.cdap.plugin + marketo-entity-generator + 1.0.0-SNAPSHOT + + + + org.apache.maven.plugins + maven-compiler-plugin + + 8 + 8 + + + + + + + 6.1.0-SNAPSHOT + + + + + io.cdap.cdap + cdap-api-common + ${cdap.version} + + + com.google.guava + guava + 28.1-jre + + + com.squareup + javapoet + 1.11.1 + + + io.cdap.plugin + marketo-entity-plugin + system + 1.0.0-SNAPSHOT + ${basedir}/../target/marketo-entity-plugin-1.0.0-SNAPSHOT.jar + + + \ No newline at end of file diff --git a/contrib/src/main/java/io/cdap/plugin/generate/Generate.java b/contrib/src/main/java/io/cdap/plugin/generate/Generate.java new file mode 100644 index 0000000..5c15348 --- /dev/null +++ b/contrib/src/main/java/io/cdap/plugin/generate/Generate.java @@ -0,0 +1,335 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.generate; + +import com.google.common.collect.ImmutableList; +import com.google.common.reflect.ClassPath; +import com.squareup.javapoet.ClassName; +import com.squareup.javapoet.JavaFile; +import com.squareup.javapoet.MethodSpec; +import com.squareup.javapoet.TypeSpec; +import io.cdap.cdap.api.data.format.StructuredRecord; +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.plugin.marketo.common.api.entities.asset.gen.Entity; + +import java.io.IOException; +import java.lang.annotation.Annotation; +import java.lang.reflect.Field; +import java.lang.reflect.ParameterizedType; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.stream.Collectors; +import javax.lang.model.element.Modifier; + +/** + * Class that generates schema for entities. + * This class is not required anymore in near future and not intended to be used by end users... + * ...but can be used later again if some massive changes to schema will be required. + *

+ * Requires guava. + */ +public class Generate { + public static void main(String... args) throws IOException { + List methods = new ArrayList<>(); + + MethodSpec.Builder schemaForEntityNameBuilder = MethodSpec.methodBuilder("schemaForEntityName") + .addModifiers(Modifier.PUBLIC, Modifier.STATIC) + .returns(Schema.class) + .addParameter(String.class, "entityName"); + + schemaForEntityNameBuilder.beginControlFlow("switch(entityName)"); + getEntityClasses().forEach(aClass -> { + schemaForEntityNameBuilder.beginControlFlow("case $S:", aClass.getSimpleName()); + schemaForEntityNameBuilder.addStatement("return $L()", generateSchemaGetterMethod(aClass)); + schemaForEntityNameBuilder.endControlFlow(); + }); + schemaForEntityNameBuilder.beginControlFlow("default:"); + schemaForEntityNameBuilder.addStatement( + "throw new IllegalArgumentException(\"Unknown entity name:\" + entityName)"); + schemaForEntityNameBuilder.endControlFlow(); + schemaForEntityNameBuilder.endControlFlow(); + + methods.add(schemaForEntityNameBuilder.build()); + + // individual schemas + getEntityClasses().forEach(aClass -> { + MethodSpec.Builder entitySchemaGenBuilder = MethodSpec.methodBuilder(generateSchemaGetterMethod(aClass)) + .addModifiers(Modifier.PUBLIC, Modifier.STATIC) + .returns(Schema.class) + .addStatement("$T fields = new $T<>()", List.class, ArrayList.class); + for (Field field : aClass.getDeclaredFields()) { + if (isSimpleType(field)) { + entitySchemaGenBuilder.addStatement("fields.add(Schema.Field.of($S, $L))", field.getName(), + simpleTypeToCdapType(field)); + } else if (isComplexType(field)) { + entitySchemaGenBuilder.addStatement( + "fields.add(Schema.Field.of($S, Schema.nullableOf(EntitySchemaHelper.$L())))", + field.getName(), generateSchemaGetterMethod(field.getType())); + } else if (isListType(field)) { + entitySchemaGenBuilder.addStatement( + "fields.add(Schema.Field.of($S, $L))", + field.getName(), listTypeToCdapType(field)); + } else { + throw new RuntimeException("Unknown field type."); + } + } + entitySchemaGenBuilder.addStatement( + "return Schema.recordOf(UUID.randomUUID().toString().replace(\"-\", \"\"), fields)"); + methods.add(entitySchemaGenBuilder.build()); + }); + + // entity object to structured record + MethodSpec.Builder recordFromEntityBuilder = MethodSpec.methodBuilder("structuredRecordFromEntity") + .addModifiers(Modifier.PUBLIC, Modifier.STATIC) + .returns(StructuredRecord.class) + .addParameter(String.class, "entityName") + .addParameter(Object.class, "entity") + .addParameter(Schema.class, "schema") + .addStatement("StructuredRecord.Builder builder = StructuredRecord.builder(schema)") + .beginControlFlow("switch(entityName)"); + getEntityClasses().forEach(aClass -> { + recordFromEntityBuilder.beginControlFlow("case $S:", aClass.getSimpleName()); + recordFromEntityBuilder.addStatement("$T $L = ($L) entity", + aClass, aClass.getSimpleName().toLowerCase(), + aClass.getSimpleName()); + + recordFromEntityBuilder.beginControlFlow("if (entity == null)"); + recordFromEntityBuilder.addStatement("break"); + recordFromEntityBuilder.endControlFlow(); + for (Field field : aClass.getDeclaredFields()) { + if (isSimpleType(field)) { + recordFromEntityBuilder.addStatement("builder.set($S, $L.$L())", + field.getName(), + aClass.getSimpleName().toLowerCase(), + findGetterMethod(field, aClass)); + } else if (isComplexType(field)) { + recordFromEntityBuilder.addStatement("builder.set($S, EntitySchemaHelper.structuredRecordFromEntity(\n" + + " $S,\n" + + " $L.$L(),\n" + + " schema.getField($S).getSchema().getNonNullable()))", + field.getName(), + field.getType().getSimpleName(), + aClass.getSimpleName().toLowerCase(), + findGetterMethod(field, aClass), + field.getName()); + } else if (isListType(field)) { + ParameterizedType integerListType = (ParameterizedType) field.getGenericType(); + Class listType = (Class) integerListType.getActualTypeArguments()[0]; + if (isSimpleType(listType)) { + recordFromEntityBuilder.addStatement("builder.set($S, $L.$L())", + field.getName(), + aClass.getSimpleName().toLowerCase(), + findGetterMethod(field, aClass)); + } else { + recordFromEntityBuilder.addStatement( + "builder.set(\n" + + " $S,\n" + + " $L.$L().stream()\n" + + " .map(ent -> EntitySchemaHelper.structuredRecordFromEntity(\n" + + " $S,\n" + + " ent,\n" + + " schema.getField($S).getSchema().getNonNullable().getComponentSchema()))\n" + + " .collect(Collectors.toList())\n" + + ")", + field.getName(), aClass.getSimpleName().toLowerCase(), findGetterMethod(field, aClass), + listType.getSimpleName(), field.getName() + ); + } + } else { + throw new RuntimeException("Unknown field type."); + } + } + recordFromEntityBuilder.addStatement("break"); + recordFromEntityBuilder.endControlFlow(); + }); + + recordFromEntityBuilder.beginControlFlow("default:"); + recordFromEntityBuilder.addStatement( + "throw new IllegalArgumentException(\"Unknown entity name:\" + entityName)"); + recordFromEntityBuilder.endControlFlow(); + recordFromEntityBuilder.endControlFlow(); // switch statement + recordFromEntityBuilder.addStatement("return builder.build()"); + methods.add(recordFromEntityBuilder.build()); + + TypeSpec.Builder entityTypeEnumBuilder = TypeSpec.enumBuilder("EntityType") + .addModifiers(Modifier.PUBLIC, Modifier.STATIC) + .addField(String.class, "value", Modifier.PRIVATE, Modifier.FINAL); + + getTopLevelEntityClasses().forEach(aClass -> { + entityTypeEnumBuilder.addEnumConstant( + aClass.getSimpleName().toUpperCase(), + TypeSpec.anonymousClassBuilder("$S", aClass.getSimpleName()).build() + ); + }); + + entityTypeEnumBuilder.addMethod( + MethodSpec.constructorBuilder() + .addParameter(String.class, "value") + .addStatement("this.$N = $N", "value", "value") + .build() + ); + + entityTypeEnumBuilder.addMethod( + MethodSpec.methodBuilder("fromString") + .addModifiers(Modifier.PUBLIC, Modifier.STATIC) + .returns(ClassName.bestGuess("EntityType")) + .addParameter(String.class, "value") + .beginControlFlow("for (EntityType entityType : EntityType.values())") + .beginControlFlow("if (entityType.value.equals(value))") + .addStatement("return entityType") + .endControlFlow() + .endControlFlow() + .addStatement("throw new IllegalArgumentException(\"Unknown entity type type: \" + value)") + .build() + ); + + TypeSpec schemaHelperSpec = TypeSpec.classBuilder("EntitySchemaHelper") + .addModifiers(Modifier.PUBLIC, Modifier.FINAL) + .addMethods(methods) + .addType(entityTypeEnumBuilder.build()) + .build(); + + JavaFile javaFile = JavaFile.builder("io.cdap.plugin.marketo.source.batch.entity", schemaHelperSpec) + .skipJavaLangImports(true) + .build(); + System.out.println(javaFile.toString()); + } + + private static final List SIMPLE_TYPES = ImmutableList.of(Boolean.class, Integer.class, int.class, + boolean.class, String.class); + + private static final List COLLECTION_TYPES = ImmutableList.of(List.class); + + public static boolean isSimpleType(Field field) { + return SIMPLE_TYPES.contains(field.getType()); + } + + public static boolean isSimpleType(Class cls) { + return SIMPLE_TYPES.contains(cls); + } + + public static String generateSchemaGetterMethod(Class cls) { + return "get" + capitalize(cls.getSimpleName()) + "Schema"; + } + + public static boolean isListType(Field field) { + return COLLECTION_TYPES.contains(field.getType()); + } + + public static boolean isComplexType(Field field) { + return !SIMPLE_TYPES.contains(field.getType()) && !COLLECTION_TYPES.contains(field.getType()); + } + + public static String simpleTypeToCdapType(Field field) { + if (field.getType().equals(Boolean.class) || field.getType().equals(boolean.class)) { + return "Schema.nullableOf(Schema.of(Schema.Type.BOOLEAN))"; + } else if (field.getType().equals(String.class)) { + return "Schema.nullableOf(Schema.of(Schema.Type.STRING))"; + } else if (field.getType().equals(Integer.class) || field.getType().equals(int.class)) { + return "Schema.nullableOf(Schema.of(Schema.Type.INT))"; + } else { + throw new RuntimeException("Unsupported simple type"); + } + } + + public static String listTypeToCdapType(Field field) { + ParameterizedType integerListType = (ParameterizedType) field.getGenericType(); + Class listType = (Class) integerListType.getActualTypeArguments()[0]; + if (listType.equals(Boolean.class) || listType.equals(boolean.class)) { + return "SSchema.nullableOf(Schema.arrayOf(Schema.of(Schema.Type.BOOLEAN)))"; + } else if (listType.equals(String.class)) { + return "Schema.nullableOf(Schema.arrayOf(Schema.of(Schema.Type.STRING)))"; + } else if (listType.equals(Integer.class) || listType.equals(int.class)) { + return "Schema.nullableOf(Schema.arrayOf(Schema.of(Schema.Type.INT)))"; + } else { + return "Schema.nullableOf(Schema.arrayOf(EntitySchemaHelper." + generateSchemaGetterMethod(listType) + "()))"; + } + } + + public static String findGetterMethod(Field field, Class cls) { + if (field.getType().equals(Boolean.class) || field.getType().equals(boolean.class)) { + String getterName = "get" + capitalize(field.getName()); + try { + cls.getMethod(getterName); + return getterName; + } catch (NoSuchMethodException e) { + if (field.getName().startsWith("is")) { + String alternateGetter = "get" + capitalize(field.getName().substring(2)); + try { + cls.getMethod(alternateGetter); + return alternateGetter; + } catch (NoSuchMethodException altE) { + throw new RuntimeException(altE); + } + } + } + } else { + String getterName = "get" + capitalize(field.getName()); + try { + cls.getMethod(getterName); + return getterName; + } catch (NoSuchMethodException altE) { + throw new RuntimeException(altE); + } + } + throw new RuntimeException(); + } + + public static String capitalize(String str) { + return str.substring(0, 1).toUpperCase() + str.substring(1); + } + + public static List getEntityClasses() throws IOException { + return ClassPath.from(Generate.class.getClassLoader()) + .getTopLevelClasses("io.cdap.plugin.marketo.common.api.entities.asset").stream() + .map(ClassPath.ClassInfo::load) + .filter(Generate::isEntity) + .sorted(Comparator.comparing(Class::getSimpleName)) + .collect(Collectors.toList()); + } + + public static List getTopLevelEntityClasses() throws IOException { + return ClassPath.from(Generate.class.getClassLoader()) + .getTopLevelClasses("io.cdap.plugin.marketo.common.api.entities.asset").stream() + .map(ClassPath.ClassInfo::load) + .filter(Generate::isTopLevelEntity) + .sorted(Comparator.comparing(Class::getSimpleName)) + .collect(Collectors.toList()); + } + + public static boolean isEntity(Class cls) { + for (Annotation a : cls.getAnnotations()) { + if (a.annotationType().equals(Entity.class)) { + return true; + } + } + return false; + } + + public static boolean isTopLevelEntity(Class cls) { + for (Annotation a : cls.getAnnotations()) { + if (a.annotationType().equals(Entity.class)) { + Entity e = (Entity) a; + if (e.topLevel()) { + return true; + } + } + } + return false; + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Email.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Email.java index a1bbd91..d7da1d2 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Email.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Email.java @@ -25,7 +25,7 @@ /** * Email entity. */ -@Entity +@Entity(topLevel = true) public class Email { String createdAt; String description; diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/EmailTemplate.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/EmailTemplate.java index 9b7209b..b54e3f7 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/EmailTemplate.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/EmailTemplate.java @@ -21,7 +21,7 @@ /** * Email template entity. */ -@Entity +@Entity(topLevel = true) public class EmailTemplate { String createdAt; String description; diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/File.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/File.java index 2fcc0b2..3e54994 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/File.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/File.java @@ -21,7 +21,7 @@ /** * File entity. */ -@Entity +@Entity(topLevel = true) public class File { String createdAt; String description; diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Folder.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Folder.java index f07b693..84eb790 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Folder.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Folder.java @@ -21,7 +21,7 @@ /** * Folder entity. */ -@Entity +@Entity(topLevel = true) public class Folder { Integer accessZoneId; String createdAt; diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Form.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Form.java index 74eed78..3893896 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Form.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Form.java @@ -24,7 +24,7 @@ /** * Form entity. */ -@Entity +@Entity(topLevel = true) public class Form { String buttonLabel; Integer buttonLocation; diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/FormField.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/FormField.java index a97bdaf..a268fac 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/FormField.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/FormField.java @@ -21,7 +21,7 @@ /** * Form filed entity. */ -@Entity +@Entity(topLevel = true) public class FormField { String dataType; String defaultValue; diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/LandingPage.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/LandingPage.java index 959eb55..f12e8fa 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/LandingPage.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/LandingPage.java @@ -21,7 +21,7 @@ /** * Landing page entity. */ -@Entity +@Entity(topLevel = true) public class LandingPage { String url; String computedUrl; diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/LandingPageTemplate.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/LandingPageTemplate.java index 2179236..ea692cb 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/LandingPageTemplate.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/LandingPageTemplate.java @@ -21,7 +21,7 @@ /** * Landing page template entity. */ -@Entity +@Entity(topLevel = true) public class LandingPageTemplate { String createdAt; String description; diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Program.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Program.java index 0f325ee..4dbb36d 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Program.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Program.java @@ -21,7 +21,7 @@ /** * Program entity. */ -@Entity +@Entity(topLevel = true) public class Program { String channel; String createdAt; diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Segmentation.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Segmentation.java index a91e0a1..0dcbeb7 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Segmentation.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Segmentation.java @@ -21,7 +21,7 @@ /** * Segmentation entity. */ -@Entity +@Entity(topLevel = true) public class Segmentation { String createdAt; String description; diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/SmartCampaign.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/SmartCampaign.java index 338d23e..d6e9a8d 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/SmartCampaign.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/SmartCampaign.java @@ -21,7 +21,7 @@ /** * SmartCampaign entity. */ -@Entity +@Entity(topLevel = true) public class SmartCampaign { Integer id; String name; diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/SmartList.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/SmartList.java index cfcddf4..e64bdc4 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/SmartList.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/SmartList.java @@ -21,7 +21,7 @@ /** * SmartList entity. */ -@Entity +@Entity(topLevel = true) public class SmartList { Integer id; String name; diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Snippet.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Snippet.java index d8b108e..b4dbea6 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Snippet.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Snippet.java @@ -21,7 +21,7 @@ /** * Snippet entity. */ -@Entity +@Entity(topLevel = true) public class Snippet { String createdAt; String description; diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/StaticList.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/StaticList.java index 7d2b6cd..7737703 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/StaticList.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/StaticList.java @@ -21,7 +21,7 @@ /** * StaticList entity. */ -@Entity +@Entity(topLevel = true) public class StaticList { Integer id; String name; diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Tag.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Tag.java index ce5bd70..b001370 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Tag.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/Tag.java @@ -21,7 +21,7 @@ /** * Tag entity. */ -@Entity +@Entity(topLevel = true) public class Tag { String applicableProgramTypes; String required; diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/gen/Entity.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/gen/Entity.java index 135e87b..9e28c1a 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/gen/Entity.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/gen/Entity.java @@ -24,4 +24,5 @@ */ @Retention(RetentionPolicy.RUNTIME) public @interface Entity { + boolean topLevel() default false; } diff --git a/src/main/java/io/cdap/plugin/marketo/source/batch/entity/EntitySchemaHelper.java b/src/main/java/io/cdap/plugin/marketo/source/batch/entity/EntitySchemaHelper.java index 72f2661..785c1c2 100644 --- a/src/main/java/io/cdap/plugin/marketo/source/batch/entity/EntitySchemaHelper.java +++ b/src/main/java/io/cdap/plugin/marketo/source/batch/entity/EntitySchemaHelper.java @@ -50,553 +50,150 @@ * Schema helper for entity plugin. */ @SuppressWarnings("DuplicatedCode") -public class EntitySchemaHelper { +public final class EntitySchemaHelper { public static Schema schemaForEntityName(String entityName) { switch (entityName) { - case "Program": - return getProgramSchema(); - case "Folder": - return getFolderSchema(); - case "FileFolder": + case "Email": { + return getEmailSchema(); + } + case "EmailCCField": { + return getEmailCCFieldSchema(); + } + case "EmailField": { + return getEmailFieldSchema(); + } + case "EmailTemplate": { + return getEmailTemplateSchema(); + } + case "File": { + return getFileSchema(); + } + case "FileFolder": { return getFileFolderSchema(); - case "LandingPage": - return getLandingPageSchema(); - case "FormField": + } + case "Folder": { + return getFolderSchema(); + } + case "FolderDescriptor": { + return getFolderDescriptorSchema(); + } + case "Form": { + return getFormSchema(); + } + case "FormField": { return getFormFieldSchema(); - case "StaticList": - return getStaticListSchema(); - case "FormThankYouPageDTO": + } + case "FormKnownVisitorDTO": { + return getFormKnownVisitorDTOSchema(); + } + case "FormThankYouPageDTO": { return getFormThankYouPageDTOSchema(); - case "LandingPageTemplate": + } + case "LandingPage": { + return getLandingPageSchema(); + } + case "LandingPageTemplate": { return getLandingPageTemplateSchema(); - case "Email": - return getEmailSchema(); - case "SmartList": - return getSmartListSchema(); - case "Snippet": - return getSnippetSchema(); - case "FormKnownVisitorDTO": - return getFormKnownVisitorDTOSchema(); - case "EmailField": - return getEmailFieldSchema(); - case "SmartCampaign": - return getSmartCampaignSchema(); - case "Form": - return getFormSchema(); - case "FolderDescriptor": - return getFolderDescriptorSchema(); - case "Recurrence": + } + case "Program": { + return getProgramSchema(); + } + case "Recurrence": { return getRecurrenceSchema(); - case "File": - return getFileSchema(); - case "Segmentation": + } + case "Segmentation": { return getSegmentationSchema(); - case "Tag": + } + case "SmartCampaign": { + return getSmartCampaignSchema(); + } + case "SmartList": { + return getSmartListSchema(); + } + case "Snippet": { + return getSnippetSchema(); + } + case "StaticList": { + return getStaticListSchema(); + } + case "Tag": { return getTagSchema(); - case "EmailTemplate": - return getEmailTemplateSchema(); - case "EmailCCField": - return getEmailCCFieldSchema(); - default: - + } + default: { throw new IllegalArgumentException("Unknown entity name:" + entityName); + } } } - public static StructuredRecord structuredRecordFromEntity(String entityName, Object entity, Schema schema) { - StructuredRecord.Builder builder = StructuredRecord.builder(schema); - switch (entityName) { - case "Program": - Program program = (Program) entity; - if (entity == null) { - break; - } - builder.set("channel", program.getChannel()); - builder.set("createdAt", program.getCreatedAt()); - builder.set("description", program.getDescription()); - builder.set("folder", EntitySchemaHelper.structuredRecordFromEntity( - "FolderDescriptor", - program.getFolder(), - schema.getField("folder").getSchema().getNonNullable()) - ); - builder.set("id", program.getId()); - builder.set("name", program.getName()); - builder.set("sfdcId", program.getSfdcId()); - builder.set("sfdcName", program.getSfdcName()); - builder.set("status", program.getStatus()); - builder.set("type", program.getType()); - builder.set("updatedAt", program.getUpdatedAt()); - builder.set("url", program.getUrl()); - builder.set("workspace", program.getWorkspace()); - break; - case "Folder": - Folder folder = (Folder) entity; - if (entity == null) { - break; - } - builder.set("accessZoneId", folder.getAccessZoneId()); - builder.set("createdAt", folder.getCreatedAt()); - builder.set("description", folder.getDescription()); - builder.set("folderId", EntitySchemaHelper.structuredRecordFromEntity( - "FolderDescriptor", - folder.getFolderId(), - schema.getField("folderId").getSchema().getNonNullable()) - ); - builder.set("folderType", folder.getFolderType()); - builder.set("id", folder.getId()); - builder.set("isArchive", folder.getArchive()); - builder.set("isSystem", folder.getSystem()); - builder.set("name", folder.getName()); - builder.set("parent", EntitySchemaHelper.structuredRecordFromEntity( - "FolderDescriptor", - folder.getParent(), - schema.getField("parent").getSchema().getNonNullable()) - ); - builder.set("path", folder.getPath()); - builder.set("updatedAt", folder.getUpdatedAt()); - builder.set("url", folder.getUrl()); - builder.set("workspace", folder.getWorkspace()); - break; - case "FileFolder": - FileFolder filefolder = (FileFolder) entity; - if (entity == null) { - break; - } - builder.set("id", filefolder.getId()); - builder.set("name", filefolder.getName()); - builder.set("type", filefolder.getType()); - break; - case "LandingPage": - LandingPage landingpage = (LandingPage) entity; - if (entity == null) { - break; - } - builder.set("url", landingpage.getUrl()); - builder.set("computedUrl", landingpage.getComputedUrl()); - builder.set("createdAt", landingpage.getCreatedAt()); - builder.set("customHeadHTML", landingpage.getCustomHeadHTML()); - builder.set("description", landingpage.getDescription()); - builder.set("facebookOgTags", landingpage.getFacebookOgTags()); - builder.set("folder", EntitySchemaHelper.structuredRecordFromEntity( - "FolderDescriptor", - landingpage.getFolder(), - schema.getField("folder").getSchema().getNonNullable()) - ); - builder.set("formPrefill", landingpage.getFormPrefill()); - builder.set("id", landingpage.getId()); - builder.set("keywords", landingpage.getKeywords()); - builder.set("mobileEnabled", landingpage.getMobileEnabled()); - builder.set("name", landingpage.getName()); - builder.set("robots", landingpage.getRobots()); - builder.set("status", landingpage.getStatus()); - builder.set("template", landingpage.getTemplate()); - builder.set("title", landingpage.getTitle()); - builder.set("updatedAt", landingpage.getUpdatedAt()); - builder.set("workspace", landingpage.getWorkspace()); - break; - case "FormField": - FormField formfield = (FormField) entity; - if (entity == null) { - break; - } - builder.set("dataType", formfield.getDataType()); - builder.set("defaultValue", formfield.getDefaultValue()); - builder.set("description", formfield.getDescription()); - builder.set("fieldMaskValues", formfield.getFieldMaskValues()); - builder.set("fieldWidth", formfield.getFieldWidth()); - builder.set("id", formfield.getId()); - builder.set("initiallyChecked", formfield.getInitiallyChecked()); - builder.set("isLabelToRight", formfield.getLabelToRight()); - builder.set("isMultiselect", formfield.getMultiselect()); - builder.set("isRequired", formfield.getRequired()); - builder.set("labelWidth", formfield.getLabelWidth()); - builder.set("maxLength", formfield.getMaxLength()); - builder.set("maximumNumber", formfield.getMaximumNumber()); - builder.set("minimumNumber", formfield.getMinimumNumber()); - builder.set("picklistValues", formfield.getPicklistValues()); - builder.set("placeholderText", formfield.getPlaceholderText()); - builder.set("validationMessage", formfield.getValidationMessage()); - builder.set("visibleRows", formfield.getVisibleRows()); - break; - case "StaticList": - StaticList staticlist = (StaticList) entity; - if (entity == null) { - break; - } - builder.set("id", staticlist.getId()); - builder.set("name", staticlist.getName()); - builder.set("description", staticlist.getDescription()); - builder.set("createdAt", staticlist.getCreatedAt()); - builder.set("updatedAt", staticlist.getUpdatedAt()); - builder.set("url", staticlist.getUrl()); - builder.set("folder", EntitySchemaHelper.structuredRecordFromEntity( - "FolderDescriptor", - staticlist.getFolder(), - schema.getField("folder").getSchema().getNonNullable()) - ); - builder.set("workspace", staticlist.getWorkspace()); - builder.set("computedUrl", staticlist.getComputedUrl()); - break; - case "FormThankYouPageDTO": - FormThankYouPageDTO formthankyoupagedto = (FormThankYouPageDTO) entity; - if (entity == null) { - break; - } - builder.set("isDefault", formthankyoupagedto.getDefault()); - builder.set("followupType", formthankyoupagedto.getFollowupType()); - builder.set("followupValue", formthankyoupagedto.getFollowupValue()); - builder.set("operator", formthankyoupagedto.getOperator()); - builder.set("subjectField", formthankyoupagedto.getSubjectField()); - builder.set("values", formthankyoupagedto.getValues()); - break; - case "LandingPageTemplate": - LandingPageTemplate landingpagetemplate = (LandingPageTemplate) entity; - if (entity == null) { - break; - } - builder.set("createdAt", landingpagetemplate.getCreatedAt()); - builder.set("description", landingpagetemplate.getDescription()); - builder.set("enableMunchkin", landingpagetemplate.getEnableMunchkin()); - builder.set("folder", EntitySchemaHelper.structuredRecordFromEntity( - "FolderDescriptor", - landingpagetemplate.getFolder(), - schema.getField("folder").getSchema().getNonNullable()) - ); - builder.set("id", landingpagetemplate.getId()); - builder.set("name", landingpagetemplate.getName()); - builder.set("status", landingpagetemplate.getStatus()); - builder.set("templateType", landingpagetemplate.getTemplateType()); - builder.set("updatedAt", landingpagetemplate.getUpdatedAt()); - builder.set("url", landingpagetemplate.getUrl()); - builder.set("workspace", landingpagetemplate.getWorkspace()); - break; - case "Email": - Email email = (Email) entity; - if (entity == null) { - break; - } - builder.set("createdAt", email.getCreatedAt()); - builder.set("description", email.getDescription()); - builder.set("folder", EntitySchemaHelper.structuredRecordFromEntity( - "FolderDescriptor", - email.getFolder(), - schema.getField("folder").getSchema().getNonNullable()) - ); - builder.set("fromEmail", EntitySchemaHelper.structuredRecordFromEntity( - "EmailField", - email.getFromEmail(), - schema.getField("fromEmail").getSchema().getNonNullable()) - ); - builder.set("fromName", EntitySchemaHelper.structuredRecordFromEntity( - "EmailField", - email.getFromName(), - schema.getField("fromName").getSchema().getNonNullable()) - ); - builder.set("id", email.getId()); - builder.set("name", email.getName()); - builder.set("operational", email.getOperational()); - builder.set("publishToMSI", email.getPublishToMSI()); - builder.set("replyEmail", EntitySchemaHelper.structuredRecordFromEntity( - "EmailField", - email.getReplyEmail(), - schema.getField("replyEmail").getSchema().getNonNullable()) - ); - builder.set("status", email.getStatus()); - builder.set("subject", EntitySchemaHelper.structuredRecordFromEntity( - "EmailField", - email.getSubject(), - schema.getField("subject").getSchema().getNonNullable()) - ); - builder.set("template", email.getTemplate()); - builder.set("textOnly", email.getTextOnly()); - builder.set("updatedAt", email.getUpdatedAt()); - builder.set("url", email.getUrl()); - builder.set("version", email.getVersion()); - builder.set("webView", email.getWebView()); - builder.set("workspace", email.getWorkspace()); - builder.set("autoCopyToText", email.getAutoCopyToText()); - builder.set( - "ccFields", - email.getCcFields().stream() - .map(ent -> EntitySchemaHelper.structuredRecordFromEntity( - "EmailCCField", - ent, - schema.getField("ccFields").getSchema().getNonNullable().getComponentSchema())) - .collect(Collectors.toList()) - ); - break; - case "SmartList": - SmartList smartlist = (SmartList) entity; - if (entity == null) { - break; - } - builder.set("id", smartlist.getId()); - builder.set("name", smartlist.getName()); - builder.set("description", smartlist.getDescription()); - builder.set("createdAt", smartlist.getCreatedAt()); - builder.set("updatedAt", smartlist.getUpdatedAt()); - builder.set("url", smartlist.getUrl()); - builder.set("folder", EntitySchemaHelper.structuredRecordFromEntity( - "FolderDescriptor", - smartlist.getFolder(), - schema.getField("folder").getSchema().getNonNullable()) - ); - builder.set("workspace", smartlist.getWorkspace()); - break; - case "Snippet": - Snippet snippet = (Snippet) entity; - if (entity == null) { - break; - } - builder.set("createdAt", snippet.getCreatedAt()); - builder.set("description", snippet.getDescription()); - builder.set("folder", EntitySchemaHelper.structuredRecordFromEntity( - "FolderDescriptor", - snippet.getFolder(), - schema.getField("folder").getSchema().getNonNullable()) - ); - builder.set("id", snippet.getId()); - builder.set("name", snippet.getName()); - builder.set("status", snippet.getStatus()); - builder.set("updatedAt", snippet.getUpdatedAt()); - builder.set("url", snippet.getUrl()); - builder.set("workspace", snippet.getWorkspace()); - break; - case "FormKnownVisitorDTO": - FormKnownVisitorDTO formknownvisitordto = (FormKnownVisitorDTO) entity; - if (entity == null) { - break; - } - builder.set("template", formknownvisitordto.getTemplate()); - builder.set("type", formknownvisitordto.getType()); - break; - case "EmailField": - EmailField emailfield = (EmailField) entity; - if (entity == null) { - break; - } - builder.set("type", emailfield.getType()); - builder.set("value", emailfield.getValue()); - break; - case "SmartCampaign": - SmartCampaign smartcampaign = (SmartCampaign) entity; - if (entity == null) { - break; - } - builder.set("id", smartcampaign.getId()); - builder.set("name", smartcampaign.getName()); - builder.set("description", smartcampaign.getDescription()); - builder.set("type", smartcampaign.getType()); - builder.set("isSystem", smartcampaign.getSystem()); - builder.set("isActive", smartcampaign.getActive()); - builder.set("isRequestable", smartcampaign.getRequestable()); - builder.set("recurrence", EntitySchemaHelper.structuredRecordFromEntity( - "Recurrence", - smartcampaign.getRecurrence(), - schema.getField("recurrence").getSchema().getNonNullable()) - ); - builder.set("qualificationRuleType", smartcampaign.getQualificationRuleType()); - builder.set("qualificationRuleInterval", smartcampaign.getQualificationRuleInterval()); - builder.set("qualificationRuleUnit", smartcampaign.getQualificationRuleUnit()); - builder.set("maxMembers", smartcampaign.getMaxMembers()); - builder.set("isCommunicationLimitEnabled", smartcampaign.getCommunicationLimitEnabled()); - builder.set("smartListId", smartcampaign.getSmartListId()); - builder.set("flowId", smartcampaign.getFlowId()); - builder.set("folder", EntitySchemaHelper.structuredRecordFromEntity( - "FolderDescriptor", - smartcampaign.getFolder(), - schema.getField("folder").getSchema().getNonNullable()) - ); - builder.set("createdAt", smartcampaign.getCreatedAt()); - builder.set("updatedAt", smartcampaign.getUpdatedAt()); - builder.set("workspace", smartcampaign.getWorkspace()); - builder.set("status", smartcampaign.getStatus()); - break; - case "Form": - Form form = (Form) entity; - if (entity == null) { - break; - } - builder.set("buttonLabel", form.getButtonLabel()); - builder.set("buttonLocation", form.getButtonLocation()); - builder.set("createdAt", form.getCreatedAt()); - builder.set("description", form.getDescription()); - builder.set("folder", EntitySchemaHelper.structuredRecordFromEntity( - "FolderDescriptor", - form.getFolder(), - schema.getField("folder").getSchema().getNonNullable()) - ); - builder.set("fontFamily", form.getFontFamily()); - builder.set("fontSize", form.getFontSize()); - builder.set("id", form.getId()); - builder.set("knownVisitor", EntitySchemaHelper.structuredRecordFromEntity( - "FormKnownVisitorDTO", - form.getKnownVisitor(), - schema.getField("knownVisitor").getSchema().getNonNullable()) - ); - builder.set("labelPosition", form.getLabelPosition()); - builder.set("language", form.getLanguage()); - builder.set("locale", form.getLocale()); - builder.set("name", form.getName()); - builder.set("progressiveProfiling", form.getProgressiveProfiling()); - builder.set("status", form.getStatus()); - builder.set( - "thankYouList", - form.getThankYouList().stream() - .map(ent -> EntitySchemaHelper.structuredRecordFromEntity( - "FormThankYouPageDTO", - ent, - schema.getField("thankYouList").getSchema().getNonNullable().getComponentSchema())) - .collect(Collectors.toList()) - ); - builder.set("theme", form.getTheme()); - builder.set("updatedAt", form.getUpdatedAt()); - builder.set("url", form.getUrl()); - builder.set("waitingLabel", form.getWaitingLabel()); - break; - case "FolderDescriptor": - FolderDescriptor folderdescriptor = (FolderDescriptor) entity; - if (entity == null) { - break; - } - builder.set("id", folderdescriptor.getId()); - builder.set("type", folderdescriptor.getType()); - builder.set("folderName", folderdescriptor.getFolderName()); - break; - case "Recurrence": - Recurrence recurrence = (Recurrence) entity; - if (entity == null) { - break; - } - builder.set("startAt", recurrence.getStartAt()); - builder.set("endAt", recurrence.getEndAt()); - builder.set("intervalType", recurrence.getIntervalType()); - builder.set("interval", recurrence.getInterval()); - builder.set("weekdayOnly", recurrence.getWeekdayOnly()); - builder.set("weekdayMask", recurrence.getWeekdayMask()); - builder.set("dayOfMonth", recurrence.getDayOfMonth()); - builder.set("dayOfWeek", recurrence.getDayOfWeek()); - builder.set("weekOfMonth", recurrence.getWeekOfMonth()); - break; - case "File": - File file = (File) entity; - if (entity == null) { - break; - } - builder.set("createdAt", file.getCreatedAt()); - builder.set("description", file.getDescription()); - builder.set("folder", EntitySchemaHelper.structuredRecordFromEntity( - "FileFolder", - file.getFolder(), - schema.getField("folder").getSchema().getNonNullable()) - ); - builder.set("id", file.getId()); - builder.set("mimeType", file.getMimeType()); - builder.set("name", file.getName()); - builder.set("size", file.getSize()); - builder.set("updatedAt", file.getUpdatedAt()); - builder.set("url", file.getUrl()); - break; - case "Segmentation": - Segmentation segmentation = (Segmentation) entity; - if (entity == null) { - break; - } - builder.set("createdAt", segmentation.getCreatedAt()); - builder.set("description", segmentation.getDescription()); - builder.set("folder", EntitySchemaHelper.structuredRecordFromEntity( - "FolderDescriptor", - segmentation.getFolder(), - schema.getField("folder").getSchema().getNonNullable()) - ); - builder.set("id", segmentation.getId()); - builder.set("name", segmentation.getName()); - builder.set("status", segmentation.getStatus()); - builder.set("updatedAt", segmentation.getUpdatedAt()); - builder.set("url", segmentation.getUrl()); - builder.set("workspace", segmentation.getWorkspace()); - break; - case "Tag": - Tag tag = (Tag) entity; - if (entity == null) { - break; - } - builder.set("applicableProgramTypes", tag.getApplicableProgramTypes()); - builder.set("required", tag.getRequired()); - builder.set("tagType", tag.getTagType()); - break; - case "EmailTemplate": - EmailTemplate emailtemplate = (EmailTemplate) entity; - if (entity == null) { - break; - } - builder.set("createdAt", emailtemplate.getCreatedAt()); - builder.set("description", emailtemplate.getDescription()); - builder.set("folder", EntitySchemaHelper.structuredRecordFromEntity( - "FolderDescriptor", - emailtemplate.getFolder(), - schema.getField("folder").getSchema().getNonNullable()) - ); - builder.set("id", emailtemplate.getId()); - builder.set("name", emailtemplate.getName()); - builder.set("status", emailtemplate.getStatus()); - builder.set("updatedAt", emailtemplate.getUpdatedAt()); - builder.set("url", emailtemplate.getUrl()); - builder.set("version", emailtemplate.getVersion()); - builder.set("workspace", emailtemplate.getWorkspace()); - break; - case "EmailCCField": - EmailCCField emailccfield = (EmailCCField) entity; - if (entity == null) { - break; - } - builder.set("attributeId", emailccfield.getAttributeId()); - builder.set("objectName", emailccfield.getObjectName()); - builder.set("displayName", emailccfield.getDisplayName()); - builder.set("apiName", emailccfield.getApiName()); - break; - default: + public static Schema getEmailSchema() { + List fields = new ArrayList<>(); + fields.add(Schema.Field.of("createdAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("description", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("folder", Schema.nullableOf(EntitySchemaHelper.getFolderDescriptorSchema()))); + fields.add(Schema.Field.of("fromEmail", Schema.nullableOf(EntitySchemaHelper.getEmailFieldSchema()))); + fields.add(Schema.Field.of("fromName", Schema.nullableOf(EntitySchemaHelper.getEmailFieldSchema()))); + fields.add(Schema.Field.of("id", Schema.nullableOf(Schema.of(Schema.Type.INT)))); + fields.add(Schema.Field.of("name", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("operational", Schema.nullableOf(Schema.of(Schema.Type.BOOLEAN)))); + fields.add(Schema.Field.of("publishToMSI", Schema.nullableOf(Schema.of(Schema.Type.BOOLEAN)))); + fields.add(Schema.Field.of("replyEmail", Schema.nullableOf(EntitySchemaHelper.getEmailFieldSchema()))); + fields.add(Schema.Field.of("status", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("subject", Schema.nullableOf(EntitySchemaHelper.getEmailFieldSchema()))); + fields.add(Schema.Field.of("template", Schema.nullableOf(Schema.of(Schema.Type.INT)))); + fields.add(Schema.Field.of("textOnly", Schema.nullableOf(Schema.of(Schema.Type.BOOLEAN)))); + fields.add(Schema.Field.of("updatedAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("url", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("version", Schema.nullableOf(Schema.of(Schema.Type.INT)))); + fields.add(Schema.Field.of("webView", Schema.nullableOf(Schema.of(Schema.Type.BOOLEAN)))); + fields.add(Schema.Field.of("workspace", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("autoCopyToText", Schema.nullableOf(Schema.of(Schema.Type.BOOLEAN)))); + fields.add(Schema.Field.of("ccFields", + Schema.nullableOf(Schema.arrayOf(EntitySchemaHelper.getEmailCCFieldSchema())))); + return Schema.recordOf(UUID.randomUUID().toString().replace("-", ""), fields); + } - throw new IllegalArgumentException("Unknown entity name:" + entityName); - } + public static Schema getEmailCCFieldSchema() { + List fields = new ArrayList<>(); + fields.add(Schema.Field.of("attributeId", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("objectName", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("displayName", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("apiName", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + return Schema.recordOf(UUID.randomUUID().toString().replace("-", ""), fields); + } - return builder.build(); + public static Schema getEmailFieldSchema() { + List fields = new ArrayList<>(); + fields.add(Schema.Field.of("type", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("value", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + return Schema.recordOf(UUID.randomUUID().toString().replace("-", ""), fields); } - public static Schema getProgramSchema() { + public static Schema getEmailTemplateSchema() { List fields = new ArrayList<>(); - fields.add(Schema.Field.of("channel", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); fields.add(Schema.Field.of("createdAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); fields.add(Schema.Field.of("description", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); fields.add(Schema.Field.of("folder", Schema.nullableOf(EntitySchemaHelper.getFolderDescriptorSchema()))); fields.add(Schema.Field.of("id", Schema.nullableOf(Schema.of(Schema.Type.INT)))); fields.add(Schema.Field.of("name", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); - fields.add(Schema.Field.of("sfdcId", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); - fields.add(Schema.Field.of("sfdcName", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); fields.add(Schema.Field.of("status", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); - fields.add(Schema.Field.of("type", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); fields.add(Schema.Field.of("updatedAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); fields.add(Schema.Field.of("url", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("version", Schema.nullableOf(Schema.of(Schema.Type.INT)))); fields.add(Schema.Field.of("workspace", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); return Schema.recordOf(UUID.randomUUID().toString().replace("-", ""), fields); } - public static Schema getFolderSchema() { + public static Schema getFileSchema() { List fields = new ArrayList<>(); - fields.add(Schema.Field.of("accessZoneId", Schema.nullableOf(Schema.of(Schema.Type.INT)))); fields.add(Schema.Field.of("createdAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); fields.add(Schema.Field.of("description", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); - fields.add(Schema.Field.of("folderId", Schema.nullableOf(EntitySchemaHelper.getFolderDescriptorSchema()))); - fields.add(Schema.Field.of("folderType", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("folder", Schema.nullableOf(EntitySchemaHelper.getFileFolderSchema()))); fields.add(Schema.Field.of("id", Schema.nullableOf(Schema.of(Schema.Type.INT)))); - fields.add(Schema.Field.of("isArchive", Schema.nullableOf(Schema.of(Schema.Type.BOOLEAN)))); - fields.add(Schema.Field.of("isSystem", Schema.nullableOf(Schema.of(Schema.Type.BOOLEAN)))); + fields.add(Schema.Field.of("mimeType", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); fields.add(Schema.Field.of("name", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); - fields.add(Schema.Field.of("parent", Schema.nullableOf(EntitySchemaHelper.getFolderDescriptorSchema()))); - fields.add(Schema.Field.of("path", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("size", Schema.nullableOf(Schema.of(Schema.Type.INT)))); fields.add(Schema.Field.of("updatedAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); fields.add(Schema.Field.of("url", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); - fields.add(Schema.Field.of("workspace", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); return Schema.recordOf(UUID.randomUUID().toString().replace("-", ""), fields); } @@ -608,29 +205,60 @@ public static Schema getFileFolderSchema() { return Schema.recordOf(UUID.randomUUID().toString().replace("-", ""), fields); } - public static Schema getLandingPageSchema() { + public static Schema getFolderSchema() { List fields = new ArrayList<>(); - fields.add(Schema.Field.of("url", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); - fields.add(Schema.Field.of("computedUrl", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("accessZoneId", Schema.nullableOf(Schema.of(Schema.Type.INT)))); fields.add(Schema.Field.of("createdAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); - fields.add(Schema.Field.of("customHeadHTML", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); fields.add(Schema.Field.of("description", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); - fields.add(Schema.Field.of("facebookOgTags", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); - fields.add(Schema.Field.of("folder", Schema.nullableOf(EntitySchemaHelper.getFolderDescriptorSchema()))); - fields.add(Schema.Field.of("formPrefill", Schema.nullableOf(Schema.of(Schema.Type.BOOLEAN)))); + fields.add(Schema.Field.of("folderId", Schema.nullableOf(EntitySchemaHelper.getFolderDescriptorSchema()))); + fields.add(Schema.Field.of("folderType", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); fields.add(Schema.Field.of("id", Schema.nullableOf(Schema.of(Schema.Type.INT)))); - fields.add(Schema.Field.of("keywords", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); - fields.add(Schema.Field.of("mobileEnabled", Schema.nullableOf(Schema.of(Schema.Type.BOOLEAN)))); + fields.add(Schema.Field.of("isArchive", Schema.nullableOf(Schema.of(Schema.Type.BOOLEAN)))); + fields.add(Schema.Field.of("isSystem", Schema.nullableOf(Schema.of(Schema.Type.BOOLEAN)))); fields.add(Schema.Field.of("name", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); - fields.add(Schema.Field.of("robots", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); - fields.add(Schema.Field.of("status", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); - fields.add(Schema.Field.of("template", Schema.nullableOf(Schema.of(Schema.Type.INT)))); - fields.add(Schema.Field.of("title", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("parent", Schema.nullableOf(EntitySchemaHelper.getFolderDescriptorSchema()))); + fields.add(Schema.Field.of("path", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); fields.add(Schema.Field.of("updatedAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("url", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); fields.add(Schema.Field.of("workspace", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); return Schema.recordOf(UUID.randomUUID().toString().replace("-", ""), fields); } + public static Schema getFolderDescriptorSchema() { + List fields = new ArrayList<>(); + fields.add(Schema.Field.of("id", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("type", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("folderName", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + return Schema.recordOf(UUID.randomUUID().toString().replace("-", ""), fields); + } + + public static Schema getFormSchema() { + List fields = new ArrayList<>(); + fields.add(Schema.Field.of("buttonLabel", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("buttonLocation", Schema.nullableOf(Schema.of(Schema.Type.INT)))); + fields.add(Schema.Field.of("createdAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("description", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("folder", Schema.nullableOf(EntitySchemaHelper.getFolderDescriptorSchema()))); + fields.add(Schema.Field.of("fontFamily", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("fontSize", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("id", Schema.nullableOf(Schema.of(Schema.Type.INT)))); + fields.add(Schema.Field.of("knownVisitor", + Schema.nullableOf(EntitySchemaHelper.getFormKnownVisitorDTOSchema()))); + fields.add(Schema.Field.of("labelPosition", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("language", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("locale", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("name", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("progressiveProfiling", Schema.nullableOf(Schema.of(Schema.Type.BOOLEAN)))); + fields.add(Schema.Field.of("status", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("thankYouList", + Schema.nullableOf(Schema.arrayOf(EntitySchemaHelper.getFormThankYouPageDTOSchema())))); + fields.add(Schema.Field.of("theme", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("updatedAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("url", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("waitingLabel", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + return Schema.recordOf(UUID.randomUUID().toString().replace("-", ""), fields); + } + public static Schema getFormFieldSchema() { List fields = new ArrayList<>(); fields.add(Schema.Field.of("dataType", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); @@ -654,17 +282,10 @@ public static Schema getFormFieldSchema() { return Schema.recordOf(UUID.randomUUID().toString().replace("-", ""), fields); } - public static Schema getStaticListSchema() { + public static Schema getFormKnownVisitorDTOSchema() { List fields = new ArrayList<>(); - fields.add(Schema.Field.of("id", Schema.nullableOf(Schema.of(Schema.Type.INT)))); - fields.add(Schema.Field.of("name", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); - fields.add(Schema.Field.of("description", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); - fields.add(Schema.Field.of("createdAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); - fields.add(Schema.Field.of("updatedAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); - fields.add(Schema.Field.of("url", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); - fields.add(Schema.Field.of("folder", Schema.nullableOf(EntitySchemaHelper.getFolderDescriptorSchema()))); - fields.add(Schema.Field.of("workspace", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); - fields.add(Schema.Field.of("computedUrl", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("template", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("type", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); return Schema.recordOf(UUID.randomUUID().toString().replace("-", ""), fields); } @@ -679,63 +300,79 @@ public static Schema getFormThankYouPageDTOSchema() { return Schema.recordOf(UUID.randomUUID().toString().replace("-", ""), fields); } - public static Schema getLandingPageTemplateSchema() { + public static Schema getLandingPageSchema() { List fields = new ArrayList<>(); + fields.add(Schema.Field.of("url", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("computedUrl", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); fields.add(Schema.Field.of("createdAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("customHeadHTML", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); fields.add(Schema.Field.of("description", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); - fields.add(Schema.Field.of("enableMunchkin", Schema.nullableOf(Schema.of(Schema.Type.BOOLEAN)))); + fields.add(Schema.Field.of("facebookOgTags", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); fields.add(Schema.Field.of("folder", Schema.nullableOf(EntitySchemaHelper.getFolderDescriptorSchema()))); + fields.add(Schema.Field.of("formPrefill", Schema.nullableOf(Schema.of(Schema.Type.BOOLEAN)))); fields.add(Schema.Field.of("id", Schema.nullableOf(Schema.of(Schema.Type.INT)))); + fields.add(Schema.Field.of("keywords", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("mobileEnabled", Schema.nullableOf(Schema.of(Schema.Type.BOOLEAN)))); fields.add(Schema.Field.of("name", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("robots", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); fields.add(Schema.Field.of("status", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); - fields.add(Schema.Field.of("templateType", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("template", Schema.nullableOf(Schema.of(Schema.Type.INT)))); + fields.add(Schema.Field.of("title", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); fields.add(Schema.Field.of("updatedAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); - fields.add(Schema.Field.of("url", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); fields.add(Schema.Field.of("workspace", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); return Schema.recordOf(UUID.randomUUID().toString().replace("-", ""), fields); } - public static Schema getEmailSchema() { + public static Schema getLandingPageTemplateSchema() { List fields = new ArrayList<>(); fields.add(Schema.Field.of("createdAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); fields.add(Schema.Field.of("description", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("enableMunchkin", Schema.nullableOf(Schema.of(Schema.Type.BOOLEAN)))); fields.add(Schema.Field.of("folder", Schema.nullableOf(EntitySchemaHelper.getFolderDescriptorSchema()))); - fields.add(Schema.Field.of("fromEmail", Schema.nullableOf(EntitySchemaHelper.getEmailFieldSchema()))); - fields.add(Schema.Field.of("fromName", Schema.nullableOf(EntitySchemaHelper.getEmailFieldSchema()))); fields.add(Schema.Field.of("id", Schema.nullableOf(Schema.of(Schema.Type.INT)))); fields.add(Schema.Field.of("name", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); - fields.add(Schema.Field.of("operational", Schema.nullableOf(Schema.of(Schema.Type.BOOLEAN)))); - fields.add(Schema.Field.of("publishToMSI", Schema.nullableOf(Schema.of(Schema.Type.BOOLEAN)))); - fields.add(Schema.Field.of("replyEmail", Schema.nullableOf(EntitySchemaHelper.getEmailFieldSchema()))); fields.add(Schema.Field.of("status", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); - fields.add(Schema.Field.of("subject", Schema.nullableOf(EntitySchemaHelper.getEmailFieldSchema()))); - fields.add(Schema.Field.of("template", Schema.nullableOf(Schema.of(Schema.Type.INT)))); - fields.add(Schema.Field.of("textOnly", Schema.nullableOf(Schema.of(Schema.Type.BOOLEAN)))); + fields.add(Schema.Field.of("templateType", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); fields.add(Schema.Field.of("updatedAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); fields.add(Schema.Field.of("url", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); - fields.add(Schema.Field.of("version", Schema.nullableOf(Schema.of(Schema.Type.INT)))); - fields.add(Schema.Field.of("webView", Schema.nullableOf(Schema.of(Schema.Type.BOOLEAN)))); fields.add(Schema.Field.of("workspace", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); - fields.add(Schema.Field.of("autoCopyToText", Schema.nullableOf(Schema.of(Schema.Type.BOOLEAN)))); - fields.add(Schema.Field.of("ccFields", Schema.nullableOf( - Schema.arrayOf(EntitySchemaHelper.getEmailCCFieldSchema())))); return Schema.recordOf(UUID.randomUUID().toString().replace("-", ""), fields); } - public static Schema getSmartListSchema() { + public static Schema getProgramSchema() { List fields = new ArrayList<>(); + fields.add(Schema.Field.of("channel", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("createdAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("description", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("folder", Schema.nullableOf(EntitySchemaHelper.getFolderDescriptorSchema()))); fields.add(Schema.Field.of("id", Schema.nullableOf(Schema.of(Schema.Type.INT)))); fields.add(Schema.Field.of("name", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); - fields.add(Schema.Field.of("description", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); - fields.add(Schema.Field.of("createdAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("sfdcId", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("sfdcName", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("status", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("type", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); fields.add(Schema.Field.of("updatedAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); fields.add(Schema.Field.of("url", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); - fields.add(Schema.Field.of("folder", Schema.nullableOf(EntitySchemaHelper.getFolderDescriptorSchema()))); fields.add(Schema.Field.of("workspace", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); return Schema.recordOf(UUID.randomUUID().toString().replace("-", ""), fields); } - public static Schema getSnippetSchema() { + public static Schema getRecurrenceSchema() { + List fields = new ArrayList<>(); + fields.add(Schema.Field.of("startAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("endAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("intervalType", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("interval", Schema.nullableOf(Schema.of(Schema.Type.INT)))); + fields.add(Schema.Field.of("weekdayOnly", Schema.nullableOf(Schema.of(Schema.Type.BOOLEAN)))); + fields.add(Schema.Field.of("weekdayMask", + Schema.nullableOf(Schema.arrayOf(Schema.of(Schema.Type.STRING))))); + fields.add(Schema.Field.of("dayOfMonth", Schema.nullableOf(Schema.of(Schema.Type.INT)))); + fields.add(Schema.Field.of("dayOfWeek", Schema.nullableOf(Schema.of(Schema.Type.INT)))); + fields.add(Schema.Field.of("weekOfMonth", Schema.nullableOf(Schema.of(Schema.Type.INT)))); + return Schema.recordOf(UUID.randomUUID().toString().replace("-", ""), fields); + } + + public static Schema getSegmentationSchema() { List fields = new ArrayList<>(); fields.add(Schema.Field.of("createdAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); fields.add(Schema.Field.of("description", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); @@ -749,20 +386,6 @@ public static Schema getSnippetSchema() { return Schema.recordOf(UUID.randomUUID().toString().replace("-", ""), fields); } - public static Schema getFormKnownVisitorDTOSchema() { - List fields = new ArrayList<>(); - fields.add(Schema.Field.of("template", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); - fields.add(Schema.Field.of("type", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); - return Schema.recordOf(UUID.randomUUID().toString().replace("-", ""), fields); - } - - public static Schema getEmailFieldSchema() { - List fields = new ArrayList<>(); - fields.add(Schema.Field.of("type", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); - fields.add(Schema.Field.of("value", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); - return Schema.recordOf(UUID.randomUUID().toString().replace("-", ""), fields); - } - public static Schema getSmartCampaignSchema() { List fields = new ArrayList<>(); fields.add(Schema.Field.of("id", Schema.nullableOf(Schema.of(Schema.Type.INT)))); @@ -789,69 +412,20 @@ public static Schema getSmartCampaignSchema() { return Schema.recordOf(UUID.randomUUID().toString().replace("-", ""), fields); } - public static Schema getFormSchema() { + public static Schema getSmartListSchema() { List fields = new ArrayList<>(); - fields.add(Schema.Field.of("buttonLabel", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); - fields.add(Schema.Field.of("buttonLocation", Schema.nullableOf(Schema.of(Schema.Type.INT)))); - fields.add(Schema.Field.of("createdAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); - fields.add(Schema.Field.of("description", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); - fields.add(Schema.Field.of("folder", Schema.nullableOf(EntitySchemaHelper.getFolderDescriptorSchema()))); - fields.add(Schema.Field.of("fontFamily", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); - fields.add(Schema.Field.of("fontSize", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); fields.add(Schema.Field.of("id", Schema.nullableOf(Schema.of(Schema.Type.INT)))); - fields.add(Schema.Field.of("knownVisitor", Schema.nullableOf(EntitySchemaHelper.getFormKnownVisitorDTOSchema()))); - fields.add(Schema.Field.of("labelPosition", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); - fields.add(Schema.Field.of("language", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); - fields.add(Schema.Field.of("locale", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); fields.add(Schema.Field.of("name", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); - fields.add(Schema.Field.of("progressiveProfiling", Schema.nullableOf(Schema.of(Schema.Type.BOOLEAN)))); - fields.add(Schema.Field.of("status", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); - fields.add(Schema.Field.of("thankYouList", Schema.nullableOf( - Schema.arrayOf(EntitySchemaHelper.getFormThankYouPageDTOSchema())))); - fields.add(Schema.Field.of("theme", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); - fields.add(Schema.Field.of("updatedAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); - fields.add(Schema.Field.of("url", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); - fields.add(Schema.Field.of("waitingLabel", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); - return Schema.recordOf(UUID.randomUUID().toString().replace("-", ""), fields); - } - - public static Schema getFolderDescriptorSchema() { - List fields = new ArrayList<>(); - fields.add(Schema.Field.of("id", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); - fields.add(Schema.Field.of("type", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); - fields.add(Schema.Field.of("folderName", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); - return Schema.recordOf(UUID.randomUUID().toString().replace("-", ""), fields); - } - - public static Schema getRecurrenceSchema() { - List fields = new ArrayList<>(); - fields.add(Schema.Field.of("startAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); - fields.add(Schema.Field.of("endAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); - fields.add(Schema.Field.of("intervalType", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); - fields.add(Schema.Field.of("interval", Schema.nullableOf(Schema.of(Schema.Type.INT)))); - fields.add(Schema.Field.of("weekdayOnly", Schema.nullableOf(Schema.of(Schema.Type.BOOLEAN)))); - fields.add(Schema.Field.of("weekdayMask", Schema.nullableOf(Schema.arrayOf(Schema.of(Schema.Type.STRING))))); - fields.add(Schema.Field.of("dayOfMonth", Schema.nullableOf(Schema.of(Schema.Type.INT)))); - fields.add(Schema.Field.of("dayOfWeek", Schema.nullableOf(Schema.of(Schema.Type.INT)))); - fields.add(Schema.Field.of("weekOfMonth", Schema.nullableOf(Schema.of(Schema.Type.INT)))); - return Schema.recordOf(UUID.randomUUID().toString().replace("-", ""), fields); - } - - public static Schema getFileSchema() { - List fields = new ArrayList<>(); - fields.add(Schema.Field.of("createdAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); fields.add(Schema.Field.of("description", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); - fields.add(Schema.Field.of("folder", Schema.nullableOf(EntitySchemaHelper.getFileFolderSchema()))); - fields.add(Schema.Field.of("id", Schema.nullableOf(Schema.of(Schema.Type.INT)))); - fields.add(Schema.Field.of("mimeType", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); - fields.add(Schema.Field.of("name", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); - fields.add(Schema.Field.of("size", Schema.nullableOf(Schema.of(Schema.Type.INT)))); + fields.add(Schema.Field.of("createdAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); fields.add(Schema.Field.of("updatedAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); fields.add(Schema.Field.of("url", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("folder", Schema.nullableOf(EntitySchemaHelper.getFolderDescriptorSchema()))); + fields.add(Schema.Field.of("workspace", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); return Schema.recordOf(UUID.randomUUID().toString().replace("-", ""), fields); } - public static Schema getSegmentationSchema() { + public static Schema getSnippetSchema() { List fields = new ArrayList<>(); fields.add(Schema.Field.of("createdAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); fields.add(Schema.Field.of("description", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); @@ -865,36 +439,523 @@ public static Schema getSegmentationSchema() { return Schema.recordOf(UUID.randomUUID().toString().replace("-", ""), fields); } - public static Schema getTagSchema() { - List fields = new ArrayList<>(); - fields.add(Schema.Field.of("applicableProgramTypes", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); - fields.add(Schema.Field.of("required", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); - fields.add(Schema.Field.of("tagType", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); - return Schema.recordOf(UUID.randomUUID().toString().replace("-", ""), fields); - } - - public static Schema getEmailTemplateSchema() { + public static Schema getStaticListSchema() { List fields = new ArrayList<>(); - fields.add(Schema.Field.of("createdAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); - fields.add(Schema.Field.of("description", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); - fields.add(Schema.Field.of("folder", Schema.nullableOf(EntitySchemaHelper.getFolderDescriptorSchema()))); fields.add(Schema.Field.of("id", Schema.nullableOf(Schema.of(Schema.Type.INT)))); fields.add(Schema.Field.of("name", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); - fields.add(Schema.Field.of("status", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("description", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("createdAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); fields.add(Schema.Field.of("updatedAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); fields.add(Schema.Field.of("url", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); - fields.add(Schema.Field.of("version", Schema.nullableOf(Schema.of(Schema.Type.INT)))); + fields.add(Schema.Field.of("folder", Schema.nullableOf(EntitySchemaHelper.getFolderDescriptorSchema()))); fields.add(Schema.Field.of("workspace", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("computedUrl", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); return Schema.recordOf(UUID.randomUUID().toString().replace("-", ""), fields); } - public static Schema getEmailCCFieldSchema() { + public static Schema getTagSchema() { List fields = new ArrayList<>(); - fields.add(Schema.Field.of("attributeId", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); - fields.add(Schema.Field.of("objectName", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); - fields.add(Schema.Field.of("displayName", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); - fields.add(Schema.Field.of("apiName", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("applicableProgramTypes", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("required", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); + fields.add(Schema.Field.of("tagType", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); return Schema.recordOf(UUID.randomUUID().toString().replace("-", ""), fields); } + public static StructuredRecord structuredRecordFromEntity(String entityName, Object entity, + Schema schema) { + StructuredRecord.Builder builder = StructuredRecord.builder(schema); + switch (entityName) { + case "Email": { + Email email = (Email) entity; + if (entity == null) { + break; + } + builder.set("createdAt", email.getCreatedAt()); + builder.set("description", email.getDescription()); + builder.set("folder", EntitySchemaHelper.structuredRecordFromEntity( + "FolderDescriptor", + email.getFolder(), + schema.getField("folder").getSchema().getNonNullable())); + builder.set("fromEmail", EntitySchemaHelper.structuredRecordFromEntity( + "EmailField", + email.getFromEmail(), + schema.getField("fromEmail").getSchema().getNonNullable())); + builder.set("fromName", EntitySchemaHelper.structuredRecordFromEntity( + "EmailField", + email.getFromName(), + schema.getField("fromName").getSchema().getNonNullable())); + builder.set("id", email.getId()); + builder.set("name", email.getName()); + builder.set("operational", email.getOperational()); + builder.set("publishToMSI", email.getPublishToMSI()); + builder.set("replyEmail", EntitySchemaHelper.structuredRecordFromEntity( + "EmailField", + email.getReplyEmail(), + schema.getField("replyEmail").getSchema().getNonNullable())); + builder.set("status", email.getStatus()); + builder.set("subject", EntitySchemaHelper.structuredRecordFromEntity( + "EmailField", + email.getSubject(), + schema.getField("subject").getSchema().getNonNullable())); + builder.set("template", email.getTemplate()); + builder.set("textOnly", email.getTextOnly()); + builder.set("updatedAt", email.getUpdatedAt()); + builder.set("url", email.getUrl()); + builder.set("version", email.getVersion()); + builder.set("webView", email.getWebView()); + builder.set("workspace", email.getWorkspace()); + builder.set("autoCopyToText", email.getAutoCopyToText()); + builder.set( + "ccFields", + email.getCcFields().stream() + .map(ent -> EntitySchemaHelper.structuredRecordFromEntity( + "EmailCCField", + ent, + schema.getField("ccFields").getSchema().getNonNullable().getComponentSchema())) + .collect(Collectors.toList()) + ); + break; + } + case "EmailCCField": { + EmailCCField emailccfield = (EmailCCField) entity; + if (entity == null) { + break; + } + builder.set("attributeId", emailccfield.getAttributeId()); + builder.set("objectName", emailccfield.getObjectName()); + builder.set("displayName", emailccfield.getDisplayName()); + builder.set("apiName", emailccfield.getApiName()); + break; + } + case "EmailField": { + EmailField emailfield = (EmailField) entity; + if (entity == null) { + break; + } + builder.set("type", emailfield.getType()); + builder.set("value", emailfield.getValue()); + break; + } + case "EmailTemplate": { + EmailTemplate emailtemplate = (EmailTemplate) entity; + if (entity == null) { + break; + } + builder.set("createdAt", emailtemplate.getCreatedAt()); + builder.set("description", emailtemplate.getDescription()); + builder.set("folder", EntitySchemaHelper.structuredRecordFromEntity( + "FolderDescriptor", + emailtemplate.getFolder(), + schema.getField("folder").getSchema().getNonNullable())); + builder.set("id", emailtemplate.getId()); + builder.set("name", emailtemplate.getName()); + builder.set("status", emailtemplate.getStatus()); + builder.set("updatedAt", emailtemplate.getUpdatedAt()); + builder.set("url", emailtemplate.getUrl()); + builder.set("version", emailtemplate.getVersion()); + builder.set("workspace", emailtemplate.getWorkspace()); + break; + } + case "File": { + File file = (File) entity; + if (entity == null) { + break; + } + builder.set("createdAt", file.getCreatedAt()); + builder.set("description", file.getDescription()); + builder.set("folder", EntitySchemaHelper.structuredRecordFromEntity( + "FileFolder", + file.getFolder(), + schema.getField("folder").getSchema().getNonNullable())); + builder.set("id", file.getId()); + builder.set("mimeType", file.getMimeType()); + builder.set("name", file.getName()); + builder.set("size", file.getSize()); + builder.set("updatedAt", file.getUpdatedAt()); + builder.set("url", file.getUrl()); + break; + } + case "FileFolder": { + FileFolder filefolder = (FileFolder) entity; + if (entity == null) { + break; + } + builder.set("id", filefolder.getId()); + builder.set("name", filefolder.getName()); + builder.set("type", filefolder.getType()); + break; + } + case "Folder": { + Folder folder = (Folder) entity; + if (entity == null) { + break; + } + builder.set("accessZoneId", folder.getAccessZoneId()); + builder.set("createdAt", folder.getCreatedAt()); + builder.set("description", folder.getDescription()); + builder.set("folderId", EntitySchemaHelper.structuredRecordFromEntity( + "FolderDescriptor", + folder.getFolderId(), + schema.getField("folderId").getSchema().getNonNullable())); + builder.set("folderType", folder.getFolderType()); + builder.set("id", folder.getId()); + builder.set("isArchive", folder.getArchive()); + builder.set("isSystem", folder.getSystem()); + builder.set("name", folder.getName()); + builder.set("parent", EntitySchemaHelper.structuredRecordFromEntity( + "FolderDescriptor", + folder.getParent(), + schema.getField("parent").getSchema().getNonNullable())); + builder.set("path", folder.getPath()); + builder.set("updatedAt", folder.getUpdatedAt()); + builder.set("url", folder.getUrl()); + builder.set("workspace", folder.getWorkspace()); + break; + } + case "FolderDescriptor": { + FolderDescriptor folderdescriptor = (FolderDescriptor) entity; + if (entity == null) { + break; + } + builder.set("id", folderdescriptor.getId()); + builder.set("type", folderdescriptor.getType()); + builder.set("folderName", folderdescriptor.getFolderName()); + break; + } + case "Form": { + Form form = (Form) entity; + if (entity == null) { + break; + } + builder.set("buttonLabel", form.getButtonLabel()); + builder.set("buttonLocation", form.getButtonLocation()); + builder.set("createdAt", form.getCreatedAt()); + builder.set("description", form.getDescription()); + builder.set("folder", EntitySchemaHelper.structuredRecordFromEntity( + "FolderDescriptor", + form.getFolder(), + schema.getField("folder").getSchema().getNonNullable())); + builder.set("fontFamily", form.getFontFamily()); + builder.set("fontSize", form.getFontSize()); + builder.set("id", form.getId()); + builder.set("knownVisitor", EntitySchemaHelper.structuredRecordFromEntity( + "FormKnownVisitorDTO", + form.getKnownVisitor(), + schema.getField("knownVisitor").getSchema().getNonNullable())); + builder.set("labelPosition", form.getLabelPosition()); + builder.set("language", form.getLanguage()); + builder.set("locale", form.getLocale()); + builder.set("name", form.getName()); + builder.set("progressiveProfiling", form.getProgressiveProfiling()); + builder.set("status", form.getStatus()); + builder.set( + "thankYouList", + form.getThankYouList().stream() + .map(ent -> EntitySchemaHelper.structuredRecordFromEntity( + "FormThankYouPageDTO", + ent, + schema.getField("thankYouList").getSchema().getNonNullable().getComponentSchema())) + .collect(Collectors.toList()) + ); + builder.set("theme", form.getTheme()); + builder.set("updatedAt", form.getUpdatedAt()); + builder.set("url", form.getUrl()); + builder.set("waitingLabel", form.getWaitingLabel()); + break; + } + case "FormField": { + FormField formfield = (FormField) entity; + if (entity == null) { + break; + } + builder.set("dataType", formfield.getDataType()); + builder.set("defaultValue", formfield.getDefaultValue()); + builder.set("description", formfield.getDescription()); + builder.set("fieldMaskValues", formfield.getFieldMaskValues()); + builder.set("fieldWidth", formfield.getFieldWidth()); + builder.set("id", formfield.getId()); + builder.set("initiallyChecked", formfield.getInitiallyChecked()); + builder.set("isLabelToRight", formfield.getLabelToRight()); + builder.set("isMultiselect", formfield.getMultiselect()); + builder.set("isRequired", formfield.getRequired()); + builder.set("labelWidth", formfield.getLabelWidth()); + builder.set("maxLength", formfield.getMaxLength()); + builder.set("maximumNumber", formfield.getMaximumNumber()); + builder.set("minimumNumber", formfield.getMinimumNumber()); + builder.set("picklistValues", formfield.getPicklistValues()); + builder.set("placeholderText", formfield.getPlaceholderText()); + builder.set("validationMessage", formfield.getValidationMessage()); + builder.set("visibleRows", formfield.getVisibleRows()); + break; + } + case "FormKnownVisitorDTO": { + FormKnownVisitorDTO formknownvisitordto = (FormKnownVisitorDTO) entity; + if (entity == null) { + break; + } + builder.set("template", formknownvisitordto.getTemplate()); + builder.set("type", formknownvisitordto.getType()); + break; + } + case "FormThankYouPageDTO": { + FormThankYouPageDTO formthankyoupagedto = (FormThankYouPageDTO) entity; + if (entity == null) { + break; + } + builder.set("isDefault", formthankyoupagedto.getDefault()); + builder.set("followupType", formthankyoupagedto.getFollowupType()); + builder.set("followupValue", formthankyoupagedto.getFollowupValue()); + builder.set("operator", formthankyoupagedto.getOperator()); + builder.set("subjectField", formthankyoupagedto.getSubjectField()); + builder.set("values", formthankyoupagedto.getValues()); + break; + } + case "LandingPage": { + LandingPage landingpage = (LandingPage) entity; + if (entity == null) { + break; + } + builder.set("url", landingpage.getUrl()); + builder.set("computedUrl", landingpage.getComputedUrl()); + builder.set("createdAt", landingpage.getCreatedAt()); + builder.set("customHeadHTML", landingpage.getCustomHeadHTML()); + builder.set("description", landingpage.getDescription()); + builder.set("facebookOgTags", landingpage.getFacebookOgTags()); + builder.set("folder", EntitySchemaHelper.structuredRecordFromEntity( + "FolderDescriptor", + landingpage.getFolder(), + schema.getField("folder").getSchema().getNonNullable())); + builder.set("formPrefill", landingpage.getFormPrefill()); + builder.set("id", landingpage.getId()); + builder.set("keywords", landingpage.getKeywords()); + builder.set("mobileEnabled", landingpage.getMobileEnabled()); + builder.set("name", landingpage.getName()); + builder.set("robots", landingpage.getRobots()); + builder.set("status", landingpage.getStatus()); + builder.set("template", landingpage.getTemplate()); + builder.set("title", landingpage.getTitle()); + builder.set("updatedAt", landingpage.getUpdatedAt()); + builder.set("workspace", landingpage.getWorkspace()); + break; + } + case "LandingPageTemplate": { + LandingPageTemplate landingpagetemplate = (LandingPageTemplate) entity; + if (entity == null) { + break; + } + builder.set("createdAt", landingpagetemplate.getCreatedAt()); + builder.set("description", landingpagetemplate.getDescription()); + builder.set("enableMunchkin", landingpagetemplate.getEnableMunchkin()); + builder.set("folder", EntitySchemaHelper.structuredRecordFromEntity( + "FolderDescriptor", + landingpagetemplate.getFolder(), + schema.getField("folder").getSchema().getNonNullable())); + builder.set("id", landingpagetemplate.getId()); + builder.set("name", landingpagetemplate.getName()); + builder.set("status", landingpagetemplate.getStatus()); + builder.set("templateType", landingpagetemplate.getTemplateType()); + builder.set("updatedAt", landingpagetemplate.getUpdatedAt()); + builder.set("url", landingpagetemplate.getUrl()); + builder.set("workspace", landingpagetemplate.getWorkspace()); + break; + } + case "Program": { + Program program = (Program) entity; + if (entity == null) { + break; + } + builder.set("channel", program.getChannel()); + builder.set("createdAt", program.getCreatedAt()); + builder.set("description", program.getDescription()); + builder.set("folder", EntitySchemaHelper.structuredRecordFromEntity( + "FolderDescriptor", + program.getFolder(), + schema.getField("folder").getSchema().getNonNullable())); + builder.set("id", program.getId()); + builder.set("name", program.getName()); + builder.set("sfdcId", program.getSfdcId()); + builder.set("sfdcName", program.getSfdcName()); + builder.set("status", program.getStatus()); + builder.set("type", program.getType()); + builder.set("updatedAt", program.getUpdatedAt()); + builder.set("url", program.getUrl()); + builder.set("workspace", program.getWorkspace()); + break; + } + case "Recurrence": { + Recurrence recurrence = (Recurrence) entity; + if (entity == null) { + break; + } + builder.set("startAt", recurrence.getStartAt()); + builder.set("endAt", recurrence.getEndAt()); + builder.set("intervalType", recurrence.getIntervalType()); + builder.set("interval", recurrence.getInterval()); + builder.set("weekdayOnly", recurrence.getWeekdayOnly()); + builder.set("weekdayMask", recurrence.getWeekdayMask()); + builder.set("dayOfMonth", recurrence.getDayOfMonth()); + builder.set("dayOfWeek", recurrence.getDayOfWeek()); + builder.set("weekOfMonth", recurrence.getWeekOfMonth()); + break; + } + case "Segmentation": { + Segmentation segmentation = (Segmentation) entity; + if (entity == null) { + break; + } + builder.set("createdAt", segmentation.getCreatedAt()); + builder.set("description", segmentation.getDescription()); + builder.set("folder", EntitySchemaHelper.structuredRecordFromEntity( + "FolderDescriptor", + segmentation.getFolder(), + schema.getField("folder").getSchema().getNonNullable())); + builder.set("id", segmentation.getId()); + builder.set("name", segmentation.getName()); + builder.set("status", segmentation.getStatus()); + builder.set("updatedAt", segmentation.getUpdatedAt()); + builder.set("url", segmentation.getUrl()); + builder.set("workspace", segmentation.getWorkspace()); + break; + } + case "SmartCampaign": { + SmartCampaign smartcampaign = (SmartCampaign) entity; + if (entity == null) { + break; + } + builder.set("id", smartcampaign.getId()); + builder.set("name", smartcampaign.getName()); + builder.set("description", smartcampaign.getDescription()); + builder.set("type", smartcampaign.getType()); + builder.set("isSystem", smartcampaign.getSystem()); + builder.set("isActive", smartcampaign.getActive()); + builder.set("isRequestable", smartcampaign.getRequestable()); + builder.set("recurrence", EntitySchemaHelper.structuredRecordFromEntity( + "Recurrence", + smartcampaign.getRecurrence(), + schema.getField("recurrence").getSchema().getNonNullable())); + builder.set("qualificationRuleType", smartcampaign.getQualificationRuleType()); + builder.set("qualificationRuleInterval", smartcampaign.getQualificationRuleInterval()); + builder.set("qualificationRuleUnit", smartcampaign.getQualificationRuleUnit()); + builder.set("maxMembers", smartcampaign.getMaxMembers()); + builder.set("isCommunicationLimitEnabled", smartcampaign.getCommunicationLimitEnabled()); + builder.set("smartListId", smartcampaign.getSmartListId()); + builder.set("flowId", smartcampaign.getFlowId()); + builder.set("folder", EntitySchemaHelper.structuredRecordFromEntity( + "FolderDescriptor", + smartcampaign.getFolder(), + schema.getField("folder").getSchema().getNonNullable())); + builder.set("createdAt", smartcampaign.getCreatedAt()); + builder.set("updatedAt", smartcampaign.getUpdatedAt()); + builder.set("workspace", smartcampaign.getWorkspace()); + builder.set("status", smartcampaign.getStatus()); + break; + } + case "SmartList": { + SmartList smartlist = (SmartList) entity; + if (entity == null) { + break; + } + builder.set("id", smartlist.getId()); + builder.set("name", smartlist.getName()); + builder.set("description", smartlist.getDescription()); + builder.set("createdAt", smartlist.getCreatedAt()); + builder.set("updatedAt", smartlist.getUpdatedAt()); + builder.set("url", smartlist.getUrl()); + builder.set("folder", EntitySchemaHelper.structuredRecordFromEntity( + "FolderDescriptor", + smartlist.getFolder(), + schema.getField("folder").getSchema().getNonNullable())); + builder.set("workspace", smartlist.getWorkspace()); + break; + } + case "Snippet": { + Snippet snippet = (Snippet) entity; + if (entity == null) { + break; + } + builder.set("createdAt", snippet.getCreatedAt()); + builder.set("description", snippet.getDescription()); + builder.set("folder", EntitySchemaHelper.structuredRecordFromEntity( + "FolderDescriptor", + snippet.getFolder(), + schema.getField("folder").getSchema().getNonNullable())); + builder.set("id", snippet.getId()); + builder.set("name", snippet.getName()); + builder.set("status", snippet.getStatus()); + builder.set("updatedAt", snippet.getUpdatedAt()); + builder.set("url", snippet.getUrl()); + builder.set("workspace", snippet.getWorkspace()); + break; + } + case "StaticList": { + StaticList staticlist = (StaticList) entity; + if (entity == null) { + break; + } + builder.set("id", staticlist.getId()); + builder.set("name", staticlist.getName()); + builder.set("description", staticlist.getDescription()); + builder.set("createdAt", staticlist.getCreatedAt()); + builder.set("updatedAt", staticlist.getUpdatedAt()); + builder.set("url", staticlist.getUrl()); + builder.set("folder", EntitySchemaHelper.structuredRecordFromEntity( + "FolderDescriptor", + staticlist.getFolder(), + schema.getField("folder").getSchema().getNonNullable())); + builder.set("workspace", staticlist.getWorkspace()); + builder.set("computedUrl", staticlist.getComputedUrl()); + break; + } + case "Tag": { + Tag tag = (Tag) entity; + if (entity == null) { + break; + } + builder.set("applicableProgramTypes", tag.getApplicableProgramTypes()); + builder.set("required", tag.getRequired()); + builder.set("tagType", tag.getTagType()); + break; + } + default: { + throw new IllegalArgumentException("Unknown entity name:" + entityName); + } + } + return builder.build(); + } + + /** + * Valid entity types. + */ + public enum EntityType { + EMAIL("Email"), + EMAILTEMPLATE("EmailTemplate"), + FILE("File"), + FOLDER("Folder"), + FORM("Form"), + FORMFIELD("FormField"), + LANDINGPAGE("LandingPage"), + LANDINGPAGETEMPLATE("LandingPageTemplate"), + PROGRAM("Program"), + SEGMENTATION("Segmentation"), + SMARTCAMPAIGN("SmartCampaign"), + SMARTLIST("SmartList"), + SNIPPET("Snippet"), + STATICLIST("StaticList"), + TAG("Tag"); + + private final String value; + + EntityType(String value) { + this.value = value; + } + + public static EntityType fromString(String value) { + for (EntityType entityType : EntityType.values()) { + if (entityType.value.equals(value)) { + return entityType; + } + } + throw new IllegalArgumentException("Unknown entity type type: " + value); + } + } } From 847d6eb27d80a46fa4c77fa0aef669fe44367cbf Mon Sep 17 00:00:00 2001 From: Yevhenii Chekanskyi Date: Fri, 6 Dec 2019 15:31:47 +0200 Subject: [PATCH 16/16] PLUGIN-75 Marketo Plugin - implement entity plugin. --- .../io/cdap/plugin/generate/Generate.java | 122 +++++- docs/MarketoEntityPlugin-batchsource.md | 36 ++ icons/MarketoEntityPlugin-batchsource.png | Bin 0 -> 996 bytes .../api/entities/asset/EmailResponse.java | 3 + .../entities/asset/EmailTemplateResponse.java | 3 + .../api/entities/asset/FileResponse.java | 3 + .../api/entities/asset/FolderResponse.java | 3 + .../entities/asset/FormFieldsResponse.java | 3 + .../api/entities/asset/FormResponse.java | 3 + .../entities/asset/LandingPageResponse.java | 3 + .../asset/LandingPageTemplateResponse.java | 3 + .../api/entities/asset/ProgramResponse.java | 3 + .../entities/asset/SegmentationResponse.java | 3 + .../asset/SmartCampaignsResponse.java | 3 + .../entities/asset/SmartListsResponse.java | 3 + .../api/entities/asset/SnippetsResponse.java | 3 + .../entities/asset/StaticListsResponse.java | 3 + .../api/entities/asset/TagsResponse.java | 3 + .../api/entities/asset/gen/Response.java | 29 ++ ...itySchemaHelper.java => EntityHelper.java} | 389 ++++++++++++++---- .../entity/MarketoEntityFormatProvider.java | 51 +++ .../entity/MarketoEntityInputFormat.java | 62 +++ .../batch/entity/MarketoEntityPlugin.java | 80 ++++ .../entity/MarketoEntityRecordReader.java | 119 ++++++ .../entity/MarketoEntitySourceConfig.java | 160 +++++++ .../batch/entity/MarketoEntitySplit.java | 78 ++++ ...aHelperTest.java => EntityHelperTest.java} | 14 +- widgets/MarketoEntityPlugin-batchsource.json | 94 +++++ 28 files changed, 1195 insertions(+), 84 deletions(-) create mode 100644 docs/MarketoEntityPlugin-batchsource.md create mode 100644 icons/MarketoEntityPlugin-batchsource.png create mode 100644 src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/gen/Response.java rename src/main/java/io/cdap/plugin/marketo/source/batch/entity/{EntitySchemaHelper.java => EntityHelper.java} (79%) create mode 100644 src/main/java/io/cdap/plugin/marketo/source/batch/entity/MarketoEntityFormatProvider.java create mode 100644 src/main/java/io/cdap/plugin/marketo/source/batch/entity/MarketoEntityInputFormat.java create mode 100644 src/main/java/io/cdap/plugin/marketo/source/batch/entity/MarketoEntityPlugin.java create mode 100644 src/main/java/io/cdap/plugin/marketo/source/batch/entity/MarketoEntityRecordReader.java create mode 100644 src/main/java/io/cdap/plugin/marketo/source/batch/entity/MarketoEntitySourceConfig.java create mode 100644 src/main/java/io/cdap/plugin/marketo/source/batch/entity/MarketoEntitySplit.java rename src/test/java/io/cdap/plugin/marketo/source/batch/entity/{EntitySchemaHelperTest.java => EntityHelperTest.java} (81%) create mode 100644 widgets/MarketoEntityPlugin-batchsource.json diff --git a/contrib/src/main/java/io/cdap/plugin/generate/Generate.java b/contrib/src/main/java/io/cdap/plugin/generate/Generate.java index 5c15348..90642fb 100644 --- a/contrib/src/main/java/io/cdap/plugin/generate/Generate.java +++ b/contrib/src/main/java/io/cdap/plugin/generate/Generate.java @@ -17,14 +17,18 @@ package io.cdap.plugin.generate; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.google.common.reflect.ClassPath; import com.squareup.javapoet.ClassName; +import com.squareup.javapoet.FieldSpec; import com.squareup.javapoet.JavaFile; import com.squareup.javapoet.MethodSpec; import com.squareup.javapoet.TypeSpec; import io.cdap.cdap.api.data.format.StructuredRecord; import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.plugin.marketo.common.api.Marketo; import io.cdap.plugin.marketo.common.api.entities.asset.gen.Entity; +import io.cdap.plugin.marketo.common.api.entities.asset.gen.Response; import java.io.IOException; import java.lang.annotation.Annotation; @@ -32,6 +36,7 @@ import java.lang.reflect.ParameterizedType; import java.util.ArrayList; import java.util.Comparator; +import java.util.Iterator; import java.util.List; import java.util.stream.Collectors; import javax.lang.model.element.Modifier; @@ -46,6 +51,53 @@ public class Generate { public static void main(String... args) throws IOException { List methods = new ArrayList<>(); + List withoutPaging = getResponseClasses().stream() + .filter(Generate::isNotPaged) + .map(Generate::getItemClsForResponseCls) + .map(aClass -> "EntityType."+aClass.getSimpleName().toUpperCase()) + .collect(Collectors.toList()); + + FieldSpec withoutPagingField = FieldSpec.builder(ClassName.bestGuess("List"), "ENTITIES_WITHOUT_PAGING") + .addModifiers(Modifier.PRIVATE, Modifier.STATIC, Modifier.FINAL) + .initializer("$T.of(\n$L\n)", ImmutableList.class, String.join(",\n", withoutPaging)) + .build(); + + MethodSpec supportPagingMethod = MethodSpec.methodBuilder("supportPaging") + .addModifiers(Modifier.PUBLIC, Modifier.STATIC) + .returns(boolean.class) + .addParameter(ClassName.bestGuess("EntityType"), "entityType") + .addStatement("return !ENTITIES_WITHOUT_PAGING.contains(entityType)") + .build(); + + MethodSpec.Builder iteratorForEntityTypeBuilder = MethodSpec.methodBuilder("iteratorForEntityType") + .addModifiers(Modifier.PUBLIC, Modifier.STATIC) + .returns(Iterator.class) + .addParameter(Marketo.class, "marketo") + .addParameter(ClassName.bestGuess("EntityType"), "entityType") + .addParameter(int.class, "offset") + .addParameter(int.class, "maxResults") + .beginControlFlow("switch (entityType)"); + + getResponseClasses().forEach(aClass -> { + ParameterizedType baseT = (ParameterizedType) aClass.getGenericSuperclass(); + Class responseItemClass = (Class) baseT.getActualTypeArguments()[0]; + iteratorForEntityTypeBuilder.beginControlFlow("case $L:", + responseItemClass.getSimpleName().toUpperCase()); + iteratorForEntityTypeBuilder.addStatement("return marketo.iteratePage(\n$S,\n $T.class,\n $T::getResult,\n " + + "$T.of(\n\"maxReturn\",\n Integer.toString(maxResults),\n" + + " \"offset\",\n Integer.toString(offset)\n)\n)", + getFetchUrl(aClass), + aClass, aClass, + ImmutableMap.class); + iteratorForEntityTypeBuilder.endControlFlow(); + }); + iteratorForEntityTypeBuilder.beginControlFlow("default:"); + iteratorForEntityTypeBuilder.addStatement( + "throw new IllegalArgumentException(\"Unknown entity name:\" + entityType.getValue())"); + iteratorForEntityTypeBuilder.endControlFlow(); + iteratorForEntityTypeBuilder.endControlFlow(); + + methods.add(iteratorForEntityTypeBuilder.build()); MethodSpec.Builder schemaForEntityNameBuilder = MethodSpec.methodBuilder("schemaForEntityName") .addModifiers(Modifier.PUBLIC, Modifier.STATIC) @@ -78,7 +130,7 @@ public static void main(String... args) throws IOException { simpleTypeToCdapType(field)); } else if (isComplexType(field)) { entitySchemaGenBuilder.addStatement( - "fields.add(Schema.Field.of($S, Schema.nullableOf(EntitySchemaHelper.$L())))", + "fields.add(Schema.Field.of($S, Schema.nullableOf(EntityHelper.$L())))", field.getName(), generateSchemaGetterMethod(field.getType())); } else if (isListType(field)) { entitySchemaGenBuilder.addStatement( @@ -89,7 +141,7 @@ public static void main(String... args) throws IOException { } } entitySchemaGenBuilder.addStatement( - "return Schema.recordOf(UUID.randomUUID().toString().replace(\"-\", \"\"), fields)"); + "return Schema.recordOf(\"Schema\" + UUID.randomUUID().toString().replace(\"-\", \"\"), fields)"); methods.add(entitySchemaGenBuilder.build()); }); @@ -118,7 +170,7 @@ public static void main(String... args) throws IOException { aClass.getSimpleName().toLowerCase(), findGetterMethod(field, aClass)); } else if (isComplexType(field)) { - recordFromEntityBuilder.addStatement("builder.set($S, EntitySchemaHelper.structuredRecordFromEntity(\n" + + recordFromEntityBuilder.addStatement("builder.set($S, EntityHelper.structuredRecordFromEntity(\n" + " $S,\n" + " $L.$L(),\n" + " schema.getField($S).getSchema().getNonNullable()))", @@ -140,7 +192,7 @@ public static void main(String... args) throws IOException { "builder.set(\n" + " $S,\n" + " $L.$L().stream()\n" + - " .map(ent -> EntitySchemaHelper.structuredRecordFromEntity(\n" + + " .map(ent -> EntityHelper.structuredRecordFromEntity(\n" + " $S,\n" + " ent,\n" + " schema.getField($S).getSchema().getNonNullable().getComponentSchema()))\n" + @@ -184,6 +236,14 @@ public static void main(String... args) throws IOException { .build() ); + entityTypeEnumBuilder.addMethod( + MethodSpec.methodBuilder("getValue") + .addModifiers(Modifier.PUBLIC) + .returns(String.class) + .addStatement("return value") + .build() + ); + entityTypeEnumBuilder.addMethod( MethodSpec.methodBuilder("fromString") .addModifiers(Modifier.PUBLIC, Modifier.STATIC) @@ -198,8 +258,10 @@ public static void main(String... args) throws IOException { .build() ); - TypeSpec schemaHelperSpec = TypeSpec.classBuilder("EntitySchemaHelper") + TypeSpec schemaHelperSpec = TypeSpec.classBuilder("EntityHelper") .addModifiers(Modifier.PUBLIC, Modifier.FINAL) + .addField(withoutPagingField) + .addMethod(supportPagingMethod) .addMethods(methods) .addType(entityTypeEnumBuilder.build()) .build(); @@ -257,7 +319,7 @@ public static String listTypeToCdapType(Field field) { } else if (listType.equals(Integer.class) || listType.equals(int.class)) { return "Schema.nullableOf(Schema.arrayOf(Schema.of(Schema.Type.INT)))"; } else { - return "Schema.nullableOf(Schema.arrayOf(EntitySchemaHelper." + generateSchemaGetterMethod(listType) + "()))"; + return "Schema.nullableOf(Schema.arrayOf(EntityHelper." + generateSchemaGetterMethod(listType) + "()))"; } } @@ -312,6 +374,15 @@ public static List getTopLevelEntityClasses() throws IOException { .collect(Collectors.toList()); } + public static List getResponseClasses() throws IOException { + return ClassPath.from(Generate.class.getClassLoader()) + .getTopLevelClasses("io.cdap.plugin.marketo.common.api.entities.asset").stream() + .map(ClassPath.ClassInfo::load) + .filter(Generate::isResponses) + .sorted(Comparator.comparing(Class::getSimpleName)) + .collect(Collectors.toList()); + } + public static boolean isEntity(Class cls) { for (Annotation a : cls.getAnnotations()) { if (a.annotationType().equals(Entity.class)) { @@ -321,6 +392,45 @@ public static boolean isEntity(Class cls) { return false; } + public static boolean isResponses(Class cls) { + for (Annotation a : cls.getAnnotations()) { + if (a.annotationType().equals(Response.class)) { + return true; + } + } + return false; + } + + public static String getFetchUrl(Class cls) { + for (Annotation a : cls.getAnnotations()) { + if (a.annotationType().equals(Response.class)) { + Response r = (Response) a; + return r.fetchUrl(); + } + } + throw new RuntimeException(); + } + + public static boolean getPaged(Class cls) { + for (Annotation a : cls.getAnnotations()) { + if (a.annotationType().equals(Response.class)) { + Response r = (Response) a; + return r.paged(); + } + } + throw new RuntimeException(); + } + + public static boolean isNotPaged(Class cls) { + return !getPaged(cls); + } + + public static Class getItemClsForResponseCls(Class cls) { + ParameterizedType baseT = (ParameterizedType) cls.getGenericSuperclass(); + Class responseItemClass = (Class) baseT.getActualTypeArguments()[0]; + return responseItemClass; + } + public static boolean isTopLevelEntity(Class cls) { for (Annotation a : cls.getAnnotations()) { if (a.annotationType().equals(Entity.class)) { diff --git a/docs/MarketoEntityPlugin-batchsource.md b/docs/MarketoEntityPlugin-batchsource.md new file mode 100644 index 0000000..4db2a24 --- /dev/null +++ b/docs/MarketoEntityPlugin-batchsource.md @@ -0,0 +1,36 @@ +# Marketo Entity Batch Source + +Description +----------- +This plugin is used to query Leads or Activities entities for specified date range from Marketo. + +Properties +---------- +### General + +**Reference Name:** Name used to uniquely identify this source for lineage, annotating metadata, etc. + +**Rest API endpoint:** Marketo rest API endpoint, unique for each client. + +**Entity Type:** Type of entity. +### Authentication + +**Client ID:** Client ID. + +**Client Secret:** Client secret. + +### Advanced + +**Splits Count:** Parallel splits count. + +**Max Results Per Page:** Maximum number of records to retrieve by single request. + +API limits notes +---------------- +All entities will be fetched at once. + +Total requests count depends on **Max Results Per Page**, leave it with default +value(200 - maximum possible value) to minimize risk of reaching limits. + +Default request limit is 50000 per day, this will allow to fetch ~10.000.000 entities, but you must consider this limit +if there are several pipelines or other applications that uses API. \ No newline at end of file diff --git a/icons/MarketoEntityPlugin-batchsource.png b/icons/MarketoEntityPlugin-batchsource.png new file mode 100644 index 0000000000000000000000000000000000000000..4abe7e74fa2422e7bba39b77d633ece4ccce275f GIT binary patch literal 996 zcmVPx#1ZP1_K>z@;j|==^1poj532;bRa{vGi!vFvd!vV){sAK>D19?eAK~!i%?U`Rl zQ&Akp&$-)_Ec-(dC7B6E_0)srgAronLv9*^270gIw$hNEq8BNuy-4&Jh!wh_?qXya z=w=B~4l*dRhh8!a0yC_vh{`|vcYpn+C*fwB(_Qr}_QCDm^F5bwKKK02xxytpAQjp3 zq|&ynthpovc3fR36~PyLE1Db^rNXwgEL{|Qh#-`bR7!}ABn5^=lfAYk`-XvaMFlq^ zb5OdS5_)}do8gscY$;{wf@mTl6}M%Um*3vNn%Y3=UP^^7EVqHmV#h(2hDDnhQ35qd zQxoHo$?*uw;%tS{zMJ*L43=A-0OSe&c2`wV>pm9G1Fwr6(bUpFiBQj)A%Fn90YY5> zwCBUbyQdzHEx`KbAsHQwIFdTn3=0tA$8QJm`!36Ur^i@4!*Z<=Eb~xlkaL0np7hj= z&p24Q^60`P5(&KP2K?a~YDgNwQchy=3@fzQ=_VXSrJy6_EQk@hKKc8!YFAT~1vf>NmeNjEy()~l?WJJcdN@&gCoglA>m0Hnz- zOODfRsc&~%AGUhT)3X=xiAqGiS2c!%uXDQ0ckeu~jJ$65Kn*xa~6^au@bx1lE6}G%=)$K$gQ4JpI8(T?nF>v8b8&zQJk@P z3c|jd{~NAEP+&w*ltU=d%A%;@NCf(WLNK@e{>c|Ch(9D_%l`cQ$1%M6Cge97Rx1D) SQdJ880000 { } diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/EmailTemplateResponse.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/EmailTemplateResponse.java index 186fa6a..0ba9987 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/EmailTemplateResponse.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/EmailTemplateResponse.java @@ -16,9 +16,12 @@ package io.cdap.plugin.marketo.common.api.entities.asset; +import io.cdap.plugin.marketo.common.api.entities.asset.gen.Response; + /** * GET /rest/asset/v1/emailTemplates.json */ +@Response(fetchUrl = "/rest/asset/v1/emailTemplates.json") public class EmailTemplateResponse extends SimpleBaseResponse { } diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/FileResponse.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/FileResponse.java index 9f8cf78..f6eed56 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/FileResponse.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/FileResponse.java @@ -16,9 +16,12 @@ package io.cdap.plugin.marketo.common.api.entities.asset; +import io.cdap.plugin.marketo.common.api.entities.asset.gen.Response; + /** * GET /rest/asset/v1/files.json */ +@Response(fetchUrl = "/rest/asset/v1/files.json") public class FileResponse extends SimpleBaseResponse { } diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/FolderResponse.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/FolderResponse.java index 553538e..efac548 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/FolderResponse.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/FolderResponse.java @@ -16,9 +16,12 @@ package io.cdap.plugin.marketo.common.api.entities.asset; +import io.cdap.plugin.marketo.common.api.entities.asset.gen.Response; + /** * GET /rest/asset/v1/folders.json */ +@Response(fetchUrl = "/rest/asset/v1/folders.json") public class FolderResponse extends SimpleBaseResponse { } diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/FormFieldsResponse.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/FormFieldsResponse.java index 8e38999..ff406bc 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/FormFieldsResponse.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/FormFieldsResponse.java @@ -16,9 +16,12 @@ package io.cdap.plugin.marketo.common.api.entities.asset; +import io.cdap.plugin.marketo.common.api.entities.asset.gen.Response; + /** * GET /rest/asset/v1/form/fields.json */ +@Response(fetchUrl = "/rest/asset/v1/form/fields.json") public class FormFieldsResponse extends SimpleBaseResponse { } diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/FormResponse.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/FormResponse.java index 9cee13c..6351369 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/FormResponse.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/FormResponse.java @@ -16,9 +16,12 @@ package io.cdap.plugin.marketo.common.api.entities.asset; +import io.cdap.plugin.marketo.common.api.entities.asset.gen.Response; + /** * GET /rest/asset/v1/forms.json */ +@Response(fetchUrl = "/rest/asset/v1/forms.json") public class FormResponse extends SimpleBaseResponse { } diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/LandingPageResponse.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/LandingPageResponse.java index c5f14e8..c577992 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/LandingPageResponse.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/LandingPageResponse.java @@ -16,9 +16,12 @@ package io.cdap.plugin.marketo.common.api.entities.asset; +import io.cdap.plugin.marketo.common.api.entities.asset.gen.Response; + /** * GET /rest/asset/v1/landingPages.json */ +@Response(fetchUrl = "/rest/asset/v1/landingPages.json") public class LandingPageResponse extends SimpleBaseResponse { } diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/LandingPageTemplateResponse.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/LandingPageTemplateResponse.java index 4912f9b..47fbfca 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/LandingPageTemplateResponse.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/LandingPageTemplateResponse.java @@ -16,9 +16,12 @@ package io.cdap.plugin.marketo.common.api.entities.asset; +import io.cdap.plugin.marketo.common.api.entities.asset.gen.Response; + /** * GET /rest/asset/v1/landingPageTemplates.json */ +@Response(fetchUrl = "/rest/asset/v1/landingPageTemplates.json") public class LandingPageTemplateResponse extends SimpleBaseResponse { } diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/ProgramResponse.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/ProgramResponse.java index a69b142..eb827a5 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/ProgramResponse.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/ProgramResponse.java @@ -16,8 +16,11 @@ package io.cdap.plugin.marketo.common.api.entities.asset; +import io.cdap.plugin.marketo.common.api.entities.asset.gen.Response; + /** * GET /rest/asset/v1/programs.json */ +@Response(fetchUrl = "/rest/asset/v1/programs.json") public class ProgramResponse extends SimpleBaseResponse { } diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/SegmentationResponse.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/SegmentationResponse.java index e2c4d80..e91ae07 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/SegmentationResponse.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/SegmentationResponse.java @@ -16,9 +16,12 @@ package io.cdap.plugin.marketo.common.api.entities.asset; +import io.cdap.plugin.marketo.common.api.entities.asset.gen.Response; + /** * GET /rest/asset/v1/segmentation.json * NO PAGING */ +@Response(fetchUrl = "/rest/asset/v1/segmentation.json", paged = false) public class SegmentationResponse extends SimpleBaseResponse { } diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/SmartCampaignsResponse.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/SmartCampaignsResponse.java index bdfd0dd..55a3828 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/SmartCampaignsResponse.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/SmartCampaignsResponse.java @@ -16,9 +16,12 @@ package io.cdap.plugin.marketo.common.api.entities.asset; +import io.cdap.plugin.marketo.common.api.entities.asset.gen.Response; + /** * GET /rest/asset/v1/smartCampaigns.json */ +@Response(fetchUrl = "/rest/asset/v1/smartCampaigns.json") public class SmartCampaignsResponse extends SimpleBaseResponse { } diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/SmartListsResponse.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/SmartListsResponse.java index a2ca48a..612afe4 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/SmartListsResponse.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/SmartListsResponse.java @@ -16,8 +16,11 @@ package io.cdap.plugin.marketo.common.api.entities.asset; +import io.cdap.plugin.marketo.common.api.entities.asset.gen.Response; + /** * GET /rest/asset/v1/smartLists.json */ +@Response(fetchUrl = "/rest/asset/v1/smartLists.json") public class SmartListsResponse extends SimpleBaseResponse { } diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/SnippetsResponse.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/SnippetsResponse.java index fc8f442..0a83ccd 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/SnippetsResponse.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/SnippetsResponse.java @@ -16,8 +16,11 @@ package io.cdap.plugin.marketo.common.api.entities.asset; +import io.cdap.plugin.marketo.common.api.entities.asset.gen.Response; + /** * GET /rest/asset/v1/snippets.json */ +@Response(fetchUrl = "/rest/asset/v1/snippets.json") public class SnippetsResponse extends SimpleBaseResponse { } diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/StaticListsResponse.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/StaticListsResponse.java index c5d8f99..3cb9c29 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/StaticListsResponse.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/StaticListsResponse.java @@ -16,8 +16,11 @@ package io.cdap.plugin.marketo.common.api.entities.asset; +import io.cdap.plugin.marketo.common.api.entities.asset.gen.Response; + /** * GET /rest/asset/v1/staticLists.json */ +@Response(fetchUrl = "/rest/asset/v1/staticLists.json") public class StaticListsResponse extends SimpleBaseResponse { } diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/TagsResponse.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/TagsResponse.java index bf7e8ef..862aee6 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/TagsResponse.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/TagsResponse.java @@ -16,8 +16,11 @@ package io.cdap.plugin.marketo.common.api.entities.asset; +import io.cdap.plugin.marketo.common.api.entities.asset.gen.Response; + /** * GET /rest/asset/v1/tagTypes.json */ +@Response(fetchUrl = "/rest/asset/v1/tagTypes.json") public class TagsResponse extends SimpleBaseResponse { } diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/gen/Response.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/gen/Response.java new file mode 100644 index 0000000..ae0cc4a --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/asset/gen/Response.java @@ -0,0 +1,29 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api.entities.asset.gen; + +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +/** + * Marks class as response class. + */ +@Retention(RetentionPolicy.RUNTIME) +public @interface Response { + String fetchUrl(); + boolean paged() default true; +} diff --git a/src/main/java/io/cdap/plugin/marketo/source/batch/entity/EntitySchemaHelper.java b/src/main/java/io/cdap/plugin/marketo/source/batch/entity/EntityHelper.java similarity index 79% rename from src/main/java/io/cdap/plugin/marketo/source/batch/entity/EntitySchemaHelper.java rename to src/main/java/io/cdap/plugin/marketo/source/batch/entity/EntityHelper.java index 785c1c2..82ef757 100644 --- a/src/main/java/io/cdap/plugin/marketo/source/batch/entity/EntitySchemaHelper.java +++ b/src/main/java/io/cdap/plugin/marketo/source/batch/entity/EntityHelper.java @@ -16,32 +16,51 @@ package io.cdap.plugin.marketo.source.batch.entity; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import io.cdap.cdap.api.data.format.StructuredRecord; import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.plugin.marketo.common.api.Marketo; import io.cdap.plugin.marketo.common.api.entities.asset.Email; import io.cdap.plugin.marketo.common.api.entities.asset.EmailCCField; import io.cdap.plugin.marketo.common.api.entities.asset.EmailField; +import io.cdap.plugin.marketo.common.api.entities.asset.EmailResponse; import io.cdap.plugin.marketo.common.api.entities.asset.EmailTemplate; +import io.cdap.plugin.marketo.common.api.entities.asset.EmailTemplateResponse; import io.cdap.plugin.marketo.common.api.entities.asset.File; import io.cdap.plugin.marketo.common.api.entities.asset.FileFolder; +import io.cdap.plugin.marketo.common.api.entities.asset.FileResponse; import io.cdap.plugin.marketo.common.api.entities.asset.Folder; import io.cdap.plugin.marketo.common.api.entities.asset.FolderDescriptor; +import io.cdap.plugin.marketo.common.api.entities.asset.FolderResponse; import io.cdap.plugin.marketo.common.api.entities.asset.Form; import io.cdap.plugin.marketo.common.api.entities.asset.FormField; +import io.cdap.plugin.marketo.common.api.entities.asset.FormFieldsResponse; import io.cdap.plugin.marketo.common.api.entities.asset.FormKnownVisitorDTO; +import io.cdap.plugin.marketo.common.api.entities.asset.FormResponse; import io.cdap.plugin.marketo.common.api.entities.asset.FormThankYouPageDTO; import io.cdap.plugin.marketo.common.api.entities.asset.LandingPage; +import io.cdap.plugin.marketo.common.api.entities.asset.LandingPageResponse; import io.cdap.plugin.marketo.common.api.entities.asset.LandingPageTemplate; +import io.cdap.plugin.marketo.common.api.entities.asset.LandingPageTemplateResponse; import io.cdap.plugin.marketo.common.api.entities.asset.Program; +import io.cdap.plugin.marketo.common.api.entities.asset.ProgramResponse; import io.cdap.plugin.marketo.common.api.entities.asset.Recurrence; import io.cdap.plugin.marketo.common.api.entities.asset.Segmentation; +import io.cdap.plugin.marketo.common.api.entities.asset.SegmentationResponse; import io.cdap.plugin.marketo.common.api.entities.asset.SmartCampaign; +import io.cdap.plugin.marketo.common.api.entities.asset.SmartCampaignsResponse; import io.cdap.plugin.marketo.common.api.entities.asset.SmartList; +import io.cdap.plugin.marketo.common.api.entities.asset.SmartListsResponse; import io.cdap.plugin.marketo.common.api.entities.asset.Snippet; +import io.cdap.plugin.marketo.common.api.entities.asset.SnippetsResponse; import io.cdap.plugin.marketo.common.api.entities.asset.StaticList; +import io.cdap.plugin.marketo.common.api.entities.asset.StaticListsResponse; import io.cdap.plugin.marketo.common.api.entities.asset.Tag; +import io.cdap.plugin.marketo.common.api.entities.asset.TagsResponse; import java.util.ArrayList; +import java.util.Iterator; import java.util.List; import java.util.UUID; import java.util.stream.Collectors; @@ -50,7 +69,219 @@ * Schema helper for entity plugin. */ @SuppressWarnings("DuplicatedCode") -public final class EntitySchemaHelper { +public final class EntityHelper { + private static final List ENTITIES_WITHOUT_PAGING = ImmutableList.of( + EntityType.SEGMENTATION + ); + + public static boolean supportPaging(EntityType entityType) { + return !ENTITIES_WITHOUT_PAGING.contains(entityType); + } + + public static Iterator iteratorForEntityType(Marketo marketo, EntityType entityType, int offset, + int maxResults) { + switch (entityType) { + case EMAIL: { + return marketo.iteratePage( + "/rest/asset/v1/emails.json", + EmailResponse.class, + EmailResponse::getResult, + ImmutableMap.of( + "maxReturn", + Integer.toString(maxResults), + "offset", + Integer.toString(offset) + ) + ); + } + case EMAILTEMPLATE: { + return marketo.iteratePage( + "/rest/asset/v1/emailTemplates.json", + EmailTemplateResponse.class, + EmailTemplateResponse::getResult, + ImmutableMap.of( + "maxReturn", + Integer.toString(maxResults), + "offset", + Integer.toString(offset) + ) + ); + } + case FILE: { + return marketo.iteratePage( + "/rest/asset/v1/files.json", + FileResponse.class, + FileResponse::getResult, + ImmutableMap.of( + "maxReturn", + Integer.toString(maxResults), + "offset", + Integer.toString(offset) + ) + ); + } + case FOLDER: { + return marketo.iteratePage( + "/rest/asset/v1/folders.json", + FolderResponse.class, + FolderResponse::getResult, + ImmutableMap.of( + "maxReturn", + Integer.toString(maxResults), + "offset", + Integer.toString(offset) + ) + ); + } + case FORMFIELD: { + return marketo.iteratePage( + "/rest/asset/v1/form/fields.json", + FormFieldsResponse.class, + FormFieldsResponse::getResult, + ImmutableMap.of( + "maxReturn", + Integer.toString(maxResults), + "offset", + Integer.toString(offset) + ) + ); + } + case FORM: { + return marketo.iteratePage( + "/rest/asset/v1/forms.json", + FormResponse.class, + FormResponse::getResult, + ImmutableMap.of( + "maxReturn", + Integer.toString(maxResults), + "offset", + Integer.toString(offset) + ) + ); + } + case LANDINGPAGE: { + return marketo.iteratePage( + "/rest/asset/v1/landingPages.json", + LandingPageResponse.class, + LandingPageResponse::getResult, + ImmutableMap.of( + "maxReturn", + Integer.toString(maxResults), + "offset", + Integer.toString(offset) + ) + ); + } + case LANDINGPAGETEMPLATE: { + return marketo.iteratePage( + "/rest/asset/v1/landingPageTemplates.json", + LandingPageTemplateResponse.class, + LandingPageTemplateResponse::getResult, + ImmutableMap.of( + "maxReturn", + Integer.toString(maxResults), + "offset", + Integer.toString(offset) + ) + ); + } + case PROGRAM: { + return marketo.iteratePage( + "/rest/asset/v1/programs.json", + ProgramResponse.class, + ProgramResponse::getResult, + ImmutableMap.of( + "maxReturn", + Integer.toString(maxResults), + "offset", + Integer.toString(offset) + ) + ); + } + case SEGMENTATION: { + return marketo.iteratePage( + "/rest/asset/v1/segmentation.json", + SegmentationResponse.class, + SegmentationResponse::getResult, + ImmutableMap.of( + "maxReturn", + Integer.toString(maxResults), + "offset", + Integer.toString(offset) + ) + ); + } + case SMARTCAMPAIGN: { + return marketo.iteratePage( + "/rest/asset/v1/smartCampaigns.json", + SmartCampaignsResponse.class, + SmartCampaignsResponse::getResult, + ImmutableMap.of( + "maxReturn", + Integer.toString(maxResults), + "offset", + Integer.toString(offset) + ) + ); + } + case SMARTLIST: { + return marketo.iteratePage( + "/rest/asset/v1/smartLists.json", + SmartListsResponse.class, + SmartListsResponse::getResult, + ImmutableMap.of( + "maxReturn", + Integer.toString(maxResults), + "offset", + Integer.toString(offset) + ) + ); + } + case SNIPPET: { + return marketo.iteratePage( + "/rest/asset/v1/snippets.json", + SnippetsResponse.class, + SnippetsResponse::getResult, + ImmutableMap.of( + "maxReturn", + Integer.toString(maxResults), + "offset", + Integer.toString(offset) + ) + ); + } + case STATICLIST: { + return marketo.iteratePage( + "/rest/asset/v1/staticLists.json", + StaticListsResponse.class, + StaticListsResponse::getResult, + ImmutableMap.of( + "maxReturn", + Integer.toString(maxResults), + "offset", + Integer.toString(offset) + ) + ); + } + case TAG: { + return marketo.iteratePage( + "/rest/asset/v1/tagTypes.json", + TagsResponse.class, + TagsResponse::getResult, + ImmutableMap.of( + "maxReturn", + Integer.toString(maxResults), + "offset", + Integer.toString(offset) + ) + ); + } + default: { + throw new IllegalArgumentException("Unknown entity name:" + entityType.getValue()); + } + } + } + public static Schema schemaForEntityName(String entityName) { switch (entityName) { case "Email": { @@ -129,16 +360,16 @@ public static Schema getEmailSchema() { List fields = new ArrayList<>(); fields.add(Schema.Field.of("createdAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); fields.add(Schema.Field.of("description", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); - fields.add(Schema.Field.of("folder", Schema.nullableOf(EntitySchemaHelper.getFolderDescriptorSchema()))); - fields.add(Schema.Field.of("fromEmail", Schema.nullableOf(EntitySchemaHelper.getEmailFieldSchema()))); - fields.add(Schema.Field.of("fromName", Schema.nullableOf(EntitySchemaHelper.getEmailFieldSchema()))); + fields.add(Schema.Field.of("folder", Schema.nullableOf(EntityHelper.getFolderDescriptorSchema()))); + fields.add(Schema.Field.of("fromEmail", Schema.nullableOf(EntityHelper.getEmailFieldSchema()))); + fields.add(Schema.Field.of("fromName", Schema.nullableOf(EntityHelper.getEmailFieldSchema()))); fields.add(Schema.Field.of("id", Schema.nullableOf(Schema.of(Schema.Type.INT)))); fields.add(Schema.Field.of("name", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); fields.add(Schema.Field.of("operational", Schema.nullableOf(Schema.of(Schema.Type.BOOLEAN)))); fields.add(Schema.Field.of("publishToMSI", Schema.nullableOf(Schema.of(Schema.Type.BOOLEAN)))); - fields.add(Schema.Field.of("replyEmail", Schema.nullableOf(EntitySchemaHelper.getEmailFieldSchema()))); + fields.add(Schema.Field.of("replyEmail", Schema.nullableOf(EntityHelper.getEmailFieldSchema()))); fields.add(Schema.Field.of("status", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); - fields.add(Schema.Field.of("subject", Schema.nullableOf(EntitySchemaHelper.getEmailFieldSchema()))); + fields.add(Schema.Field.of("subject", Schema.nullableOf(EntityHelper.getEmailFieldSchema()))); fields.add(Schema.Field.of("template", Schema.nullableOf(Schema.of(Schema.Type.INT)))); fields.add(Schema.Field.of("textOnly", Schema.nullableOf(Schema.of(Schema.Type.BOOLEAN)))); fields.add(Schema.Field.of("updatedAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); @@ -148,8 +379,8 @@ public static Schema getEmailSchema() { fields.add(Schema.Field.of("workspace", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); fields.add(Schema.Field.of("autoCopyToText", Schema.nullableOf(Schema.of(Schema.Type.BOOLEAN)))); fields.add(Schema.Field.of("ccFields", - Schema.nullableOf(Schema.arrayOf(EntitySchemaHelper.getEmailCCFieldSchema())))); - return Schema.recordOf(UUID.randomUUID().toString().replace("-", ""), fields); + Schema.nullableOf(Schema.arrayOf(EntityHelper.getEmailCCFieldSchema())))); + return Schema.recordOf("Schema" + UUID.randomUUID().toString().replace("-", ""), fields); } public static Schema getEmailCCFieldSchema() { @@ -158,21 +389,21 @@ public static Schema getEmailCCFieldSchema() { fields.add(Schema.Field.of("objectName", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); fields.add(Schema.Field.of("displayName", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); fields.add(Schema.Field.of("apiName", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); - return Schema.recordOf(UUID.randomUUID().toString().replace("-", ""), fields); + return Schema.recordOf("Schema" + UUID.randomUUID().toString().replace("-", ""), fields); } public static Schema getEmailFieldSchema() { List fields = new ArrayList<>(); fields.add(Schema.Field.of("type", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); fields.add(Schema.Field.of("value", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); - return Schema.recordOf(UUID.randomUUID().toString().replace("-", ""), fields); + return Schema.recordOf("Schema" + UUID.randomUUID().toString().replace("-", ""), fields); } public static Schema getEmailTemplateSchema() { List fields = new ArrayList<>(); fields.add(Schema.Field.of("createdAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); fields.add(Schema.Field.of("description", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); - fields.add(Schema.Field.of("folder", Schema.nullableOf(EntitySchemaHelper.getFolderDescriptorSchema()))); + fields.add(Schema.Field.of("folder", Schema.nullableOf(EntityHelper.getFolderDescriptorSchema()))); fields.add(Schema.Field.of("id", Schema.nullableOf(Schema.of(Schema.Type.INT)))); fields.add(Schema.Field.of("name", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); fields.add(Schema.Field.of("status", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); @@ -180,21 +411,21 @@ public static Schema getEmailTemplateSchema() { fields.add(Schema.Field.of("url", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); fields.add(Schema.Field.of("version", Schema.nullableOf(Schema.of(Schema.Type.INT)))); fields.add(Schema.Field.of("workspace", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); - return Schema.recordOf(UUID.randomUUID().toString().replace("-", ""), fields); + return Schema.recordOf("Schema" + UUID.randomUUID().toString().replace("-", ""), fields); } public static Schema getFileSchema() { List fields = new ArrayList<>(); fields.add(Schema.Field.of("createdAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); fields.add(Schema.Field.of("description", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); - fields.add(Schema.Field.of("folder", Schema.nullableOf(EntitySchemaHelper.getFileFolderSchema()))); + fields.add(Schema.Field.of("folder", Schema.nullableOf(EntityHelper.getFileFolderSchema()))); fields.add(Schema.Field.of("id", Schema.nullableOf(Schema.of(Schema.Type.INT)))); fields.add(Schema.Field.of("mimeType", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); fields.add(Schema.Field.of("name", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); fields.add(Schema.Field.of("size", Schema.nullableOf(Schema.of(Schema.Type.INT)))); fields.add(Schema.Field.of("updatedAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); fields.add(Schema.Field.of("url", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); - return Schema.recordOf(UUID.randomUUID().toString().replace("-", ""), fields); + return Schema.recordOf("Schema" + UUID.randomUUID().toString().replace("-", ""), fields); } public static Schema getFileFolderSchema() { @@ -202,7 +433,7 @@ public static Schema getFileFolderSchema() { fields.add(Schema.Field.of("id", Schema.nullableOf(Schema.of(Schema.Type.INT)))); fields.add(Schema.Field.of("name", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); fields.add(Schema.Field.of("type", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); - return Schema.recordOf(UUID.randomUUID().toString().replace("-", ""), fields); + return Schema.recordOf("Schema" + UUID.randomUUID().toString().replace("-", ""), fields); } public static Schema getFolderSchema() { @@ -210,18 +441,18 @@ public static Schema getFolderSchema() { fields.add(Schema.Field.of("accessZoneId", Schema.nullableOf(Schema.of(Schema.Type.INT)))); fields.add(Schema.Field.of("createdAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); fields.add(Schema.Field.of("description", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); - fields.add(Schema.Field.of("folderId", Schema.nullableOf(EntitySchemaHelper.getFolderDescriptorSchema()))); + fields.add(Schema.Field.of("folderId", Schema.nullableOf(EntityHelper.getFolderDescriptorSchema()))); fields.add(Schema.Field.of("folderType", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); fields.add(Schema.Field.of("id", Schema.nullableOf(Schema.of(Schema.Type.INT)))); fields.add(Schema.Field.of("isArchive", Schema.nullableOf(Schema.of(Schema.Type.BOOLEAN)))); fields.add(Schema.Field.of("isSystem", Schema.nullableOf(Schema.of(Schema.Type.BOOLEAN)))); fields.add(Schema.Field.of("name", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); - fields.add(Schema.Field.of("parent", Schema.nullableOf(EntitySchemaHelper.getFolderDescriptorSchema()))); + fields.add(Schema.Field.of("parent", Schema.nullableOf(EntityHelper.getFolderDescriptorSchema()))); fields.add(Schema.Field.of("path", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); fields.add(Schema.Field.of("updatedAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); fields.add(Schema.Field.of("url", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); fields.add(Schema.Field.of("workspace", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); - return Schema.recordOf(UUID.randomUUID().toString().replace("-", ""), fields); + return Schema.recordOf("Schema" + UUID.randomUUID().toString().replace("-", ""), fields); } public static Schema getFolderDescriptorSchema() { @@ -229,7 +460,7 @@ public static Schema getFolderDescriptorSchema() { fields.add(Schema.Field.of("id", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); fields.add(Schema.Field.of("type", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); fields.add(Schema.Field.of("folderName", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); - return Schema.recordOf(UUID.randomUUID().toString().replace("-", ""), fields); + return Schema.recordOf("Schema" + UUID.randomUUID().toString().replace("-", ""), fields); } public static Schema getFormSchema() { @@ -238,12 +469,11 @@ public static Schema getFormSchema() { fields.add(Schema.Field.of("buttonLocation", Schema.nullableOf(Schema.of(Schema.Type.INT)))); fields.add(Schema.Field.of("createdAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); fields.add(Schema.Field.of("description", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); - fields.add(Schema.Field.of("folder", Schema.nullableOf(EntitySchemaHelper.getFolderDescriptorSchema()))); + fields.add(Schema.Field.of("folder", Schema.nullableOf(EntityHelper.getFolderDescriptorSchema()))); fields.add(Schema.Field.of("fontFamily", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); fields.add(Schema.Field.of("fontSize", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); fields.add(Schema.Field.of("id", Schema.nullableOf(Schema.of(Schema.Type.INT)))); - fields.add(Schema.Field.of("knownVisitor", - Schema.nullableOf(EntitySchemaHelper.getFormKnownVisitorDTOSchema()))); + fields.add(Schema.Field.of("knownVisitor", Schema.nullableOf(EntityHelper.getFormKnownVisitorDTOSchema()))); fields.add(Schema.Field.of("labelPosition", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); fields.add(Schema.Field.of("language", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); fields.add(Schema.Field.of("locale", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); @@ -251,12 +481,12 @@ public static Schema getFormSchema() { fields.add(Schema.Field.of("progressiveProfiling", Schema.nullableOf(Schema.of(Schema.Type.BOOLEAN)))); fields.add(Schema.Field.of("status", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); fields.add(Schema.Field.of("thankYouList", - Schema.nullableOf(Schema.arrayOf(EntitySchemaHelper.getFormThankYouPageDTOSchema())))); + Schema.nullableOf(Schema.arrayOf(EntityHelper.getFormThankYouPageDTOSchema())))); fields.add(Schema.Field.of("theme", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); fields.add(Schema.Field.of("updatedAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); fields.add(Schema.Field.of("url", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); fields.add(Schema.Field.of("waitingLabel", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); - return Schema.recordOf(UUID.randomUUID().toString().replace("-", ""), fields); + return Schema.recordOf("Schema" + UUID.randomUUID().toString().replace("-", ""), fields); } public static Schema getFormFieldSchema() { @@ -279,14 +509,14 @@ public static Schema getFormFieldSchema() { fields.add(Schema.Field.of("placeholderText", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); fields.add(Schema.Field.of("validationMessage", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); fields.add(Schema.Field.of("visibleRows", Schema.nullableOf(Schema.of(Schema.Type.INT)))); - return Schema.recordOf(UUID.randomUUID().toString().replace("-", ""), fields); + return Schema.recordOf("Schema" + UUID.randomUUID().toString().replace("-", ""), fields); } public static Schema getFormKnownVisitorDTOSchema() { List fields = new ArrayList<>(); fields.add(Schema.Field.of("template", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); fields.add(Schema.Field.of("type", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); - return Schema.recordOf(UUID.randomUUID().toString().replace("-", ""), fields); + return Schema.recordOf("Schema" + UUID.randomUUID().toString().replace("-", ""), fields); } public static Schema getFormThankYouPageDTOSchema() { @@ -297,7 +527,7 @@ public static Schema getFormThankYouPageDTOSchema() { fields.add(Schema.Field.of("operator", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); fields.add(Schema.Field.of("subjectField", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); fields.add(Schema.Field.of("values", Schema.nullableOf(Schema.arrayOf(Schema.of(Schema.Type.STRING))))); - return Schema.recordOf(UUID.randomUUID().toString().replace("-", ""), fields); + return Schema.recordOf("Schema" + UUID.randomUUID().toString().replace("-", ""), fields); } public static Schema getLandingPageSchema() { @@ -308,7 +538,7 @@ public static Schema getLandingPageSchema() { fields.add(Schema.Field.of("customHeadHTML", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); fields.add(Schema.Field.of("description", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); fields.add(Schema.Field.of("facebookOgTags", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); - fields.add(Schema.Field.of("folder", Schema.nullableOf(EntitySchemaHelper.getFolderDescriptorSchema()))); + fields.add(Schema.Field.of("folder", Schema.nullableOf(EntityHelper.getFolderDescriptorSchema()))); fields.add(Schema.Field.of("formPrefill", Schema.nullableOf(Schema.of(Schema.Type.BOOLEAN)))); fields.add(Schema.Field.of("id", Schema.nullableOf(Schema.of(Schema.Type.INT)))); fields.add(Schema.Field.of("keywords", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); @@ -320,7 +550,7 @@ public static Schema getLandingPageSchema() { fields.add(Schema.Field.of("title", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); fields.add(Schema.Field.of("updatedAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); fields.add(Schema.Field.of("workspace", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); - return Schema.recordOf(UUID.randomUUID().toString().replace("-", ""), fields); + return Schema.recordOf("Schema" + UUID.randomUUID().toString().replace("-", ""), fields); } public static Schema getLandingPageTemplateSchema() { @@ -328,7 +558,7 @@ public static Schema getLandingPageTemplateSchema() { fields.add(Schema.Field.of("createdAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); fields.add(Schema.Field.of("description", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); fields.add(Schema.Field.of("enableMunchkin", Schema.nullableOf(Schema.of(Schema.Type.BOOLEAN)))); - fields.add(Schema.Field.of("folder", Schema.nullableOf(EntitySchemaHelper.getFolderDescriptorSchema()))); + fields.add(Schema.Field.of("folder", Schema.nullableOf(EntityHelper.getFolderDescriptorSchema()))); fields.add(Schema.Field.of("id", Schema.nullableOf(Schema.of(Schema.Type.INT)))); fields.add(Schema.Field.of("name", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); fields.add(Schema.Field.of("status", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); @@ -336,7 +566,7 @@ public static Schema getLandingPageTemplateSchema() { fields.add(Schema.Field.of("updatedAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); fields.add(Schema.Field.of("url", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); fields.add(Schema.Field.of("workspace", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); - return Schema.recordOf(UUID.randomUUID().toString().replace("-", ""), fields); + return Schema.recordOf("Schema" + UUID.randomUUID().toString().replace("-", ""), fields); } public static Schema getProgramSchema() { @@ -344,7 +574,7 @@ public static Schema getProgramSchema() { fields.add(Schema.Field.of("channel", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); fields.add(Schema.Field.of("createdAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); fields.add(Schema.Field.of("description", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); - fields.add(Schema.Field.of("folder", Schema.nullableOf(EntitySchemaHelper.getFolderDescriptorSchema()))); + fields.add(Schema.Field.of("folder", Schema.nullableOf(EntityHelper.getFolderDescriptorSchema()))); fields.add(Schema.Field.of("id", Schema.nullableOf(Schema.of(Schema.Type.INT)))); fields.add(Schema.Field.of("name", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); fields.add(Schema.Field.of("sfdcId", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); @@ -354,7 +584,7 @@ public static Schema getProgramSchema() { fields.add(Schema.Field.of("updatedAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); fields.add(Schema.Field.of("url", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); fields.add(Schema.Field.of("workspace", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); - return Schema.recordOf(UUID.randomUUID().toString().replace("-", ""), fields); + return Schema.recordOf("Schema" + UUID.randomUUID().toString().replace("-", ""), fields); } public static Schema getRecurrenceSchema() { @@ -364,26 +594,25 @@ public static Schema getRecurrenceSchema() { fields.add(Schema.Field.of("intervalType", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); fields.add(Schema.Field.of("interval", Schema.nullableOf(Schema.of(Schema.Type.INT)))); fields.add(Schema.Field.of("weekdayOnly", Schema.nullableOf(Schema.of(Schema.Type.BOOLEAN)))); - fields.add(Schema.Field.of("weekdayMask", - Schema.nullableOf(Schema.arrayOf(Schema.of(Schema.Type.STRING))))); + fields.add(Schema.Field.of("weekdayMask", Schema.nullableOf(Schema.arrayOf(Schema.of(Schema.Type.STRING))))); fields.add(Schema.Field.of("dayOfMonth", Schema.nullableOf(Schema.of(Schema.Type.INT)))); fields.add(Schema.Field.of("dayOfWeek", Schema.nullableOf(Schema.of(Schema.Type.INT)))); fields.add(Schema.Field.of("weekOfMonth", Schema.nullableOf(Schema.of(Schema.Type.INT)))); - return Schema.recordOf(UUID.randomUUID().toString().replace("-", ""), fields); + return Schema.recordOf("Schema" + UUID.randomUUID().toString().replace("-", ""), fields); } public static Schema getSegmentationSchema() { List fields = new ArrayList<>(); fields.add(Schema.Field.of("createdAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); fields.add(Schema.Field.of("description", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); - fields.add(Schema.Field.of("folder", Schema.nullableOf(EntitySchemaHelper.getFolderDescriptorSchema()))); + fields.add(Schema.Field.of("folder", Schema.nullableOf(EntityHelper.getFolderDescriptorSchema()))); fields.add(Schema.Field.of("id", Schema.nullableOf(Schema.of(Schema.Type.INT)))); fields.add(Schema.Field.of("name", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); fields.add(Schema.Field.of("status", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); fields.add(Schema.Field.of("updatedAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); fields.add(Schema.Field.of("url", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); fields.add(Schema.Field.of("workspace", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); - return Schema.recordOf(UUID.randomUUID().toString().replace("-", ""), fields); + return Schema.recordOf("Schema" + UUID.randomUUID().toString().replace("-", ""), fields); } public static Schema getSmartCampaignSchema() { @@ -395,7 +624,7 @@ public static Schema getSmartCampaignSchema() { fields.add(Schema.Field.of("isSystem", Schema.nullableOf(Schema.of(Schema.Type.BOOLEAN)))); fields.add(Schema.Field.of("isActive", Schema.nullableOf(Schema.of(Schema.Type.BOOLEAN)))); fields.add(Schema.Field.of("isRequestable", Schema.nullableOf(Schema.of(Schema.Type.BOOLEAN)))); - fields.add(Schema.Field.of("recurrence", Schema.nullableOf(EntitySchemaHelper.getRecurrenceSchema()))); + fields.add(Schema.Field.of("recurrence", Schema.nullableOf(EntityHelper.getRecurrenceSchema()))); fields.add(Schema.Field.of("qualificationRuleType", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); fields.add(Schema.Field.of("qualificationRuleInterval", Schema.nullableOf(Schema.of(Schema.Type.INT)))); fields.add(Schema.Field.of("qualificationRuleUnit", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); @@ -404,12 +633,12 @@ public static Schema getSmartCampaignSchema() { Schema.nullableOf(Schema.of(Schema.Type.BOOLEAN)))); fields.add(Schema.Field.of("smartListId", Schema.nullableOf(Schema.of(Schema.Type.INT)))); fields.add(Schema.Field.of("flowId", Schema.nullableOf(Schema.of(Schema.Type.INT)))); - fields.add(Schema.Field.of("folder", Schema.nullableOf(EntitySchemaHelper.getFolderDescriptorSchema()))); + fields.add(Schema.Field.of("folder", Schema.nullableOf(EntityHelper.getFolderDescriptorSchema()))); fields.add(Schema.Field.of("createdAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); fields.add(Schema.Field.of("updatedAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); fields.add(Schema.Field.of("workspace", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); fields.add(Schema.Field.of("status", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); - return Schema.recordOf(UUID.randomUUID().toString().replace("-", ""), fields); + return Schema.recordOf("Schema" + UUID.randomUUID().toString().replace("-", ""), fields); } public static Schema getSmartListSchema() { @@ -420,23 +649,23 @@ public static Schema getSmartListSchema() { fields.add(Schema.Field.of("createdAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); fields.add(Schema.Field.of("updatedAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); fields.add(Schema.Field.of("url", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); - fields.add(Schema.Field.of("folder", Schema.nullableOf(EntitySchemaHelper.getFolderDescriptorSchema()))); + fields.add(Schema.Field.of("folder", Schema.nullableOf(EntityHelper.getFolderDescriptorSchema()))); fields.add(Schema.Field.of("workspace", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); - return Schema.recordOf(UUID.randomUUID().toString().replace("-", ""), fields); + return Schema.recordOf("Schema" + UUID.randomUUID().toString().replace("-", ""), fields); } public static Schema getSnippetSchema() { List fields = new ArrayList<>(); fields.add(Schema.Field.of("createdAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); fields.add(Schema.Field.of("description", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); - fields.add(Schema.Field.of("folder", Schema.nullableOf(EntitySchemaHelper.getFolderDescriptorSchema()))); + fields.add(Schema.Field.of("folder", Schema.nullableOf(EntityHelper.getFolderDescriptorSchema()))); fields.add(Schema.Field.of("id", Schema.nullableOf(Schema.of(Schema.Type.INT)))); fields.add(Schema.Field.of("name", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); fields.add(Schema.Field.of("status", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); fields.add(Schema.Field.of("updatedAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); fields.add(Schema.Field.of("url", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); fields.add(Schema.Field.of("workspace", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); - return Schema.recordOf(UUID.randomUUID().toString().replace("-", ""), fields); + return Schema.recordOf("Schema" + UUID.randomUUID().toString().replace("-", ""), fields); } public static Schema getStaticListSchema() { @@ -447,10 +676,10 @@ public static Schema getStaticListSchema() { fields.add(Schema.Field.of("createdAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); fields.add(Schema.Field.of("updatedAt", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); fields.add(Schema.Field.of("url", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); - fields.add(Schema.Field.of("folder", Schema.nullableOf(EntitySchemaHelper.getFolderDescriptorSchema()))); + fields.add(Schema.Field.of("folder", Schema.nullableOf(EntityHelper.getFolderDescriptorSchema()))); fields.add(Schema.Field.of("workspace", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); fields.add(Schema.Field.of("computedUrl", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); - return Schema.recordOf(UUID.randomUUID().toString().replace("-", ""), fields); + return Schema.recordOf("Schema" + UUID.randomUUID().toString().replace("-", ""), fields); } public static Schema getTagSchema() { @@ -458,7 +687,7 @@ public static Schema getTagSchema() { fields.add(Schema.Field.of("applicableProgramTypes", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); fields.add(Schema.Field.of("required", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); fields.add(Schema.Field.of("tagType", Schema.nullableOf(Schema.of(Schema.Type.STRING)))); - return Schema.recordOf(UUID.randomUUID().toString().replace("-", ""), fields); + return Schema.recordOf("Schema" + UUID.randomUUID().toString().replace("-", ""), fields); } public static StructuredRecord structuredRecordFromEntity(String entityName, Object entity, @@ -472,15 +701,15 @@ public static StructuredRecord structuredRecordFromEntity(String entityName, Obj } builder.set("createdAt", email.getCreatedAt()); builder.set("description", email.getDescription()); - builder.set("folder", EntitySchemaHelper.structuredRecordFromEntity( + builder.set("folder", EntityHelper.structuredRecordFromEntity( "FolderDescriptor", email.getFolder(), schema.getField("folder").getSchema().getNonNullable())); - builder.set("fromEmail", EntitySchemaHelper.structuredRecordFromEntity( + builder.set("fromEmail", EntityHelper.structuredRecordFromEntity( "EmailField", email.getFromEmail(), schema.getField("fromEmail").getSchema().getNonNullable())); - builder.set("fromName", EntitySchemaHelper.structuredRecordFromEntity( + builder.set("fromName", EntityHelper.structuredRecordFromEntity( "EmailField", email.getFromName(), schema.getField("fromName").getSchema().getNonNullable())); @@ -488,12 +717,12 @@ public static StructuredRecord structuredRecordFromEntity(String entityName, Obj builder.set("name", email.getName()); builder.set("operational", email.getOperational()); builder.set("publishToMSI", email.getPublishToMSI()); - builder.set("replyEmail", EntitySchemaHelper.structuredRecordFromEntity( + builder.set("replyEmail", EntityHelper.structuredRecordFromEntity( "EmailField", email.getReplyEmail(), schema.getField("replyEmail").getSchema().getNonNullable())); builder.set("status", email.getStatus()); - builder.set("subject", EntitySchemaHelper.structuredRecordFromEntity( + builder.set("subject", EntityHelper.structuredRecordFromEntity( "EmailField", email.getSubject(), schema.getField("subject").getSchema().getNonNullable())); @@ -508,7 +737,7 @@ public static StructuredRecord structuredRecordFromEntity(String entityName, Obj builder.set( "ccFields", email.getCcFields().stream() - .map(ent -> EntitySchemaHelper.structuredRecordFromEntity( + .map(ent -> EntityHelper.structuredRecordFromEntity( "EmailCCField", ent, schema.getField("ccFields").getSchema().getNonNullable().getComponentSchema())) @@ -543,7 +772,7 @@ public static StructuredRecord structuredRecordFromEntity(String entityName, Obj } builder.set("createdAt", emailtemplate.getCreatedAt()); builder.set("description", emailtemplate.getDescription()); - builder.set("folder", EntitySchemaHelper.structuredRecordFromEntity( + builder.set("folder", EntityHelper.structuredRecordFromEntity( "FolderDescriptor", emailtemplate.getFolder(), schema.getField("folder").getSchema().getNonNullable())); @@ -563,7 +792,7 @@ public static StructuredRecord structuredRecordFromEntity(String entityName, Obj } builder.set("createdAt", file.getCreatedAt()); builder.set("description", file.getDescription()); - builder.set("folder", EntitySchemaHelper.structuredRecordFromEntity( + builder.set("folder", EntityHelper.structuredRecordFromEntity( "FileFolder", file.getFolder(), schema.getField("folder").getSchema().getNonNullable())); @@ -593,7 +822,7 @@ public static StructuredRecord structuredRecordFromEntity(String entityName, Obj builder.set("accessZoneId", folder.getAccessZoneId()); builder.set("createdAt", folder.getCreatedAt()); builder.set("description", folder.getDescription()); - builder.set("folderId", EntitySchemaHelper.structuredRecordFromEntity( + builder.set("folderId", EntityHelper.structuredRecordFromEntity( "FolderDescriptor", folder.getFolderId(), schema.getField("folderId").getSchema().getNonNullable())); @@ -602,7 +831,7 @@ public static StructuredRecord structuredRecordFromEntity(String entityName, Obj builder.set("isArchive", folder.getArchive()); builder.set("isSystem", folder.getSystem()); builder.set("name", folder.getName()); - builder.set("parent", EntitySchemaHelper.structuredRecordFromEntity( + builder.set("parent", EntityHelper.structuredRecordFromEntity( "FolderDescriptor", folder.getParent(), schema.getField("parent").getSchema().getNonNullable())); @@ -631,14 +860,14 @@ public static StructuredRecord structuredRecordFromEntity(String entityName, Obj builder.set("buttonLocation", form.getButtonLocation()); builder.set("createdAt", form.getCreatedAt()); builder.set("description", form.getDescription()); - builder.set("folder", EntitySchemaHelper.structuredRecordFromEntity( + builder.set("folder", EntityHelper.structuredRecordFromEntity( "FolderDescriptor", form.getFolder(), schema.getField("folder").getSchema().getNonNullable())); builder.set("fontFamily", form.getFontFamily()); builder.set("fontSize", form.getFontSize()); builder.set("id", form.getId()); - builder.set("knownVisitor", EntitySchemaHelper.structuredRecordFromEntity( + builder.set("knownVisitor", EntityHelper.structuredRecordFromEntity( "FormKnownVisitorDTO", form.getKnownVisitor(), schema.getField("knownVisitor").getSchema().getNonNullable())); @@ -651,7 +880,7 @@ public static StructuredRecord structuredRecordFromEntity(String entityName, Obj builder.set( "thankYouList", form.getThankYouList().stream() - .map(ent -> EntitySchemaHelper.structuredRecordFromEntity( + .map(ent -> EntityHelper.structuredRecordFromEntity( "FormThankYouPageDTO", ent, schema.getField("thankYouList").getSchema().getNonNullable().getComponentSchema())) @@ -721,7 +950,7 @@ public static StructuredRecord structuredRecordFromEntity(String entityName, Obj builder.set("customHeadHTML", landingpage.getCustomHeadHTML()); builder.set("description", landingpage.getDescription()); builder.set("facebookOgTags", landingpage.getFacebookOgTags()); - builder.set("folder", EntitySchemaHelper.structuredRecordFromEntity( + builder.set("folder", EntityHelper.structuredRecordFromEntity( "FolderDescriptor", landingpage.getFolder(), schema.getField("folder").getSchema().getNonNullable())); @@ -746,7 +975,7 @@ public static StructuredRecord structuredRecordFromEntity(String entityName, Obj builder.set("createdAt", landingpagetemplate.getCreatedAt()); builder.set("description", landingpagetemplate.getDescription()); builder.set("enableMunchkin", landingpagetemplate.getEnableMunchkin()); - builder.set("folder", EntitySchemaHelper.structuredRecordFromEntity( + builder.set("folder", EntityHelper.structuredRecordFromEntity( "FolderDescriptor", landingpagetemplate.getFolder(), schema.getField("folder").getSchema().getNonNullable())); @@ -767,7 +996,7 @@ public static StructuredRecord structuredRecordFromEntity(String entityName, Obj builder.set("channel", program.getChannel()); builder.set("createdAt", program.getCreatedAt()); builder.set("description", program.getDescription()); - builder.set("folder", EntitySchemaHelper.structuredRecordFromEntity( + builder.set("folder", EntityHelper.structuredRecordFromEntity( "FolderDescriptor", program.getFolder(), schema.getField("folder").getSchema().getNonNullable())); @@ -805,7 +1034,7 @@ public static StructuredRecord structuredRecordFromEntity(String entityName, Obj } builder.set("createdAt", segmentation.getCreatedAt()); builder.set("description", segmentation.getDescription()); - builder.set("folder", EntitySchemaHelper.structuredRecordFromEntity( + builder.set("folder", EntityHelper.structuredRecordFromEntity( "FolderDescriptor", segmentation.getFolder(), schema.getField("folder").getSchema().getNonNullable())); @@ -829,7 +1058,7 @@ public static StructuredRecord structuredRecordFromEntity(String entityName, Obj builder.set("isSystem", smartcampaign.getSystem()); builder.set("isActive", smartcampaign.getActive()); builder.set("isRequestable", smartcampaign.getRequestable()); - builder.set("recurrence", EntitySchemaHelper.structuredRecordFromEntity( + builder.set("recurrence", EntityHelper.structuredRecordFromEntity( "Recurrence", smartcampaign.getRecurrence(), schema.getField("recurrence").getSchema().getNonNullable())); @@ -840,7 +1069,7 @@ public static StructuredRecord structuredRecordFromEntity(String entityName, Obj builder.set("isCommunicationLimitEnabled", smartcampaign.getCommunicationLimitEnabled()); builder.set("smartListId", smartcampaign.getSmartListId()); builder.set("flowId", smartcampaign.getFlowId()); - builder.set("folder", EntitySchemaHelper.structuredRecordFromEntity( + builder.set("folder", EntityHelper.structuredRecordFromEntity( "FolderDescriptor", smartcampaign.getFolder(), schema.getField("folder").getSchema().getNonNullable())); @@ -861,7 +1090,7 @@ public static StructuredRecord structuredRecordFromEntity(String entityName, Obj builder.set("createdAt", smartlist.getCreatedAt()); builder.set("updatedAt", smartlist.getUpdatedAt()); builder.set("url", smartlist.getUrl()); - builder.set("folder", EntitySchemaHelper.structuredRecordFromEntity( + builder.set("folder", EntityHelper.structuredRecordFromEntity( "FolderDescriptor", smartlist.getFolder(), schema.getField("folder").getSchema().getNonNullable())); @@ -875,7 +1104,7 @@ public static StructuredRecord structuredRecordFromEntity(String entityName, Obj } builder.set("createdAt", snippet.getCreatedAt()); builder.set("description", snippet.getDescription()); - builder.set("folder", EntitySchemaHelper.structuredRecordFromEntity( + builder.set("folder", EntityHelper.structuredRecordFromEntity( "FolderDescriptor", snippet.getFolder(), schema.getField("folder").getSchema().getNonNullable())); @@ -898,7 +1127,7 @@ public static StructuredRecord structuredRecordFromEntity(String entityName, Obj builder.set("createdAt", staticlist.getCreatedAt()); builder.set("updatedAt", staticlist.getUpdatedAt()); builder.set("url", staticlist.getUrl()); - builder.set("folder", EntitySchemaHelper.structuredRecordFromEntity( + builder.set("folder", EntityHelper.structuredRecordFromEntity( "FolderDescriptor", staticlist.getFolder(), schema.getField("folder").getSchema().getNonNullable())); @@ -924,23 +1153,37 @@ public static StructuredRecord structuredRecordFromEntity(String entityName, Obj } /** - * Valid entity types. + * Entity type. */ public enum EntityType { EMAIL("Email"), + EMAILTEMPLATE("EmailTemplate"), + FILE("File"), + FOLDER("Folder"), + FORM("Form"), + FORMFIELD("FormField"), + LANDINGPAGE("LandingPage"), + LANDINGPAGETEMPLATE("LandingPageTemplate"), + PROGRAM("Program"), + SEGMENTATION("Segmentation"), + SMARTCAMPAIGN("SmartCampaign"), + SMARTLIST("SmartList"), + SNIPPET("Snippet"), + STATICLIST("StaticList"), + TAG("Tag"); private final String value; @@ -949,6 +1192,10 @@ public enum EntityType { this.value = value; } + public String getValue() { + return value; + } + public static EntityType fromString(String value) { for (EntityType entityType : EntityType.values()) { if (entityType.value.equals(value)) { diff --git a/src/main/java/io/cdap/plugin/marketo/source/batch/entity/MarketoEntityFormatProvider.java b/src/main/java/io/cdap/plugin/marketo/source/batch/entity/MarketoEntityFormatProvider.java new file mode 100644 index 0000000..0b1ba00 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/source/batch/entity/MarketoEntityFormatProvider.java @@ -0,0 +1,51 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.source.batch.entity; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import io.cdap.cdap.api.data.batch.InputFormatProvider; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +/** + * InputFormatProvider used by cdap to provide configurations to mapreduce job + */ +public class MarketoEntityFormatProvider implements InputFormatProvider { + public static final String PROPERTY_CONFIG_JSON = "cdap.marketo.entity.config"; + private static final Gson gson = new GsonBuilder().create(); + private final Map conf; + + + MarketoEntityFormatProvider(MarketoEntitySourceConfig config) { + this.conf = Collections.unmodifiableMap(new HashMap() {{ + put(PROPERTY_CONFIG_JSON, gson.toJson(config)); + }}); + } + + @Override + public String getInputFormatClassName() { + return MarketoEntityInputFormat.class.getName(); + } + + @Override + public Map getInputFormatConfiguration() { + return conf; + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/source/batch/entity/MarketoEntityInputFormat.java b/src/main/java/io/cdap/plugin/marketo/source/batch/entity/MarketoEntityInputFormat.java new file mode 100644 index 0000000..6f67747 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/source/batch/entity/MarketoEntityInputFormat.java @@ -0,0 +1,62 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.source.batch.entity; + +import com.google.common.collect.ImmutableList; +import com.google.gson.Gson; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.mapreduce.InputFormat; +import org.apache.hadoop.mapreduce.InputSplit; +import org.apache.hadoop.mapreduce.JobContext; +import org.apache.hadoop.mapreduce.RecordReader; +import org.apache.hadoop.mapreduce.TaskAttemptContext; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +/** + * InputFormat for mapreduce job. + */ +public class MarketoEntityInputFormat extends InputFormat { + private static final Gson GSON = new Gson(); + private static final Logger LOG = LoggerFactory.getLogger(MarketoEntityInputFormat.class); + + @Override + public List getSplits(JobContext jobContext) { + Configuration conf = jobContext.getConfiguration(); + MarketoEntitySourceConfig config = GSON.fromJson( + conf.get(MarketoEntityFormatProvider.PROPERTY_CONFIG_JSON), MarketoEntitySourceConfig.class); + + if (EntityHelper.supportPaging(config.getEntityType())) { + return IntStream.range(0, config.getSplitsCount()) + .mapToObj(splitId -> new MarketoEntitySplit(splitId, config.getResultsPerPage(), config.getSplitsCount())) + .collect(Collectors.toList()); + } else { + LOG.info("Entity '{}' does not support paging, single split used", config.getEntityType().getValue()); + return ImmutableList.of(new MarketoEntitySplit(0, 200, 1)); + } + } + + @Override + public RecordReader createRecordReader(InputSplit inputSplit, TaskAttemptContext taskAttemptContext) { + MarketoEntitySplit split = (MarketoEntitySplit) inputSplit; + return new MarketoEntityRecordReader(split.getSplitId(), split.getResultsPerPage(), split.getTotalSplits()); + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/source/batch/entity/MarketoEntityPlugin.java b/src/main/java/io/cdap/plugin/marketo/source/batch/entity/MarketoEntityPlugin.java new file mode 100644 index 0000000..6cb2e10 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/source/batch/entity/MarketoEntityPlugin.java @@ -0,0 +1,80 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.source.batch.entity; + +import io.cdap.cdap.api.annotation.Description; +import io.cdap.cdap.api.annotation.Name; +import io.cdap.cdap.api.annotation.Plugin; +import io.cdap.cdap.api.data.batch.Input; +import io.cdap.cdap.api.data.format.StructuredRecord; +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.cdap.api.dataset.lib.KeyValue; +import io.cdap.cdap.etl.api.Emitter; +import io.cdap.cdap.etl.api.FailureCollector; +import io.cdap.cdap.etl.api.PipelineConfigurer; +import io.cdap.cdap.etl.api.batch.BatchSource; +import io.cdap.cdap.etl.api.batch.BatchSourceContext; +import io.cdap.plugin.common.LineageRecorder; +import org.apache.hadoop.io.NullWritable; + +import java.util.Objects; +import java.util.stream.Collectors; + +/** + * Plugin that reads entities from Marketo api. + */ +@Plugin(type = BatchSource.PLUGIN_TYPE) +@Name(MarketoEntityPlugin.NAME) +@Description("Reads Entities from Marketo.") +public class MarketoEntityPlugin extends BatchSource { + public static final String NAME = "MarketoEntityPlugin"; + + private final MarketoEntitySourceConfig config; + + public MarketoEntityPlugin(MarketoEntitySourceConfig config) { + this.config = config; + } + + @Override + public void configurePipeline(PipelineConfigurer pipelineConfigurer) { + validateConfiguration(pipelineConfigurer.getStageConfigurer().getFailureCollector()); + pipelineConfigurer.getStageConfigurer().setOutputSchema(config.getSchema()); + } + + @Override + public void prepareRun(BatchSourceContext batchSourceContext) { + validateConfiguration(batchSourceContext.getFailureCollector()); + LineageRecorder lineageRecorder = new LineageRecorder(batchSourceContext, config.referenceName); + lineageRecorder.createExternalDataset(config.getSchema()); + lineageRecorder.recordRead("Read", "Reading Marketo entities", + Objects.requireNonNull(config.getSchema().getFields()).stream() + .map(Schema.Field::getName) + .collect(Collectors.toList())); + + batchSourceContext.setInput(Input.of(config.referenceName, new MarketoEntityFormatProvider(config))); + } + + @Override + public void transform(KeyValue input, Emitter emitter) { + emitter.emit(input.getValue()); + } + + private void validateConfiguration(FailureCollector failureCollector) { + config.validate(failureCollector); + failureCollector.getOrThrowException(); + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/source/batch/entity/MarketoEntityRecordReader.java b/src/main/java/io/cdap/plugin/marketo/source/batch/entity/MarketoEntityRecordReader.java new file mode 100644 index 0000000..5bd98bb --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/source/batch/entity/MarketoEntityRecordReader.java @@ -0,0 +1,119 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.source.batch.entity; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import io.cdap.cdap.api.data.format.StructuredRecord; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.io.NullWritable; +import org.apache.hadoop.mapreduce.InputSplit; +import org.apache.hadoop.mapreduce.RecordReader; +import org.apache.hadoop.mapreduce.TaskAttemptContext; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.Iterator; + +/** + * RecordReader implementation, which reads events from Marketo api. + */ +public class MarketoEntityRecordReader extends RecordReader { + private static final Logger LOG = LoggerFactory.getLogger(MarketoEntityRecordReader.class); + private static final Gson GSON = new GsonBuilder().create(); + private Iterator iterator = null; + private Object current = null; + private MarketoEntitySourceConfig config; + private int splitId; + private int resultsPerPage; + private int totalSplits; + private int currentOffset = -1; + private boolean needSwapPage = false; + private boolean supportPaging = true; + + public MarketoEntityRecordReader(int splitId, int resultsPerPage, int totalSplits) { + this.splitId = splitId; + this.resultsPerPage = resultsPerPage; + this.totalSplits = totalSplits; + this.currentOffset = splitId * resultsPerPage; + } + + @Override + public void initialize(InputSplit inputSplit, TaskAttemptContext taskAttemptContext) throws IOException { + deserializeConfiguration(taskAttemptContext.getConfiguration()); + iterator = EntityHelper.iteratorForEntityType(config.getMarketo(), config.getEntityType(), + currentOffset, resultsPerPage); + LOG.info("Split '{}' fetched data by offset '{}'", splitId, currentOffset); + } + + @Override + public boolean nextKeyValue() { + if (iterator.hasNext()) { + current = iterator.next(); + // this page is not empty, try next page as well + needSwapPage = true; + return true; + } else if (swapPage()) { + // page swapped, try to get new item + return nextKeyValue(); + } else { + return false; + } + } + + private boolean swapPage() { + if (needSwapPage && supportPaging) { + needSwapPage = false; + currentOffset += totalSplits * resultsPerPage; + iterator = EntityHelper.iteratorForEntityType(config.getMarketo(), config.getEntityType(), currentOffset, + resultsPerPage); + LOG.info("Split '{}' fetched data by offset '{}'", splitId, currentOffset); + return true; + } + return false; + } + + @Override + public NullWritable getCurrentKey() { + return null; + } + + @Override + public StructuredRecord getCurrentValue() { + return EntityHelper.structuredRecordFromEntity( + config.getEntityType().getValue(), + current, + config.getSchema() + ); + } + + @Override + public float getProgress() { + return 0; + } + + @Override + public void close() { + } + + private void deserializeConfiguration(Configuration conf) { + String configJson = conf.get(MarketoEntityFormatProvider.PROPERTY_CONFIG_JSON); + config = GSON.fromJson(configJson, MarketoEntitySourceConfig.class); + supportPaging = EntityHelper.supportPaging(config.getEntityType()); + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/source/batch/entity/MarketoEntitySourceConfig.java b/src/main/java/io/cdap/plugin/marketo/source/batch/entity/MarketoEntitySourceConfig.java new file mode 100644 index 0000000..5e7a353 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/source/batch/entity/MarketoEntitySourceConfig.java @@ -0,0 +1,160 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.source.batch.entity; + +import com.google.common.base.Strings; +import io.cdap.cdap.api.annotation.Description; +import io.cdap.cdap.api.annotation.Macro; +import io.cdap.cdap.api.annotation.Name; +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.cdap.etl.api.FailureCollector; +import io.cdap.plugin.common.IdUtils; +import io.cdap.plugin.common.ReferencePluginConfig; +import io.cdap.plugin.marketo.common.api.Marketo; + +import java.net.MalformedURLException; +import java.net.URL; + +/** + * Provides all required configuration for reading Marketo entities. + */ +public class MarketoEntitySourceConfig extends ReferencePluginConfig { + public static final String PROPERTY_CLIENT_ID = "clientId"; + public static final String PROPERTY_CLIENT_SECRET = "clientSecret"; + public static final String PROPERTY_REST_API_ENDPOINT = "restApiEndpoint"; + public static final String PROPERTY_ENTITY_TYPE = "entityType"; + public static final String PROPERTY_SPLITS_COUNT = "splitsCount"; + public static final String PROPERTY_RESULTS_PER_PAGE = "resultsPerPage"; + + @Name(PROPERTY_CLIENT_ID) + @Description("Marketo Client ID.") + @Macro + protected String clientId; + + @Name(PROPERTY_CLIENT_SECRET) + @Description("Marketo Client secret.") + @Macro + protected String clientSecret; + + @Name(PROPERTY_REST_API_ENDPOINT) + @Description("REST API endpoint URL.") + @Macro + protected String restApiEndpoint; + + @Name(PROPERTY_ENTITY_TYPE) + @Description("Type of entity.") + @Macro + protected String entityType; + + @Name(PROPERTY_SPLITS_COUNT) + @Description("Number of splits to fetch data.") + @Macro + protected int splitsCount; + + @Name(PROPERTY_RESULTS_PER_PAGE) + @Description("Max results per page.") + @Macro + protected int resultsPerPage; + + private transient Schema schema = null; + private transient Marketo marketo = null; + + public MarketoEntitySourceConfig(String referenceName) { + super(referenceName); + } + + public EntityHelper.EntityType getEntityType() { + return EntityHelper.EntityType.fromString(entityType); + } + + public Schema getSchema() { + if (schema == null) { + schema = EntityHelper.schemaForEntityName(getEntityType().getValue()); + } + return schema; + } + + public Marketo getMarketo() { + if (marketo == null) { + marketo = new Marketo(getRestApiEndpoint(), getClientId(), getClientSecret()); + } + return marketo; + } + + public String getClientId() { + return clientId; + } + + public String getClientSecret() { + return clientSecret; + } + + public String getRestApiEndpoint() { + return restApiEndpoint; + } + + public int getSplitsCount() { + return splitsCount; + } + + public int getResultsPerPage() { + return resultsPerPage; + } + + void validate(FailureCollector failureCollector) { + IdUtils.validateReferenceName(referenceName, failureCollector); + validateEntityType(failureCollector); + validateMarketoEndpoint(failureCollector); + validateSecrets(failureCollector); + } + + void validateEntityType(FailureCollector failureCollector) { + if (!containsMacro(PROPERTY_ENTITY_TYPE)) { + try { + getEntityType(); + } catch (IllegalArgumentException ex) { + failureCollector.addFailure(String.format("Incorrect entity type '%s'.", entityType), + "Set entity type to valid.") + .withConfigProperty(PROPERTY_ENTITY_TYPE); + } + } + } + + void validateSecrets(FailureCollector failureCollector) { + if (!containsMacro(PROPERTY_CLIENT_ID) && Strings.isNullOrEmpty(getClientId())) { + failureCollector.addFailure("Client ID is empty.", null) + .withConfigProperty(PROPERTY_CLIENT_ID); + } + + if (!containsMacro(PROPERTY_CLIENT_SECRET) && Strings.isNullOrEmpty(getClientSecret())) { + failureCollector.addFailure("Client Secret is empty.", null) + .withConfigProperty(PROPERTY_CLIENT_SECRET); + } + } + + void validateMarketoEndpoint(FailureCollector failureCollector) { + if (!containsMacro(PROPERTY_REST_API_ENDPOINT)) { + try { + new URL(getRestApiEndpoint()); + } catch (MalformedURLException e) { + failureCollector + .addFailure(String.format("Malformed Marketo Rest API endpoint URL '%s'.", getRestApiEndpoint()), null) + .withConfigProperty(PROPERTY_REST_API_ENDPOINT); + } + } + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/source/batch/entity/MarketoEntitySplit.java b/src/main/java/io/cdap/plugin/marketo/source/batch/entity/MarketoEntitySplit.java new file mode 100644 index 0000000..044465d --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/source/batch/entity/MarketoEntitySplit.java @@ -0,0 +1,78 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.source.batch.entity; + +import org.apache.hadoop.io.Writable; +import org.apache.hadoop.mapreduce.InputSplit; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; + +/** + * A no-op split. + */ +public class MarketoEntitySplit extends InputSplit implements Writable { + private int splitId; + private int resultsPerPage; + private int totalSplits; + + public MarketoEntitySplit() { + } + + public MarketoEntitySplit(int splitId, int resultsPerPage, int totalSplits) { + this.splitId = splitId; + this.resultsPerPage = resultsPerPage; + this.totalSplits = totalSplits; + } + + @Override + public void readFields(DataInput dataInput) throws IOException { + splitId = dataInput.readInt(); + resultsPerPage = dataInput.readInt(); + totalSplits = dataInput.readInt(); + } + + @Override + public void write(DataOutput dataOutput) throws IOException { + dataOutput.writeInt(splitId); + dataOutput.writeInt(resultsPerPage); + dataOutput.writeInt(totalSplits); + } + + @Override + public long getLength() { + return 0; + } + + @Override + public String[] getLocations() { + return new String[0]; + } + + public int getSplitId() { + return splitId; + } + + public int getResultsPerPage() { + return resultsPerPage; + } + + public int getTotalSplits() { + return totalSplits; + } +} diff --git a/src/test/java/io/cdap/plugin/marketo/source/batch/entity/EntitySchemaHelperTest.java b/src/test/java/io/cdap/plugin/marketo/source/batch/entity/EntityHelperTest.java similarity index 81% rename from src/test/java/io/cdap/plugin/marketo/source/batch/entity/EntitySchemaHelperTest.java rename to src/test/java/io/cdap/plugin/marketo/source/batch/entity/EntityHelperTest.java index cc29097..16205fb 100644 --- a/src/test/java/io/cdap/plugin/marketo/source/batch/entity/EntitySchemaHelperTest.java +++ b/src/test/java/io/cdap/plugin/marketo/source/batch/entity/EntityHelperTest.java @@ -28,24 +28,24 @@ import java.util.List; -public class EntitySchemaHelperTest { +public class EntityHelperTest { @Test public void testNestedRecords() { - Schema programSchema = EntitySchemaHelper.getProgramSchema(); + Schema programSchema = EntityHelper.getProgramSchema(); Program program = Program.builder().folder(new FolderDescriptor("1234", "Hello", null)) .build(); - StructuredRecord programRecord = EntitySchemaHelper.structuredRecordFromEntity("Program", program, - programSchema); + StructuredRecord programRecord = EntityHelper.structuredRecordFromEntity("Program", program, + programSchema); Assert.assertEquals(programRecord.get("folder").get("id"), "1234"); - Schema emailSchema = EntitySchemaHelper.getEmailSchema(); + Schema emailSchema = EntityHelper.getEmailSchema(); Email email = Email.builder().ccFields(ImmutableList.of( new EmailCCField("attr1", "cc1", "cc 1", "cc1"), new EmailCCField("attr2", "cc2", "cc 2", "cc2") )).build(); - StructuredRecord emailRecord = EntitySchemaHelper.structuredRecordFromEntity("Email", email, - emailSchema); + StructuredRecord emailRecord = EntityHelper.structuredRecordFromEntity("Email", email, + emailSchema); List ccRecords = emailRecord.>get("ccFields"); Assert.assertEquals(2, ccRecords.size()); diff --git a/widgets/MarketoEntityPlugin-batchsource.json b/widgets/MarketoEntityPlugin-batchsource.json new file mode 100644 index 0000000..b7e4940 --- /dev/null +++ b/widgets/MarketoEntityPlugin-batchsource.json @@ -0,0 +1,94 @@ +{ + "metadata": { + "spec-version": "1.0" + }, + "display-name": "Marketo Entity", + "configuration-groups": [ + { + "label": "General", + "properties": [ + { + "widget-type": "textbox", + "label": "Reference Name", + "name": "referenceName" + }, + { + "widget-type": "textbox", + "label": "Rest API endpoint", + "name": "restApiEndpoint" + }, + { + "widget-type": "select", + "label": "Entity Type", + "name": "entityType", + "widget-attributes": { + "default": "File", + "values": [ + "Email", + "EmailTemplate", + "File", + "Folder", + "Form", + "FormField", + "LandingPage", + "LandingPageTemplate", + "Program", + "Segmentation", + "SmartCampaign", + "SmartList", + "Snippet", + "StaticList", + "Tag" + ] + } + } + ] + }, + { + "label": "Authentication", + "properties": [ + { + "widget-type": "textbox", + "label": "Client ID", + "name": "clientId" + }, + { + "widget-type": "password", + "label": "Client Secret", + "name": "clientSecret" + } + ] + }, + { + "label": "Advanced", + "properties": [ + { + "widget-type": "number", + "label": "Splits Count", + "name": "splitsCount", + "widget-attributes": { + "default": 2, + "min": 1, + "max": 5 + } + }, + { + "widget-type": "number", + "label": "Max Results Per Page", + "name": "resultsPerPage", + "widget-attributes": { + "default": 200, + "min": 20, + "max": 200 + } + } + ] + } + ], + "outputs": [ + { + "widget-type": "non-editable-schema-editor", + "schema": {} + } + ] +} \ No newline at end of file