diff --git a/precog/java/src/main/java/com/precog/api/Client.java b/precog/java/src/main/java/com/precog/api/Client.java index 0fc2ce4..2d13f76 100644 --- a/precog/java/src/main/java/com/precog/api/Client.java +++ b/precog/java/src/main/java/com/precog/api/Client.java @@ -135,7 +135,7 @@ public String describeAccount(String email, String password, String accountId) t * @param The type of the record object. This type must be serializable to JSON using a ToJson instance * for some supertype of the specified type. * @param path The path at which the record should be placed in the virtual file system. - * @param record The record being storeed. + * @param record The record being stored. * @param serializer The function used to serialize the record to a JSON string. * @throws IOException */ @@ -151,28 +151,6 @@ public void store(Path path, String recordJson) throws IOException { ingest(path, recordJson, options); } - /** - * Builds the async/sync data storage path - * - * @param async boolean, true to do an async storage call - * @param path The path at which the record should be placed in the virtual file system. - * @return full path - */ - public Path buildStoragePath(boolean async, Path path) { - return new Path(async ? "async" : "sync").append(Paths.FS).append(path); - } - - /** - * Builds a sync data storage path - * - * @param path The path at which the record should be placed in the virtual file system. - * @return full path - */ - public Path buildStoragePath(Path path) { - return buildStoragePath(false, path); - } - - /** * Ingest data in the specified path * Ingest behavior is controlled by the ingest options @@ -192,10 +170,10 @@ public String ingest(Path path, String content, IngestOptions options) throws IO throw new IllegalArgumentException("argument 'content' must contain a non empty value formatted as described by type"); } Request request = new Request(); - request.getHeader().putAll(options.asMap()); + request.getParams().putAll(options.asMap()); request.setBody(content); request.setContentType(options.getDataType()); - return rest.request(Rest.Method.POST, actionPath(Services.INGEST, buildStoragePath(options.isAsync(), path)).getPath(), request); + return rest.request(Rest.Method.POST, actionPath(Services.INGEST, (Paths.FS).append(path)).getPath(), request); } /** @@ -207,7 +185,7 @@ public String ingest(Path path, String content, IngestOptions options) throws IO */ public String delete(Path path) throws IOException { Request request = new Request(); - return rest.request(Rest.Method.DELETE, actionPath(Services.INGEST, buildStoragePath(path)).getPath(), request); + return rest.request(Rest.Method.DELETE, actionPath(Services.INGEST, (Paths.FS).append(path)).getPath(), request); } /** @@ -220,7 +198,7 @@ public String delete(Path path) throws IOException { * @throws IOException */ public String query(Path path, String q) throws IOException { - if (!path.getPrefix().equals(Paths.FS)) { + if (!Paths.FS.equals(path.getPrefix())) { path = Paths.FS.append(path); } Request request = new Request(); diff --git a/precog/java/src/main/java/com/precog/api/dto/AccountInfo.java b/precog/java/src/main/java/com/precog/api/dto/AccountInfo.java index fa38f30..fed0e6a 100644 --- a/precog/java/src/main/java/com/precog/api/dto/AccountInfo.java +++ b/precog/java/src/main/java/com/precog/api/dto/AccountInfo.java @@ -11,7 +11,7 @@ public class AccountInfo { private String accountId; private String email; - private int accountCreationDate; + private String accountCreationDate; private String apiKey; private String rootPath; private Map plan; @@ -32,11 +32,11 @@ public void setEmail(String email) { this.email = email; } - public int getAccountCreationDate() { + public String getAccountCreationDate() { return accountCreationDate; } - public void setAccountCreationDate(int accountCreationDate) { + public void setAccountCreationDate(String accountCreationDate) { this.accountCreationDate = accountCreationDate; } diff --git a/precog/java/src/main/java/com/precog/api/options/CSVIngestOptions.java b/precog/java/src/main/java/com/precog/api/options/CSVIngestOptions.java index c9a7aeb..ce7b1f6 100644 --- a/precog/java/src/main/java/com/precog/api/options/CSVIngestOptions.java +++ b/precog/java/src/main/java/com/precog/api/options/CSVIngestOptions.java @@ -15,9 +15,9 @@ public class CSVIngestOptions extends IngestOptions { public static String ESCAPE = "escape"; public static String DELIMITER = "delimiter"; - private String delimiter; - private String quote; - private String escape; + private String delimiter="'"; + private String quote="\""; + private String escape="\\"; public CSVIngestOptions() { super(ContentType.CSV); diff --git a/precog/java/src/main/java/com/precog/api/options/IngestOptions.java b/precog/java/src/main/java/com/precog/api/options/IngestOptions.java index f17ebbb..ab2cf8d 100644 --- a/precog/java/src/main/java/com/precog/api/options/IngestOptions.java +++ b/precog/java/src/main/java/com/precog/api/options/IngestOptions.java @@ -14,9 +14,31 @@ public class IngestOptions { public static String OWNER_ACCOUNT_ID = "ownerAccountId"; + public static String RECEIPT = "receipt"; + public static String MODE="mode"; + public static String BATCH = "batch"; + public static String STREAMING="streaming"; + private ContentType dataType; private String ownerAccountId; - private boolean async; + private boolean batch=true; + private boolean receipt=true; + + public boolean isBatch() { + return batch; + } + + public void setBatch(boolean batch) { + this.batch = batch; + } + + public boolean isReceipt() { + return receipt; + } + + public void setReceipt(boolean receipt) { + this.receipt = receipt; + } public IngestOptions(ContentType dataType) { this.dataType = dataType; @@ -27,6 +49,13 @@ public Map asMap() { if (ownerAccountId != null) { map.put(OWNER_ACCOUNT_ID, ownerAccountId); } + map.put(BATCH,Boolean.toString(batch)); + if(batch){ + map.put(MODE,BATCH); + map.put(RECEIPT,Boolean.toString(receipt)); + } else { + map.put(MODE,STREAMING); + } return map; } @@ -42,11 +71,4 @@ public void setOwnerAccountId(String ownerAccountId) { this.ownerAccountId = ownerAccountId; } - public boolean isAsync() { - return async; - } - - public void setAsync(boolean async) { - this.async = async; - } } diff --git a/precog/java/src/test/java/com/precog/api/ClientTest.java b/precog/java/src/test/java/com/precog/api/ClientTest.java index 6b73c6b..2545fc8 100644 --- a/precog/java/src/test/java/com/precog/api/ClientTest.java +++ b/precog/java/src/test/java/com/precog/api/ClientTest.java @@ -40,7 +40,6 @@ private static class TestData { @SerializedName("~raw") public final RawJson testRaw; - public TestData(int testInt, String testStr, RawJson testRaw) { this.testInt = testInt; this.testStr = testStr; @@ -124,7 +123,7 @@ public void testStoreRawUTF8() throws IOException { public void testIngestCSV() throws IOException { IngestOptions options = new CSVIngestOptions(); - String response = testClient.ingest(testPath, "blah,\n\n", options); + String response = testClient.ingest(testPath, "head1,head2\nblah,bleh\n", options); IngestResult result = GsonFromJson.of(new TypeToken() { }).deserialize(response); assertEquals(1, result.getIngested()); @@ -141,6 +140,27 @@ public void testIngestJSON() throws IOException { assertEquals(1, result.getIngested()); } + @Test + public void testIngestJSONBatchNoReceipt() throws IOException { + + IngestOptions options = new IngestOptions(ContentType.JSON); + options.setReceipt(false); + String rawJson = "{\"test\":[{\"v\": 1}, {\"v\": 2}]}"; + String response = testClient.ingest(testPath, rawJson, options); + assertEquals("{\"content-length\":29}", response); + } + + @Test + public void testIngestJSONStreaming() throws IOException { + IngestOptions options = new IngestOptions(ContentType.JSON); + options.setBatch(false); + String rawJson = "{\"test\":[{\"v\": 1}, {\"v\": 2}]}"; + String response = testClient.ingest(testPath, rawJson, options); + IngestResult result = GsonFromJson.of(new TypeToken() { + }).deserialize(response); + assertEquals(1, result.getIngested()); + } + @Test public void testIngestCsvWithOptions() throws IOException { @@ -148,7 +168,7 @@ public void testIngestCsvWithOptions() throws IOException { options.setDelimiter(","); options.setQuote("'"); options.setEscape("\\"); - String response = testClient.ingest(testPath, "blah\n\n", options); + String response = testClient.ingest(testPath, "head\nblah\n", options); IngestResult result = GsonFromJson.of(new TypeToken() { }).deserialize(response); assertEquals(1, result.getIngested()); @@ -157,11 +177,12 @@ public void testIngestCsvWithOptions() throws IOException { @Test public void testIngestAsync() throws IOException { - IngestOptions options = new CSVIngestOptions(); - options.setAsync(true); - String response = testClient.ingest(testPath, "blah,\n\n", options); + IngestOptions options = new IngestOptions(ContentType.JSON); + String rawJson = "{\"test\":[{\"v\": 1}, {\"v\": 2}]}"; + options.setBatch(false); + String response = testClient.ingest(testPath, rawJson, options); //is async, so we don't expect results - assertEquals("", response); + assertEquals("{\"ingested\":1,\"errors\":[]}", response); } @Test @@ -225,7 +246,6 @@ public void testDescribeAccount() throws IOException { } @Test - @Ignore public void testQuery() throws IOException { //just test the query was sent and executed successfully @@ -249,4 +269,14 @@ public void testFromHeroku() throws UnsupportedEncodingException { assertNotNull(precogApi); } + @Test + public void testQueryLoad() throws IOException { + Path path=testPath.append(new Path("load")); + String rawJson = "{\"test\":[{\"v\": 1}, {\"v\": 2}]}"; + testClient.store(testPath.append(new Path("load")), rawJson); + //just test the query was sent and executed successfully + String result = testClient.query(new Path(testAccountId), "load(\"//"+ path +"\")"); + assertNotNull(result); + } + }