diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 40bd49b..ad0e2c8 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -23,10 +23,10 @@ jobs: steps: - uses: actions/checkout@v3 - - name: Set up JDK 11 + - name: Set up JDK 17 uses: actions/setup-java@v3 with: - java-version: '11' + java-version: '17' distribution: 'temurin' - name: Validate Gradle wrapper diff --git a/Dockerfile b/Dockerfile index b85590b..cc0814d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,13 @@ -FROM mcr.microsoft.com/openjdk/jdk:17-ubuntu as dev -ENV TZ=America/New_York -RUN apt update -RUN apt -y install wget -RUN apt -y install git -#FROM ubuntu:20.04 as prod +FROM --platform=linux/amd64 ubuntu:24.04 + +ENV TZ=America/New_York + +RUN echo 'Acquire::http::Pipeline-Depth 0;\nAcquire::http::No-Cache true; \nAcquire::BrokenProxy true;\n' > /etc/apt/apt.conf.d/99fixbadproxy + +RUN apt update &&\ + apt -y install wget &&\ + apt -y install git &&\ + apt -y install curl &&\ + apt -y install vim &&\ + apt -y install unzip &&\ + apt -y install openjdk-17-jdk diff --git a/build.gradle b/build.gradle index 1a91eab..51a21e7 100644 --- a/build.gradle +++ b/build.gradle @@ -18,8 +18,8 @@ dependencies { implementation 'com.amazonaws:aws-java-sdk-s3' } jar { - sourceCompatibility = JavaVersion.VERSION_11 - targetCompatibility = JavaVersion.VERSION_11 + sourceCompatibility = JavaVersion.VERSION_17 + targetCompatibility = JavaVersion.VERSION_17 } publishing { repositories { @@ -51,5 +51,5 @@ publishing { } group 'mil.army.usace.hec' - version '0.0.56' + version '0.0.59' } diff --git a/build/tmp/compileJava/previous-compilation-data.bin b/build/tmp/compileJava/previous-compilation-data.bin deleted file mode 100644 index 568a150..0000000 Binary files a/build/tmp/compileJava/previous-compilation-data.bin and /dev/null differ diff --git a/build/tmp/jar/MANIFEST.MF b/build/tmp/jar/MANIFEST.MF deleted file mode 100644 index 58630c0..0000000 --- a/build/tmp/jar/MANIFEST.MF +++ /dev/null @@ -1,2 +0,0 @@ -Manifest-Version: 1.0 - diff --git a/src/main/java/usace/cc/plugin/AWSConfig.java b/src/main/java/usace/cc/plugin/AWSConfig.java index f2ecd48..cde21de 100644 --- a/src/main/java/usace/cc/plugin/AWSConfig.java +++ b/src/main/java/usace/cc/plugin/AWSConfig.java @@ -1,22 +1,30 @@ package usace.cc.plugin; + import com.fasterxml.jackson.annotation.JsonProperty; + public class AWSConfig { @JsonProperty public String aws_config_name; + @JsonProperty public String aws_access_key_id; + @JsonProperty public String aws_secret_access_key_id; + @JsonProperty public String aws_region; + @JsonProperty public String aws_bucket; - @JsonProperty - public Boolean aws_mock; + @JsonProperty public String aws_endpoint; + @JsonProperty public Boolean aws_disable_ssl; + @JsonProperty public Boolean aws_force_path_style; + } diff --git a/src/main/java/usace/cc/plugin/Action.java b/src/main/java/usace/cc/plugin/Action.java index a5754fe..6d425be 100644 --- a/src/main/java/usace/cc/plugin/Action.java +++ b/src/main/java/usace/cc/plugin/Action.java @@ -1,36 +1,21 @@ package usace.cc.plugin; -import java.util.Map; - import com.fasterxml.jackson.annotation.JsonProperty; -public class Action { - @JsonProperty - private String name; +public class Action extends IOManager{ + @JsonProperty + private String type; + + @JsonProperty("description") private String desc; - @JsonProperty - private Map params; - public String getName(){ - return name; - } + public String getType(){ + return type; + } + public String getDescription(){ return desc; } - - public Map getParameters(){ - return params; - } - public void UpdateActionPaths(){ - PluginManager pm = PluginManager.getInstance(); - this.name = pm.SubstitutePath(this.name); - this.desc = pm.SubstitutePath(this.desc); - if(params!=null){ - for(Map.Entry apb : params.entrySet()){ - params.replace(apb.getKey(),apb.getValue().UpdatePaths()); - } - } - } } diff --git a/src/main/java/usace/cc/plugin/CcStore.java b/src/main/java/usace/cc/plugin/CcStore.java index 0d76bd7..9ef1b25 100644 --- a/src/main/java/usace/cc/plugin/CcStore.java +++ b/src/main/java/usace/cc/plugin/CcStore.java @@ -1,11 +1,11 @@ package usace.cc.plugin; public interface CcStore { - public boolean PutObject(PutObjectInput input); - public boolean PullObject(PullObjectInput input); - public byte[] GetObject(GetObjectInput input) throws Exception; - public Payload GetPayload() throws Exception; + public boolean putObject(PutObjectInput input); + public boolean pullObject(PullObjectInput input); + public byte[] getObject(GetObjectInput input) throws Exception; + public Payload getPayload() throws Exception; //public void SetPayload(Payload payload); only used in the go sdk to support cloudcompute which is written in go. - public String RootPath(); - public boolean HandlesDataStoreType(StoreType datastoretype); + public String rootPath(); + public boolean handlesDataStoreType(StoreType datastoretype); } diff --git a/src/main/java/usace/cc/plugin/CcStoreS3.java b/src/main/java/usace/cc/plugin/CcStoreS3.java index 54087f7..9c42f56 100644 --- a/src/main/java/usace/cc/plugin/CcStoreS3.java +++ b/src/main/java/usace/cc/plugin/CcStoreS3.java @@ -34,37 +34,27 @@ public class CcStoreS3 implements CcStore { String bucket; String root; String manifestId; + String payloadId; StoreType storeType; AmazonS3 awsS3; AWSConfig config; - @Override - public String RootPath() { - return bucket; - } + public CcStoreS3(){ AWSConfig acfg = new AWSConfig(); acfg.aws_access_key_id = System.getenv(EnvironmentVariables.CC_PROFILE + "_" + EnvironmentVariables.AWS_ACCESS_KEY_ID); acfg.aws_secret_access_key_id = System.getenv(EnvironmentVariables.CC_PROFILE + "_" + EnvironmentVariables.AWS_SECRET_ACCESS_KEY); acfg.aws_region = System.getenv(EnvironmentVariables.CC_PROFILE + "_" + EnvironmentVariables.AWS_DEFAULT_REGION); acfg.aws_bucket = System.getenv(EnvironmentVariables.CC_PROFILE + "_" + EnvironmentVariables.AWS_S3_BUCKET); - acfg.aws_mock = Boolean.parseBoolean(System.getenv(EnvironmentVariables.CC_PROFILE + "_" +"S3_MOCK"));//convert to boolean;//stringformat - acfg.aws_endpoint = System.getenv(EnvironmentVariables.CC_PROFILE + "_" +"S3_ENDPOINT"); - acfg.aws_disable_ssl = Boolean.parseBoolean(System.getenv(EnvironmentVariables.CC_PROFILE + "_" +"S3_DISABLE_SSL"));//convert to bool? - acfg.aws_force_path_style = Boolean.parseBoolean(System.getenv(EnvironmentVariables.CC_PROFILE + "_" +"S3_FORCE_PATH_STYLE"));//convert to bool + acfg.aws_endpoint = System.getenv(EnvironmentVariables.CC_PROFILE + "_" +"AWS_ENDPOINT"); config = acfg; - //System.out.println(EnvironmentVariables.CC_PROFILE + "_" + EnvironmentVariables.AWS_DEFAULT_REGION+"::"+config.aws_region); - //System.out.println(EnvironmentVariables.CC_PROFILE + "_" + EnvironmentVariables.AWS_ACCESS_KEY_ID+"::"+config.aws_access_key_id); - //System.out.println(EnvironmentVariables.CC_PROFILE + "_" + EnvironmentVariables.AWS_SECRET_ACCESS_KEY+"::"+config.aws_secret_access_key_id); - //System.out.println(EnvironmentVariables.CC_PROFILE + "_" + EnvironmentVariables.AWS_S3_BUCKET+"::"+config.aws_bucket); + Region clientRegion = RegionUtils.getRegion(config.aws_region);//.toUpperCase().replace("-", "_"));//Regions.valueOf(config.aws_region.toUpperCase().replace("-", "_")); try { AmazonS3 s3Client = null; - if(config.aws_mock){ - System.out.println("mocking s3 with minio"); - //System.out.println(EnvironmentVariables.CC_PROFILE + "_S3_MOCK::"+config.aws_mock); - //System.out.println(EnvironmentVariables.CC_PROFILE + "_S3_ENDPOINT::"+config.aws_endpoint); - //System.out.println(EnvironmentVariables.CC_PROFILE + "_S3_DISABLE_SSL::"+config.aws_disable_ssl); - //System.out.println(EnvironmentVariables.CC_PROFILE + "_S3_FORCE_PATH_STYLE::"+config.aws_force_path_style); + if(!config.aws_endpoint.equals("")){ + System.out.println(String.format("Using alt endpoint: %s",config.aws_endpoint)); + config.aws_force_path_style=true; + config.aws_disable_ssl=true; AWSCredentials credentials = new BasicAWSCredentials(config.aws_access_key_id, config.aws_secret_access_key_id); ClientConfiguration clientConfiguration = new ClientConfiguration(); clientConfiguration.setSignerOverride("AWSS3V4SignerType"); @@ -97,20 +87,28 @@ public CcStoreS3(){ } storeType = StoreType.S3; manifestId = System.getenv(EnvironmentVariables.CC_MANIFEST_ID); - localRootPath = Constants.LocalRootPath; + payloadId = System.getenv(EnvironmentVariables.CC_PAYLOAD_ID); + localRootPath = Constants.LOCAL_ROOT_PATH; bucket = config.aws_bucket;// + Constants.RemoteRootPath; root = System.getenv(EnvironmentVariables.CC_ROOT); } + @Override - public boolean HandlesDataStoreType(StoreType storeType){ + public String rootPath() { + return bucket; + } + + @Override + public boolean handlesDataStoreType(StoreType storeType){ return this.storeType == storeType; } + @Override - public boolean PutObject(PutObjectInput input) { + public boolean putObject(PutObjectInput input) { String path = root + "/" + manifestId + "/" + input.getFileName() + "." + input.getFileExtension(); byte[] data; switch(input.getObjectState()){ - case LocalDisk: + case LOCAL_DISK: //read from local File file = new File(path); data = new byte[(int) file.length()]; @@ -120,11 +118,11 @@ public boolean PutObject(PutObjectInput input) { catch(Exception e){ //@TODOprint? } - UploadToS3(config.aws_bucket, path, data); + uploadToS3(config.aws_bucket, path, data); break; - case Memory: + case MEMORY: data = input.getData(); - UploadToS3(config.aws_bucket, path, data); + uploadToS3(config.aws_bucket, path, data); break; default: return false; @@ -132,14 +130,15 @@ public boolean PutObject(PutObjectInput input) { return true; } + @Override - public boolean PullObject(PullObjectInput input) { + public boolean pullObject(PullObjectInput input) { String path = root + "/" + manifestId + "/" + input.getFileName() + "." + input.getFileExtension(); byte[] data; String localPath = input.getDestRootPath() + "/" + input.getFileName() + "." + input.getFileExtension(); try { //get the object from s3 - data = DownloadBytesFromS3(path); + data = downloadBytesFromS3(path); //create localpath writer InputStream stream = new ByteArrayInputStream(data); //write it. @@ -149,6 +148,7 @@ public boolean PullObject(PullObjectInput input) { } return false; } + private void writeInputStreamToDisk(InputStream input, String outputDestination) throws IOException { String directory = new File(outputDestination).getParent(); File f = new File(directory); @@ -156,31 +156,37 @@ private void writeInputStreamToDisk(InputStream input, String outputDestination) f.mkdirs(); } byte[] bytes = input.readAllBytes(); - OutputStream os = new FileOutputStream(new File(outputDestination)); - os.write(bytes); + try (OutputStream os = new FileOutputStream(new File(outputDestination))) { + os.write(bytes); + } catch (Exception e) { + e.printStackTrace(); + }; } + @Override - public byte[] GetObject(GetObjectInput input) throws AmazonS3Exception { - String path = root + "/" + manifestId + "/" + input.getFileName() + "." + input.getFileExtension(); + public byte[] getObject(GetObjectInput input) throws AmazonS3Exception { + String path = root + "/" + payloadId + "/" + input.getFileName() + "." + input.getFileExtension(); byte[] data; try { - data = DownloadBytesFromS3(path); + data = downloadBytesFromS3(path); } catch (Exception e) { throw new AmazonS3Exception(e.toString()); } return data; } + @Override - public Payload GetPayload() throws AmazonS3Exception { - String filepath = root + "/" + manifestId + "/" + Constants.PayloadFileName; + public Payload getPayload() throws AmazonS3Exception { + String filepath = root + "/" + payloadId + "/" + Constants.PAYLOAD_FILE_NAME; try{ - byte[] body = DownloadBytesFromS3(filepath); - return ReadJsonModelPayloadFromBytes(body); + byte[] body = downloadBytesFromS3(filepath); + return readJsonModelPayloadFromBytes(body); } catch (Exception e){ throw new AmazonS3Exception(e.toString()); } } - private byte[] DownloadBytesFromS3(String key) throws Exception{ + + private byte[] downloadBytesFromS3(String key) throws Exception{ S3Object fullObject = null; boolean spaces = key.contains(" "); if(spaces){ @@ -205,7 +211,8 @@ private byte[] DownloadBytesFromS3(String key) throws Exception{ } } } - private Payload ReadJsonModelPayloadFromBytes(byte[] bytes) throws Exception { + + private Payload readJsonModelPayloadFromBytes(byte[] bytes) throws Exception { final ObjectMapper mapper = new ObjectMapper(); // jackson databind try { return mapper.readValue(bytes, Payload.class); @@ -213,7 +220,8 @@ private Payload ReadJsonModelPayloadFromBytes(byte[] bytes) throws Exception { throw e; } } - private void UploadToS3(String bucketName, String objectKey, byte[] fileBytes) { + + private void uploadToS3(String bucketName, String objectKey, byte[] fileBytes) { try { //File file = new File(objectPath); InputStream stream = new ByteArrayInputStream(fileBytes); diff --git a/src/main/java/usace/cc/plugin/Config.java b/src/main/java/usace/cc/plugin/Config.java index 892d996..749ac27 100644 --- a/src/main/java/usace/cc/plugin/Config.java +++ b/src/main/java/usace/cc/plugin/Config.java @@ -1,6 +1,10 @@ package usace.cc.plugin; + import com.fasterxml.jackson.annotation.JsonProperty; + public class Config { + @JsonProperty public AWSConfig[] aws_configs; + } \ No newline at end of file diff --git a/src/main/java/usace/cc/plugin/ConnectionDataStore.java b/src/main/java/usace/cc/plugin/ConnectionDataStore.java new file mode 100644 index 0000000..c6de7b2 --- /dev/null +++ b/src/main/java/usace/cc/plugin/ConnectionDataStore.java @@ -0,0 +1,6 @@ +package usace.cc.plugin; + +public interface ConnectionDataStore { + public ConnectionDataStore connect(DataStore ds); + public Object rawSession(); +} diff --git a/src/main/java/usace/cc/plugin/Constants.java b/src/main/java/usace/cc/plugin/Constants.java index 825f3db..3c9610f 100644 --- a/src/main/java/usace/cc/plugin/Constants.java +++ b/src/main/java/usace/cc/plugin/Constants.java @@ -1,6 +1,6 @@ package usace.cc.plugin; public final class Constants { - public static String PayloadFileName = "payload"; - public static String LocalRootPath = "/data"; + public static String PAYLOAD_FILE_NAME = "payload"; + public static String LOCAL_ROOT_PATH = "/data"; } diff --git a/src/main/java/usace/cc/plugin/DataSource.java b/src/main/java/usace/cc/plugin/DataSource.java index 99359a4..c3afb5e 100644 --- a/src/main/java/usace/cc/plugin/DataSource.java +++ b/src/main/java/usace/cc/plugin/DataSource.java @@ -1,49 +1,55 @@ package usace.cc.plugin; + +import java.util.Map; import com.fasterxml.jackson.annotation.JsonProperty; + public class DataSource { + @JsonProperty - private String Name; - @JsonProperty - private String ID; - @JsonProperty - private String StoreName; + private String name; + @JsonProperty - private String[] Paths; + private String id; + + @JsonProperty("store_name") + private String storeName; + @JsonProperty - private String[] DataPaths; + private Map paths; + + @JsonProperty("data_paths") + private Map dataPaths; + public String getId(){ - return ID; + return id; } + public String getName(){ - return Name; + return name; + } + + public void setName(String name){ + this.name=name; } - public String[] getPaths(){ - return Paths; + + public Map getPaths(){ + return this.paths; } - public String[] getDataPaths(){ - return DataPaths; + + public void setPaths(Map paths){ + this.paths=paths; + } + + public Map getDataPaths(){ + return dataPaths; } - public String getStoreName(){ - return StoreName; - } - public DataSource UpdatePaths(){ - DataSource dest = this; - PluginManager pm = PluginManager.getInstance(); - dest.Name = pm.SubstitutePath(this.getName()); - if(this.getPaths()!=null){ - for(int j=0; j datapaths){ + this.dataPaths=datapaths; + } - return dest; + public String getStoreName(){ + return storeName; } + } \ No newline at end of file diff --git a/src/main/java/usace/cc/plugin/DataSourceIOType.java b/src/main/java/usace/cc/plugin/DataSourceIOType.java new file mode 100644 index 0000000..faab2b0 --- /dev/null +++ b/src/main/java/usace/cc/plugin/DataSourceIOType.java @@ -0,0 +1,7 @@ +package usace.cc.plugin; + +public enum DataSourceIOType { + INPUT, + OUTPUT, + ANY, +} diff --git a/src/main/java/usace/cc/plugin/DataStore.java b/src/main/java/usace/cc/plugin/DataStore.java index 2c482e7..f08b902 100644 --- a/src/main/java/usace/cc/plugin/DataStore.java +++ b/src/main/java/usace/cc/plugin/DataStore.java @@ -1,41 +1,53 @@ package usace.cc.plugin; + import java.util.Map; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; + +@JsonIgnoreProperties public class DataStore { @JsonProperty - private String Name; - @JsonProperty - private String ID; - @JsonProperty - private Map Parameters; - @JsonProperty - private StoreType StoreType; - @JsonProperty - private String DsProfile; + private String name; + @JsonProperty - private Object Session; + @JsonIgnoreProperties(ignoreUnknown = true) + private String id; + + @JsonProperty("params") + @JsonIgnoreProperties(ignoreUnknown = true) + private Map parameters; + //private PayloadAttributes parameters; + + @JsonProperty("store_type") + private StoreType storeType; + + @JsonProperty("profile") + private String dsProfile; + + //@TODO can session be a narrower type or a generic? + private Object session; public String getName(){ - return Name; + return name; } public String getId(){ - return ID; + return id; } public StoreType getStoreType(){ - return StoreType; + return storeType; } - public Map getParameters(){ - return Parameters; + public PayloadAttributes getParameters(){ + return new PayloadAttributes(parameters); } public String getDsProfile(){ - return DsProfile; + return dsProfile; } public Object getSession(){ - return Session; + return session; } public void setSession(Object session){ - this.Session = session; + this.session = session; } } \ No newline at end of file diff --git a/src/main/java/usace/cc/plugin/DataStoreTypeRegistry.java b/src/main/java/usace/cc/plugin/DataStoreTypeRegistry.java new file mode 100644 index 0000000..19f975c --- /dev/null +++ b/src/main/java/usace/cc/plugin/DataStoreTypeRegistry.java @@ -0,0 +1,31 @@ +package usace.cc.plugin; + +import java.lang.reflect.Type; +import java.util.HashMap; +import java.util.Map; + +public final class DataStoreTypeRegistry { + private Map registry; + private static DataStoreTypeRegistry instance = null; + + private DataStoreTypeRegistry(){ + registry = new HashMap(); + registry.put(StoreType.S3, FileDataStoreS3.class); + } + + public static DataStoreTypeRegistry getInstance(){ + if (instance==null){ + instance = new DataStoreTypeRegistry(); + } + return instance; + } + + public static void register(StoreType storeType, Object storeInstance){ + DataStoreTypeRegistry.getInstance().registry.put(storeType, storeInstance.getClass()); + } + + public Object newStore(StoreType s)throws Exception{ + Type type = DataStoreTypeRegistry.getInstance().registry.get(s); + return type.getClass().getDeclaredConstructor().newInstance(); + } +} diff --git a/src/main/java/usace/cc/plugin/EnvironmentVariables.java b/src/main/java/usace/cc/plugin/EnvironmentVariables.java index 9e3f4a0..00d1822 100644 --- a/src/main/java/usace/cc/plugin/EnvironmentVariables.java +++ b/src/main/java/usace/cc/plugin/EnvironmentVariables.java @@ -2,7 +2,9 @@ public final class EnvironmentVariables { public static String CC_MANIFEST_ID = "CC_MANIFEST_ID"; - public static String CC_EVENT_NUMBER = "CC_EVENT_NUMBER"; + public static String CC_PAYLOAD_ID = "CC_PAYLOAD_ID"; + //public static String CC_EVENT_NUMBER = "CC_EVENT_NUMBER"; + public static String CC_EVENT_IDENTIFIER = "CC_EVENT_IDENTIFIER"; public static String CC_EVENT_ID = "CC_EVENT_ID"; public static String CC_ROOT = "CC_ROOT"; public static String CC_PLUGIN_DEFINITION = "CC_PLUGIN_DEFINITION"; @@ -12,4 +14,7 @@ public final class EnvironmentVariables { public static String AWS_SECRET_ACCESS_KEY = "AWS_SECRET_ACCESS_KEY"; public static String AWS_DEFAULT_REGION = "AWS_DEFAULT_REGION"; public static String AWS_S3_BUCKET = "AWS_S3_BUCKET"; + public static String S3_ENDPOINT = "S3_ENDPOINT"; + public static String S3_DISABLE_SSL = "S3_DISABLE_SSL"; + public static String S3_FORCE_PATH_STYLE = "S3_FORCE_PATH_STYLE"; } diff --git a/src/main/java/usace/cc/plugin/Error.java b/src/main/java/usace/cc/plugin/Error.java index 20e0134..d0a2a6c 100644 --- a/src/main/java/usace/cc/plugin/Error.java +++ b/src/main/java/usace/cc/plugin/Error.java @@ -15,13 +15,17 @@ enum ErrorLevel { DISABLED; public static final EnumSet All_Opts = EnumSet.allOf(ErrorLevel.class); } + @JsonProperty private String error; + @JsonProperty private ErrorLevel errorlevel; + public String getError(){ return error; } + public ErrorLevel getErrorLevel(){ return errorlevel; } diff --git a/src/main/java/usace/cc/plugin/FileDataStore.java b/src/main/java/usace/cc/plugin/FileDataStore.java index 6b2d578..6dc6310 100644 --- a/src/main/java/usace/cc/plugin/FileDataStore.java +++ b/src/main/java/usace/cc/plugin/FileDataStore.java @@ -1,9 +1,10 @@ package usace.cc.plugin; + import java.io.InputStream; public interface FileDataStore { - public Boolean Copy(FileDataStore destStore, String srcPath, String destPath); - public InputStream Get(String path); - public Boolean Put(InputStream data, String path); - public Boolean Delete(String path); + public Boolean copy(FileDataStore destStore, String srcPath, String destPath); + public InputStream get(String path); + public Boolean put(InputStream data, String path); + public Boolean delete(String path); } \ No newline at end of file diff --git a/src/main/java/usace/cc/plugin/FileDataStoreS3.java b/src/main/java/usace/cc/plugin/FileDataStoreS3.java index b9d0bf4..a62ff5d 100644 --- a/src/main/java/usace/cc/plugin/FileDataStoreS3.java +++ b/src/main/java/usace/cc/plugin/FileDataStoreS3.java @@ -4,6 +4,8 @@ import java.io.IOException; import java.io.InputStream; import java.rmi.RemoteException; +import java.util.Optional; + import com.amazonaws.AmazonServiceException; import com.amazonaws.ClientConfiguration; import com.amazonaws.Protocol; @@ -13,7 +15,6 @@ import com.amazonaws.auth.AWSStaticCredentialsProvider; import com.amazonaws.auth.BasicAWSCredentials; import com.amazonaws.client.builder.AwsClientBuilder; -import com.amazonaws.regions.Regions; import com.amazonaws.regions.Region; import com.amazonaws.regions.RegionUtils; import com.amazonaws.services.s3.AmazonS3; @@ -25,28 +26,32 @@ import com.amazonaws.services.s3.model.PutObjectResult; import com.amazonaws.services.s3.model.S3Object; - -public class FileDataStoreS3 implements FileDataStore { +//@TODO move all package private vars to class private vars +public class FileDataStoreS3 implements FileDataStore, ConnectionDataStore { String bucket; String postFix; StoreType storeType; AmazonS3 awsS3; AWSConfig config; private static String S3ROOT = "root"; + + public FileDataStoreS3(){} + @Override - public Boolean Copy(FileDataStore destStore, String srcPath, String destPath) { + public Boolean copy(FileDataStore destStore, String srcPath, String destPath) { byte[] data; try { - data = GetObject(srcPath); + data = getObject(srcPath); ByteArrayInputStream bias = new ByteArrayInputStream(data); - return destStore.Put(bias, destPath); + return destStore.put(bias, destPath); } catch (RemoteException e) { e.printStackTrace(); return false; } } + @Override - public InputStream Get(String path) { + public InputStream get(String path) { S3Object fullObject = null; String key = postFix + "/" + path; System.out.println(path); @@ -62,11 +67,11 @@ public InputStream Get(String path) { } @Override - public Boolean Put(InputStream data, String path) { + public Boolean put(InputStream data, String path) { byte[] bytes; try { bytes = data.readAllBytes(); - return UploadToS3(config.aws_bucket, postFix + "/" + path, bytes); + return uploadToS3(config.aws_bucket, postFix + "/" + path, bytes); } catch (IOException e) { e.printStackTrace(); return false; @@ -75,7 +80,7 @@ public Boolean Put(InputStream data, String path) { } @Override - public Boolean Delete(String path) { + public Boolean delete(String path) { DeleteObjectRequest dor = new DeleteObjectRequest(config.aws_bucket,postFix + "/" + path); try{ awsS3.deleteObject(dor); @@ -90,58 +95,64 @@ public Boolean Delete(String path) { } } - public FileDataStoreS3(DataStore ds){ - AWSConfig acfg = new AWSConfig(); - acfg.aws_access_key_id = System.getenv(ds.getDsProfile() + "_" + EnvironmentVariables.AWS_ACCESS_KEY_ID); - acfg.aws_secret_access_key_id = System.getenv(ds.getDsProfile() + "_" + EnvironmentVariables.AWS_SECRET_ACCESS_KEY); - acfg.aws_region = System.getenv(ds.getDsProfile() + "_" + EnvironmentVariables.AWS_DEFAULT_REGION); - acfg.aws_bucket = System.getenv(ds.getDsProfile() + "_" + EnvironmentVariables.AWS_S3_BUCKET); - acfg.aws_mock = Boolean.parseBoolean(System.getenv(ds.getDsProfile() + "_"+ "S3_MOCK"));//convert to boolean; - acfg.aws_endpoint = System.getenv(ds.getDsProfile() + "_"+ "S3_ENDPOINT"); - acfg.aws_disable_ssl = Boolean.parseBoolean(System.getenv(ds.getDsProfile() + "_"+ "S3_DISABLE_SSL"));//convert to bool? - acfg.aws_force_path_style = Boolean.parseBoolean(System.getenv(ds.getDsProfile() + "_"+ "S3_FORCE_PATH_STYLE"));//convert to bool - config = acfg; - //System.out.println(ds.getDsProfile() + "_" + EnvironmentVariables.AWS_DEFAULT_REGION+"::"+config.aws_region); - //System.out.println(ds.getDsProfile() + "_" + EnvironmentVariables.AWS_ACCESS_KEY_ID+"::"+config.aws_access_key_id); - //System.out.println(ds.getDsProfile() + "_" + EnvironmentVariables.AWS_SECRET_ACCESS_KEY+"::"+config.aws_secret_access_key_id); - //System.out.println(ds.getDsProfile() + "_" + EnvironmentVariables.AWS_S3_BUCKET+"::"+config.aws_bucket); + @Override + public Object rawSession(){ + return awsS3; + } + + @Override + public ConnectionDataStore connect(DataStore ds) { + config = new AWSConfig(); + config.aws_access_key_id = System.getenv(ds.getDsProfile() + "_" + EnvironmentVariables.AWS_ACCESS_KEY_ID); + config.aws_secret_access_key_id = System.getenv(ds.getDsProfile() + "_" + EnvironmentVariables.AWS_SECRET_ACCESS_KEY); + config.aws_region = System.getenv(ds.getDsProfile() + "_" + EnvironmentVariables.AWS_DEFAULT_REGION); + config.aws_bucket = System.getenv(ds.getDsProfile() + "_" + EnvironmentVariables.AWS_S3_BUCKET); + config.aws_endpoint = System.getenv(ds.getDsProfile() + "_"+ EnvironmentVariables.S3_ENDPOINT); + config.aws_disable_ssl = Boolean.parseBoolean(System.getenv(ds.getDsProfile() + "_"+ EnvironmentVariables.S3_DISABLE_SSL));//convert to bool? + config.aws_force_path_style = Boolean.parseBoolean(System.getenv(ds.getDsProfile() + "_"+ EnvironmentVariables.S3_FORCE_PATH_STYLE));//convert to bool Region clientRegion = RegionUtils.getRegion(config.aws_region);//.toUpperCase().replace("-", "_"));//Regions.valueOf(config.aws_region.toUpperCase().replace("-", "_")); try { AmazonS3 s3Client = null; - if(config.aws_mock){ - AWSCredentials credentials = new BasicAWSCredentials(config.aws_access_key_id, config.aws_secret_access_key_id); + AWSCredentials credentials = new BasicAWSCredentials(config.aws_access_key_id, config.aws_secret_access_key_id); + + var clientBuilder = AmazonS3ClientBuilder.standard() + .withRegion(clientRegion.getName()) + .withCredentials(new AWSStaticCredentialsProvider(credentials)); + + if (!config.aws_endpoint.equals("")){ ClientConfiguration clientConfiguration = new ClientConfiguration(); clientConfiguration.setSignerOverride("AWSS3V4SignerType"); clientConfiguration.setProtocol(Protocol.HTTP); - s3Client = AmazonS3ClientBuilder - .standard() - .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(config.aws_endpoint, clientRegion.getName())) + clientBuilder.withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(config.aws_endpoint, clientRegion.getName())) .withPathStyleAccessEnabled(config.aws_force_path_style) - .withClientConfiguration(clientConfiguration) - .withCredentials(new AWSStaticCredentialsProvider(credentials)) - .build(); - }else{ - AWSCredentials credentials = new BasicAWSCredentials(config.aws_access_key_id, config.aws_secret_access_key_id); - s3Client = AmazonS3ClientBuilder - .standard() - .withRegion(clientRegion.getName()) - .withCredentials(new AWSStaticCredentialsProvider(credentials)) - .build(); + .withClientConfiguration(clientConfiguration); + } - awsS3 = s3Client; - } catch (AmazonServiceException e) { - // The call was transmitted successfully, but Amazon S3 couldn't process - // it, so it returned an error response. - e.printStackTrace(); - } catch (SdkClientException e) { - // Amazon S3 couldn't be contacted for a response, or the client - // couldn't parse the response from Amazon S3. + + awsS3 = clientBuilder.build(); + + } catch (SdkClientException e ) { + //@TODO do we want to print the stackstrace? + // I'm converting to a RuntimeException and throwing it up the stack e.printStackTrace(); + throw new RuntimeException(e); } + storeType = StoreType.S3; - String tmpRoot = ds.getParameters().get(FileDataStoreS3.S3ROOT); + + String tmpRoot=""; + + try { + Optional optParam = ds.getParameters().get(FileDataStoreS3.S3ROOT); + if (optParam.isPresent()){ + tmpRoot = optParam.get(); + } + } catch (Exception e) { + e.printStackTrace(); + //@TODO UGH....NOT HANDLING THE ERRORS!!!!!!!!!!!!!!!! + } if (tmpRoot == ""){ //error out? System.out.print("Missing S3 Root Paramter. Cannot create the store."); @@ -149,17 +160,20 @@ public FileDataStoreS3(DataStore ds){ this.bucket = config.aws_bucket; tmpRoot = tmpRoot.replaceFirst("^/+", ""); this.postFix = tmpRoot; + return this; } - private byte[] GetObject(String path) throws RemoteException { + + private byte[] getObject(String path) throws RemoteException { byte[] data; try { - data = DownloadBytesFromS3(path); + data = downloadBytesFromS3(path); } catch (Exception e) { throw new RemoteException(e.toString()); } return data; } - private byte[] DownloadBytesFromS3(String key) throws Exception{ + + private byte[] downloadBytesFromS3(String key) throws Exception{ S3Object fullObject = null; key = postFix + "/" + key; System.out.println(key); @@ -182,7 +196,7 @@ private byte[] DownloadBytesFromS3(String key) throws Exception{ } } - private boolean UploadToS3(String bucketName, String objectKey, byte[] fileBytes) { + private boolean uploadToS3(String bucketName, String objectKey, byte[] fileBytes) { try { InputStream stream = new ByteArrayInputStream(fileBytes); ObjectMetadata meta = new ObjectMetadata(); @@ -196,4 +210,5 @@ private boolean UploadToS3(String bucketName, String objectKey, byte[] fileBytes return false; } } + } diff --git a/src/main/java/usace/cc/plugin/GetDataSourceInput.java b/src/main/java/usace/cc/plugin/GetDataSourceInput.java new file mode 100644 index 0000000..39c18a9 --- /dev/null +++ b/src/main/java/usace/cc/plugin/GetDataSourceInput.java @@ -0,0 +1,20 @@ +package usace.cc.plugin; + +public class GetDataSourceInput { + + private DataSourceIOType dataSourceType; + private String dataSourceName; + + public DataSourceIOType getDataSourceIOType(){ + return dataSourceType; + } + + public String getDataSourceName(){ + return dataSourceName; + } + + public GetDataSourceInput(String name, DataSourceIOType type){ + dataSourceType = type; + dataSourceName = name; + } +} diff --git a/src/main/java/usace/cc/plugin/GetObjectInput.java b/src/main/java/usace/cc/plugin/GetObjectInput.java index fb6c513..95ef467 100644 --- a/src/main/java/usace/cc/plugin/GetObjectInput.java +++ b/src/main/java/usace/cc/plugin/GetObjectInput.java @@ -5,25 +5,28 @@ public class GetObjectInput { private String fileExtension; private StoreType sourceStoreType; private String sourceRootPath; + + public GetObjectInput(String fileName, StoreType sourceStoreType, String sourceRootPath, String fileExtension){ + this.fileName = fileName; + this.sourceStoreType = sourceStoreType; + this.sourceRootPath = sourceRootPath; + this.fileExtension = fileExtension; + } + public String getFileName(){ return fileName; } + public String getFileExtension(){ return fileExtension; } + public StoreType getSourceStoreType(){ return sourceStoreType; } + public String getSourceRootPath(){ return sourceRootPath; } - /** - * - */ - public GetObjectInput(String fileName, StoreType sourceStoreType, String sourceRootPath, String fileExtension){ - this.fileName = fileName; - this.sourceStoreType = sourceStoreType; - this.sourceRootPath = sourceRootPath; - this.fileExtension = fileExtension; - } + } \ No newline at end of file diff --git a/src/main/java/usace/cc/plugin/IOManager.java b/src/main/java/usace/cc/plugin/IOManager.java new file mode 100644 index 0000000..889ec74 --- /dev/null +++ b/src/main/java/usace/cc/plugin/IOManager.java @@ -0,0 +1,370 @@ +package usace.cc.plugin; + +import java.util.Arrays; +import java.util.Map; +import java.util.Optional; +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; + +public class IOManager { + + //IO Manager Error Types + static class InvalidDataSourceException extends RuntimeException { + public InvalidDataSourceException(String message) { + super(message); + } + + public InvalidDataSourceException(Exception ex){ + super(ex); + } + } + + static class InvalidDataStoreException extends Exception { + public InvalidDataStoreException(String message) { + super(message); + } + + public InvalidDataStoreException(Exception ex){ + super(ex); + } + } + + @JsonProperty + @JsonIgnoreProperties(ignoreUnknown = true) + private Map attributes; + //private PayloadAttributes attributes; + + @JsonProperty + @JsonIgnoreProperties(ignoreUnknown = true) + private DataStore[] stores; + + @JsonProperty + @JsonIgnoreProperties(ignoreUnknown = true) + private DataSource[] inputs; + + @JsonProperty + @JsonIgnoreProperties(ignoreUnknown = true) + private DataSource[] outputs; + + private IOManager parent; + + public PayloadAttributes getAttributes(){ + return new PayloadAttributes(attributes); + } + + public DataStore[] getStores(){ + return this.stores; + } + + public DataSource[] getInputs(){ + return this.inputs; + } + + public DataSource[] getOutputs(){ + return this.outputs; + } + + public void setParent(IOManager parent){ + this.parent=parent; + } + + + /** + * Retrieves a {@link DataStore} by its name. + *

+ * This method searches the current list of data stores for one with a matching name. + * If not found and a parent is defined, it will recursively search the parent. + *

+ * + * @param name the name of the data store to retrieve + * @return an {@code Optional} containing the matching {@code DataStore}, or + * {@code Optional.empty()} if no matching store is found in this instance + * or its parent chain + */ + public Optional getStore(String name){ + for (DataStore store:this.stores){ + if (name.equals(store.getName())){ + return Optional.of(store); + } + } + if (this.parent!=null){ + return parent.getStore(name); + } + return Optional.empty(); + } + + + /** + * Retrieves a {@link DataSource} based on the provided input criteria. + *

+ * This method selects from input sources, output sources, or both depending on the + * {@link GetDataSourceInput#getDataSourceIOType()} value. It then searches for a + * {@code DataSource} with a matching name. If no match is found locally and a parent + * exists, the search will continue recursively in the parent. + *

+ * + * @param gdsi an object encapsulating the data source name and the I/O type to search + * @return an {@code Optional} containing the matching {@code DataSource}, or {@code Optional.empty()} + * if no such data source exists in this instance or its parent chain + * @throws InvalidDataSourceException runtime exception if the {@code DataSourceIOType} in the input is not recognized + */ + public Optional getDataSource(GetDataSourceInput gdsi) throws InvalidDataSourceException{ + DataSource[] sources; + switch (gdsi.getDataSourceIOType()) { + case INPUT: + sources = this.inputs; + break; + case OUTPUT: + sources = this.outputs; + break; + case ANY: + sources = Arrays.copyOf(this.inputs,this.inputs.length + this.outputs.length); + System.arraycopy(this.outputs, 0, sources, this.inputs.length, this.outputs.length); + break; + default: + throw new InvalidDataSourceException("data source input type not recognized"); + } + + for(DataSource ds : sources){ + if(ds.getName() == gdsi.getDataSourceName()){ + return Optional.of(ds); + } + } + + if (this.parent!=null){ + return this.parent.getDataSource(gdsi); + } + + return Optional.empty(); + } + + /** + * Retrieves an input {@link DataSource} by name. + *

+ * This is a convenience method that constructs a {@link GetDataSourceInput} with the specified + * name and an I/O type of {@code INPUT}, then delegates to {@link #getDataSource(GetDataSourceInput)}. + *

+ * + * @param name the name of the input data source to retrieve + * @return an {@code Optional} containing the matching input {@code DataSource}, or + * {@code Optional.empty()} if no matching source is found locally or in a parent + * @throws InvalidDataSourceException if the underlying data source lookup encounters an error + */ + public Optional getInputDataSource(String name) throws InvalidDataSourceException { + var gdsi = new GetDataSourceInput(name, DataSourceIOType.INPUT); + return getDataSource(gdsi); + } + + /** + * Retrieves an output {@link DataSource} by name. + *

+ * This is a convenience method that constructs a {@link GetDataSourceInput} with the specified + * name and an I/O type of {@code OUTPUT}, then delegates to {@link #getDataSource(GetDataSourceInput)}. + *

+ * + * @param name the name of the output data source to retrieve + * @return an {@code Optional} containing the matching output {@code DataSource}, or + * {@code Optional.empty()} if no matching source is found locally or in a parent + * @throws InvalidDataSourceException if the underlying data source lookup encounters an error + */ + public Optional getOutputDataSource(String name) throws InvalidDataSourceException { + var gdsi = new GetDataSourceInput(name, DataSourceIOType.OUTPUT); + return getDataSource(gdsi); + } + + //@TODO....I include a data path here + /** + * Copies a file from an input {@link DataSource} to a local file path. + *

+ * This method locates the input data source by name, retrieves the input stream + * for the specified path key, reads the file's contents, and writes them to the + * specified local file path. + *

+ * + * @param dataSourceName the name of the input data source + * @param pathkey the key or path identifying the file within the data source + * @param localPath the path on the local filesystem where the file will be written + * @throws InvalidDataSourceException if the data source is not found or if an error occurs while accessing it + * @throws IOException if an I/O error occurs during reading from the source or writing to the local file + */ + public void copyFileToLocal(String dataSourceName, String pathkey, String localPath) throws IOException, InvalidDataSourceException{ + Optional indsOpt = getDataSource(new GetDataSourceInput(dataSourceName, DataSourceIOType.INPUT)); + if (!indsOpt.isPresent()){ + throw new InvalidDataSourceException("Data source not found"); + } + + DataSource inds = indsOpt.get(); + + try (InputStream is = getInputStream(inds, pathkey)) { + byte[] bytes = is.readAllBytes(); + File outfile = new File(localPath); + try(OutputStream writer = new FileOutputStream(outfile)){ + writer.write(bytes); + } + } + } + + // public void write(InputStream writer, String datasourcename, String pathName, String datapathName) throws Exception{ + // DataSource ds = this.getDataSource(new GetDataSourceInput(datasourcename, DataSourceIOType.OUTPUT)); + // FileDataStore fds = GetStoreSession(ds.getStoreName()); + // fds.Put(writer, pathName); + // } + + + //@TODO dataPathName is not being used!!!!! + + /** + * Retrieves the contents of a file from an input {@link DataSource} as a byte array. + *

+ * This method locates the data source by name and attempts to read the contents of the file + * identified by the given {@code pathName}. If the data source is not found, or an error occurs + * while accessing it, an exception is thrown. The {@code dataPathName} is included for error + * context but not used directly in the lookup. + *

+ * + * @param dataSourceName the name of the input data source + * @param pathName the key or identifier of the file within the data source + * @param dataPathName an additional path descriptor for context in error reporting + * @return a byte array containing the contents of the specified file + * @throws InvalidDataSourceException if the data source is not found or cannot be accessed + * @throws IOException if an I/O error occurs while reading from the data source + */ + public byte[] get(String dataSourceName, String pathName, String dataPathName) throws IOException, InvalidDataSourceException{ + Optional dsOpt = this.getDataSource(new GetDataSourceInput(dataSourceName, DataSourceIOType.INPUT)); + if (dsOpt.isPresent()){ + var ds = dsOpt.get(); + return getInputStream(ds, pathName).readAllBytes(); + } + throw new InvalidDataSourceException(String.format("invalid data source. Name: %s, Path: %s, DataPath: %s",dataSourceName,pathName,dataPathName)); + } + + /** + * Writes the provided byte array to a specified path in a named output data source. + * + *

This method looks up the data source by name and attempts to retrieve a corresponding + * {@code FileDataStore} session. If both are found, it writes the data to the given path + * within that data store. If either the data source or the data store is not found, + * an appropriate checked exception is thrown.

+ * + * @param data the data to write + * @param dataSourceName the name of the output data source to target + * @param pathName the path within the data store to write the data to + * @param dataPathName currently unused, reserved for future expansion or routing + * + * @throws IOException if an I/O error occurs during the write + * @throws InvalidDataSourceException if the specified data source cannot be found + * @throws InvalidDataStoreException if the data store session for the data source cannot be obtained + */ + public void put(byte[] data, String dataSourceName, String pathName, String dataPathName) throws IOException, InvalidDataSourceException, InvalidDataStoreException{ + Optional dsOpt = this.getDataSource(new GetDataSourceInput(dataSourceName, DataSourceIOType.OUTPUT)); + if (dsOpt.isPresent()){ + var ds = dsOpt.get(); + Optional fdsOpt = getStoreSession(ds.getStoreName()); + if (fdsOpt.isPresent()){ + var fds = fdsOpt.get(); + ByteArrayInputStream bais = new ByteArrayInputStream(data); + fds.put(bais, pathName); + } else { + throw new InvalidDataStoreException("Datastore not found"); + } + } else { + throw new InvalidDataSourceException("Datasource not found"); + } + } + + /** + * Retrieves an {@link InputStream} from the given {@link DataSource} using the specified path key. + *

+ * This method attempts to obtain a {@code FileDataStore} session from the data source's store name, + * and then retrieves the corresponding input stream using the provided {@code pathKey}. + *

+ * + * @param dataSource the data source from which to retrieve the input stream + * @param pathKey the key or identifier used to locate the desired file within the store + * @return an {@code InputStream} for reading the data associated with the given key + * @throws InvalidDataSourceException if the store session cannot be obtained, if the store is not + * present, or if an underlying {@code InvalidDataStoreException} occurs + */ + public InputStream getInputStream(DataSource dataSource, String pathKey) throws InvalidDataSourceException{ + try{ + Optional fdsOpt = getStoreSession(dataSource.getStoreName()); + if(fdsOpt.isPresent()){ + var fds = fdsOpt.get(); + return fds.get(pathKey); + } + throw new InvalidDataSourceException("Unable to get input stream from the data source"); + } catch(InvalidDataStoreException ex) { + throw new InvalidDataSourceException(ex); + } + } + + /** + * Copies a local file to a remote data store. + * + *

This method retrieves a {@link DataSource} configured for output using the given + * destination name, then locates the associated {@link FileDataStore} session. + * It reads the local file from the specified {@code localPath} and uploads its contents + * to the remote store under the provided {@code pathKey}.

+ * + * @param destinationName the name of the remote destination to which the file will be copied + * @param pathKey the key (path) under which the file will be stored remotely + * @param localPath the file system path to the local file that needs to be copied + * @throws InvalidDataSourceException if the specified data source is invalid or cannot be retrieved + * @throws InvalidDataStoreException if the data store session cannot be established or is invalid + * @throws IOException if an I/O error occurs while reading the local file or writing to the remote store + */ + public void copyFileToRemote(String destinationName, String pathKey, String localPath) throws InvalidDataSourceException, InvalidDataStoreException, IOException{ + Optional dsOpt = this.getDataSource(new GetDataSourceInput(destinationName, DataSourceIOType.OUTPUT)); + if(dsOpt.isPresent()){ + var ds = dsOpt.get(); + Optional fdsOpt = getStoreSession(ds.getStoreName()); + if(fdsOpt.isPresent()){ + var fdstore = fdsOpt.get(); + File localFile = new File(localPath); + InputStream reader = new FileInputStream(localFile); + fdstore.put(reader, pathKey); + } + } + } + + + /** + * Retrieves a session object of type {@code T} from a data store with the specified name. + *

+ * This method searches through the available {@code DataStore} instances to find one + * matching the given {@code name}. If found, it attempts to retrieve and cast the session + * object to the desired type {@code T}. + *

+ * + * @param The expected type of the session object. + * @param name The name of the data store to retrieve the session from. + * @return An {@code Optional} containing the session object if present and of the correct type, + * or {@code Optional.empty()} if no matching store is found or the session is {@code null}. + * @throws InvalidDataStoreException if the session cannot be cast to the specified type {@code T}. + */ + public Optional getStoreSession(String name) throws InvalidDataStoreException{ + for(DataStore ds : stores){ + if (ds.getName()==name){ + try{ + T session = (T)ds.getSession(); + if (session==null){ + return Optional.empty(); + } + return Optional.of(session); + } catch(ClassCastException ex){ + throw new InvalidDataStoreException(ex); + } + } + } + return Optional.empty(); + } +} diff --git a/src/main/java/usace/cc/plugin/Logger.java b/src/main/java/usace/cc/plugin/Logger.java index a8a13c6..6dc6e14 100644 --- a/src/main/java/usace/cc/plugin/Logger.java +++ b/src/main/java/usace/cc/plugin/Logger.java @@ -1,33 +1,35 @@ package usace.cc.plugin; import java.time.LocalDate; - import usace.cc.plugin.Error.ErrorLevel; public class Logger { //this is an aggregator, it is anticipated that this will get replaced but the api will remain. private ErrorLevel errorLevel; private String sender; + public Logger(String sender, ErrorLevel level){ this.sender = sender; this.errorLevel = level; } + public void setErrorLevel(ErrorLevel level){ this.errorLevel = level; } - public void LogMessage(Message message){ + + public void logMessage(Message message){ String line = this.sender + ":" + LocalDate.now() + "\n\t" + message.getMessage() + "\n"; System.out.println(line); } - public void LogError(Error error){ + + public void logError(Error error){ if (error.getErrorLevel().compareTo(this.errorLevel)>=0){ String line = sender + "issues a " + error.getErrorLevel().toString() + " error:" + LocalDate.now() + "\n\t" + error.getError() + "\n"; System.out.println(line); - } - - + } } - public void ReportStatus(Status report){ + + public void reportStatus(Status report){ String line = this.sender + ":" + report.getStatus().toString() + ":" + LocalDate.now() + "\n\t" + report.getProgress() + " percent complete." + "\n"; System.out.println(line); } diff --git a/src/main/java/usace/cc/plugin/Message.java b/src/main/java/usace/cc/plugin/Message.java index 57f0837..3d5996d 100644 --- a/src/main/java/usace/cc/plugin/Message.java +++ b/src/main/java/usace/cc/plugin/Message.java @@ -1,12 +1,16 @@ package usace.cc.plugin; import com.fasterxml.jackson.annotation.JsonProperty; + public class Message { + @JsonProperty private String message; + public String getMessage(){ return message; } + public Message(String message){ this.message = message; } diff --git a/src/main/java/usace/cc/plugin/ObjectState.java b/src/main/java/usace/cc/plugin/ObjectState.java index 89a170b..041ea69 100644 --- a/src/main/java/usace/cc/plugin/ObjectState.java +++ b/src/main/java/usace/cc/plugin/ObjectState.java @@ -1,7 +1,7 @@ package usace.cc.plugin; public enum ObjectState { - Memory, - LocalDisk, + MEMORY, + LOCAL_DISK, //RemoteDisk //ToDo } diff --git a/src/main/java/usace/cc/plugin/Payload.java b/src/main/java/usace/cc/plugin/Payload.java index 3febdb9..83c5049 100644 --- a/src/main/java/usace/cc/plugin/Payload.java +++ b/src/main/java/usace/cc/plugin/Payload.java @@ -1,35 +1,15 @@ package usace.cc.plugin; import com.fasterxml.jackson.annotation.JsonProperty; -import java.util.Map; -public class Payload { - @JsonProperty - private Map attributes; - @JsonProperty - private DataStore[] stores; - @JsonProperty - private DataSource[] inputs; - @JsonProperty - private DataSource[] outputs; + +public class Payload extends IOManager{ + @JsonProperty private Action[] actions; - public Map getAttributes(){ - return attributes; - } - public DataStore[] getStores(){ - return stores; - } - public void setStore(int index, DataStore store){ - stores[index] = store; - } - public DataSource[] getInputs(){ - return inputs; - } - public DataSource[] getOutputs(){ - return outputs; - } + public Action[] getActions(){ return actions; } + } diff --git a/src/main/java/usace/cc/plugin/PayloadAttributes.java b/src/main/java/usace/cc/plugin/PayloadAttributes.java new file mode 100644 index 0000000..d59df71 --- /dev/null +++ b/src/main/java/usace/cc/plugin/PayloadAttributes.java @@ -0,0 +1,58 @@ +package usace.cc.plugin; + +import java.util.Map; +import java.util.Map.Entry; +import java.util.Optional; +import java.util.Set; + +import com.fasterxml.jackson.annotation.JsonProperty; + +public class PayloadAttributes { + + public PayloadAttributes(Map attrs){ + this.attributes=attrs; + } + + @JsonProperty + private Map attributes; + + public Map getAttributes(){ + return attributes; + } + + public Set> entrySet(){ + return attributes.entrySet(); + } + + public Set keySet(){ + return attributes.keySet(); + } + + public Optional get(String name) throws IllegalArgumentException{ + Object val = attributes.get(name); + if (val==null){ + return Optional.empty(); + }else{ + try { + @SuppressWarnings("unchecked") + T tval = (T) val; + return Optional.of(tval); + } catch (ClassCastException e) { + throw new IllegalArgumentException("Invalid type cast.", e); + } + } + } + + //@TODO delete if uneccesary + public Optional getAlt1(String name, Class clazz) throws IllegalArgumentException{ + Object val = attributes.get(name); + if (val==null){ + return Optional.empty(); + }else{ + if (clazz.isInstance(val)){ + return Optional.of((T)val); + } + throw new IllegalArgumentException("Incorrect type"); + } + } +} \ No newline at end of file diff --git a/src/main/java/usace/cc/plugin/PluginManager.java b/src/main/java/usace/cc/plugin/PluginManager.java index 3a5a95a..4f15db5 100644 --- a/src/main/java/usace/cc/plugin/PluginManager.java +++ b/src/main/java/usace/cc/plugin/PluginManager.java @@ -1,184 +1,182 @@ package usace.cc.plugin; -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.io.InputStream; +import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.regex.Matcher; import java.util.regex.Pattern; import usace.cc.plugin.Error.ErrorLevel; public final class PluginManager { + private CcStore cs; - private Payload _payload; - private Logger _logger; - private static PluginManager _instance = null; - private boolean _hasUpdatedPaths = false; + private Payload payload; + private Logger logger; + private String eventIdentifier; + private static PluginManager instance = null; + private boolean hasUpdatedPaths = false; + private Pattern p; + private final String pathPattern = "(?<=\\{).+?(?=\\})"; + + public static PluginManager getInstance(){ - if (_instance==null){ - _instance = new PluginManager(); + if (instance==null){ + instance = new PluginManager(); } - return _instance; + return instance; } - Pattern p; + + private PluginManager(){ - p = Pattern.compile("(?<=\\{).+?(?=\\})"); + p = Pattern.compile(pathPattern); String sender = System.getenv(EnvironmentVariables.CC_PLUGIN_DEFINITION); - _logger = new Logger(sender, ErrorLevel.WARN); + eventIdentifier=System.getenv(EnvironmentVariables.CC_EVENT_IDENTIFIER); + logger = new Logger(sender, ErrorLevel.WARN); cs = new CcStoreS3(); try { - _payload = cs.GetPayload(); - int i = 0; - for (DataStore store : _payload.getStores()) { - switch (store.getStoreType()){ - case S3: - store.setSession(new FileDataStoreS3(store)); - _payload.setStore(i, store); - break; - case WS://does java work with fallthrough? - case RDBMS: - System.out.println("WS and RDBMS session instantiation is the responsibility of the plugin."); - break; - default: - System.out.println("Invalid Store type");//what type was provided? - break; - } - i ++; + this.payload = cs.getPayload(); + this.connectStores(payload.getStores()); + for (Action action : payload.getActions()) { + this.connectStores(action.getStores()); } - //substitutePathVariables(); + substitutePathVariables(); } catch (Exception e) { e.printStackTrace(); } } - private void substitutePathVariables() { - if (_payload.getInputs()!=null){ - for (int i= 0; i<_payload.getInputs().length; i++){ - _payload.getInputs()[i] = _payload.getInputs()[i].UpdatePaths(); - } + + + public Payload getPayload(){ + //@TODO why substitute here? + // if (!hasUpdatedPaths){ + // try{ + // substitutePathVariables(); + // }catch(Exception e){ + // e.printStackTrace(); + // System.exit(1); + // } + // hasUpdatedPaths = true; + // } + return payload; + } + + public String getEventIdentifier() { + return this.eventIdentifier; + } + + public void setLogLevel(ErrorLevel level){ + logger.setErrorLevel(level); + } + + public void logMessage(Message message){ + logger.logMessage(message); + } + + public void logError(Error error){ + logger.logError(error); + } + + public void reportProgress(Status report){ + logger.reportStatus(report); + } + + + private void connectStores(DataStore[] stores) throws Exception{ + DataStoreTypeRegistry registry = DataStoreTypeRegistry.getInstance(); + for (int i = 0; i entry : attrs.entrySet()) { + //var key = entry.getKey(); + var val = entry.getValue(); + if (val instanceof String){ + parameterSubstitute((String)val, pattrs); } } + } + //a.k.a. pathssubstitute + private void substitutePaths(DataSource ds, PayloadAttributes attrs){ + var param = parameterSubstitute(ds.getName(),attrs); + ds.setName(param); + + var paths = ds.getPaths(); + for (String key : paths.keySet()){ + paths.put(key, parameterSubstitute(paths.get(key), attrs)); + } + ds.setPaths(paths); + + var datapaths = ds.getDataPaths(); + for (String key : datapaths.keySet()){ + datapaths.put(key, parameterSubstitute(datapaths.get(key), attrs)); + } + ds.setDataPaths(datapaths); } - public String SubstitutePath(String path) { - Matcher m = p.matcher(path); + + private String parameterSubstitute(String param, PayloadAttributes attrs) { + Matcher m = p.matcher(param); while(m.find()){ String result = m.group(); String[] parts = result.split("::", 0); String prefix = parts[0]; + String subname = parts[1]; switch(prefix){ case "ENV": - String val = System.getenv(parts[1]); - path = path.replaceFirst("\\{"+result+"\\}", val);//? - m = p.matcher(path); + String val = System.getenv(subname); + param = param.replaceFirst("\\{"+result+"\\}", val);//? + m = p.matcher(param); break; case "ATTR": - String valattr = _payload.getAttributes().get(parts[1]).toString(); - path = path.replaceFirst("\\{"+result+"\\}", valattr);//? - m = p.matcher(path); + //Optional valattr = attrs.get(subname); + //Optional valattr = (Optional)attrs.get(subname); + //Optional valattr = attrs.getAlt1(subname, String.class); + Optional optVal = attrs.get(subname); + if (optVal.isPresent()){ + param = param.replaceFirst("\\{"+result+"\\}", optVal.get());//? + m = p.matcher(param); + } else{ + //@TODO logging is kind of annoying + logger.logMessage(new Message(String.format("Attribute %s not found", subname))); + } break; - default: - break; - } - } - return path; - } - public Payload getPayload(){ - if (!_hasUpdatedPaths){ - substitutePathVariables(); - _hasUpdatedPaths = true; - } - return _payload; - } - public FileDataStore getFileStore(String storeName){ - return (FileDataStore) findDataStore(storeName).getSession();//check for nil? - } - public DataStore getStore(String storeName){ - return findDataStore(storeName); - } - public DataSource getInputDataSource(String name){ - return findDataSource(name, getInputDataSources()); - } - public DataSource getOutputDataSource(String name){ - return findDataSource(name, getOutputDataSources()); - } - public DataSource[] getInputDataSources(){ - return _payload.getInputs(); - } - public DataSource[] getOutputDataSources(){ - return _payload.getOutputs(); - } - public byte[] getFile(DataSource ds, int path){ - FileDataStore store = getFileStore(ds.getStoreName()); - InputStream reader = store.Get(ds.getPaths()[path]); - byte[] data; - try { - data = reader.readAllBytes(); - return data; - } catch (IOException e) { - e.printStackTrace(); - return null; - } - } - public boolean putFile(byte[] data, DataSource ds, int path){ - FileDataStore store = getFileStore(ds.getStoreName()); - return store.Put(new ByteArrayInputStream(data), ds.getPaths()[path]); - } - public boolean fileWriter(InputStream inputstream, DataSource destDs, int destPath){ - FileDataStore store = getFileStore(destDs.getStoreName()); - return store.Put(inputstream, destDs.getPaths()[destPath]); - } - public InputStream fileReader(DataSource ds, int path){ - FileDataStore store = getFileStore(ds.getStoreName()); - return store.Get(ds.getPaths()[path]); - } - public InputStream fileReaderByName(String dataSourceName, int path){ - DataSource ds = findDataSource(dataSourceName, getInputDataSources()); - return fileReader(ds, path); - } - public void setLogLevel(ErrorLevel level){ - _logger.setErrorLevel(level); - } - public void LogMessage(Message message){ - _logger.LogMessage(message); - } - public void LogError(Error error){ - _logger.LogError(error); - } - public void ReportProgress(Status report){ - _logger.ReportStatus(report); - } - public int EventNumber(){ - //Object result = _payload.getAttributes().get(EnvironmentVariables.CC_EVENT_NUMBER); - String val = System.getenv(EnvironmentVariables.CC_EVENT_NUMBER); - int eventNumber = Integer.parseInt(val); - return eventNumber; - } - private DataSource findDataSource(String name, DataSource[] dataSources){ - for (DataSource dataSource : dataSources) { - if (dataSource.getName().equalsIgnoreCase(name)){ - return dataSource; - } - } - return null; - } - private DataStore findDataStore(String name){ - for (DataStore dataStore : _payload.getStores()) { - if (dataStore.getName().equalsIgnoreCase(name)){ - return dataStore; } } - System.out.println(name + " store not found."); - return null; + return param; } } \ No newline at end of file diff --git a/src/main/java/usace/cc/plugin/PullObjectInput.java b/src/main/java/usace/cc/plugin/PullObjectInput.java index 8ee42bd..8fb2d51 100644 --- a/src/main/java/usace/cc/plugin/PullObjectInput.java +++ b/src/main/java/usace/cc/plugin/PullObjectInput.java @@ -7,30 +7,34 @@ public class PullObjectInput { private StoreType sourceStoreType; private String sourceRootPath; private String destRootPath; + + public PullObjectInput(String fileName, StoreType sourceStoreType, String sourceRootPath, String destRootPath, String fileExtension){ + this.fileName = fileName; + this.sourceStoreType = sourceStoreType; + this.sourceRootPath = sourceRootPath; + this.destRootPath = destRootPath; + this.fileExtension = fileExtension; + } + public String getFileName(){ return fileName; } + public String getFileExtension(){ return fileExtension; } + public StoreType getSourceStoreType(){ return sourceStoreType; } + public String getSourceRootPath(){ return sourceRootPath; } + public String getDestRootPath(){ return destRootPath; } - /** - * - */ - public PullObjectInput(String fileName, StoreType sourceStoreType, String sourceRootPath, String destRootPath, String fileExtension){ - this.fileName = fileName; - this.sourceStoreType = sourceStoreType; - this.sourceRootPath = sourceRootPath; - this.destRootPath = destRootPath; - this.fileExtension = fileExtension; - } + } diff --git a/src/main/java/usace/cc/plugin/PutObjectInput.java b/src/main/java/usace/cc/plugin/PutObjectInput.java index 6ed19c3..560cf68 100644 --- a/src/main/java/usace/cc/plugin/PutObjectInput.java +++ b/src/main/java/usace/cc/plugin/PutObjectInput.java @@ -8,37 +8,43 @@ public class PutObjectInput { private byte[] data; private String sourcePath; private String destPath; + + public PutObjectInput(String fileName, StoreType destStoreType, String sourcePath, String destPath, String fileExtension, ObjectState state, byte[] data){ + this.fileName = fileName; + this.destStoreType = destStoreType; + this.sourcePath = sourcePath; + this.destPath = destPath; + this.fileExtension = fileExtension; + this.data = data; + this.objectState = state; + } + public String getFileName(){ return fileName; } + public String getFileExtension(){ return fileExtension; } + public StoreType getDestinationStoreType(){ return destStoreType; } + public ObjectState getObjectState(){ return objectState; } + public byte[] getData(){ return data; } + public String getSourcePath(){ return sourcePath; } + public String getDestinationPath(){ return destPath; } - /** - * - */ - public PutObjectInput(String fileName, StoreType destStoreType, String sourcePath, String destPath, String fileExtension, ObjectState state, byte[] data){ - this.fileName = fileName; - this.destStoreType = destStoreType; - this.sourcePath = sourcePath; - this.destPath = destPath; - this.fileExtension = fileExtension; - this.data = data; - this.objectState = state; - } + } diff --git a/src/main/java/usace/cc/plugin/SeedSet.java b/src/main/java/usace/cc/plugin/SeedSet.java index 4173200..e814805 100644 --- a/src/main/java/usace/cc/plugin/SeedSet.java +++ b/src/main/java/usace/cc/plugin/SeedSet.java @@ -1,13 +1,18 @@ package usace.cc.plugin; + import com.fasterxml.jackson.annotation.JsonProperty; + public class SeedSet { @JsonProperty private long event_seed; + @JsonProperty private long realization_seed; + public long getEventSeed(){ return event_seed; } + public long getRealizationSeed(){ return realization_seed; } diff --git a/src/main/java/usace/cc/plugin/Status.java b/src/main/java/usace/cc/plugin/Status.java index 09f89bf..efecb03 100644 --- a/src/main/java/usace/cc/plugin/Status.java +++ b/src/main/java/usace/cc/plugin/Status.java @@ -8,13 +8,17 @@ enum StatusLevel { FAILED, //Status = "Failed" SUCCEEDED, //Status = "Succeeded" } + @JsonProperty private int progress; + @JsonProperty private StatusLevel status; + public int getProgress(){ return progress; } + public StatusLevel getStatus(){ return status; } diff --git a/src/main/java/usace/cc/plugin/StoreType.java b/src/main/java/usace/cc/plugin/StoreType.java index 770f681..db5bb62 100644 --- a/src/main/java/usace/cc/plugin/StoreType.java +++ b/src/main/java/usace/cc/plugin/StoreType.java @@ -4,5 +4,6 @@ public enum StoreType { S3, WS, RDBMS, - EBS + EBS, + TILEDB }