-
Notifications
You must be signed in to change notification settings - Fork 10
Added sample EXI program using Agile Delta #27
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: master
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,17 @@ | ||
| <?xml version="1.0" encoding="UTF-8"?> | ||
| <projectDescription> | ||
| <name>helloExi</name> | ||
| <comment></comment> | ||
| <projects> | ||
| </projects> | ||
| <buildSpec> | ||
| <buildCommand> | ||
| <name>org.eclipse.jdt.core.javabuilder</name> | ||
| <arguments> | ||
| </arguments> | ||
| </buildCommand> | ||
| </buildSpec> | ||
| <natures> | ||
| <nature>org.eclipse.jdt.core.javanature</nature> | ||
| </natures> | ||
| </projectDescription> |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,18 @@ | ||
| This is a Daffodil/DFDL example program using Agile Delta's EXI implementation. | ||
|
|
||
| The build.sbt file is a build script for the sbt - simple build tool. | ||
|
|
||
| You will need version 0.13.x or higher. | ||
|
|
||
| You will also need Java 1.8 (aka Java 8) or higher JDK. | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Say what you tested it with. |
||
|
|
||
| You will need an internet connection to pull down the dependent libraries, including daffodil itself. | ||
|
|
||
| With those installed you can type 'sbt run' and it will download all dependencies as jar files, and then compile and run the HelloWorld.java program. | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Update HelloEXI.java. Also rephrase. Suggest part about using an AgileDelta eval/full library should go first. Given that, the rest will pull down automatically. |
||
|
|
||
| You will need either an evaluation or full version of the Agile Delta Efficient XML SDK, with all of it's jar files placed in the "lib/" directory. | ||
| NOTE: The standard evaluation package from Agile Delta only includes command line tools and does not include access to the SDK. You must ask for the SDK. | ||
|
|
||
| If you would like the source and javadoc jars for the dependencies downloaded also, then type 'sbt updateClassifiers'. | ||
|
|
||
| The jar files are cached in the lib_managed/ directory (which is removed by 'sbt clean') | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,23 @@ | ||
| name := "dfdl-helloexi" | ||
|
|
||
| organization := "com.tresys" | ||
|
|
||
| version := "0.1.0" | ||
|
|
||
| scalaVersion := "2.12.15" | ||
|
|
||
| // People use this project to study what the dependencies actually are needed | ||
| // so having them put into lib_managed is helpful. | ||
| retrieveManaged := true | ||
| useCoursier := false // Workaround becauuse retrieveManaged doesn't work in some sbt versions. | ||
|
|
||
| Compile / run / mainClass := Some("HelloEXI") | ||
|
|
||
| libraryDependencies ++= Seq( | ||
| "org.apache.daffodil" %% "daffodil-japi" % "3.2.1", | ||
| "jaxen" % "jaxen" % "1.2.0", | ||
| "junit" % "junit" % "4.13.2" % "test", | ||
| "com.github.sbt" % "junit-interface" % "0.13.2" % "test" | ||
| ) | ||
|
|
||
| testOptions += Tests.Argument(TestFrameworks.JUnit, "-v") |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1 @@ | ||
| sbt.version=1.6.1 |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,167 @@ | ||
|
|
||
| import java.io.InputStream; | ||
| import java.io.OutputStream; | ||
| import java.io.File; | ||
| import java.io.FileInputStream; | ||
| import java.io.FileOutputStream; | ||
| import java.io.IOException; | ||
| import java.net.URISyntaxException; | ||
| import java.net.URL; | ||
| import java.util.Collections; | ||
| import java.util.List; | ||
| import java.util.Map; | ||
|
|
||
| import org.xml.sax.ContentHandler; | ||
| import org.xml.sax.XMLReader; | ||
| import org.xml.sax.InputSource; | ||
| import org.xml.sax.SAXException; | ||
|
|
||
| import org.apache.daffodil.japi.Compiler; | ||
| import org.apache.daffodil.japi.Daffodil; | ||
| import org.apache.daffodil.japi.DataProcessor; | ||
| import org.apache.daffodil.japi.Diagnostic; | ||
| import org.apache.daffodil.japi.ParseResult; | ||
| import org.apache.daffodil.japi.ProcessorFactory; | ||
| import org.apache.daffodil.japi.UnparseResult; | ||
| import org.apache.daffodil.japi.DaffodilParseXMLReader; | ||
| import org.apache.daffodil.japi.DaffodilUnparseContentHandler; | ||
| import org.apache.daffodil.japi.io.InputSourceDataInputStream; | ||
| import org.apache.daffodil.japi.DaffodilUnparseErrorSAXException; | ||
|
|
||
| import com.agiledelta.efx.EFXProperty; | ||
| import com.agiledelta.efx.EFXFactory; | ||
| import com.agiledelta.efx.EFXException; | ||
| import com.agiledelta.efx.sax.EFXSAXSerializer; | ||
|
|
||
| /** | ||
| * Demonstrates using the Daffodil DFDL processor to | ||
| * <ul> | ||
| * <li>compile a DFDL schema | ||
| * <li>parse non-XML data into EXI using Agile Delta and | ||
| * <li>unparse the data back to non-XML form. | ||
| * </ul> | ||
| */ | ||
| public class HelloEXI { | ||
|
|
||
| public static void main(String[] args) throws IOException, URISyntaxException, EFXException { | ||
|
|
||
| URL schemaFileURL = HelloEXI.class.getResource("/helloWorld.dfdl.xsd"); | ||
| URL dataFileURL = HelloEXI.class.getResource("/helloWorld.dat"); | ||
| String exiFilePath = "src/main/resources/helloWorld.exi"; | ||
| String unpFilePath = "src/main/resources/helloWorld.exi.dat"; | ||
|
|
||
| // | ||
| // First compile the DFDL Schema | ||
| // | ||
| Compiler c = Daffodil.compiler(); | ||
| ProcessorFactory pf = c.compileSource(schemaFileURL.toURI()); | ||
| if (pf.isError()) { | ||
| // didn't compile schema. Must be diagnostic of some sort. | ||
| List<Diagnostic> diags = pf.getDiagnostics(); | ||
| for (Diagnostic d : diags) { | ||
| System.err.println(d.getSomeMessage()); | ||
| } | ||
| System.exit(1); | ||
| } | ||
| DataProcessor dp = pf.onPath("/"); | ||
| if (dp.isError()) { | ||
| // didn't compile schema. Must be diagnostic of some sort. | ||
| List<Diagnostic> diags = dp.getDiagnostics(); | ||
| for (Diagnostic d : diags) { | ||
| System.err.println(d.getSomeMessage()); | ||
| } | ||
| System.exit(1); | ||
| } | ||
|
|
||
| // | ||
| // Parse - parse data to EXI | ||
| // | ||
| System.out.println("**** Parsing data into EXI *****"); | ||
| InputStream is = dataFileURL.openStream(); | ||
| OutputStream os = new FileOutputStream(exiFilePath); | ||
| InputSourceDataInputStream dis = new InputSourceDataInputStream(is); | ||
|
|
||
| // | ||
| // Setup Agile Delta EXI content handler | ||
| // | ||
| EFXFactory factory = EFXFactory.newInstance(); | ||
| //SchemaResolver resolver = new SchemaResolver() { | ||
| // public SchemaSource getSchema(String schemaID, String schemaPath, String namespaceURI) throws java.io.IOException { | ||
| // } | ||
| //}; | ||
| ////factory.setProperty(EFXProperty.SCHEMA_RESOLVER, resolver); | ||
| //factory.setSchema(schemaId); | ||
|
Comment on lines
+88
to
+93
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This is where I ran into an issue with schema aware EXI. Agile Delta's SDK has an interface for a SchemaResolver which would be pretty straightforward to implement using the DFDLCatalogResolver under the hood. However, the DFDLCatalogResolver is not included in the Java API of Daffodil. I'm thinking that the right move is to expose the DFDLCatalogResolver in the Java API, but wanted to bring it up for discussion first. This was also the case when using Exificient in the Daffodil Main.scala program, but since it is written in Scala it wasn't a big deal to just import it. The DFDLCatalogResolver can still be used in Java without adding it to the Java API, but using Scala objects in Java gets a bit ugly.
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. We should expose the resolver in japi and sapi packages. This really is just a pointer to the DFDLCatalogResolver which is already just using scala code but implementing a Java interface. I.e., I would not bother with a per-method proxy/delegation. |
||
| factory.setProperty(EFXProperty.HEADER, true); | ||
|
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Doing some basic testing, it appears that Agile Delta and Exificient use different default values when encoding files. For example, compression is enabled by default in AD, but not in Exificient. Including this HEADER option adds a header to the EXI file specifying what options were used to encode it. This allows Exificient to handle AD's encoded files. We will probably also want to enable this option in the Daffodil Main program as well when we use Exificient. In a quick test, enabling the HEADER option in this program only adds 1 byte to the outputted EXI file, which is definitely worth it to ensure compatibility between EXI implementations.
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Example should illustrate how to turn on this compression feature explicitly, not just add header so defaults are recorded. Are there other interesting options as well. |
||
| EFXSAXSerializer exiContentHandler = new EFXSAXSerializer(factory, os); | ||
| DaffodilParseXMLReader reader = dp.newXMLReaderInstance(); | ||
| reader.setContentHandler(exiContentHandler); | ||
| reader.parse(dis); | ||
| os.close(); | ||
|
|
||
| // | ||
| // Check for errors | ||
| // | ||
| ParseResult pr = (ParseResult) reader.getProperty("urn:ogf:dfdl:2013:imp:daffodil.apache.org:2018:sax:ParseResult"); | ||
| boolean err = pr.isError(); | ||
| if (err) { | ||
| // didn't parse the data. Must be diagnostic of some sort. | ||
| List<Diagnostic> diags = pr.getDiagnostics(); | ||
| for (Diagnostic d : diags) { | ||
| System.err.println(d.getSomeMessage()); | ||
| } | ||
| System.exit(2); | ||
| } | ||
|
|
||
| is.close(); | ||
| os.close(); | ||
|
|
||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I think the example should show how to transform the EXI data via XSLT to say Mars instead of World, so as to show how one can feed EXI data directly to XSLT without converting to XML text first. |
||
| // | ||
| // Unparse back to native format | ||
| // | ||
|
|
||
| // If you need to also convert XML back into the native data format | ||
| // you need to "unparse" the infoset back to data. | ||
| // | ||
| // Not all DFDL schemas are setup for unparsing. There are some things | ||
| // you need for unparsing that just don't need to be present in the | ||
| // schema if you only intend to do parsing. | ||
| // | ||
| // But let's assume your DFDL schema is one that is able to be used both | ||
| // for parsing and unparsing data. | ||
| // | ||
| // So let's try unparsing | ||
| // | ||
| // We'll just store the result of unparsing into this | ||
| // ByteArrayOutputStream. | ||
| // | ||
| System.out.println("**** Unparsing XML infoset back into data *****"); | ||
|
|
||
| is = new FileInputStream(exiFilePath); | ||
| FileOutputStream fos = new FileOutputStream(unpFilePath); | ||
| java.nio.channels.WritableByteChannel wbc = java.nio.channels.Channels.newChannel(fos); | ||
| XMLReader xmlReader = factory.createXMLReader(); | ||
| DaffodilUnparseContentHandler handler = dp.newContentHandlerInstance(wbc); | ||
| xmlReader.setContentHandler(handler); | ||
| try { | ||
| xmlReader.parse(new InputSource(is)); | ||
| } catch (SAXException ex) { | ||
| // Do nothing, error is detailed in unparse result | ||
| } finally { | ||
| is.close(); | ||
| wbc.close(); | ||
| fos.close(); | ||
| } | ||
|
|
||
| UnparseResult ur = handler.getUnparseResult(); | ||
| err = ur.isError(); | ||
|
|
||
| if (err) { | ||
| // didn't unparse. Must be diagnostic of some sort. | ||
| List<Diagnostic> diags = ur.getDiagnostics(); | ||
| for (Diagnostic d : diags) { | ||
| System.err.println(d.getSomeMessage()); | ||
| } | ||
| System.exit(3); | ||
| } | ||
| } | ||
| } | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1 @@ | ||
| Hello world! |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,32 @@ | ||
| <?xml version="1.0" encoding="UTF-8"?> | ||
|
|
||
| <xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema" | ||
| xmlns:tns="http://example.com/dfdl/helloworld/" | ||
| targetNamespace="http://example.com/dfdl/helloworld/" | ||
| xmlns:dfdl="http://www.ogf.org/dfdl/dfdl-1.0/" | ||
| xmlns:fn="http://www.w3.org/2005/xpath-functions" | ||
| xmlns:daf="urn:ogf:dfdl:2013:imp:daffodil.apache.org:2018:ext"> | ||
|
|
||
| <xs:include schemaLocation="org/apache/daffodil/xsd/DFDLGeneralFormat.dfdl.xsd" /> | ||
|
|
||
| <xs:annotation> | ||
| <xs:appinfo source="http://www.ogf.org/dfdl/"> | ||
|
|
||
| <dfdl:format ref="GeneralFormat" | ||
| representation="text" encoding="utf-8" lengthKind="delimited"/> | ||
|
|
||
| </xs:appinfo> | ||
| </xs:annotation> | ||
|
|
||
|
|
||
| <xs:element name="helloWorld"> | ||
| <xs:complexType> | ||
| <xs:sequence> | ||
| <xs:element name="word" type="xs:string" maxOccurs="unbounded" | ||
| dfdl:lengthKind="delimited" dfdl:terminator="%WSP+;" | ||
| dfdl:occursCountKind="parsed"/> | ||
| </xs:sequence> | ||
| </xs:complexType> | ||
| </xs:element> | ||
|
|
||
| </xs:schema> |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,15 @@ | ||
| import org.junit.Test; | ||
| import org.junit.Assert; | ||
|
|
||
| public class TestHelloEXI { | ||
|
|
||
| @Test | ||
| public void testMain() { | ||
| try{ | ||
| HelloEXI.main(new String[0]); | ||
| } | ||
| catch(Exception e){ | ||
| Assert.fail("Should not have thrown any exception"); | ||
| } | ||
| } | ||
| } |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
No point suggesting version 0.13.x unless you want to test that far back! Just say "Tested with sbt 1.6.1" (Which you could upgrade to 1.7.2)