Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
target
target/
lib_managed
lib/
src_managed
resource_managed
.history
Expand All @@ -12,6 +13,7 @@ resource_managed
.bsp/
errors
project/errors
.metals
.~*#
out
.scala_dependencies
Expand Down
17 changes: 17 additions & 0 deletions helloExi/.project
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
<?xml version="1.0" encoding="UTF-8"?>
<projectDescription>
<name>helloExi</name>
<comment></comment>
<projects>
</projects>
<buildSpec>
<buildCommand>
<name>org.eclipse.jdt.core.javabuilder</name>
<arguments>
</arguments>
</buildCommand>
</buildSpec>
<natures>
<nature>org.eclipse.jdt.core.javanature</nature>
</natures>
</projectDescription>
18 changes: 18 additions & 0 deletions helloExi/README.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
This is a Daffodil/DFDL example program using Agile Delta's EXI implementation.

The build.sbt file is a build script for the sbt - simple build tool.

You will need version 0.13.x or higher.
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

No point suggesting version 0.13.x unless you want to test that far back! Just say "Tested with sbt 1.6.1" (Which you could upgrade to 1.7.2)


You will also need Java 1.8 (aka Java 8) or higher JDK.
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Say what you tested it with.


You will need an internet connection to pull down the dependent libraries, including daffodil itself.

With those installed you can type 'sbt run' and it will download all dependencies as jar files, and then compile and run the HelloWorld.java program.
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Update HelloEXI.java.

Also rephrase. Suggest part about using an AgileDelta eval/full library should go first. Given that, the rest will pull down automatically.


You will need either an evaluation or full version of the Agile Delta Efficient XML SDK, with all of it's jar files placed in the "lib/" directory.
NOTE: The standard evaluation package from Agile Delta only includes command line tools and does not include access to the SDK. You must ask for the SDK.

If you would like the source and javadoc jars for the dependencies downloaded also, then type 'sbt updateClassifiers'.

The jar files are cached in the lib_managed/ directory (which is removed by 'sbt clean')
23 changes: 23 additions & 0 deletions helloExi/build.sbt
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
name := "dfdl-helloexi"

organization := "com.tresys"

version := "0.1.0"

scalaVersion := "2.12.15"

// People use this project to study what the dependencies actually are needed
// so having them put into lib_managed is helpful.
retrieveManaged := true
useCoursier := false // Workaround becauuse retrieveManaged doesn't work in some sbt versions.

Compile / run / mainClass := Some("HelloEXI")

libraryDependencies ++= Seq(
"org.apache.daffodil" %% "daffodil-japi" % "3.2.1",
"jaxen" % "jaxen" % "1.2.0",
"junit" % "junit" % "4.13.2" % "test",
"com.github.sbt" % "junit-interface" % "0.13.2" % "test"
)

testOptions += Tests.Argument(TestFrameworks.JUnit, "-v")
1 change: 1 addition & 0 deletions helloExi/project/build.properties
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
sbt.version=1.6.1
167 changes: 167 additions & 0 deletions helloExi/src/main/java/HelloEXI.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,167 @@

import java.io.InputStream;
import java.io.OutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.net.URISyntaxException;
import java.net.URL;
import java.util.Collections;
import java.util.List;
import java.util.Map;

import org.xml.sax.ContentHandler;
import org.xml.sax.XMLReader;
import org.xml.sax.InputSource;
import org.xml.sax.SAXException;

import org.apache.daffodil.japi.Compiler;
import org.apache.daffodil.japi.Daffodil;
import org.apache.daffodil.japi.DataProcessor;
import org.apache.daffodil.japi.Diagnostic;
import org.apache.daffodil.japi.ParseResult;
import org.apache.daffodil.japi.ProcessorFactory;
import org.apache.daffodil.japi.UnparseResult;
import org.apache.daffodil.japi.DaffodilParseXMLReader;
import org.apache.daffodil.japi.DaffodilUnparseContentHandler;
import org.apache.daffodil.japi.io.InputSourceDataInputStream;
import org.apache.daffodil.japi.DaffodilUnparseErrorSAXException;

import com.agiledelta.efx.EFXProperty;
import com.agiledelta.efx.EFXFactory;
import com.agiledelta.efx.EFXException;
import com.agiledelta.efx.sax.EFXSAXSerializer;

/**
* Demonstrates using the Daffodil DFDL processor to
* <ul>
* <li>compile a DFDL schema
* <li>parse non-XML data into EXI using Agile Delta and
* <li>unparse the data back to non-XML form.
* </ul>
*/
public class HelloEXI {

public static void main(String[] args) throws IOException, URISyntaxException, EFXException {

URL schemaFileURL = HelloEXI.class.getResource("/helloWorld.dfdl.xsd");
URL dataFileURL = HelloEXI.class.getResource("/helloWorld.dat");
String exiFilePath = "src/main/resources/helloWorld.exi";
String unpFilePath = "src/main/resources/helloWorld.exi.dat";

//
// First compile the DFDL Schema
//
Compiler c = Daffodil.compiler();
ProcessorFactory pf = c.compileSource(schemaFileURL.toURI());
if (pf.isError()) {
// didn't compile schema. Must be diagnostic of some sort.
List<Diagnostic> diags = pf.getDiagnostics();
for (Diagnostic d : diags) {
System.err.println(d.getSomeMessage());
}
System.exit(1);
}
DataProcessor dp = pf.onPath("/");
if (dp.isError()) {
// didn't compile schema. Must be diagnostic of some sort.
List<Diagnostic> diags = dp.getDiagnostics();
for (Diagnostic d : diags) {
System.err.println(d.getSomeMessage());
}
System.exit(1);
}

//
// Parse - parse data to EXI
//
System.out.println("**** Parsing data into EXI *****");
InputStream is = dataFileURL.openStream();
OutputStream os = new FileOutputStream(exiFilePath);
InputSourceDataInputStream dis = new InputSourceDataInputStream(is);

//
// Setup Agile Delta EXI content handler
//
EFXFactory factory = EFXFactory.newInstance();
//SchemaResolver resolver = new SchemaResolver() {
// public SchemaSource getSchema(String schemaID, String schemaPath, String namespaceURI) throws java.io.IOException {
// }
//};
////factory.setProperty(EFXProperty.SCHEMA_RESOLVER, resolver);
//factory.setSchema(schemaId);
Comment on lines +88 to +93
Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is where I ran into an issue with schema aware EXI. Agile Delta's SDK has an interface for a SchemaResolver which would be pretty straightforward to implement using the DFDLCatalogResolver under the hood. However, the DFDLCatalogResolver is not included in the Java API of Daffodil. I'm thinking that the right move is to expose the DFDLCatalogResolver in the Java API, but wanted to bring it up for discussion first.

This was also the case when using Exificient in the Daffodil Main.scala program, but since it is written in Scala it wasn't a big deal to just import it. The DFDLCatalogResolver can still be used in Java without adding it to the Java API, but using Scala objects in Java gets a bit ugly.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We should expose the resolver in japi and sapi packages.

This really is just a pointer to the DFDLCatalogResolver which is already just using scala code but implementing a Java interface.

I.e., I would not bother with a per-method proxy/delegation.

factory.setProperty(EFXProperty.HEADER, true);
Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Doing some basic testing, it appears that Agile Delta and Exificient use different default values when encoding files. For example, compression is enabled by default in AD, but not in Exificient. Including this HEADER option adds a header to the EXI file specifying what options were used to encode it. This allows Exificient to handle AD's encoded files.

We will probably also want to enable this option in the Daffodil Main program as well when we use Exificient. In a quick test, enabling the HEADER option in this program only adds 1 byte to the outputted EXI file, which is definitely worth it to ensure compatibility between EXI implementations.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Example should illustrate how to turn on this compression feature explicitly, not just add header so defaults are recorded.

Are there other interesting options as well.

EFXSAXSerializer exiContentHandler = new EFXSAXSerializer(factory, os);
DaffodilParseXMLReader reader = dp.newXMLReaderInstance();
reader.setContentHandler(exiContentHandler);
reader.parse(dis);
os.close();

//
// Check for errors
//
ParseResult pr = (ParseResult) reader.getProperty("urn:ogf:dfdl:2013:imp:daffodil.apache.org:2018:sax:ParseResult");
boolean err = pr.isError();
if (err) {
// didn't parse the data. Must be diagnostic of some sort.
List<Diagnostic> diags = pr.getDiagnostics();
for (Diagnostic d : diags) {
System.err.println(d.getSomeMessage());
}
System.exit(2);
}

is.close();
os.close();

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think the example should show how to transform the EXI data via XSLT to say Mars instead of World, so as to show how one can feed EXI data directly to XSLT without converting to XML text first.

//
// Unparse back to native format
//

// If you need to also convert XML back into the native data format
// you need to "unparse" the infoset back to data.
//
// Not all DFDL schemas are setup for unparsing. There are some things
// you need for unparsing that just don't need to be present in the
// schema if you only intend to do parsing.
//
// But let's assume your DFDL schema is one that is able to be used both
// for parsing and unparsing data.
//
// So let's try unparsing
//
// We'll just store the result of unparsing into this
// ByteArrayOutputStream.
//
System.out.println("**** Unparsing XML infoset back into data *****");

is = new FileInputStream(exiFilePath);
FileOutputStream fos = new FileOutputStream(unpFilePath);
java.nio.channels.WritableByteChannel wbc = java.nio.channels.Channels.newChannel(fos);
XMLReader xmlReader = factory.createXMLReader();
DaffodilUnparseContentHandler handler = dp.newContentHandlerInstance(wbc);
xmlReader.setContentHandler(handler);
try {
xmlReader.parse(new InputSource(is));
} catch (SAXException ex) {
// Do nothing, error is detailed in unparse result
} finally {
is.close();
wbc.close();
fos.close();
}

UnparseResult ur = handler.getUnparseResult();
err = ur.isError();

if (err) {
// didn't unparse. Must be diagnostic of some sort.
List<Diagnostic> diags = ur.getDiagnostics();
for (Diagnostic d : diags) {
System.err.println(d.getSomeMessage());
}
System.exit(3);
}
}
}
1 change: 1 addition & 0 deletions helloExi/src/main/resources/helloWorld.dat
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Hello world!
32 changes: 32 additions & 0 deletions helloExi/src/main/resources/helloWorld.dfdl.xsd
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
<?xml version="1.0" encoding="UTF-8"?>

<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema"
xmlns:tns="http://example.com/dfdl/helloworld/"
targetNamespace="http://example.com/dfdl/helloworld/"
xmlns:dfdl="http://www.ogf.org/dfdl/dfdl-1.0/"
xmlns:fn="http://www.w3.org/2005/xpath-functions"
xmlns:daf="urn:ogf:dfdl:2013:imp:daffodil.apache.org:2018:ext">

<xs:include schemaLocation="org/apache/daffodil/xsd/DFDLGeneralFormat.dfdl.xsd" />

<xs:annotation>
<xs:appinfo source="http://www.ogf.org/dfdl/">

<dfdl:format ref="GeneralFormat"
representation="text" encoding="utf-8" lengthKind="delimited"/>

</xs:appinfo>
</xs:annotation>


<xs:element name="helloWorld">
<xs:complexType>
<xs:sequence>
<xs:element name="word" type="xs:string" maxOccurs="unbounded"
dfdl:lengthKind="delimited" dfdl:terminator="%WSP+;"
dfdl:occursCountKind="parsed"/>
</xs:sequence>
</xs:complexType>
</xs:element>

</xs:schema>
15 changes: 15 additions & 0 deletions helloExi/src/test/java/TestHelloWorld.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
import org.junit.Test;
import org.junit.Assert;

public class TestHelloEXI {

@Test
public void testMain() {
try{
HelloEXI.main(new String[0]);
}
catch(Exception e){
Assert.fail("Should not have thrown any exception");
}
}
}