From 29086d32293534cdf631a871687abe9dbe0693d4 Mon Sep 17 00:00:00 2001 From: Nathan Baltzell Date: Wed, 8 Oct 2025 20:53:19 -0400 Subject: [PATCH 01/30] add new YAML file --- etc/services/rgl-clarode.yml | 57 ++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 etc/services/rgl-clarode.yml diff --git a/etc/services/rgl-clarode.yml b/etc/services/rgl-clarode.yml new file mode 100644 index 0000000000..cbbac41ac2 --- /dev/null +++ b/etc/services/rgl-clarode.yml @@ -0,0 +1,57 @@ +mime-types: + - binary/data-hipo +configuration: + global: + variation: rge_spring2024 + timestamp: 10/08/2025-17:00:00 + io-services: + writer: + schema_dir: "/scigroup/cvmfs/hallb/clas12/sw/noarch/coatjava/13.3.0/etc/bankdefs/hipo4/singles/mon" + services: + MAGFIELDS: + magfieldSolenoidMap: Symm_solenoid_r601_phi1_z1201_13June2018.dat + magfieldTorusMap: Full_torus_r251_phi181_z251_25Jan2021.dat +io-services: + reader: + class: org.jlab.io.clara.DecoderReader + name: DecoderReader + writer: + class: org.jlab.io.clara.DecoderWriter + name: DecoderWriter +services: + - class: org.jlab.clas.swimtools.MagFieldsEngine + name: MAGFIELDS + - class: org.jlab.rec.ft.cal.FTCALEngine + name: FTCAL + - class: org.jlab.rec.ft.hodo.FTHODOEngine + name: FTHODO + - class: org.jlab.rec.ft.FTEBEngine + name: FTEB + - class: org.jlab.service.dc.DCHBClustering + name: DCCR + - class: org.jlab.service.dc.DCHBPostClusterConv + name: DCHB + - class: org.jlab.service.ftof.FTOFHBEngine + name: FTOFHB + - class: org.jlab.service.ec.ECEngine + name: EC + - class: org.jlab.service.ctof.CTOFEngine + name: CTOF + - class: org.jlab.service.cnd.CNDCalibrationEngine + name: CND + - class: org.jlab.service.band.BANDEngine + name: BAND + - class: org.jlab.service.htcc.HTCCReconstructionService + name: HTCC + - class: org.jlab.service.ltcc.LTCCEngine + name: LTCC + - class: org.jlab.service.eb.EBHBEngine + name: EBHB + - class: org.jlab.service.dc.DCTBEngine + name: DCTB + - class: org.jlab.service.ftof.FTOFTBEngine + name: FTOFTB + - class: org.jlab.service.eb.EBTBEngine + name: EBTB + - class: org.jlab.rec.rich.RICHEBEngine + name: RICH From c43cc361d285628a798f437442f0f7e779cdaa5a Mon Sep 17 00:00:00 2001 From: Nathan Baltzell Date: Fri, 10 Oct 2025 17:40:47 -0400 Subject: [PATCH 02/30] use stock schema in yamls --- .github/workflows/ci.yml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b115ec2670..07cbac1a73 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -154,6 +154,29 @@ jobs: run: | ./coatjava/bin/decoder -n 10000 -o dog.hipo ./clas_005038.evio.00000 + test_clara: + needs: [ build ] + strategy: + fail-fast: true + steps: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + - name: Set up JDK + uses: actions/setup-java@v5 + with: + java-version: ${{ env.java_version }} + distribution: ${{ env.java_distribution }} + - uses: actions/download-artifact@v5 + with: + name: build_${{ matrix.runner }} + - name: untar build + run: tar xzvf coatjava.tar.gz + - name: run test + run: | + cd validation/advanced-tests + ls -lhtr + test_coatjava: needs: [ build ] strategy: From 2853f7d71cdbebe5b6239cc8286470a3ff71631b Mon Sep 17 00:00:00 2001 From: Nathan Baltzell Date: Fri, 10 Oct 2025 19:14:08 -0400 Subject: [PATCH 03/30] .gitignore clara installation --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index e06912d790..d88c305087 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ # installed: /coatjava +/clara target /publish /docs/javadoc/* From cd3119dfbf717075655749e2b32430dc6aef347e Mon Sep 17 00:00:00 2001 From: Nathan Baltzell Date: Fri, 10 Oct 2025 19:14:15 -0400 Subject: [PATCH 04/30] run clara test --- .github/workflows/ci.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 07cbac1a73..7bc5fce05e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,7 +39,9 @@ jobs: id: cache with: key: raw_test_data # fixed key will always hit; clear cache to trigger cache miss - path: clas_005038.evio.00000 + path: | + clas_005038.evio.00000 + clas_021559.evio.00000 lookup-only: true - name: install xrootd-client if: ${{ steps.cache.outputs.cache-hit != 'true' }} @@ -50,6 +52,7 @@ jobs: if: ${{ steps.cache.outputs.cache-hit != 'true' }} run: | xrdcp xroot://sci-xrootd.jlab.org///osgpool/hallb/clas12/validation/clas_005038.evio.00000 ./ + xrdcp xroot://sci-xrootd.jlab.org///osgpool/hallb/clas12/validation/raw/rg-l/clas_021559.evio.00000 ./ # build ############################################################################# @@ -171,10 +174,11 @@ jobs: with: name: build_${{ matrix.runner }} - name: untar build - run: tar xzvf coatjava.tar.gz + run: | + tar xzvf clara.tar.gz - name: run test run: | - cd validation/advanced-tests + ./coatjava/bin/run-clara -y ./etc/services/rgl-clarode.yml ${{ env.nthreads }} -n 1000 -c ./clara ./clas_021559.evio.00000 ls -lhtr test_coatjava: From 98be61a05dd32466016fe5eada34541ff4411801 Mon Sep 17 00:00:00 2001 From: Nathan Baltzell Date: Fri, 10 Oct 2025 19:17:59 -0400 Subject: [PATCH 05/30] disable validation workflows --- .github/workflows/validation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/validation.yml b/.github/workflows/validation.yml index 6c42d6d924..6a521c8705 100644 --- a/.github/workflows/validation.yml +++ b/.github/workflows/validation.yml @@ -3,7 +3,7 @@ name: Validation on: pull_request: push: - branches: [ development ] + branches: [ validation ] tags: [ '*' ] workflow_dispatch: From f160e73a5329c312b915c537ff395f84e0f34c8a Mon Sep 17 00:00:00 2001 From: Nathan Baltzell Date: Fri, 10 Oct 2025 19:30:10 -0400 Subject: [PATCH 06/30] add denoising engine --- etc/services/rgl-clarode.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/etc/services/rgl-clarode.yml b/etc/services/rgl-clarode.yml index cbbac41ac2..a0796f97be 100644 --- a/etc/services/rgl-clarode.yml +++ b/etc/services/rgl-clarode.yml @@ -19,6 +19,8 @@ io-services: class: org.jlab.io.clara.DecoderWriter name: DecoderWriter services: + - class: org.jlab.service.ai.DCDenoiseEngine + name: DCDN - class: org.jlab.clas.swimtools.MagFieldsEngine name: MAGFIELDS - class: org.jlab.rec.ft.cal.FTCALEngine From 983bc1930a7bdb065242a3188f754eb501316de5 Mon Sep 17 00:00:00 2001 From: Nathan Baltzell Date: Fri, 10 Oct 2025 19:43:42 -0400 Subject: [PATCH 07/30] disable validatin --- .github/workflows/validation.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/validation.yml b/.github/workflows/validation.yml index 6a521c8705..0f48339155 100644 --- a/.github/workflows/validation.yml +++ b/.github/workflows/validation.yml @@ -1,11 +1,11 @@ name: Validation -on: - pull_request: - push: - branches: [ validation ] - tags: [ '*' ] - workflow_dispatch: +#on: +# pull_request: +# push: +# branches: [ development ] +# tags: [ '*' ] +# workflow_dispatch: jobs: validation: From 39298705b04e89300095848d5300d11093283992 Mon Sep 17 00:00:00 2001 From: Nathan Baltzell Date: Fri, 10 Oct 2025 21:01:52 -0400 Subject: [PATCH 08/30] disable validation --- .../workflows/{validation.yml => .validation.yml} | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) rename .github/workflows/{validation.yml => .validation.yml} (76%) diff --git a/.github/workflows/validation.yml b/.github/workflows/.validation.yml similarity index 76% rename from .github/workflows/validation.yml rename to .github/workflows/.validation.yml index 0f48339155..6c42d6d924 100644 --- a/.github/workflows/validation.yml +++ b/.github/workflows/.validation.yml @@ -1,11 +1,11 @@ name: Validation -#on: -# pull_request: -# push: -# branches: [ development ] -# tags: [ '*' ] -# workflow_dispatch: +on: + pull_request: + push: + branches: [ development ] + tags: [ '*' ] + workflow_dispatch: jobs: validation: From 677a0708844ef16096b0d4bdc830a25d29bb91d6 Mon Sep 17 00:00:00 2001 From: Nathan Baltzell Date: Mon, 13 Oct 2025 16:03:52 -0400 Subject: [PATCH 09/30] reactivate --- .github/workflows/{.validation.yml => validation.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{.validation.yml => validation.yml} (100%) diff --git a/.github/workflows/.validation.yml b/.github/workflows/validation.yml similarity index 100% rename from .github/workflows/.validation.yml rename to .github/workflows/validation.yml From 9f6805d0c4bddae0332e123bf472ddb04b1d2a5a Mon Sep 17 00:00:00 2001 From: Nathan Baltzell Date: Mon, 13 Oct 2025 16:07:24 -0400 Subject: [PATCH 10/30] schema dir in yaml --- etc/services/rgl-clarode.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/services/rgl-clarode.yml b/etc/services/rgl-clarode.yml index a0796f97be..ae762980a7 100644 --- a/etc/services/rgl-clarode.yml +++ b/etc/services/rgl-clarode.yml @@ -6,7 +6,7 @@ configuration: timestamp: 10/08/2025-17:00:00 io-services: writer: - schema_dir: "/scigroup/cvmfs/hallb/clas12/sw/noarch/coatjava/13.3.0/etc/bankdefs/hipo4/singles/mon" + schema_dir: full services: MAGFIELDS: magfieldSolenoidMap: Symm_solenoid_r601_phi1_z1201_13June2018.dat From d177892fce66d68b32134d09c425a327ac555e17 Mon Sep 17 00:00:00 2001 From: Nathan Baltzell Date: Mon, 13 Oct 2025 18:07:55 -0400 Subject: [PATCH 11/30] set memory limit --- bin/run-clara | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/run-clara b/bin/run-clara index 29e20b8a6e..4d9260e16a 100755 --- a/bin/run-clara +++ b/bin/run-clara @@ -72,7 +72,7 @@ done [ $(cat $CLARA_USER_DATA/filelist.txt | wc -l) -gt 0 ] || error "Found no input files" # Set some JVM options: -export JAVA_OPTS="$JAVA_OPTS -XX:+IgnoreUnrecognizedVMOptions" +export JAVA_OPTS="$JAVA_OPTS -Xmx${threads}g -XX:+IgnoreUnrecognizedVMOptions" export JAVA_OPTS="$JAVA_OPTS -Djava.io.tmpdir=$CLARA_USER_DATA -Dorg.sqlite.tmpdir=$CLARA_USER_DATA" # Set verbosity: [ -z ${quiet+x} ] && stub=fine || stub=info From 3338f19fce16497799d3e8eb784f3a32c5d14951 Mon Sep 17 00:00:00 2001 From: Nathan Baltzell Date: Mon, 13 Oct 2025 18:08:25 -0400 Subject: [PATCH 12/30] keep memory contiguous --- .../ai/src/main/java/org/jlab/service/ai/DCDenoiseEngine.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/reconstruction/ai/src/main/java/org/jlab/service/ai/DCDenoiseEngine.java b/reconstruction/ai/src/main/java/org/jlab/service/ai/DCDenoiseEngine.java index 12560c7a95..b530438b2d 100644 --- a/reconstruction/ai/src/main/java/org/jlab/service/ai/DCDenoiseEngine.java +++ b/reconstruction/ai/src/main/java/org/jlab/service/ai/DCDenoiseEngine.java @@ -17,7 +17,7 @@ import ai.djl.translate.TranslateException; import java.io.IOException; import java.util.concurrent.BlockingQueue; -import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ArrayBlockingQueue; import org.jlab.clas.reco.ReconstructionEngine; import org.jlab.io.base.DataBank; @@ -39,7 +39,7 @@ public class DCDenoiseEngine extends ReconstructionEngine { public static class PredictorPool { final BlockingQueue pool; public PredictorPool(int size, ZooModel model) { - pool = new LinkedBlockingQueue<>(size); + pool = new ArrayBlockingQueue<>(size); for (int i=0; i Date: Mon, 13 Oct 2025 18:42:35 -0400 Subject: [PATCH 13/30] benchmark --- .../src/main/java/org/jlab/service/ai/DCDenoiseEngine.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/reconstruction/ai/src/main/java/org/jlab/service/ai/DCDenoiseEngine.java b/reconstruction/ai/src/main/java/org/jlab/service/ai/DCDenoiseEngine.java index b530438b2d..eda1b98b3f 100644 --- a/reconstruction/ai/src/main/java/org/jlab/service/ai/DCDenoiseEngine.java +++ b/reconstruction/ai/src/main/java/org/jlab/service/ai/DCDenoiseEngine.java @@ -22,6 +22,7 @@ import org.jlab.clas.reco.ReconstructionEngine; import org.jlab.io.base.DataBank; import org.jlab.io.base.DataEvent; +import org.jlab.utils.benchmark.Benchmark; import org.jlab.utils.system.ClasUtilsFile; public class DCDenoiseEngine extends ReconstructionEngine { @@ -85,10 +86,14 @@ public boolean processDataEvent(DataEvent event) { DataBank bank = event.getBank(BANK_NAMES[i]); try { // WARNING: Predictor is *not* thread safe. + Benchmark.getInstance().resume("GETPRED"); Predictor predictor = predictors.get(); + Benchmark.getInstance().pause("GETPRED"); for (int sector=0; sector<6; sector++) { float[][] input = DCDenoiseEngine.read(bank, sector+1); + Benchmark.getInstance().resume("PREDICT"); float[][] output = predictor.predict(input); + Benchmark.getInstance().pause("PREDICT"); //System.out.println("IN:");show(input); //System.out.println("OUT:");show(output); update(bank, threshold, output, sector); From 8e2bd6e01878397d84255eb7256514853dfcfd84 Mon Sep 17 00:00:00 2001 From: Nathan Baltzell Date: Mon, 13 Oct 2025 18:42:45 -0400 Subject: [PATCH 14/30] fail fast --- .../ai/src/main/java/org/jlab/service/ai/DCDenoiseEngine.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reconstruction/ai/src/main/java/org/jlab/service/ai/DCDenoiseEngine.java b/reconstruction/ai/src/main/java/org/jlab/service/ai/DCDenoiseEngine.java index eda1b98b3f..1c7b5ef228 100644 --- a/reconstruction/ai/src/main/java/org/jlab/service/ai/DCDenoiseEngine.java +++ b/reconstruction/ai/src/main/java/org/jlab/service/ai/DCDenoiseEngine.java @@ -41,7 +41,7 @@ public static class PredictorPool { final BlockingQueue pool; public PredictorPool(int size, ZooModel model) { pool = new ArrayBlockingQueue<>(size); - for (int i=0; i Date: Mon, 13 Oct 2025 18:50:56 -0400 Subject: [PATCH 15/30] let it be null, no waiting --- .../ai/src/main/java/org/jlab/service/ai/DCDenoiseEngine.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reconstruction/ai/src/main/java/org/jlab/service/ai/DCDenoiseEngine.java b/reconstruction/ai/src/main/java/org/jlab/service/ai/DCDenoiseEngine.java index 1c7b5ef228..4a15461cea 100644 --- a/reconstruction/ai/src/main/java/org/jlab/service/ai/DCDenoiseEngine.java +++ b/reconstruction/ai/src/main/java/org/jlab/service/ai/DCDenoiseEngine.java @@ -44,7 +44,7 @@ public PredictorPool(int size, ZooModel model) { for (int i=0; i Date: Mon, 13 Oct 2025 18:52:24 -0400 Subject: [PATCH 16/30] configure threads --- .../ai/src/main/java/org/jlab/service/ai/DCDenoiseEngine.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/reconstruction/ai/src/main/java/org/jlab/service/ai/DCDenoiseEngine.java b/reconstruction/ai/src/main/java/org/jlab/service/ai/DCDenoiseEngine.java index 4a15461cea..4d9c240c97 100644 --- a/reconstruction/ai/src/main/java/org/jlab/service/ai/DCDenoiseEngine.java +++ b/reconstruction/ai/src/main/java/org/jlab/service/ai/DCDenoiseEngine.java @@ -29,6 +29,7 @@ public class DCDenoiseEngine extends ReconstructionEngine { final static String[] BANK_NAMES = {"DC::tot","DC::tdc"}; final static String CONF_THRESHOLD = "threshold"; + final static String CONF_THREADS = "threads"; final static int LAYERS = 36; final static int WIRES = 112; @@ -68,7 +69,8 @@ public boolean init() { .optProgress(new ProgressBar()) .build(); model = criteria.loadModel(); - predictors = new PredictorPool(64, model); + int threads = Integer.parseInt(getEngineConfigString(CONF_THREADS,"64")); + predictors = new PredictorPool(threads, model); return true; } catch (NullPointerException | MalformedModelException | IOException | ModelNotFoundException ex) { System.getLogger(DCDenoiseEngine.class.getName()).log(System.Logger.Level.ERROR, (String) null, ex); From 4f411d051cf6a61b975d6a5324eadcca6b49f58e Mon Sep 17 00:00:00 2001 From: Nathan Baltzell Date: Mon, 13 Oct 2025 19:10:53 -0400 Subject: [PATCH 17/30] fall back to $HOME --- .../src/main/java/org/jlab/utils/system/ClasUtilsFile.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/common-tools/clas-utils/src/main/java/org/jlab/utils/system/ClasUtilsFile.java b/common-tools/clas-utils/src/main/java/org/jlab/utils/system/ClasUtilsFile.java index d47d2eb9c3..e6da2bdffa 100644 --- a/common-tools/clas-utils/src/main/java/org/jlab/utils/system/ClasUtilsFile.java +++ b/common-tools/clas-utils/src/main/java/org/jlab/utils/system/ClasUtilsFile.java @@ -60,6 +60,9 @@ public static String getCoatjavaRuntimeDir() { ret = "/" + String.join("/", Arrays.copyOfRange(d,0,d.length-3)); } + if (!(new File(ret)).isDirectory()) { + ret = System.getenv("HOME")+"/sw/coatjava/coatjava"; + } return ret; } From de6677da860db2c09aa3fe7d95ea8f63cc9c2904 Mon Sep 17 00:00:00 2001 From: Nathan Baltzell Date: Mon, 13 Oct 2025 19:11:12 -0400 Subject: [PATCH 18/30] add test --- .../main/java/org/jlab/service/ai/DCDenoiseEngine.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/reconstruction/ai/src/main/java/org/jlab/service/ai/DCDenoiseEngine.java b/reconstruction/ai/src/main/java/org/jlab/service/ai/DCDenoiseEngine.java index 4d9c240c97..26152d93e1 100644 --- a/reconstruction/ai/src/main/java/org/jlab/service/ai/DCDenoiseEngine.java +++ b/reconstruction/ai/src/main/java/org/jlab/service/ai/DCDenoiseEngine.java @@ -78,6 +78,14 @@ public boolean init() { } } + public static void main(String args[]){ + DCDenoiseEngine dn = new DCDenoiseEngine(); + dn.init(); + for (int i=0; i<100; i++) { + dn.processFakeEvent(); + } + } + @Override public boolean processDataEvent(DataEvent event) { From 946a0a40bac1532789837a879fc449104a9331f3 Mon Sep 17 00:00:00 2001 From: Nathan Baltzell Date: Mon, 13 Oct 2025 19:24:05 -0400 Subject: [PATCH 19/30] single-threaded --- .../src/main/java/org/jlab/service/ai/DCDenoiseEngine.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/reconstruction/ai/src/main/java/org/jlab/service/ai/DCDenoiseEngine.java b/reconstruction/ai/src/main/java/org/jlab/service/ai/DCDenoiseEngine.java index 26152d93e1..5e1bac43c5 100644 --- a/reconstruction/ai/src/main/java/org/jlab/service/ai/DCDenoiseEngine.java +++ b/reconstruction/ai/src/main/java/org/jlab/service/ai/DCDenoiseEngine.java @@ -58,6 +58,9 @@ public DCDenoiseEngine() { @Override public boolean init() { + System.setProperty("ai.djl.pytorch.num_interop_threads", "1"); + System.setProperty("ai.djl.pytorch.num_threads", "1"); + System.setProperty("ai.djl.pytorch.graph_optimizer", "false"); if (getEngineConfigString(CONF_THRESHOLD) != null) threshold = Float.parseFloat(getEngineConfigString(CONF_THRESHOLD)); try { @@ -81,7 +84,7 @@ public boolean init() { public static void main(String args[]){ DCDenoiseEngine dn = new DCDenoiseEngine(); dn.init(); - for (int i=0; i<100; i++) { + for (int i=0; i<10000; i++) { dn.processFakeEvent(); } } From 26cb9b8b5c4e3be3af0c97802b60c7dd73639020 Mon Sep 17 00:00:00 2001 From: Nathan Baltzell Date: Mon, 13 Oct 2025 20:10:30 -0400 Subject: [PATCH 20/30] reduce verbosity --- install-clara | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/install-clara b/install-clara index e5baec09d0..876d0c408a 100755 --- a/install-clara +++ b/install-clara @@ -34,7 +34,7 @@ function litter() { then echo -e "\nSee temporary build directory for more info:\n\t$tmp_dir" else - rm -rf $tmp_dir && echo -e "\nRemoved temporary build directory:\n\t$tmp_dir" + rm -rf $tmp_dir fi fi } @@ -117,7 +117,6 @@ fi tmp_dir=$(mktemp -d tmp.install-clara.XXXXXX) [ $? -ne 0 ] && error "Cannot create temporary directory: $tmp_dir" tmp_dir=$(cd $tmp_dir && pwd) -echo -e "\nCreating temporary build directory:\n\t$tmp_dir" cd $tmp_dir # CLARA: From 245d1425d5e8966a3809f127472930c6359e7286 Mon Sep 17 00:00:00 2001 From: Nathan Baltzell Date: Tue, 14 Oct 2025 10:58:16 -0400 Subject: [PATCH 21/30] add gitlab clara test --- .gitlab-ci.yml | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c2dd8719b6..4e0a287380 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -23,13 +23,15 @@ stages: build: stage: build script: - - ./build-coatjava.sh -T$JL_RUNNER_AVAIL_CPU --quiet --no-progress + - ./build-coatjava.sh --clara -T$JL_RUNNER_AVAIL_CPU --quiet --no-progress - tar -czf coatjava.tar.gz coatjava + - tar -czf clara.tar.gz clara artifacts: when: always expire_in: 1 day paths: - coatjava.tar.gz + - clara.tar.gz depana: allow_failure: true @@ -38,6 +40,18 @@ depana: - libexec/dependency-tree.sh - libexec/dependency-analysis.sh +download: + stage: build + script: + - xrdcp xroot://sci-xrootd.jlab.org///osgpool/hallb/clas12/validation/clas_005038.evio.00000 ./ + - xrdcp xroot://sci-xrootd.jlab.org///osgpool/hallb/clas12/validation/raw/rg-l/clas_021559.evio.00000 ./ + artifacts: + when: always + expire_in: 1 day + paths: + - clas_005038.evio.00000 + - clas_021559.evio.00000 + spotbugs: stage: test needs: [build] @@ -100,6 +114,18 @@ eb: - ARG: electronneutronC - ARG: electronFTpion +clara: + allow_failure: true + stage: test + needs: [build] + dependencies: [build] + script: + - > + tar -xzf clara.tar.gz && + ./clara/plugins/clas12/bin/run-clara -c ./clara + -t $JL_RUNNER_AVAIL_CPU -y ./etc/services/rgl-clarode.yml + -n 100 -o ./out clas_02*.hipo + profile: stage: test needs: [build] From 1378a9eacbf8cc38feb19a26b325cbc25ff0949d Mon Sep 17 00:00:00 2001 From: Nathan Baltzell Date: Tue, 14 Oct 2025 15:54:05 -0400 Subject: [PATCH 22/30] fix ci test --- .github/workflows/ci.yml | 24 ++++++++++++++++-------- .gitlab-ci.yml | 4 ++-- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7bc5fce05e..99a0c88608 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -52,7 +52,7 @@ jobs: if: ${{ steps.cache.outputs.cache-hit != 'true' }} run: | xrdcp xroot://sci-xrootd.jlab.org///osgpool/hallb/clas12/validation/clas_005038.evio.00000 ./ - xrdcp xroot://sci-xrootd.jlab.org///osgpool/hallb/clas12/validation/raw/rg-l/clas_021559.evio.00000 ./ + xrdcp xroot://sci-xrootd.jlab.org///osgpool/hallb/clas12/validation/raw/rg-l/clas_021559.evio.00001 ./ # build ############################################################################# @@ -89,14 +89,18 @@ jobs: if: ${{ github.ref_type == 'tag' }} run: libexec/version-bump.sh ${{ github.ref_name }} - name: build - run: ./build-coatjava.sh --xrootd --no-progress -T${{ env.nthreads }} + run: ./build-coatjava.sh --clara --xrootd --no-progress -T${{ env.nthreads }} - name: tar # tarball to preserve permissions - run: tar czvf coatjava.tar.gz coatjava + run: | + tar czvf coatjava.tar.gz coatjava + tar czvf clara.tar.gz clara - uses: actions/upload-artifact@v4 with: name: build_${{ matrix.runner }} retention-days: 1 - path: coatjava.tar.gz + path: | + coatjava.tar.gz + clara.tar.gz # tests ############################################################################# @@ -158,10 +162,9 @@ jobs: ./coatjava/bin/decoder -n 10000 -o dog.hipo ./clas_005038.evio.00000 test_clara: - needs: [ build ] + needs: [ build, download_test_data ] strategy: fail-fast: true - steps: runs-on: ubuntu-latest steps: - uses: actions/checkout@v5 @@ -172,13 +175,18 @@ jobs: distribution: ${{ env.java_distribution }} - uses: actions/download-artifact@v5 with: - name: build_${{ matrix.runner }} + name: build_ubuntu-latest + - uses: actions/cache/restore@v4 + with: + key: raw_test_data + path: clas_021559.evio.00001 - name: untar build run: | tar xzvf clara.tar.gz - name: run test run: | - ./coatjava/bin/run-clara -y ./etc/services/rgl-clarode.yml ${{ env.nthreads }} -n 1000 -c ./clara ./clas_021559.evio.00000 + ls -lhtr + ./clara/plugins/clas12/bin/run-clara -y ./etc/services/rgl-clarode.yml -t ${{ env.nthreads }} -n 200 -c ./clara -o ./tmp ./clas_021559.evio.00001 ls -lhtr test_coatjava: diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4e0a287380..69a6b73db0 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -44,7 +44,7 @@ download: stage: build script: - xrdcp xroot://sci-xrootd.jlab.org///osgpool/hallb/clas12/validation/clas_005038.evio.00000 ./ - - xrdcp xroot://sci-xrootd.jlab.org///osgpool/hallb/clas12/validation/raw/rg-l/clas_021559.evio.00000 ./ + - xrdcp xroot://sci-xrootd.jlab.org///osgpool/hallb/clas12/validation/raw/rg-l/clas_021559.evio.00001 ./ artifacts: when: always expire_in: 1 day @@ -124,7 +124,7 @@ clara: tar -xzf clara.tar.gz && ./clara/plugins/clas12/bin/run-clara -c ./clara -t $JL_RUNNER_AVAIL_CPU -y ./etc/services/rgl-clarode.yml - -n 100 -o ./out clas_02*.hipo + -n 100 -o ./out clas_021*.hipo profile: stage: test From bd3172d63bf19000af660bed134cc1681dfa7dd4 Mon Sep 17 00:00:00 2001 From: Nathan Baltzell Date: Tue, 14 Oct 2025 15:56:48 -0400 Subject: [PATCH 23/30] remove benchmarks --- .../src/main/java/org/jlab/service/ai/DCDenoiseEngine.java | 5 ----- 1 file changed, 5 deletions(-) diff --git a/reconstruction/ai/src/main/java/org/jlab/service/ai/DCDenoiseEngine.java b/reconstruction/ai/src/main/java/org/jlab/service/ai/DCDenoiseEngine.java index 5e1bac43c5..cee3b2daad 100644 --- a/reconstruction/ai/src/main/java/org/jlab/service/ai/DCDenoiseEngine.java +++ b/reconstruction/ai/src/main/java/org/jlab/service/ai/DCDenoiseEngine.java @@ -22,7 +22,6 @@ import org.jlab.clas.reco.ReconstructionEngine; import org.jlab.io.base.DataBank; import org.jlab.io.base.DataEvent; -import org.jlab.utils.benchmark.Benchmark; import org.jlab.utils.system.ClasUtilsFile; public class DCDenoiseEngine extends ReconstructionEngine { @@ -99,14 +98,10 @@ public boolean processDataEvent(DataEvent event) { DataBank bank = event.getBank(BANK_NAMES[i]); try { // WARNING: Predictor is *not* thread safe. - Benchmark.getInstance().resume("GETPRED"); Predictor predictor = predictors.get(); - Benchmark.getInstance().pause("GETPRED"); for (int sector=0; sector<6; sector++) { float[][] input = DCDenoiseEngine.read(bank, sector+1); - Benchmark.getInstance().resume("PREDICT"); float[][] output = predictor.predict(input); - Benchmark.getInstance().pause("PREDICT"); //System.out.println("IN:");show(input); //System.out.println("OUT:");show(output); update(bank, threshold, output, sector); From b68e32ad7f2d01b25bf473ce7189b76bac510b29 Mon Sep 17 00:00:00 2001 From: Nathan Baltzell Date: Tue, 14 Oct 2025 16:06:35 -0400 Subject: [PATCH 24/30] fix file name --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 99a0c88608..e35ebd141e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -41,7 +41,7 @@ jobs: key: raw_test_data # fixed key will always hit; clear cache to trigger cache miss path: | clas_005038.evio.00000 - clas_021559.evio.00000 + clas_021559.evio.00001 lookup-only: true - name: install xrootd-client if: ${{ steps.cache.outputs.cache-hit != 'true' }} From 20d98317d93709790ed37d0b25c9758e904eac94 Mon Sep 17 00:00:00 2001 From: Nathan Baltzell Date: Tue, 14 Oct 2025 16:56:26 -0400 Subject: [PATCH 25/30] asdf --- .github/workflows/ci.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e35ebd141e..638c02633b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -53,6 +53,7 @@ jobs: run: | xrdcp xroot://sci-xrootd.jlab.org///osgpool/hallb/clas12/validation/clas_005038.evio.00000 ./ xrdcp xroot://sci-xrootd.jlab.org///osgpool/hallb/clas12/validation/raw/rg-l/clas_021559.evio.00001 ./ + ls -lhtr # build ############################################################################# @@ -154,7 +155,6 @@ jobs: - uses: actions/cache/restore@v4 with: key: raw_test_data - path: clas_005038.evio.00000 - name: untar build run: tar xzvf coatjava.tar.gz - name: run test @@ -179,7 +179,6 @@ jobs: - uses: actions/cache/restore@v4 with: key: raw_test_data - path: clas_021559.evio.00001 - name: untar build run: | tar xzvf clara.tar.gz From 502e1ea35007d0a41664309701aa664ac220506e Mon Sep 17 00:00:00 2001 From: Nathan Baltzell Date: Tue, 14 Oct 2025 17:03:37 -0400 Subject: [PATCH 26/30] Revert "asdf" This reverts commit f648034ce51395c22e06694bbd422d6442b6abbc. --- .github/workflows/ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 638c02633b..e35ebd141e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -53,7 +53,6 @@ jobs: run: | xrdcp xroot://sci-xrootd.jlab.org///osgpool/hallb/clas12/validation/clas_005038.evio.00000 ./ xrdcp xroot://sci-xrootd.jlab.org///osgpool/hallb/clas12/validation/raw/rg-l/clas_021559.evio.00001 ./ - ls -lhtr # build ############################################################################# @@ -155,6 +154,7 @@ jobs: - uses: actions/cache/restore@v4 with: key: raw_test_data + path: clas_005038.evio.00000 - name: untar build run: tar xzvf coatjava.tar.gz - name: run test @@ -179,6 +179,7 @@ jobs: - uses: actions/cache/restore@v4 with: key: raw_test_data + path: clas_021559.evio.00001 - name: untar build run: | tar xzvf clara.tar.gz From d82ab15567c7491e9d90a4d942d888dead3d53f3 Mon Sep 17 00:00:00 2001 From: Nathan Baltzell Date: Tue, 14 Oct 2025 17:04:54 -0400 Subject: [PATCH 27/30] yes --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e35ebd141e..31e19921ff 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -159,6 +159,7 @@ jobs: run: tar xzvf coatjava.tar.gz - name: run test run: | + ls -lhtr ./coatjava/bin/decoder -n 10000 -o dog.hipo ./clas_005038.evio.00000 test_clara: From aae8a4d160ae9c7c4650050724b72a87394b34c7 Mon Sep 17 00:00:00 2001 From: Nathan Baltzell Date: Tue, 14 Oct 2025 17:51:24 -0400 Subject: [PATCH 28/30] fix --- .github/workflows/ci.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 31e19921ff..048a26b442 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -154,7 +154,9 @@ jobs: - uses: actions/cache/restore@v4 with: key: raw_test_data - path: clas_005038.evio.00000 + path: | + clas_005038.evio.00000 + clas_021559.evio.00001 - name: untar build run: tar xzvf coatjava.tar.gz - name: run test @@ -180,7 +182,9 @@ jobs: - uses: actions/cache/restore@v4 with: key: raw_test_data - path: clas_021559.evio.00001 + path: | + clas_005038.evio.00000 + clas_021559.evio.00001 - name: untar build run: | tar xzvf clara.tar.gz From f850a552cf418ffc290cc843a4ec25c34cafca9c Mon Sep 17 00:00:00 2001 From: Nathan Baltzell Date: Tue, 14 Oct 2025 17:58:49 -0400 Subject: [PATCH 29/30] needs cvmfs for ccdb --- .github-ci.yaml | 1 + .github/workflows/ci.yml | 4 ++++ 2 files changed, 5 insertions(+) create mode 120000 .github-ci.yaml diff --git a/.github-ci.yaml b/.github-ci.yaml new file mode 120000 index 0000000000..e978df9245 --- /dev/null +++ b/.github-ci.yaml @@ -0,0 +1 @@ +.github/workflows/ci.yml \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 048a26b442..dbc0480328 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -176,6 +176,10 @@ jobs: with: java-version: ${{ env.java_version }} distribution: ${{ env.java_distribution }} + - name: setup cvmfs + uses: cvmfs-contrib/github-action-cvmfs@v5 + with: + cvmfs_repositories: 'oasis.opensciencegrid.org' - uses: actions/download-artifact@v5 with: name: build_ubuntu-latest From 55534389f0fc22063d5e408aef200fbc2084c879 Mon Sep 17 00:00:00 2001 From: Nathan Baltzell Date: Tue, 14 Oct 2025 19:16:06 -0400 Subject: [PATCH 30/30] remove hack --- .../src/main/java/org/jlab/utils/system/ClasUtilsFile.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/common-tools/clas-utils/src/main/java/org/jlab/utils/system/ClasUtilsFile.java b/common-tools/clas-utils/src/main/java/org/jlab/utils/system/ClasUtilsFile.java index e6da2bdffa..d47d2eb9c3 100644 --- a/common-tools/clas-utils/src/main/java/org/jlab/utils/system/ClasUtilsFile.java +++ b/common-tools/clas-utils/src/main/java/org/jlab/utils/system/ClasUtilsFile.java @@ -60,9 +60,6 @@ public static String getCoatjavaRuntimeDir() { ret = "/" + String.join("/", Arrays.copyOfRange(d,0,d.length-3)); } - if (!(new File(ret)).isDirectory()) { - ret = System.getenv("HOME")+"/sw/coatjava/coatjava"; - } return ret; }