From a88edfbd48dde5d22d0dfc55960f5068a0830a91 Mon Sep 17 00:00:00 2001 From: Silvio Tomatis Date: Thu, 5 Dec 2024 15:01:16 +0100 Subject: [PATCH 1/4] Fix git command to initialize submodules in docs --- how-to-test-alto-xml.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/how-to-test-alto-xml.md b/how-to-test-alto-xml.md index dee0819..14e676e 100644 --- a/how-to-test-alto-xml.md +++ b/how-to-test-alto-xml.md @@ -2,7 +2,7 @@ Setup Open ONI as submodule. This is needed since the open-oni folder will be mounted in the `web` container. - `git submodule --init --checkout` + `git submodule update --init --checkout` Than you should be able to test an XML Alto file with: From b8cb23e0a6c7ca7fd5dc12badd96f738e4b3c0f8 Mon Sep 17 00:00:00 2001 From: "Silvio Tomatis (aider)" Date: Thu, 5 Dec 2024 16:17:19 +0100 Subject: [PATCH 2/4] ci: Add GitHub Actions job to test Alto XML processing --- .github/workflows/test.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b2f42f9..2dc3dd2 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -38,3 +38,15 @@ jobs: - name: Run pytest run: pytest + + test-alto: + name: Test Alto XML + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + submodules: true + + - name: Run Alto XML test + run: bash test-alto-batch.sh test-alto/0001.xml From ed7acafab2e2e1aa6de85fe3cf2b065cfcedf1ae Mon Sep 17 00:00:00 2001 From: Silvio Tomatis Date: Thu, 5 Dec 2024 16:58:08 +0100 Subject: [PATCH 3/4] Move alto related stuff to the test-alto folder and improve waiting logic for container initialization --- .github/workflows/test.yml | 7 +++++-- .gitmodules | 2 +- docker-compose.yml => test-alto/docker-compose.yml | 0 test-alto/{ => example-file}/0001.jpeg | Bin test-alto/{ => example-file}/0001.xml | 0 .../how-to-test-alto-xml.md | 9 +++++++++ open-oni => test-alto/open-oni | 0 test-alto-batch.sh => test-alto/test-alto-batch.sh | 11 ++++++++++- .../test_batches}/create_awardee_title.py | 0 .../test_batches}/default_mets.xml | 0 10 files changed, 25 insertions(+), 4 deletions(-) rename docker-compose.yml => test-alto/docker-compose.yml (100%) rename test-alto/{ => example-file}/0001.jpeg (100%) rename test-alto/{ => example-file}/0001.xml (100%) rename how-to-test-alto-xml.md => test-alto/how-to-test-alto-xml.md (81%) rename open-oni => test-alto/open-oni (100%) rename test-alto-batch.sh => test-alto/test-alto-batch.sh (94%) rename {test_batches => test-alto/test_batches}/create_awardee_title.py (100%) rename {test_batches => test-alto/test_batches}/default_mets.xml (100%) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2dc3dd2..e8d924f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -48,5 +48,8 @@ jobs: with: submodules: true - - name: Run Alto XML test - run: bash test-alto-batch.sh test-alto/0001.xml + - name: Pull docker images to run ALTO XML test + run: cd test-alto && docker compose pull + + - name: Run ALTO XML test + run: cd test-alto && bash test-alto-batch.sh example-file/0001.xml diff --git a/.gitmodules b/.gitmodules index ddfd64d..3a2c8db 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,4 +1,4 @@ [submodule "open-oni"] - path = open-oni + path = test-alto/open-oni url = git@github.com:codemyriad/open-oni.git branch = masca diff --git a/docker-compose.yml b/test-alto/docker-compose.yml similarity index 100% rename from docker-compose.yml rename to test-alto/docker-compose.yml diff --git a/test-alto/0001.jpeg b/test-alto/example-file/0001.jpeg similarity index 100% rename from test-alto/0001.jpeg rename to test-alto/example-file/0001.jpeg diff --git a/test-alto/0001.xml b/test-alto/example-file/0001.xml similarity index 100% rename from test-alto/0001.xml rename to test-alto/example-file/0001.xml diff --git a/how-to-test-alto-xml.md b/test-alto/how-to-test-alto-xml.md similarity index 81% rename from how-to-test-alto-xml.md rename to test-alto/how-to-test-alto-xml.md index 14e676e..16cc31c 100644 --- a/how-to-test-alto-xml.md +++ b/test-alto/how-to-test-alto-xml.md @@ -21,3 +21,12 @@ The script will lookup for a .jpeg image in the same directory and with the same |-- 0001.jpeg One can reset all data by removing the docker peristent volumes `data-mariadb` and `data-solr`. + + +Data reset +---------- + +To reset local state run + + docker compose down + docker volume rm test-alto_data-solr test-alto_data-mariadb diff --git a/open-oni b/test-alto/open-oni similarity index 100% rename from open-oni rename to test-alto/open-oni diff --git a/test-alto-batch.sh b/test-alto/test-alto-batch.sh similarity index 94% rename from test-alto-batch.sh rename to test-alto/test-alto-batch.sh index 9141354..cbb69c0 100644 --- a/test-alto-batch.sh +++ b/test-alto/test-alto-batch.sh @@ -1,6 +1,7 @@ #!/bin/bash set -e +cd "$(dirname "$0")" # Check if input file is provided if [ $# -eq 0 ]; then @@ -84,7 +85,15 @@ docker compose -f "$DOCKER_COMPOSE_PATH" down docker compose -f "$DOCKER_COMPOSE_PATH" up -d # Wait for the container to fully start -sleep 15 +timeout=90 +until docker compose logs web | grep -q "ONI setup successful"; do + if [ $timeout -le 0 ]; then + echo "Timeout waiting for ONI setup" + exit 1 + fi + sleep 3 + timeout=$((timeout-5)) +done # Create Awardee and Title objects if they don't exists docker compose -f "$DOCKER_COMPOSE_PATH" exec -T web bash -c "source ENV/bin/activate && python manage.py shell < /opt/create_awardee_title.py" diff --git a/test_batches/create_awardee_title.py b/test-alto/test_batches/create_awardee_title.py similarity index 100% rename from test_batches/create_awardee_title.py rename to test-alto/test_batches/create_awardee_title.py diff --git a/test_batches/default_mets.xml b/test-alto/test_batches/default_mets.xml similarity index 100% rename from test_batches/default_mets.xml rename to test-alto/test_batches/default_mets.xml From a5994e88846db5033d07b26d3b1852407755e6f7 Mon Sep 17 00:00:00 2001 From: Chiruzzi Marco Date: Fri, 6 Dec 2024 09:31:09 +0100 Subject: [PATCH 4/4] Check if the batch exists before trying to purge it; Do not destroy and recreate the same container every time the test-alto-batch script is run --- .gitignore | 4 ---- test-alto/.gitignore | 3 +++ test-alto/test-alto-batch.sh | 5 +---- 3 files changed, 4 insertions(+), 8 deletions(-) create mode 100644 test-alto/.gitignore diff --git a/.gitignore b/.gitignore index 95148c3..57bc039 100644 --- a/.gitignore +++ b/.gitignore @@ -12,7 +12,3 @@ dist/ src/lp_labelstudio/web_server/test_images/*.json .env - -test_batches/* -!test_batches/default_mets.xml -!test_batches/create_awardee_title.py diff --git a/test-alto/.gitignore b/test-alto/.gitignore new file mode 100644 index 0000000..a4772bb --- /dev/null +++ b/test-alto/.gitignore @@ -0,0 +1,3 @@ +test_batches/* +!test_batches/default_mets.xml +!test_batches/create_awardee_title.py diff --git a/test-alto/test-alto-batch.sh b/test-alto/test-alto-batch.sh index cbb69c0..c651c75 100644 --- a/test-alto/test-alto-batch.sh +++ b/test-alto/test-alto-batch.sh @@ -78,9 +78,6 @@ if [ ! -f "$DOCKER_COMPOSE_PATH" ]; then exit 1 fi -# Stop any existing Open ONI containers -docker compose -f "$DOCKER_COMPOSE_PATH" down - # Start Docker Compose with batch mounted docker compose -f "$DOCKER_COMPOSE_PATH" up -d @@ -99,7 +96,7 @@ done docker compose -f "$DOCKER_COMPOSE_PATH" exec -T web bash -c "source ENV/bin/activate && python manage.py shell < /opt/create_awardee_title.py" # Purge the batch if it exists in Open ONI -docker compose -f "$DOCKER_COMPOSE_PATH" exec -T web bash -c "source ENV/bin/activate && python manage.py purge_batch $BATCH_NAME" +docker compose -f "$DOCKER_COMPOSE_PATH" exec -T web bash -c "source ENV/bin/activate && (python manage.py batches | grep -q '^$BATCH_NAME$' && python manage.py purge_batch $BATCH_NAME; true)" # Load the batch into Open ONI docker compose -f "$DOCKER_COMPOSE_PATH" exec -T web bash -c "source ENV/bin/activate && python manage.py load_batch /opt/openoni/data/batches/$BATCH_NAME"