From b4cf660ef02f404a3731a055107b424a9c722c45 Mon Sep 17 00:00:00 2001 From: mkottakota1 <149763406+mkottakota1@users.noreply.github.com> Date: Tue, 13 Jan 2026 08:59:39 +0000 Subject: [PATCH 01/16] Using customized private repo --- .github/workflows/ci.yml | 59 +++++++++++++++++++++------------------- 1 file changed, 31 insertions(+), 28 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 73a23dc..998b56f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,64 +2,67 @@ name: CI on: push: - branches: [ "main" ] pull_request: - branches: [ "main" ] - # Allows you to run this workflow manually from the Actions tab workflow_dispatch: jobs: build: - runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 + + # 1. Login to Docker Hub to pull your private image + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: mkottakota419 + password: ${{ secrets.DOCKER_PASSWORD }} # Ensure this secret is set in GitHub Settings + - name: Set up MySQL server run: | docker run -d -p 3306:3306 -e MYSQL_ALLOW_EMPTY_PASSWORD=yes --name test-mysql mysql:8.0 echo "MySQL startup ..." - until docker exec test-mysql mysql -u root -e "SELECT 1;"; do \ - echo "..."; \ - sleep 1; \ - done; + until docker exec test-mysql mysql -u root -e "SELECT 1;"; do + echo "..." + sleep 1 + done docker exec -u root test-mysql mysql -u root -e "CREATE DATABASE testdb;" docker exec -u root test-mysql mysql -u root testdb -e "CREATE TABLE test_source (i integer, b boolean, f float, v varchar(32), c char(32), lv varchar(9999), bn binary(32), vb varbinary(32), lvb varbinary(9999), d date, t time, ts timestamp null, tz varchar(80), tsz varchar(80), n numeric(20,4));" docker exec -u root test-mysql /bin/bash -c "(echo \"INSERT INTO test_source VALUES (null, null, null, null, null, null, null, null, null, null, null, null, null, null, null);\"; for i in \`seq 1 9\`; do echo \"INSERT INTO test_source VALUES (\$i, 1, \$i.5, 'test \$i', 'test \$i', 'test \$i', 'test \$i', 'test \$i', 'test \$i', '\$((\$i+11))00/1/\$i', '4:0\$i', '2038-01-0\$i 03:14:07', '1:2\$i:00', 'June 1, \$((\$i+11))00 03:2\$i EST', '123456.7890');\"; done) | mysql -u root testdb" - docker exec -u root test-mysql mysql -u root testdb -e "select * from test_source;" - - name: Set up a Vertica server - timeout-minutes: 15 + + - name: Set up Vertica server (Private Image) + timeout-minutes: 10 run: | docker run -d -p 5433:5433 -p 5444:5444 \ - -e ODBCSYSINI=/var/odbc-loader/tests/config \ --add-host=host.docker.internal:host-gateway \ --name vertica_docker \ - opentext/vertica-ce:24.2.0-1 - echo "Vertica startup ..." - until docker exec vertica_docker test -f /data/vertica/VMart/agent_start.out; do \ - echo "..."; \ - sleep 3; \ - done; + mkottakota419/vertica-ci:latest + + echo "Starting Vertica Database (mydb) ..." + # Since the DB is created, we just need to start the process + docker exec -u dbadmin vertica_docker admintools -t start_db -d mydb -p "MySecretPass" + echo "Vertica is up" - docker exec -u dbadmin vertica_docker /opt/vertica/bin/vsql -c "\l" docker exec -u dbadmin vertica_docker /opt/vertica/bin/vsql -c "select version()" + - name: Build & Install UDx run: | - docker exec -u root vertica_docker dnf -y install gcc-toolset-9-gcc-c++.x86_64 - docker exec -u root vertica_docker dnf -y install unixODBC-devel - docker exec -u root vertica_docker dnf -y install pcre-devel - docker exec -u root vertica_docker dnf -y install perl + # All dnf installs (gcc, odbc-devel, make) are now skipped as they are in the image docker cp ${{ github.workspace }} vertica_docker:/var/odbc-loader - docker exec -u root vertica_docker /bin/bash -c "sudo chown -R dbadmin:verticadba /var/odbc-loader" - docker exec -u root -w /var/odbc-loader vertica_docker dnf install -y make - docker exec -u dbadmin -w /var/odbc-loader vertica_docker scl enable gcc-toolset-9 "bash -c 'make; make install'" + docker exec -u root vertica_docker /bin/bash -c "chown -R dbadmin:verticadba /var/odbc-loader" + + # Using toolset-12 as installed in our Dockerfile + docker exec -u dbadmin -w /var/odbc-loader vertica_docker scl enable gcc-toolset-12 "bash -c 'make; make install'" - name: Install ODBC clients run: | + # We still install the specific MySQL connector at runtime to keep image size down docker exec -w /var/odbc-loader -u root vertica_docker wget https://repo.mysql.com/mysql80-community-release-el8-9.noarch.rpm docker exec -w /var/odbc-loader -u root vertica_docker dnf install -y mysql80-community-release-el8-9.noarch.rpm docker exec -w /var/odbc-loader -u root vertica_docker dnf install -y mysql-connector-odbc - name: Run Tests run: | - docker exec -w /var/odbc-loader -u dbadmin vertica_docker make test + # Ensure the test config knows where to find host.docker.internal (MySQL) + docker exec -w /var/odbc-loader -u dbadmin vertica_docker make test \ No newline at end of file From 2bc6aee601e492bcc14aef0a076c2ffa5860ba55 Mon Sep 17 00:00:00 2001 From: mkottakota1 <149763406+mkottakota1@users.noreply.github.com> Date: Tue, 13 Jan 2026 09:07:29 +0000 Subject: [PATCH 02/16] Rebuild DB --- .github/workflows/ci.yml | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 998b56f..375ef9f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,9 +39,16 @@ jobs: --name vertica_docker \ mkottakota419/vertica-ci:latest - echo "Starting Vertica Database (mydb) ..." - # Since the DB is created, we just need to start the process - docker exec -u dbadmin vertica_docker admintools -t start_db -d mydb -p "MySecretPass" + echo "Re-binding and Starting Vertica Database (mydb) ..." + + # This command re-registers the existing files into the admintools.conf + docker exec -u dbadmin vertica_docker admintools -t create_db \ + --skip-prepare \ + -s localhost \ + -d mydb \ + -p "MySecretPass" \ + -c /catalog -D /data \ + --force-rebuild echo "Vertica is up" docker exec -u dbadmin vertica_docker /opt/vertica/bin/vsql -c "select version()" From 107d3f19b5496d5db0bcb63d501ca5e25f2d3cc4 Mon Sep 17 00:00:00 2001 From: mkottakota1 <149763406+mkottakota1@users.noreply.github.com> Date: Tue, 13 Jan 2026 09:12:26 +0000 Subject: [PATCH 03/16] Create Fresh DB --- .github/workflows/ci.yml | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 375ef9f..d7af8c7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,19 +39,20 @@ jobs: --name vertica_docker \ mkottakota419/vertica-ci:latest - echo "Re-binding and Starting Vertica Database (mydb) ..." + echo "Creating fresh Vertica Database (mydb) ..." + + # We use a simple create_db. + # If /catalog or /data exist from your previous image, + # we'll clear them first to ensure success. + docker exec -u root vertica_docker bash -c "rm -rf /catalog/* /data/*" - # This command re-registers the existing files into the admintools.conf docker exec -u dbadmin vertica_docker admintools -t create_db \ - --skip-prepare \ -s localhost \ -d mydb \ - -p "MySecretPass" \ - -c /catalog -D /data \ - --force-rebuild + -p "MySecretPass" echo "Vertica is up" - docker exec -u dbadmin vertica_docker /opt/vertica/bin/vsql -c "select version()" + docker exec -u dbadmin vertica_docker vsql -c "SELECT version();" - name: Build & Install UDx run: | From ab79fc6f984d65d01eb95540e5b1c1959ac87767 Mon Sep 17 00:00:00 2001 From: mkottakota1 <149763406+mkottakota1@users.noreply.github.com> Date: Tue, 13 Jan 2026 09:18:09 +0000 Subject: [PATCH 04/16] Installing Iproute package --- .github/workflows/ci.yml | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d7af8c7..a55269c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -32,27 +32,33 @@ jobs: docker exec -u root test-mysql /bin/bash -c "(echo \"INSERT INTO test_source VALUES (null, null, null, null, null, null, null, null, null, null, null, null, null, null, null);\"; for i in \`seq 1 9\`; do echo \"INSERT INTO test_source VALUES (\$i, 1, \$i.5, 'test \$i', 'test \$i', 'test \$i', 'test \$i', 'test \$i', 'test \$i', '\$((\$i+11))00/1/\$i', '4:0\$i', '2038-01-0\$i 03:14:07', '1:2\$i:00', 'June 1, \$((\$i+11))00 03:2\$i EST', '123456.7890');\"; done) | mysql -u root testdb" - name: Set up Vertica server (Private Image) - timeout-minutes: 10 + timeout-minutes: 15 run: | docker run -d -p 5433:5433 -p 5444:5444 \ --add-host=host.docker.internal:host-gateway \ --name vertica_docker \ mkottakota419/vertica-ci:latest - echo "Creating fresh Vertica Database (mydb) ..." + echo "Preparing system environment..." + # 1. Install tools. + # 2. Try to link, but don't fail if it's already there (|| true). + # 3. Verify the path exists so we can see it in the logs. + docker exec -u root vertica_docker bash -c " + dnf install -y iproute procps-ng && \ + ln -sf /usr/sbin/ip /sbin/ip || true && \ + ls -l /sbin/ip" - # We use a simple create_db. - # If /catalog or /data exist from your previous image, - # we'll clear them first to ensure success. + echo "Cleaning stale files..." docker exec -u root vertica_docker bash -c "rm -rf /catalog/* /data/*" + echo "Creating Vertica Database (mydb) ..." docker exec -u dbadmin vertica_docker admintools -t create_db \ -s localhost \ -d mydb \ -p "MySecretPass" echo "Vertica is up" - docker exec -u dbadmin vertica_docker vsql -c "SELECT version();" + docker exec -u dbadmin vertica_docker vsql -w "MySecretPass" -c "SELECT version();" - name: Build & Install UDx run: | From c126833c4b474a7edbbc77c830dee7ba7577a2b4 Mon Sep 17 00:00:00 2001 From: mkottakota1 <149763406+mkottakota1@users.noreply.github.com> Date: Tue, 13 Jan 2026 09:30:52 +0000 Subject: [PATCH 05/16] ensure we are running make in the exact folder where the Makefile sits --- .github/workflows/ci.yml | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a55269c..934a0d5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -62,12 +62,22 @@ jobs: - name: Build & Install UDx run: | - # All dnf installs (gcc, odbc-devel, make) are now skipped as they are in the image - docker cp ${{ github.workspace }} vertica_docker:/var/odbc-loader - docker exec -u root vertica_docker /bin/bash -c "chown -R dbadmin:verticadba /var/odbc-loader" + # 1. Create the directory + docker exec -u root vertica_docker mkdir -p /var/odbc-loader - # Using toolset-12 as installed in our Dockerfile - docker exec -u dbadmin -w /var/odbc-loader vertica_docker scl enable gcc-toolset-12 "bash -c 'make; make install'" + # 2. Copy the CONTENTS of the workspace (note the trailing /.) + # This ensures the Makefile is at /var/odbc-loader/Makefile + docker cp ${{ github.workspace }}/. vertica_docker:/var/odbc-loader + + # 3. Fix permissions + docker exec -u root vertica_docker chown -R dbadmin:verticadba /var/odbc-loader + + # 4. List files to verify (helps debugging) + docker exec -u dbadmin -w /var/odbc-loader vertica_docker ls -R + + # 5. Run the build + docker exec -u dbadmin -w /var/odbc-loader vertica_docker scl enable gcc-toolset-12 "make" + docker exec -u dbadmin -w /var/odbc-loader vertica_docker scl enable gcc-toolset-12 "make install" - name: Install ODBC clients run: | From 96f20db25647d7b0490a30627dc64e3312744504 Mon Sep 17 00:00:00 2001 From: mkottakota1 <149763406+mkottakota1@users.noreply.github.com> Date: Tue, 13 Jan 2026 09:38:25 +0000 Subject: [PATCH 06/16] Passing DB Password --- .github/workflows/ci.yml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 934a0d5..7f83b11 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -75,9 +75,14 @@ jobs: # 4. List files to verify (helps debugging) docker exec -u dbadmin -w /var/odbc-loader vertica_docker ls -R - # 5. Run the build - docker exec -u dbadmin -w /var/odbc-loader vertica_docker scl enable gcc-toolset-12 "make" - docker exec -u dbadmin -w /var/odbc-loader vertica_docker scl enable gcc-toolset-12 "make install" + # 5. Run the build (make) + docker exec -u dbadmin -w /var/odbc-loader vertica_docker \ + scl enable gcc-toolset-12 "make" + + # 6. Run install with the password provided to vsql + # We set PGPASSWORD which vsql recognizes automatically + docker exec -u dbadmin -w /var/odbc-loader -e PGPASSWORD="MySecretPass" vertica_docker \ + scl enable gcc-toolset-12 "make install" - name: Install ODBC clients run: | From 71987002df20b5fbc41241a651c98157930361b3 Mon Sep 17 00:00:00 2001 From: mkottakota1 <149763406+mkottakota1@users.noreply.github.com> Date: Tue, 13 Jan 2026 09:42:42 +0000 Subject: [PATCH 07/16] Add wget installable --- .github/workflows/ci.yml | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7f83b11..a9e26ad 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -80,16 +80,21 @@ jobs: scl enable gcc-toolset-12 "make" # 6. Run install with the password provided to vsql - # We set PGPASSWORD which vsql recognizes automatically - docker exec -u dbadmin -w /var/odbc-loader -e PGPASSWORD="MySecretPass" vertica_docker \ + # We set VSQL_PASSWORD which vsql recognizes automatically + docker exec -u dbadmin -w /var/odbc-loader -e VSQL_PASSWORD="MySecretPass" vertica_docker \ scl enable gcc-toolset-12 "make install" - name: Install ODBC clients run: | - # We still install the specific MySQL connector at runtime to keep image size down + # 1. Install wget so we can download the repo file + docker exec -u root vertica_docker dnf install -y wget + + # 2. Download and install MySQL repo docker exec -w /var/odbc-loader -u root vertica_docker wget https://repo.mysql.com/mysql80-community-release-el8-9.noarch.rpm docker exec -w /var/odbc-loader -u root vertica_docker dnf install -y mysql80-community-release-el8-9.noarch.rpm - docker exec -w /var/odbc-loader -u root vertica_docker dnf install -y mysql-connector-odbc + + # 3. Install the connector (disabling GPG check to avoid prompt blocks) + docker exec -w /var/odbc-loader -u root vertica_docker dnf install -y --nogpgcheck mysql-connector-odbc - name: Run Tests run: | From 7e00911ca08ae6c814e7f9a2c267198b0a4c2a9e Mon Sep 17 00:00:00 2001 From: mkottakota1 <149763406+mkottakota1@users.noreply.github.com> Date: Tue, 13 Jan 2026 09:47:13 +0000 Subject: [PATCH 08/16] Add VSQL ENV Var for running tests --- .github/workflows/ci.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a9e26ad..0365322 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -98,5 +98,11 @@ jobs: - name: Run Tests run: | - # Ensure the test config knows where to find host.docker.internal (MySQL) - docker exec -w /var/odbc-loader -u dbadmin vertica_docker make test \ No newline at end of file + # We need to tell the ODBC Driver Manager where to find your odbc.ini and odbcinst.ini + # These are located in /var/odbc-loader/tests/config based on your 'ls -R' earlier + + docker exec -w /var/odbc-loader \ + -u dbadmin \ + -e ODBCSYSINI=/var/odbc-loader/tests/config \ + -e VSQL_PASSWORD="MySecretPass" \ + vertica_docker make test \ No newline at end of file From 5abdc1222ec0fc9cdbdb68bfa30ed78c3fad7020 Mon Sep 17 00:00:00 2001 From: mkottakota1 <149763406+mkottakota1@users.noreply.github.com> Date: Tue, 13 Jan 2026 09:56:24 +0000 Subject: [PATCH 09/16] Coping odbc.ini files to etc folder --- .github/workflows/ci.yml | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0365322..676a1bc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -98,11 +98,18 @@ jobs: - name: Run Tests run: | - # We need to tell the ODBC Driver Manager where to find your odbc.ini and odbcinst.ini - # These are located in /var/odbc-loader/tests/config based on your 'ls -R' earlier - + # 1. Point the system-wide ODBC config to your repo files + # We use 'cat' to overwrite the system files with your test configs + docker exec -u root vertica_docker bash -c "cat /var/odbc-loader/tests/config/odbcinst.ini > /etc/odbcinst.ini" + docker exec -u root vertica_docker bash -c "cat /var/odbc-loader/tests/config/odbc.ini > /etc/odbc.ini" + + # 2. Verify the Driver Manager sees the DSN (helpful for logs) + echo "Verifying ODBC Data Sources:" + docker exec -u dbadmin vertica_docker odbcinst -q -s + + # 3. Run the test + # We still provide VSQL_PASSWORD for the test script to talk to Vertica docker exec -w /var/odbc-loader \ -u dbadmin \ - -e ODBCSYSINI=/var/odbc-loader/tests/config \ -e VSQL_PASSWORD="MySecretPass" \ vertica_docker make test \ No newline at end of file From 17a56e9ab81d02687b638d59317e06aad7c470fc Mon Sep 17 00:00:00 2001 From: mkottakota1 <149763406+mkottakota1@users.noreply.github.com> Date: Tue, 13 Jan 2026 10:02:05 +0000 Subject: [PATCH 10/16] Finding correct so file name --- .github/workflows/ci.yml | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 676a1bc..08e5859 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -98,17 +98,20 @@ jobs: - name: Run Tests run: | - # 1. Point the system-wide ODBC config to your repo files - # We use 'cat' to overwrite the system files with your test configs + # 1. Locate the actual MySQL ODBC driver file + # It might be libmyodbc8w.so, libmyodbc8a.so, or have a version suffix + MYSQL_DRIVER_PATH=$(docker exec vertica_docker find /usr/lib64 -name "libmyodbc*.so" | head -n 1) + echo "Found MySQL Driver at: $MYSQL_DRIVER_PATH" + + # 2. Create a symlink so it matches what is in your odbcinst.ini + # If your odbcinst.ini expects /usr/lib64/libmyodbc8w.so, we make sure it's there + docker exec -u root vertica_docker ln -sf "$MYSQL_DRIVER_PATH" /usr/lib64/libmyodbc8w.so + + # 3. Point system-wide ODBC config to your repo files docker exec -u root vertica_docker bash -c "cat /var/odbc-loader/tests/config/odbcinst.ini > /etc/odbcinst.ini" docker exec -u root vertica_docker bash -c "cat /var/odbc-loader/tests/config/odbc.ini > /etc/odbc.ini" - # 2. Verify the Driver Manager sees the DSN (helpful for logs) - echo "Verifying ODBC Data Sources:" - docker exec -u dbadmin vertica_docker odbcinst -q -s - - # 3. Run the test - # We still provide VSQL_PASSWORD for the test script to talk to Vertica + # 4. Run the test docker exec -w /var/odbc-loader \ -u dbadmin \ -e VSQL_PASSWORD="MySecretPass" \ From 9a8d1f81351c676350b302d91842eaed2e54a2fa Mon Sep 17 00:00:00 2001 From: mkottakota1 <149763406+mkottakota1@users.noreply.github.com> Date: Mon, 2 Feb 2026 06:35:11 +0000 Subject: [PATCH 11/16] Introducing federted queries test and skipping copy_test --- .github/workflows/ci.yml | 2 + Makefile | 6 +-- tests/expected/federated_queries.out | 37 +++++++++++++++ tests/federated_queries.sql | 69 ++++++++++++++++++++++++++++ 4 files changed, 111 insertions(+), 3 deletions(-) create mode 100644 tests/expected/federated_queries.out create mode 100644 tests/federated_queries.sql diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 08e5859..708d3f0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,6 +30,8 @@ jobs: docker exec -u root test-mysql mysql -u root -e "CREATE DATABASE testdb;" docker exec -u root test-mysql mysql -u root testdb -e "CREATE TABLE test_source (i integer, b boolean, f float, v varchar(32), c char(32), lv varchar(9999), bn binary(32), vb varbinary(32), lvb varbinary(9999), d date, t time, ts timestamp null, tz varchar(80), tsz varchar(80), n numeric(20,4));" docker exec -u root test-mysql /bin/bash -c "(echo \"INSERT INTO test_source VALUES (null, null, null, null, null, null, null, null, null, null, null, null, null, null, null);\"; for i in \`seq 1 9\`; do echo \"INSERT INTO test_source VALUES (\$i, 1, \$i.5, 'test \$i', 'test \$i', 'test \$i', 'test \$i', 'test \$i', 'test \$i', '\$((\$i+11))00/1/\$i', '4:0\$i', '2038-01-0\$i 03:14:07', '1:2\$i:00', 'June 1, \$((\$i+11))00 03:2\$i EST', '123456.7890');\"; done) | mysql -u root testdb" + docker exec -u root test-mysql mysql -u root testdb -e "CREATE TABLE people (id integer PRIMARY KEY, name varchar(20));" + docker exec -u root test-mysql mysql -u root testdb -e "INSERT INTO people VALUES (101, 'Alice'), (102, 'Bob'), (103, 'Charlie');" - name: Set up Vertica server (Private Image) timeout-minutes: 15 diff --git a/Makefile b/Makefile index 7fa3064..f87a165 100644 --- a/Makefile +++ b/Makefile @@ -42,12 +42,12 @@ test_example: @echo ALL TESTS SUCCESSFUL test: - @$(VSQL) -f tests/copy_test.sql > $(TMPDIR)/copy_test.out 2>&1 - @diff -u tests/expected/copy_test.out <(perl -pe 's/^vsql:[\/_:\w\.]* /vsql: /; \ + @$(VSQL) -f tests/federated_queries.sql > $(TMPDIR)/federated_queries.out 2>&1 + @diff -u tests/expected/federated_queries.out <(perl -pe 's/^vsql:[\/_:\w\.]* /vsql: /; \ s/\[ODBC[^\]]*\]/[...]/g; \ s/\[mysql[^\]]*\]/[...]/g; \ s/(Error parsing .* )\(.*\)$$/$$1(...)/; \ - s/mariadb/MySQL/ig; ' $(TMPDIR)/copy_test.out) + s/mariadb/MySQL/ig; ' $(TMPDIR)/federated_queries.out) @echo ALL TESTS SUCCESSFUL .PHONY: build clean install uninstall example test_example test diff --git a/tests/expected/federated_queries.out b/tests/expected/federated_queries.out new file mode 100644 index 0000000..98bfeec --- /dev/null +++ b/tests/expected/federated_queries.out @@ -0,0 +1,37 @@ +Timing is off. +SET + +CREATE EXTERNAL TABLE + + id | name +-----+---------- + 101 | Alice + 102 | Bob + 103 | Charlie +(3 rows) + + id +----- + 101 + 102 + 103 +(3 rows) + +CREATE TABLE +INSERT + id | name | status +-----+---------+--------- + 101 | Alice | inactive + 102 | Bob | active + 103 | Charlie | inactive +(3 rows) + + id | name +-----+---------- + 101 | Alice + 102 | Bob + 103 | Charlie +(3 rows) + +DROP TABLE +DROP TABLE diff --git a/tests/federated_queries.sql b/tests/federated_queries.sql new file mode 100644 index 0000000..c8cdeb6 --- /dev/null +++ b/tests/federated_queries.sql @@ -0,0 +1,69 @@ +\timing off + +-- Set output to a fixed timezone regardless of where this is being tested +set time zone to 'EST'; + +-- ===================================================== +-- Federated Queries Test Case +-- ===================================================== +-- This test demonstrates the use of ODBCLoader for federated queries +-- allowing Vertica to query external databases (e.g., MySQL) while +-- leveraging predicate pushdown and column pruning to minimize data movement + +-- Create an External Table in Vertica that acts as a gateway to MySQL +-- The External Table definition stores metadata about the external source +-- but does not retrieve any data until queried +CREATE EXTERNAL TABLE public.epeople ( + id INTEGER, + name VARCHAR(20) +) AS COPY WITH + SOURCE ODBCSource() + PARSER ODBCLoader( + connect='DSN=MySQL', + query='SELECT * FROM testdb.people' +); + +-- Test 1: Query with predicate pushdown +-- When executing a query with a WHERE clause, ODBCLoader rewrites the original +-- external query to include the predicate, pushing the filter to the external database +-- This minimizes the amount of data transferred from MySQL to Vertica +-- Expected: Only people with id > 100 are retrieved from MySQL +SELECT * FROM public.epeople WHERE id > 100; + +-- Test 2: Query with column pruning +-- When only specific columns are selected, ODBCLoader optimizes the external query +-- to fetch only the required columns. Columns not in the Vertica query are replaced +-- with NULL values to match the External Table schema +-- Expected: Only id column is fetched from MySQL (name is replaced with NULL) +SELECT id FROM public.epeople WHERE id > 100; + +-- Test 3: Federated join query +-- Demonstrates joining an external table (MySQL) with a Vertica-managed table +-- This leverages both databases' query engines for optimal performance +-- Create a temporary Vertica table for joining +CREATE TABLE public.employee_status ( + id INTEGER, + status VARCHAR(20) +); + +INSERT INTO public.employee_status VALUES + (1, 'active'), + (101, 'inactive'), + (102, 'active'), + (103, 'inactive'); + +-- Perform a federated join combining data from MySQL (epeople) and Vertica (employee_status) +-- The ODBCLoader will push down predicates to MySQL when possible +SELECT epeople.id, epeople.name, employee_status.status +FROM public.epeople +JOIN public.employee_status ON epeople.id = employee_status.id +WHERE epeople.id > 100; + +-- Test 4: Complex predicate pushdown +-- Multiple predicates are combined and pushed to the external database +-- Expected: MySQL executes: SELECT * FROM testdb.people WHERE id > 50 AND id < 150 +SELECT id, name FROM public.epeople WHERE id > 50 AND id < 150; + +-- Clean up external and temporary tables +DROP TABLE public.epeople; +DROP TABLE public.employee_status; From 30c5fd9f5256d2ee34489a682bc38926e8d5cf48 Mon Sep 17 00:00:00 2001 From: mkottakota1 <149763406+mkottakota1@users.noreply.github.com> Date: Mon, 2 Feb 2026 06:46:09 +0000 Subject: [PATCH 12/16] Updated federated test out file --- tests/expected/federated_queries.out | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/tests/expected/federated_queries.out b/tests/expected/federated_queries.out index 98bfeec..111e800 100644 --- a/tests/expected/federated_queries.out +++ b/tests/expected/federated_queries.out @@ -1,16 +1,14 @@ Timing is off. SET - -CREATE EXTERNAL TABLE - - id | name ------+---------- +CREATE TABLE + id | name +-----+--------- 101 | Alice 102 | Bob 103 | Charlie (3 rows) - id + id ----- 101 102 @@ -18,16 +16,20 @@ CREATE EXTERNAL TABLE (3 rows) CREATE TABLE -INSERT - id | name | status ------+---------+--------- + OUTPUT +-------- + 4 +(1 row) + + id | name | status +-----+---------+---------- 101 | Alice | inactive 102 | Bob | active 103 | Charlie | inactive (3 rows) - id | name ------+---------- + id | name +-----+--------- 101 | Alice 102 | Bob 103 | Charlie From d313c51d8495d21f31dcfba6a9c46e3d5a9a3ac7 Mon Sep 17 00:00:00 2001 From: mkottakota1 <149763406+mkottakota1@users.noreply.github.com> Date: Mon, 2 Feb 2026 07:11:12 +0000 Subject: [PATCH 13/16] Adding more info to test run --- Makefile | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index f87a165..b98f6d3 100644 --- a/Makefile +++ b/Makefile @@ -42,13 +42,28 @@ test_example: @echo ALL TESTS SUCCESSFUL test: - @$(VSQL) -f tests/federated_queries.sql > $(TMPDIR)/federated_queries.out 2>&1 + @echo "=========================================" + @echo "Running ODBCLoader Test Suite" + @echo "=========================================" + @echo "" + @echo "[*] Running Federated Queries Test..." + @echo "" + @$(VSQL) -f tests/federated_queries.sql 2>&1 | tee $(TMPDIR)/federated_queries.out | perl -pe 's/^vsql:[\/_:\w\.]* /vsql: /; \ + s/\[ODBC[^\]]*\]/[...]/g; \ + s/\[mysql[^\]]*\]/[...]/g; \ + s/(Error parsing .* )\(.*\)$$/$$1(...)/; \ + s/mariadb/MySQL/ig; ' + @echo "" + @echo "[*] Validating test output..." @diff -u tests/expected/federated_queries.out <(perl -pe 's/^vsql:[\/_:\w\.]* /vsql: /; \ s/\[ODBC[^\]]*\]/[...]/g; \ s/\[mysql[^\]]*\]/[...]/g; \ s/(Error parsing .* )\(.*\)$$/$$1(...)/; \ - s/mariadb/MySQL/ig; ' $(TMPDIR)/federated_queries.out) - @echo ALL TESTS SUCCESSFUL + s/mariadb/MySQL/ig; ' $(TMPDIR)/federated_queries.out) && echo "[✓] Test validation passed" || (echo "[✗] Test validation failed" && exit 1) + @echo "" + @echo "=========================================" + @echo "✓ ALL TESTS SUCCESSFUL" + @echo "=========================================" .PHONY: build clean install uninstall example test_example test From c14c3f4a6f715d414886158adba28df8eb69ef5c Mon Sep 17 00:00:00 2001 From: mkottakota1 <149763406+mkottakota1@users.noreply.github.com> Date: Fri, 6 Feb 2026 11:27:29 +0000 Subject: [PATCH 14/16] Added error handling and removed hardcoded DB password --- .github/workflows/ci.yml | 71 +++++++++++++++++++++++----------------- 1 file changed, 41 insertions(+), 30 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 708d3f0..048fd9b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,15 +27,18 @@ jobs: echo "..." sleep 1 done - docker exec -u root test-mysql mysql -u root -e "CREATE DATABASE testdb;" - docker exec -u root test-mysql mysql -u root testdb -e "CREATE TABLE test_source (i integer, b boolean, f float, v varchar(32), c char(32), lv varchar(9999), bn binary(32), vb varbinary(32), lvb varbinary(9999), d date, t time, ts timestamp null, tz varchar(80), tsz varchar(80), n numeric(20,4));" - docker exec -u root test-mysql /bin/bash -c "(echo \"INSERT INTO test_source VALUES (null, null, null, null, null, null, null, null, null, null, null, null, null, null, null);\"; for i in \`seq 1 9\`; do echo \"INSERT INTO test_source VALUES (\$i, 1, \$i.5, 'test \$i', 'test \$i', 'test \$i', 'test \$i', 'test \$i', 'test \$i', '\$((\$i+11))00/1/\$i', '4:0\$i', '2038-01-0\$i 03:14:07', '1:2\$i:00', 'June 1, \$((\$i+11))00 03:2\$i EST', '123456.7890');\"; done) | mysql -u root testdb" - docker exec -u root test-mysql mysql -u root testdb -e "CREATE TABLE people (id integer PRIMARY KEY, name varchar(20));" - docker exec -u root test-mysql mysql -u root testdb -e "INSERT INTO people VALUES (101, 'Alice'), (102, 'Bob'), (103, 'Charlie');" + echo "Setting up MySQL databases and tables..." + docker exec -u root test-mysql mysql -u root -e "CREATE DATABASE testdb;" || (echo "Failed to create testdb" && exit 1) + docker exec -u root test-mysql mysql -u root testdb -e "CREATE TABLE test_source (i integer, b boolean, f float, v varchar(32), c char(32), lv varchar(9999), bn binary(32), vb varbinary(32), lvb varbinary(9999), d date, t time, ts timestamp null, tz varchar(80), tsz varchar(80), n numeric(20,4));" || (echo "Failed to create test_source table" && exit 1) + docker exec -u root test-mysql /bin/bash -c "(echo \"INSERT INTO test_source VALUES (null, null, null, null, null, null, null, null, null, null, null, null, null, null, null);\"; for i in \`seq 1 9\`; do echo \"INSERT INTO test_source VALUES (\$i, 1, \$i.5, 'test \$i', 'test \$i', 'test \$i', 'test \$i', 'test \$i', 'test \$i', '\$((\$i+11))00/1/\$i', '4:0\$i', '2038-01-0\$i 03:14:07', '1:2\$i:00', 'June 1, \$((\$i+11))00 03:2\$i EST', '123456.7890');\"; done) | mysql -u root testdb" || (echo "Failed to insert test data" && exit 1) + docker exec -u root test-mysql mysql -u root testdb -e "CREATE TABLE people (id integer PRIMARY KEY, name varchar(20));" || (echo "Failed to create people table" && exit 1) + docker exec -u root test-mysql mysql -u root testdb -e "INSERT INTO people VALUES (101, 'Alice'), (102, 'Bob'), (103, 'Charlie');" || (echo "Failed to insert people data" && exit 1) + echo "MySQL setup completed successfully" - name: Set up Vertica server (Private Image) timeout-minutes: 15 run: | + set -e docker run -d -p 5433:5433 -p 5444:5444 \ --add-host=host.docker.internal:host-gateway \ --name vertica_docker \ @@ -48,73 +51,81 @@ jobs: docker exec -u root vertica_docker bash -c " dnf install -y iproute procps-ng && \ ln -sf /usr/sbin/ip /sbin/ip || true && \ - ls -l /sbin/ip" + ls -l /sbin/ip" || (echo "Failed to prepare system environment" && exit 1) echo "Cleaning stale files..." - docker exec -u root vertica_docker bash -c "rm -rf /catalog/* /data/*" + docker exec -u root vertica_docker bash -c "rm -rf /catalog/* /data/*" || (echo "Failed to clean stale files" && exit 1) echo "Creating Vertica Database (mydb) ..." docker exec -u dbadmin vertica_docker admintools -t create_db \ -s localhost \ - -d mydb \ - -p "MySecretPass" + -d mydb || (echo "Failed to create Vertica database" && exit 1) - echo "Vertica is up" - docker exec -u dbadmin vertica_docker vsql -w "MySecretPass" -c "SELECT version();" + echo "Vertica database is up" + docker exec -u dbadmin vertica_docker vsql -c "SELECT version();" || (echo "Failed to connect to Vertica database" && exit 1) + echo "Vertica is ready" - name: Build & Install UDx run: | + set -e # 1. Create the directory - docker exec -u root vertica_docker mkdir -p /var/odbc-loader + docker exec -u root vertica_docker mkdir -p /var/odbc-loader || (echo "Failed to create directory" && exit 1) # 2. Copy the CONTENTS of the workspace (note the trailing /.) # This ensures the Makefile is at /var/odbc-loader/Makefile - docker cp ${{ github.workspace }}/. vertica_docker:/var/odbc-loader + docker cp ${{ github.workspace }}/. vertica_docker:/var/odbc-loader || (echo "Failed to copy workspace" && exit 1) # 3. Fix permissions - docker exec -u root vertica_docker chown -R dbadmin:verticadba /var/odbc-loader + docker exec -u root vertica_docker chown -R dbadmin:verticadba /var/odbc-loader || (echo "Failed to change permissions" && exit 1) # 4. List files to verify (helps debugging) - docker exec -u dbadmin -w /var/odbc-loader vertica_docker ls -R + docker exec -u dbadmin -w /var/odbc-loader vertica_docker ls -R || (echo "Failed to list files" && exit 1) # 5. Run the build (make) docker exec -u dbadmin -w /var/odbc-loader vertica_docker \ - scl enable gcc-toolset-12 "make" + scl enable gcc-toolset-12 "make" || (echo "Build failed" && exit 1) - # 6. Run install with the password provided to vsql - # We set VSQL_PASSWORD which vsql recognizes automatically - docker exec -u dbadmin -w /var/odbc-loader -e VSQL_PASSWORD="MySecretPass" vertica_docker \ - scl enable gcc-toolset-12 "make install" + # 6. Run install without password (passwordless authentication) + docker exec -u dbadmin -w /var/odbc-loader vertica_docker \ + scl enable gcc-toolset-12 "make install" || (echo "UDx installation failed" && exit 1) + echo "UDx build and installation successful" - name: Install ODBC clients run: | + set -e # 1. Install wget so we can download the repo file - docker exec -u root vertica_docker dnf install -y wget + docker exec -u root vertica_docker dnf install -y wget || (echo "Failed to install wget" && exit 1) # 2. Download and install MySQL repo - docker exec -w /var/odbc-loader -u root vertica_docker wget https://repo.mysql.com/mysql80-community-release-el8-9.noarch.rpm - docker exec -w /var/odbc-loader -u root vertica_docker dnf install -y mysql80-community-release-el8-9.noarch.rpm + docker exec -w /var/odbc-loader -u root vertica_docker wget https://repo.mysql.com/mysql80-community-release-el8-9.noarch.rpm || (echo "Failed to download MySQL repo" && exit 1) + docker exec -w /var/odbc-loader -u root vertica_docker dnf install -y mysql80-community-release-el8-9.noarch.rpm || (echo "Failed to install MySQL repo" && exit 1) # 3. Install the connector (disabling GPG check to avoid prompt blocks) - docker exec -w /var/odbc-loader -u root vertica_docker dnf install -y --nogpgcheck mysql-connector-odbc + docker exec -w /var/odbc-loader -u root vertica_docker dnf install -y --nogpgcheck mysql-connector-odbc || (echo "Failed to install MySQL connector" && exit 1) + echo "ODBC clients installed successfully" - name: Run Tests run: | + set -e # 1. Locate the actual MySQL ODBC driver file # It might be libmyodbc8w.so, libmyodbc8a.so, or have a version suffix MYSQL_DRIVER_PATH=$(docker exec vertica_docker find /usr/lib64 -name "libmyodbc*.so" | head -n 1) + if [ -z "$MYSQL_DRIVER_PATH" ]; then + echo "Error: MySQL ODBC driver not found" + exit 1 + fi echo "Found MySQL Driver at: $MYSQL_DRIVER_PATH" # 2. Create a symlink so it matches what is in your odbcinst.ini # If your odbcinst.ini expects /usr/lib64/libmyodbc8w.so, we make sure it's there - docker exec -u root vertica_docker ln -sf "$MYSQL_DRIVER_PATH" /usr/lib64/libmyodbc8w.so + docker exec -u root vertica_docker ln -sf "$MYSQL_DRIVER_PATH" /usr/lib64/libmyodbc8w.so || (echo "Failed to create driver symlink" && exit 1) # 3. Point system-wide ODBC config to your repo files - docker exec -u root vertica_docker bash -c "cat /var/odbc-loader/tests/config/odbcinst.ini > /etc/odbcinst.ini" - docker exec -u root vertica_docker bash -c "cat /var/odbc-loader/tests/config/odbc.ini > /etc/odbc.ini" + docker exec -u root vertica_docker bash -c "cat /var/odbc-loader/tests/config/odbcinst.ini > /etc/odbcinst.ini" || (echo "Failed to configure odbcinst.ini" && exit 1) + docker exec -u root vertica_docker bash -c "cat /var/odbc-loader/tests/config/odbc.ini > /etc/odbc.ini" || (echo "Failed to configure odbc.ini" && exit 1) - # 4. Run the test + # 4. Run the test (passwordless Vertica connection) docker exec -w /var/odbc-loader \ -u dbadmin \ - -e VSQL_PASSWORD="MySecretPass" \ - vertica_docker make test \ No newline at end of file + vertica_docker make test || (echo "Tests failed" && exit 1) + echo "All tests passed" \ No newline at end of file From 4af44534ad8df417693dde45f58e75942ea7b550 Mon Sep 17 00:00:00 2001 From: mkottakota1 <149763406+mkottakota1@users.noreply.github.com> Date: Fri, 6 Feb 2026 11:38:07 +0000 Subject: [PATCH 15/16] corrected the commane for column pruning test --- tests/federated_queries.sql | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/federated_queries.sql b/tests/federated_queries.sql index c8cdeb6..c988f26 100644 --- a/tests/federated_queries.sql +++ b/tests/federated_queries.sql @@ -31,10 +31,10 @@ CREATE EXTERNAL TABLE public.epeople ( SELECT * FROM public.epeople WHERE id > 100; -- Test 2: Query with column pruning --- When only specific columns are selected, ODBCLoader optimizes the external query --- to fetch only the required columns. Columns not in the Vertica query are replaced --- with NULL values to match the External Table schema --- Expected: Only id column is fetched from MySQL (name is replaced with NULL) +-- When only specific columns are selected, ODBCLoader optimizes the external source query +-- to fetch only the required columns, reducing data transfer from the remote database. +-- Unselected columns are not fetched and do not appear in the query result. +-- Expected: Only id column is fetched from MySQL (name column is not selected, so it is not fetched) SELECT id FROM public.epeople WHERE id > 100; -- Test 3: Federated join query From acc40115da28ad3094fbdc01bec62157e7f36b53 Mon Sep 17 00:00:00 2001 From: mkottakota1 <149763406+mkottakota1@users.noreply.github.com> Date: Mon, 9 Feb 2026 06:21:53 +0000 Subject: [PATCH 16/16] Removing hardcoded USERNAME --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 048fd9b..7f3669b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,8 +16,8 @@ jobs: - name: Login to Docker Hub uses: docker/login-action@v3 with: - username: mkottakota419 - password: ${{ secrets.DOCKER_PASSWORD }} # Ensure this secret is set in GitHub Settings + username: ${{secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} - name: Set up MySQL server run: |