Migrate to Private Docker Image & Add Federated Queries Test Suite

.github/workflows/ci.yml

@@ -2,64 +2,130 @@ name: CI
 
 on:
   push:
-    branches: [ "main" ]
   pull_request:
-    branches: [ "main" ]
-  # Allows you to run this workflow manually from the Actions tab
   workflow_dispatch:
 
 jobs:
   build:
-
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
+
+    # 1. Login to Docker Hub to pull your private image
+    - name: Login to Docker Hub
+      uses: docker/login-action@v3
+      with:
+        username: ${{secrets.DOCKER_USERNAME }}
+        password: ${{ secrets.DOCKER_PASSWORD }}
+
     - name: Set up MySQL server
       run: |
         docker run -d -p 3306:3306 -e MYSQL_ALLOW_EMPTY_PASSWORD=yes --name test-mysql mysql:8.0
         echo "MySQL startup ..."
-        until docker exec test-mysql mysql -u root -e "SELECT 1;"; do \
-          echo "..."; \
-          sleep 1; \
-        done;
-        docker exec -u root test-mysql mysql -u root -e "CREATE DATABASE testdb;"
-        docker exec -u root test-mysql mysql -u root testdb -e "CREATE TABLE test_source (i integer, b boolean, f float, v varchar(32), c char(32), lv varchar(9999), bn binary(32), vb varbinary(32), lvb varbinary(9999), d date, t time, ts timestamp null, tz varchar(80), tsz varchar(80), n numeric(20,4));"
-        docker exec -u root test-mysql /bin/bash -c "(echo \"INSERT INTO test_source VALUES (null, null, null, null, null, null, null, null, null, null, null, null, null, null, null);\"; for i in \`seq 1 9\`; do echo \"INSERT INTO test_source VALUES (\$i, 1, \$i.5, 'test \$i', 'test \$i', 'test \$i', 'test \$i', 'test \$i', 'test \$i', '\$((\$i+11))00/1/\$i', '4:0\$i', '2038-01-0\$i 03:14:07', '1:2\$i:00', 'June 1, \$((\$i+11))00 03:2\$i EST', '123456.7890');\"; done) | mysql -u root testdb"
-        docker exec -u root test-mysql mysql -u root testdb -e "select * from test_source;"
-    - name: Set up a Vertica server
+        until docker exec test-mysql mysql -u root -e "SELECT 1;"; do
+          echo "..."
+          sleep 1
+        done
+        echo "Setting up MySQL databases and tables..."
+        docker exec -u root test-mysql mysql -u root -e "CREATE DATABASE testdb;" || (echo "Failed to create testdb" && exit 1)
+        docker exec -u root test-mysql mysql -u root testdb -e "CREATE TABLE test_source (i integer, b boolean, f float, v varchar(32), c char(32), lv varchar(9999), bn binary(32), vb varbinary(32), lvb varbinary(9999), d date, t time, ts timestamp null, tz varchar(80), tsz varchar(80), n numeric(20,4));" || (echo "Failed to create test_source table" && exit 1)
+        docker exec -u root test-mysql /bin/bash -c "(echo \"INSERT INTO test_source VALUES (null, null, null, null, null, null, null, null, null, null, null, null, null, null, null);\"; for i in \`seq 1 9\`; do echo \"INSERT INTO test_source VALUES (\$i, 1, \$i.5, 'test \$i', 'test \$i', 'test \$i', 'test \$i', 'test \$i', 'test \$i', '\$((\$i+11))00/1/\$i', '4:0\$i', '2038-01-0\$i 03:14:07', '1:2\$i:00', 'June 1, \$((\$i+11))00 03:2\$i EST', '123456.7890');\"; done) | mysql -u root testdb" || (echo "Failed to insert test data" && exit 1)
+        docker exec -u root test-mysql mysql -u root testdb -e "CREATE TABLE people (id integer PRIMARY KEY, name varchar(20));" || (echo "Failed to create people table" && exit 1)
+        docker exec -u root test-mysql mysql -u root testdb -e "INSERT INTO people VALUES (101, 'Alice'), (102, 'Bob'), (103, 'Charlie');" || (echo "Failed to insert people data" && exit 1)
+        echo "MySQL setup completed successfully"
+
+    - name: Set up Vertica server (Private Image)
       timeout-minutes: 15
       run: |
+        set -e
         docker run -d -p 5433:5433 -p 5444:5444 \
-          -e ODBCSYSINI=/var/odbc-loader/tests/config \
           --add-host=host.docker.internal:host-gateway \
           --name vertica_docker \
-          opentext/vertica-ce:24.2.0-1
-        echo "Vertica startup ..."
-        until docker exec vertica_docker test -f /data/vertica/VMart/agent_start.out; do \
-          echo "..."; \
-          sleep 3; \
-        done;
-        echo "Vertica is up"
-        docker exec -u dbadmin vertica_docker /opt/vertica/bin/vsql -c "\l"
-        docker exec -u dbadmin vertica_docker /opt/vertica/bin/vsql -c "select version()"
+          mkottakota419/vertica-ci:latest
+
+        echo "Preparing system environment..."
+        # 1. Install tools. 
+        # 2. Try to link, but don't fail if it's already there (|| true).
+        # 3. Verify the path exists so we can see it in the logs.
+        docker exec -u root vertica_docker bash -c "
+          dnf install -y iproute procps-ng && \
+          ln -sf /usr/sbin/ip /sbin/ip || true && \
+          ls -l /sbin/ip" || (echo "Failed to prepare system environment" && exit 1)
+
+        echo "Cleaning stale files..."
+        docker exec -u root vertica_docker bash -c "rm -rf /catalog/* /data/*" || (echo "Failed to clean stale files" && exit 1)
+
+        echo "Creating Vertica Database (mydb) ..."
+        docker exec -u dbadmin vertica_docker admintools -t create_db \
+          -s localhost \
+          -d mydb || (echo "Failed to create Vertica database" && exit 1)
+
+        echo "Vertica database is up"
+        docker exec -u dbadmin vertica_docker vsql -c "SELECT version();" || (echo "Failed to connect to Vertica database" && exit 1)
+        echo "Vertica is ready"
+
     - name: Build & Install UDx
       run: |
-        docker exec -u root vertica_docker dnf -y install gcc-toolset-9-gcc-c++.x86_64 
-        docker exec -u root vertica_docker dnf -y install unixODBC-devel
-        docker exec -u root vertica_docker dnf -y install pcre-devel
-        docker exec -u root vertica_docker dnf -y install perl
-        docker cp ${{ github.workspace }} vertica_docker:/var/odbc-loader
-        docker exec -u root vertica_docker /bin/bash -c "sudo chown -R dbadmin:verticadba /var/odbc-loader"
-        docker exec -u root -w /var/odbc-loader vertica_docker dnf install -y make
-        docker exec -u dbadmin -w /var/odbc-loader vertica_docker scl enable gcc-toolset-9 "bash -c 'make; make install'"
+        set -e
+        # 1. Create the directory
+        docker exec -u root vertica_docker mkdir -p /var/odbc-loader || (echo "Failed to create directory" && exit 1)
+
+        # 2. Copy the CONTENTS of the workspace (note the trailing /.)
+        # This ensures the Makefile is at /var/odbc-loader/Makefile
+        docker cp ${{ github.workspace }}/. vertica_docker:/var/odbc-loader || (echo "Failed to copy workspace" && exit 1)
+
+        # 3. Fix permissions
+        docker exec -u root vertica_docker chown -R dbadmin:verticadba /var/odbc-loader || (echo "Failed to change permissions" && exit 1)
+
+        # 4. List files to verify (helps debugging)
+        docker exec -u dbadmin -w /var/odbc-loader vertica_docker ls -R || (echo "Failed to list files" && exit 1)
+
+        # 5. Run the build (make)
+        docker exec -u dbadmin -w /var/odbc-loader vertica_docker \
+          scl enable gcc-toolset-12 "make" || (echo "Build failed" && exit 1)
+
+        # 6. Run install without password (passwordless authentication)
+        docker exec -u dbadmin -w /var/odbc-loader vertica_docker \
+          scl enable gcc-toolset-12 "make install" || (echo "UDx installation failed" && exit 1)
+        echo "UDx build and installation successful"
 
     - name: Install ODBC clients
       run: |
-        docker exec -w /var/odbc-loader -u root vertica_docker wget https://repo.mysql.com/mysql80-community-release-el8-9.noarch.rpm
-        docker exec -w /var/odbc-loader -u root vertica_docker dnf install -y mysql80-community-release-el8-9.noarch.rpm
-        docker exec -w /var/odbc-loader -u root vertica_docker dnf install -y mysql-connector-odbc
+        set -e
+        # 1. Install wget so we can download the repo file
+        docker exec -u root vertica_docker dnf install -y wget || (echo "Failed to install wget" && exit 1)
+
+        # 2. Download and install MySQL repo
+        docker exec -w /var/odbc-loader -u root vertica_docker wget https://repo.mysql.com/mysql80-community-release-el8-9.noarch.rpm || (echo "Failed to download MySQL repo" && exit 1)
+        docker exec -w /var/odbc-loader -u root vertica_docker dnf install -y mysql80-community-release-el8-9.noarch.rpm || (echo "Failed to install MySQL repo" && exit 1)
+
+        # 3. Install the connector (disabling GPG check to avoid prompt blocks)
+        docker exec -w /var/odbc-loader -u root vertica_docker dnf install -y --nogpgcheck mysql-connector-odbc || (echo "Failed to install MySQL connector" && exit 1)
+        echo "ODBC clients installed successfully"
 
     - name: Run Tests
       run: |
-        docker exec -w /var/odbc-loader -u dbadmin vertica_docker make test
+        set -e
+        # 1. Locate the actual MySQL ODBC driver file
+        # It might be libmyodbc8w.so, libmyodbc8a.so, or have a version suffix
+        MYSQL_DRIVER_PATH=$(docker exec vertica_docker find /usr/lib64 -name "libmyodbc*.so" | head -n 1)
+        if [ -z "$MYSQL_DRIVER_PATH" ]; then
+          echo "Error: MySQL ODBC driver not found"
+          exit 1
+        fi
+        echo "Found MySQL Driver at: $MYSQL_DRIVER_PATH"
+
+        # 2. Create a symlink so it matches what is in your odbcinst.ini
+        # If your odbcinst.ini expects /usr/lib64/libmyodbc8w.so, we make sure it's there
+        docker exec -u root vertica_docker ln -sf "$MYSQL_DRIVER_PATH" /usr/lib64/libmyodbc8w.so || (echo "Failed to create driver symlink" && exit 1)
+
+        # 3. Point system-wide ODBC config to your repo files
+        docker exec -u root vertica_docker bash -c "cat /var/odbc-loader/tests/config/odbcinst.ini > /etc/odbcinst.ini" || (echo "Failed to configure odbcinst.ini" && exit 1)
+        docker exec -u root vertica_docker bash -c "cat /var/odbc-loader/tests/config/odbc.ini > /etc/odbc.ini" || (echo "Failed to configure odbc.ini" && exit 1)
+
+        # 4. Run the test (passwordless Vertica connection)
+        docker exec -w /var/odbc-loader \
+          -u dbadmin \
+          vertica_docker make test || (echo "Tests failed" && exit 1)
+        echo "All tests passed"

Makefile

@@ -42,13 +42,28 @@ test_example:
 	@echo ALL TESTS SUCCESSFUL
 
 test:
-	@$(VSQL) -f tests/copy_test.sql > $(TMPDIR)/copy_test.out 2>&1
-	@diff -u tests/expected/copy_test.out <(perl -pe 's/^vsql:[\/_:\w\.]* /vsql: /; \
+	@echo "========================================="
+	@echo "Running ODBCLoader Test Suite"
+	@echo "========================================="
+	@echo ""
+	@echo "[*] Running Federated Queries Test..."
+	@echo ""
+	@$(VSQL) -f tests/federated_queries.sql 2>&1 | tee $(TMPDIR)/federated_queries.out | perl -pe 's/^vsql:[\/_:\w\.]* /vsql: /; \
 	              s/\[ODBC[^\]]*\]/[...]/g; \
 		      s/\[mysql[^\]]*\]/[...]/g; \
 		      s/(Error parsing .* )\(.*\)$$/$$1(...)/; \
-		      s/mariadb/MySQL/ig; ' $(TMPDIR)/copy_test.out)
-	@echo ALL TESTS SUCCESSFUL
+		      s/mariadb/MySQL/ig; '
+	@echo ""
+	@echo "[*] Validating test output..."
+	@diff -u tests/expected/federated_queries.out <(perl -pe 's/^vsql:[\/_:\w\.]* /vsql: /; \
+	              s/\[ODBC[^\]]*\]/[...]/g; \
+		      s/\[mysql[^\]]*\]/[...]/g; \
+		      s/(Error parsing .* )\(.*\)$$/$$1(...)/; \
+		      s/mariadb/MySQL/ig; ' $(TMPDIR)/federated_queries.out) && echo "[✓] Test validation passed" || (echo "[✗] Test validation failed" && exit 1)
+	@echo ""
+	@echo "========================================="
+	@echo "✓ ALL TESTS SUCCESSFUL"
+	@echo "========================================="
 
 .PHONY: build clean install uninstall example test_example test
 

tests/expected/federated_queries.out

@@ -0,0 +1,39 @@
+Timing is off.
+SET
+CREATE TABLE
+ id  |  name   
+-----+---------
+ 101 | Alice
+ 102 | Bob
+ 103 | Charlie
+(3 rows)
+
+ id  
+-----
+ 101
+ 102
+ 103
+(3 rows)
+
+CREATE TABLE
+ OUTPUT 
+--------
+      4
+(1 row)
+
+ id  |  name   |  status  
+-----+---------+----------
+ 101 | Alice   | inactive
+ 102 | Bob     | active
+ 103 | Charlie | inactive
+(3 rows)
+
+ id  |  name   
+-----+---------
+ 101 | Alice
+ 102 | Bob
+ 103 | Charlie
+(3 rows)
+
+DROP TABLE
+DROP TABLE

tests/federated_queries.sql

@@ -0,0 +1,69 @@
+\timing off
+
+-- Set output to a fixed timezone regardless of where this is being tested
+set time zone to 'EST';
+
+-- =====================================================
+-- Federated Queries Test Case
+-- =====================================================
+-- This test demonstrates the use of ODBCLoader for federated queries
+-- allowing Vertica to query external databases (e.g., MySQL) while
+-- leveraging predicate pushdown and column pruning to minimize data movement
+
+-- Create an External Table in Vertica that acts as a gateway to MySQL
+-- The External Table definition stores metadata about the external source
+-- but does not retrieve any data until queried
+CREATE EXTERNAL TABLE public.epeople (
+    id INTEGER,
+    name VARCHAR(20)
+) AS COPY WITH
+    SOURCE ODBCSource()
+    PARSER ODBCLoader(
+        connect='DSN=MySQL',
+        query='SELECT * FROM testdb.people'
+);
+
+-- Test 1: Query with predicate pushdown
+-- When executing a query with a WHERE clause, ODBCLoader rewrites the original
+-- external query to include the predicate, pushing the filter to the external database
+-- This minimizes the amount of data transferred from MySQL to Vertica
+-- Expected: Only people with id > 100 are retrieved from MySQL
+SELECT * FROM public.epeople WHERE id > 100;
+
+-- Test 2: Query with column pruning
+-- When only specific columns are selected, ODBCLoader optimizes the external source query
+-- to fetch only the required columns, reducing data transfer from the remote database.
+-- Unselected columns are not fetched and do not appear in the query result.
+-- Expected: Only id column is fetched from MySQL (name column is not selected, so it is not fetched)
+SELECT id FROM public.epeople WHERE id > 100;
+
+-- Test 3: Federated join query
+-- Demonstrates joining an external table (MySQL) with a Vertica-managed table
+-- This leverages both databases' query engines for optimal performance
+-- Create a temporary Vertica table for joining
+CREATE TABLE public.employee_status (
+    id INTEGER,
+    status VARCHAR(20)
+);
+
+INSERT INTO public.employee_status VALUES 
+    (1, 'active'),
+    (101, 'inactive'),
+    (102, 'active'),
+    (103, 'inactive');
+
+-- Perform a federated join combining data from MySQL (epeople) and Vertica (employee_status)
+-- The ODBCLoader will push down predicates to MySQL when possible
+SELECT epeople.id, epeople.name, employee_status.status
+FROM public.epeople 
+JOIN public.employee_status ON epeople.id = employee_status.id
+WHERE epeople.id > 100;
+
+-- Test 4: Complex predicate pushdown
+-- Multiple predicates are combined and pushed to the external database
+-- Expected: MySQL executes: SELECT * FROM testdb.people WHERE id > 50 AND id < 150
+SELECT id, name FROM public.epeople WHERE id > 50 AND id < 150;
+
+-- Clean up external and temporary tables
+DROP TABLE public.epeople;
+DROP TABLE public.employee_status;