diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 73a23dc..7f3669b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,64 +2,130 @@ name: CI on: push: - branches: [ "main" ] pull_request: - branches: [ "main" ] - # Allows you to run this workflow manually from the Actions tab workflow_dispatch: jobs: build: - runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 + + # 1. Login to Docker Hub to pull your private image + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + - name: Set up MySQL server run: | docker run -d -p 3306:3306 -e MYSQL_ALLOW_EMPTY_PASSWORD=yes --name test-mysql mysql:8.0 echo "MySQL startup ..." - until docker exec test-mysql mysql -u root -e "SELECT 1;"; do \ - echo "..."; \ - sleep 1; \ - done; - docker exec -u root test-mysql mysql -u root -e "CREATE DATABASE testdb;" - docker exec -u root test-mysql mysql -u root testdb -e "CREATE TABLE test_source (i integer, b boolean, f float, v varchar(32), c char(32), lv varchar(9999), bn binary(32), vb varbinary(32), lvb varbinary(9999), d date, t time, ts timestamp null, tz varchar(80), tsz varchar(80), n numeric(20,4));" - docker exec -u root test-mysql /bin/bash -c "(echo \"INSERT INTO test_source VALUES (null, null, null, null, null, null, null, null, null, null, null, null, null, null, null);\"; for i in \`seq 1 9\`; do echo \"INSERT INTO test_source VALUES (\$i, 1, \$i.5, 'test \$i', 'test \$i', 'test \$i', 'test \$i', 'test \$i', 'test \$i', '\$((\$i+11))00/1/\$i', '4:0\$i', '2038-01-0\$i 03:14:07', '1:2\$i:00', 'June 1, \$((\$i+11))00 03:2\$i EST', '123456.7890');\"; done) | mysql -u root testdb" - docker exec -u root test-mysql mysql -u root testdb -e "select * from test_source;" - - name: Set up a Vertica server + until docker exec test-mysql mysql -u root -e "SELECT 1;"; do + echo "..." + sleep 1 + done + echo "Setting up MySQL databases and tables..." + docker exec -u root test-mysql mysql -u root -e "CREATE DATABASE testdb;" || (echo "Failed to create testdb" && exit 1) + docker exec -u root test-mysql mysql -u root testdb -e "CREATE TABLE test_source (i integer, b boolean, f float, v varchar(32), c char(32), lv varchar(9999), bn binary(32), vb varbinary(32), lvb varbinary(9999), d date, t time, ts timestamp null, tz varchar(80), tsz varchar(80), n numeric(20,4));" || (echo "Failed to create test_source table" && exit 1) + docker exec -u root test-mysql /bin/bash -c "(echo \"INSERT INTO test_source VALUES (null, null, null, null, null, null, null, null, null, null, null, null, null, null, null);\"; for i in \`seq 1 9\`; do echo \"INSERT INTO test_source VALUES (\$i, 1, \$i.5, 'test \$i', 'test \$i', 'test \$i', 'test \$i', 'test \$i', 'test \$i', '\$((\$i+11))00/1/\$i', '4:0\$i', '2038-01-0\$i 03:14:07', '1:2\$i:00', 'June 1, \$((\$i+11))00 03:2\$i EST', '123456.7890');\"; done) | mysql -u root testdb" || (echo "Failed to insert test data" && exit 1) + docker exec -u root test-mysql mysql -u root testdb -e "CREATE TABLE people (id integer PRIMARY KEY, name varchar(20));" || (echo "Failed to create people table" && exit 1) + docker exec -u root test-mysql mysql -u root testdb -e "INSERT INTO people VALUES (101, 'Alice'), (102, 'Bob'), (103, 'Charlie');" || (echo "Failed to insert people data" && exit 1) + echo "MySQL setup completed successfully" + + - name: Set up Vertica server (Private Image) timeout-minutes: 15 run: | + set -e docker run -d -p 5433:5433 -p 5444:5444 \ - -e ODBCSYSINI=/var/odbc-loader/tests/config \ --add-host=host.docker.internal:host-gateway \ --name vertica_docker \ - opentext/vertica-ce:24.2.0-1 - echo "Vertica startup ..." - until docker exec vertica_docker test -f /data/vertica/VMart/agent_start.out; do \ - echo "..."; \ - sleep 3; \ - done; - echo "Vertica is up" - docker exec -u dbadmin vertica_docker /opt/vertica/bin/vsql -c "\l" - docker exec -u dbadmin vertica_docker /opt/vertica/bin/vsql -c "select version()" + mkottakota419/vertica-ci:latest + + echo "Preparing system environment..." + # 1. Install tools. + # 2. Try to link, but don't fail if it's already there (|| true). + # 3. Verify the path exists so we can see it in the logs. + docker exec -u root vertica_docker bash -c " + dnf install -y iproute procps-ng && \ + ln -sf /usr/sbin/ip /sbin/ip || true && \ + ls -l /sbin/ip" || (echo "Failed to prepare system environment" && exit 1) + + echo "Cleaning stale files..." + docker exec -u root vertica_docker bash -c "rm -rf /catalog/* /data/*" || (echo "Failed to clean stale files" && exit 1) + + echo "Creating Vertica Database (mydb) ..." + docker exec -u dbadmin vertica_docker admintools -t create_db \ + -s localhost \ + -d mydb || (echo "Failed to create Vertica database" && exit 1) + + echo "Vertica database is up" + docker exec -u dbadmin vertica_docker vsql -c "SELECT version();" || (echo "Failed to connect to Vertica database" && exit 1) + echo "Vertica is ready" + - name: Build & Install UDx run: | - docker exec -u root vertica_docker dnf -y install gcc-toolset-9-gcc-c++.x86_64 - docker exec -u root vertica_docker dnf -y install unixODBC-devel - docker exec -u root vertica_docker dnf -y install pcre-devel - docker exec -u root vertica_docker dnf -y install perl - docker cp ${{ github.workspace }} vertica_docker:/var/odbc-loader - docker exec -u root vertica_docker /bin/bash -c "sudo chown -R dbadmin:verticadba /var/odbc-loader" - docker exec -u root -w /var/odbc-loader vertica_docker dnf install -y make - docker exec -u dbadmin -w /var/odbc-loader vertica_docker scl enable gcc-toolset-9 "bash -c 'make; make install'" + set -e + # 1. Create the directory + docker exec -u root vertica_docker mkdir -p /var/odbc-loader || (echo "Failed to create directory" && exit 1) + + # 2. Copy the CONTENTS of the workspace (note the trailing /.) + # This ensures the Makefile is at /var/odbc-loader/Makefile + docker cp ${{ github.workspace }}/. vertica_docker:/var/odbc-loader || (echo "Failed to copy workspace" && exit 1) + + # 3. Fix permissions + docker exec -u root vertica_docker chown -R dbadmin:verticadba /var/odbc-loader || (echo "Failed to change permissions" && exit 1) + + # 4. List files to verify (helps debugging) + docker exec -u dbadmin -w /var/odbc-loader vertica_docker ls -R || (echo "Failed to list files" && exit 1) + + # 5. Run the build (make) + docker exec -u dbadmin -w /var/odbc-loader vertica_docker \ + scl enable gcc-toolset-12 "make" || (echo "Build failed" && exit 1) + + # 6. Run install without password (passwordless authentication) + docker exec -u dbadmin -w /var/odbc-loader vertica_docker \ + scl enable gcc-toolset-12 "make install" || (echo "UDx installation failed" && exit 1) + echo "UDx build and installation successful" - name: Install ODBC clients run: | - docker exec -w /var/odbc-loader -u root vertica_docker wget https://repo.mysql.com/mysql80-community-release-el8-9.noarch.rpm - docker exec -w /var/odbc-loader -u root vertica_docker dnf install -y mysql80-community-release-el8-9.noarch.rpm - docker exec -w /var/odbc-loader -u root vertica_docker dnf install -y mysql-connector-odbc + set -e + # 1. Install wget so we can download the repo file + docker exec -u root vertica_docker dnf install -y wget || (echo "Failed to install wget" && exit 1) + + # 2. Download and install MySQL repo + docker exec -w /var/odbc-loader -u root vertica_docker wget https://repo.mysql.com/mysql80-community-release-el8-9.noarch.rpm || (echo "Failed to download MySQL repo" && exit 1) + docker exec -w /var/odbc-loader -u root vertica_docker dnf install -y mysql80-community-release-el8-9.noarch.rpm || (echo "Failed to install MySQL repo" && exit 1) + + # 3. Install the connector (disabling GPG check to avoid prompt blocks) + docker exec -w /var/odbc-loader -u root vertica_docker dnf install -y --nogpgcheck mysql-connector-odbc || (echo "Failed to install MySQL connector" && exit 1) + echo "ODBC clients installed successfully" - name: Run Tests run: | - docker exec -w /var/odbc-loader -u dbadmin vertica_docker make test + set -e + # 1. Locate the actual MySQL ODBC driver file + # It might be libmyodbc8w.so, libmyodbc8a.so, or have a version suffix + MYSQL_DRIVER_PATH=$(docker exec vertica_docker find /usr/lib64 -name "libmyodbc*.so" | head -n 1) + if [ -z "$MYSQL_DRIVER_PATH" ]; then + echo "Error: MySQL ODBC driver not found" + exit 1 + fi + echo "Found MySQL Driver at: $MYSQL_DRIVER_PATH" + + # 2. Create a symlink so it matches what is in your odbcinst.ini + # If your odbcinst.ini expects /usr/lib64/libmyodbc8w.so, we make sure it's there + docker exec -u root vertica_docker ln -sf "$MYSQL_DRIVER_PATH" /usr/lib64/libmyodbc8w.so || (echo "Failed to create driver symlink" && exit 1) + + # 3. Point system-wide ODBC config to your repo files + docker exec -u root vertica_docker bash -c "cat /var/odbc-loader/tests/config/odbcinst.ini > /etc/odbcinst.ini" || (echo "Failed to configure odbcinst.ini" && exit 1) + docker exec -u root vertica_docker bash -c "cat /var/odbc-loader/tests/config/odbc.ini > /etc/odbc.ini" || (echo "Failed to configure odbc.ini" && exit 1) + + # 4. Run the test (passwordless Vertica connection) + docker exec -w /var/odbc-loader \ + -u dbadmin \ + vertica_docker make test || (echo "Tests failed" && exit 1) + echo "All tests passed" \ No newline at end of file diff --git a/Makefile b/Makefile index 7fa3064..b98f6d3 100644 --- a/Makefile +++ b/Makefile @@ -42,13 +42,28 @@ test_example: @echo ALL TESTS SUCCESSFUL test: - @$(VSQL) -f tests/copy_test.sql > $(TMPDIR)/copy_test.out 2>&1 - @diff -u tests/expected/copy_test.out <(perl -pe 's/^vsql:[\/_:\w\.]* /vsql: /; \ + @echo "=========================================" + @echo "Running ODBCLoader Test Suite" + @echo "=========================================" + @echo "" + @echo "[*] Running Federated Queries Test..." + @echo "" + @$(VSQL) -f tests/federated_queries.sql 2>&1 | tee $(TMPDIR)/federated_queries.out | perl -pe 's/^vsql:[\/_:\w\.]* /vsql: /; \ s/\[ODBC[^\]]*\]/[...]/g; \ s/\[mysql[^\]]*\]/[...]/g; \ s/(Error parsing .* )\(.*\)$$/$$1(...)/; \ - s/mariadb/MySQL/ig; ' $(TMPDIR)/copy_test.out) - @echo ALL TESTS SUCCESSFUL + s/mariadb/MySQL/ig; ' + @echo "" + @echo "[*] Validating test output..." + @diff -u tests/expected/federated_queries.out <(perl -pe 's/^vsql:[\/_:\w\.]* /vsql: /; \ + s/\[ODBC[^\]]*\]/[...]/g; \ + s/\[mysql[^\]]*\]/[...]/g; \ + s/(Error parsing .* )\(.*\)$$/$$1(...)/; \ + s/mariadb/MySQL/ig; ' $(TMPDIR)/federated_queries.out) && echo "[✓] Test validation passed" || (echo "[✗] Test validation failed" && exit 1) + @echo "" + @echo "=========================================" + @echo "✓ ALL TESTS SUCCESSFUL" + @echo "=========================================" .PHONY: build clean install uninstall example test_example test diff --git a/tests/expected/federated_queries.out b/tests/expected/federated_queries.out new file mode 100644 index 0000000..111e800 --- /dev/null +++ b/tests/expected/federated_queries.out @@ -0,0 +1,39 @@ +Timing is off. +SET +CREATE TABLE + id | name +-----+--------- + 101 | Alice + 102 | Bob + 103 | Charlie +(3 rows) + + id +----- + 101 + 102 + 103 +(3 rows) + +CREATE TABLE + OUTPUT +-------- + 4 +(1 row) + + id | name | status +-----+---------+---------- + 101 | Alice | inactive + 102 | Bob | active + 103 | Charlie | inactive +(3 rows) + + id | name +-----+--------- + 101 | Alice + 102 | Bob + 103 | Charlie +(3 rows) + +DROP TABLE +DROP TABLE diff --git a/tests/federated_queries.sql b/tests/federated_queries.sql new file mode 100644 index 0000000..c988f26 --- /dev/null +++ b/tests/federated_queries.sql @@ -0,0 +1,69 @@ +\timing off + +-- Set output to a fixed timezone regardless of where this is being tested +set time zone to 'EST'; + +-- ===================================================== +-- Federated Queries Test Case +-- ===================================================== +-- This test demonstrates the use of ODBCLoader for federated queries +-- allowing Vertica to query external databases (e.g., MySQL) while +-- leveraging predicate pushdown and column pruning to minimize data movement + +-- Create an External Table in Vertica that acts as a gateway to MySQL +-- The External Table definition stores metadata about the external source +-- but does not retrieve any data until queried +CREATE EXTERNAL TABLE public.epeople ( + id INTEGER, + name VARCHAR(20) +) AS COPY WITH + SOURCE ODBCSource() + PARSER ODBCLoader( + connect='DSN=MySQL', + query='SELECT * FROM testdb.people' +); + +-- Test 1: Query with predicate pushdown +-- When executing a query with a WHERE clause, ODBCLoader rewrites the original +-- external query to include the predicate, pushing the filter to the external database +-- This minimizes the amount of data transferred from MySQL to Vertica +-- Expected: Only people with id > 100 are retrieved from MySQL +SELECT * FROM public.epeople WHERE id > 100; + +-- Test 2: Query with column pruning +-- When only specific columns are selected, ODBCLoader optimizes the external source query +-- to fetch only the required columns, reducing data transfer from the remote database. +-- Unselected columns are not fetched and do not appear in the query result. +-- Expected: Only id column is fetched from MySQL (name column is not selected, so it is not fetched) +SELECT id FROM public.epeople WHERE id > 100; + +-- Test 3: Federated join query +-- Demonstrates joining an external table (MySQL) with a Vertica-managed table +-- This leverages both databases' query engines for optimal performance +-- Create a temporary Vertica table for joining +CREATE TABLE public.employee_status ( + id INTEGER, + status VARCHAR(20) +); + +INSERT INTO public.employee_status VALUES + (1, 'active'), + (101, 'inactive'), + (102, 'active'), + (103, 'inactive'); + +-- Perform a federated join combining data from MySQL (epeople) and Vertica (employee_status) +-- The ODBCLoader will push down predicates to MySQL when possible +SELECT epeople.id, epeople.name, employee_status.status +FROM public.epeople +JOIN public.employee_status ON epeople.id = employee_status.id +WHERE epeople.id > 100; + +-- Test 4: Complex predicate pushdown +-- Multiple predicates are combined and pushed to the external database +-- Expected: MySQL executes: SELECT * FROM testdb.people WHERE id > 50 AND id < 150 +SELECT id, name FROM public.epeople WHERE id > 50 AND id < 150; + +-- Clean up external and temporary tables +DROP TABLE public.epeople; +DROP TABLE public.employee_status;