From e035df0f8e10cd070c8b78f5527e5d41d386d103 Mon Sep 17 00:00:00 2001 From: Nick Woolmer <29717167+nwoolmer@users.noreply.github.com> Date: Wed, 4 Mar 2026 15:46:14 +0000 Subject: [PATCH 1/2] feat: SQLAlchemy 2.0 compatibility with 1.4 backward compat - Add import_dbapi() classmethod alongside dbapi() (fixes #9, #31) - Version-conditional create_engine params (future/implicit_returning only passed on SA <2.0) - Fix inspector to use connection-based execution (Engine.execute() removed in SA 2.0) - Declare runtime dependencies in pyproject.toml - Remove SQLAlchemy <2 cap from test dependencies - Add Python 3.12/3.13 classifiers, bump version to 1.2.0 - Update Dockerfile from EOL buster to bookworm, Python 3.13 - Add SA 2.0 compatibility test suite Closes #9, closes #31 Refs #35 Co-Authored-By: Claude Opus 4.6 --- Dockerfile | 15 ++-- Makefile | 2 +- pyproject.toml | 13 ++- src/questdb_connect/dialect.py | 34 ++++++-- src/questdb_connect/inspector.py | 136 ++++++++++++++++++------------- tests/test_sa2_compat.py | 64 +++++++++++++++ 6 files changed, 186 insertions(+), 78 deletions(-) create mode 100644 tests/test_sa2_compat.py diff --git a/Dockerfile b/Dockerfile index a514e5e..2722443 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,17 +1,16 @@ -FROM python:3.10-slim-buster +FROM python:3.13-slim-bookworm ENV ARCHITECTURE=x64 ENV PYTHONDONTWRITEBYTECODE 1 # Keeps Python from generating .pyc files in the container ENV PYTHONUNBUFFERED 1 # Turns off buffering for easier container logging -ENV SQLALCHEMY_SILENCE_UBER_WARNING 1 # because we really should upgrade to SQLAlchemy 2.x ENV QUESTDB_CONNECT_HOST "host.docker.internal" -RUN apt-get -y update -RUN apt-get -y upgrade -RUN apt-get -y --no-install-recommends install syslog-ng ca-certificates vim procps unzip less tar gzip iputils-ping gcc build-essential -RUN apt-get clean -RUN rm -rf /var/lib/apt/lists/* +RUN apt-get -y update \ + && apt-get -y upgrade \ + && apt-get -y --no-install-recommends install ca-certificates vim procps unzip less tar gzip iputils-ping gcc build-essential \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* COPY . /app WORKDIR /app -RUN pip install -U pip && pip install psycopg2-binary 'SQLAlchemy<=1.4.47' . +RUN pip install -U pip && pip install . CMD ["python", "src/examples/sqlalchemy_orm.py"] diff --git a/Makefile b/Makefile index 25238a1..9a95640 100644 --- a/Makefile +++ b/Makefile @@ -33,7 +33,7 @@ compose-down: echo "y" | docker volume prune docker-test: - docker run -e QUESTDB_CONNECT_HOST='host.docker.internal' -e SQLALCHEMY_SILENCE_UBER_WARNING=1 questdb/questdb-connect:latest + docker run -e QUESTDB_CONNECT_HOST='host.docker.internal' questdb/questdb-connect:latest test: python3 -m pytest diff --git a/pyproject.toml b/pyproject.toml index 2df337a..c8c1906 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] # https://pip.pypa.io/en/stable/reference/build-system/pyproject-toml/ name = 'questdb-connect' -version = '1.1.5' # Standalone production version (with engine) +version = '1.2.0' # SA 1.4 + 2.0 dual compat # version = '0.0.113' # testing version authors = [{ name = 'questdb.io', email = 'support@questdb.io' }] description = "SqlAlchemy library" @@ -14,8 +14,14 @@ classifiers = [ 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', + 'Programming Language :: Python :: 3.13', +] +dependencies = [ + 'SQLAlchemy>=1.4', + 'psycopg2-binary>=2.9', + 'packaging', ] -dependencies = [] [project.urls] 'Homepage' = "https://github.com/questdb/questdb-connect/" @@ -32,7 +38,7 @@ questdb = 'qdb_superset.db_engine_specs.questdb:QuestDbEngineSpec' [project.optional-dependencies] test = [ 'psycopg2-binary~=2.9.6', - 'SQLAlchemy>=1.4, <2', + 'SQLAlchemy>=1.4', 'apache-superset>=3.0.0', 'sqlparse==0.4.4', 'pytest~=7.3.0', @@ -63,6 +69,7 @@ max-args = 10 'tests/test_dialect.py' = ['S101', 'PLR2004'] 'tests/test_types.py' = ['S101'] 'tests/test_superset.py' = ['S101'] +'tests/test_sa2_compat.py' = ['S101', 'PLR2004'] 'tests/conftest.py' = ['S608'] 'src/examples/sqlalchemy_raw.py' = ['S608'] 'src/examples/server_utilisation.py' = ['S311'] diff --git a/src/questdb_connect/dialect.py b/src/questdb_connect/dialect.py index 4a236b4..9d7bd61 100644 --- a/src/questdb_connect/dialect.py +++ b/src/questdb_connect/dialect.py @@ -1,6 +1,7 @@ import abc import sqlalchemy +from packaging.version import Version from sqlalchemy.dialects.postgresql.psycopg2 import PGDialect_psycopg2 from sqlalchemy.sql.compiler import GenericTypeCompiler @@ -9,7 +10,10 @@ from .inspector import QDBInspector # ===== SQLAlchemy Dialect ====== -# https://docs.sqlalchemy.org/en/14/ apache-superset requires SQLAlchemy 1.4 +# https://docs.sqlalchemy.org/en/20/ + +SA_VERSION = Version(sqlalchemy.__version__) +SA_V2 = SA_VERSION >= Version("2.0") def connection_uri( @@ -21,24 +25,32 @@ def connection_uri( def create_engine( host: str, port: str, username: str, password: str, database: str = "main" ): + kwargs = { + "hide_parameters": False, + "isolation_level": "REPEATABLE READ", + } + if not SA_V2: + kwargs["future"] = True + kwargs["implicit_returning"] = False return sqlalchemy.create_engine( connection_uri(host, port, username, password, database), - future=True, - hide_parameters=False, - implicit_returning=False, - isolation_level="REPEATABLE READ", + **kwargs, ) def create_superset_engine( host: str, port: str, username: str, password: str, database: str = "main" ): + kwargs = { + "hide_parameters": False, + "isolation_level": "REPEATABLE READ", + } + if not SA_V2: + kwargs["future"] = False + kwargs["implicit_returning"] = True return sqlalchemy.create_engine( connection_uri(host, port, username, password, database), - future=False, - hide_parameters=False, - implicit_returning=True, - isolation_level="REPEATABLE READ", + **kwargs, ) @@ -73,6 +85,10 @@ def dbapi(cls): return dbapi + @classmethod + def import_dbapi(cls): + return cls.dbapi() + def get_schema_names(self, conn, **kw): return ["public"] diff --git a/src/questdb_connect/inspector.py b/src/questdb_connect/inspector.py index b6b5dc7..8bccbb5 100644 --- a/src/questdb_connect/inspector.py +++ b/src/questdb_connect/inspector.py @@ -2,6 +2,7 @@ import psycopg2 import sqlalchemy +from sqlalchemy.engine import Connection from .common import PartitionBy from .table_engine import QDBTableEngine @@ -9,6 +10,16 @@ class QDBInspector(sqlalchemy.engine.reflection.Inspector, abc.ABC): + def _get_connection(self): + """Get a usable connection from self.bind. + + In SA 1.4, self.bind may be an Engine (with .execute()). + In SA 2.0, Engine.execute() is removed, so we must use .connect(). + """ + if isinstance(self.bind, Connection): + return self.bind + return self.bind.connect() + def reflecttable( self, table, @@ -32,71 +43,82 @@ def reflect_table( _reflect_info=None, ): table_name = table.name + conn = self._get_connection() try: - result_set = self.bind.execute( - sqlalchemy.text( - "SELECT designatedTimestamp, partitionBy, walEnabled FROM tables() WHERE table_name = :tn" - ), - {"tn": table_name}, - ) - except psycopg2.DatabaseError: - # older version - result_set = self.bind.execute( + try: + result_set = conn.execute( + sqlalchemy.text( + "SELECT designatedTimestamp, partitionBy, walEnabled FROM tables() WHERE table_name = :tn" + ), + {"tn": table_name}, + ) + except psycopg2.DatabaseError: + # older QuestDB version uses 'name' instead of 'table_name' + result_set = conn.execute( + sqlalchemy.text( + "SELECT designatedTimestamp, partitionBy, walEnabled FROM tables() WHERE name = :tn" + ), + {"tn": table_name}, + ) + if not result_set: + self._panic_table(table_name) + table_attrs = result_set.first() + if table_attrs: + col_ts_name = table_attrs[0] + partition_by = PartitionBy[table_attrs[1]] + is_wal = True if table_attrs[2] else False + else: + col_ts_name = None + partition_by = PartitionBy.NONE + is_wal = True + dedup_upsert_keys = [] + for row in conn.execute( sqlalchemy.text( - "SELECT designatedTimestamp, partitionBy, walEnabled FROM tables() WHERE name = :tn" + 'SELECT "column", "type", "upsertKey" FROM table_columns(:tn)' ), {"tn": table_name}, - ) - if not result_set: - self._panic_table(table_name) - table_attrs = result_set.first() - if table_attrs: - col_ts_name = table_attrs[0] - partition_by = PartitionBy[table_attrs[1]] - is_wal = True if table_attrs[2] else False - else: - col_ts_name = None - partition_by = PartitionBy.NONE - is_wal = True - dedup_upsert_keys = [] - for row in self.bind.execute( - sqlalchemy.text( - 'SELECT "column", "type", "upsertKey" FROM table_columns(:tn)' - ), - {"tn": table_name}, - ): - col_name = row[0] - if include_columns and col_name not in include_columns: - continue - if exclude_columns and col_name in exclude_columns: - continue - if row[2]: # upsertKey - dedup_upsert_keys.append(col_name) - col_type = resolve_type_from_name(row[1]) - table.append_column( - sqlalchemy.Column( - col_name, - col_type, - primary_key=( - col_ts_name and col_ts_name.upper() == col_name.upper() - ), + ): + col_name = row[0] + if include_columns and col_name not in include_columns: + continue + if exclude_columns and col_name in exclude_columns: + continue + if row[2]: # upsertKey + dedup_upsert_keys.append(col_name) + col_type = resolve_type_from_name(row[1]) + table.append_column( + sqlalchemy.Column( + col_name, + col_type, + primary_key=( + col_ts_name and col_ts_name.upper() == col_name.upper() + ), + ) ) + table.engine = QDBTableEngine( + table_name, + col_ts_name, + partition_by, + is_wal, + tuple(dedup_upsert_keys) if dedup_upsert_keys else None, ) - table.engine = QDBTableEngine( - table_name, - col_ts_name, - partition_by, - is_wal, - tuple(dedup_upsert_keys) if dedup_upsert_keys else None, - ) - table.metadata = sqlalchemy.MetaData() + table.metadata = sqlalchemy.MetaData() + finally: + # Close the connection if we opened it (bind was an Engine) + if not isinstance(self.bind, Connection): + conn.close() def get_columns(self, table_name, schema=None, **kw): - result_set = self.bind.execute( - sqlalchemy.text('SELECT "column", "type" FROM table_columns(:tn)'), - {"tn": table_name}, - ) - return self.format_table_columns(table_name, result_set) + conn = self._get_connection() + try: + result_set = conn.execute( + sqlalchemy.text('SELECT "column", "type" FROM table_columns(:tn)'), + {"tn": table_name}, + ) + return self.format_table_columns(table_name, result_set) + finally: + if not isinstance(self.bind, Connection): + conn.close() def get_schema_names(self): return ["public"] diff --git a/tests/test_sa2_compat.py b/tests/test_sa2_compat.py new file mode 100644 index 0000000..7021a69 --- /dev/null +++ b/tests/test_sa2_compat.py @@ -0,0 +1,64 @@ +"""Tests for SQLAlchemy 2.0 compatibility. + +These tests verify the SA 2.0 migration changes work correctly. +""" +import questdb_connect as qdbc +import sqlalchemy +from questdb_connect.dialect import SA_V2, QuestDBDialect + + +def test_import_dbapi(): + """import_dbapi() must exist and return the questdb_connect module.""" + dbapi = QuestDBDialect.import_dbapi() + assert dbapi is qdbc + + +def test_dbapi_still_works(): + """dbapi() must still work for SA 1.4 backward compat.""" + dbapi = QuestDBDialect.dbapi() + assert dbapi is qdbc + + +def test_create_engine_no_deprecated_params(test_config): + """create_engine() must not pass deprecated params on SA 2.0.""" + import warnings + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + engine = qdbc.create_engine( + test_config.host, + test_config.port, + test_config.username, + test_config.password, + test_config.database, + ) + sa_dep = [x for x in w if issubclass(x.category, DeprecationWarning) + and ("implicit_returning" in str(x.message) or "future" in str(x.message))] + assert len(sa_dep) == 0, f"Unexpected deprecation warnings: {sa_dep}" + engine.dispose() + + +def test_create_superset_engine_no_deprecated_params(test_config): + """create_superset_engine() must not pass deprecated params on SA 2.0.""" + import warnings + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + engine = qdbc.create_superset_engine( + test_config.host, + test_config.port, + test_config.username, + test_config.password, + test_config.database, + ) + sa_dep = [x for x in w if issubclass(x.category, DeprecationWarning) + and ("implicit_returning" in str(x.message) or "future" in str(x.message))] + assert len(sa_dep) == 0, f"Unexpected deprecation warnings: {sa_dep}" + engine.dispose() + + +def test_sa_version_detection(): + """SA_V2 flag must match the installed SQLAlchemy version.""" + major = int(sqlalchemy.__version__.split(".")[0]) + if major >= 2: + assert SA_V2 is True + else: + assert SA_V2 is False From f7d7ace097fe9b6e43877308ed9f63c3b37fce87 Mon Sep 17 00:00:00 2001 From: Nick Woolmer <29717167+nwoolmer@users.noreply.github.com> Date: Wed, 4 Mar 2026 17:43:30 +0000 Subject: [PATCH 2/2] feat: Superset 4.1+ and SA 2.0-only support (v2.0.0) Breaking change: drops SQLAlchemy 1.4 and Superset <4.1 support. - Update select_star signature: table_name/schema -> table: Table (4.1+ API) - Update execute signature: add database positional param (4.1+ API) - Update strip_comments_from_sql call with engine= kwarg - Drop SA 1.4 compat: remove SA_V2 flag, version conditionals, dbapi() - Remove packaging dependency (no longer needed for version detection) - Pin SQLAlchemy>=2.0, apache-superset>=4.1.0 - Bump version to 2.0.0 Co-Authored-By: Claude Opus 4.6 --- pyproject.toml | 9 +++---- src/qdb_superset/db_engine_specs/questdb.py | 27 ++++++++++--------- src/questdb_connect/dialect.py | 30 ++++----------------- tests/test_sa2_compat.py | 20 +++++--------- 4 files changed, 29 insertions(+), 57 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index c8c1906..cd4e1dd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] # https://pip.pypa.io/en/stable/reference/build-system/pyproject-toml/ name = 'questdb-connect' -version = '1.2.0' # SA 1.4 + 2.0 dual compat +version = '2.0.0' # SA 2.0 + Superset 4.1+ # version = '0.0.113' # testing version authors = [{ name = 'questdb.io', email = 'support@questdb.io' }] description = "SqlAlchemy library" @@ -18,9 +18,8 @@ classifiers = [ 'Programming Language :: Python :: 3.13', ] dependencies = [ - 'SQLAlchemy>=1.4', + 'SQLAlchemy>=2.0', 'psycopg2-binary>=2.9', - 'packaging', ] [project.urls] @@ -38,8 +37,8 @@ questdb = 'qdb_superset.db_engine_specs.questdb:QuestDbEngineSpec' [project.optional-dependencies] test = [ 'psycopg2-binary~=2.9.6', - 'SQLAlchemy>=1.4', - 'apache-superset>=3.0.0', + 'SQLAlchemy>=2.0', + 'apache-superset>=4.1.0', 'sqlparse==0.4.4', 'pytest~=7.3.0', 'pytest_mock~=3.11.1', diff --git a/src/qdb_superset/db_engine_specs/questdb.py b/src/qdb_superset/db_engine_specs/questdb.py index 335f120..18bc127 100644 --- a/src/qdb_superset/db_engine_specs/questdb.py +++ b/src/qdb_superset/db_engine_specs/questdb.py @@ -1,29 +1,31 @@ from __future__ import annotations +import logging import re from datetime import datetime from typing import Any -import questdb_connect.types as qdbc_types from flask_babel import gettext as __ -from marshmallow import fields, Schema -from questdb_connect.common import remove_public_schema +from marshmallow import Schema, fields from sqlalchemy.engine.base import Engine from sqlalchemy.engine.reflection import Inspector -from sqlalchemy.sql.expression import text, TextClause +from sqlalchemy.sql.expression import TextClause, text from sqlalchemy.types import TypeEngine -import logging + +import questdb_connect.types as qdbc_types +from questdb_connect.common import remove_public_schema # Configure the logging logging.basicConfig(level=logging.ERROR) logger = logging.getLogger(__name__) +from superset import sql_parse from superset.db_engine_specs.base import ( BaseEngineSpec, BasicParametersMixin, BasicParametersType, ) -from superset import sql_parse +from superset.sql.parse import Table from superset.utils import core as utils from superset.utils.core import GenericDataType @@ -269,9 +271,8 @@ def get_sqla_column_type( def select_star( # pylint: disable=too-many-arguments cls, database: Any, - table_name: str, + table: Table, engine: Engine, - schema: str | None = None, limit: int = 100, show_cols: bool = False, indent: bool = True, @@ -280,9 +281,8 @@ def select_star( # pylint: disable=too-many-arguments ) -> str: """Generate a "SELECT * from table_name" query with appropriate limit. :param database: Database instance - :param table_name: Table name, unquoted + :param table: Table instance :param engine: SqlAlchemy Engine instance - :param schema: Schema, unquoted :param limit: limit to impose on query :param show_cols: Show columns in query; otherwise use "*" :param indent: Add indentation to query @@ -292,9 +292,8 @@ def select_star( # pylint: disable=too-many-arguments """ return super().select_star( database, - table_name, + Table(table=table.table, schema=None, catalog=table.catalog), engine, - None, limit, show_cols, indent, @@ -332,16 +331,18 @@ def execute( # pylint: disable=unused-argument cls, cursor: Any, query: str, + database: Any, **kwargs: Any, ) -> None: """Execute a SQL query :param cursor: Cursor instance :param query: Query to execute + :param database: Database instance :param kwargs: kwargs to be passed to cursor.execute() :return: """ try: - sql = sql_parse.strip_comments_from_sql(query) + sql = sql_parse.strip_comments_from_sql(query, engine=cls.engine) cursor.execute(sql) except Exception as ex: # Log the exception with traceback diff --git a/src/questdb_connect/dialect.py b/src/questdb_connect/dialect.py index 9d7bd61..603d610 100644 --- a/src/questdb_connect/dialect.py +++ b/src/questdb_connect/dialect.py @@ -1,7 +1,6 @@ import abc import sqlalchemy -from packaging.version import Version from sqlalchemy.dialects.postgresql.psycopg2 import PGDialect_psycopg2 from sqlalchemy.sql.compiler import GenericTypeCompiler @@ -12,9 +11,6 @@ # ===== SQLAlchemy Dialect ====== # https://docs.sqlalchemy.org/en/20/ -SA_VERSION = Version(sqlalchemy.__version__) -SA_V2 = SA_VERSION >= Version("2.0") - def connection_uri( host: str, port: str, username: str, password: str, database: str = "main" @@ -25,32 +21,20 @@ def connection_uri( def create_engine( host: str, port: str, username: str, password: str, database: str = "main" ): - kwargs = { - "hide_parameters": False, - "isolation_level": "REPEATABLE READ", - } - if not SA_V2: - kwargs["future"] = True - kwargs["implicit_returning"] = False return sqlalchemy.create_engine( connection_uri(host, port, username, password, database), - **kwargs, + hide_parameters=False, + isolation_level="REPEATABLE READ", ) def create_superset_engine( host: str, port: str, username: str, password: str, database: str = "main" ): - kwargs = { - "hide_parameters": False, - "isolation_level": "REPEATABLE READ", - } - if not SA_V2: - kwargs["future"] = False - kwargs["implicit_returning"] = True return sqlalchemy.create_engine( connection_uri(host, port, username, password, database), - **kwargs, + hide_parameters=False, + isolation_level="REPEATABLE READ", ) @@ -80,15 +64,11 @@ class QuestDBDialect(PGDialect_psycopg2, abc.ABC): supports_is_distinct_from = False @classmethod - def dbapi(cls): + def import_dbapi(cls): import questdb_connect as dbapi return dbapi - @classmethod - def import_dbapi(cls): - return cls.dbapi() - def get_schema_names(self, conn, **kw): return ["public"] diff --git a/tests/test_sa2_compat.py b/tests/test_sa2_compat.py index 7021a69..7b058ea 100644 --- a/tests/test_sa2_compat.py +++ b/tests/test_sa2_compat.py @@ -2,9 +2,10 @@ These tests verify the SA 2.0 migration changes work correctly. """ -import questdb_connect as qdbc import sqlalchemy -from questdb_connect.dialect import SA_V2, QuestDBDialect + +import questdb_connect as qdbc +from questdb_connect.dialect import QuestDBDialect def test_import_dbapi(): @@ -13,12 +14,6 @@ def test_import_dbapi(): assert dbapi is qdbc -def test_dbapi_still_works(): - """dbapi() must still work for SA 1.4 backward compat.""" - dbapi = QuestDBDialect.dbapi() - assert dbapi is qdbc - - def test_create_engine_no_deprecated_params(test_config): """create_engine() must not pass deprecated params on SA 2.0.""" import warnings @@ -55,10 +50,7 @@ def test_create_superset_engine_no_deprecated_params(test_config): engine.dispose() -def test_sa_version_detection(): - """SA_V2 flag must match the installed SQLAlchemy version.""" +def test_sa2_required(): + """SQLAlchemy 2.0+ must be installed.""" major = int(sqlalchemy.__version__.split(".")[0]) - if major >= 2: - assert SA_V2 is True - else: - assert SA_V2 is False + assert major >= 2