From 5b1303a470d2b048048cee5597edc90c4a2ed5a9 Mon Sep 17 00:00:00 2001 From: Jsalz2000 Date: Fri, 11 Oct 2024 17:18:54 -0500 Subject: [PATCH 01/15] Adds remote path stripping to azure list_files function --- tests/unit/destination/az/test_list_files.py | 9 +++++++++ twindb_backup/destination/az.py | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/tests/unit/destination/az/test_list_files.py b/tests/unit/destination/az/test_list_files.py index 1e69322a..aa5553cd 100644 --- a/tests/unit/destination/az/test_list_files.py +++ b/tests/unit/destination/az/test_list_files.py @@ -84,3 +84,12 @@ def test_list_files_prefix(): blob_names_recursive = c._list_files(PREFIX, False, False) assert blob_names == blob_names_recursive + +def test_list_files_remote_path(): + """Tests AZ.list_files method, strips remote path from blob names""" + c = mocked_az() + c._container_client.list_blobs.return_value = BLOBS + [BlobProperties(name="himom/blob4")] + + blob_names = c._list_files(PREFIX, False, True) + + assert blob_names == ["blob2", "blob3", "blob4"] \ No newline at end of file diff --git a/twindb_backup/destination/az.py b/twindb_backup/destination/az.py index e6ba0a59..9024091f 100644 --- a/twindb_backup/destination/az.py +++ b/twindb_backup/destination/az.py @@ -239,7 +239,7 @@ def _list_files(self, prefix: str = "", recursive: bool = False, files_only: boo raise err return [ - blob.name + blob.name.strip(self._remote_path).strip('/') for blob in blobs if not files_only or not (bool(blob.get("metadata")) and blob.get("metadata", {}).get("hdi_isfolder") == "true") From 261a0f53112b6ed5fd0014e3d7eca6d5dc7965f7 Mon Sep 17 00:00:00 2001 From: Jsalz2000 Date: Sun, 13 Oct 2024 23:05:04 -0500 Subject: [PATCH 02/15] Adds path stripping to remote_path and prefix in azure blob destination --- README.rst | 6 +++--- docs/installation.rst | 2 +- omnibus/config/projects/twindb-backup.rb | 2 +- setup.cfg | 2 +- setup.py | 2 +- tests/unit/destination/az/test_init.py | 2 +- tests/unit/destination/az/test_list_files.py | 12 ++++++------ tests/unit/destination/az/util.py | 4 ++-- twindb_backup/__init__.py | 2 +- twindb_backup/destination/az.py | 8 ++++---- 10 files changed, 21 insertions(+), 21 deletions(-) diff --git a/README.rst b/README.rst index 10b8e53b..afec8e0d 100644 --- a/README.rst +++ b/README.rst @@ -126,9 +126,9 @@ Install TwinDB Backup. .. code-block:: console # Download the package - wget https://twindb-release.s3.amazonaws.com/twindb-backup/3.3.0/focal/twindb-backup_3.3.0-1_amd64.deb + wget https://twindb-release.s3.amazonaws.com/twindb-backup/3.3.2/focal/twindb-backup_3.3.2-1_amd64.deb # Install TwinDB Backup - apt install ./twindb-backup_3.3.0-1_amd64.deb + apt install ./twindb-backup_3.3.2-1_amd64.deb Configuring TwinDB Backup ~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -157,7 +157,7 @@ The package file will be generated in ``omnibus/pkg/``: .. code-block:: console $ ls omnibus/pkg/*.deb - omnibus/pkg/twindb-backup_3.3.0-1_amd64.deb + omnibus/pkg/twindb-backup_3.3.2-1_amd64.deb Once the package is built you can install it with rpm/dpkg or upload it to your repository and install it with apt or yum. diff --git a/docs/installation.rst b/docs/installation.rst index 8e2aff97..98da3ee9 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -38,7 +38,7 @@ The package file will be generated in ``omnibus/pkg/``: .. code-block:: console $ ls omnibus/pkg/*.deb - omnibus/pkg/twindb-backup_3.3.0-1_amd64.deb + omnibus/pkg/twindb-backup_3.3.2-1_amd64.deb Once the package is built you can install it with rpm/dpkg or upload it to your repository and install it with apt or yum. diff --git a/omnibus/config/projects/twindb-backup.rb b/omnibus/config/projects/twindb-backup.rb index 1d381364..231e895f 100644 --- a/omnibus/config/projects/twindb-backup.rb +++ b/omnibus/config/projects/twindb-backup.rb @@ -23,7 +23,7 @@ # and /opt/twindb-backup on all other platforms install_dir '/opt/twindb-backup' -build_version '3.3.0' +build_version '3.3.2' build_iteration 1 diff --git a/setup.cfg b/setup.cfg index 5cfa9fa5..7a1df71d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 3.3.0 +current_version = 3.3.2 commit = True tag = False diff --git a/setup.py b/setup.py index df20a2a2..d2022212 100644 --- a/setup.py +++ b/setup.py @@ -20,7 +20,7 @@ setup( name="twindb-backup", - version="3.3.0", + version="3.3.2", description="TwinDB Backup tool for files, MySQL et al.", long_description=readme + "\n\n" + history, author="TwinDB Development Team", diff --git a/tests/unit/destination/az/test_init.py b/tests/unit/destination/az/test_init.py index 9361533c..53aa8632 100644 --- a/tests/unit/destination/az/test_init.py +++ b/tests/unit/destination/az/test_init.py @@ -21,7 +21,7 @@ def test_init_param(): assert c._connection_string == p.connection_string assert c._hostname == p.hostname assert c._chunk_size == p.chunk_size - assert c._remote_path == p.remote_path + assert c._remote_path == p.remote_path.strip('/') if p.remote_path != '/' else p.remote_path assert isinstance(c._container_client, ContainerClient) az.AZ._connect.assert_called_once() diff --git a/tests/unit/destination/az/test_list_files.py b/tests/unit/destination/az/test_list_files.py index aa5553cd..62385d9c 100644 --- a/tests/unit/destination/az/test_list_files.py +++ b/tests/unit/destination/az/test_list_files.py @@ -35,7 +35,7 @@ def test_list_files_fail(): with pytest.raises(Exception): c._list_files(PREFIX, False, False) - c._container_client.list_blobs.assert_called_once_with(name_starts_with=PREFIX, include=["metadata"]) + c._container_client.list_blobs.assert_called_once_with(name_starts_with=PREFIX.strip('/'), include=["metadata"]) def test_list_files_files_only(): @@ -47,7 +47,7 @@ def test_list_files_files_only(): assert blob_names == ["blob2", "blob3"] - c._container_client.list_blobs.assert_called_once_with(name_starts_with=PREFIX, include=["metadata"]) + c._container_client.list_blobs.assert_called_once_with(name_starts_with=PREFIX.strip('/'), include=["metadata"]) def test_list_files_all_files(): @@ -59,7 +59,7 @@ def test_list_files_all_files(): assert blob_names == [b.name for b in BLOBS] - c._container_client.list_blobs.assert_called_once_with(name_starts_with=PREFIX, include=["metadata"]) + c._container_client.list_blobs.assert_called_once_with(name_starts_with=PREFIX.strip('/'), include=["metadata"]) def test_list_files_recursive(): @@ -71,7 +71,7 @@ def test_list_files_recursive(): blob_names_recursive = c._list_files(PREFIX, True, False) assert blob_names == blob_names_recursive - c._container_client.list_blobs.assert_called_with(name_starts_with=PREFIX, include=["metadata"]) + c._container_client.list_blobs.assert_called_with(name_starts_with=PREFIX.strip('/'), include=["metadata"]) def test_list_files_prefix(): @@ -88,8 +88,8 @@ def test_list_files_prefix(): def test_list_files_remote_path(): """Tests AZ.list_files method, strips remote path from blob names""" c = mocked_az() - c._container_client.list_blobs.return_value = BLOBS + [BlobProperties(name="himom/blob4")] + c._container_client.list_blobs.return_value = BLOBS + [BlobProperties(name="himom/backups/blob4")] blob_names = c._list_files(PREFIX, False, True) - assert blob_names == ["blob2", "blob3", "blob4"] \ No newline at end of file + assert blob_names == ["blob2", "blob3", "backups/blob4"] \ No newline at end of file diff --git a/tests/unit/destination/az/util.py b/tests/unit/destination/az/util.py index 8b221f9f..d6c37b78 100644 --- a/tests/unit/destination/az/util.py +++ b/tests/unit/destination/az/util.py @@ -14,7 +14,7 @@ def __init__(self, only_required=False) -> None: if not only_required: self.hostname = "test_host" self.chunk_size = 123 - self.remote_path = "/himom" + self.remote_path = "/himom/" def __iter__(self): return iter(self.__dict__) @@ -33,7 +33,7 @@ def __init__(self, only_required=False) -> None: if not only_required: self.chunk_size = 123 - self.remote_path = "/himom" + self.remote_path = "/himom/" def __iter__(self): return iter(self.__dict__) diff --git a/twindb_backup/__init__.py b/twindb_backup/__init__.py index ab7bfe4b..d65070e2 100644 --- a/twindb_backup/__init__.py +++ b/twindb_backup/__init__.py @@ -40,7 +40,7 @@ class and saves the backup copy in something defined in a destination class. __author__ = "TwinDB Development Team" __email__ = "dev@twindb.com" -__version__ = "3.3.0" +__version__ = "3.3.2" STATUS_FORMAT_VERSION = 1 LOCK_FILE = "/var/run/twindb-backup.lock" LOG_FILE = "/var/log/twindb-backup-measures.log" diff --git a/twindb_backup/destination/az.py b/twindb_backup/destination/az.py index 9024091f..6cef5d86 100644 --- a/twindb_backup/destination/az.py +++ b/twindb_backup/destination/az.py @@ -47,7 +47,7 @@ def __init__( self._connection_string = connection_string self._hostname = hostname self._chunk_size = chunk_size - self._remote_path = remote_path + self._remote_path = remote_path.strip('/') if remote_path != '/' else remote_path super(AZ, self).__init__(self._remote_path) self._container_client = self._connect() @@ -94,7 +94,7 @@ def render_path(self, path: str) -> str: Returns: str: Absolute path to the blob in the container """ - return f"{self._remote_path}/{path}" + return f"{self._remote_path}/{path}".strip('/') def _download_to_pipe(self, blob_key: str, pipe_in: int, pipe_out: int) -> None: """Downloads a blob from Azure Blob Storage and writes it to a pipe @@ -226,12 +226,12 @@ def _list_files(self, prefix: str = "", recursive: bool = False, files_only: boo otherwise includes files and directories. Defaults to False. """ LOG.debug( - f"""Listing files in container {self._container_name} with prefix={prefix}, + f"""Listing files in container {self._container_name} with prefix={prefix.strip('/')}, recursive={recursive}, files_only={files_only}""" ) try: - blobs = self._container_client.list_blobs(name_starts_with=prefix, include=["metadata"]) + blobs = self._container_client.list_blobs(name_starts_with=prefix.strip('/'), include=["metadata"]) except builtins.Exception as err: LOG.error( f"Failed to list files in container {self._container_name}. Error: {type(err).__name__}, Reason: {err}" From 2ea04fd3a276cc98ed42fd56f515c75126f72dc8 Mon Sep 17 00:00:00 2001 From: Jsalz2000 Date: Sun, 13 Oct 2024 23:16:57 -0500 Subject: [PATCH 03/15] Formats changes with black to pass make lint --- tests/unit/destination/az/test_init.py | 2 +- tests/unit/destination/az/test_list_files.py | 11 ++++++----- twindb_backup/destination/az.py | 8 ++++---- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/tests/unit/destination/az/test_init.py b/tests/unit/destination/az/test_init.py index 53aa8632..ff4722c7 100644 --- a/tests/unit/destination/az/test_init.py +++ b/tests/unit/destination/az/test_init.py @@ -21,7 +21,7 @@ def test_init_param(): assert c._connection_string == p.connection_string assert c._hostname == p.hostname assert c._chunk_size == p.chunk_size - assert c._remote_path == p.remote_path.strip('/') if p.remote_path != '/' else p.remote_path + assert c._remote_path == p.remote_path.strip("/") if p.remote_path != "/" else p.remote_path assert isinstance(c._container_client, ContainerClient) az.AZ._connect.assert_called_once() diff --git a/tests/unit/destination/az/test_list_files.py b/tests/unit/destination/az/test_list_files.py index 62385d9c..e87cf87a 100644 --- a/tests/unit/destination/az/test_list_files.py +++ b/tests/unit/destination/az/test_list_files.py @@ -35,7 +35,7 @@ def test_list_files_fail(): with pytest.raises(Exception): c._list_files(PREFIX, False, False) - c._container_client.list_blobs.assert_called_once_with(name_starts_with=PREFIX.strip('/'), include=["metadata"]) + c._container_client.list_blobs.assert_called_once_with(name_starts_with=PREFIX.strip("/"), include=["metadata"]) def test_list_files_files_only(): @@ -47,7 +47,7 @@ def test_list_files_files_only(): assert blob_names == ["blob2", "blob3"] - c._container_client.list_blobs.assert_called_once_with(name_starts_with=PREFIX.strip('/'), include=["metadata"]) + c._container_client.list_blobs.assert_called_once_with(name_starts_with=PREFIX.strip("/"), include=["metadata"]) def test_list_files_all_files(): @@ -59,7 +59,7 @@ def test_list_files_all_files(): assert blob_names == [b.name for b in BLOBS] - c._container_client.list_blobs.assert_called_once_with(name_starts_with=PREFIX.strip('/'), include=["metadata"]) + c._container_client.list_blobs.assert_called_once_with(name_starts_with=PREFIX.strip("/"), include=["metadata"]) def test_list_files_recursive(): @@ -71,7 +71,7 @@ def test_list_files_recursive(): blob_names_recursive = c._list_files(PREFIX, True, False) assert blob_names == blob_names_recursive - c._container_client.list_blobs.assert_called_with(name_starts_with=PREFIX.strip('/'), include=["metadata"]) + c._container_client.list_blobs.assert_called_with(name_starts_with=PREFIX.strip("/"), include=["metadata"]) def test_list_files_prefix(): @@ -85,6 +85,7 @@ def test_list_files_prefix(): assert blob_names == blob_names_recursive + def test_list_files_remote_path(): """Tests AZ.list_files method, strips remote path from blob names""" c = mocked_az() @@ -92,4 +93,4 @@ def test_list_files_remote_path(): blob_names = c._list_files(PREFIX, False, True) - assert blob_names == ["blob2", "blob3", "backups/blob4"] \ No newline at end of file + assert blob_names == ["blob2", "blob3", "backups/blob4"] diff --git a/twindb_backup/destination/az.py b/twindb_backup/destination/az.py index 6cef5d86..748e7dc9 100644 --- a/twindb_backup/destination/az.py +++ b/twindb_backup/destination/az.py @@ -47,7 +47,7 @@ def __init__( self._connection_string = connection_string self._hostname = hostname self._chunk_size = chunk_size - self._remote_path = remote_path.strip('/') if remote_path != '/' else remote_path + self._remote_path = remote_path.strip("/") if remote_path != "/" else remote_path super(AZ, self).__init__(self._remote_path) self._container_client = self._connect() @@ -94,7 +94,7 @@ def render_path(self, path: str) -> str: Returns: str: Absolute path to the blob in the container """ - return f"{self._remote_path}/{path}".strip('/') + return f"{self._remote_path}/{path}".strip("/") def _download_to_pipe(self, blob_key: str, pipe_in: int, pipe_out: int) -> None: """Downloads a blob from Azure Blob Storage and writes it to a pipe @@ -231,7 +231,7 @@ def _list_files(self, prefix: str = "", recursive: bool = False, files_only: boo ) try: - blobs = self._container_client.list_blobs(name_starts_with=prefix.strip('/'), include=["metadata"]) + blobs = self._container_client.list_blobs(name_starts_with=prefix.strip("/"), include=["metadata"]) except builtins.Exception as err: LOG.error( f"Failed to list files in container {self._container_name}. Error: {type(err).__name__}, Reason: {err}" @@ -239,7 +239,7 @@ def _list_files(self, prefix: str = "", recursive: bool = False, files_only: boo raise err return [ - blob.name.strip(self._remote_path).strip('/') + blob.name.strip(self._remote_path).strip("/") for blob in blobs if not files_only or not (bool(blob.get("metadata")) and blob.get("metadata", {}).get("hdi_isfolder") == "true") From 59ad4f9f81a3c4db670c8838e6118d65f5e1f168 Mon Sep 17 00:00:00 2001 From: Jsalz2000 Date: Sun, 13 Oct 2024 23:23:43 -0500 Subject: [PATCH 04/15] Bumps version to 3.4.0 --- README.rst | 6 +++--- docs/installation.rst | 2 +- omnibus/config/projects/twindb-backup.rb | 2 +- setup.cfg | 2 +- setup.py | 2 +- twindb_backup/__init__.py | 2 +- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/README.rst b/README.rst index afec8e0d..cf9d21a6 100644 --- a/README.rst +++ b/README.rst @@ -126,9 +126,9 @@ Install TwinDB Backup. .. code-block:: console # Download the package - wget https://twindb-release.s3.amazonaws.com/twindb-backup/3.3.2/focal/twindb-backup_3.3.2-1_amd64.deb + wget https://twindb-release.s3.amazonaws.com/twindb-backup/3.4.0/focal/twindb-backup_3.4.0-1_amd64.deb # Install TwinDB Backup - apt install ./twindb-backup_3.3.2-1_amd64.deb + apt install ./twindb-backup_3.4.0-1_amd64.deb Configuring TwinDB Backup ~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -157,7 +157,7 @@ The package file will be generated in ``omnibus/pkg/``: .. code-block:: console $ ls omnibus/pkg/*.deb - omnibus/pkg/twindb-backup_3.3.2-1_amd64.deb + omnibus/pkg/twindb-backup_3.4.0-1_amd64.deb Once the package is built you can install it with rpm/dpkg or upload it to your repository and install it with apt or yum. diff --git a/docs/installation.rst b/docs/installation.rst index 98da3ee9..15edff99 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -38,7 +38,7 @@ The package file will be generated in ``omnibus/pkg/``: .. code-block:: console $ ls omnibus/pkg/*.deb - omnibus/pkg/twindb-backup_3.3.2-1_amd64.deb + omnibus/pkg/twindb-backup_3.4.0-1_amd64.deb Once the package is built you can install it with rpm/dpkg or upload it to your repository and install it with apt or yum. diff --git a/omnibus/config/projects/twindb-backup.rb b/omnibus/config/projects/twindb-backup.rb index 231e895f..bef6b2cd 100644 --- a/omnibus/config/projects/twindb-backup.rb +++ b/omnibus/config/projects/twindb-backup.rb @@ -23,7 +23,7 @@ # and /opt/twindb-backup on all other platforms install_dir '/opt/twindb-backup' -build_version '3.3.2' +build_version '3.4.0' build_iteration 1 diff --git a/setup.cfg b/setup.cfg index 7a1df71d..08be9c7c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 3.3.2 +current_version = 3.4.0 commit = True tag = False diff --git a/setup.py b/setup.py index d2022212..b5216d47 100644 --- a/setup.py +++ b/setup.py @@ -20,7 +20,7 @@ setup( name="twindb-backup", - version="3.3.2", + version="3.4.0", description="TwinDB Backup tool for files, MySQL et al.", long_description=readme + "\n\n" + history, author="TwinDB Development Team", diff --git a/twindb_backup/__init__.py b/twindb_backup/__init__.py index d65070e2..019fb496 100644 --- a/twindb_backup/__init__.py +++ b/twindb_backup/__init__.py @@ -40,7 +40,7 @@ class and saves the backup copy in something defined in a destination class. __author__ = "TwinDB Development Team" __email__ = "dev@twindb.com" -__version__ = "3.3.2" +__version__ = "3.4.0" STATUS_FORMAT_VERSION = 1 LOCK_FILE = "/var/run/twindb-backup.lock" LOG_FILE = "/var/log/twindb-backup-measures.log" From 62b93be85e134e80850c6de8d3f413b14c8d9a1c Mon Sep 17 00:00:00 2001 From: Jsalz2000 Date: Mon, 14 Oct 2024 00:37:48 -0500 Subject: [PATCH 05/15] Adds mysql hostname config argument, removes hardcoded address --- docs/usage.rst | 1 + support/twindb-backup.cfg | 2 +- tests/unit/configuration/test_mysql.py | 23 ++++++++++++++++++- twindb_backup/backup.py | 12 ++++++---- twindb_backup/configuration/mysql.py | 7 ++++++ twindb_backup/source/mysql_source.py | 2 +- .../modules/profile/files/twindb-backup.cfg | 1 + 7 files changed, 40 insertions(+), 8 deletions(-) diff --git a/docs/usage.rst b/docs/usage.rst index f10b0aa0..b0ffcbb6 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -151,6 +151,7 @@ The ``expire_log_days`` options specifies the retention period for MySQL binlogs mysql_defaults_file = /etc/twindb/my.cnf full_backup = daily expire_log_days = 7 + hostname = localhost # optional, defaults to 127.0.0.1 Backing up MySQL Binlog ----------------------- diff --git a/support/twindb-backup.cfg b/support/twindb-backup.cfg index e2f9b0e5..44c6130b 100644 --- a/support/twindb-backup.cfg +++ b/support/twindb-backup.cfg @@ -60,8 +60,8 @@ ssh_key=/root/.ssh/id_rsa # MySQL mysql_defaults_file=/etc/twindb/my.cnf - full_backup=daily +#hostname=localhost # optional, defaults to 127.0.0.1 [retention] diff --git a/tests/unit/configuration/test_mysql.py b/tests/unit/configuration/test_mysql.py index afe1105c..2765c950 100644 --- a/tests/unit/configuration/test_mysql.py +++ b/tests/unit/configuration/test_mysql.py @@ -1,10 +1,31 @@ from twindb_backup.configuration.mysql import MySQLConfig -def test_mysql(): +def test_mysql_defaults(): mc = MySQLConfig() assert mc.defaults_file == "/root/.my.cnf" assert mc.full_backup == "daily" + assert mc.expire_log_days == 7 + assert mc.xtrabackup_binary is None + assert mc.xbstream_binary is None + assert mc.hostname == "127.0.0.1" + + +def test_mysql_init(): + mc = MySQLConfig( + mysql_defaults_file="/foo/bar", + full_backup="weekly", + expire_log_days=3, + xtrabackup_binary="/foo/xtrabackup", + xbstream_binary="/foo/xbstream", + hostname="foo", + ) + assert mc.defaults_file == "/foo/bar" + assert mc.full_backup == "weekly" + assert mc.expire_log_days == 3 + assert mc.xtrabackup_binary == "/foo/xtrabackup" + assert mc.xbstream_binary == "/foo/xbstream" + assert mc.hostname == "foo" def test_mysql_set_xtrabackup_binary(): diff --git a/twindb_backup/backup.py b/twindb_backup/backup.py index 2a58150b..b977d81c 100644 --- a/twindb_backup/backup.py +++ b/twindb_backup/backup.py @@ -102,7 +102,7 @@ def backup_files(run_type, config: TwinDBBackupConfig): ) -def backup_mysql(run_type, config): +def backup_mysql(run_type, config: TwinDBBackupConfig): """Take backup of local MySQL instance :param run_type: Run type @@ -129,8 +129,10 @@ def backup_mysql(run_type, config): kwargs["parent_lsn"] = parent.lsn LOG.debug("Creating source %r", kwargs) - mysql_client = MySQLClient(config.mysql.defaults_file) - src = MYSQL_SRC_MAP[mysql_client.server_vendor](MySQLConnectInfo(config.mysql.defaults_file), run_type, **kwargs) + mysql_client = MySQLClient(defaults_file=config.mysql.defaults_file, hostname=config.mysql.hostname) + src = MYSQL_SRC_MAP[mysql_client.server_vendor]( + MySQLConnectInfo(defaults_file=config.mysql.defaults_file, hostname=config.mysql.hostname), run_type, **kwargs + ) callbacks = [] try: @@ -173,7 +175,7 @@ def backup_mysql(run_type, config): callback[0].callback(**callback[1]) -def backup_binlogs(run_type, config): # pylint: disable=too-many-locals +def backup_binlogs(run_type, config: TwinDBBackupConfig): # pylint: disable=too-many-locals """Copy MySQL binlog files to the backup destination. :param run_type: Run type @@ -187,7 +189,7 @@ def backup_binlogs(run_type, config): # pylint: disable=too-many-locals dst = config.destination() status = BinlogStatus(dst=dst) - mysql_client = MySQLClient(defaults_file=config.mysql.defaults_file) + mysql_client = MySQLClient(defaults_file=config.mysql.defaults_file, hostname=config.mysql.hostname) log_bin_basename = mysql_client.variable("log_bin_basename") if log_bin_basename is None: return diff --git a/twindb_backup/configuration/mysql.py b/twindb_backup/configuration/mysql.py index e4d8b180..7647c8d3 100644 --- a/twindb_backup/configuration/mysql.py +++ b/twindb_backup/configuration/mysql.py @@ -15,6 +15,7 @@ def __init__(self, **kwargs): self._expire_log_days = int(kwargs.get("expire_log_days", 7)) self._xtrabackup_binary = kwargs.get("xtrabackup_binary") self._xbstream_binary = kwargs.get("xbstream_binary") + self._hostname = kwargs.get("hostname", "127.0.0.1") @property def defaults_file(self): @@ -56,3 +57,9 @@ def xbstream_binary(self, path): """Set path to xbstream""" self._xbstream_binary = path + + @property + def hostname(self): + """MySQL hostname to connect to""" + + return self._hostname diff --git a/twindb_backup/source/mysql_source.py b/twindb_backup/source/mysql_source.py index b39ba8e3..0361bb90 100644 --- a/twindb_backup/source/mysql_source.py +++ b/twindb_backup/source/mysql_source.py @@ -216,7 +216,7 @@ def get_stream(self): self._xtrabackup, "--defaults-file=%s" % self._connect_info.defaults_file, "--stream=xbstream", - "--host=127.0.0.1", + f"--host={self._connect_info.hostname}", "--backup", ] cmd += ["--target-dir", "."] diff --git a/vagrant/environment/puppet/modules/profile/files/twindb-backup.cfg b/vagrant/environment/puppet/modules/profile/files/twindb-backup.cfg index e7f5c697..09bc007b 100644 --- a/vagrant/environment/puppet/modules/profile/files/twindb-backup.cfg +++ b/vagrant/environment/puppet/modules/profile/files/twindb-backup.cfg @@ -44,6 +44,7 @@ backup_dir=/path/to/twindb-server-backups [mysql] mysql_defaults_file=/root/.my.cnf full_backup=daily +#hostname=localhost # optional, defaults to 127.0.0.1 # Retention [retention] From c05b415da642f5e39321db731648c66745ee2fc1 Mon Sep 17 00:00:00 2001 From: Jsalz2000 Date: Mon, 30 Dec 2024 14:37:53 -0600 Subject: [PATCH 06/15] Adds Azure Client config parameters, refactors Azure destination --- tests/unit/destination/az/test_config.py | 136 +++++++++++++--- tests/unit/destination/az/test_delete.py | 6 +- .../destination/az/test_download_to_pipe.py | 8 +- tests/unit/destination/az/test_init.py | 100 +++++++----- tests/unit/destination/az/test_list_files.py | 24 +-- tests/unit/destination/az/test_read.py | 14 +- tests/unit/destination/az/test_save.py | 6 +- tests/unit/destination/az/test_write.py | 6 +- tests/unit/destination/az/util.py | 30 ++-- twindb_backup/configuration/__init__.py | 12 +- .../configuration/destinations/az.py | 145 +++++++++++++----- twindb_backup/destination/az.py | 60 ++++---- 12 files changed, 370 insertions(+), 177 deletions(-) diff --git a/tests/unit/destination/az/test_config.py b/tests/unit/destination/az/test_config.py index 8fa56c8f..4e499c03 100644 --- a/tests/unit/destination/az/test_config.py +++ b/tests/unit/destination/az/test_config.py @@ -1,37 +1,135 @@ +from dataclasses import asdict + import pytest -from twindb_backup.configuration.destinations.az import AZConfig +from twindb_backup.configuration.destinations.az import AZClientConfig, AZConfig, drop_empty_dict_factory -from .util import AZConfigParams +from .util import AZClientConfigParams, AZConfigParams def test_initialization_success(): """Test initialization of AZConfig with all parameters set.""" - p = AZConfigParams() - c = AZConfig(**dict(p)) - assert c.connection_string == p.connection_string - assert c.container_name == p.container_name - assert c.chunk_size == p.chunk_size - assert c.remote_path == p.remote_path + client_params = AZClientConfigParams() + config_params = AZConfigParams() + client_config = AZClientConfig(**dict(client_params)) + + c = AZConfig(client_config=client_config, **dict(config_params)) + + # AZConfig Assertions + assert c.client_config == client_config + assert c.connection_string == config_params.connection_string + assert c.container_name == config_params.container_name + assert ( + c.remote_path == config_params.remote_path.strip("/") + if config_params.remote_path != "/" + else config_params.remote_path + ) + + # AZClientConfig Assertions + assert c.client_config.api_version == client_params.api_version + assert c.client_config.secondary_hostname == client_params.secondary_hostname + assert c.client_config.max_block_size == client_params.max_block_size + assert c.client_config.max_single_put_size == client_params.max_single_put_size + assert c.client_config.min_large_block_upload_threshold == client_params.min_large_block_upload_threshold + assert c.client_config.use_byte_buffer == client_params.use_byte_buffer + assert c.client_config.max_page_size == client_params.max_page_size + assert c.client_config.max_single_get_size == client_params.max_single_get_size + assert c.client_config.max_chunk_get_size == client_params.max_chunk_get_size + assert c.client_config.audience == client_params.audience def test_initialization_success_defaults(): """Test initialization of AZConfig with only required parameters set and ensure default values.""" - p = AZConfigParams(only_required=True) - c = AZConfig(**dict(p)) - assert c.connection_string == p.connection_string - assert c.container_name == p.container_name - assert c.chunk_size == 4 * 1024 * 1024 + client_params = AZClientConfigParams(only_required=True) + config_params = AZConfigParams(only_required=True) + client_config = AZClientConfig(**dict(client_params)) + + c = AZConfig(client_config=client_config, **dict(config_params)) + + # AZConfig Assertions + assert c.client_config == client_config + assert c.connection_string == config_params.connection_string + assert c.container_name == config_params.container_name assert c.remote_path == "/" + # AZClientConfig Assertions + assert c.client_config.api_version == None + assert c.client_config.secondary_hostname == None + assert c.client_config.max_block_size == 4 * 1024 * 1024 # 4MB + assert c.client_config.max_single_put_size == 64 * 1024 * 1024 # 64MB + assert c.client_config.min_large_block_upload_threshold == (4 * 1024 * 1024) + 1 # 4MB + 1 + assert c.client_config.use_byte_buffer == False + assert c.client_config.max_page_size == 4 * 1024 * 1024 # 4MB + assert c.client_config.max_single_get_size == 32 * 1024 * 1024 # 32MB + assert c.client_config.max_chunk_get_size == 4 * 1024 * 1024 # 4MB + assert c.client_config.audience == None + def test_invalid_params(): """Test initialization of AZConfig with invalid parameters.""" - with pytest.raises(ValueError): + + # Invalidate AZConfig + with pytest.raises(ValueError): # Invalid client_config + AZConfig(client_config={}, connection_string="test_connection_string", container_name="test_container") + with pytest.raises(ValueError): # Invalid connection_string + AZConfig(client_config=AZClientConfig(), connection_string=123, container_name="test_container") + with pytest.raises(ValueError): # Invalid remote_path AZConfig( - connection_string="test_connection_string", container_name="test_container", chunk_size="invalid_chunk_size" + client_config=AZClientConfig(), + connection_string="test_connection_string", + container_name="test_container", + remote_path=1, ) - with pytest.raises(ValueError): - AZConfig(connection_string="test_connection_string", container_name="test_container", remote_path=1) - with pytest.raises(TypeError): - AZConfig(connection_string="test_connection_string") + with pytest.raises(ValueError): # Invalid container_name + AZConfig(client_config=AZClientConfig(), connection_string="test_connection_string", container_name=1) + + # Invalidate AZClientConfig + with pytest.raises(ValueError): # Invalid api_version + AZClientConfig(api_version=123) + with pytest.raises(ValueError): # Invalid secondary_hostname + AZClientConfig(secondary_hostname=123) + with pytest.raises(ValueError): # Invalid max_block_size + AZClientConfig(max_block_size="123") + with pytest.raises(ValueError): # Invalid max_single_put_size + AZClientConfig(max_single_put_size="123") + with pytest.raises(ValueError): # Invalid min_large_block_upload_threshold + AZClientConfig(min_large_block_upload_threshold="123") + with pytest.raises(ValueError): # Invalid use_byte_buffer + AZClientConfig(use_byte_buffer="123") + with pytest.raises(ValueError): # Invalid max_page_size + AZClientConfig(max_page_size="123") + with pytest.raises(ValueError): # Invalid max_single_get_size + AZClientConfig(max_single_get_size="123") + with pytest.raises(ValueError): # Invalid max_chunk_get_size + AZClientConfig(max_chunk_get_size="123") + with pytest.raises(ValueError): # Invalid audience + AZClientConfig(audience=123) + + +def test_drop_empty_dicts_some_undefined(): + """Test drop_empty_dict_factory helper function.""" + + client_config = AZClientConfig(**dict(AZClientConfigParams(only_required=True))) + + # Convert to dict and drop attributes with None values + client_config_dict = asdict(client_config, dict_factory=drop_empty_dict_factory) + + # Assert that the dict does not contain any None values + assert "api_version" not in client_config_dict + assert "secondary_hostname" not in client_config_dict + assert "audience" not in client_config_dict + + +def test_drop_empty_dicts_all_defined(): + """Test drop_empty_dict_factory helper function doesn't drop any attributes when all are defined.""" + + client_config = AZClientConfig(**dict(AZClientConfigParams())) + + # Convert to dict and drop attributes with None values + client_config_dict_drop_empty = asdict(client_config, dict_factory=drop_empty_dict_factory) + + # Convert to dict + client_config_dict = asdict(client_config) + + # Assert that the dicts are the same + assert client_config_dict == client_config_dict_drop_empty diff --git a/tests/unit/destination/az/test_delete.py b/tests/unit/destination/az/test_delete.py index 357ac84f..f2456815 100644 --- a/tests/unit/destination/az/test_delete.py +++ b/tests/unit/destination/az/test_delete.py @@ -10,14 +10,14 @@ def test_delete_success(): c = mocked_az() c.delete("test") - c._container_client.delete_blob.assert_called_once_with(c.render_path("test")) + c.container_client.delete_blob.assert_called_once_with(c.render_path("test")) def test_delete_fail(): """Tests AZ.delete method, re-raising an exception on failure""" c = mocked_az() - c._container_client.delete_blob.side_effect = Exception() + c.container_client.delete_blob.side_effect = Exception() with pytest.raises(Exception): c.delete("test") - c._container_client.delete_blob.assert_called_once_with(c.render_path("test")) + c.container_client.delete_blob.assert_called_once_with(c.render_path("test")) diff --git a/tests/unit/destination/az/test_download_to_pipe.py b/tests/unit/destination/az/test_download_to_pipe.py index 837da0c4..7f92cb56 100644 --- a/tests/unit/destination/az/test_download_to_pipe.py +++ b/tests/unit/destination/az/test_download_to_pipe.py @@ -15,13 +15,13 @@ def test_download_to_pipe_success(): c = mocked_az() mc_dbr = MagicMock() - c._container_client.download_blob.return_value = mc_dbr + c.container_client.download_blob.return_value = mc_dbr c._download_to_pipe(c.render_path("foo-key"), 100, 200) mc_os.close.assert_called_once_with(100) mc_os.fdopen.assert_called_once_with(200, "wb") - c._container_client.download_blob.assert_called_once_with(c.render_path("foo-key")) + c.container_client.download_blob.assert_called_once_with(c.render_path("foo-key")) mc_dbr.readinto.assert_called_once_with(mc_fdopen.__enter__()) @@ -30,11 +30,11 @@ def test_download_to_pipe_fail(): with patch("twindb_backup.destination.az.os") as mc_os: c = mocked_az() - c._container_client.download_blob.side_effect = ae.HttpResponseError() + c.container_client.download_blob.side_effect = ae.HttpResponseError() with pytest.raises(Exception): c._download_to_pipe(c.render_path("foo-key"), 100, 200) mc_os.close.assert_called_once_with(100) mc_os.fdopen.assert_called_once_with(200, "wb") - c._container_client.download_blob.assert_called_once_with(c.render_path("foo-key")) + c.container_client.download_blob.assert_called_once_with(c.render_path("foo-key")) diff --git a/tests/unit/destination/az/test_init.py b/tests/unit/destination/az/test_init.py index ff4722c7..fade15a6 100644 --- a/tests/unit/destination/az/test_init.py +++ b/tests/unit/destination/az/test_init.py @@ -1,4 +1,5 @@ import socket +from dataclasses import asdict from unittest.mock import MagicMock, patch import azure.core.exceptions as ae @@ -6,23 +7,24 @@ from azure.storage.blob import ContainerClient import twindb_backup.destination.az as az +from twindb_backup.configuration.destinations.az import AZClientConfig, AZConfig, drop_empty_dict_factory -from .util import AZParams +from .util import AZClientConfigParams, AZConfigParams def test_init_param(): """Test initialization of AZ with all parameters set, mocking the _connect method.""" with patch("twindb_backup.destination.az.AZ._connect") as mc: mc.return_value = MagicMock(spec=ContainerClient) - p = AZParams() - c = az.AZ(**dict(p)) - - assert c._container_name == p.container_name - assert c._connection_string == p.connection_string - assert c._hostname == p.hostname - assert c._chunk_size == p.chunk_size - assert c._remote_path == p.remote_path.strip("/") if p.remote_path != "/" else p.remote_path - assert isinstance(c._container_client, ContainerClient) + + client_params = AZClientConfigParams() + config_params = AZConfigParams() + client_config = AZClientConfig(**dict(client_params)) + config = AZConfig(client_config=client_config, **dict(config_params)) + + c = az.AZ(config=config) + + assert isinstance(c.container_client, ContainerClient) az.AZ._connect.assert_called_once() @@ -30,15 +32,15 @@ def test_init_param_defaults(): """Test initialization of AZ with only required parameters set, ensuring default values, mocking the _connect method.""" with patch("twindb_backup.destination.az.AZ._connect") as mc: mc.return_value = MagicMock(spec=ContainerClient) - p = AZParams(only_required=True) - c = az.AZ(**dict(p)) - - assert c._container_name == p.container_name - assert c._connection_string == p.connection_string - assert c._hostname == socket.gethostname() - assert c._chunk_size == 4 * 1024 * 1024 - assert c._remote_path == "/" - assert isinstance(c._container_client, ContainerClient) + + client_params = AZClientConfigParams() + config_params = AZConfigParams() + client_config = AZClientConfig(**dict(client_params)) + config = AZConfig(client_config=client_config, **dict(config_params)) + + c = az.AZ(config=config) + + assert isinstance(c.container_client, ContainerClient) az.AZ._connect.assert_called_once() @@ -46,21 +48,31 @@ def test_init_conn_string_valid(): """Test initialization of AZ with valid connection string.""" with patch("twindb_backup.destination.az.ContainerClient.exists") as mc: mc.return_value = True - p = AZParams() - c = az.AZ(**dict(p)) + + client_params = AZClientConfigParams() + config_params = AZConfigParams() + client_config = AZClientConfig(**dict(client_params)) + config = AZConfig(client_config=client_config, **dict(config_params)) + + c = az.AZ(config=config) az.ContainerClient.exists.assert_called_once() - assert isinstance(c._container_client, ContainerClient) + assert isinstance(c.container_client, ContainerClient) def test_init_conn_string_invalid(): """Test initialization of AZ with invalid connection string, expecting ValueError.""" with patch("twindb_backup.destination.az.ContainerClient.exists") as mc: mc.return_value = True - p = AZParams() - p.connection_string = "invalid_connection_string" + + client_params = AZClientConfigParams() + config_params = AZConfigParams() + client_config = AZClientConfig(**dict(client_params)) + config = AZConfig(client_config=client_config, **dict(config_params)) + config.connection_string = "invalid_connection_string" + with pytest.raises(ValueError, match="Connection string is either blank or malformed."): - _ = az.AZ(**dict(p)) + _ = az.AZ(config=config) def test_init_container_not_exists(): @@ -69,12 +81,17 @@ def test_init_container_not_exists(): mc.return_value = False with patch("twindb_backup.destination.az.ContainerClient.create_container") as mc_create_container: mc_create_container.return_value = MagicMock(spec=ContainerClient) - p = AZParams() - c = az.AZ(**dict(p)) + + client_params = AZClientConfigParams() + config_params = AZConfigParams() + client_config = AZClientConfig(**dict(client_params)) + config = AZConfig(client_config=client_config, **dict(config_params)) + + c = az.AZ(config=config) az.ContainerClient.exists.assert_called_once() az.ContainerClient.create_container.assert_called_once() - assert isinstance(c._container_client, ContainerClient) + assert isinstance(c.container_client, ContainerClient) def test_init_container_create_fails(): @@ -84,23 +101,36 @@ def test_init_container_create_fails(): with patch("twindb_backup.destination.az.ContainerClient.create_container") as mc_create_container: mc_create_container.side_effect = ae.HttpResponseError() - p = AZParams() + client_params = AZClientConfigParams() + config_params = AZConfigParams() + client_config = AZClientConfig(**dict(client_params)) + config = AZConfig(client_config=client_config, **dict(config_params)) + with pytest.raises(Exception): - c = az.AZ(**dict(p)) + c = az.AZ(config=config) az.ContainerClient.exists.assert_called_once() az.ContainerClient.create_container.assert_called_once() - assert isinstance(c._container_client, ContainerClient) + assert isinstance(c.container_client, ContainerClient) def test_init_success(): """Test initialization of AZ with existing container, mocking the from_connection_string method.""" with patch("twindb_backup.destination.az.ContainerClient.from_connection_string") as mc: mc.return_value = MagicMock(spec=ContainerClient) - p = AZParams() - c = az.AZ(**dict(p)) - az.ContainerClient.from_connection_string.assert_called_once_with(p.connection_string, p.container_name) + client_params = AZClientConfigParams() + config_params = AZConfigParams() + client_config = AZClientConfig(**dict(client_params)) + config = AZConfig(client_config=client_config, **dict(config_params)) + + c = az.AZ(config=config) + + az.ContainerClient.from_connection_string.assert_called_once_with( + conn_str=config.connection_string, + container_name=config.container_name, + **asdict(config.client_config, dict_factory=drop_empty_dict_factory) + ) mc.return_value.exists.assert_called_once() mc.return_value.create_container.assert_not_called() - assert isinstance(c._container_client, ContainerClient) + assert isinstance(c.container_client, ContainerClient) diff --git a/tests/unit/destination/az/test_list_files.py b/tests/unit/destination/az/test_list_files.py index e87cf87a..fdb7a0d5 100644 --- a/tests/unit/destination/az/test_list_files.py +++ b/tests/unit/destination/az/test_list_files.py @@ -19,65 +19,65 @@ def test_list_files_success(): """Tests AZ.list_files method, reading a list of blob names from azure.""" c = mocked_az() - c._container_client.list_blobs.return_value = BLOBS + c.container_client.list_blobs.return_value = BLOBS blobs = c._list_files() assert blobs == [b.name for b in BLOBS] - c._container_client.list_blobs.assert_called_once() + c.container_client.list_blobs.assert_called_once() def test_list_files_fail(): """Tests AZ.list_files method, re-raises an exception on failure""" c = mocked_az() - c._container_client.list_blobs.side_effect = ae.HttpResponseError() + c.container_client.list_blobs.side_effect = ae.HttpResponseError() with pytest.raises(Exception): c._list_files(PREFIX, False, False) - c._container_client.list_blobs.assert_called_once_with(name_starts_with=PREFIX.strip("/"), include=["metadata"]) + c.container_client.list_blobs.assert_called_once_with(name_starts_with=PREFIX.strip("/"), include=["metadata"]) def test_list_files_files_only(): """Tests AZ.list_files method, listing only file blobs""" c = mocked_az() - c._container_client.list_blobs.return_value = BLOBS + c.container_client.list_blobs.return_value = BLOBS blob_names = c._list_files(PREFIX, False, True) assert blob_names == ["blob2", "blob3"] - c._container_client.list_blobs.assert_called_once_with(name_starts_with=PREFIX.strip("/"), include=["metadata"]) + c.container_client.list_blobs.assert_called_once_with(name_starts_with=PREFIX.strip("/"), include=["metadata"]) def test_list_files_all_files(): """Tests AZ.list_files method, listing all blobs, including directories""" c = mocked_az() - c._container_client.list_blobs.return_value = BLOBS + c.container_client.list_blobs.return_value = BLOBS blob_names = c._list_files(PREFIX, False, False) assert blob_names == [b.name for b in BLOBS] - c._container_client.list_blobs.assert_called_once_with(name_starts_with=PREFIX.strip("/"), include=["metadata"]) + c.container_client.list_blobs.assert_called_once_with(name_starts_with=PREFIX.strip("/"), include=["metadata"]) def test_list_files_recursive(): """Tests AZ.list_files method, recursive option is ignored""" c = mocked_az() - c._container_client.list_blobs.return_value = BLOBS + c.container_client.list_blobs.return_value = BLOBS blob_names = c._list_files(PREFIX, False, False) blob_names_recursive = c._list_files(PREFIX, True, False) assert blob_names == blob_names_recursive - c._container_client.list_blobs.assert_called_with(name_starts_with=PREFIX.strip("/"), include=["metadata"]) + c.container_client.list_blobs.assert_called_with(name_starts_with=PREFIX.strip("/"), include=["metadata"]) def test_list_files_prefix(): """Tests AZ.list_files method, prefix is used as a filter in list_blobs only""" c = mocked_az() - c._container_client.list_blobs.return_value = BLOBS + c.container_client.list_blobs.return_value = BLOBS # Prefix is used as a filter in list_blobs, and because its mocked - it makes no difference. blob_names = c._list_files("".join(random.SystemRandom().choices(string.ascii_lowercase, k=10)), False, False) @@ -89,7 +89,7 @@ def test_list_files_prefix(): def test_list_files_remote_path(): """Tests AZ.list_files method, strips remote path from blob names""" c = mocked_az() - c._container_client.list_blobs.return_value = BLOBS + [BlobProperties(name="himom/backups/blob4")] + c.container_client.list_blobs.return_value = BLOBS + [BlobProperties(name="himom/backups/blob4")] blob_names = c._list_files(PREFIX, False, True) diff --git a/tests/unit/destination/az/test_read.py b/tests/unit/destination/az/test_read.py index 052cafca..0ce3597b 100644 --- a/tests/unit/destination/az/test_read.py +++ b/tests/unit/destination/az/test_read.py @@ -15,31 +15,31 @@ def test_read_success(): """Tests AZ.read method, ensuring the blob is read from azure.""" c = mocked_az() mock = MagicMock(StorageStreamDownloader) - c._container_client.download_blob.return_value = mock + c.container_client.download_blob.return_value = mock c.read(EXAMPLE_FILE) - c._container_client.download_blob.assert_called_once_with(c.render_path(EXAMPLE_FILE), encoding="utf-8") + c.container_client.download_blob.assert_called_once_with(c.render_path(EXAMPLE_FILE), encoding="utf-8") mock.read.assert_called_once() def test_read_fail(): """Tests AZ.read method, re-raises an exception on failure""" c = mocked_az() - c._container_client.download_blob.side_effect = ae.HttpResponseError() + c.container_client.download_blob.side_effect = ae.HttpResponseError() with pytest.raises(Exception): c.read(EXAMPLE_FILE) - c._container_client.download_blob.assert_called_once_with(c.render_path(EXAMPLE_FILE), encoding="utf-8") + c.container_client.download_blob.assert_called_once_with(c.render_path(EXAMPLE_FILE), encoding="utf-8") def test_read_fail_not_found(): """Tests AZ.read method, raising a twindb_backup.destination.exceptions.FileNotFound exception on ResourceNotFoundError""" c = mocked_az() - c._container_client.download_blob.side_effect = ae.ResourceNotFoundError() + c.container_client.download_blob.side_effect = ae.ResourceNotFoundError() with pytest.raises( - FileNotFound, match=f"File {c.render_path(EXAMPLE_FILE)} does not exist in container {c._container_name}" + FileNotFound, match=f"File {c.render_path(EXAMPLE_FILE)} does not exist in container {c.config.container_name}" ): c.read(EXAMPLE_FILE) - c._container_client.download_blob.assert_called_once_with(c.render_path(EXAMPLE_FILE), encoding="utf-8") + c.container_client.download_blob.assert_called_once_with(c.render_path(EXAMPLE_FILE), encoding="utf-8") diff --git a/tests/unit/destination/az/test_save.py b/tests/unit/destination/az/test_save.py index 0cafd271..68d98486 100644 --- a/tests/unit/destination/az/test_save.py +++ b/tests/unit/destination/az/test_save.py @@ -19,7 +19,7 @@ def test_save_success(): c.save(handler, EXAMPLE_FILE) - c._container_client.upload_blob.assert_called_once_with(c.render_path(EXAMPLE_FILE), file_obj) + c.container_client.upload_blob.assert_called_once_with(c.render_path(EXAMPLE_FILE), file_obj) def test_save_fail(): @@ -29,9 +29,9 @@ def test_save_fail(): file_obj = MagicMock() handler.__enter__.return_value = file_obj handler.__exit__.return_value = None - c._container_client.upload_blob.side_effect = ae.HttpResponseError() + c.container_client.upload_blob.side_effect = ae.HttpResponseError() with pytest.raises(Exception): c.save(handler, EXAMPLE_FILE) - c._container_client.upload_blob.assert_called_once_with(c.render_path(EXAMPLE_FILE), file_obj) + c.container_client.upload_blob.assert_called_once_with(c.render_path(EXAMPLE_FILE), file_obj) diff --git a/tests/unit/destination/az/test_write.py b/tests/unit/destination/az/test_write.py index 99303939..839f4e75 100644 --- a/tests/unit/destination/az/test_write.py +++ b/tests/unit/destination/az/test_write.py @@ -13,15 +13,15 @@ def test_write_success(): c.write(CONTENT, EXAMPLE_FILE) - c._container_client.upload_blob.assert_called_once_with(c.render_path(EXAMPLE_FILE), CONTENT, overwrite=True) + c.container_client.upload_blob.assert_called_once_with(c.render_path(EXAMPLE_FILE), CONTENT, overwrite=True) def test_write_fail(): """Tests AZ.write method, re-raises an exception on failure""" c = mocked_az() - c._container_client.upload_blob.side_effect = ae.HttpResponseError() + c.container_client.upload_blob.side_effect = ae.HttpResponseError() with pytest.raises(Exception): c.write(CONTENT, EXAMPLE_FILE) - c._container_client.upload_blob.assert_called_once_with(c.render_path(EXAMPLE_FILE), CONTENT, overwrite=True) + c.container_client.upload_blob.assert_called_once_with(c.render_path(EXAMPLE_FILE), CONTENT, overwrite=True) diff --git a/tests/unit/destination/az/util.py b/tests/unit/destination/az/util.py index d6c37b78..fe10a4b2 100644 --- a/tests/unit/destination/az/util.py +++ b/tests/unit/destination/az/util.py @@ -4,17 +4,23 @@ from azure.storage.blob import ContainerClient import twindb_backup.destination.az as az +from twindb_backup.configuration.destinations.az import AZClientConfig, AZConfig -class AZParams(collections.Mapping): +class AZClientConfigParams(collections.Mapping): def __init__(self, only_required=False) -> None: - self.container_name = "test_container" - self.connection_string = "DefaultEndpointsProtocol=https;AccountName=ACCOUNT_NAME;AccountKey=ACCOUNT_KEY;EndpointSuffix=core.windows.net" if not only_required: - self.hostname = "test_host" - self.chunk_size = 123 - self.remote_path = "/himom/" + self.api_version = "2021-04-10" + self.secondary_hostname = "secondary.example.com" + self.max_block_size = 128 * 1024 * 1024 # 128MB + self.max_single_put_size = 128 * 1024 * 1024 # 128MB + self.min_large_block_upload_threshold = 128 * 1024 * 1024 # 128MB + self.use_byte_buffer = False + self.max_page_size = 128 * 1024 * 1024 # 128MB + self.max_single_get_size = 128 * 1024 * 1024 # 128MB + self.max_chunk_get_size = 128 * 1024 * 1024 # 128MB + self.audience = "https://example.com" def __iter__(self): return iter(self.__dict__) @@ -28,11 +34,10 @@ def __getitem__(self, key): class AZConfigParams(collections.Mapping): def __init__(self, only_required=False) -> None: - self.connection_string = "test_connection_string" self.container_name = "test_container" + self.connection_string = "DefaultEndpointsProtocol=https;AccountName=ACCOUNT_NAME;AccountKey=ACCOUNT_KEY;EndpointSuffix=core.windows.net" if not only_required: - self.chunk_size = 123 self.remote_path = "/himom/" def __iter__(self): @@ -48,7 +53,12 @@ def __getitem__(self, key): def mocked_az(): with patch("twindb_backup.destination.az.AZ._connect") as mc: mc.return_value = MagicMock(spec=ContainerClient) - p = AZParams() - c = az.AZ(**dict(p)) + + client_params = AZClientConfigParams() + config_params = AZConfigParams() + + az_config = AZConfig(client_config=AZClientConfig(**dict(client_params)), **dict(config_params)) + + c = az.AZ(config=az_config) return c diff --git a/twindb_backup/configuration/__init__.py b/twindb_backup/configuration/__init__.py index 7cd760d6..2b4f922c 100644 --- a/twindb_backup/configuration/__init__.py +++ b/twindb_backup/configuration/__init__.py @@ -8,7 +8,7 @@ from twindb_backup import INTERVALS, LOG from twindb_backup.configuration.compression import CompressionConfig -from twindb_backup.configuration.destinations.az import AZConfig +from twindb_backup.configuration.destinations.az import AZClientConfig, AZConfig from twindb_backup.configuration.destinations.gcs import GCSConfig from twindb_backup.configuration.destinations.s3 import S3Config from twindb_backup.configuration.destinations.ssh import SSHConfig @@ -103,7 +103,9 @@ def ssh(self): def az(self): # pylint: disable=invalid-name """Azure Blob configuration""" try: - return AZConfig(**self.__read_options_from_section("az")) + az_config = self.__read_options_from_section("az") + az_client_config = self.__read_options_from_section("az.client") + return AZConfig(client_config=AZClientConfig(**az_client_config), **az_config) except NoSectionError: return None @@ -254,11 +256,7 @@ def destination(self, backup_source=socket.gethostname()): ) elif backup_destination == "az": return AZ( - connection_string=self.az.connection_string, - container_name=self.az.container_name, - chunk_size=self.az.chunk_size, - remote_path=self.az.remote_path, - hostname=backup_source, + config=self.az, ) else: raise ConfigurationError(f"Unsupported destination '{backup_destination}'") diff --git a/twindb_backup/configuration/destinations/az.py b/twindb_backup/configuration/destinations/az.py index 6d3c03ab..c88d1450 100644 --- a/twindb_backup/configuration/destinations/az.py +++ b/twindb_backup/configuration/destinations/az.py @@ -1,45 +1,108 @@ """Azure Blob Storage destination configuration""" +import typing as t +from dataclasses import dataclass +# Parameters taken from: +# https://learn.microsoft.com/en-us/python/api/azure-storage-blob/azure.storage.blob.containerclient?view=azure-python#keyword-only-parameters +@dataclass +class AZClientConfig: + """Azure Blob Container Client Configuration + + Attributes: + api_version (str, optional): The version of the Azure Storage API to use. Defaults to None. + secondary_hostname (str, optional): The secondary hostname to use for the storage account. Defaults to None. + max_block_size (int): The maximum size of a block in bytes. Defaults to 4MB. + max_single_put_size (int): The maximum size of a single put operation in bytes. Defaults to 64MB. + min_large_block_upload_threshold (int): The minimum size threshold for large block uploads in bytes. + Defaults to 4MB + 1. + use_byte_buffer (bool): Whether to use a byte buffer for uploads. Defaults to False. + max_page_size (int): The maximum size of a page in bytes. Defaults to 4MB. + max_single_get_size (int): The maximum size of a single get operation in bytes. Defaults to 32MB. + max_chunk_get_size (int): The maximum size of a chunk in bytes for get operations. Defaults to 4MB. + audience (str, optional): The audience for the Azure Storage account. Defaults to None. + """ + + api_version: t.Optional[str] = None + secondary_hostname: t.Optional[str] = None + max_block_size: int = 4 * 1024 * 1024 # 4MB + max_single_put_size: int = 64 * 1024 * 1024 # 64MB + min_large_block_upload_threshold: int = (4 * 1024 * 1024) + 1 # 4MB + 1 + use_byte_buffer: bool = False + max_page_size: int = 4 * 1024 * 1024 # 4MB + max_single_get_size: int = 32 * 1024 * 1024 # 32MB + max_chunk_get_size: int = 4 * 1024 * 1024 # 4MB + audience: t.Optional[str] = None + + def validate(self) -> None: + """Validates the configuration parameters for the Azure destination. + + Raises: + ValueError: Raises a ValueError if the type hint or value is incorrect for any of the parameters. + """ + + if self.api_version is not None and not isinstance(self.api_version, str): + raise ValueError("api_version must be a string or undefined") + if self.secondary_hostname is not None and not isinstance(self.secondary_hostname, str): + raise ValueError("secondary_hostname must be a string or undefined") + if not isinstance(self.max_block_size, int) or self.max_block_size <= 0: + raise ValueError("max_block_size must be a positive integer") + if not isinstance(self.max_single_put_size, int) or self.max_single_put_size <= 0: + raise ValueError("max_single_put_size must be a positive integer") + if not isinstance(self.min_large_block_upload_threshold, int) or self.min_large_block_upload_threshold <= 0: + raise ValueError("min_large_block_upload_threshold must be a positive integer") + if not isinstance(self.use_byte_buffer, bool): + raise ValueError("use_byte_buffer must be a boolean") + if not isinstance(self.max_page_size, int) or self.max_page_size <= 0: + raise ValueError("max_page_size must be a positive integer") + if not isinstance(self.max_single_get_size, int) or self.max_single_get_size <= 0: + raise ValueError("max_single_get_size must be a positive integer") + if not isinstance(self.max_chunk_get_size, int) or self.max_chunk_get_size <= 0: + raise ValueError("max_chunk_get_size must be a positive integer") + if self.audience is not None and not isinstance(self.audience, str): + raise ValueError("audience must be a string or undefined") + + def __post_init__(self) -> None: + self.validate() + + +@dataclass class AZConfig: - """Azure Blob Storage Configuration.""" - - def __init__( - self, connection_string: str, container_name: str, chunk_size: int = 1024 * 1024 * 4, remote_path: str = "/" - ): - self._connection_string = connection_string - self._container_name = container_name - self._chunk_size = chunk_size - self._remote_path = remote_path - self.validate_config() - - def validate_config(self): - """Validate configuration.""" - if not isinstance(self._connection_string, str): - raise ValueError("CONNECTION_STRING must be a string") - if not isinstance(self._container_name, str): - raise ValueError("CONTAINER_NAME must be a string") - if not isinstance(self._chunk_size, int): - raise ValueError("CHUNK_SIZE must be an integer") - if not isinstance(self._remote_path, str): - raise ValueError("REMOTE_PATH must be a string") - - @property - def connection_string(self) -> str: - """CONNECTION_STRING""" - return self._connection_string - - @property - def container_name(self) -> str: - """CONTAINER_NAME""" - return self._container_name - - @property - def chunk_size(self) -> int: - """CHUNK_SIZE""" - return self._chunk_size - - @property - def remote_path(self) -> str: - """REMOTE_PATH""" - return self._remote_path + """Azure Blob Storage Configuration + + Attributes: + client_config (AZClientConfig): Configuration for the Azure Blob Container Client. + connection_string (str): Connection string for the Azure storage account. + container_name (str): Name of the container in the Azure storage account. + remote_path (str, optional): Remote base path in the container to store backups. Defaults to "/". + """ + + client_config: AZClientConfig + connection_string: str + container_name: str + remote_path: str = "/" + + def validate(self) -> None: + """Validates the configuration parameters for the Azure destination. + + Raises: + ValueError: Raises a ValueError if the type hint or value is incorrect for any of the parameters. + """ + + if not isinstance(self.client_config, AZClientConfig): + raise ValueError("client_config must be an instance of AZClientConfig") + if not isinstance(self.connection_string, str): + raise ValueError("connection_string must be a string") + if not isinstance(self.container_name, str): + raise ValueError("container_name must be a string") + if not isinstance(self.remote_path, str): + raise ValueError("remote_path must be a string") + + def __post_init__(self) -> None: + self.validate() + self.remote_path = self.remote_path.strip("/") if self.remote_path != "/" else self.remote_path + + +def drop_empty_dict_factory(d): + """Drop empty values from a dictionary""" + return {k: v for k, v in d if v is not None} diff --git a/twindb_backup/destination/az.py b/twindb_backup/destination/az.py index 748e7dc9..e9fe9941 100644 --- a/twindb_backup/destination/az.py +++ b/twindb_backup/destination/az.py @@ -7,12 +7,14 @@ import socket import typing as t from contextlib import contextmanager +from dataclasses import asdict from multiprocessing import Process import azure.core.exceptions as ae from azure.storage.blob import ContainerClient from twindb_backup import LOG +from twindb_backup.configuration.destinations.az import AZConfig, drop_empty_dict_factory from twindb_backup.copy.base_copy import BaseCopy from twindb_backup.destination.base_destination import BaseDestination from twindb_backup.destination.exceptions import FileNotFound @@ -21,36 +23,21 @@ class AZ(BaseDestination): """Azure Blob Storage Destination class""" - def __init__( - self, - container_name: str, - connection_string: str, - hostname: str = socket.gethostname(), - chunk_size: int = 4 * 1024 * 1024, # TODO: Add support for chunk size - remote_path: str = "/", - ) -> None: + def __init__(self, config: AZConfig) -> None: """Creates an instance of the Azure Blob Storage Destination class, initializes the ContainerClient and validates the connection settings Args: - container_name (str): Name of the container in the Azure storage account - connection_string (str): Connection string for the Azure storage account - hostname (str, optional): Hostname of the host performing the backup. Defaults to socket.gethostname(). - chunk_size (int, optional): Size in bytes for read/write streams. Defaults to 4*1024*1024. - remote_path (str, optional): Remote base path in the container to store backups. Defaults to "/". + config (AZConfig): Azure Blob Storage Configuration Raises: err: Raises an error if the client cannot be initialized """ + self.config = config - self._container_name = container_name - self._connection_string = connection_string - self._hostname = hostname - self._chunk_size = chunk_size - self._remote_path = remote_path.strip("/") if remote_path != "/" else remote_path - super(AZ, self).__init__(self._remote_path) + super(AZ, self).__init__(self.config.remote_path) - self._container_client = self._connect() + self.container_client = self._connect() """HELPER FUNCTIONS """ @@ -70,7 +57,11 @@ def _connect(self) -> ContainerClient: # Create the container client - validates connection string format try: - client = ContainerClient.from_connection_string(self._connection_string, self._container_name) + client = ContainerClient.from_connection_string( + conn_str=self.config.connection_string, + container_name=self.config.container_name, + **asdict(self.config.client_config, dict_factory=drop_empty_dict_factory), + ) except builtins.ValueError as err: LOG.error(f"Failed to create Azure Client. Error: {type(err).__name__}, Reason: {err}") raise err @@ -94,7 +85,7 @@ def render_path(self, path: str) -> str: Returns: str: Absolute path to the blob in the container """ - return f"{self._remote_path}/{path}".strip("/") + return f"{self.config.remote_path}/{path}".strip("/") def _download_to_pipe(self, blob_key: str, pipe_in: int, pipe_out: int) -> None: """Downloads a blob from Azure Blob Storage and writes it to a pipe @@ -107,7 +98,7 @@ def _download_to_pipe(self, blob_key: str, pipe_in: int, pipe_out: int) -> None: os.close(pipe_in) with os.fdopen(pipe_out, "wb") as pipe_out_file: try: - self._container_client.download_blob(blob_key).readinto(pipe_out_file) + self.container_client.download_blob(blob_key).readinto(pipe_out_file) except builtins.Exception as err: LOG.error(f"Failed to download blob {blob_key}. Error: {type(err).__name__}, Reason: {err}") raise err @@ -126,7 +117,7 @@ def delete(self, path: str) -> None: """ LOG.debug(f"Attempting to delete blob: {self.render_path(path)}") try: - self._container_client.delete_blob(self.render_path(path)) + self.container_client.delete_blob(self.render_path(path)) except builtins.Exception as err: LOG.error(f"Failed to delete blob {self.render_path(path)}. Error: {type(err).__name__}, Reason: {err}") raise err @@ -171,10 +162,12 @@ def read(self, filepath: str) -> bytes: """ LOG.debug(f"Attempting to read blob: {self.render_path(filepath)}") try: - return self._container_client.download_blob(self.render_path(filepath), encoding="utf-8").read() + return self.container_client.download_blob(self.render_path(filepath), encoding="utf-8").read() except ae.ResourceNotFoundError: - LOG.debug(f"File {self.render_path(filepath)} does not exist in container {self._container_name}") - raise FileNotFound(f"File {self.render_path(filepath)} does not exist in container {self._container_name}") + LOG.debug(f"File {self.render_path(filepath)} does not exist in container {self.config.container_name}") + raise FileNotFound( + f"File {self.render_path(filepath)} does not exist in container {self.config.container_name}" + ) except builtins.Exception as err: LOG.error(f"Failed to read blob {self.render_path(filepath)}. Error: {type(err).__name__}, Reason: {err}") raise err @@ -193,7 +186,7 @@ def save(self, handler: t.BinaryIO, filepath: str) -> None: LOG.debug(f"Attempting to save blob: {self.render_path(filepath)}") with handler as file_obj: try: - self._container_client.upload_blob(self.render_path(filepath), file_obj) + self.container_client.upload_blob(self.render_path(filepath), file_obj) except builtins.Exception as err: LOG.error(f"Failed to upload blob or it already exists. Error {type(err).__name__}, Reason: {err}") raise err @@ -211,7 +204,7 @@ def write(self, content: str, filepath: str) -> None: LOG.debug(f"Attempting to write blob: {self.render_path(filepath)}") try: - self._container_client.upload_blob(self.render_path(filepath), content, overwrite=True) + self.container_client.upload_blob(self.render_path(filepath), content, overwrite=True) except builtins.Exception as err: LOG.error(f"Failed to upload or overwrite blob. Error {type(err).__name__}, Reason: {err}") raise err @@ -226,20 +219,21 @@ def _list_files(self, prefix: str = "", recursive: bool = False, files_only: boo otherwise includes files and directories. Defaults to False. """ LOG.debug( - f"""Listing files in container {self._container_name} with prefix={prefix.strip('/')}, + f"""Listing files in container {self.config.container_name} with prefix={prefix.strip('/')}, recursive={recursive}, files_only={files_only}""" ) try: - blobs = self._container_client.list_blobs(name_starts_with=prefix.strip("/"), include=["metadata"]) + blobs = self.container_client.list_blobs(name_starts_with=prefix.strip("/"), include=["metadata"]) except builtins.Exception as err: LOG.error( - f"Failed to list files in container {self._container_name}. Error: {type(err).__name__}, Reason: {err}" + f"Failed to list files in container {self.config.container_name}. " + f"Error: {type(err).__name__}, Reason: {err}" ) raise err return [ - blob.name.strip(self._remote_path).strip("/") + blob.name.strip(self.config.remote_path).strip("/") for blob in blobs if not files_only or not (bool(blob.get("metadata")) and blob.get("metadata", {}).get("hdi_isfolder") == "true") From a71ef36cb866c3647ea57f5bee0039c75aa076cb Mon Sep 17 00:00:00 2001 From: Jsalz2000 Date: Mon, 30 Dec 2024 14:43:44 -0600 Subject: [PATCH 07/15] Bumps version to 3.4.1 --- README.rst | 6 +++--- docs/installation.rst | 2 +- omnibus/config/projects/twindb-backup.rb | 2 +- setup.cfg | 2 +- setup.py | 2 +- twindb_backup/__init__.py | 2 +- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/README.rst b/README.rst index cf9d21a6..feaa3f04 100644 --- a/README.rst +++ b/README.rst @@ -126,9 +126,9 @@ Install TwinDB Backup. .. code-block:: console # Download the package - wget https://twindb-release.s3.amazonaws.com/twindb-backup/3.4.0/focal/twindb-backup_3.4.0-1_amd64.deb + wget https://twindb-release.s3.amazonaws.com/twindb-backup/3.4.1/focal/twindb-backup_3.4.1-1_amd64.deb # Install TwinDB Backup - apt install ./twindb-backup_3.4.0-1_amd64.deb + apt install ./twindb-backup_3.4.1-1_amd64.deb Configuring TwinDB Backup ~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -157,7 +157,7 @@ The package file will be generated in ``omnibus/pkg/``: .. code-block:: console $ ls omnibus/pkg/*.deb - omnibus/pkg/twindb-backup_3.4.0-1_amd64.deb + omnibus/pkg/twindb-backup_3.4.1-1_amd64.deb Once the package is built you can install it with rpm/dpkg or upload it to your repository and install it with apt or yum. diff --git a/docs/installation.rst b/docs/installation.rst index 15edff99..461f967c 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -38,7 +38,7 @@ The package file will be generated in ``omnibus/pkg/``: .. code-block:: console $ ls omnibus/pkg/*.deb - omnibus/pkg/twindb-backup_3.4.0-1_amd64.deb + omnibus/pkg/twindb-backup_3.4.1-1_amd64.deb Once the package is built you can install it with rpm/dpkg or upload it to your repository and install it with apt or yum. diff --git a/omnibus/config/projects/twindb-backup.rb b/omnibus/config/projects/twindb-backup.rb index bef6b2cd..64aad887 100644 --- a/omnibus/config/projects/twindb-backup.rb +++ b/omnibus/config/projects/twindb-backup.rb @@ -23,7 +23,7 @@ # and /opt/twindb-backup on all other platforms install_dir '/opt/twindb-backup' -build_version '3.4.0' +build_version '3.4.1' build_iteration 1 diff --git a/setup.cfg b/setup.cfg index 08be9c7c..cfe58c68 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 3.4.0 +current_version = 3.4.1 commit = True tag = False diff --git a/setup.py b/setup.py index b5216d47..c8b6a721 100644 --- a/setup.py +++ b/setup.py @@ -20,7 +20,7 @@ setup( name="twindb-backup", - version="3.4.0", + version="3.4.1", description="TwinDB Backup tool for files, MySQL et al.", long_description=readme + "\n\n" + history, author="TwinDB Development Team", diff --git a/twindb_backup/__init__.py b/twindb_backup/__init__.py index 019fb496..86075b09 100644 --- a/twindb_backup/__init__.py +++ b/twindb_backup/__init__.py @@ -40,7 +40,7 @@ class and saves the backup copy in something defined in a destination class. __author__ = "TwinDB Development Team" __email__ = "dev@twindb.com" -__version__ = "3.4.0" +__version__ = "3.4.1" STATUS_FORMAT_VERSION = 1 LOCK_FILE = "/var/run/twindb-backup.lock" LOG_FILE = "/var/log/twindb-backup-measures.log" From 2ecf1c81ee1a2cb57cd31163552b259746635d28 Mon Sep 17 00:00:00 2001 From: Jsalz2000 Date: Mon, 30 Dec 2024 15:26:28 -0600 Subject: [PATCH 08/15] Documents optional az.client configuration block --- docs/usage.rst | 16 ++++++++++++++++ support/twindb-backup.cfg | 15 +++++++++++++++ tests/unit/conftest.py | 12 ++++++++++++ .../modules/profile/files/twindb-backup.cfg | 13 +++++++++++++ 4 files changed, 56 insertions(+) diff --git a/docs/usage.rst b/docs/usage.rst index b0ffcbb6..c02bcb3b 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -102,6 +102,22 @@ In the ``[az]`` section you specify Azure credentials as well as Azure Blob Stor container_name = twindb-backups remote_path = /backups/mysql # optional +In the ``[az.client]`` section you specify optional Azure Blob Storage client options. + +.. code-block:: ini + + [az.client] + + api_version = "2019-02-02" + secondary_hostname = "ACCOUNT_NAME-secondary.blob.core.windows.net" + max_block_size = 4194304 + max_single_put_size = 67108864 + min_large_block_upload_threshold = 4194305 + use_byte_buffer = yes + max_page_size = 4194304 + max_single_get_size = 33554432 + max_chunk_get_size = 4194304 + audience = "https://storage.azure.com/" Google Cloud Storage ~~~~~~~~~~~~~~~~~~~~ diff --git a/support/twindb-backup.cfg b/support/twindb-backup.cfg index 44c6130b..dd81ab6a 100644 --- a/support/twindb-backup.cfg +++ b/support/twindb-backup.cfg @@ -39,6 +39,21 @@ connection_string="DefaultEndpointsProtocol=https;AccountName=ACCOUNT_NAME;Accou container_name=twindb-backups #remote_path = /backups/mysql # optional +[az.client] + +# Azure client optional settings + +# api_version="2019-02-02" # optional +# secondary_hostname="ACCOUNT_NAME-secondary.blob.core.windows.net" # optional +# max_block_size=4194304 # optional +# max_single_put_size=67108864 # optional +# min_large_block_upload_threshold=4194305 # optional +# use_byte_buffer=yes # optional +# max_page_size=4194304 # optional +# max_single_get_size=33554432 # optional +# max_chunk_get_size=4194304 # optional +# audience="https://storage.azure.com/" # optional + [gcs] # GCS destination settings diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index 9c2aaea9..9447b13c 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -33,6 +33,18 @@ def config_content(): container_name="twindb-backups" remote_path="/backups/mysql" +[az.client] +api_version="2019-02-02" +secondary_hostname="ACCOUNT_NAME-secondary.blob.core.windows.net" +max_block_size=4194304 +max_single_put_size=67108864 +min_large_block_upload_threshold=4194305 +use_byte_buffer=yes +max_page_size=4194304 +max_single_get_size=33554432 +max_chunk_get_size=4194304 +audience="https://storage.azure.com/" + [gcs] GC_CREDENTIALS_FILE="XXXXX" GC_ENCRYPTION_KEY= diff --git a/vagrant/environment/puppet/modules/profile/files/twindb-backup.cfg b/vagrant/environment/puppet/modules/profile/files/twindb-backup.cfg index 09bc007b..c2c93cd3 100644 --- a/vagrant/environment/puppet/modules/profile/files/twindb-backup.cfg +++ b/vagrant/environment/puppet/modules/profile/files/twindb-backup.cfg @@ -28,6 +28,19 @@ connection_string="DefaultEndpointsProtocol=https;AccountName=ACCOUNT_NAME;Accou container_name="twindb-backups" #remote_path = /backups/mysql # optional +# Azure client optional settings +#[az.client] +# api_version="2019-02-02" # optional +# secondary_hostname="ACCOUNT_NAME-secondary.blob.core.windows.net" # optional +# max_block_size=4194304 # optional +# max_single_put_size=67108864 # optional +# min_large_block_upload_threshold=4194305 # optional +# use_byte_buffer=yes # optional +# max_page_size=4194304 # optional +# max_single_get_size=33554432 # optional +# max_chunk_get_size=4194304 # optional +# audience="https://storage.azure.com/" # optional + # GCS destination settings [gcs] GC_CREDENTIALS_FILE=/twindb_backup/env/My Project 17339-bbbc43d1bee3.json From 6094d6713b4847b76b35babbcda6affba7c9d920 Mon Sep 17 00:00:00 2001 From: Jsalz2000 Date: Tue, 31 Dec 2024 12:30:11 -0600 Subject: [PATCH 09/15] Adds whitespace below file header in az config --- twindb_backup/configuration/destinations/az.py | 1 + 1 file changed, 1 insertion(+) diff --git a/twindb_backup/configuration/destinations/az.py b/twindb_backup/configuration/destinations/az.py index c88d1450..35df62b3 100644 --- a/twindb_backup/configuration/destinations/az.py +++ b/twindb_backup/configuration/destinations/az.py @@ -1,4 +1,5 @@ """Azure Blob Storage destination configuration""" + import typing as t from dataclasses import dataclass From 0c3957d121a6706c012a23d46cad5e0cb743657f Mon Sep 17 00:00:00 2001 From: Jsalz2000 Date: Tue, 31 Dec 2024 15:55:16 -0600 Subject: [PATCH 10/15] Adds __cast_options function in config parsing --- docs/usage.rst | 2 +- support/twindb-backup.cfg | 2 +- tests/unit/conftest.py | 2 +- twindb_backup/configuration/__init__.py | 22 ++++++++++++++++++- .../modules/profile/files/twindb-backup.cfg | 2 +- 5 files changed, 25 insertions(+), 5 deletions(-) diff --git a/docs/usage.rst b/docs/usage.rst index c02bcb3b..caa0755b 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -113,7 +113,7 @@ In the ``[az.client]`` section you specify optional Azure Blob Storage client op max_block_size = 4194304 max_single_put_size = 67108864 min_large_block_upload_threshold = 4194305 - use_byte_buffer = yes + use_byte_buffer = true max_page_size = 4194304 max_single_get_size = 33554432 max_chunk_get_size = 4194304 diff --git a/support/twindb-backup.cfg b/support/twindb-backup.cfg index dd81ab6a..22bf38d2 100644 --- a/support/twindb-backup.cfg +++ b/support/twindb-backup.cfg @@ -48,7 +48,7 @@ container_name=twindb-backups # max_block_size=4194304 # optional # max_single_put_size=67108864 # optional # min_large_block_upload_threshold=4194305 # optional -# use_byte_buffer=yes # optional +# use_byte_buffer=true # optional # max_page_size=4194304 # optional # max_single_get_size=33554432 # optional # max_chunk_get_size=4194304 # optional diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index 9447b13c..dcf85cf3 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -39,7 +39,7 @@ def config_content(): max_block_size=4194304 max_single_put_size=67108864 min_large_block_upload_threshold=4194305 -use_byte_buffer=yes +use_byte_buffer=true max_page_size=4194304 max_single_get_size=33554432 max_chunk_get_size=4194304 diff --git a/twindb_backup/configuration/__init__.py b/twindb_backup/configuration/__init__.py index 2b4f922c..ac3a340b 100644 --- a/twindb_backup/configuration/__init__.py +++ b/twindb_backup/configuration/__init__.py @@ -3,6 +3,7 @@ Module to process configuration file. """ import socket +import typing as t from configparser import ConfigParser, NoOptionError, NoSectionError from shlex import split @@ -104,7 +105,7 @@ def az(self): # pylint: disable=invalid-name """Azure Blob configuration""" try: az_config = self.__read_options_from_section("az") - az_client_config = self.__read_options_from_section("az.client") + az_client_config = self.__cast_options(self.__read_options_from_section("az.client")) return AZConfig(client_config=AZClientConfig(**az_client_config), **az_config) except NoSectionError: @@ -278,3 +279,22 @@ def __read_options_from_section(self, section): def __repr__(self): return f"{self.__class__.__name__}: {self._config_file}" + + def __cast_options(self, options: t.Dict[str, str]) -> t.Dict[str, t.Union[str, int, bool]]: + """Cast options to their correct types + + Args: + options (t.Dict[str, str]): A dictionary of kwargs to cast + + Returns: + t.Dict[str, t.Union[str, int, bool]]: An updated dictionary with the correct types + """ + for k,v in options.items(): + if v.lower() == "true": + options[k] = True + elif v.lower() == "false": + options[k] = False + elif v.isdigit(): + options[k] = int(v) + + return options diff --git a/vagrant/environment/puppet/modules/profile/files/twindb-backup.cfg b/vagrant/environment/puppet/modules/profile/files/twindb-backup.cfg index c2c93cd3..82fd9308 100644 --- a/vagrant/environment/puppet/modules/profile/files/twindb-backup.cfg +++ b/vagrant/environment/puppet/modules/profile/files/twindb-backup.cfg @@ -35,7 +35,7 @@ container_name="twindb-backups" # max_block_size=4194304 # optional # max_single_put_size=67108864 # optional # min_large_block_upload_threshold=4194305 # optional -# use_byte_buffer=yes # optional +# use_byte_buffer=true # optional # max_page_size=4194304 # optional # max_single_get_size=33554432 # optional # max_chunk_get_size=4194304 # optional From fc0582b30ffdd8bd9733430ec17c6ad791259f06 Mon Sep 17 00:00:00 2001 From: Jsalz2000 Date: Thu, 2 Jan 2025 13:57:42 -0600 Subject: [PATCH 11/15] Makes az.client config optional in config parser --- twindb_backup/configuration/__init__.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/twindb_backup/configuration/__init__.py b/twindb_backup/configuration/__init__.py index ac3a340b..6a7608cc 100644 --- a/twindb_backup/configuration/__init__.py +++ b/twindb_backup/configuration/__init__.py @@ -105,12 +105,17 @@ def az(self): # pylint: disable=invalid-name """Azure Blob configuration""" try: az_config = self.__read_options_from_section("az") - az_client_config = self.__cast_options(self.__read_options_from_section("az.client")) - return AZConfig(client_config=AZClientConfig(**az_client_config), **az_config) - except NoSectionError: return None + az_client_config = {} + try: + az_client_config = self.__cast_options(self.__read_options_from_section("az.client")) + except: + pass + + return AZConfig(client_config=AZClientConfig(**az_client_config), **az_config) + @property def s3(self): # pylint: disable=invalid-name """Amazon S3 configuration""" From 1e5cbba968e0480b03e1279f9f1652e15cf9bfca Mon Sep 17 00:00:00 2001 From: Jsalz2000 Date: Thu, 2 Jan 2025 15:31:27 -0600 Subject: [PATCH 12/15] Adds connection_timeout and max_concurrency settings to Azure Blob destination --- twindb_backup/configuration/destinations/az.py | 8 ++++++++ twindb_backup/destination/az.py | 8 ++++---- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/twindb_backup/configuration/destinations/az.py b/twindb_backup/configuration/destinations/az.py index 35df62b3..7316d46f 100644 --- a/twindb_backup/configuration/destinations/az.py +++ b/twindb_backup/configuration/destinations/az.py @@ -22,6 +22,7 @@ class AZClientConfig: max_single_get_size (int): The maximum size of a single get operation in bytes. Defaults to 32MB. max_chunk_get_size (int): The maximum size of a chunk in bytes for get operations. Defaults to 4MB. audience (str, optional): The audience for the Azure Storage account. Defaults to None. + connection_timeout (int): The connection timeout in seconds. Defaults to 20. """ api_version: t.Optional[str] = None @@ -34,6 +35,7 @@ class AZClientConfig: max_single_get_size: int = 32 * 1024 * 1024 # 32MB max_chunk_get_size: int = 4 * 1024 * 1024 # 4MB audience: t.Optional[str] = None + connection_timeout: int = 20 def validate(self) -> None: """Validates the configuration parameters for the Azure destination. @@ -62,6 +64,8 @@ def validate(self) -> None: raise ValueError("max_chunk_get_size must be a positive integer") if self.audience is not None and not isinstance(self.audience, str): raise ValueError("audience must be a string or undefined") + if not isinstance(self.connection_timeout, int) or self.connection_timeout <= 0: + raise ValueError("connection_timeout must be a positive integer") def __post_init__(self) -> None: self.validate() @@ -76,12 +80,14 @@ class AZConfig: connection_string (str): Connection string for the Azure storage account. container_name (str): Name of the container in the Azure storage account. remote_path (str, optional): Remote base path in the container to store backups. Defaults to "/". + max_concurrency (int, optional): Maximum number of concurrent requests to the Azure Storage service. Defaults to 1. """ client_config: AZClientConfig connection_string: str container_name: str remote_path: str = "/" + max_concurrency: int = 1 def validate(self) -> None: """Validates the configuration parameters for the Azure destination. @@ -98,6 +104,8 @@ def validate(self) -> None: raise ValueError("container_name must be a string") if not isinstance(self.remote_path, str): raise ValueError("remote_path must be a string") + if not isinstance(self.max_concurrency, int) or self.max_concurrency <= 0: + raise ValueError("max_concurrency must be a positive integer") def __post_init__(self) -> None: self.validate() diff --git a/twindb_backup/destination/az.py b/twindb_backup/destination/az.py index e9fe9941..44759fe7 100644 --- a/twindb_backup/destination/az.py +++ b/twindb_backup/destination/az.py @@ -98,7 +98,7 @@ def _download_to_pipe(self, blob_key: str, pipe_in: int, pipe_out: int) -> None: os.close(pipe_in) with os.fdopen(pipe_out, "wb") as pipe_out_file: try: - self.container_client.download_blob(blob_key).readinto(pipe_out_file) + self.container_client.download_blob(blob_key, max_concurrency=self.config.max_concurrency).readinto(pipe_out_file) except builtins.Exception as err: LOG.error(f"Failed to download blob {blob_key}. Error: {type(err).__name__}, Reason: {err}") raise err @@ -162,7 +162,7 @@ def read(self, filepath: str) -> bytes: """ LOG.debug(f"Attempting to read blob: {self.render_path(filepath)}") try: - return self.container_client.download_blob(self.render_path(filepath), encoding="utf-8").read() + return self.container_client.download_blob(self.render_path(filepath), encoding="utf-8", max_concurrency=self.config.max_concurrency).read() except ae.ResourceNotFoundError: LOG.debug(f"File {self.render_path(filepath)} does not exist in container {self.config.container_name}") raise FileNotFound( @@ -186,7 +186,7 @@ def save(self, handler: t.BinaryIO, filepath: str) -> None: LOG.debug(f"Attempting to save blob: {self.render_path(filepath)}") with handler as file_obj: try: - self.container_client.upload_blob(self.render_path(filepath), file_obj) + self.container_client.upload_blob(self.render_path(filepath), file_obj, max_concurrency=self.config.max_concurrency) except builtins.Exception as err: LOG.error(f"Failed to upload blob or it already exists. Error {type(err).__name__}, Reason: {err}") raise err @@ -204,7 +204,7 @@ def write(self, content: str, filepath: str) -> None: LOG.debug(f"Attempting to write blob: {self.render_path(filepath)}") try: - self.container_client.upload_blob(self.render_path(filepath), content, overwrite=True) + self.container_client.upload_blob(self.render_path(filepath), content, overwrite=True, max_concurrency=self.config.max_concurrency) except builtins.Exception as err: LOG.error(f"Failed to upload or overwrite blob. Error {type(err).__name__}, Reason: {err}") raise err From 925d98c40060a657e020518b485c886ac3093630 Mon Sep 17 00:00:00 2001 From: Jsalz2000 Date: Thu, 2 Jan 2025 15:41:13 -0600 Subject: [PATCH 13/15] Formats with black, isort. Casts az config params --- twindb_backup/configuration/__init__.py | 6 +++--- twindb_backup/configuration/destinations/az.py | 3 ++- twindb_backup/destination/az.py | 16 ++++++++++++---- 3 files changed, 17 insertions(+), 8 deletions(-) diff --git a/twindb_backup/configuration/__init__.py b/twindb_backup/configuration/__init__.py index 6a7608cc..d9623707 100644 --- a/twindb_backup/configuration/__init__.py +++ b/twindb_backup/configuration/__init__.py @@ -104,14 +104,14 @@ def ssh(self): def az(self): # pylint: disable=invalid-name """Azure Blob configuration""" try: - az_config = self.__read_options_from_section("az") + az_config = self.__cast_options(self.__read_options_from_section("az")) except NoSectionError: return None az_client_config = {} try: az_client_config = self.__cast_options(self.__read_options_from_section("az.client")) - except: + except Exception: pass return AZConfig(client_config=AZClientConfig(**az_client_config), **az_config) @@ -294,7 +294,7 @@ def __cast_options(self, options: t.Dict[str, str]) -> t.Dict[str, t.Union[str, Returns: t.Dict[str, t.Union[str, int, bool]]: An updated dictionary with the correct types """ - for k,v in options.items(): + for k, v in options.items(): if v.lower() == "true": options[k] = True elif v.lower() == "false": diff --git a/twindb_backup/configuration/destinations/az.py b/twindb_backup/configuration/destinations/az.py index 7316d46f..ea0ff1ea 100644 --- a/twindb_backup/configuration/destinations/az.py +++ b/twindb_backup/configuration/destinations/az.py @@ -80,7 +80,8 @@ class AZConfig: connection_string (str): Connection string for the Azure storage account. container_name (str): Name of the container in the Azure storage account. remote_path (str, optional): Remote base path in the container to store backups. Defaults to "/". - max_concurrency (int, optional): Maximum number of concurrent requests to the Azure Storage service. Defaults to 1. + max_concurrency (int, optional): Maximum number of concurrent requests to the Azure Storage service. + Defaults to 1. """ client_config: AZClientConfig diff --git a/twindb_backup/destination/az.py b/twindb_backup/destination/az.py index 44759fe7..397f1d97 100644 --- a/twindb_backup/destination/az.py +++ b/twindb_backup/destination/az.py @@ -98,7 +98,9 @@ def _download_to_pipe(self, blob_key: str, pipe_in: int, pipe_out: int) -> None: os.close(pipe_in) with os.fdopen(pipe_out, "wb") as pipe_out_file: try: - self.container_client.download_blob(blob_key, max_concurrency=self.config.max_concurrency).readinto(pipe_out_file) + self.container_client.download_blob(blob_key, max_concurrency=self.config.max_concurrency).readinto( + pipe_out_file + ) except builtins.Exception as err: LOG.error(f"Failed to download blob {blob_key}. Error: {type(err).__name__}, Reason: {err}") raise err @@ -162,7 +164,9 @@ def read(self, filepath: str) -> bytes: """ LOG.debug(f"Attempting to read blob: {self.render_path(filepath)}") try: - return self.container_client.download_blob(self.render_path(filepath), encoding="utf-8", max_concurrency=self.config.max_concurrency).read() + return self.container_client.download_blob( + self.render_path(filepath), encoding="utf-8", max_concurrency=self.config.max_concurrency + ).read() except ae.ResourceNotFoundError: LOG.debug(f"File {self.render_path(filepath)} does not exist in container {self.config.container_name}") raise FileNotFound( @@ -186,7 +190,9 @@ def save(self, handler: t.BinaryIO, filepath: str) -> None: LOG.debug(f"Attempting to save blob: {self.render_path(filepath)}") with handler as file_obj: try: - self.container_client.upload_blob(self.render_path(filepath), file_obj, max_concurrency=self.config.max_concurrency) + self.container_client.upload_blob( + self.render_path(filepath), file_obj, max_concurrency=self.config.max_concurrency + ) except builtins.Exception as err: LOG.error(f"Failed to upload blob or it already exists. Error {type(err).__name__}, Reason: {err}") raise err @@ -204,7 +210,9 @@ def write(self, content: str, filepath: str) -> None: LOG.debug(f"Attempting to write blob: {self.render_path(filepath)}") try: - self.container_client.upload_blob(self.render_path(filepath), content, overwrite=True, max_concurrency=self.config.max_concurrency) + self.container_client.upload_blob( + self.render_path(filepath), content, overwrite=True, max_concurrency=self.config.max_concurrency + ) except builtins.Exception as err: LOG.error(f"Failed to upload or overwrite blob. Error {type(err).__name__}, Reason: {err}") raise err From ff93bc76a1b081a87390a2bf34f277e39dcd35f0 Mon Sep 17 00:00:00 2001 From: Jsalz2000 Date: Thu, 2 Jan 2025 15:41:57 -0600 Subject: [PATCH 14/15] Adds connection_timeout and max_concurrency Azure Blob unit tests --- tests/unit/destination/az/test_config.py | 13 +++++++++++++ tests/unit/destination/az/test_download_to_pipe.py | 8 ++++++-- tests/unit/destination/az/test_read.py | 12 +++++++++--- tests/unit/destination/az/test_save.py | 8 ++++++-- tests/unit/destination/az/test_write.py | 8 ++++++-- tests/unit/destination/az/util.py | 2 ++ 6 files changed, 42 insertions(+), 9 deletions(-) diff --git a/tests/unit/destination/az/test_config.py b/tests/unit/destination/az/test_config.py index 4e499c03..b6a01f40 100644 --- a/tests/unit/destination/az/test_config.py +++ b/tests/unit/destination/az/test_config.py @@ -24,6 +24,7 @@ def test_initialization_success(): if config_params.remote_path != "/" else config_params.remote_path ) + assert c.max_concurrency == config_params.max_concurrency # AZClientConfig Assertions assert c.client_config.api_version == client_params.api_version @@ -36,6 +37,7 @@ def test_initialization_success(): assert c.client_config.max_single_get_size == client_params.max_single_get_size assert c.client_config.max_chunk_get_size == client_params.max_chunk_get_size assert c.client_config.audience == client_params.audience + assert c.client_config.connection_timeout == client_params.connection_timeout def test_initialization_success_defaults(): @@ -51,6 +53,7 @@ def test_initialization_success_defaults(): assert c.connection_string == config_params.connection_string assert c.container_name == config_params.container_name assert c.remote_path == "/" + assert c.max_concurrency == 1 # AZClientConfig Assertions assert c.client_config.api_version == None @@ -63,6 +66,7 @@ def test_initialization_success_defaults(): assert c.client_config.max_single_get_size == 32 * 1024 * 1024 # 32MB assert c.client_config.max_chunk_get_size == 4 * 1024 * 1024 # 4MB assert c.client_config.audience == None + assert c.client_config.connection_timeout == 20 def test_invalid_params(): @@ -82,6 +86,13 @@ def test_invalid_params(): ) with pytest.raises(ValueError): # Invalid container_name AZConfig(client_config=AZClientConfig(), connection_string="test_connection_string", container_name=1) + with pytest.raises(ValueError): # Invalid max_concurrency + AZConfig( + client_config=AZClientConfig(), + connection_string="test_connection_string", + container_name="test_container", + max_concurrency="1", + ) # Invalidate AZClientConfig with pytest.raises(ValueError): # Invalid api_version @@ -104,6 +115,8 @@ def test_invalid_params(): AZClientConfig(max_chunk_get_size="123") with pytest.raises(ValueError): # Invalid audience AZClientConfig(audience=123) + with pytest.raises(ValueError): # Invalid connection_timeout + AZClientConfig(connection_timeout="123") def test_drop_empty_dicts_some_undefined(): diff --git a/tests/unit/destination/az/test_download_to_pipe.py b/tests/unit/destination/az/test_download_to_pipe.py index 7f92cb56..7f8c63ee 100644 --- a/tests/unit/destination/az/test_download_to_pipe.py +++ b/tests/unit/destination/az/test_download_to_pipe.py @@ -21,7 +21,9 @@ def test_download_to_pipe_success(): mc_os.close.assert_called_once_with(100) mc_os.fdopen.assert_called_once_with(200, "wb") - c.container_client.download_blob.assert_called_once_with(c.render_path("foo-key")) + c.container_client.download_blob.assert_called_once_with( + c.render_path("foo-key"), max_concurrency=c.config.max_concurrency + ) mc_dbr.readinto.assert_called_once_with(mc_fdopen.__enter__()) @@ -37,4 +39,6 @@ def test_download_to_pipe_fail(): mc_os.close.assert_called_once_with(100) mc_os.fdopen.assert_called_once_with(200, "wb") - c.container_client.download_blob.assert_called_once_with(c.render_path("foo-key")) + c.container_client.download_blob.assert_called_once_with( + c.render_path("foo-key"), max_concurrency=c.config.max_concurrency + ) diff --git a/tests/unit/destination/az/test_read.py b/tests/unit/destination/az/test_read.py index 0ce3597b..110a8cc7 100644 --- a/tests/unit/destination/az/test_read.py +++ b/tests/unit/destination/az/test_read.py @@ -19,7 +19,9 @@ def test_read_success(): c.read(EXAMPLE_FILE) - c.container_client.download_blob.assert_called_once_with(c.render_path(EXAMPLE_FILE), encoding="utf-8") + c.container_client.download_blob.assert_called_once_with( + c.render_path(EXAMPLE_FILE), encoding="utf-8", max_concurrency=c.config.max_concurrency + ) mock.read.assert_called_once() @@ -30,7 +32,9 @@ def test_read_fail(): with pytest.raises(Exception): c.read(EXAMPLE_FILE) - c.container_client.download_blob.assert_called_once_with(c.render_path(EXAMPLE_FILE), encoding="utf-8") + c.container_client.download_blob.assert_called_once_with( + c.render_path(EXAMPLE_FILE), encoding="utf-8", max_concurrency=c.config.max_concurrency + ) def test_read_fail_not_found(): @@ -42,4 +46,6 @@ def test_read_fail_not_found(): FileNotFound, match=f"File {c.render_path(EXAMPLE_FILE)} does not exist in container {c.config.container_name}" ): c.read(EXAMPLE_FILE) - c.container_client.download_blob.assert_called_once_with(c.render_path(EXAMPLE_FILE), encoding="utf-8") + c.container_client.download_blob.assert_called_once_with( + c.render_path(EXAMPLE_FILE), encoding="utf-8", max_concurrency=c.config.max_concurrency + ) diff --git a/tests/unit/destination/az/test_save.py b/tests/unit/destination/az/test_save.py index 68d98486..2965c321 100644 --- a/tests/unit/destination/az/test_save.py +++ b/tests/unit/destination/az/test_save.py @@ -19,7 +19,9 @@ def test_save_success(): c.save(handler, EXAMPLE_FILE) - c.container_client.upload_blob.assert_called_once_with(c.render_path(EXAMPLE_FILE), file_obj) + c.container_client.upload_blob.assert_called_once_with( + c.render_path(EXAMPLE_FILE), file_obj, max_concurrency=c.config.max_concurrency + ) def test_save_fail(): @@ -34,4 +36,6 @@ def test_save_fail(): with pytest.raises(Exception): c.save(handler, EXAMPLE_FILE) - c.container_client.upload_blob.assert_called_once_with(c.render_path(EXAMPLE_FILE), file_obj) + c.container_client.upload_blob.assert_called_once_with( + c.render_path(EXAMPLE_FILE), file_obj, max_concurrency=c.config.max_concurrency + ) diff --git a/tests/unit/destination/az/test_write.py b/tests/unit/destination/az/test_write.py index 839f4e75..282daf0f 100644 --- a/tests/unit/destination/az/test_write.py +++ b/tests/unit/destination/az/test_write.py @@ -13,7 +13,9 @@ def test_write_success(): c.write(CONTENT, EXAMPLE_FILE) - c.container_client.upload_blob.assert_called_once_with(c.render_path(EXAMPLE_FILE), CONTENT, overwrite=True) + c.container_client.upload_blob.assert_called_once_with( + c.render_path(EXAMPLE_FILE), CONTENT, overwrite=True, max_concurrency=c.config.max_concurrency + ) def test_write_fail(): @@ -24,4 +26,6 @@ def test_write_fail(): with pytest.raises(Exception): c.write(CONTENT, EXAMPLE_FILE) - c.container_client.upload_blob.assert_called_once_with(c.render_path(EXAMPLE_FILE), CONTENT, overwrite=True) + c.container_client.upload_blob.assert_called_once_with( + c.render_path(EXAMPLE_FILE), CONTENT, overwrite=True, max_concurrency=c.config.max_concurrency + ) diff --git a/tests/unit/destination/az/util.py b/tests/unit/destination/az/util.py index fe10a4b2..44be2bd7 100644 --- a/tests/unit/destination/az/util.py +++ b/tests/unit/destination/az/util.py @@ -21,6 +21,7 @@ def __init__(self, only_required=False) -> None: self.max_single_get_size = 128 * 1024 * 1024 # 128MB self.max_chunk_get_size = 128 * 1024 * 1024 # 128MB self.audience = "https://example.com" + self.connection_timeout = 30 def __iter__(self): return iter(self.__dict__) @@ -39,6 +40,7 @@ def __init__(self, only_required=False) -> None: if not only_required: self.remote_path = "/himom/" + self.max_concurrency = 4 def __iter__(self): return iter(self.__dict__) From 91275a1bec1b983e26b3eac97458bf7b5397c35e Mon Sep 17 00:00:00 2001 From: Jsalz2000 Date: Thu, 2 Jan 2025 16:00:17 -0600 Subject: [PATCH 15/15] Adds connection_timeout and max_concurrency Azure Blob options to docs --- docs/usage.rst | 2 ++ support/twindb-backup.cfg | 2 ++ tests/unit/conftest.py | 2 ++ .../environment/puppet/modules/profile/files/twindb-backup.cfg | 2 ++ 4 files changed, 8 insertions(+) diff --git a/docs/usage.rst b/docs/usage.rst index caa0755b..97a455bd 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -101,6 +101,7 @@ In the ``[az]`` section you specify Azure credentials as well as Azure Blob Stor connection_string = "DefaultEndpointsProtocol=https;AccountName=ACCOUNT_NAME;AccountKey=ACCOUNT_KEY;EndpointSuffix=core.windows.net" container_name = twindb-backups remote_path = /backups/mysql # optional + max_concurrency = 1 # optional In the ``[az.client]`` section you specify optional Azure Blob Storage client options. @@ -118,6 +119,7 @@ In the ``[az.client]`` section you specify optional Azure Blob Storage client op max_single_get_size = 33554432 max_chunk_get_size = 4194304 audience = "https://storage.azure.com/" + connection_timeout = 20 Google Cloud Storage ~~~~~~~~~~~~~~~~~~~~ diff --git a/support/twindb-backup.cfg b/support/twindb-backup.cfg index 22bf38d2..62d0c3bc 100644 --- a/support/twindb-backup.cfg +++ b/support/twindb-backup.cfg @@ -38,6 +38,7 @@ BUCKET=twindb-backups connection_string="DefaultEndpointsProtocol=https;AccountName=ACCOUNT_NAME;AccountKey=ACCOUNT_KEY;EndpointSuffix=core.windows.net" container_name=twindb-backups #remote_path = /backups/mysql # optional +#max_concurrency = 1 # optional [az.client] @@ -53,6 +54,7 @@ container_name=twindb-backups # max_single_get_size=33554432 # optional # max_chunk_get_size=4194304 # optional # audience="https://storage.azure.com/" # optional +# connection_timeout=20 # optional [gcs] diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index dcf85cf3..30422753 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -32,6 +32,7 @@ def config_content(): connection_string="DefaultEndpointsProtocol=https;AccountName=ACCOUNT_NAME;AccountKey=ACCOUNT_KEY;EndpointSuffix=core.windows.net" container_name="twindb-backups" remote_path="/backups/mysql" +max_concurrency=1 [az.client] api_version="2019-02-02" @@ -44,6 +45,7 @@ def config_content(): max_single_get_size=33554432 max_chunk_get_size=4194304 audience="https://storage.azure.com/" +connection_timeout=20 [gcs] GC_CREDENTIALS_FILE="XXXXX" diff --git a/vagrant/environment/puppet/modules/profile/files/twindb-backup.cfg b/vagrant/environment/puppet/modules/profile/files/twindb-backup.cfg index 82fd9308..55b2c7f2 100644 --- a/vagrant/environment/puppet/modules/profile/files/twindb-backup.cfg +++ b/vagrant/environment/puppet/modules/profile/files/twindb-backup.cfg @@ -27,6 +27,7 @@ BUCKET="twindb-backups" connection_string="DefaultEndpointsProtocol=https;AccountName=ACCOUNT_NAME;AccountKey=ACCOUNT_KEY;EndpointSuffix=core.windows.net" container_name="twindb-backups" #remote_path = /backups/mysql # optional +#max_concurrency = 1 # optional # Azure client optional settings #[az.client] @@ -40,6 +41,7 @@ container_name="twindb-backups" # max_single_get_size=33554432 # optional # max_chunk_get_size=4194304 # optional # audience="https://storage.azure.com/" # optional +# connection_timeout=20 # optional # GCS destination settings [gcs]