From 70bb231bb225f61c16087180a21602a4abdcee2a Mon Sep 17 00:00:00 2001 From: Martin Raiber Date: Tue, 21 Jan 2025 11:14:27 +0100 Subject: [PATCH 1/2] Add callback/class to track upload and download progress --- .vscode/settings.json | 95 +++++++++++++++++++++------------------- filecloudapi/fcserver.py | 59 ++++++++++++++++++++++--- pyproject.toml | 2 +- 3 files changed, 105 insertions(+), 51 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index e06fa34..efc406c 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,45 +1,52 @@ { - "files.exclude": { - ".cache/": true, - ".venv/": true, - "*.egg-info": true, - "pip-wheel-metadata/": true, - "**/__pycache__": true, - "**/*.pyc": true, - "**/.ipynb_checkpoints": true, - "**/tmp/": true, - "dist/": true, - "htmlcov/": true, - "notebooks/*.yml": true, - "notebooks/files/": true, - "notebooks/inventory/": true, - "prof/": true, - "site/": true, - "geckodriver.log": true, - "targets.log": true, - "bin/verchew": true - }, - "editor.formatOnSave": true, - "pylint.args": ["--rcfile=.pylint.ini"], - "cSpell.words": [ - "asdf", - "builtins", - "codecov", - "codehilite", - "choco", - "cygstart", - "cygwin", - "dataclasses", - "Graphviz", - "ipython", - "mkdocs", - "noclasses", - "pipx", - "pyenv", - "ruamel", - "showfspath", - "USERPROFILE", - "venv", - "verchew" - ] - } + "files.exclude": { + ".cache/": true, + ".venv/": true, + "*.egg-info": true, + "pip-wheel-metadata/": true, + "**/__pycache__": true, + "**/*.pyc": true, + "**/.ipynb_checkpoints": true, + "**/tmp/": true, + "dist/": true, + "htmlcov/": true, + "notebooks/*.yml": true, + "notebooks/files/": true, + "notebooks/inventory/": true, + "prof/": true, + "site/": true, + "geckodriver.log": true, + "targets.log": true, + "bin/verchew": true + }, + "editor.formatOnSave": true, + "pylint.args": [ + "--rcfile=.pylint.ini" + ], + "cSpell.words": [ + "asdf", + "builtins", + "codecov", + "codehilite", + "choco", + "cygstart", + "cygwin", + "dataclasses", + "Graphviz", + "ipython", + "mkdocs", + "noclasses", + "pipx", + "pyenv", + "ruamel", + "showfspath", + "USERPROFILE", + "venv", + "verchew" + ], + "python.testing.pytestArgs": [ + "" + ], + "python.testing.unittestEnabled": false, + "python.testing.pytestEnabled": true +} \ No newline at end of file diff --git a/filecloudapi/fcserver.py b/filecloudapi/fcserver.py index 9d0498c..e04dc8e 100644 --- a/filecloudapi/fcserver.py +++ b/filecloudapi/fcserver.py @@ -1,5 +1,6 @@ # Copyright (c) 2024 FileCloud. All Rights Reserved. import datetime +import threading import logging import pathlib import re @@ -49,6 +50,35 @@ def str_to_bool(value): log = logging.getLogger(__name__) +class Progress: + """ + Way to track progress of uploads/downloads. + + Either use this object in another thread or + override update() to get progress updates. + """ + + def __init__(self) -> None: + self._completed_bytes = 0 + self._total_bytes = 0 + self._lock = threading.Lock() + + """ + Progress callback of uploads/downloads + """ + def update(self, completed_bytes: int, total_bytes: int, chunk_complete: bool) -> None: + with self._lock: + self._completed_bytes = completed_bytes + self._total_bytes = total_bytes + + def completed_bytes(self) -> int: + with self._lock: + return self._completed_bytes + + def total_bytes(self) -> int: + with self._lock: + return self._total_bytes + class FCServer: """ @@ -496,7 +526,8 @@ def waitforfileremoval(self, path: str, maxwaits: float = 30): raise TimeoutError(f"File {path} not removed after {maxwaits} seconds") def downloadfile_no_retry( - self, path: str, dstPath: Union[pathlib.Path, str], redirect: bool = True + self, path: str, dstPath: Union[pathlib.Path, str], redirect: bool = True, + progress: Optional[Progress] = None ) -> None: """ Download file at 'path' to local 'dstPath' @@ -511,23 +542,29 @@ def downloadfile_no_retry( stream=True, ) as resp: resp.raise_for_status() + content_length = int(resp.headers.get("Content-Length", "-1")) + completed_bytes = 0 with open(dstPath, "wb") as dstF: for chunk in resp.iter_content(128 * 1024): + completed_bytes += len(chunk) dstF.write(chunk) + if progress is not None: + progress.update(completed_bytes, content_length, False) def downloadfile( - self, path: str, dstPath: Union[pathlib.Path, str], redirect: bool = True + self, path: str, dstPath: Union[pathlib.Path, str], redirect: bool = True, + progress: Optional[Progress] = None ) -> None: """ Download file at 'path' to local 'dstPath'. Retries. """ if self.retries is None: - return self.downloadfile_no_retry(path, dstPath, redirect) + return self.downloadfile_no_retry(path, dstPath, redirect, progress) retries = self.retries while True: try: - self.downloadfile_no_retry(path, dstPath, redirect) + self.downloadfile_no_retry(path, dstPath, redirect, progress) return except: retries = retries.increment() @@ -568,22 +605,24 @@ def upload_bytes( data: bytes, serverpath: str, datemodified: datetime.datetime = datetime.datetime.now(), + progress: Optional[Progress] = None ) -> None: """ Upload bytes 'data' to server at 'serverpath'. """ - self.upload(BufferedReader(BytesIO(data)), serverpath, datemodified) # type: ignore + self.upload(BufferedReader(BytesIO(data)), serverpath, datemodified, progress=progress) # type: ignore def upload_str( self, data: str, serverpath: str, datemodified: datetime.datetime = datetime.datetime.now(), + progress: Optional[Progress] = None ) -> None: """ Upload str 'data' UTF-8 encoded to server at 'serverpath'. """ - self.upload_bytes(data.encode("utf-8"), serverpath, datemodified) + self.upload_bytes(data.encode("utf-8"), serverpath, datemodified, progress=progress) def upload_file( self, @@ -591,6 +630,7 @@ def upload_file( serverpath: str, datemodified: datetime.datetime = datetime.datetime.now(), adminproxyuserid: Optional[str] = None, + progress: Optional[Progress] = None ) -> None: """ Upload file at 'localpath' to server at 'serverpath'. @@ -601,6 +641,7 @@ def upload_file( serverpath, datemodified, adminproxyuserid=adminproxyuserid, + progress=progress ) def _serverdatetime(self, dt: datetime.datetime): @@ -619,6 +660,7 @@ def upload( serverpath: str, datemodified: datetime.datetime, adminproxyuserid: Optional[str] = None, + progress: Optional[Progress] = None, ) -> None: """ Upload seekable stream at uploadf to server at 'serverpath' @@ -681,6 +723,8 @@ def read(self, size=-1): size = min(size, max_read) data = super().read(size) self.pos += len(data) + if progress is not None: + progress.update(self.pos, data_size, False) return data def __len__(self) -> int: @@ -811,6 +855,9 @@ def close(self): pos += curr_slice_size + if progress is not None: + progress.update(pos, data_size, True) + def share(self, path: str, adminproxyuserid: str = "") -> FCShare: """ Share 'path' diff --git a/pyproject.toml b/pyproject.toml index 2ffdf4f..4d4801e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [tool.poetry] name = "filecloudapi-python" -version = "0.1.2" +version = "0.2" description = "A Python library to connect to a Filecloud server" packages = [{ include = "filecloudapi" }] From 9cbfe9c9157699f9ca0b35f45318ee5bffe9ea5d Mon Sep 17 00:00:00 2001 From: Martin Raiber Date: Tue, 21 Jan 2025 11:20:13 +0100 Subject: [PATCH 2/2] Update formatting --- filecloudapi/fcserver.py | 38 +++++++++++++++++++++++++------------- poetry.lock | 2 +- pyproject.toml | 2 +- 3 files changed, 27 insertions(+), 15 deletions(-) diff --git a/filecloudapi/fcserver.py b/filecloudapi/fcserver.py index e04dc8e..4141454 100644 --- a/filecloudapi/fcserver.py +++ b/filecloudapi/fcserver.py @@ -1,9 +1,9 @@ # Copyright (c) 2024 FileCloud. All Rights Reserved. import datetime -import threading import logging import pathlib import re +import threading import time import xml.etree.ElementTree as ET from io import SEEK_CUR, SEEK_END, SEEK_SET, BufferedReader, BytesIO @@ -50,6 +50,7 @@ def str_to_bool(value): log = logging.getLogger(__name__) + class Progress: """ Way to track progress of uploads/downloads. @@ -66,7 +67,10 @@ def __init__(self) -> None: """ Progress callback of uploads/downloads """ - def update(self, completed_bytes: int, total_bytes: int, chunk_complete: bool) -> None: + + def update( + self, completed_bytes: int, total_bytes: int, chunk_complete: bool + ) -> None: with self._lock: self._completed_bytes = completed_bytes self._total_bytes = total_bytes @@ -74,7 +78,7 @@ def update(self, completed_bytes: int, total_bytes: int, chunk_complete: bool) - def completed_bytes(self) -> int: with self._lock: return self._completed_bytes - + def total_bytes(self) -> int: with self._lock: return self._total_bytes @@ -526,8 +530,11 @@ def waitforfileremoval(self, path: str, maxwaits: float = 30): raise TimeoutError(f"File {path} not removed after {maxwaits} seconds") def downloadfile_no_retry( - self, path: str, dstPath: Union[pathlib.Path, str], redirect: bool = True, - progress: Optional[Progress] = None + self, + path: str, + dstPath: Union[pathlib.Path, str], + redirect: bool = True, + progress: Optional[Progress] = None, ) -> None: """ Download file at 'path' to local 'dstPath' @@ -548,12 +555,15 @@ def downloadfile_no_retry( for chunk in resp.iter_content(128 * 1024): completed_bytes += len(chunk) dstF.write(chunk) - if progress is not None: + if progress is not None: progress.update(completed_bytes, content_length, False) def downloadfile( - self, path: str, dstPath: Union[pathlib.Path, str], redirect: bool = True, - progress: Optional[Progress] = None + self, + path: str, + dstPath: Union[pathlib.Path, str], + redirect: bool = True, + progress: Optional[Progress] = None, ) -> None: """ Download file at 'path' to local 'dstPath'. Retries. @@ -605,7 +615,7 @@ def upload_bytes( data: bytes, serverpath: str, datemodified: datetime.datetime = datetime.datetime.now(), - progress: Optional[Progress] = None + progress: Optional[Progress] = None, ) -> None: """ Upload bytes 'data' to server at 'serverpath'. @@ -617,12 +627,14 @@ def upload_str( data: str, serverpath: str, datemodified: datetime.datetime = datetime.datetime.now(), - progress: Optional[Progress] = None + progress: Optional[Progress] = None, ) -> None: """ Upload str 'data' UTF-8 encoded to server at 'serverpath'. """ - self.upload_bytes(data.encode("utf-8"), serverpath, datemodified, progress=progress) + self.upload_bytes( + data.encode("utf-8"), serverpath, datemodified, progress=progress + ) def upload_file( self, @@ -630,7 +642,7 @@ def upload_file( serverpath: str, datemodified: datetime.datetime = datetime.datetime.now(), adminproxyuserid: Optional[str] = None, - progress: Optional[Progress] = None + progress: Optional[Progress] = None, ) -> None: """ Upload file at 'localpath' to server at 'serverpath'. @@ -641,7 +653,7 @@ def upload_file( serverpath, datemodified, adminproxyuserid=adminproxyuserid, - progress=progress + progress=progress, ) def _serverdatetime(self, dt: datetime.datetime): diff --git a/poetry.lock b/poetry.lock index fa5892a..f820803 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1656,4 +1656,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.0" python-versions = "^3.11" -content-hash = "c7e8802d3603111c06a6f361a2b19f6b801095e303d54e5bc9b1e99977eb0af8" +content-hash = "edceff6102d294e792a10b0b244352d089f3bf260e430bff9a41c29cb6474065" diff --git a/pyproject.toml b/pyproject.toml index 4d4801e..dda49a4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,7 +33,7 @@ python = "^3.11" click = "*" requests = "*" -[tool.poetry.dev-dependencies] +[tool.poetry.group.dev.dependencies] # Formatters black = "^22.1"