From 6984e886b643950bdd29f56f9c25154ebecbc583 Mon Sep 17 00:00:00 2001 From: Hasier Date: Fri, 2 Feb 2024 15:29:40 +0000 Subject: [PATCH 1/6] Incrementally build iter actions list --- tenacity/__init__.py | 99 +++++++++++++++++++++++++++++++++----------- 1 file changed, 75 insertions(+), 24 deletions(-) diff --git a/tenacity/__init__.py b/tenacity/__init__.py index bd60556b..64ecaf22 100644 --- a/tenacity/__init__.py +++ b/tenacity/__init__.py @@ -279,6 +279,14 @@ def statistics(self) -> t.Dict[str, t.Any]: self._local.statistics = t.cast(t.Dict[str, t.Any], {}) return self._local.statistics + @property + def iter_state(self) -> t.Dict[str, t.Any]: + try: + return self._local.iter_state # type: ignore[no-any-return] + except AttributeError: + self._local.iter_state = t.cast(t.Dict[str, t.Any], {}) + return self._local.iter_state + def wraps(self, f: WrappedFn) -> WrappedFn: """Wrap a function for retrying. @@ -303,20 +311,13 @@ def begin(self) -> None: self.statistics["attempt_number"] = 1 self.statistics["idle_for"] = 0 - def iter(self, retry_state: "RetryCallState") -> t.Union[DoAttempt, DoSleep, t.Any]: # noqa - fut = retry_state.outcome - if fut is None: - if self.before is not None: - self.before(retry_state) - return DoAttempt() - - is_explicit_retry = fut.failed and isinstance(fut.exception(), TryAgain) - if not (is_explicit_retry or self.retry(retry_state)): - return fut.result() + def _add_action_func(self, fn: t.Callable[..., t.Any]) -> None: + self.iter_state["actions"].append(fn) - if self.after is not None: - self.after(retry_state) + def _run_retry(self, retry_state: "RetryCallState") -> None: + self.iter_state["retry_run_result"] = self.retry(retry_state) + def _run_wait(self, retry_state: "RetryCallState") -> None: if self.wait: sleep = self.wait(retry_state) else: @@ -324,24 +325,74 @@ def iter(self, retry_state: "RetryCallState") -> t.Union[DoAttempt, DoSleep, t.A retry_state.upcoming_sleep = sleep + def _run_stop(self, retry_state: "RetryCallState") -> None: self.statistics["delay_since_first_attempt"] = retry_state.seconds_since_start - if self.stop(retry_state): + self.iter_state["stop_run_result"] = self.stop(retry_state) + + def iter(self, retry_state: "RetryCallState") -> t.Union[DoAttempt, DoSleep, t.Any]: # noqa + self._begin_iter(retry_state) + result = None + for action in self.iter_state["actions"]: + result = action(retry_state) + return result + + def _begin_iter(self, retry_state: "RetryCallState") -> None: # noqa + self.iter_state.clear() + self.iter_state["actions"] = [] + + fut = retry_state.outcome + if fut is None: + if self.before is not None: + self._add_action_func(self.before) + self._add_action_func(lambda rs: DoAttempt()) + return + + self.iter_state["is_explicit_retry"] = fut.failed and isinstance(fut.exception(), TryAgain) + if not self.iter_state["is_explicit_retry"]: + self._add_action_func(self._run_retry) + self._add_action_func(self._post_retry_check_actions) + + def _post_retry_check_actions(self, retry_state: "RetryCallState") -> None: + if not (self.iter_state["is_explicit_retry"] or self.iter_state.get("retry_run_result")): + self._add_action_func(lambda rs: rs.outcome.result()) + return + + if self.after is not None: + self._add_action_func(self.after) + + self._add_action_func(self._run_wait) + self._add_action_func(self._run_stop) + self._add_action_func(self._post_stop_check_actions) + + def _post_stop_check_actions(self, retry_state: "RetryCallState") -> None: + if self.iter_state["stop_run_result"]: if self.retry_error_callback: - return self.retry_error_callback(retry_state) - retry_exc = self.retry_error_cls(fut) - if self.reraise: - raise retry_exc.reraise() - raise retry_exc from fut.exception() + self._add_action_func(self.retry_error_callback) + return + + def exc_check(rs: "RetryCallState") -> None: + fut = t.cast(Future, rs.outcome) + retry_exc = self.retry_error_cls(fut) + if self.reraise: + raise retry_exc.reraise() + raise retry_exc from fut.exception() + + self._add_action_func(exc_check) + return + + def next_action(rs: "RetryCallState") -> None: + sleep = rs.upcoming_sleep + rs.next_action = RetryAction(sleep) + rs.idle_for += sleep + self.statistics["idle_for"] += sleep + self.statistics["attempt_number"] += 1 - retry_state.next_action = RetryAction(sleep) - retry_state.idle_for += sleep - self.statistics["idle_for"] += sleep - self.statistics["attempt_number"] += 1 + self._add_action_func(next_action) if self.before_sleep is not None: - self.before_sleep(retry_state) + self._add_action_func(self.before_sleep) - return DoSleep(sleep) + self._add_action_func(lambda rs: DoSleep(rs.upcoming_sleep)) def __iter__(self) -> t.Generator[AttemptManager, None, None]: self.begin() From 538686b75bd90f15c6c7ead7999904620fa1f910 Mon Sep 17 00:00:00 2001 From: Hasier Date: Sat, 3 Feb 2024 15:32:43 +0000 Subject: [PATCH 2/6] Add TypedDict for iter_state --- tenacity/__init__.py | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/tenacity/__init__.py b/tenacity/__init__.py index 64ecaf22..d02fb3cd 100644 --- a/tenacity/__init__.py +++ b/tenacity/__init__.py @@ -97,6 +97,14 @@ WrappedFn = t.TypeVar("WrappedFn", bound=t.Callable[..., t.Any]) +class IterState(t.TypedDict): + actions: list[t.Callable[["RetryCallState"], t.Any]] + retry_run_result: bool + delay_since_first_attempt: int + stop_run_result: bool + is_explicit_retry: bool + + class TryAgain(Exception): """Always retry the executed function when raised.""" @@ -280,11 +288,11 @@ def statistics(self) -> t.Dict[str, t.Any]: return self._local.statistics @property - def iter_state(self) -> t.Dict[str, t.Any]: + def iter_state(self) -> IterState: try: return self._local.iter_state # type: ignore[no-any-return] except AttributeError: - self._local.iter_state = t.cast(t.Dict[str, t.Any], {}) + self._local.iter_state = t.cast(IterState, {}) return self._local.iter_state def wraps(self, f: WrappedFn) -> WrappedFn: @@ -337,8 +345,15 @@ def iter(self, retry_state: "RetryCallState") -> t.Union[DoAttempt, DoSleep, t.A return result def _begin_iter(self, retry_state: "RetryCallState") -> None: # noqa - self.iter_state.clear() - self.iter_state["actions"] = [] + self.iter_state.update( + { + "actions": [], + "retry_run_result": False, + "delay_since_first_attempt": 0, + "stop_run_result": False, + "is_explicit_retry": False, + } + ) fut = retry_state.outcome if fut is None: @@ -353,7 +368,7 @@ def _begin_iter(self, retry_state: "RetryCallState") -> None: # noqa self._add_action_func(self._post_retry_check_actions) def _post_retry_check_actions(self, retry_state: "RetryCallState") -> None: - if not (self.iter_state["is_explicit_retry"] or self.iter_state.get("retry_run_result")): + if not (self.iter_state["is_explicit_retry"] or self.iter_state["retry_run_result"]): self._add_action_func(lambda rs: rs.outcome.result()) return From 5591e7d316bc558de1af9a018883bb5d81090f9e Mon Sep 17 00:00:00 2001 From: Hasier Date: Mon, 5 Feb 2024 09:31:13 +0000 Subject: [PATCH 3/6] Format with ruff --- tenacity/__init__.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tenacity/__init__.py b/tenacity/__init__.py index 527585ca..47ab4464 100644 --- a/tenacity/__init__.py +++ b/tenacity/__init__.py @@ -372,13 +372,17 @@ def _begin_iter(self, retry_state: "RetryCallState") -> None: # noqa self._add_action_func(lambda rs: DoAttempt()) return - self.iter_state["is_explicit_retry"] = fut.failed and isinstance(fut.exception(), TryAgain) + self.iter_state["is_explicit_retry"] = fut.failed and isinstance( + fut.exception(), TryAgain + ) if not self.iter_state["is_explicit_retry"]: self._add_action_func(self._run_retry) self._add_action_func(self._post_retry_check_actions) def _post_retry_check_actions(self, retry_state: "RetryCallState") -> None: - if not (self.iter_state["is_explicit_retry"] or self.iter_state["retry_run_result"]): + if not ( + self.iter_state["is_explicit_retry"] or self.iter_state["retry_run_result"] + ): self._add_action_func(lambda rs: rs.outcome.result()) return From dfd09074e62684e376264a5eb548645759797edd Mon Sep 17 00:00:00 2001 From: Hasier Date: Mon, 5 Feb 2024 14:56:43 +0000 Subject: [PATCH 4/6] Make IterState a dataclass --- tenacity/__init__.py | 55 ++++++++++++++++++++++---------------------- 1 file changed, 27 insertions(+), 28 deletions(-) diff --git a/tenacity/__init__.py b/tenacity/__init__.py index 47ab4464..e9b0df24 100644 --- a/tenacity/__init__.py +++ b/tenacity/__init__.py @@ -15,8 +15,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - - +import dataclasses import functools import sys import threading @@ -97,12 +96,22 @@ WrappedFn = t.TypeVar("WrappedFn", bound=t.Callable[..., t.Any]) -class IterState(t.TypedDict): - actions: list[t.Callable[["RetryCallState"], t.Any]] - retry_run_result: bool - delay_since_first_attempt: int - stop_run_result: bool - is_explicit_retry: bool +@dataclasses.dataclass(slots=True) +class IterState: + actions: list[t.Callable[["RetryCallState"], t.Any]] = dataclasses.field( + default_factory=list + ) + retry_run_result: bool = False + delay_since_first_attempt: int = 0 + stop_run_result: bool = False + is_explicit_retry: bool = False + + def reset(self) -> None: + self.actions = [] + self.retry_run_result = False + self.delay_since_first_attempt = 0 + self.stop_run_result = False + self.is_explicit_retry = False class TryAgain(Exception): @@ -300,7 +309,7 @@ def iter_state(self) -> IterState: try: return self._local.iter_state # type: ignore[no-any-return] except AttributeError: - self._local.iter_state = t.cast(IterState, {}) + self._local.iter_state = IterState() return self._local.iter_state def wraps(self, f: WrappedFn) -> WrappedFn: @@ -330,10 +339,10 @@ def begin(self) -> None: self.statistics["idle_for"] = 0 def _add_action_func(self, fn: t.Callable[..., t.Any]) -> None: - self.iter_state["actions"].append(fn) + self.iter_state.actions.append(fn) def _run_retry(self, retry_state: "RetryCallState") -> None: - self.iter_state["retry_run_result"] = self.retry(retry_state) + self.iter_state.retry_run_result = self.retry(retry_state) def _run_wait(self, retry_state: "RetryCallState") -> None: if self.wait: @@ -345,25 +354,17 @@ def _run_wait(self, retry_state: "RetryCallState") -> None: def _run_stop(self, retry_state: "RetryCallState") -> None: self.statistics["delay_since_first_attempt"] = retry_state.seconds_since_start - self.iter_state["stop_run_result"] = self.stop(retry_state) + self.iter_state.stop_run_result = self.stop(retry_state) def iter(self, retry_state: "RetryCallState") -> t.Union[DoAttempt, DoSleep, t.Any]: # noqa self._begin_iter(retry_state) result = None - for action in self.iter_state["actions"]: + for action in self.iter_state.actions: result = action(retry_state) return result def _begin_iter(self, retry_state: "RetryCallState") -> None: # noqa - self.iter_state.update( - { - "actions": [], - "retry_run_result": False, - "delay_since_first_attempt": 0, - "stop_run_result": False, - "is_explicit_retry": False, - } - ) + self.iter_state.reset() fut = retry_state.outcome if fut is None: @@ -372,17 +373,15 @@ def _begin_iter(self, retry_state: "RetryCallState") -> None: # noqa self._add_action_func(lambda rs: DoAttempt()) return - self.iter_state["is_explicit_retry"] = fut.failed and isinstance( + self.iter_state.is_explicit_retry = fut.failed and isinstance( fut.exception(), TryAgain ) - if not self.iter_state["is_explicit_retry"]: + if not self.iter_state.is_explicit_retry: self._add_action_func(self._run_retry) self._add_action_func(self._post_retry_check_actions) def _post_retry_check_actions(self, retry_state: "RetryCallState") -> None: - if not ( - self.iter_state["is_explicit_retry"] or self.iter_state["retry_run_result"] - ): + if not (self.iter_state.is_explicit_retry or self.iter_state.retry_run_result): self._add_action_func(lambda rs: rs.outcome.result()) return @@ -394,7 +393,7 @@ def _post_retry_check_actions(self, retry_state: "RetryCallState") -> None: self._add_action_func(self._post_stop_check_actions) def _post_stop_check_actions(self, retry_state: "RetryCallState") -> None: - if self.iter_state["stop_run_result"]: + if self.iter_state.stop_run_result: if self.retry_error_callback: self._add_action_func(self.retry_error_callback) return From 175059eb4cff7d4cae23b7afa6efc47402118e40 Mon Sep 17 00:00:00 2001 From: Hasier Date: Mon, 5 Feb 2024 14:58:59 +0000 Subject: [PATCH 5/6] Fix typing --- tenacity/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tenacity/__init__.py b/tenacity/__init__.py index e9b0df24..47527eb9 100644 --- a/tenacity/__init__.py +++ b/tenacity/__init__.py @@ -98,7 +98,7 @@ @dataclasses.dataclass(slots=True) class IterState: - actions: list[t.Callable[["RetryCallState"], t.Any]] = dataclasses.field( + actions: t.List[t.Callable[["RetryCallState"], t.Any]] = dataclasses.field( default_factory=list ) retry_run_result: bool = False From 5b11340f997b8db6e48728006bd0dc6257c446fe Mon Sep 17 00:00:00 2001 From: Hasier Date: Mon, 5 Feb 2024 15:26:40 +0000 Subject: [PATCH 6/6] Conditionally add slots to dataclass --- tenacity/__init__.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tenacity/__init__.py b/tenacity/__init__.py index 47527eb9..dc8dfbcc 100644 --- a/tenacity/__init__.py +++ b/tenacity/__init__.py @@ -96,7 +96,12 @@ WrappedFn = t.TypeVar("WrappedFn", bound=t.Callable[..., t.Any]) -@dataclasses.dataclass(slots=True) +dataclass_kwargs = {} +if sys.version_info >= (3, 10): + dataclass_kwargs.update({"slots": True}) + + +@dataclasses.dataclass(**dataclass_kwargs) class IterState: actions: t.List[t.Callable[["RetryCallState"], t.Any]] = dataclasses.field( default_factory=list