From f290193ea15b68d2f7151ac8cadb194e0a5febae Mon Sep 17 00:00:00 2001 From: "jsoros (aider)" Date: Mon, 19 May 2025 20:50:17 -0400 Subject: [PATCH 01/10] fix: limit Gmail API batch requests to 100 items --- src/processor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/processor.py b/src/processor.py index e6c384c..eccc20b 100644 --- a/src/processor.py +++ b/src/processor.py @@ -108,7 +108,7 @@ def get_metadata(self, messages): max=len(messages), ) - for messages_batch in helpers.chunks(messages, 250): + for messages_batch in helpers.chunks(messages, 100): # for messages_batch in [messages[0:1000]]: batch = self.service.new_batch_http_request() From d9829a1f9952cb9cf8006d2884590ce66aa9b62e Mon Sep 17 00:00:00 2001 From: "jsoros (aider)" Date: Mon, 19 May 2025 20:54:16 -0400 Subject: [PATCH 02/10] fix: handle missing labelIds key in message metadata processing --- src/processor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/processor.py b/src/processor.py index eccc20b..0de8873 100644 --- a/src/processor.py +++ b/src/processor.py @@ -76,7 +76,7 @@ def process_message(self, request_id, response, exception): self.messagesQueue.append( { "id": response["id"], - "labels": response["labelIds"], + "labels": response.get("labelIds", []), "fields": {"from": _from, "date": _date}, } ) From ab48dc4e3d38a085a614d1b307e357e91dc0fa5f Mon Sep 17 00:00:00 2001 From: "jsoros (aider)" Date: Mon, 19 May 2025 21:03:52 -0400 Subject: [PATCH 03/10] feat: implement caching for processed messages and metadata --- src/processor.py | 40 +++++++++++++++++++++++++++++++--------- 1 file changed, 31 insertions(+), 9 deletions(-) diff --git a/src/processor.py b/src/processor.py index 0de8873..79a1224 100644 --- a/src/processor.py +++ b/src/processor.py @@ -1,6 +1,7 @@ import collections import os.path import pickle +import time from progress.counter import Counter from progress.bar import IncrementalBar @@ -8,6 +9,7 @@ from src.service import Service _progressPadding = 29 +_CACHE_DIR = "cache" class Processor: @@ -17,16 +19,24 @@ def __init__(self): self.user_id = "me" self.messagesQueue = collections.deque() self.failedMessagesQueue = collections.deque() + + # Create cache directory if it doesn't exist + if not os.path.exists(_CACHE_DIR): + os.makedirs(_CACHE_DIR) def get_messages(self): # Get all messages of user # Output format: # [{'id': '13c...7', 'threadId': '13c...7'}, ...] - - # if os.path.exists("messages.pickle"): - # with open("messages.pickle", "rb") as token: - # messages = pickle.load(token) - # return messages + + cache_file = os.path.join(_CACHE_DIR, "messages.pickle") + + # Check if cache exists and is not older than 24 hours + if os.path.exists(cache_file) and (time.time() - os.path.getmtime(cache_file) < 86400): + print(f"{helpers.loader_icn} Loading messages from cache") + with open(cache_file, "rb") as token: + messages = pickle.load(token) + return messages # includeSpamTrash # labelIds @@ -56,6 +66,10 @@ def get_messages(self): progress.next() progress.finish() + + # Cache the messages + with open(cache_file, "wb") as token: + pickle.dump(messages, token) return messages @@ -96,10 +110,14 @@ def get_metadata(self, messages): # ] # } - # if os.path.exists("success.pickle"): - # with open("success.pickle", "rb") as token: - # self.messagesQueue = pickle.load(token) - # return + cache_file = os.path.join(_CACHE_DIR, "metadata.pickle") + + # Check if cache exists and is not older than 24 hours + if os.path.exists(cache_file) and (time.time() - os.path.getmtime(cache_file) < 86400): + print(f"{helpers.loader_icn} Loading message metadata from cache") + with open(cache_file, "rb") as token: + self.messagesQueue = pickle.load(token) + return progress = IncrementalBar( f"{helpers.loader_icn} Fetching messages meta data ".ljust( @@ -123,3 +141,7 @@ def get_metadata(self, messages): progress.next(len(messages_batch)) progress.finish() + + # Cache the metadata + with open(cache_file, "wb") as token: + pickle.dump(self.messagesQueue, token) From 8f0a6c4d6370e516c056773c20baf1f7fb4b3043 Mon Sep 17 00:00:00 2001 From: "jsoros (aider)" Date: Mon, 19 May 2025 21:08:56 -0400 Subject: [PATCH 04/10] fix: Add fallback output and error handling for chart visualizations --- src/metrics.py | 45 +++++++++++++++++++++++++++++++++------------ 1 file changed, 33 insertions(+), 12 deletions(-) diff --git a/src/metrics.py b/src/metrics.py index 4217883..32cbaac 100644 --- a/src/metrics.py +++ b/src/metrics.py @@ -47,17 +47,26 @@ def _analyze_senders(self, event): event.set() print(f"\n\n{helpers.h1_icn} Senders (top {self.resultsLimit})\n") - args = { - "stacked": False, - "width": 55, - "no_labels": False, - "format": "{:<,d}", - "suffix": "", - "vertical": False, - "different_scale": False, - } - - chart(colors=[94], data=data_count, args=args, labels=data_keys) + + # Print a simple table if chart function fails + for i in range(len(data_keys)): + if i < len(data_count): + print(f"{data_keys[i]}: {data_count[i][0]:,}") + + try: + args = { + "stacked": False, + "width": 55, + "no_labels": False, + "format": "{:<,d}", + "suffix": "", + "vertical": False, + "different_scale": False, + } + + chart(colors=[94], data=data_count, args=args, labels=data_keys) + except Exception as e: + print(f"Note: Could not display chart. Using simple output instead.") def _analyze_count(self, event): # Average emails per day @@ -167,7 +176,16 @@ def _analyze_date(self, event): args = {"color": False, "custom_tick": False, "start_dt": f"{year}-01-01"} print(f"\n{helpers.h2_icn} Year {year} ({_sum:,} emails)\n") - calendar_heatmap(data=data_count, args=args, labels=data_keys) + + # Print simple date data if visualization fails + try: + calendar_heatmap(data=data_count, args=args, labels=data_keys) + except Exception as e: + print(f"Note: Could not display calendar heatmap. Showing simple output instead.") + # Show top 10 dates with most emails + sorted_dates = sorted(zip(data_keys, [c[0] for c in data_count]), key=lambda x: x[1], reverse=True) + for date, count in sorted_dates[:10]: + print(f"{date}: {count:,} emails") def analyse(self): """ @@ -226,6 +244,9 @@ def analyse(self): time.sleep(0.1) progress.finish() + + # Print completion message + print("\nAnalysis complete!") def start(self): messages = self.processor.get_messages() From 0e087ee2cf70f2bb9415b27d54ba7e731e4113b7 Mon Sep 17 00:00:00 2001 From: "jsoros (aider)" Date: Mon, 19 May 2025 21:22:42 -0400 Subject: [PATCH 05/10] fix: Add robust error handling for chart rendering and package dependencies --- src/metrics.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/src/metrics.py b/src/metrics.py index 32cbaac..11441b3 100644 --- a/src/metrics.py +++ b/src/metrics.py @@ -1,7 +1,13 @@ import time +import sys from progress.spinner import Spinner -from ascii_graph import Pyasciigraph -from termgraph.termgraph import chart, calendar_heatmap +try: + from ascii_graph import Pyasciigraph + from termgraph.termgraph import chart, calendar_heatmap +except ImportError: + print("Error: Required visualization packages not found.") + print("Please run: pip install ascii-graph termgraph") + sys.exit(1) import agate import warnings import concurrent.futures @@ -48,11 +54,6 @@ def _analyze_senders(self, event): print(f"\n\n{helpers.h1_icn} Senders (top {self.resultsLimit})\n") - # Print a simple table if chart function fails - for i in range(len(data_keys)): - if i < len(data_count): - print(f"{data_keys[i]}: {data_count[i][0]:,}") - try: args = { "stacked": False, @@ -67,6 +68,10 @@ def _analyze_senders(self, event): chart(colors=[94], data=data_count, args=args, labels=data_keys) except Exception as e: print(f"Note: Could not display chart. Using simple output instead.") + # Print a simple table if chart function fails + for i in range(len(data_keys)): + if i < len(data_count): + print(f"{data_keys[i]}: {data_count[i][0]:,}") def _analyze_count(self, event): # Average emails per day @@ -177,7 +182,6 @@ def _analyze_date(self, event): print(f"\n{helpers.h2_icn} Year {year} ({_sum:,} emails)\n") - # Print simple date data if visualization fails try: calendar_heatmap(data=data_count, args=args, labels=data_keys) except Exception as e: From 19d1ba5679b59e0b9e6fe67f98db9705fc386897 Mon Sep 17 00:00:00 2001 From: jsoros Date: Mon, 19 May 2025 21:25:16 -0400 Subject: [PATCH 06/10] chore: update Python version requirement to 3.8 --- Pipfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Pipfile b/Pipfile index f1a8c50..1404cd8 100644 --- a/Pipfile +++ b/Pipfile @@ -19,4 +19,4 @@ termcolor = "*" argparse = "*" [requires] -python_version = "3.7" +python_version = "3.8" From bb5ecc054d33c268a34f7d8796843923b8c51319 Mon Sep 17 00:00:00 2001 From: "jsoros (aider)" Date: Mon, 19 May 2025 21:25:18 -0400 Subject: [PATCH 07/10] chore: update chart rendering dependencies and error handling --- Pipfile | 1 + src/helpers.py | 28 ++++++++++++++++++++-------- src/metrics.py | 6 +++--- 3 files changed, 24 insertions(+), 11 deletions(-) diff --git a/Pipfile b/Pipfile index 1404cd8..0c58e9e 100644 --- a/Pipfile +++ b/Pipfile @@ -17,6 +17,7 @@ ascii-graph = "*" agate = "*" termcolor = "*" argparse = "*" +python-dateutil = "*" [requires] python_version = "3.8" diff --git a/src/helpers.py b/src/helpers.py index a0221a5..290bbe9 100644 --- a/src/helpers.py +++ b/src/helpers.py @@ -14,19 +14,31 @@ def remove_dup_timezone(date_str): def convert_date(date_str): # Dates comes multiple formats, this function tries to guess it + from dateutil import parser + + if not date_str: + return datetime.now() + clean_date = remove_dup_timezone(date_str) - _val = None - try: - _val = datetime.strptime(clean_date, "%d %b %Y %H:%M:%S %z") - except ValueError: + # Try using dateutil parser which handles many date formats + return parser.parse(clean_date) + except: + # Fall back to original method + _val = None try: - _val = datetime.strptime(clean_date, "%d %b %Y %H:%M:%S %Z") + _val = datetime.strptime(clean_date, "%d %b %Y %H:%M:%S %z") except ValueError: - _val = datetime.strptime(clean_date, "%d %b %Y %H:%M:%S") - - return _val + try: + _val = datetime.strptime(clean_date, "%d %b %Y %H:%M:%S %Z") + except ValueError: + try: + _val = datetime.strptime(clean_date, "%d %b %Y %H:%M:%S") + except ValueError: + # If all parsing fails, return current date + return datetime.now() + return _val def reduce_to_date(date_str): diff --git a/src/metrics.py b/src/metrics.py index 11441b3..5ea75a9 100644 --- a/src/metrics.py +++ b/src/metrics.py @@ -6,7 +6,7 @@ from termgraph.termgraph import chart, calendar_heatmap except ImportError: print("Error: Required visualization packages not found.") - print("Please run: pip install ascii-graph termgraph") + print("Please run: pipenv install ascii-graph termgraph") sys.exit(1) import agate import warnings @@ -67,7 +67,7 @@ def _analyze_senders(self, event): chart(colors=[94], data=data_count, args=args, labels=data_keys) except Exception as e: - print(f"Note: Could not display chart. Using simple output instead.") + print(f"Note: Could not display chart. Using simple output instead. Error: {str(e)}") # Print a simple table if chart function fails for i in range(len(data_keys)): if i < len(data_count): @@ -185,7 +185,7 @@ def _analyze_date(self, event): try: calendar_heatmap(data=data_count, args=args, labels=data_keys) except Exception as e: - print(f"Note: Could not display calendar heatmap. Showing simple output instead.") + print(f"Note: Could not display calendar heatmap. Showing simple output instead. Error: {str(e)}") # Show top 10 dates with most emails sorted_dates = sorted(zip(data_keys, [c[0] for c in data_count]), key=lambda x: x[1], reverse=True) for date, count in sorted_dates[:10]: From 63483631ddc769b9d183e6190dcacb34be8be5b9 Mon Sep 17 00:00:00 2001 From: "jsoros (aider)" Date: Mon, 19 May 2025 22:48:38 -0400 Subject: [PATCH 08/10] feat: add feature to find senders inactive for more than X days --- analyzer.py | 3 ++ src/metrics.py | 91 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 94 insertions(+) diff --git a/analyzer.py b/analyzer.py index 8b59cfb..ceed6a9 100644 --- a/analyzer.py +++ b/analyzer.py @@ -21,6 +21,9 @@ def init_args(): parser.add_argument( "--version", action="store_true", help="Display version and exit" ) + parser.add_argument( + "--inactive", type=int, default=0, help="Show senders inactive for more than X days" + ) args = vars(parser.parse_args()) diff --git a/src/metrics.py b/src/metrics.py index 5ea75a9..2f6be7e 100644 --- a/src/metrics.py +++ b/src/metrics.py @@ -1,5 +1,6 @@ import time import sys +from datetime import datetime from progress.spinner import Spinner try: from ascii_graph import Pyasciigraph @@ -26,6 +27,7 @@ def __init__(self, args): self.processor = Processor() self.user_id = args["user"] self.resultsLimit = args["top"] + self.inactive_days = args["inactive"] self.table = None def _load_table(self, event): @@ -129,6 +131,81 @@ def _analyze_count(self, event): print(f"\n\n{helpers.h1_icn} Stats\n") print(termtables.to_string(metrics)) + def _analyze_inactive_senders(self, event): + """Analyze senders who haven't sent emails in X days""" + if self.inactive_days <= 0: + event.set() + return + + # Get current date for comparison + current_date = datetime.now() + + # Process the data to get the last email date for each sender + sender_last_dates = {} + + # Filter out rows with no sender or date + filtered_table = self.table.where( + lambda row: row["fields/from"] is not None and row["fields/date"] is not None + ) + + # Convert dates to datetime objects for comparison + date_table = filtered_table.compute([ + ( + "datetime", + agate.Formula( + agate.DateTime(datetime_format="%Y-%m-%d %H:%M:%S"), + lambda row: helpers.reduce_to_datetime(row["fields/date"]), + ), + ) + ]) + + # Group by sender and find the most recent email + for row in date_table.rows: + sender = row["fields/from"] + date_obj = helpers.convert_date(row["fields/date"]) + + if sender not in sender_last_dates or date_obj > sender_last_dates[sender]["date"]: + sender_last_dates[sender] = { + "date": date_obj, + "date_str": row["fields/date"] + } + + # Find senders inactive for more than X days + inactive_senders = [] + for sender, data in sender_last_dates.items(): + days_since = (current_date - data["date"]).days + if days_since > self.inactive_days: + inactive_senders.append({ + "sender": sender, + "last_email_date": data["date_str"], + "days_since": days_since + }) + + # Sort by days_since in descending order + inactive_senders.sort(key=lambda x: x["days_since"], reverse=True) + + # Limit to top results + inactive_senders = inactive_senders[:self.resultsLimit] + + event.set() + + if inactive_senders: + print(f"\n\n{helpers.h1_icn} Senders inactive for more than {self.inactive_days} days\n") + + # Prepare data for table display + table_data = [] + for sender in inactive_senders: + table_data.append([ + sender["sender"], + sender["last_email_date"], + f"{sender['days_since']} days" + ]) + + headers = ["Sender", "Last Email Date", "Days Since"] + print(termtables.to_string(table_data, header=headers)) + else: + print(f"\n\n{helpers.h1_icn} No senders inactive for more than {self.inactive_days} days found") + def _analyze_date(self, event): table = self.table.where(lambda row: row["fields/date"] is not None).compute( [ @@ -249,6 +326,20 @@ def analyse(self): progress.finish() + # Only run inactive senders analysis if the threshold is set + if self.inactive_days > 0: + progress = Spinner(f"{helpers.loader_icn} Analysing inactive senders ") + + event = Event() + + future = executor.submit(self._analyze_inactive_senders, event) + + while not event.isSet() and future.running(): + progress.next() + time.sleep(0.1) + + progress.finish() + # Print completion message print("\nAnalysis complete!") From 54ae47a69e5423168d82b8ab47a69981e14d0d48 Mon Sep 17 00:00:00 2001 From: jsoros Date: Tue, 23 Dec 2025 19:38:22 -0500 Subject: [PATCH 09/10] add .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 122bfa9..fac7e8a 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,4 @@ __pycache__/ venv/ target/ .DS_Store +.aider* From 4cca0e1dfc5b7b1867efd344ce0075224da4f446 Mon Sep 17 00:00:00 2001 From: jsoros Date: Tue, 23 Dec 2025 20:14:14 -0500 Subject: [PATCH 10/10] feat: add query-based fetch and export modes --- Pipfile.lock | 366 +++++++++++++++++++++++++++++++++++------------ README.md | 38 ++++- analyzer.py | 44 ++++++ src/metrics.py | 30 +++- src/processor.py | 310 +++++++++++++++++++++++++++++++++++---- src/service.py | 10 +- 6 files changed, 666 insertions(+), 132 deletions(-) diff --git a/Pipfile.lock b/Pipfile.lock index 0e09d1d..ec29311 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,11 +1,11 @@ { "_meta": { "hash": { - "sha256": "2b98faaf1aeaff58bd8263fa6fcd4b76cc5985e48f53a868b5849f7af1e72e04" + "sha256": "55cd924a01772d171cf1caaeeb3e0c6cc75502976fbe8bf2c4cca2bd7680f6bc" }, "pipfile-spec": 6, "requires": { - "python_version": "3.7" + "python_version": "3.8" }, "sources": [ { @@ -18,11 +18,11 @@ "default": { "agate": { "hashes": [ - "sha256:48d6f80b35611c1ba25a642cbc5b90fcbdeeb2a54711c4a8d062ee2809334d1c", - "sha256:c93aaa500b439d71e4a5cf088d0006d2ce2c76f1950960c8843114e5f361dfd3" + "sha256:0726e3883e157da2b6bb58cc40bd9b810a5df4a11dea3bcd3de0f422a3819294", + "sha256:24bc3d3cbd165aa3ab0ef9e798dd4c53ad703012d450fe89b9c26b239505c445" ], "index": "pypi", - "version": "==1.6.1" + "version": "==1.13.0" }, "argparse": { "hashes": [ @@ -34,147 +34,311 @@ }, "ascii-graph": { "hashes": [ - "sha256:c1844fe309cd221f35f19efc58c5c7157941e35172d486d7c824ba5ad1d05f71" + "sha256:32ec5e8af8ac1b285cb0c3d1715b3e44d2152bff71f27cde60c3b5d23f98e0a4", + "sha256:82a128971cf189e37131239f89fdf2dac769672e2c30de187e7aa5c345e103e2" ], "index": "pypi", - "version": "==1.5.1" + "version": "==1.5.2" }, "babel": { "hashes": [ - "sha256:1aac2ae2d0d8ea368fa90906567f5c08463d98ade155c0c4bfedd6a0f7160e38", - "sha256:d670ea0b10f8b723672d3a6abeb87b565b244da220d76b4dba1b66269ec152d4" + "sha256:0c54cffb19f690cdcc52a3b50bcbf71e07a808d1c80d549f2459b9d2cf0afb9d", + "sha256:4d0b53093fdfb4b21c92b5213dba5a1b23885afa8383709427046b21c366e5f2" ], - "version": "==2.8.0" + "markers": "python_version >= '3.8'", + "version": "==2.17.0" }, "cachetools": { "hashes": [ - "sha256:1d057645db16ca7fe1f3bd953558897603d6f0b9c51ed9d11eb4d071ec4e2aab", - "sha256:de5d88f87781602201cde465d3afe837546663b168e8b39df67411b0bf10cefc" + "sha256:1a661caa9175d26759571b2e19580f9d6393969e5dfca11fdb1f947a23e640d4", + "sha256:d26a22bcc62eb95c3beabd9f1ee5e820d3d2704fe2967cbe350e20c8ffcd3f0a" ], - "version": "==4.1.0" + "markers": "python_version >= '3.7'", + "version": "==5.5.2" }, "certifi": { "hashes": [ - "sha256:1d987a998c75633c40847cc966fcf5904906c920a7f17ef374f5aa4282abd304", - "sha256:51fcb31174be6e6664c5f69e3e1691a2d72a1a12e90f872cbdb1567eb47b6519" - ], - "version": "==2020.4.5.1" + "sha256:0a816057ea3cdefcef70270d2c515e4506bbc954f417fa5ade2021213bb8f0c6", + "sha256:30350364dfe371162649852c63336a15c70c6510c2ad5015b21c2345311805f3" + ], + "markers": "python_version >= '3.6'", + "version": "==2025.4.26" + }, + "charset-normalizer": { + "hashes": [ + "sha256:005fa3432484527f9732ebd315da8da8001593e2cf46a3d817669f062c3d9ed4", + "sha256:046595208aae0120559a67693ecc65dd75d46f7bf687f159127046628178dc45", + "sha256:0c29de6a1a95f24b9a1aa7aefd27d2487263f00dfd55a77719b530788f75cff7", + "sha256:0c8c57f84ccfc871a48a47321cfa49ae1df56cd1d965a09abe84066f6853b9c0", + "sha256:0f5d9ed7f254402c9e7d35d2f5972c9bbea9040e99cd2861bd77dc68263277c7", + "sha256:18dd2e350387c87dabe711b86f83c9c78af772c748904d372ade190b5c7c9d4d", + "sha256:1b1bde144d98e446b056ef98e59c256e9294f6b74d7af6846bf5ffdafd687a7d", + "sha256:1c95a1e2902a8b722868587c0e1184ad5c55631de5afc0eb96bc4b0d738092c0", + "sha256:1cad5f45b3146325bb38d6855642f6fd609c3f7cad4dbaf75549bf3b904d3184", + "sha256:21b2899062867b0e1fde9b724f8aecb1af14f2778d69aacd1a5a1853a597a5db", + "sha256:24498ba8ed6c2e0b56d4acbf83f2d989720a93b41d712ebd4f4979660db4417b", + "sha256:25a23ea5c7edc53e0f29bae2c44fcb5a1aa10591aae107f2a2b2583a9c5cbc64", + "sha256:289200a18fa698949d2b39c671c2cc7a24d44096784e76614899a7ccf2574b7b", + "sha256:28a1005facc94196e1fb3e82a3d442a9d9110b8434fc1ded7a24a2983c9888d8", + "sha256:32fc0341d72e0f73f80acb0a2c94216bd704f4f0bce10aedea38f30502b271ff", + "sha256:36b31da18b8890a76ec181c3cf44326bf2c48e36d393ca1b72b3f484113ea344", + "sha256:3c21d4fca343c805a52c0c78edc01e3477f6dd1ad7c47653241cf2a206d4fc58", + "sha256:3fddb7e2c84ac87ac3a947cb4e66d143ca5863ef48e4a5ecb83bd48619e4634e", + "sha256:43e0933a0eff183ee85833f341ec567c0980dae57c464d8a508e1b2ceb336471", + "sha256:4a476b06fbcf359ad25d34a057b7219281286ae2477cc5ff5e3f70a246971148", + "sha256:4e594135de17ab3866138f496755f302b72157d115086d100c3f19370839dd3a", + "sha256:50bf98d5e563b83cc29471fa114366e6806bc06bc7a25fd59641e41445327836", + "sha256:5a9979887252a82fefd3d3ed2a8e3b937a7a809f65dcb1e068b090e165bbe99e", + "sha256:5baececa9ecba31eff645232d59845c07aa030f0c81ee70184a90d35099a0e63", + "sha256:5bf4545e3b962767e5c06fe1738f951f77d27967cb2caa64c28be7c4563e162c", + "sha256:6333b3aa5a12c26b2a4d4e7335a28f1475e0e5e17d69d55141ee3cab736f66d1", + "sha256:65c981bdbd3f57670af8b59777cbfae75364b483fa8a9f420f08094531d54a01", + "sha256:68a328e5f55ec37c57f19ebb1fdc56a248db2e3e9ad769919a58672958e8f366", + "sha256:6a0289e4589e8bdfef02a80478f1dfcb14f0ab696b5a00e1f4b8a14a307a3c58", + "sha256:6b66f92b17849b85cad91259efc341dce9c1af48e2173bf38a85c6329f1033e5", + "sha256:6c9379d65defcab82d07b2a9dfbfc2e95bc8fe0ebb1b176a3190230a3ef0e07c", + "sha256:6fc1f5b51fa4cecaa18f2bd7a003f3dd039dd615cd69a2afd6d3b19aed6775f2", + "sha256:70f7172939fdf8790425ba31915bfbe8335030f05b9913d7ae00a87d4395620a", + "sha256:721c76e84fe669be19c5791da68232ca2e05ba5185575086e384352e2c309597", + "sha256:7222ffd5e4de8e57e03ce2cef95a4c43c98fcb72ad86909abdfc2c17d227fc1b", + "sha256:75d10d37a47afee94919c4fab4c22b9bc2a8bf7d4f46f87363bcf0573f3ff4f5", + "sha256:76af085e67e56c8816c3ccf256ebd136def2ed9654525348cfa744b6802b69eb", + "sha256:770cab594ecf99ae64c236bc9ee3439c3f46be49796e265ce0cc8bc17b10294f", + "sha256:7a6ab32f7210554a96cd9e33abe3ddd86732beeafc7a28e9955cdf22ffadbab0", + "sha256:7c48ed483eb946e6c04ccbe02c6b4d1d48e51944b6db70f697e089c193404941", + "sha256:7f56930ab0abd1c45cd15be65cc741c28b1c9a34876ce8c17a2fa107810c0af0", + "sha256:8075c35cd58273fee266c58c0c9b670947c19df5fb98e7b66710e04ad4e9ff86", + "sha256:8272b73e1c5603666618805fe821edba66892e2870058c94c53147602eab29c7", + "sha256:82d8fd25b7f4675d0c47cf95b594d4e7b158aca33b76aa63d07186e13c0e0ab7", + "sha256:844da2b5728b5ce0e32d863af26f32b5ce61bc4273a9c720a9f3aa9df73b1455", + "sha256:8755483f3c00d6c9a77f490c17e6ab0c8729e39e6390328e42521ef175380ae6", + "sha256:915f3849a011c1f593ab99092f3cecfcb4d65d8feb4a64cf1bf2d22074dc0ec4", + "sha256:926ca93accd5d36ccdabd803392ddc3e03e6d4cd1cf17deff3b989ab8e9dbcf0", + "sha256:982bb1e8b4ffda883b3d0a521e23abcd6fd17418f6d2c4118d257a10199c0ce3", + "sha256:98f862da73774290f251b9df8d11161b6cf25b599a66baf087c1ffe340e9bfd1", + "sha256:9cbfacf36cb0ec2897ce0ebc5d08ca44213af24265bd56eca54bee7923c48fd6", + "sha256:a370b3e078e418187da8c3674eddb9d983ec09445c99a3a263c2011993522981", + "sha256:a955b438e62efdf7e0b7b52a64dc5c3396e2634baa62471768a64bc2adb73d5c", + "sha256:aa6af9e7d59f9c12b33ae4e9450619cf2488e2bbe9b44030905877f0b2324980", + "sha256:aa88ca0b1932e93f2d961bf3addbb2db902198dca337d88c89e1559e066e7645", + "sha256:aaeeb6a479c7667fbe1099af9617c83aaca22182d6cf8c53966491a0f1b7ffb7", + "sha256:aaf27faa992bfee0264dc1f03f4c75e9fcdda66a519db6b957a3f826e285cf12", + "sha256:b2680962a4848b3c4f155dc2ee64505a9c57186d0d56b43123b17ca3de18f0fa", + "sha256:b2d318c11350e10662026ad0eb71bb51c7812fc8590825304ae0bdd4ac283acd", + "sha256:b33de11b92e9f75a2b545d6e9b6f37e398d86c3e9e9653c4864eb7e89c5773ef", + "sha256:b3daeac64d5b371dea99714f08ffc2c208522ec6b06fbc7866a450dd446f5c0f", + "sha256:be1e352acbe3c78727a16a455126d9ff83ea2dfdcbc83148d2982305a04714c2", + "sha256:bee093bf902e1d8fc0ac143c88902c3dfc8941f7ea1d6a8dd2bcb786d33db03d", + "sha256:c72fbbe68c6f32f251bdc08b8611c7b3060612236e960ef848e0a517ddbe76c5", + "sha256:c9e36a97bee9b86ef9a1cf7bb96747eb7a15c2f22bdb5b516434b00f2a599f02", + "sha256:cddf7bd982eaa998934a91f69d182aec997c6c468898efe6679af88283b498d3", + "sha256:cf713fe9a71ef6fd5adf7a79670135081cd4431c2943864757f0fa3a65b1fafd", + "sha256:d11b54acf878eef558599658b0ffca78138c8c3655cf4f3a4a673c437e67732e", + "sha256:d41c4d287cfc69060fa91cae9683eacffad989f1a10811995fa309df656ec214", + "sha256:d524ba3f1581b35c03cb42beebab4a13e6cdad7b36246bd22541fa585a56cccd", + "sha256:daac4765328a919a805fa5e2720f3e94767abd632ae410a9062dff5412bae65a", + "sha256:db4c7bf0e07fc3b7d89ac2a5880a6a8062056801b83ff56d8464b70f65482b6c", + "sha256:dc7039885fa1baf9be153a0626e337aa7ec8bf96b0128605fb0d77788ddc1681", + "sha256:dccab8d5fa1ef9bfba0590ecf4d46df048d18ffe3eec01eeb73a42e0d9e7a8ba", + "sha256:dedb8adb91d11846ee08bec4c8236c8549ac721c245678282dcb06b221aab59f", + "sha256:e45ba65510e2647721e35323d6ef54c7974959f6081b58d4ef5d87c60c84919a", + "sha256:e53efc7c7cee4c1e70661e2e112ca46a575f90ed9ae3fef200f2a25e954f4b28", + "sha256:e635b87f01ebc977342e2697d05b56632f5f879a4f15955dfe8cef2448b51691", + "sha256:e70e990b2137b29dc5564715de1e12701815dacc1d056308e2b17e9095372a82", + "sha256:e8082b26888e2f8b36a042a58307d5b917ef2b1cacab921ad3323ef91901c71a", + "sha256:e8323a9b031aa0393768b87f04b4164a40037fb2a3c11ac06a03ffecd3618027", + "sha256:e92fca20c46e9f5e1bb485887d074918b13543b1c2a1185e69bb8d17ab6236a7", + "sha256:eb30abc20df9ab0814b5a2524f23d75dcf83cde762c161917a2b4b7b55b1e518", + "sha256:eba9904b0f38a143592d9fc0e19e2df0fa2e41c3c3745554761c5f6447eedabf", + "sha256:ef8de666d6179b009dce7bcb2ad4c4a779f113f12caf8dc77f0162c29d20490b", + "sha256:efd387a49825780ff861998cd959767800d54f8308936b21025326de4b5a42b9", + "sha256:f0aa37f3c979cf2546b73e8222bbfa3dc07a641585340179d768068e3455e544", + "sha256:f4074c5a429281bf056ddd4c5d3b740ebca4d43ffffe2ef4bf4d2d05114299da", + "sha256:f69a27e45c43520f5487f27627059b64aaf160415589230992cec34c5e18a509", + "sha256:fb707f3e15060adf5b7ada797624a6c6e0138e2a26baa089df64c68ee98e040f", + "sha256:fcbe676a55d7445b22c10967bceaaf0ee69407fbe0ece4d032b6eb8d4565982a", + "sha256:fdb20a30fe1175ecabed17cbf7812f7b804b8a315a25f24678bcdf120a90077f" + ], + "markers": "python_version >= '3.7'", + "version": "==3.4.2" }, - "chardet": { + "colorama": { "hashes": [ - "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae", - "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691" + "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", + "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6" ], - "version": "==3.0.4" + "index": "pypi", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5, 3.6'", + "version": "==0.4.6" }, - "colorama": { + "google-api-core": { "hashes": [ - "sha256:7d73d2a99753107a36ac6b455ee49046802e59d9d076ef8e47b61499fa29afff", - "sha256:e96da0d330793e2cb9485e9ddfd918d456036c7149416295932478192f4436a1" + "sha256:810a63ac95f3c441b7c0e43d344e372887f62ce9071ba972eacf32672e072de9", + "sha256:81718493daf06d96d6bc76a91c23874dbf2fac0adbbf542831b805ee6e974696" ], - "index": "pypi", - "version": "==0.4.3" + "markers": "python_version >= '3.7'", + "version": "==2.24.2" }, "google-api-python-client": { "hashes": [ - "sha256:3121d55d106ef1a2756e8074239512055bd99eb44da417b3dd680f9a1385adec", - "sha256:a8a88174f66d92aed7ebbd73744c2c319b4b1ce828e565f9ec721352d2e2fb8c" + "sha256:0585bb97bd5f5bf3ed8d4bf624593e4c5a14d06c811d1952b07a1f94b4d12c51", + "sha256:dae3e882dc0e6f28e60cf09c1f13fedfd881db84f824dd418aa9e44def2fe00d" ], "index": "pypi", - "version": "==1.7.11" + "markers": "python_version >= '3.7'", + "version": "==2.169.0" }, "google-auth": { "hashes": [ - "sha256:73b141d122942afe12e8bfdcb6900d5df35c27d39700f078363ba0b1298ad33b", - "sha256:fbf25fee328c0828ef293459d9c649ef84ee44c0b932bb999d19df0ead1b40cf" + "sha256:58f0e8416a9814c1d86c9b7f6acf6816b51aba167b2c76821965271bac275540", + "sha256:ed4cae4f5c46b41bae1d19c036e06f6c371926e97b19e816fc854eff811974ee" ], - "version": "==1.15.0" + "markers": "python_version >= '3.7'", + "version": "==2.40.1" }, "google-auth-httplib2": { "hashes": [ - "sha256:098fade613c25b4527b2c08fa42d11f3c2037dda8995d86de0745228e965d445", - "sha256:f1c437842155680cf9918df9bc51c1182fda41feef88c34004bd1978c8157e08" + "sha256:38aa7badf48f974f1eb9861794e9c0cb2a0511a4ec0679b1f886d108f5640e05", + "sha256:b65a0a2123300dd71281a7bf6e64d65a0759287df52729bdd1ae2e47dc311a3d" ], "index": "pypi", - "version": "==0.0.3" + "version": "==0.2.0" }, "google-auth-oauthlib": { "hashes": [ - "sha256:88d2cd115e3391eb85e1243ac6902e76e77c5fe438b7276b297fbe68015458dd", - "sha256:a92a0f6f41a0fb6138454fbc02674e64f89d82a244ea32f98471733c8ef0e0e1" + "sha256:11046fb8d3348b296302dd939ace8af0a724042e8029c1b872d87fabc9f41684", + "sha256:fd619506f4b3908b5df17b65f39ca8d66ea56986e5472eb5978fd8f3786f00a2" ], "index": "pypi", - "version": "==0.4.1" + "markers": "python_version >= '3.6'", + "version": "==1.2.2" + }, + "googleapis-common-protos": { + "hashes": [ + "sha256:0e1b44e0ea153e6594f9f394fef15193a68aaaea2d843f83e2742717ca753257", + "sha256:b8bfcca8c25a2bb253e0e0b0adaf8c00773e5e6af6fd92397576680b807e0fd8" + ], + "markers": "python_version >= '3.7'", + "version": "==1.70.0" }, "httplib2": { "hashes": [ - "sha256:4f6988e6399a2546b525a037d56da34aed4d149bbdc0e78523018d5606c26e74", - "sha256:b0e1f3ed76c97380fe2485bc47f25235453b40ef33ca5921bb2897e257a49c4c" + "sha256:14ae0a53c1ba8f3d37e9e27cf37eabb0fb9980f435ba405d546948b009dd64dc", + "sha256:d7a10bc5ef5ab08322488bde8c726eeee5c8618723fdb399597ec58f3d82df81" ], - "index": "pypi", - "version": "==0.18.0" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==0.22.0" }, "idna": { "hashes": [ - "sha256:7588d1c14ae4c77d74036e8c22ff447b26d0fde8f007354fd48a7814db15b7cb", - "sha256:a068a21ceac8a4d63dbfd964670474107f541babbd2250d61922f029858365fa" + "sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9", + "sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3" ], - "version": "==2.9" + "markers": "python_version >= '3.6'", + "version": "==3.10" }, "isodate": { "hashes": [ - "sha256:2e364a3d5759479cdb2d37cce6b9376ea504db2ff90252a2e5b7cc89cc9ff2d8", - "sha256:aa4d33c06640f5352aca96e4b81afd8ab3b47337cc12089822d6f322ac772c81" + "sha256:28009937d8031054830160fce6d409ed342816b543597cece116d966c6d99e15", + "sha256:4cd1aa0f43ca76f4a6c6c0292a85f40b35ec2e43e315b59f06e6d32171a953e6" ], - "version": "==0.6.0" + "markers": "python_version >= '3.7'", + "version": "==0.7.2" }, "leather": { "hashes": [ - "sha256:076d1603b5281488285718ce1a5ce78cf1027fe1e76adf9c548caf83c519b988", - "sha256:e0bb36a6d5f59fbf3c1a6e75e7c8bee29e67f06f5b48c0134407dde612eba5e2" + "sha256:18290bc93749ae39039af5e31e871fcfad74d26c4c3ea28ea4f681f4571b3a2b", + "sha256:f964bec2086f3153a6c16e707f20cb718f811f57af116075f4c0f4805c608b95" ], - "version": "==0.3.3" + "version": "==0.4.0" }, "oauthlib": { "hashes": [ - "sha256:bee41cc35fcca6e988463cacc3bcb8a96224f470ca547e697b604cc697b2f889", - "sha256:df884cd6cbe20e32633f1db1072e9356f53638e4361bef4e8b03c9127c9328ea" + "sha256:8139f29aac13e25d502680e9e19963e83f16838d48a0d71c287fe40e7067fbca", + "sha256:9859c40929662bec5d64f34d01c99e093149682a3f38915dc0655d5a633dd918" ], - "version": "==3.1.0" + "markers": "python_version >= '3.6'", + "version": "==3.2.2" }, "parsedatetime": { "hashes": [ - "sha256:3b835fc54e472c17ef447be37458b400e3fefdf14bb1ffdedb5d2c853acf4ba1", - "sha256:d2e9ddb1e463de871d32088a3f3cea3dc8282b1b2800e081bd0ef86900451667" + "sha256:4cb368fbb18a0b7231f4d76119165451c8d2e35951455dfee97c62a87b04d455", + "sha256:cb96edd7016872f58479e35879294258c71437195760746faffedb692aef000b" ], - "version": "==2.5" + "version": "==2.6" }, "progress": { "hashes": [ - "sha256:69ecedd1d1bbe71bf6313d88d1e6c4d2957b7f1d4f71312c211257f7dae64372" + "sha256:c9c86e98b5c03fa1fe11e3b67c1feda4788b8d0fe7336c2ff7d5644ccfba34cd" ], "index": "pypi", - "version": "==1.5" + "version": "==1.6" + }, + "proto-plus": { + "hashes": [ + "sha256:13285478c2dcf2abb829db158e1047e2f1e8d63a077d94263c2b88b043c75a66", + "sha256:21a515a4c4c0088a773899e23c7bbade3d18f9c66c73edd4c7ee3816bc96a012" + ], + "markers": "python_version >= '3.7'", + "version": "==1.26.1" + }, + "protobuf": { + "hashes": [ + "sha256:13eb236f8eb9ec34e63fc8b1d6efd2777d062fa6aaa68268fb67cf77f6839ad7", + "sha256:1832f0515b62d12d8e6ffc078d7e9eb06969aa6dc13c13e1036e39d73bebc2de", + "sha256:307ecba1d852ec237e9ba668e087326a67564ef83e45a0189a772ede9e854dd0", + "sha256:3fde11b505e1597f71b875ef2fc52062b6a9740e5f7c8997ce878b6009145862", + "sha256:476cb7b14914c780605a8cf62e38c2a85f8caff2e28a6a0bad827ec7d6c85d68", + "sha256:4f1dfcd7997b31ef8f53ec82781ff434a28bf71d9102ddde14d076adcfc78c99", + "sha256:678974e1e3a9b975b8bc2447fca458db5f93a2fb6b0c8db46b6675b5b5346812", + "sha256:aec4962f9ea93c431d5714ed1be1c93f13e1a8618e70035ba2b0564d9e633f2e", + "sha256:bcefcdf3976233f8a502d265eb65ea740c989bacc6c30a58290ed0e519eb4b8d", + "sha256:d7d3f7d1d5a66ed4942d4fefb12ac4b14a29028b209d4bfb25c68ae172059922", + "sha256:fd32223020cb25a2cc100366f1dedc904e2d71d9322403224cdde5fdced0dabe" + ], + "markers": "python_version >= '3.8'", + "version": "==5.29.4" }, "pyasn1": { "hashes": [ - "sha256:39c7e2ec30515947ff4e87fb6f456dfc6e84857d34be479c9d4a4ba4bf46aa5d", - "sha256:aef77c9fb94a3ac588e87841208bdec464471d9871bd5050a287cc9a475cd0ba" + "sha256:0d632f46f2ba09143da3a8afe9e33fb6f92fa2320ab7e886e2d0f7672af84629", + "sha256:6f580d2bdd84365380830acf45550f2511469f673cb4a5ae3857a3170128b034" ], - "version": "==0.4.8" + "markers": "python_version >= '3.8'", + "version": "==0.6.1" }, "pyasn1-modules": { "hashes": [ - "sha256:905f84c712230b2c592c19470d3ca8d552de726050d1d1716282a1f6146be65e", - "sha256:a50b808ffeb97cb3601dd25981f6b016cbb3d31fbf57a8b8a87428e6158d0c74" + "sha256:29253a9207ce32b64c3ac6600edc75368f98473906e8fd1043bd6b5b1de2c14a", + "sha256:677091de870a80aae844b1ca6134f54652fa2c8c5a52aa396440ac3106e941e6" ], - "version": "==0.2.8" + "markers": "python_version >= '3.8'", + "version": "==0.4.2" + }, + "pyparsing": { + "hashes": [ + "sha256:a6a7ee4235a3f944aa1fa2249307708f893fe5717dc603503c6c7969c070fb7c", + "sha256:f86ec8d1a83f11977c9a6ea7598e8c27fc5cddfa5b07ea2241edbbde1d7bc032" + ], + "markers": "python_version >= '3.1'", + "version": "==3.1.4" + }, + "python-dateutil": { + "hashes": [ + "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", + "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427" + ], + "index": "pypi", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==2.9.0.post0" }, "python-slugify": { "hashes": [ - "sha256:a8fc3433821140e8f409a9831d13ae5deccd0b033d4744d94b31fea141bdd84c" + "sha256:276540b79961052b66b7d116620b36518847f52d5fd9e3a70164fc8c50faa6b8", + "sha256:59202371d1d05b54a9e7720c5e038f928f45daaffe41dd10822f3907b937c856" ], - "version": "==4.0.0" + "markers": "python_version >= '3.7'", + "version": "==8.0.4" }, "pytimeparse": { "hashes": [ @@ -185,60 +349,70 @@ }, "pytz": { "hashes": [ - "sha256:a494d53b6d39c3c6e44c3bec237336e14305e4f29bbf800b599253057fbb79ed", - "sha256:c35965d010ce31b23eeb663ed3cc8c906275d6be1a34393a1d73a41febf4a048" + "sha256:360b9e3dbb49a209c21ad61809c7fb453643e048b38924c765813546746e81c3", + "sha256:5ddf76296dd8c44c26eb8f4b6f35488f3ccbf6fbbd7adee0b7262d43f0ec2f00" ], - "version": "==2020.1" + "markers": "python_version < '3.9'", + "version": "==2025.2" }, "requests": { "hashes": [ - "sha256:43999036bfa82904b6af1d99e4882b560e5e2c68e5c4b0aa03b655f3d7d73fee", - "sha256:b3f43d496c6daba4493e7c431722aeb7dbc6288f52a6e04e7b6023b0247817e6" + "sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760", + "sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6" ], - "version": "==2.23.0" + "markers": "python_version >= '3.8'", + "version": "==2.32.3" }, "requests-oauthlib": { "hashes": [ - "sha256:7f71572defaecd16372f9006f33c2ec8c077c3cfa6f5911a9a90202beb513f3d", - "sha256:b4261601a71fd721a8bd6d7aa1cc1d6a8a93b4a9f5e96626f8e4d91e8beeaa6a" + "sha256:7dd8a5c40426b779b0868c404bdef9768deccf22749cde15852df527e6269b36", + "sha256:b3dffaebd884d8cd778494369603a9e7b58d29111bf6b41bdc2dcd87203af4e9" ], - "version": "==1.3.0" + "markers": "python_version >= '3.4'", + "version": "==2.0.0" }, "rsa": { "hashes": [ - "sha256:14ba45700ff1ec9eeb206a2ce76b32814958a98e372006c8fb76ba820211be66", - "sha256:1a836406405730121ae9823e19c6e806c62bbad73f890574fff50efa4122c487" + "sha256:68635866661c6836b8d39430f97a996acbd61bfa49406748ea243539fe239762", + "sha256:e7bdbfdb5497da4c07dfd35530e1a902659db6ff241e39d9953cad06ebd0ae75" ], - "version": "==4.0" + "markers": "python_version >= '3.6' and python_version < '4'", + "version": "==4.9.1" }, "six": { "hashes": [ - "sha256:236bdbdce46e6e6a3d61a337c0f8b763ca1e8717c03b369e87a7ec7ce1319c0a", - "sha256:8f3cd2e254d8f793e7f3d6d9df77b92252b52637291d0f0da013c76ea2724b6c" + "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", + "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81" ], - "version": "==1.14.0" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==1.17.0" }, "termcolor": { "hashes": [ - "sha256:1d6d69ce66211143803fbc56652b41d73b4a400a2891d7bf7a1cdf4c02de613b" + "sha256:9297c0df9c99445c2412e832e882a7884038a25617c60cea2ad69488d4040d63", + "sha256:aab9e56047c8ac41ed798fa36d892a37aca6b3e9159f3e0c24bc64a9b3ac7b7a" ], "index": "pypi", - "version": "==1.1.0" + "markers": "python_version >= '3.8'", + "version": "==2.4.0" }, "termgraph": { "hashes": [ - "sha256:3ea73b7659faec1a420ed3515d0b67e1bd7cb915f36f2016c4e524926d3eb0ae" + "sha256:36ff2098e41eeab1e7cdda7366dc3e5b514ea799fa3e77537564492a7edefdd5", + "sha256:c181c168c2278a7f7161de05b1033396a2f6b06ab260ddf987b0320fae93ff8e" ], "index": "pypi", - "version": "==0.2.1" + "markers": "python_version >= '3.7'", + "version": "==0.5.3" }, "termtables": { "hashes": [ - "sha256:0fd321f7f478001f824896b4dbc5af0cf296ecf3c4aa450018cdcd31b4880b6e", - "sha256:762ba718cae224917fd8e1eda5cbfb33027ab6c86cfb9dd676e35f0c6516b4bf" + "sha256:0ca860eed8957db0045020bcdfcf6620e188b0d776e8543a57ff539fcfeec26d", + "sha256:797c6afeb78abdab97cd5bfbbd2fc1bfbd9630052699dc881b27b334bcc6a73f" ], "index": "pypi", - "version": "==0.1.1" + "markers": "python_version >= '3.6'", + "version": "==0.2.4" }, "text-unidecode": { "hashes": [ @@ -249,17 +423,19 @@ }, "uritemplate": { "hashes": [ - "sha256:07620c3f3f8eed1f12600845892b0e036a2420acf513c53f7de0abd911a5894f", - "sha256:5af8ad10cec94f215e3f48112de2022e1d5a37ed427fbd88652fa908f2ab7cae" + "sha256:4346edfc5c3b79f694bccd6d6099a322bbeb628dbf2cd86eea55a456ce5124f0", + "sha256:830c08b8d99bdd312ea4ead05994a38e8936266f84b9a7878232db50b044e02e" ], - "version": "==3.0.1" + "markers": "python_version >= '3.6'", + "version": "==4.1.1" }, "urllib3": { "hashes": [ - "sha256:3018294ebefce6572a474f0604c2021e33b3fd8006ecd11d62107a5d2a963527", - "sha256:88206b0eb87e6d677d424843ac5209e3fb9d0190d0ee169599165ec25e9d9115" + "sha256:ca899ca043dcb1bafa3e262d73aa25c465bfb49e0bd9dd5d59f1d0acba2f8fac", + "sha256:e7d814a81dad81e6caf2ec9fdedb284ecc9c73076b62654547cc64ccdcae26e9" ], - "version": "==1.25.9" + "markers": "python_version >= '3.8'", + "version": "==2.2.3" } }, "develop": {} diff --git a/README.md b/README.md index eb95ef3..93aa2cd 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ ![screenshot](screenshots/tool.png) -This tool will analyze your gmail account to show you statics of your emails. e.g. +This tool will analyze your gmail account to show you statistics of your emails. e.g. - Total number of emails - First email received @@ -28,14 +28,38 @@ $ python analyzer.py --help ``` $ python analyzer.py --help -usage: analyzer.py [-h] [--top TOP] [--user USER] [--verbose] [--version] +usage: analyzer.py [-h] [--top TOP] [--user USER] [--query QUERY] + [--inactive INACTIVE] [--max-retry-rounds MAX_RETRY_ROUNDS] + [--pull-data] [--refresh-data] [--analyze-only] + [--export-csv EXPORT_CSV] [--verbose] [--version] Simple Gmail Analyzer optional arguments: - -h, --help show this help message and exit - --top TOP Number of results to show - --user USER User ID to fetch data for - --verbose Verbose output, helpful for debugging - --version Display version and exit + -h, --help show this help message and exit + --top TOP Number of results to show + --user USER User ID to fetch data for + --query QUERY Gmail search query (e.g., 'label:work after:2023/01/01') + --inactive INACTIVE Show senders inactive for more than X days + --max-retry-rounds MAX_RETRY_ROUNDS + Max retry rounds for failed message fetches (0 for unlimited) + --pull-data Fetch and cache data, then exit + --refresh-data Force refresh cached data, then exit + --analyze-only Analyze using cached data only (no API calls) + --export-csv EXPORT_CSV + Export message metadata to CSV at the given path + --verbose Verbose output, helpful for debugging + --version Display version and exit +``` + +# Caching & Data Pulls + +The analyzer caches message and metadata pickles in `cache/` for 24 hours. Queries +create separate cache files, so cached data stays scoped to each Gmail search. + +Examples: + +``` +$ python analyzer.py --pull-data --query "label:work after:2023/01/01" +$ python analyzer.py --analyze-only --export-csv out/messages.csv ``` diff --git a/analyzer.py b/analyzer.py index ceed6a9..d3b3bd7 100644 --- a/analyzer.py +++ b/analyzer.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python3 + import argparse import sys import colorama @@ -21,9 +23,42 @@ def init_args(): parser.add_argument( "--version", action="store_true", help="Display version and exit" ) + parser.add_argument( + "--query", + type=str, + default=None, + help="Gmail search query (e.g., 'label:work after:2023/01/01')", + ) parser.add_argument( "--inactive", type=int, default=0, help="Show senders inactive for more than X days" ) + parser.add_argument( + "--max-retry-rounds", + type=int, + default=5, + help="Max retry rounds for failed message fetches (0 for unlimited)", + ) + parser.add_argument( + "--pull-data", + action="store_true", + help="Fetch and cache data, then exit", + ) + parser.add_argument( + "--refresh-data", + action="store_true", + help="Force refresh cached data, then exit", + ) + parser.add_argument( + "--analyze-only", + action="store_true", + help="Analyze using cached data only (no API calls)", + ) + parser.add_argument( + "--export-csv", + type=str, + default=None, + help="Export message metadata to CSV at the given path", + ) args = vars(parser.parse_args()) @@ -39,4 +74,13 @@ def init_args(): print("gmail analyzer v{}".format(VERSION)) sys.exit() + mode_flags = [ + args["pull_data"], + args["refresh_data"], + args["analyze_only"], + ] + if sum(1 for flag in mode_flags if flag) > 1: + print("Error: --pull-data, --refresh-data, and --analyze-only are mutually exclusive.") + sys.exit(1) + Metrics(args).start() diff --git a/src/metrics.py b/src/metrics.py index 2f6be7e..8a19b6f 100644 --- a/src/metrics.py +++ b/src/metrics.py @@ -24,10 +24,17 @@ def __init__(self, args): # Ignore warnings about SSL connections warnings.simplefilter("ignore", ResourceWarning) - self.processor = Processor() + self.processor = Processor( + query=args.get("query"), + max_retry_rounds=args.get("max_retry_rounds"), + ) self.user_id = args["user"] self.resultsLimit = args["top"] self.inactive_days = args["inactive"] + self.pull_data = args.get("pull_data", False) + self.refresh_data = args.get("refresh_data", False) + self.analyze_only = args.get("analyze_only", False) + self.export_csv = args.get("export_csv") self.table = None def _load_table(self, event): @@ -344,8 +351,25 @@ def analyse(self): print("\nAnalysis complete!") def start(self): - messages = self.processor.get_messages() + if self.analyze_only: + if not self.processor.load_cached_metadata(): + print("No cached metadata found. Run with --pull-data first.") + return + if self.export_csv: + self.processor.export_csv(self.export_csv) + self.analyse() + return + + force_refresh = self.refresh_data + messages = self.processor.get_messages(force_refresh=force_refresh) - self.processor.get_metadata(messages) + self.processor.get_metadata(messages, force_refresh=force_refresh) + + if self.export_csv: + self.processor.export_csv(self.export_csv) + + if self.pull_data or self.refresh_data: + print("Data pull complete.") + return self.analyse() diff --git a/src/processor.py b/src/processor.py index 79a1224..8a43589 100644 --- a/src/processor.py +++ b/src/processor.py @@ -1,7 +1,12 @@ import collections +import csv +import hashlib +import json import os.path import pickle +import random import time +from googleapiclient.errors import HttpError from progress.counter import Counter from progress.bar import IncrementalBar @@ -10,13 +15,26 @@ _progressPadding = 29 _CACHE_DIR = "cache" +_CACHE_TTL_SECONDS = 86400 +_MAX_RETRIES = 5 +_RETRY_BASE_DELAY_SECONDS = 1.0 +_RETRY_MAX_DELAY_SECONDS = 60.0 +_DEFAULT_MAX_RETRY_ROUNDS = 5 class Processor: # Talk to google api, fetch results and decorate them - def __init__(self): + def __init__(self, query=None, max_retry_rounds=None): self.service = Service().instance() self.user_id = "me" + self.query = query + self.cache_key = self._build_cache_key(query) + if max_retry_rounds is None: + self.max_retry_rounds = _DEFAULT_MAX_RETRY_ROUNDS + elif max_retry_rounds <= 0: + self.max_retry_rounds = None + else: + self.max_retry_rounds = max_retry_rounds self.messagesQueue = collections.deque() self.failedMessagesQueue = collections.deque() @@ -24,15 +42,82 @@ def __init__(self): if not os.path.exists(_CACHE_DIR): os.makedirs(_CACHE_DIR) - def get_messages(self): + def _build_cache_key(self, query): + if not query: + return None + return hashlib.sha1(query.encode("utf-8")).hexdigest()[:10] + + def _cache_path(self, prefix): + if self.cache_key: + filename = f"{prefix}_{self.cache_key}.pickle" + else: + filename = f"{prefix}.pickle" + return os.path.join(_CACHE_DIR, filename) + + def _should_retry(self, exception): + if not isinstance(exception, HttpError): + return False + + status = getattr(exception.resp, "status", None) + if status in (429, 500, 503): + return True + + if status == 403: + reason = self._extract_error_reason(exception) + if reason in ("rateLimitExceeded", "userRateLimitExceeded"): + return True + + return False + + def _extract_error_reason(self, exception): + try: + payload = json.loads(exception.content.decode("utf-8")) + except (ValueError, AttributeError, UnicodeDecodeError): + return None + + try: + return payload["error"]["errors"][0]["reason"] + except (KeyError, IndexError, TypeError): + return None + + def _sleep_with_backoff(self, attempt): + base_delay = min(_RETRY_MAX_DELAY_SECONDS, _RETRY_BASE_DELAY_SECONDS * (2 ** attempt)) + jitter = random.uniform(0, base_delay * 0.1) + time.sleep(base_delay + jitter) + + def _execute_with_backoff(self, request_callable, context): + attempt = 0 + while True: + try: + return request_callable() + except HttpError as exception: + if not self._should_retry(exception) or attempt >= _MAX_RETRIES: + raise + wait_time = min( + _RETRY_MAX_DELAY_SECONDS, + _RETRY_BASE_DELAY_SECONDS * (2 ** attempt), + ) + jitter = random.uniform(0, wait_time * 0.1) + print( + f"{helpers.loader_icn} Rate limit or transient error while {context}. " + f"Retrying in {wait_time + jitter:.1f}s" + ) + time.sleep(wait_time + jitter) + attempt += 1 + + def get_messages(self, force_refresh=False): # Get all messages of user # Output format: # [{'id': '13c...7', 'threadId': '13c...7'}, ...] - - cache_file = os.path.join(_CACHE_DIR, "messages.pickle") - + + cache_file = self._cache_path("messages") + # Check if cache exists and is not older than 24 hours - if os.path.exists(cache_file) and (time.time() - os.path.getmtime(cache_file) < 86400): + if ( + not force_refresh + and os.path.exists(cache_file) + and (time.time() - os.path.getmtime(cache_file) < _CACHE_TTL_SECONDS) + ): print(f"{helpers.loader_icn} Loading messages from cache") with open(cache_file, "rb") as token: messages = pickle.load(token) @@ -41,7 +126,17 @@ def get_messages(self): # includeSpamTrash # labelIds - response = self.service.users().messages().list(userId=self.user_id).execute() + list_kwargs = { + "userId": self.user_id, + "fields": "messages(id,threadId),nextPageToken,resultSizeEstimate", + } + if self.query: + list_kwargs["q"] = self.query + + response = self._execute_with_backoff( + lambda: self.service.users().messages().list(**list_kwargs).execute(), + "listing messages", + ) messages = [] est_max = response["resultSizeEstimate"] * 5 @@ -55,11 +150,10 @@ def get_messages(self): while "nextPageToken" in response: page_token = response["nextPageToken"] - response = ( - self.service.users() - .messages() - .list(userId=self.user_id, pageToken=page_token) - .execute() + list_kwargs["pageToken"] = page_token + response = self._execute_with_backoff( + lambda: self.service.users().messages().list(**list_kwargs).execute(), + "listing messages", ) messages.extend(response["messages"]) @@ -75,10 +169,15 @@ def get_messages(self): def process_message(self, request_id, response, exception): if exception is not None: - self.failedMessagesQueue.append(exception.uri) + if self._should_retry(exception): + self.failedMessagesQueue.append(request_id) + else: + print( + f"{helpers.loader_icn} Skipping message {request_id} due to error: {exception}" + ) return - headers = response["payload"]["headers"] + headers = response.get("payload", {}).get("headers", []) _date = next( (header["value"] for header in headers if header["name"] == "Date"), None @@ -86,16 +185,19 @@ def process_message(self, request_id, response, exception): _from = next( (header["value"] for header in headers if header["name"] == "From"), None ) + _subject = next( + (header["value"] for header in headers if header["name"] == "Subject"), None + ) self.messagesQueue.append( { "id": response["id"], "labels": response.get("labelIds", []), - "fields": {"from": _from, "date": _date}, + "fields": {"from": _from, "date": _date, "subject": _subject}, } ) - def get_metadata(self, messages): + def get_metadata(self, messages, force_refresh=False): # Get metadata for all messages: # 1. Create a batch get message request for all messages # 2. Process the returned output @@ -110,13 +212,26 @@ def get_metadata(self, messages): # ] # } - cache_file = os.path.join(_CACHE_DIR, "metadata.pickle") - - # Check if cache exists and is not older than 24 hours - if os.path.exists(cache_file) and (time.time() - os.path.getmtime(cache_file) < 86400): - print(f"{helpers.loader_icn} Loading message metadata from cache") + cache_file = self._cache_path("metadata") + + cache_exists = os.path.exists(cache_file) + cache_fresh = cache_exists and ( + time.time() - os.path.getmtime(cache_file) < _CACHE_TTL_SECONDS + ) + + if cache_exists and not force_refresh: + if cache_fresh: + print(f"{helpers.loader_icn} Loading message metadata from cache") + else: + print(f"{helpers.loader_icn} Loading stale metadata cache to resume") with open(cache_file, "rb") as token: - self.messagesQueue = pickle.load(token) + cached = pickle.load(token) + self.messagesQueue = ( + cached if isinstance(cached, collections.deque) else collections.deque(cached) + ) + cached_ids = {message["id"] for message in self.messagesQueue} + messages = [message for message in messages if message["id"] not in cached_ids] + if not messages: return progress = IncrementalBar( @@ -133,15 +248,162 @@ def get_metadata(self, messages): for message in messages_batch: msg_id = message["id"] batch.add( - self.service.users().messages().get(userId=self.user_id, id=msg_id), + self.service.users() + .messages() + .get( + userId=self.user_id, + id=msg_id, + format="metadata", + metadataHeaders=["From", "Date", "Subject"], + fields="id,labelIds,payload/headers", + ), callback=self.process_message, + request_id=msg_id, ) - batch.execute() + self._execute_with_backoff(batch.execute, "fetching message metadata") progress.next(len(messages_batch)) progress.finish() + + self._retry_failed_messages() # Cache the metadata with open(cache_file, "wb") as token: pickle.dump(self.messagesQueue, token) + + def load_cached_metadata(self): + cache_file = self._cache_path("metadata") + if not os.path.exists(cache_file): + return False + print(f"{helpers.loader_icn} Loading message metadata from cache") + with open(cache_file, "rb") as token: + cached = pickle.load(token) + self.messagesQueue = ( + cached if isinstance(cached, collections.deque) else collections.deque(cached) + ) + return True + + def export_csv(self, path): + if not self.messagesQueue: + print(f"{helpers.loader_icn} No metadata loaded; skipping CSV export.") + return + + rows = list(self.messagesQueue) + fieldnames = ["id", "from", "date", "subject", "labels"] + with open(path, "w", newline="") as csvfile: + writer = csv.DictWriter(csvfile, fieldnames=fieldnames) + writer.writeheader() + for row in rows: + fields = row.get("fields", {}) + writer.writerow( + { + "id": row.get("id"), + "from": fields.get("from"), + "date": fields.get("date"), + "subject": fields.get("subject"), + "labels": ",".join(row.get("labels", [])), + } + ) + print(f"{helpers.loader_icn} Exported CSV to {path}") + + def get_query_message_ids(self): + if not self.query: + return None + + response = self._execute_with_backoff( + lambda: self.service.users() + .messages() + .list( + userId=self.user_id, + q=self.query, + fields="messages(id,threadId),nextPageToken", + ) + .execute(), + "listing query messages", + ) + messages = [] + + if "messages" in response: + messages.extend(response["messages"]) + + while "nextPageToken" in response: + page_token = response["nextPageToken"] + response = self._execute_with_backoff( + lambda: self.service.users() + .messages() + .list( + userId=self.user_id, + pageToken=page_token, + q=self.query, + fields="messages(id,threadId),nextPageToken", + ) + .execute(), + "listing query messages", + ) + messages.extend(response["messages"]) + + return {message["id"] for message in messages} + + def filter_messages_queue(self, message_ids): + if message_ids is None: + return + if not message_ids: + self.messagesQueue = collections.deque() + return + self.messagesQueue = collections.deque( + message for message in self.messagesQueue if message["id"] in message_ids + ) + + def _retry_failed_messages(self): + retry_round = 0 + while self.failedMessagesQueue and ( + self.max_retry_rounds is None or retry_round < self.max_retry_rounds + ): + failed_ids = list(self.failedMessagesQueue) + self.failedMessagesQueue = collections.deque() + + print( + f"{helpers.loader_icn} Retrying {len(failed_ids)} failed messages " + f"(round {retry_round + 1}/" + f"{self.max_retry_rounds if self.max_retry_rounds is not None else '∞'})" + ) + + self._sleep_with_backoff(retry_round) + + for messages_batch in helpers.chunks(failed_ids, 50): + batch = self.service.new_batch_http_request() + for msg_id in messages_batch: + batch.add( + self.service.users() + .messages() + .get( + userId=self.user_id, + id=msg_id, + format="metadata", + metadataHeaders=["From", "Date", "Subject"], + fields="id,labelIds,payload/headers", + ), + callback=self.process_message, + request_id=msg_id, + ) + + try: + self._execute_with_backoff(batch.execute, "retrying message metadata") + except HttpError as exception: + if not self._should_retry(exception): + print(f"{helpers.loader_icn} Batch retry failed: {exception}") + return + + retry_round += 1 + if self.failedMessagesQueue and len(self.failedMessagesQueue) >= len(failed_ids): + print( + f"{helpers.loader_icn} Retry did not reduce failures; stopping retries." + ) + return + + if self.failedMessagesQueue: + print( + f"{helpers.loader_icn} Unable to fetch {len(self.failedMessagesQueue)} " + "messages after retries" + ) diff --git a/src/service.py b/src/service.py index 0b6aea4..7856ab3 100644 --- a/src/service.py +++ b/src/service.py @@ -6,8 +6,8 @@ class Service: - def __init__(self): - self.scopes = ["https://www.googleapis.com/auth/gmail.readonly"] + def __init__(self, scopes=None): + self.scopes = scopes or ["https://www.googleapis.com/auth/gmail.readonly"] def instance(self): service = build("gmail", "v1", credentials=self._get_creds()) @@ -25,7 +25,11 @@ def _get_creds(self): creds = pickle.load(token) # If there are no (valid) credentials available, let the user log in. - if not creds or not creds.valid: + scopes_mismatch = False + if creds and getattr(creds, "scopes", None): + scopes_mismatch = not set(self.scopes).issubset(set(creds.scopes)) + + if not creds or not creds.valid or scopes_mismatch: if creds and creds.expired and creds.refresh_token: creds.refresh(Request()) else: