From b78533968eec24710f19f96dc3edbf6a79d7749a Mon Sep 17 00:00:00 2001 From: Mason Date: Fri, 15 Mar 2024 17:24:34 -0400 Subject: [PATCH 01/19] WIP, pass down user_ids to the children of Team --- tap_clickup/streams.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tap_clickup/streams.py b/tap_clickup/streams.py index d4abd88..5d12ac8 100644 --- a/tap_clickup/streams.py +++ b/tap_clickup/streams.py @@ -20,8 +20,10 @@ class TeamsStream(ClickUpStream): def get_child_context(self, record: dict, context: Optional[dict]) -> dict: """Return a context dictionary for child streams.""" + user_ids = [member.get("user", {}).get("id") for member in record.get("members", []) if isinstance(member, dict)] return { "team_id": record["id"], + "user_ids": user_ids } @@ -37,6 +39,14 @@ class TimeEntries(ClickUpStream): parent_stream_type = TeamsStream # TODO not clear why this is needed partitions = None + def get_url_params( + self, context: Optional[dict], next_page_token: Optional[Any] + ) -> Dict[str, Any]: + """Return a dictionary of values to be used in URL parameterization.""" + params = super().get_url_params(context, next_page_token) + if 'user_ids' in context: + params["user_ids"] = ",".join(context["user_ids"]) + return params class SpacesStream(ClickUpStream): From de8232336b02ca7e6410dd4ac7ab9d9ebf64b5db Mon Sep 17 00:00:00 2001 From: Mason Date: Fri, 22 Mar 2024 11:13:15 -0400 Subject: [PATCH 02/19] Remove if for now --- tap_clickup/streams.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tap_clickup/streams.py b/tap_clickup/streams.py index 5d12ac8..70f0fac 100644 --- a/tap_clickup/streams.py +++ b/tap_clickup/streams.py @@ -20,7 +20,8 @@ class TeamsStream(ClickUpStream): def get_child_context(self, record: dict, context: Optional[dict]) -> dict: """Return a context dictionary for child streams.""" - user_ids = [member.get("user", {}).get("id") for member in record.get("members", []) if isinstance(member, dict)] + user_ids = [str(member.get("user", {}).get("id")) for member in record.get("members", []) if isinstance(member, dict)] + return { "team_id": record["id"], "user_ids": user_ids @@ -40,12 +41,11 @@ class TimeEntries(ClickUpStream): # TODO not clear why this is needed partitions = None def get_url_params( - self, context: Optional[dict], next_page_token: Optional[Any] + self, context: Optional[dict], next_page_token: Optional[Any] ) -> Dict[str, Any]: """Return a dictionary of values to be used in URL parameterization.""" params = super().get_url_params(context, next_page_token) - if 'user_ids' in context: - params["user_ids"] = ",".join(context["user_ids"]) + params["assignee"] = ",".join(context["user_ids"]) return params From d39ad162c5c08d5c269507365b7a28af31235153 Mon Sep 17 00:00:00 2001 From: Mason Date: Fri, 22 Mar 2024 12:29:26 -0400 Subject: [PATCH 03/19] Fix time entry schema --- tap_clickup/schemas/time_entries.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tap_clickup/schemas/time_entries.json b/tap_clickup/schemas/time_entries.json index b8b9793..8560582 100644 --- a/tap_clickup/schemas/time_entries.json +++ b/tap_clickup/schemas/time_entries.json @@ -20,7 +20,7 @@ "type": "string" }, "color": { - "type": "string" + "type": ["string", "null"] }, "type": { "type": "string" @@ -49,7 +49,7 @@ "type": "string" }, "color": { - "type": "string" + "type": ["string", "null"] }, "initials": { "type": "string" From e59ec0f07ab1524739e3bde06d1b24a8756b5e41 Mon Sep 17 00:00:00 2001 From: Mason Date: Fri, 22 Mar 2024 12:33:58 -0400 Subject: [PATCH 04/19] Another schema fix --- tap_clickup/schemas/time_entries.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tap_clickup/schemas/time_entries.json b/tap_clickup/schemas/time_entries.json index 8560582..4a91cec 100644 --- a/tap_clickup/schemas/time_entries.json +++ b/tap_clickup/schemas/time_entries.json @@ -55,7 +55,7 @@ "type": "string" }, "profilePicture": { - "type": "null" + "type": ["string", "null"] } } }, "billable": { From 437335e2eee91b1871f475df699b92590a1ce0a8 Mon Sep 17 00:00:00 2001 From: Mason Date: Fri, 22 Mar 2024 13:57:39 -0400 Subject: [PATCH 05/19] Formatting --- tap_clickup/schemas/time_entries.json | 128 ++++++++++++++------------ 1 file changed, 70 insertions(+), 58 deletions(-) diff --git a/tap_clickup/schemas/time_entries.json b/tap_clickup/schemas/time_entries.json index 4a91cec..2cf61bd 100644 --- a/tap_clickup/schemas/time_entries.json +++ b/tap_clickup/schemas/time_entries.json @@ -20,7 +20,10 @@ "type": "string" }, "color": { - "type": ["string", "null"] + "type": [ + "string", + "null" + ] }, "type": { "type": "string" @@ -28,76 +31,85 @@ "orderindex": { "type": "integer" } - } }, - "custom_type": { - "type": "null" } - } }, - "wid": { - "type": "string" - }, - "user": { - "type": "object", - "properties": { - "id": { - "type": "integer" - }, - "username": { - "type": "string" - }, - "email": { - "type": "string" - }, - "color": { - "type": ["string", "null"] - }, - "initials": { - "type": "string" - }, - "profilePicture": { - "type": ["string", "null"] - } - } }, - "billable": { - "type": "boolean" - }, - "start": { - "type": "string" }, - "end": { - "type": "string" + "custom_type": { + "type": "null" + } + } + }, + "wid": { + "type": "string" + }, + "user": { + "type": "object", + "properties": { + "id": { + "type": "integer" }, - "duration": { + "username": { "type": "string" }, - "description": { + "email": { "type": "string" }, - "tags": { - "type": "array" + "color": { + "type": [ + "string", + "null" + ] }, - "source": { + "initials": { "type": "string" }, - "at": { + "profilePicture": { + "type": [ + "string", + "null" + ] + } + } + }, + "billable": { + "type": "boolean" + }, + "start": { + "type": "string" + }, + "end": { + "type": "string" + }, + "duration": { + "type": "string" + }, + "description": { + "type": "string" + }, + "tags": { + "type": "array" + }, + "source": { + "type": "string" + }, + "at": { + "type": "string" + }, + "task_location": { + "type": "object", + "properties": { + "list_id": { "type": "string" }, - "task_location": { - "type": "object", - "properties": { - "list_id": { - "type": "string" - }, - "folder_id": { - "type": "string" - }, - "space_id": { - "type": "string" - } - } + "folder_id": { + "type": "string" }, - "task_url": { + "space_id": { "type": "string" } + } + }, + "task_url": { + "type": "string" + } } -} +} \ No newline at end of file From 5661eabcce18f68fcab0b1e4ae9ed6f712736eab Mon Sep 17 00:00:00 2001 From: Mason Date: Fri, 22 Mar 2024 13:58:45 -0400 Subject: [PATCH 06/19] Super weird, but when we don't have access to the task related to the entry we get a '0' value. --- tap_clickup/schemas/time_entries.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tap_clickup/schemas/time_entries.json b/tap_clickup/schemas/time_entries.json index 2cf61bd..893a6a2 100644 --- a/tap_clickup/schemas/time_entries.json +++ b/tap_clickup/schemas/time_entries.json @@ -5,7 +5,7 @@ "type": "string" }, "task": { - "type": "object", + "type": ["object", "string"], "properties": { "id": { "type": "string" From 0e39e9cf528b4ffc22b63686c1ce625a739e5a61 Mon Sep 17 00:00:00 2001 From: Mason Date: Fri, 22 Mar 2024 14:22:15 -0400 Subject: [PATCH 07/19] task custom type can be a number --- tap_clickup/schemas/time_entries.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tap_clickup/schemas/time_entries.json b/tap_clickup/schemas/time_entries.json index 893a6a2..9b80f27 100644 --- a/tap_clickup/schemas/time_entries.json +++ b/tap_clickup/schemas/time_entries.json @@ -34,7 +34,7 @@ } }, "custom_type": { - "type": "null" + "type": ["null", "integer"] } } }, From ae8d960789f3e7701c03a3b64b193de4a30e9da2 Mon Sep 17 00:00:00 2001 From: Mason Date: Fri, 22 Mar 2024 14:27:48 -0400 Subject: [PATCH 08/19] list ids are nullable --- tap_clickup/schemas/time_entries.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tap_clickup/schemas/time_entries.json b/tap_clickup/schemas/time_entries.json index 9b80f27..abf9976 100644 --- a/tap_clickup/schemas/time_entries.json +++ b/tap_clickup/schemas/time_entries.json @@ -98,7 +98,7 @@ "type": "object", "properties": { "list_id": { - "type": "string" + "type": ["string", "null"] }, "folder_id": { "type": "string" From b208d6892895c55e682f06038894fbc8394a0a96 Mon Sep 17 00:00:00 2001 From: Mason Date: Fri, 22 Mar 2024 14:32:09 -0400 Subject: [PATCH 09/19] Folder id nullable --- tap_clickup/schemas/time_entries.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tap_clickup/schemas/time_entries.json b/tap_clickup/schemas/time_entries.json index abf9976..8ae5b67 100644 --- a/tap_clickup/schemas/time_entries.json +++ b/tap_clickup/schemas/time_entries.json @@ -101,7 +101,7 @@ "type": ["string", "null"] }, "folder_id": { - "type": "string" + "type": ["string", "null"] }, "space_id": { "type": "string" From c0d378365e28faf955871519066638c9a9462770 Mon Sep 17 00:00:00 2001 From: Mason Date: Fri, 22 Mar 2024 14:49:40 -0400 Subject: [PATCH 10/19] Another null field. It seems this may be a bug with the clickup api. I submitted a bug report to clickup --- tap_clickup/schemas/time_entries.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tap_clickup/schemas/time_entries.json b/tap_clickup/schemas/time_entries.json index 8ae5b67..2ad29eb 100644 --- a/tap_clickup/schemas/time_entries.json +++ b/tap_clickup/schemas/time_entries.json @@ -104,7 +104,7 @@ "type": ["string", "null"] }, "space_id": { - "type": "string" + "type": ["string", "null"] } } }, From cd3a7fe9e4c6567f45e2f0e7cba808d671e8af13 Mon Sep 17 00:00:00 2001 From: Mason Date: Fri, 22 Mar 2024 17:27:18 -0400 Subject: [PATCH 11/19] Pass assignees via setting / conf --- meltano.yml | 2 ++ tap_clickup/streams.py | 4 +++- tap_clickup/tap.py | 8 ++++++++ 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/meltano.yml b/meltano.yml index 8b6c32c..47a1dbd 100644 --- a/meltano.yml +++ b/meltano.yml @@ -13,6 +13,8 @@ plugins: settings: - name: api_token kind: password + - name: time_entry_assignees + kind: string # config: select: - '!shared_hierarchy.*' diff --git a/tap_clickup/streams.py b/tap_clickup/streams.py index 70f0fac..529606b 100644 --- a/tap_clickup/streams.py +++ b/tap_clickup/streams.py @@ -45,7 +45,9 @@ def get_url_params( ) -> Dict[str, Any]: """Return a dictionary of values to be used in URL parameterization.""" params = super().get_url_params(context, next_page_token) - params["assignee"] = ",".join(context["user_ids"]) + # TODO: Use env vars for these otherwise use the values as we do below + params["assignee"] = self.config.get("time_entry_assignees") + # params["assignee"] = ",".join(context["user_ids"]) return params diff --git a/tap_clickup/tap.py b/tap_clickup/tap.py index 1c371b3..ef4b0f4 100644 --- a/tap_clickup/tap.py +++ b/tap_clickup/tap.py @@ -46,6 +46,14 @@ class TapClickUp(Tap): config_jsonschema = th.PropertiesList( th.Property( "api_token", th.StringType, required=True, description="Example: 'pk_12345" + ), + th.Property( + "time_entry_assignees", + th.StringType, + required=False, + description="""By default, the extractor will get all user ids from your + team and use them when fetching time entries. If you want to fetch time entries + assigned to specific users, provide a comma-separated list of user IDs here. Ex. '420230,452346,784219'""" ), # Removing "official" start_date support re https://github.com/AutoIDM/tap-clickup/issues/118 # th.Property( From bef907059f1b8d061bff54a6cd3dbb225354f434 Mon Sep 17 00:00:00 2001 From: Mason Date: Fri, 22 Mar 2024 17:57:46 -0400 Subject: [PATCH 12/19] Allow for time_entry start date to know when to start fetching time entries --- tap_clickup/streams.py | 14 +++++++++++--- tap_clickup/tap.py | 10 +++++++++- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/tap_clickup/streams.py b/tap_clickup/streams.py index 529606b..485c903 100644 --- a/tap_clickup/streams.py +++ b/tap_clickup/streams.py @@ -1,5 +1,6 @@ """Stream type classes for tap-clickup.""" from pathlib import Path +from time import mktime, strptime from typing import Optional, Any, Dict import requests from singer_sdk.helpers.jsonpath import extract_jsonpath @@ -45,9 +46,16 @@ def get_url_params( ) -> Dict[str, Any]: """Return a dictionary of values to be used in URL parameterization.""" params = super().get_url_params(context, next_page_token) - # TODO: Use env vars for these otherwise use the values as we do below - params["assignee"] = self.config.get("time_entry_assignees") - # params["assignee"] = ",".join(context["user_ids"]) + + if "time_entry_start_date" in self.config: + # Formatted in ISO 8601, it must now be converted to milliseconds + print("MY TEST DATE", self.config["time_entry_start_date"]) + start_date_in_ms = int(mktime(strptime(self.config["time_entry_start_date"], "%Y-%m-%d").timetuple()) * 1000) + params["start_date"] = start_date_in_ms + if "time_entry_assignees" in self.config: + params["assignee"] = self.config["time_entry_assignees"] + else: + params["assignee"] = ",".join(context["user_ids"]) return params diff --git a/tap_clickup/tap.py b/tap_clickup/tap.py index ef4b0f4..1b836b0 100644 --- a/tap_clickup/tap.py +++ b/tap_clickup/tap.py @@ -47,7 +47,7 @@ class TapClickUp(Tap): th.Property( "api_token", th.StringType, required=True, description="Example: 'pk_12345" ), - th.Property( + th.Property( "time_entry_assignees", th.StringType, required=False, @@ -55,6 +55,14 @@ class TapClickUp(Tap): team and use them when fetching time entries. If you want to fetch time entries assigned to specific users, provide a comma-separated list of user IDs here. Ex. '420230,452346,784219'""" ), + th.Property( + "time_entry_start_date", + th.StringType, + required=False, + description="""The start date that determines how far back in time the extractor gets time entries. + Without this, only the last thirty days of time entries will be fetched. + Ex. '2023-01-01T00:00:00Z' to follow singer date format.""" + ), # Removing "official" start_date support re https://github.com/AutoIDM/tap-clickup/issues/118 # th.Property( # "start_date", From efdf2fec59dd30662b423cf07d07dc8d5e63e768 Mon Sep 17 00:00:00 2001 From: Mason Date: Fri, 22 Mar 2024 18:03:26 -0400 Subject: [PATCH 13/19] Add new conf to meltano.yml --- meltano.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/meltano.yml b/meltano.yml index 47a1dbd..290976d 100644 --- a/meltano.yml +++ b/meltano.yml @@ -14,6 +14,8 @@ plugins: - name: api_token kind: password - name: time_entry_assignees + kind: string + - name: time_entry_start_date kind: string # config: select: From e18b6322c6f4fa2529533f0fe7735cb9e1da6160 Mon Sep 17 00:00:00 2001 From: Mason Date: Fri, 22 Mar 2024 18:13:22 -0400 Subject: [PATCH 14/19] Fix conversion issue --- tap_clickup/streams.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tap_clickup/streams.py b/tap_clickup/streams.py index 485c903..87d9afb 100644 --- a/tap_clickup/streams.py +++ b/tap_clickup/streams.py @@ -1,6 +1,6 @@ """Stream type classes for tap-clickup.""" from pathlib import Path -from time import mktime, strptime +from datetime import strptime from typing import Optional, Any, Dict import requests from singer_sdk.helpers.jsonpath import extract_jsonpath @@ -49,9 +49,9 @@ def get_url_params( if "time_entry_start_date" in self.config: # Formatted in ISO 8601, it must now be converted to milliseconds - print("MY TEST DATE", self.config["time_entry_start_date"]) - start_date_in_ms = int(mktime(strptime(self.config["time_entry_start_date"], "%Y-%m-%d").timetuple()) * 1000) - params["start_date"] = start_date_in_ms + start_date = strptime(self.config["time_entry_start_date"], "%Y-%m-%dT%H:%M:%SZ") + # Convert the datetime object to milliseconds + params["start_date"] = int(start_date.timestamp() * 1000) if "time_entry_assignees" in self.config: params["assignee"] = self.config["time_entry_assignees"] else: From f1ab1ce1429c6d9be07b5bbd2a6b7ed09073925e Mon Sep 17 00:00:00 2001 From: Mason Date: Fri, 22 Mar 2024 18:17:12 -0400 Subject: [PATCH 15/19] Bad import --- tap_clickup/streams.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tap_clickup/streams.py b/tap_clickup/streams.py index 87d9afb..28246dd 100644 --- a/tap_clickup/streams.py +++ b/tap_clickup/streams.py @@ -1,6 +1,6 @@ """Stream type classes for tap-clickup.""" from pathlib import Path -from datetime import strptime +from time import strptime from typing import Optional, Any, Dict import requests from singer_sdk.helpers.jsonpath import extract_jsonpath From 061b55c571b3c52c158826706b482c0a1ad899c8 Mon Sep 17 00:00:00 2001 From: Mason Date: Fri, 22 Mar 2024 18:18:58 -0400 Subject: [PATCH 16/19] Take 59 with format --- tap_clickup/streams.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tap_clickup/streams.py b/tap_clickup/streams.py index 28246dd..9186672 100644 --- a/tap_clickup/streams.py +++ b/tap_clickup/streams.py @@ -1,6 +1,6 @@ """Stream type classes for tap-clickup.""" +from datetime import datetime from pathlib import Path -from time import strptime from typing import Optional, Any, Dict import requests from singer_sdk.helpers.jsonpath import extract_jsonpath @@ -49,7 +49,7 @@ def get_url_params( if "time_entry_start_date" in self.config: # Formatted in ISO 8601, it must now be converted to milliseconds - start_date = strptime(self.config["time_entry_start_date"], "%Y-%m-%dT%H:%M:%SZ") + start_date = datetime.strptime(self.config["time_entry_start_date"], "%Y-%m-%dT%H:%M:%SZ") # Convert the datetime object to milliseconds params["start_date"] = int(start_date.timestamp() * 1000) if "time_entry_assignees" in self.config: From 9aa2d07fc6f21ce070c82c58774da2d196c63ae9 Mon Sep 17 00:00:00 2001 From: Mason Date: Fri, 22 Mar 2024 18:28:57 -0400 Subject: [PATCH 17/19] Attempt to use 'at' field from time entry as replication key --- tap_clickup/streams.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tap_clickup/streams.py b/tap_clickup/streams.py index 9186672..2f49ab2 100644 --- a/tap_clickup/streams.py +++ b/tap_clickup/streams.py @@ -35,7 +35,7 @@ class TimeEntries(ClickUpStream): name = "time_entries" path = "/team/{team_id}/time_entries" primary_keys = ["id"] - replication_key = None + replication_key = "at" schema_filepath = SCHEMAS_DIR / "time_entries.json" records_jsonpath = "$.data[*]" parent_stream_type = TeamsStream From cc47a89f55c0ad76c835df034ac5251df0c24945 Mon Sep 17 00:00:00 2001 From: mlahp7 <72365595+mlahp7@users.noreply.github.com> Date: Fri, 17 May 2024 16:29:34 -0400 Subject: [PATCH 18/19] Feedback tweaks (#1) * Per reviewer suggestion, use replication key as query for start date * Bring back time_entry_start_date, in the case someone is just starting fresh, they would need to enter a date for the very first sync. The at field is set in milliseconds as the replication_key, we will plug that in when it is provided in the state * Bring back if statement for initial execution falling back to config date * Formatting --- README.md | 16 +++++++++------- tap_clickup/streams.py | 13 +++++++++---- tap_clickup/tap.py | 5 +++-- 3 files changed, 21 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index aae5685..9abd666 100644 --- a/README.md +++ b/README.md @@ -30,13 +30,15 @@ Want to become a sponsor? Reach out to us at [autoidm.com](https://autoidm.com) ## Settings -| Setting | Required | Default | Description | -|:--------------------|:--------:|:-------:|:------------| -| api_token | True | None | Example: 'pk_12345 | -| stream_maps | False | None | Config object for stream maps capability. | -| stream_map_config | False | None | User-defined config values to be used within map expressions. | -| flattening_enabled | False | None | 'True' to enable schema flattening and automatically expand nested properties. | -| flattening_max_depth| False | None | The max depth to flatten schemas. | +| Setting | Required | Default | Description | +|:----------------------|:--------:|:-------:|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| api_token | True | None | Example: 'pk_12345 | +| time_entry_assignees | False | None | By default, the extractor will get all user ids from your team and use them when fetching time entries. If you want to fetch time entries assigned to specific users, provide a comma-separated list of user IDs here. Ex. '420230,452346,784219' | +| time_entry_start_date | False | None | The start date that determines how far back in time the extractor gets time entries. Without this, only the last thirty days of time entries will be fetched. After the initial run, this value will be ignored in favor of the state, using the replication_key of 'at' to determine the start date. Ex. '2023-01-01T00:00:00Z' to follow singer date format. | +| stream_maps | False | None | Config object for stream maps capability. | +| stream_map_config | False | None | User-defined config values to be used within map expressions. | +| flattening_enabled | False | None | 'True' to enable schema flattening and automatically expand nested properties. | +| flattening_max_depth | False | None | The max depth to flatten schemas. | A full list of supported settings and capabilities is available by running: `tap-clickup --about` diff --git a/tap_clickup/streams.py b/tap_clickup/streams.py index 2f49ab2..13e29fd 100644 --- a/tap_clickup/streams.py +++ b/tap_clickup/streams.py @@ -21,7 +21,8 @@ class TeamsStream(ClickUpStream): def get_child_context(self, record: dict, context: Optional[dict]) -> dict: """Return a context dictionary for child streams.""" - user_ids = [str(member.get("user", {}).get("id")) for member in record.get("members", []) if isinstance(member, dict)] + user_ids = [str(member.get("user", {}).get("id")) for member in record.get("members", []) if + isinstance(member, dict)] return { "team_id": record["id"], @@ -41,17 +42,21 @@ class TimeEntries(ClickUpStream): parent_stream_type = TeamsStream # TODO not clear why this is needed partitions = None + def get_url_params( self, context: Optional[dict], next_page_token: Optional[Any] ) -> Dict[str, Any]: """Return a dictionary of values to be used in URL parameterization.""" params = super().get_url_params(context, next_page_token) - if "time_entry_start_date" in self.config: - # Formatted in ISO 8601, it must now be converted to milliseconds + state_based_date = self.get_starting_replication_key_value(context) + # In the case of the first run, we need to use the start date from the config + if not state_based_date: start_date = datetime.strptime(self.config["time_entry_start_date"], "%Y-%m-%dT%H:%M:%SZ") - # Convert the datetime object to milliseconds params["start_date"] = int(start_date.timestamp() * 1000) + else: + # Because the state date is already in milliseconds, we can just use it + params["start_date"] = state_based_date if "time_entry_assignees" in self.config: params["assignee"] = self.config["time_entry_assignees"] else: diff --git a/tap_clickup/tap.py b/tap_clickup/tap.py index 1b836b0..438c3e2 100644 --- a/tap_clickup/tap.py +++ b/tap_clickup/tap.py @@ -60,8 +60,9 @@ class TapClickUp(Tap): th.StringType, required=False, description="""The start date that determines how far back in time the extractor gets time entries. - Without this, only the last thirty days of time entries will be fetched. - Ex. '2023-01-01T00:00:00Z' to follow singer date format.""" + Without this, only the last thirty days of time entries will be fetched. After the initial run, + this value will be ignored in favor of the state, using the replication_key of 'at' to determine the + start date. Ex. '2023-01-01T00:00:00Z' to follow singer date format.""" ), # Removing "official" start_date support re https://github.com/AutoIDM/tap-clickup/issues/118 # th.Property( From 527af20e629940276c829642c97c2885802b0466 Mon Sep 17 00:00:00 2001 From: Mason Date: Fri, 17 May 2024 16:34:22 -0400 Subject: [PATCH 19/19] Update current status of Time Entries data in the readme --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 9abd666..ffda6ce 100644 --- a/README.md +++ b/README.md @@ -74,8 +74,8 @@ Note that the most up to date information is located in tap_clickup/streams.py. - Table name: time_entries - Description: All time entries are pulled for every team. Currently only pulls the last 30 days of time_entries see https://github.com/AutoIDM/tap-clickup/issues/134 for an issue addressing this! - Primary key column(s): id -- Replicated fully or incrementally: Full -- Bookmark column(s): N/A +- Replicated fully or incrementally: Incremental +- Bookmark column(s): at. _Please note that you must set the start date in the config to get time entries older than 30 days._ - Link to API endpoint documentation: [Time Entries](https://jsapi.apiary.io/apis/clickup20/reference/0/time-tracking-legacy/get-time-entries-within-a-date-range.html) ### Folders