From c2437ce44ecc5eaee809eb709e3da33142080241 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mindaugas=20Ni=C5=BEauskas?= Date: Thu, 4 Jul 2024 09:38:55 +0300 Subject: [PATCH 1/4] create CustomPropertiesStream class --- tap_github/repository_streams.py | 22 ++++++++++++++++++++++ tap_github/streams.py | 2 ++ 2 files changed, 24 insertions(+) diff --git a/tap_github/repository_streams.py b/tap_github/repository_streams.py index de8371e8..2766665a 100644 --- a/tap_github/repository_streams.py +++ b/tap_github/repository_streams.py @@ -2441,3 +2441,25 @@ class TrafficPageViewsStream(TrafficRestStream): th.Property("count", th.IntegerType), th.Property("uniques", th.IntegerType), ).to_dict() + + +class CustomPropertiesStream(RepositoryStream): + """Defines 'custom_properties' stream.""" + + name = "custom_properties" + path = "/repos/{org}/{repo}/properties/values" + primary_keys = ["property_name"] + parent_stream_type = RepositoryStream + ignore_parent_replication_key = False + state_partitioning_keys = ["repo", "org"] + selected_by_default = True + + schema = th.PropertiesList( + # Parent Keys + th.Property("repo", th.StringType), + th.Property("org", th.StringType), + th.Property("repo_id", th.IntegerType), + # Custom Property Keys + th.Property("property_name", th.StringType), + th.Property("value", th.StringType), + ).to_dict() \ No newline at end of file diff --git a/tap_github/streams.py b/tap_github/streams.py index e1b05e58..a0690b9b 100644 --- a/tap_github/streams.py +++ b/tap_github/streams.py @@ -17,6 +17,7 @@ CommitsStream, CommunityProfileStream, ContributorsStream, + CustomPropertiesStream, DependenciesStream, DependentsStream, EventsStream, @@ -74,6 +75,7 @@ def __init__(self, valid_queries: Set[str], streams: List[Type[Stream]]): CommitsStream, CommunityProfileStream, ContributorsStream, + CustomPropertiesStream, DependenciesStream, DependentsStream, EventsStream, From 65e002e4528d4853c4377a4bb08a184d7464d621 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mindaugas=20Ni=C5=BEauskas?= Date: Thu, 4 Jul 2024 09:45:16 +0300 Subject: [PATCH 2/4] add empty line --- tap_github/repository_streams.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tap_github/repository_streams.py b/tap_github/repository_streams.py index 2766665a..4dce9a0e 100644 --- a/tap_github/repository_streams.py +++ b/tap_github/repository_streams.py @@ -2462,4 +2462,4 @@ class CustomPropertiesStream(RepositoryStream): # Custom Property Keys th.Property("property_name", th.StringType), th.Property("value", th.StringType), - ).to_dict() \ No newline at end of file + ).to_dict() From 14ec150025171a9107dc59c80bc3de61a24097b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mindaugas=20Ni=C5=BEauskas?= Date: Thu, 4 Jul 2024 09:47:10 +0300 Subject: [PATCH 3/4] remove selected_by_default --- tap_github/repository_streams.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tap_github/repository_streams.py b/tap_github/repository_streams.py index 4dce9a0e..17915595 100644 --- a/tap_github/repository_streams.py +++ b/tap_github/repository_streams.py @@ -2452,7 +2452,6 @@ class CustomPropertiesStream(RepositoryStream): parent_stream_type = RepositoryStream ignore_parent_replication_key = False state_partitioning_keys = ["repo", "org"] - selected_by_default = True schema = th.PropertiesList( # Parent Keys From 8ada871fa87c5bb40300405804564126b749849c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mindaugas=20Ni=C5=BEauskas?= Date: Thu, 4 Jul 2024 10:19:37 +0300 Subject: [PATCH 4/4] set replication_key = None --- tap_github/repository_streams.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tap_github/repository_streams.py b/tap_github/repository_streams.py index 17915595..d2942695 100644 --- a/tap_github/repository_streams.py +++ b/tap_github/repository_streams.py @@ -2443,14 +2443,15 @@ class TrafficPageViewsStream(TrafficRestStream): ).to_dict() -class CustomPropertiesStream(RepositoryStream): +class CustomPropertiesStream(GitHubRestStream): """Defines 'custom_properties' stream.""" name = "custom_properties" path = "/repos/{org}/{repo}/properties/values" - primary_keys = ["property_name"] + primary_keys = ["repo", "org", "property_name"] + replication_key = None parent_stream_type = RepositoryStream - ignore_parent_replication_key = False + ignore_parent_replication_key = True state_partitioning_keys = ["repo", "org"] schema = th.PropertiesList(