diff --git a/tap_github/repository_streams.py b/tap_github/repository_streams.py index 2e03497c..40a86502 100644 --- a/tap_github/repository_streams.py +++ b/tap_github/repository_streams.py @@ -2191,6 +2191,7 @@ class DependenciesStream(GitHubGraphqlStream): parent_stream_type = RepositoryStream state_partitioning_keys = ["repo_id"] ignore_parent_replication_key = True + do_not_paginate = False @property def http_headers(self) -> dict: @@ -2220,7 +2221,7 @@ def query(self) -> str: """Return dynamic GraphQL query.""" # Graphql id is equivalent to REST node_id. To keep the tap consistent, we rename "id" to "node_id". # Due to GrapQl nested-pagination limitations, we loop through the top level dependencyGraphManifests one by one. - return """ + initial_query = """ query repositoryDependencies($repo: String! $org: String! $nextPageCursor_0: String $nextPageCursor_1: String) { repository(name: $repo owner: $org) { dependencyGraphManifests (first: 1 withDependencies: true after: $nextPageCursor_0) { @@ -2263,9 +2264,25 @@ def query(self) -> str: cost } } - """ + if self.do_not_paginate: + no_pagination_query = initial_query.replace( + " $nextPageCursor_0: String $nextPageCursor_1: String", "" + ) + no_pagination_query = no_pagination_query.replace( + "after: $nextPageCursor_0", "" + ) + no_pagination_query = no_pagination_query.replace( + "after: $nextPageCursor_1", "" + ) + no_pagination_query = no_pagination_query.replace("first: 1", "first: 10") + no_pagination_query = no_pagination_query.replace("first: 50", "first: 100") + + return no_pagination_query + + return initial_query + schema = th.PropertiesList( # Parent Keys th.Property("repo", th.StringType), @@ -2296,6 +2313,12 @@ def query(self) -> str: ).to_dict() +class DependenciesStreamIncomplete(DependenciesStream): + """Defines 'DependenciesStreamDirty' stream to limit pagination.""" + + do_not_paginate = True + + class TrafficRestStream(GitHubRestStream): """Base class for Traffic Streams""" diff --git a/tap_github/streams.py b/tap_github/streams.py index e1b05e58..5f71d809 100644 --- a/tap_github/streams.py +++ b/tap_github/streams.py @@ -18,6 +18,7 @@ CommunityProfileStream, ContributorsStream, DependenciesStream, + DependenciesStreamIncomplete, DependentsStream, EventsStream, ExtraMetricsStream, @@ -75,6 +76,7 @@ def __init__(self, valid_queries: Set[str], streams: List[Type[Stream]]): CommunityProfileStream, ContributorsStream, DependenciesStream, + DependenciesStreamIncomplete, DependentsStream, EventsStream, IssueCommentsStream,