Galaxy server clean-up script along with some changes to galaxykit to support it

galaxykit/client.py

@@ -159,14 +159,18 @@ def _refresh_jwt_token(self):
             "User-Agent": user_agent(),
             "Content-Type": "application/x-www-form-urlencoded",
         }
-        json = self._http(
+        json, resp = self._http(
             "post",
             self.auth_url,
             data=payload,
             headers=headers,
+            include_response=True,
         )
-        self.token = json["access_token"]
-        self.token_type = "Bearer"
+        try:
+            self.token = json["access_token"]
+            self.token_type = "Bearer"
+        except KeyError:
+            raise GalaxyClientError("Unexpected error in JWT token refresh.", resp)
 
     def _update_auth_headers(self):
         self.headers.update(
@@ -187,6 +191,10 @@ def _http(self, method, path, *args, **kwargs):
         url = urljoin(self.galaxy_root, path)
         headers = kwargs.pop("headers", self.headers)
         parse_json = kwargs.pop("parse_json", True)
+        include_response = kwargs.pop("include_response", False)
+
+        if include_response and not parse_json:
+            raise ValueError("GalaxyClient._http() called with include_response=True only valid when parse_json=True!")
 
         resp = requests.request(
             method, url, headers=headers, verify=self.https_verify, *args, **kwargs
@@ -209,9 +217,13 @@ def _http(self, method, path, *args, **kwargs):
                 raise ValueError("Failed to parse JSON response from API") from exc
             if "errors" in json:
                 raise GalaxyClientError(resp, *json["errors"])
+
             if resp.status_code >= 400:
                 raise GalaxyClientError(resp, resp.status_code)
-            return json
+            elif include_response:
+                return json, resp
+            else:
+                return json
         else:
             if resp.status_code >= 400:
                 raise GalaxyClientError(resp, resp.status_code)
@@ -234,6 +246,8 @@ def _payload(self, method, path, body, *args, **kwargs):
         return self._http(method, path, *args, **kwargs)
 
     def get(self, path, *args, **kwargs):
+        if args:
+            kwargs["params"], *args = args
         return self._http("get", path, *args, **kwargs)
 
     def post(self, *args, **kwargs):

galaxykit/collections.py

@@ -4,6 +4,7 @@
 
 import uuid
 import os
+import sys
 import json
 from time import sleep
 from urllib.parse import urljoin
@@ -24,9 +25,23 @@ def get_collection(client, namespace, collection_name, version):
     return client.get(collection_url)
 
 
-def get_collection_list(client):
-    url = "_ui/v1/collection-versions/?limit=999999"
-    return client.get(url)
+def get_collection_list(client, repo="published", limit=None, offset=None, keywords=None):
+    # url = "_ui/v1/collection-versions/?limit=999999"
+    url = f"_ui/v1/repo/{repo}/"
+    params = {}
+
+    if limit is not None:
+        params["limit"] = limit
+    if offset is not None:
+        params["offset"] = offset
+
+    if isinstance(keywords, str):
+        keywords = [keywords]
+    elif keywords is None:
+        keywords = []
+    if keywords:
+        params["keywords"] = keywords
+    return client.get(url, params)
 
 
 def upload_test_collection(
@@ -201,19 +216,75 @@ def move_or_copy_collection(
 
 
 def delete_collection(
-    client, namespace, collection, version=None, repository="published"
+    client, namespace, collection, version=None, repositories=("published",), dependents=False
 ):
     """
     Delete collection version
     """
+    assert isinstance(repositories, (tuple, list))
     logger.debug(f"Deleting {collection} from {namespace} on {client.galaxy_root}")
-    if version == None:
-        delete_url = f"v3/plugin/ansible/content/{repository}/collections/index/{namespace}/{collection}/"
-    else:
-        delete_url = f"v3/plugin/ansible/content/{repository}/collections/index/{namespace}/{collection}/versions/{version}/"
-    resp = client.delete(delete_url)
-    wait_for_task(client, resp)
-    return resp
+
+    completed_repositories = set()
+
+    final_resp = {
+        "repositories": list(repositories),
+        "responses": [],
+        "delete_count": 0,
+    }
+
+    for repository in repositories:
+        if repository in completed_repositories:
+            continue
+        else:
+            completed_repositories.add(repository)
+
+        coll_name = f"{namespace}.{collection}"
+        if not version:
+            delete_url = f"v3/plugin/ansible/content/{repository}/collections/index/{namespace}/{collection}/"
+        else:
+            delete_url = f"v3/plugin/ansible/content/{repository}/collections/index/{namespace}/{collection}/versions/{version}/"
+            coll_name += f":{version}"
+        try:
+            resp = client.delete(delete_url)
+        except GalaxyClientError as e:
+            if e.response.status_code == 404:
+                logger.debug(f"Ignoring (maybe) already deleted {coll_name}", file=sys.stderr)
+                resp = {
+                    "collection": coll_name,
+                    "response_body": e.response.text,
+                    "response_status_code": e.response.status_code,
+                }
+            else:
+                raise
+        else:
+            if "dependent_collection_versions" in resp:
+                dep_count = len(resp["dependent_collection_versions"])
+                logger.debug(f"Deleting {dep_count} dependents first...")
+                if dependents:
+                    combined = {
+                        "original": None,
+                        "dependents": []
+                    }
+                    for dep in resp["dependent_collection_versions"]:
+                        dep_coll, dep_ver = dep.split(" ")
+                        dep_ns, dep_name = dep_coll.split(".")
+                        dep_resp = delete_collection(client, dep_ns, dep_name, dep_ver, repositories, dependents=True)
+                        combined["dependents"].append(dep_resp)
+                    resp = client.delete(delete_url)
+                    if "delete_count" in resp:
+                        final_resp["delete_count"] += resp["delete_count"]
+                    combined["original"] = resp
+                    return combined
+                else:
+                    raise GalaxyClientError(resp)
+
+            wait_for_task(client, resp)
+            final_resp["delete_count"] += 1
+        final_resp["responses"].append({
+            "collection": coll_name,
+            "response": resp,
+        })
+    return final_resp
 
 
 def deprecate_collection(client, namespace, collection, repository):

galaxykit/command.py

@@ -67,7 +67,34 @@ def report_error(resp):
         "ops": {
             "list": {
                 "help": "List all collections",
-                "args": None,
+                "args": {
+                    "repository": {
+                        "help": "Repository to list collections from (defaults to 'published')",
+                        "nargs": "?",
+                        "default": "published",
+                    },
+                    "limit": {
+                        "help": "Maximum number of collections to show.",
+                        "nargs": "?",
+                        "default": None,
+                        "short": "-l",
+                        "long": "--limit",
+                    },
+                    "offset": {
+                        "help": "Collection offset to begin list at. Used for paging.",
+                        "nargs": "?",
+                        "default": None,
+                        "short": "-o",
+                        "long": "--offset",
+                    },
+                    "keywords": {
+                        "help": "Filter by results that contain one or more keywords",
+                        "nargs": "*",
+                        "default": None,
+                        "short": "-k",
+                        "long": "--keywords",
+                    },
+                },
             },
             "upload": {
                 "help": "Create and upload a new collection",
@@ -117,7 +144,17 @@ def report_error(resp):
                     "namespace": {},
                     "collection": {},
                     "version": {"nargs": "?", "default": None},
-                    "repository": {"nargs": "?", "default": "published"},
+                    "repository": {
+                        "short": "-r",
+                        "long": "--repository",
+                        "default": "published,staging,rejected",
+                    },
+                    "dependents": {
+                        "short": "-d",
+                        "long": "--dependents",
+                        "default": False,
+                        "action": "store_true",
+                    },
                 },
             },
             "download": None,
@@ -489,7 +526,15 @@ def report_error(resp):
 
 def parse_args(parser, args):
     for arg in args:
-        parser.add_argument(arg, **(args[arg]))
+        flag_args=[]
+        if "short" in args[arg]:
+            flag_args.append("-" + args[arg].pop("short").strip("-"))
+        if "long" in args[arg]:
+            flag_args.append("--" + args[arg].pop("long").strip("-"))
+        if not flag_args:
+            flag_args.append(arg)
+
+        parser.add_argument(*flag_args, **(args[arg]))
 
 
 def parse_subop(subparsers, subop, subop_params):
@@ -1001,7 +1046,13 @@ def main():
 
         elif args.kind == "collection":
             if args.operation == "list":
-                print(json.dumps(collections.get_collection_list(client)))
+                resp = collections.get_collection_list(
+                    client,
+                    args.repository,
+                    keywords=args.keywords,
+                    limit=args.limit,
+                )
+                print(json.dumps(resp))
             elif args.operation == "upload":
                 namespace, collection_name, version, path = (
                     args.namespace or client.username,
@@ -1053,18 +1104,20 @@ def main():
                     "copy",
                 )
             elif args.operation == "delete":
-                namespace, collection, version, repository = (
+                namespace, collection, version, repository, dependents = (
                     args.namespace,
                     args.collection,
                     args.version,
-                    args.repository or "published",
+                    args.repository.split(","),
+                    args.dependents
                 )
                 try:
                     if version == "None":
                         version = None
                     resp = collections.delete_collection(
-                        client, namespace, collection, version, repository
+                        client, namespace, collection, version, repository, dependents
                     )
+                    print(json.dumps(resp))
                 except ValueError as e:
                     if not args.ignore:
                         logger.error(e)