Develop catalogues

astroquery/eso/__init__.py

@@ -9,17 +9,21 @@ class Conf(_config.ConfigNamespace):
     """
     Configuration parameters for `astroquery.eso`.
     """
-
+    MAX_ROW_LIMIT = _config.ConfigItem(
+        15000000,
+        'Maximum number of rows allowed by the TAP service.')
     ROW_LIMIT = _config.ConfigItem(
         1000,
-        'Maximum number of rows returned (set to -1 for unlimited).')
+        'Maximum number of rows returned (set to -1 for maximum allowed via TAP service).')
     username = _config.ConfigItem(
         "",
         'Optional default username for ESO archive.')
-    tap_url = _config.ConfigItem(
+    tap_obs_url = _config.ConfigItem(
         "https://archive.eso.org/tap_obs",
-        'URL for TAP queries.')
-
+        'URL for TAP observation queries.')
+    tap_cat_url = _config.ConfigItem(
+        "https://archive.eso.org/tap_cat",
+        'URL for TAP catalogue queries.')
 
 conf = Conf()
 

astroquery/eso/core.py

@@ -19,6 +19,7 @@
 import time
 import warnings
 import xml.etree.ElementTree as ET
+from contextlib import contextmanager
 from typing import List, Optional, Tuple, Dict, Set, Union
 
 import astropy.utils.data
@@ -71,6 +72,7 @@ class _EsoNames:
     phase3_table = "ivoa.ObsCore"
     raw_instruments_column = "instrument"
     phase3_surveys_column = "obs_collection"
+    catalogue_schema = "safcat"
 
     @staticmethod
     def ist_table(instrument_name):
@@ -94,7 +96,7 @@ def wrapper(self, *args, **kwargs):
 
         tmpvar = self.ROW_LIMIT
         try:
-            self.ROW_LIMIT = sys.maxsize
+            self.ROW_LIMIT = conf.MAX_ROW_LIMIT
             result = func(self, *args, **kwargs)
         finally:
             self.ROW_LIMIT = tmpvar
@@ -134,16 +136,35 @@ def ROW_LIMIT(self, value):
         # type check
         if not (value is None or isinstance(value, int)):
             raise TypeError(f"ROW_LIMIT attribute must be of type int or None; found {type(value)}")
+        elif value > conf.MAX_ROW_LIMIT:
+            raise ValueError(f"ROW_LIMIT cannot be higher than {conf.MAX_ROW_LIMIT}; found {value}")
 
         if value is None or value < 1:
-            mr = sys.maxsize
+            mr = conf.MAX_ROW_LIMIT
         else:
             mr = value
 
         self._ROW_LIMIT = mr
 
-    def _tap_url(self) -> str:
-        url = conf.tap_url
+    @contextmanager
+    def _temporary_row_limit(self, row_limit: Optional[int]):
+        if row_limit is None:
+            yield
+            return
+        tmpvar = self.ROW_LIMIT
+        try:
+            self.ROW_LIMIT = row_limit
+            yield
+        finally:
+            self.ROW_LIMIT = tmpvar
+
+    def _tap_url(self, which_tap: str = "tap_obs") -> str:
+        if which_tap == "tap_obs":
+            url = conf.tap_obs_url
+        elif which_tap == "tap_cat": 
+            url = conf.tap_cat_url
+        else:
+            raise ValueError("which_tap must be 'tap_obs' or 'tap_cat'.")
         return url
 
     def _authenticate(self, *, username: str, password: str) -> bool:
@@ -245,11 +266,12 @@ def _try_download_pyvo_table(self,
         table_with_an_extra_row = Table()
 
         def message(query_str):
+            which_tap = tap.baseurl.split("/")[-1]
             return (f"Error executing the following query:\n\n"
                     f"{query_str}\n\n"
                     "See examples here: https://archive.eso.org/tap_obs/examples\n\n"
                     f"For maximum query freedom use the query_tap method:\n\n"
-                    f' >>> Eso().query_tap( "{query_str}" )\n\n')
+                    f' >>> Eso().query_tap( "{query_str}", which_tap="{which_tap}")\n\n')
 
         try:
             row_limit_plus_one = self.ROW_LIMIT
@@ -267,7 +289,7 @@ def message(query_str):
 
         return table_with_an_extra_row[:self.ROW_LIMIT]
 
-    def tap(self, authenticated: bool = False) -> TAPService:
+    def tap(self, authenticated: bool = False, *, which_tap: str = "tap_obs") -> TAPService:
 
         if authenticated and not self.authenticated():
             raise LoginError(
@@ -279,32 +301,34 @@ def tap(self, authenticated: bool = False) -> TAPService:
                 "<eso_class_instance>.login(username=<your_username>"
             )
 
-        log.debug(f"Querying from {self._tap_url()}")
         if authenticated:
             h = self._get_auth_header()
             self._session.headers = {**self._session.headers, **h}
-            tap_service = TAPService(self._tap_url(), session=self._session)
+            tap_service = TAPService(self._tap_url(which_tap), session=self._session)
         else:
-            tap_service = TAPService(self._tap_url())
+            tap_service = TAPService(self._tap_url(which_tap))
 
         return tap_service
 
     def query_tap(self,
                   query: str, *,
                   authenticated: bool = False,
+                  which_tap: str = "tap_obs",
                   ) -> Table:
         """
         Query the ESO TAP service using a free ADQL string.
 
         Parameters
         ----------
-        query_str : str
+        query : str
             The ADQL query string to be executed.
         authenticated : bool, optional
             If ``True``, the query is run as an authenticated user.
             Authentication must be performed beforehand via
             :meth:`astroquery.eso.EsoClass.login`. Authenticated queries
             may be slower. Default is ``False``.
+        which_tap : {"tap_obs", "tap_cat"}, optional
+            TAP endpoint to query. Default is ``"tap_obs"``.
 
         Returns
         -------
@@ -317,7 +341,7 @@ def query_tap(self,
         eso_instance.query_tap("SELECT * FROM ivoa.ObsCore")
         """
         table_to_return = Table()
-        tap_service = self.tap(authenticated)
+        tap_service = self.tap(authenticated, which_tap=which_tap)
         table_to_return = self._try_download_pyvo_table(query, tap_service)
         return table_to_return
 
@@ -368,50 +392,97 @@ def list_surveys(self, *, cache=True) -> List[str]:
         return res
 
     @unlimited_maxrec
-    def list_column(self, table_name: str) -> None:
+    def _columns_table(self, table_name: str, *, which_tap: str = "tap_obs") -> Table:
+        if which_tap == "tap_obs":
+            help_query = (
+                "select column_name, datatype, unit, xtype "
+                f"from TAP_SCHEMA.columns where table_name = '{table_name}'")
+        else:
+            schema = _EsoNames.catalogue_schema
+            help_query = (
+                "select column_name, datatype, unit, ucd "
+                f"from TAP_SCHEMA.columns "
+                f"where table_name = '{table_name.removeprefix(schema+'.')}'")
+        return self.query_tap(help_query, which_tap=which_tap)
+
+    @unlimited_maxrec
+    def _list_column(self, table_name: str, *, which_tap: str = "tap_obs") -> Optional[Table]:
         """
-        Prints the columns contained in a given table
+        Prints the columns contained in a given table.
         """
-        help_query = (
-            f"select column_name, datatype, xtype, unit "
-            # TODO: The column description renders output unmanageable
-            # f", description "
-            f"from TAP_SCHEMA.columns "
-            f"where table_name = '{table_name}'")
-        available_cols = self.query_tap(help_query)
+        available_cols = self._columns_table(table_name, which_tap=which_tap)
 
         count_query = f"select count(*) from {table_name}"
-        num_records = list(self.query_tap(count_query)[0].values())[0]
+        num_records = list(self.query_tap(count_query, which_tap=which_tap)[0].values())[0]
 
         with (astropy.conf.set_temp(
                 "max_lines", len(available_cols) + 2),
                 astropy.conf.set_temp(
                 "max_width", sys.maxsize)):
-            log.info(f"\nColumns present in the table {table_name}:\n{available_cols}\n"
+            log.info(
+                    f"\nColumns present in the table {table_name}:\n{available_cols}\n"
                      f"\nNumber of records present in the table {table_name}:\n{num_records}\n")
 
+    @unlimited_maxrec
+    def list_catalogues(self, all_versions: bool = False, cache: bool = True) -> List[str]:
+        """
+        List available catalogue tables offered by the ESO archive.
+
+        Parameters
+        ----------
+        all_versions : bool, optional
+            If True, list all versions of each catalogue table. If False (default),
+            return only the latest version of each catalogue (as determined by the
+            TAP_CAT metadata query).
+        cache : bool, optional
+            Deprecated and unused. Retained for backwards compatibility.
+
+        Returns
+        -------
+        list[str]
+            List of catalogue table names.
+        """
+        _ = cache  # We're aware about disregarding the argument
+        schema = _EsoNames.catalogue_schema
+
+        query_str = (f"SELECT table_name FROM TAP_SCHEMA.tables as ref "
+                        "LEFT OUTER JOIN TAP_SCHEMA.keys AS k ON ref.table_name = k.from_table "
+                        "LEFT OUTER JOIN TAP_SCHEMA.key_columns AS kc ON k.key_id = kc.key_id "
+                        f"WHERE schema_name='{schema}' ")
+
+        if not all_versions:
+            query_str += ("AND cat_id IN ( "
+                            "SELECT t1.cat_id "
+                            "FROM TAP_SCHEMA.tables t1 "
+                            "LEFT JOIN TAP_SCHEMA.tables t2 ON (t1.title = t2.title AND t1.version < t2.version) "
+                            "WHERE t2.title IS NULL)")
+
+        res = self.query_tap(query_str, which_tap="tap_cat")
+        return list(res["table_name"])
+
     def _query_on_allowed_values(
         self,
-        user_params: _UserParams
+        user_params: _UserParams,
+        which_tap="tap_obs",
     ) -> Union[Table, int, str, None]:
         if user_params.print_help:
-            self.list_column(user_params.table_name)
-            return
+            return self._list_column(user_params.table_name,
+                                     which_tap=user_params.which_tap)
 
         _raise_if_has_deprecated_keys(user_params.column_filters)
 
         raise_if_coords_not_valid(user_params.cone_ra, user_params.cone_dec, user_params.cone_radius)
 
-        query = _build_adql_string(user_params)
+        query_str = _build_adql_string(user_params)
 
         if user_params.get_query_payload:
-            return query
+            return query_str
 
-        ret_table = self.query_tap(query=query, authenticated=user_params.authenticated)
+        ret_table = self.query_tap(query_str, which_tap=which_tap, authenticated=user_params.authenticated)
         return list(ret_table[0].values())[0] if user_params.count_only else ret_table
 
     @deprecated_renamed_argument(('open_form', 'cache'), (None, None),
-                                 since=['0.4.12', '0.4.12'])
+                                 since=['0.4.12', '0.4.12'])  
     def query_surveys(
             self,
             surveys: Union[List[str], str] = None, *,
@@ -496,7 +567,7 @@ def query_surveys(
                                   print_help=help,
                                   authenticated=authenticated,
                                   )
-        t = self._query_on_allowed_values(user_params=user_params)
+        t = self._query_on_allowed_values(user_params)
         t = _reorder_columns(t, DEFAULT_LEAD_COLS_PHASE3)
         return t
 
@@ -1052,5 +1123,102 @@ def query_apex_quicklooks(self,
                                   authenticated=authenticated)
         return self._query_on_allowed_values(user_params)
 
+    def query_catalogue(
+            self,
+            catalogue: str, *,
+            cone_ra: float = None, cone_dec: float = None, cone_radius: float = None,
+            columns: Union[List, str] = None,
+            column_filters: Optional[dict] = None,
+            top: int = None,
+            count_only: bool = False,
+            get_query_payload: bool = False,
+            help: bool = False,
+            authenticated: bool = False,
+            open_form: bool = False, cache: bool = False,
+            ROW_LIMIT: Optional[int] = None,
+    ) -> Union[Table, int, str]:
+        """
+        Query catalogue data contained in the ESO archive.
+
+        Parameters
+        ----------
+        catalogue : str
+            Name of the catalogue to query. Should be ONLY ONE of the names
+            returned by :meth:`~astroquery.eso.EsoClass.list_catalogues`.
+        cone_ra : float, optional
+            Not yet implemented.
+            Cone Search Center - Right Ascension in degrees.
+        cone_dec : float, optional
+            Not yet implemented.
+            Cone Search Center - Declination in degrees.
+        cone_radius : float, optional
+            Not yet implemented.
+            Cone Search Radius in degrees.
+        columns : str or list of str, optional
+            Name of the columns the query should return. If specified as a string,
+            it should be a comma-separated list of column names.
+        top : int, optional
+            When set to ``N``, returns only the top ``N`` records.
+        count_only : bool, optional
+            If ``True``, returns only an ``int``: the count of the records
+            the query would return when set to ``False``. Default is ``False``.
+        get_query_payload : bool, optional
+            If ``True``, returns only a ``str``: the query string that
+            would be issued to the TAP service. Default is ``False``.
+        help : bool, optional
+            If ``True``, prints all the parameters accepted in ``column_filters``
+            and ``columns``. Default is ``False``.
+        authenticated : bool, optional
+            If ``True``, runs the query as an authenticated user.
+            Authentication must be done beforehand via
+            :meth:`~astroquery.eso.EsoClass.login`. Note that authenticated queries
+            are slower. Default is ``False``.
+        column_filters : dict or None, optional
+            Constraints applied to the query in ADQL syntax,
+            e.g., ``{"mag": "< 20"}``.
+            Default is ``None``.
+        open_form : bool, optional
+            **Deprecated** - unused.
+        cache : bool, optional
+            **Deprecated** - unused.
+        ROW_LIMIT : int, optional
+            Overrides the configured (eso.ROW_LIMIT) row limit for this query only.
+
+        Returns
+        -------
+        astropy.table.Table, str, int, or None
+            - By default, returns an :class:`~astropy.table.Table` containing records
+              based on the specified columns and constraints. Returns ``None`` if no results.
+            - When ``count_only`` is ``True``, returns an ``int`` representing the
+              record count for the specified filters.
+            - When ``get_query_payload`` is ``True``, returns the query string that
+              would be issued to the TAP service given the specified arguments.
+        """
+        _ = open_form, cache  # make explicit that we are aware these arguments are unused
+        column_filters = column_filters if column_filters else {}
+
+        schema = _EsoNames.catalogue_schema
+        table_name = f"{schema}.{catalogue}"
+
+        with self._temporary_row_limit(ROW_LIMIT):
+            which_tap = "tap_cat"
+            user_params = _UserParams(table_name=table_name,
+                                      column_name=None,
+                                      allowed_values=None,
+                                      cone_ra=None,
+                                      cone_dec=None,
+                                      cone_radius=None,
+                                      columns=columns,
+                                      column_filters=column_filters,
+                                      top=top,
+                                      count_only=count_only,
+                                      get_query_payload=get_query_payload,
+                                      print_help=help,
+                                      authenticated=authenticated,
+                                      which_tap=which_tap
+                                      )
+
+            return self._query_on_allowed_values(user_params, which_tap=which_tap)
+
 
 Eso = EsoClass()