diff --git a/CHANGES.rst b/CHANGES.rst index c6158ea041..1c150e927f 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -72,6 +72,32 @@ vizier - Methods ``get_catalog``, ``get_catalog_async`` and ``query_*`` now always return UCD1+ instead of UCD1. [#3458] +fermi +^^^^^ + +- The module now uses the new Fermi LAT Data Query REST API + (``https://fermi.gsfc.nasa.gov/ssc/data/access/lat/query/api/v1``), which is replacing the + ``LATDataQuery.cgi`` endpoint whose HTML responses were previously scraped + with regular expressions. The user-facing ``FermiLAT.query_object()`` + signature is unchanged. [#3647] +- ``FermiLAT.query_object_async()`` now returns the server-assigned + ``query_id`` instead of the URL of an HTML results page. [#3647] +- New methods ``FermiLAT.get_status()``, ``FermiLAT.list_results()``, + ``FermiLAT.wait_for_completion()`` and ``FermiLAT.get_file_urls()`` expose + the individual steps of the asynchronous query workflow. [#3647] +- New keyword arguments: ``zenithangle`` (maximum zenith angle in degrees) and + ``coordsystem`` (``'J2000'``, ``'B1950'`` or ``'Galactic'``). All-sky + queries (radius > 60 deg, observation window <= 24 h) are now supported. [#3647] +- ``GetFermilatDatafile`` and ``get_fermilat_datafile`` are deprecated; they + now delegate to ``FermiLAT.get_file_urls()`` and take a ``query_id`` rather + than a results-page URL. [#3647] +- The module no longer emits an "Experimental" warning on import. [#3647] +- Failed queries now surface the server's error message: an HTTP error + response with a JSON body such as ``{"error": "..."}`` is raised as a + ``RemoteServiceError`` carrying that message, rather than a bare status + code. [#3647] + + mast ^^^^ - ``utils.mast_relative_path`` is now deprecated in favor of ``utils.get_cloud_paths``. [#3488] diff --git a/astroquery/fermi/__init__.py b/astroquery/fermi/__init__.py index 8db47889ca..97eedef047 100644 --- a/astroquery/fermi/__init__.py +++ b/astroquery/fermi/__init__.py @@ -4,6 +4,10 @@ https://fermi.gsfc.nasa.gov https://fermi.gsfc.nasa.gov/ssc/data/ + +As of 2026, the Fermi LAT Data Server exposes a JSON REST API. This module +targets that API; the legacy ``LATDataQuery.cgi`` HTML-scraping path has been +removed. """ from astropy import config as _config @@ -14,11 +18,18 @@ class Conf(_config.ConfigNamespace): """ url = _config.ConfigItem( - 'https://fermi.gsfc.nasa.gov/cgi-bin/ssc/LAT/LATDataQuery.cgi', - 'Fermi query URL.') + 'https://fermi.gsfc.nasa.gov/ssc/data/access/lat/query/api/v1', + 'Base URL of the Fermi LAT Data Query REST API. Endpoints ' + '(/query, /query/{id}/status, /query/{id}/results) are appended to it.') + file_base_url = _config.ConfigItem( + # NOTE: the results endpoint returns bare filenames; download URLs are + # reconstructed against this staging area. Currently a /test/ path - + # revisit if the staging location changes at production cutover. + 'https://fermi.gsfc.nasa.gov/FTP/fermi/data/lat/test/queries/', + 'Base URL under which query result files are staged.') timeout = _config.ConfigItem( 60, - 'Time limit for connecting to Fermi server.') + 'Time limit for connecting to the Fermi server.') retrieval_timeout = _config.ConfigItem( 120, 'Time limit for retrieving a data file once it has been located.') @@ -26,13 +37,9 @@ class Conf(_config.ConfigNamespace): conf = Conf() -from .core import FermiLAT, FermiLATClass, GetFermilatDatafile, get_fermilat_datafile +from .core import FermiLAT, FermiLATClass, GetFermilatDatafile, get_fermilat_datafile # noqa: E402 __all__ = ['FermiLAT', 'FermiLATClass', 'GetFermilatDatafile', 'get_fermilat_datafile', 'Conf', 'conf', ] - -import warnings -warnings.warn("Experimental: Fermi-LAT has not yet been refactored to have " - "its API match the rest of astroquery.") diff --git a/astroquery/fermi/core.py b/astroquery/fermi/core.py index 284678d382..96f94911ef 100644 --- a/astroquery/fermi/core.py +++ b/astroquery/fermi/core.py @@ -1,170 +1,392 @@ # Licensed under a 3-clause BSD style license - see LICENSE.rst -"""Download of Fermi LAT (Large Area Telescope) data""" +"""Download of Fermi LAT (Large Area Telescope) data. + +This module talks to the Fermi LAT Data Query REST API:: + + POST /query -> {"query_id": ...} + GET /query/{id}/status -> {"state": ...} + GET /query/{id}/results -> {"files": [{"name": ...}, ...]} + +which is replacing the legacy ``LATDataQuery.cgi`` form endpoint whose HTML +response pages this module used to scrape with regular expressions. +""" import re -import requests import time +from urllib.parse import urljoin + import astropy.units as u +from astropy.utils.decorators import deprecated + from ..query import BaseQuery from ..utils import commons, async_to_sync +from ..exceptions import RemoteServiceError, TimeoutError from . import conf __all__ = ['FermiLAT', 'FermiLATClass', 'GetFermilatDatafile', 'get_fermilat_datafile', ] +# The API reports free-form, human-readable states ("Query completed", +# "Query in progress", ...). Rather than hard-coding the exact strings - +# which is what made the previous scraping implementation so brittle - we +# match on substrings and treat anything unrecognised as "still running". +_DONE_TOKENS = ('complet', 'done', 'finished', 'ready') +_ERROR_TOKENS = ('error', 'fail', 'reject', 'invalid', 'abort', 'cancel') + +# "128.836,-45.1764" and friends: already a coordinate pair, no name +# resolution needed (also covers the all-sky "0.0,0.0" convention). +_COORD_PAIR_RE = re.compile(r'^\s*[-+]?\d+(\.\d*)?\s*,\s*[-+]?\d+(\.\d*)?\s*$') + @async_to_sync class FermiLATClass(BaseQuery): """ - TODO: document + Query the Fermi LAT Data Server. + + The server runs queries asynchronously: `query_object_async` submits a + query and returns its ``query_id``, and `query_object` additionally waits + for the query to finish and returns the URLs of the staged data files. """ - request_url = conf.url - result_url_re = re.compile(r'The results of your query may be found at ' - r' 60 (typically 180) + for an all-sky query - note that all-sky queries are restricted by + the server to observation windows of <= 24 hours. + .. warning:: - Defaults to 1 degree if left blank + Defaults to 1 degree if left blank. obsdates : str - Observation dates. - timesys: 'Gregorian' or 'MET' or 'MJD' - Time system associated with obsdates - energyrange_MeV: str - Energy range in MeV + Observation window, as ``"start,stop"``. + timesys : 'Gregorian' or 'MET' or 'MJD' + Time system associated with ``obsdates``. + energyrange_MeV : str + Energy range in MeV, as ``"min,max"``. + LATdatatype : 'Photon' or 'Extended' or 'None' + Which LAT event class to retrieve. + spacecraftdata : bool + Whether to also retrieve the spacecraft (SC) file. + coordsystem : 'J2000' or 'B1950' or 'Galactic' + Coordinate system of the submitted coordinates. Resolved names + and `~astropy.coordinates.SkyCoord` inputs are transformed into + this frame. + zenithangle : float, optional + Maximum zenith angle in degrees. The server default is 180. Returns ------- - payload_dict : Requests payload in a dictionary + payload : dict + The JSON payload posted to ``/query``. """ - - payload = {'shapefield': str(searchradius), - 'coordsystem': 'J2000', - 'coordfield': _parse_coordinates(name_or_coords), - 'destination': 'query', - 'timefield': obsdates, - 'timetype': timesys, - 'energyfield': energyrange_MeV, - 'photonOrExtendedOrNone': LATdatatype, - 'spacecraft': 'on' if spacecraftdata else 'off'} + # The API requires a radius; the CGI form silently defaulted to 1 deg, + # so preserve that behaviour rather than sending an empty string. + if searchradius in ('', None): + searchradius = 1 + + payload = { + 'coordfield': _parse_coordinates(name_or_coords, + coordsystem=coordsystem), + 'coordsystem': coordsystem, + # the API kept the legacy CGI-form name for the search radius + 'shapefield': searchradius, + 'timefield': obsdates, + 'timetype': timesys, + 'energyfield': energyrange_MeV, + 'photonOrExtendedOrNone': LATdatatype, + 'spacecraft': 'on' if spacecraftdata else 'off', + } + + if zenithangle is not None: + payload['zenithangle'] = zenithangle return payload + # ------------------------------------------------------------------ + # status / results + # ------------------------------------------------------------------ + def get_status(self, query_id): + """ + Return the raw status document for ``query_id``. + + Returns + ------- + status : dict + The parsed ``/query/{id}/status`` response. The overall state is + in ``status['state']``; per-server detail is in ``queue_status``, + ``running_status`` and ``servers_status``. + """ + response = self._request( + "GET", url=f"{self.base_url}/query/{query_id}/status", + timeout=self.TIMEOUT, cache=False) + _raise_for_status(response, context=f"Fermi LAT status ({query_id})") + return response.json() + + def list_results(self, query_id): + """ + Return the file metadata for a completed query. + + Returns + ------- + files : list of dict + The ``files`` entries of the ``/query/{id}/results`` response. + """ + response = self._request( + "GET", url=f"{self.base_url}/query/{query_id}/results", + timeout=self.RETRIEVAL_TIMEOUT, cache=False) + _raise_for_status(response, context=f"Fermi LAT results ({query_id})") + return response.json().get('files', []) + + def wait_for_completion(self, query_id, *, check_frequency=None, + max_wait=None, verbose=False): + """ + Poll ``/query/{id}/status`` until the query finishes. + + Parameters + ---------- + check_frequency : float, optional + Minutes between polls. Defaults to `check_frequency`. + max_wait : float, optional + Give up after this many minutes. ``None`` (default) waits + indefinitely. + verbose : bool + Print the elapsed time on completion. + + Returns + ------- + status : dict + The final status document. + """ + if check_frequency is None: + check_frequency = self.check_frequency + + elapsed_time = 0.0 + + while True: + status = self.get_status(query_id) + state = str(status.get('state', '')) + lowered = state.lower() + + if any(token in lowered for token in _ERROR_TOKENS): + raise RemoteServiceError( + f"Fermi LAT query {query_id} failed with state {state!r}") + + if any(token in lowered for token in _DONE_TOKENS): + if verbose: + print(f"Query completed in {elapsed_time:0.1f} minutes") + return status + + if max_wait is not None and elapsed_time >= max_wait: + raise TimeoutError( + f"Fermi LAT query {query_id} did not complete within " + f"{max_wait} minutes (last state: {state!r})") + + time.sleep(check_frequency * 60) + elapsed_time += check_frequency + + def get_file_urls(self, query_id, *, check_frequency=None, max_wait=None, + verbose=False): + """ + Wait for ``query_id`` to complete and return the result file URLs. + + Returns + ------- + urls : list of str + """ + self.wait_for_completion(query_id, check_frequency=check_frequency, + max_wait=max_wait, verbose=verbose) + return [_file_url(entry) for entry in self.list_results(query_id)] + def _parse_result(self, result, *, verbose=False, **kwargs): """ - Use get_fermilat_datafile to download a result URL + Turn the ``query_id`` returned by `query_object_async` into a list of + downloadable file URLs, waiting for the query to complete. """ - return get_fermilat_datafile(result) + return self.get_file_urls(result, verbose=verbose) FermiLAT = FermiLATClass() -def _parse_coordinates(coordinates): - try: - c = commons.parse_coordinates(coordinates) - # now c has some subclass of astropy.coordinate - # get ra, dec and frame - return _fermi_format_coords(c) - except (u.UnitsError, TypeError): - raise Exception("Coordinates not specified correctly") +def _raise_for_status(response, *, context): + """ + Raise a `RemoteServiceError` carrying the server's error message. + + The Fermi API signals bad requests with an HTTP error status and a JSON + body of the form ``{"error": "..."}``. ``requests.raise_for_status`` only + reports the status code, so this helper pulls the message out of the body + (trying the ``error``, ``detail`` and ``message`` keys, then falling back + to the raw text) and surfaces it, as recommended by the astroquery API + specification. + """ + status_code = getattr(response, 'status_code', None) + if status_code is None or status_code < 400: + return + message = None + try: + body = response.json() + except ValueError: + body = None + + if isinstance(body, dict): + for key in ('error', 'detail', 'message'): + if body.get(key): + message = body[key] + break + if message is None: + text = getattr(response, 'text', '') or '' + message = text.strip() or f"HTTP {status_code}" -def _fermi_format_coords(c): - c = c.transform_to('fk5') - return "{0:0.5f},{1:0.5f}".format(c.ra.degree, c.dec.degree) + raise RemoteServiceError(f"{context} failed ({status_code}): {message}") -class GetFermilatDatafile: +def _file_url(entry): """ - TODO: document - TODO: Fail with useful failure messages on genuine failures - (this doesn't need to be implemented as a class) + Build an absolute URL for one entry of the ``results`` file list. + + The API is documented as returning ``{"name": "..._PH00.fits"}``; if a + future revision also returns an absolute ``url``/``href``/``path``, prefer + that over reconstructing the location from `conf.file_base_url`. """ + if isinstance(entry, str): + name = entry + else: + for key in ('url', 'href', 'path', 'location'): + value = entry.get(key) + if value: + if value.startswith('http'): + return value + return urljoin(conf.file_base_url, value.lstrip('/')) + name = entry.get('name') + + if not name: + raise RemoteServiceError( + f"Could not determine a file URL from results entry {entry!r}") + + return urljoin(conf.file_base_url, name) + + +def _parse_coordinates(coordinates, *, coordsystem='J2000'): + # A literal "RA,Dec" pair (including the all-sky "0.0,0.0" convention) is + # passed straight through - no name resolution, no frame transform. + if isinstance(coordinates, str) and _COORD_PAIR_RE.match(coordinates): + return coordinates.replace(' ', '') - fitsfile_re = re.compile(r'Available') - fitsfile_re = re.compile(r'wget (https://fermi\.gsfc\.nasa\.gov/FTP/fermi/data/lat/queries/[A-Za-z0-9_]*.fits)') - # wget https://fermi.gsfc.nasa.gov/FTP/fermi/data/lat/queries/L1809182001077FA3883F37_SC00.fits + try: + c = commons.parse_coordinates(coordinates) + except (u.UnitsError, TypeError): + raise ValueError("Coordinates not specified correctly") - TIMEOUT = conf.retrieval_timeout + return _fermi_format_coords(c, coordsystem=coordsystem) - check_frequency = 1 # minutes - def __call__(self, result_url, *, check_frequency=1, verbose=False): - self.result_url = result_url +def _fermi_format_coords(c, *, coordsystem='J2000'): + frames = {'j2000': 'fk5', 'b1950': 'fk4', 'galactic': 'galactic'} - page_loaded = False + try: + frame = frames[coordsystem.lower()] + except KeyError: + raise ValueError( + f"Unsupported coordsystem {coordsystem!r}; " + f"expected one of {', '.join(sorted(frames))}") - elapsed_time = 0 + c = c.transform_to(frame) - while not (page_loaded): - page_loaded = fitsfile_urls = self._check_page() - if page_loaded: - # don't wait an extra N minutes for success - break - time.sleep(check_frequency * 60) - elapsed_time += check_frequency - # update progressbar here... + if frame == 'galactic': + lon, lat = c.l.degree, c.b.degree + else: + lon, lat = c.ra.degree, c.dec.degree - if verbose: - print("Query completed in %0.1f minutes" % (elapsed_time)) + return "{0:0.5f},{1:0.5f}".format(lon, lat) - return fitsfile_urls - def _check_page(self): - result_page = requests.post(url=self.result_url, - data=None, - timeout=self.TIMEOUT) +@deprecated(since='0.4.12', + alternative='FermiLAT.query_object or FermiLAT.get_file_urls') +class GetFermilatDatafile: + """ + Deprecated. Retained so that code written against the pre-REST module + keeps importing; it now delegates to `FermiLATClass.get_file_urls`. + """ - pagedata = result_page.text + TIMEOUT = conf.retrieval_timeout + check_frequency = 1 - fitsfile_urls = self.fitsfile_re.findall(pagedata) + def __call__(self, query_id, *, check_frequency=None, verbose=False): + if check_frequency is None: + check_frequency = self.check_frequency + return FermiLAT.get_file_urls(query_id, + check_frequency=check_frequency, + verbose=verbose) - if len(fitsfile_urls) == 0: - return False - else: - return fitsfile_urls +@deprecated(since='0.4.12', + alternative='FermiLAT.query_object or FermiLAT.get_file_urls') +def get_fermilat_datafile(query_id, *, check_frequency=1, verbose=False): + """ + Deprecated. Wait for a query to finish and return its data file URLs. -get_fermilat_datafile = GetFermilatDatafile() + Note that this takes a ``query_id`` - the pre-REST version took the URL of + an HTML results page, which no longer exists. + """ + return FermiLAT.get_file_urls(query_id, check_frequency=check_frequency, + verbose=verbose) diff --git a/astroquery/fermi/tests/__init__.py b/astroquery/fermi/tests/__init__.py index e69de29bb2..9dce85d06f 100644 --- a/astroquery/fermi/tests/__init__.py +++ b/astroquery/fermi/tests/__init__.py @@ -0,0 +1 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst diff --git a/astroquery/fermi/tests/data/query_result_m31.html b/astroquery/fermi/tests/data/query_result_m31.html deleted file mode 100644 index b3334a3050..0000000000 --- a/astroquery/fermi/tests/data/query_result_m31.html +++ /dev/null @@ -1,136 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - -Fermi SSC - LAT Photon, Event and Spacecraft Data - - - - - - - - -
- - -
- - - - - - -
- - - -
-Fermi Science Support Center -
- - - - - - -
- - - -

Query L13090120163429E469B432 submitted.

Please see LAT Data Caveats for important information about Fermi LAT data.

Your search criteria were:

Equatorial coordinates (degrees)(10.6847,41.2687)
Time range (MET)(384196376,399748376)
Time range (Gregorian)(2013-03-05 17:12:56,2013-09-01 17:12:56)
Energy range (MeV)(100,300000)
Search radius (degrees)15

The estimated time for your query to complete is 22 seconds. The results of your query may be found at https://fermi.gsfc.nasa.gov/cgi-bin/ssc/LAT/QueryResults.cgi?id=L13090120163429E469B432.

- - - - - - -
- - - - diff --git a/astroquery/fermi/tests/data/query_results.json b/astroquery/fermi/tests/data/query_results.json new file mode 100644 index 0000000000..62621e0f8c --- /dev/null +++ b/astroquery/fermi/tests/data/query_results.json @@ -0,0 +1,7 @@ +{ + "query_id": "L2601082002167F48EE3069", + "files": [ + { "name": "L2601082002167F48EE3069_PH00.fits" }, + { "name": "L2601082002167F48EE3069_SC00.fits" } + ] +} diff --git a/astroquery/fermi/tests/data/query_status_complete.json b/astroquery/fermi/tests/data/query_status_complete.json new file mode 100644 index 0000000000..2b46480f09 --- /dev/null +++ b/astroquery/fermi/tests/data/query_status_complete.json @@ -0,0 +1,14 @@ +{ + "query_id": "L2601082002167F48EE3069", + "queue_status": [ + { "server": "Photon Server", "queue_rank": 0, "time_remaining": "Unknown" }, + { "server": "Spacecraft Server", "queue_rank": 0, "time_remaining": "Unknown" } + ], + "running_status": [ + { "server": "Photon Server", "status": "Completed", "time_remaining": "N/A" } + ], + "servers_status": [ + { "name": "Photon Server", "position": "Query complete", "time_remaining": "N/A" } + ], + "state": "Query completed" +} diff --git a/astroquery/fermi/tests/data/query_status_failed.json b/astroquery/fermi/tests/data/query_status_failed.json new file mode 100644 index 0000000000..cc3a214b38 --- /dev/null +++ b/astroquery/fermi/tests/data/query_status_failed.json @@ -0,0 +1,7 @@ +{ + "query_id": "L2601082002167F48EE3069", + "queue_status": [], + "running_status": [], + "servers_status": [], + "state": "Query failed" +} diff --git a/astroquery/fermi/tests/data/query_status_running.json b/astroquery/fermi/tests/data/query_status_running.json new file mode 100644 index 0000000000..0f56be7d38 --- /dev/null +++ b/astroquery/fermi/tests/data/query_status_running.json @@ -0,0 +1,14 @@ +{ + "query_id": "L2601082002167F48EE3069", + "queue_status": [ + { "server": "Photon Server", "queue_rank": 2, "time_remaining": "Unknown" }, + { "server": "Spacecraft Server", "queue_rank": 1, "time_remaining": "Unknown" } + ], + "running_status": [ + { "server": "Photon Server", "status": "Running", "time_remaining": "Unknown" } + ], + "servers_status": [ + { "name": "Photon Server", "position": "Query in progress", "time_remaining": "Unknown" } + ], + "state": "Query in progress" +} diff --git a/astroquery/fermi/tests/data/query_submit.json b/astroquery/fermi/tests/data/query_submit.json new file mode 100644 index 0000000000..aa44d969a5 --- /dev/null +++ b/astroquery/fermi/tests/data/query_submit.json @@ -0,0 +1,6 @@ +{ + "query_id": "L2601082002167F48EE3069", + "status": "submitted", + "status_url": "/api/v1/query/L2601082002167F48EE3069/status", + "results_url": "/api/v1/query/L2601082002167F48EE3069/results" +} diff --git a/astroquery/fermi/tests/data/result_page_m31.html b/astroquery/fermi/tests/data/result_page_m31.html deleted file mode 100644 index 7686eb0c33..0000000000 --- a/astroquery/fermi/tests/data/result_page_m31.html +++ /dev/null @@ -1,138 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - -Fermi SSC - LAT Photon, Event and Spacecraft Data - - - - - - - - -
- - -
- - - - - - -
- - - -
-Fermi Science Support Center -
- - - - - - -
- - - -

Results for query L13090110364329E469B418

Your search criteria were:

Equatorial coordinates (degrees)(0.18648,0.72028)
Time range (MET)(378691200,378777600)
Time range (Gregorian)(2013-01-01 00:00:00,2013-01-02 00:00:00)
Energy range (MeV)(1000,100000)
Search radius (degrees)15

The state of your query is 2 (Query complete)

Server
Position in Queue
Estimated Time Remaining (sec)
Photon Server
Query complete
N/A
Spacecraft Server
Query complete
N/A

The filenames of the result files consist of the query ID string with an identifier appended to indicate which database the file came from. The identifiers are of the form: _DDNN where DD indicates the database and NN is the file number. The file number will generally be '00' unless the query resulted in a large data volume. In that case the data is broken up into multiple files. The values of the database field are:

  • PH - Photon Database
  • SC - Spacecraft Pointing, Livetime, and History Database
  • EV - Extended Database

In the event that you do not see any files with the data type you requested listed below, you should try resubmitting your query as there may have been a problem.

Filename Number of Entries Size (MB) Status
L13090110364329E469B418_PH00.fits
32
0.03
 Available
L13090110364329E469B418_SC00.fits
2436
0.40
 Available

If you would like to download the files via wget, simply copy the following commands and paste them into a terminal window. The files will be downloaded to the current directory in the terminal window.

wget https://fermi.gsfc.nasa.gov/FTP/fermi/data/lat/queries/L13090110364329E469B418_PH00.fits
-wget https://fermi.gsfc.nasa.gov/FTP/fermi/data/lat/queries/L13090110364329E469B418_SC00.fits
-
- - - - - - -
- - - - diff --git a/astroquery/fermi/tests/test_fermi.py b/astroquery/fermi/tests/test_fermi.py index 0485a7e354..d7728ee8b6 100644 --- a/astroquery/fermi/tests/test_fermi.py +++ b/astroquery/fermi/tests/test_fermi.py @@ -1,66 +1,199 @@ # Licensed under a 3-clause BSD style license - see LICENSE.rst +import json import os -import requests -import pytest + import astropy.coordinates as coord +import pytest + +from astroquery.exceptions import RemoteServiceError from astroquery.utils.mocks import MockResponse from ... import fermi -DATA_FILES = {'async': "query_result_m31.html", - 'result': 'result_page_m31.html', - 'result_url': 'https://fermi.gsfc.nasa.gov/cgi-bin/ssc/LAT/QueryResults.cgi?id=L13090120163429E469B432', - 'fits': ['https://fermi.gsfc.nasa.gov/FTP/fermi/data/lat/queries/L13090110364329E469B418_PH00.fits', - 'https://fermi.gsfc.nasa.gov/FTP/fermi/data/lat/queries/L13090110364329E469B418_SC00.fits']} +QUERY_ID = 'L2601082002167F48EE3069' + +DATA_FILES = {'submit': 'query_submit.json', + 'status_running': 'query_status_running.json', + 'status_complete': 'query_status_complete.json', + 'status_failed': 'query_status_failed.json', + 'results': 'query_results.json'} + +EXPECTED_URLS = [ + 'https://fermi.gsfc.nasa.gov/FTP/fermi/data/lat/test/queries/' + 'L2601082002167F48EE3069_PH00.fits', + 'https://fermi.gsfc.nasa.gov/FTP/fermi/data/lat/test/queries/' + 'L2601082002167F48EE3069_SC00.fits', +] + +FK5_COORDINATES = coord.SkyCoord(10.68471, 41.26875, unit=('deg', 'deg')) def data_path(filename): - data_dir = os.path.join(os.path.dirname(__file__), 'data') - return os.path.join(data_dir, filename) + return os.path.join(os.path.dirname(__file__), 'data', filename) + + +def read_data(key): + with open(data_path(DATA_FILES[key]), 'rb') as fh: + return fh.read() + + +class _Router: + """Dispatch mocked responses on (method, url), like the real API.""" + + def __init__(self, status_key='status_complete'): + self.status_key = status_key + self.calls = [] + + def __call__(self, method, url=None, **kwargs): + self.calls.append((method, url, kwargs)) + + if method == 'POST' and url.endswith('/query'): + return MockResponse(read_data('submit')) + if method == 'GET' and url.endswith(f'/query/{QUERY_ID}/status'): + return MockResponse(read_data(self.status_key)) + if method == 'GET' and url.endswith(f'/query/{QUERY_ID}/results'): + return MockResponse(read_data('results')) + + raise AssertionError(f"unexpected request: {method} {url}") @pytest.fixture -def patch_post(request): +def patch_request(request): mp = request.getfixturevalue("monkeypatch") + router = _Router() + mp.setattr(fermi.FermiLAT, '_request', router) + # don't sleep between status polls + mp.setattr(fermi.FermiLAT, 'check_frequency', 0) + return router - mp.setattr(fermi.FermiLAT, '_request', post_mockreturn) - mp.setattr(requests, 'post', post_mockreturn) - return mp +def test_payload_field_names(): + payload = fermi.core.FermiLAT.query_object_async( + FK5_COORDINATES, searchradius=15, energyrange_MeV='100,300000', + obsdates='772109936,787661936', timesys='MET', + get_query_payload=True) -def post_mockreturn(method="POST", url=None, data=None, timeout=50, **kwargs): - if data is not None: - with open(data_path(DATA_FILES['async']), 'rb') as r: - response = MockResponse(r.read(), **kwargs) - else: - with open(data_path(DATA_FILES['result']), 'rb') as r: - response = MockResponse(r.read(), **kwargs) - return response + assert payload == {'coordfield': '10.68472,41.26875', + 'coordsystem': 'J2000', + 'shapefield': 15, + 'timefield': '772109936,787661936', + 'timetype': 'MET', + 'energyfield': '100,300000', + 'photonOrExtendedOrNone': 'Photon', + 'spacecraft': 'on'} -FK5_COORDINATES = coord.SkyCoord(10.68471, 41.26875, unit=('deg', 'deg')) +def test_payload_defaults_radius_to_one_degree(): + payload = fermi.core.FermiLAT.query_object_async( + FK5_COORDINATES, get_query_payload=True) + assert payload['shapefield'] == 1 + + +def test_payload_zenithangle_only_when_given(): + payload = fermi.core.FermiLAT.query_object_async( + FK5_COORDINATES, get_query_payload=True) + assert 'zenithangle' not in payload + + payload = fermi.core.FermiLAT.query_object_async( + FK5_COORDINATES, zenithangle=85, get_query_payload=True) + assert payload['zenithangle'] == 85 + + +def test_payload_allsky_coordinates_pass_through(): + """An all-sky query submits "0.0,0.0" without hitting a name resolver.""" + payload = fermi.core.FermiLAT.query_object_async( + '0.0,0.0', searchradius=180, + obsdates='2008-08-04 15:43:36,2008-08-05 09:14:33', + get_query_payload=True) + assert payload['coordfield'] == '0.0,0.0' + assert payload['shapefield'] == 180 -# disable waiting so tests run fast -fermi.core.get_fermilat_datafile.TIMEOUT = 1 -fermi.core.get_fermilat_datafile.check_frequency = 0 +def test_payload_galactic_coordsystem(): + payload = fermi.core.FermiLAT.query_object_async( + FK5_COORDINATES, coordsystem='Galactic', get_query_payload=True) + assert payload['coordsystem'] == 'Galactic' + lon, lat = (float(x) for x in payload['coordfield'].split(',')) + assert lon == pytest.approx(121.174, abs=0.01) + assert lat == pytest.approx(-21.573, abs=0.01) -def test_FermiLAT_query_async(patch_post): + +def test_payload_bad_coordsystem(): + with pytest.raises(ValueError, match='Unsupported coordsystem'): + fermi.core.FermiLAT.query_object_async( + FK5_COORDINATES, coordsystem='Ecliptic', get_query_payload=True) + + +def test_query_object_async_returns_query_id(patch_request): result = fermi.core.FermiLAT.query_object_async( - FK5_COORDINATES, energyrange_MeV='1000, 100000', - obsdates='2013-01-01 00:00:00, 2013-01-02 00:00:00') - assert result == DATA_FILES['result_url'] + FK5_COORDINATES, energyrange_MeV='1000,100000', + obsdates='2013-01-01 00:00:00,2013-01-02 00:00:00') + assert result == QUERY_ID + + method, url, kwargs = patch_request.calls[0] + assert method == 'POST' + assert url.endswith('/query') + # the payload must go out as JSON, not as a form body + assert kwargs['json']['coordfield'] == '10.68472,41.26875' + assert kwargs.get('data') is None + + +def test_get_status(patch_request): + status = fermi.core.FermiLAT.get_status(QUERY_ID) + assert status['state'] == 'Query completed' -def test_getfermilatdatafile(patch_post): - result = fermi.core.get_fermilat_datafile(data_path(DATA_FILES['result']), - verbose=True) - assert result +def test_list_results(patch_request): + files = fermi.core.FermiLAT.list_results(QUERY_ID) + assert [f['name'] for f in files] == [ + 'L2601082002167F48EE3069_PH00.fits', + 'L2601082002167F48EE3069_SC00.fits'] -def test_FermiLAT_query(patch_post): - # Make a query that results in small SC and PH file sizes +def test_get_file_urls(patch_request): + assert fermi.core.FermiLAT.get_file_urls(QUERY_ID) == EXPECTED_URLS + + +def test_query_object(patch_request): result = fermi.core.FermiLAT.query_object( - FK5_COORDINATES, energyrange_MeV='1000, 100000', - obsdates='2013-01-01 00:00:00, 2013-01-02 00:00:00') - assert result == DATA_FILES['fits'] + FK5_COORDINATES, energyrange_MeV='1000,100000', + obsdates='2013-01-01 00:00:00,2013-01-02 00:00:00') + assert result == EXPECTED_URLS + + +def test_failed_query_raises(request): + mp = request.getfixturevalue("monkeypatch") + mp.setattr(fermi.FermiLAT, '_request', _Router(status_key='status_failed')) + mp.setattr(fermi.FermiLAT, 'check_frequency', 0) + + with pytest.raises(RemoteServiceError, match='failed with state'): + fermi.core.FermiLAT.get_file_urls(QUERY_ID) + + +def test_missing_query_id_raises(request): + mp = request.getfixturevalue("monkeypatch") + mp.setattr(fermi.FermiLAT, '_request', + lambda *a, **kw: MockResponse(json.dumps({'error': 'bad'}).encode())) + + with pytest.raises(RemoteServiceError, match="did not contain a 'query_id'"): + fermi.core.FermiLAT.query_object_async(FK5_COORDINATES) + + +def test_file_url_prefers_absolute_url_from_server(): + """Forward-compatible: honour an absolute URL if the API starts sending one.""" + entry = {'name': 'x_PH00.fits', + 'url': 'https://example.org/somewhere/x_PH00.fits'} + assert fermi.core._file_url(entry) == 'https://example.org/somewhere/x_PH00.fits' + + +def test_bad_input_surfaces_server_error_message(request): + """A 400 with {"error": ...} is reported, not swallowed into a bare status.""" + mp = request.getfixturevalue("monkeypatch") + body = json.dumps({'error': 'Invalid query parameters'}).encode() + mp.setattr(fermi.FermiLAT, '_request', + lambda *a, **kw: MockResponse(body, status_code=400)) + + with pytest.raises(RemoteServiceError, + match='Invalid query parameters'): + fermi.core.FermiLAT.query_object_async( + FK5_COORDINATES, energyrange_MeV='999999,1') diff --git a/astroquery/fermi/tests/test_fermi_remote.py b/astroquery/fermi/tests/test_fermi_remote.py index 23d2b92ef3..254cdf2e0d 100644 --- a/astroquery/fermi/tests/test_fermi_remote.py +++ b/astroquery/fermi/tests/test_fermi_remote.py @@ -1,8 +1,7 @@ # Licensed under a 3-clause BSD style license - see LICENSE.rst - -import pytest import astropy.coordinates as coord +import pytest from ... import fermi @@ -11,19 +10,43 @@ @pytest.mark.remote_data def test_FermiLAT_query_async(): - result = fermi.core.FermiLAT.query_object_async( - FK5_COORDINATES, energyrange_MeV='1000, 100000', - obsdates='2013-01-01 00:00:00, 2013-01-02 00:00:00') - assert 'https://fermi.gsfc.nasa.gov/cgi-bin/ssc/LAT/QueryResults.cgi?' in result + query_id = fermi.core.FermiLAT.query_object_async( + FK5_COORDINATES, energyrange_MeV='1000,100000', + obsdates='2013-01-01 00:00:00,2013-01-02 00:00:00') + assert isinstance(query_id, str) + assert query_id.startswith('L') + + status = fermi.core.FermiLAT.get_status(query_id) + assert 'state' in status @pytest.mark.remote_data def test_FermiLAT_query(): # Make a query that results in small SC and PH file sizes result = fermi.core.FermiLAT.query_object( - FK5_COORDINATES, energyrange_MeV='1000, 100000', - obsdates='2013-01-01 00:00:00, 2013-01-02 00:00:00') - # this test might be fragile? I'm not sure how stable the file names are + FK5_COORDINATES, energyrange_MeV='1000,100000', + obsdates='2013-01-01 00:00:00,2013-01-02 00:00:00') + + assert len(result) >= 1 for rr in result: - assert rr.startswith('https://fermi.gsfc.nasa.gov/FTP/fermi/data/lat/queries/') + assert rr.startswith('https://') assert rr.endswith('_SC00.fits') or rr.endswith('_PH00.fits') + + +@pytest.mark.remote_data +def test_FermiLAT_query_zenithangle(): + result = fermi.core.FermiLAT.query_object( + FK5_COORDINATES, energyrange_MeV='1000,100000', + obsdates='2013-01-01 00:00:00,2013-01-02 00:00:00', + zenithangle=85) + assert len(result) >= 1 + + +@pytest.mark.remote_data +def test_FermiLAT_allsky_query(): + # all-sky: radius > 60 deg, observation window <= 24 hours + result = fermi.core.FermiLAT.query_object( + '0.0,0.0', searchradius=180, + obsdates='2008-08-04 15:43:36,2008-08-05 09:14:33', + energyrange_MeV='100,300000') + assert len(result) >= 1 diff --git a/docs/fermi/fermi.rst b/docs/fermi/fermi.rst index fbcd2b38f8..71a7434c76 100644 --- a/docs/fermi/fermi.rst +++ b/docs/fermi/fermi.rst @@ -12,11 +12,45 @@ centered on M 31 for the energy range 1 to 100 GeV for the first day in 2013. >>> from astroquery.fermi import FermiLAT >>> result = FermiLAT.query_object('M31', energyrange_MeV='1000, 100000', - ... obsdates='2013-01-01 00:00:00, 2013-01-02 00:00:00') + ... obsdates='2013-01-01 00:00:00, 2013-01-02 00:00:00') >>> print(result) # doctest: +IGNORE_OUTPUT ['https://fermi.gsfc.nasa.gov/FTP/fermi/data/lat/queries/L210111120827756AAA3A88_PH00.fits', 'https://fermi.gsfc.nasa.gov/FTP/fermi/data/lat/queries/L210111120827756AAA3A88_SC00.fits'] +`~astroquery.fermi.FermiLATClass.query_object` blocks until the server has +finished staging the data. To submit a query and come back to it later, use +the asynchronous interface, which returns the server-assigned ``query_id``: + +.. doctest-remote-data:: + + >>> from astroquery.fermi import FermiLAT + >>> query_id = FermiLAT.query_object_async('M31', energyrange_MeV='1000, 100000', + ... obsdates='2013-01-01 00:00:00, 2013-01-02 00:00:00') + >>> FermiLAT.get_status(query_id)['state'] # doctest: +IGNORE_OUTPUT + 'Query completed' + >>> FermiLAT.get_file_urls(query_id) # doctest: +IGNORE_OUTPUT + ['https://fermi.gsfc.nasa.gov/FTP/fermi/data/lat/queries/L210111120827756AAA3A88_PH00.fits', + 'https://fermi.gsfc.nasa.gov/FTP/fermi/data/lat/queries/L210111120827756AAA3A88_SC00.fits'] + +A maximum zenith angle can be supplied with ``zenithangle`` (the server +default is 180 degrees), and coordinates may be given in ``J2000`` (default), +``B1950`` or ``Galactic`` frames via ``coordsystem``. + +All-sky queries +=============== + +All-sky queries are handled specially by the server: the search radius must be +greater than 60 degrees (in practice, 180), the observation window must be no +longer than 24 hours, and the coordinates are ignored. + +.. doctest-remote-data:: + + >>> from astroquery.fermi import FermiLAT + >>> result = FermiLAT.query_object('0.0,0.0', searchradius=180, + ... energyrange_MeV='100, 300000', + ... obsdates='2008-08-04 15:43:36, 2008-08-05 09:14:33') + +Queries that exceed the 24-hour window are rejected by the server. Troubleshooting =============== @@ -28,7 +62,7 @@ If you are repeatedly getting failed queries, or bad/out-of-date results, try cl >>> from astroquery.fermi import FermiLAT >>> FermiLAT.clear_cache() -If this function is unavailable, upgrade your version of astroquery. +If this function is unavailable, upgrade your version of astroquery. The ``clear_cache`` function was introduced in version 0.4.7.dev8479.