Skip to content

GitLab

Commit ab4f7188 authored by Derek Homeier's avatar Derek Homeier
Browse files

Fix and test direct Fido interface

parent f032350e
Pipeline #2519 failed with stage
in 3 minutes and 42 seconds
Hide whitespace changes
Inline Side-by-side
sdc/client.py
View file @ ab4f7188
......@@ -6,7 +6,7 @@ import os
import astropy.units as u
from astropy.time import Time
import sunpy.net.attrs as a
from sunpy.net.attr import AttrAnd, AttrOr, AttrMeta, AttrWalker, DataAttr
from sunpy.net.attr import AttrAnd, AttrOr, AttrMeta, AttrWalker, DataAttr, and_, or_
from sunpy.net.base_client import BaseClient, QueryResponseTable
from sunpy.util.exceptions import SunpyUserWarning
# from sunpy.net.attrs import Instrument, Level, Physobs, Provider, Time, Wavelength
......@@ -232,20 +232,23 @@ class KISClient(BaseClient):
return searches
def search(self, query) -> QueryResponseTable:
def search(self, *query) -> QueryResponseTable:
"""
Query the SDC RESTHeart client for a list of results in the observation collection.
Parameters
----------
query : `~sunpy.net.attr.AttrAnd, `~sunpy.net.attr.AttrOr``
Logical AND/OR combination of `sunpy.net.attrs` objects representing the query.
query : iterable or `~sunpy.net.attr.AttrAnd, `~sunpy.net.attr.AttrOr``
Logical combination of `sunpy.net.attrs` objects representing the query,
or any number of such objects to be combined using the AND (``&``) operator.
Returns
-------
results : `~sunpy.net.base_client.QueryResponseTable`
A `QueryResponseTable` instance containing the query result.
"""
query = and_(*query)
# Not very extensively tested on hierarchical queries
if not self._can_handle_query(query):
raise AttributeError(f'Query not possible: {self._status()}')
......@@ -292,7 +295,7 @@ class KISClient(BaseClient):
Parameters
----------
query : iterable of `~sunpy.net.attr.Attr`or `~sunpy.net.attr.AttrAnd, `~sunpy.net.attr.AttrOr``
query : iterable of `~sunpy.net.attr.Attr`, `~sunpy.net.attr.AttrAnd, `~sunpy.net.attr.AttrOr``
Collection of query attributes.
hasinstr : bool, optional
Assume that `~sunpy.net.attrs.Instrument` attribute is already present.
......
sdc/tests/test_client.py
View file @ ab4f7188
......@@ -6,6 +6,7 @@ import json
import astropy.units as u
from astropy.io import fits
from sunpy.net import Fido
from sunpy.net import attrs as a
from sunpy.net.base_client import QueryResponseTable
......@@ -32,6 +33,11 @@ def client():
return KISClient()
def _dockerexc(instr):
return (rf"Unable to execute search .http://dockertest:8083/sdc/{instr}_observations.filter="
rf"{{'.and':.{{'description.INSTRUMENT':'{instr}'}},")
def test_docker(client):
"""Test example queries on dockertest."""
if not HAS_DOCKERTEST:
......@@ -59,12 +65,15 @@ def test_docker(client):
assert description['BTYPE'] == 'phot.count'
links = res[0]['_embedded'][0].get('links')
oid = links['l1_data'][0]['$oid']
meta = json.loads(urllib.request.urlopen(f"{_BASE_URL}gris_l1_data.files/{oid}").read())
assert meta['_id']['$oid'] == oid
hdulist = fits.open(f"{_BASE_URL}gris_l1_data.files/{oid}/binary")
assert hdulist[0].header.get('TELESCOP') == 'GREGOR'
assert '2014-04-26T' in hdulist[0].header.get('DATE-OBS')
file_ids = [ld['$oid'] for ld in links['l1_data']]
assert len(file_ids) == 105
for oid in file_ids[0], file_ids[104]:
meta = json.loads(urllib.request.urlopen(f"{_BASE_URL}gris_l1_data.files/{oid}").read())
assert meta['_id']['$oid'] == oid
hdulist = fits.open(f"{_BASE_URL}gris_l1_data.files/{oid}/binary")
assert hdulist[0].header.get('TELESCOP') == 'GREGOR'
assert '2014-04-26T' in hdulist[0].header.get('DATE-OBS')
hdulist.close()
def test_gridfs(client):
......@@ -79,7 +88,7 @@ def test_gridfs(client):
sdc = get_sdc_connection()
gfs = gridfs.GridFS(sdc.sdc_test, "gris_l1_data")
found = gfs.find({"_id": {"$in": file_ids}})
assert(len(found)) == 50
assert(len(found)) == 105
def test_search(client):
......@@ -104,9 +113,7 @@ def test_search(client):
if len(res[0].get('_embedded')) > 0:
assert 'description' in res[0]['_embedded'][0]
else:
with pytest.raises(URLError, match=r"Unable to execute search "
r".http://dockertest:8083/sdc/bbi_observations.filter="
r"{'.and':.{'description.INSTRUMENT':'bbi'},"
with pytest.raises(URLError, match=rf"{_dockerexc('bbi')}"
r"{'description.DATE_BEG':{'.lte':{'.date':'2021-01-01T00:00:00.000'}}},"
r"{'description.DATE_END':{'.gte':{'.date':'2019-01-01T00:00:00.000'}}}"
rf".*Confirm that RESTHeart is running on {_BASE_URL} and connected"):
......@@ -116,32 +123,84 @@ def test_search(client):
assert client._can_handle_query(query)
if HAS_DOCKERTEST:
res = client.search(query)
if len(res[0].get('_embedded')) > 0:
assert 'description' in res[0]['_embedded'][0]
assert len(res[0]['_embedded']) == 100
assert 'description' in res[0]['_embedded'][0]
else:
with pytest.raises(URLError, match=r"Unable to execute search "
r".http://dockertest:8083/sdc/lars_observations.filter="
r"{'.and':.{'description.INSTRUMENT':'lars'},"
with pytest.raises(URLError, match=rf"{_dockerexc('lars')}"
r"{'description.HPLT_TAN_MIN':{'.lte':12}},"
r"{'description.HPLT_TAN_MAX':{'.gte':-10}}"):
client.search(query)
query = a.Instrument("GRIS") & (a.sdc.Theta(85*u.deg, 600*u.arcmin) | a.sdc.PolStates('iq'))
query = a.Instrument("LARS"), a.sdc.HelioProjLat(-10*u.arcsec, 0.2*u.arcmin)
assert client._can_handle_query(*query)
if HAS_DOCKERTEST:
res = client.search(*query)
assert len(res[0]['_embedded']) == 100
assert 'description' in res[0]['_embedded'][0]
else:
with pytest.raises(URLError, match=rf"{_dockerexc('lars')}"
r"{'description.HPLT_TAN_MIN':{'.lte':12}},"
r"{'description.HPLT_TAN_MAX':{'.gte':-10}}"):
client.search(*query)
query = a.Instrument("GRIS") & (a.sdc.Theta(85*u.deg, 3000*u.arcmin) | a.sdc.PolStates('iquv'))
assert client._can_handle_query(query)
if HAS_DOCKERTEST:
res = client.search(query)
assert len(res) == 2
if len(res[0].get('_embedded')) > 0:
assert 'description' in res[0]['_embedded'][0]
assert len(res[0]['_embedded']) == 100
assert 'description' in res[0]['_embedded'][0]
assert 'THETA' in res[0]['_embedded'][0]['description']
assert 'POL_STATES' in res[1]['_embedded'][0]['description']
theta = [obs['description']['THETA'] for obs in res[0]['_embedded']]
assert (min(theta) >= 50) & (max(theta) <= 85)
assert len(res[1]['_embedded']) == 100
else:
with pytest.raises(URLError, match=r"Unable to execute search ") as exc:
# Will raise on first of multi-part OR queries; somehow switches INSTRUMENT and THETA.
with pytest.raises(URLError, match=rf"{_dockerexc('gris')[:80]}") as exc:
client.search(query)
# Will raise on first of multi-part OR queries.
assert "http://dockertest:8083/sdc/gris_observations?filter=" in str(exc.value)
assert "{'description.THETA':{'$gte':10,'$lte':85}}" in str(exc.value)
assert "{'description.THETA':{'$gte':50,'$lte':85}}" in str(exc.value)
assert "{'description.POL_STATES':" not in str(exc.value)
def test_fido_search():
"""Test search using the Fido base class with AttrAnd, AttrOr and lists of *args."""
two_inst = a.Instrument("BBI") | a.Instrument("ChroTel")
if HAS_DOCKERTEST:
res = Fido.search(a.Instrument("GRIS") & a.sdc.Theta(50*u.deg, 80*u.deg))
assert len(res['kis']) > 0
assert len(res['kis', '_embedded']) > 0
theta = [obs[0]['description']['THETA'] for obs in res['kis', '_embedded']]
assert (min(theta) >= 50) & (max(theta) <= 80)
res = Fido.search(a.Instrument("GRIS"), a.sdc.Theta(50*u.deg, 80*u.deg))
assert len(res['kis']) > 0
assert len(res['kis', '_embedded']) > 0
theta = [obs[0]['description']['THETA'] for obs in res['kis']['_embedded']]
assert (min(theta) >= 50) & (max(theta) <= 80)
res = Fido.search(two_inst, a.sdc.Theta(50*u.deg, 80*u.deg))
assert len(res['kis', '_embedded']) == 2
theta = [obs for obs in res['kis', '_embedded']]
assert 'description' in theta[0]['_embedded'][0].keys()
assert (min(theta) >= 50) & (max(theta) <= 80)
assert res['kis', '_embedded'][0]['description']['INSTRUMENT'] == 'bbi'
assert res['kis', '_embedded'][1]['description']['INSTRUMENT'] == 'chrotel'
else:
with pytest.raises(URLError, match=rf"{_dockerexc('gris')}"
r"{'description.THETA':{'.gte':50,'.lte':80}}"):
Fido.search(a.Instrument("GRIS") & a.sdc.Theta(50*u.deg, 80*u.deg))
with pytest.raises(URLError, match=rf"{_dockerexc('gris')}"
r"{'description.THETA':{'.gte':50,'.lte':80}}"):
Fido.search(a.Instrument("GRIS"), a.sdc.Theta(50*u.deg, 80*u.deg))
with pytest.raises(URLError, match=rf"{_dockerexc('bbi')}") as exc:
Fido.search(two_inst, a.sdc.Theta(50*u.deg, 80*u.deg))
assert "{'description.THETA':{'$gte':50,'$lte':80}}" in str(exc.value)
assert "{'description.INSTRUMENT':'chrotel'" not in str(exc.value)
@pytest.mark.parametrize("query", ((a.Instrument("GRIS") & a.Level(3)),
(a.Instrument("ChroTel") & a.Physobs("perspective.vortex")),
(a.Level(0) & a.Instrument("Bob")),
......@@ -176,9 +235,7 @@ def test_all_queries(client, query):
if len(res[0].get('_embedded')) > 0:
assert 'description' in res[0]['_embedded'][0]
else:
with pytest.raises(URLError, match=r"Unable to execute search "
r".http://dockertest:8083/sdc/gris_observations.filter="
r"{'.and':.{'description.INSTRUMENT':'gris'},"
with pytest.raises(URLError, match=rf"{_dockerexc('gris')}"
rf"{{'description.*{query.type_name.upper()}"):
client.search(a.Instrument("GRIS") & query)
......@@ -197,9 +254,7 @@ def test_range(client):
assert max(wave_min) <= 1250
assert min(wave_max) >= 1080
else:
with pytest.raises(URLError, match=r"Unable to execute search "
r".http://dockertest:8083/sdc/gris_observations.filter="
r"{'.and':.{'description.INSTRUMENT':'gris'},"
with pytest.raises(URLError, match=rf"{_dockerexc('gris')}"
r"{'description.WAVELENGTH_MIN':{'.lte':1250}},"
r"{'description.WAVELENGTH_MAX':{'.gte':1080}}"):
client.search(query)
......@@ -219,9 +274,7 @@ def test_full_range(client):
if len(res[0].get('_embedded')) > 0:
assert 'description' in res[0]['_embedded'][0]
else:
with pytest.raises(URLError, match=r"Unable to execute search "
r".http://dockertest:8083/sdc/bbi_observations.filter="
r"{'.and':.{'description.INSTRUMENT':'bbi'},"
with pytest.raises(URLError, match=rf"{_dockerexc('bbi')}"
r"{'description.DATE_BEG':{'.lte':{'.date':'2019-01-01T00:00:00.000'}}},"
r"{'description.DATE_END':{'.gte':{'.date':'2019-01-09T00:00:00.000'}"):
client.search(query)
......@@ -236,9 +289,7 @@ def test_full_range(client):
assert max(hplt_tan_min) <= 6
assert min(hplt_tan_max) >= -2
else:
with pytest.raises(URLError, match=r"Unable to execute search "
r".http://dockertest:8083/sdc/lars_observations.filter="
r"{'.and':.{'description.INSTRUMENT':'lars'},"
with pytest.raises(URLError, match=rf"{_dockerexc('lars')}"
r"{'description.HPLT_TAN_MIN':{'.lte':6}},"
r"{'description.HPLT_TAN_MAX':{'.gte':-2}}"):
client.search(query)
......@@ -253,9 +304,7 @@ def test_full_range(client):
assert max(hplt_tan_min) <= -2
assert min(hplt_tan_max) >= 6
else:
with pytest.raises(URLError, match=r"Unable to execute search "
r".http://dockertest:8083/sdc/lars_observations.filter="
r"{'.and':.{'description.INSTRUMENT':'lars'},"
with pytest.raises(URLError, match=rf"{_dockerexc('lars')}"
r"{'description.HPLT_TAN_MIN':{'.lte':-2}},"
r"{'description.HPLT_TAN_MAX':{'.gte':6}}"):
client.search(query)
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment