Skip to content

GitLab

Commit 4d3675cb authored by Derek Homeier's avatar Derek Homeier
Browse files

Unpack ['_embedded'][:] records into QueryResponseTable, fix Fido & Time tests

parent ab4f7188
Pipeline #2548 passed with stage
in 3 minutes and 43 seconds
Hide whitespace changes
Inline Side-by-side
.gitlab-ci.yml
View file @ 4d3675cb
......@@ -8,4 +8,4 @@ test:
script:
- pip install pytest-cov
- pip install .
- pytest --cov
- pytest --cov --verbose
sdc/client.py
View file @ 4d3675cb
......@@ -77,6 +77,7 @@ def _str_val(value, regex=False):
if isinstance(value, (int, float, complex)):
return f"{value:g}"
elif isinstance(value, Time):
# Date query format? - f"{{'$date':{int(value.unix * 1000)}}}"
return f"{{'$date':'{value.isot}'}}"
elif regex:
return f"{{'$regex':'{str.lower(value)}'}}"
......@@ -260,8 +261,8 @@ class KISClient(BaseClient):
full_url = f"{self._BASE_URL}{query_string}"
try:
response = urllib.request.urlopen(full_url)
obs_dict = json.loads(response.read())
results.append(obs_dict)
obs_dict = json.loads(response.read()).get('_embedded', [])
results += obs_dict
except(HTTPError, URLError) as exc:
raise URLError(f'Unable to execute search "{full_url}": {exc}. Confirm that '
f'RESTHeart is running on {self._BASE_URL} and connected.')
......@@ -366,12 +367,12 @@ class KISClient(BaseClient):
a.Instrument: [("ChroTel", "Chromospheric Telescope CCD Imager"),
("BBI", "Broadband Context Imager @ Gregor"),
("GRIS", "Gregor Infrared Spectrograph"),
("GFPI", "Gregor Fast Processes Imaging Spectrograph"),
("M-lite", "M-lite 2M Imagers @ Gregor"),
("ECHELLE", "Echelle grating Spectrograph @ VTT"),
("HELLRIDE", "HELioseismic Large Region Interferometric Device @ VTT"),
("TESOS", "TESOS/VIP 2D Fabry-Perot interferometric spectrometer @ VTT"),
("LARS", "Lars is an Absolute Reference Spectrograph @ VTT")],
# ("GFPI", "Gregor Fast Processes Imaging Spectrograph"),
# ("HiFi", "High-resolution Fast Imager @ Gregor"), # 404 error
# ("TIP-II", "Tenerife Infrared Polarimeter @ VTT"), # name?
# ("ZIMPOL", "Zeeman Imaging Polarimeter @ Gregor"), # 404 error
......
sdc/tests/test_client.py
View file @ 4d3675cb
......@@ -34,8 +34,8 @@ def client():
def _dockerexc(instr):
return (rf"Unable to execute search .http://dockertest:8083/sdc/{instr}_observations.filter="
rf"{{'.and':.{{'description.INSTRUMENT':'{instr}'}},")
return (rf"Unable to execute search .http://dockertest:8083/sdc/{instr.lower()}_observations."
rf"filter={{'.and':.{{'description.INSTRUMENT':'{instr.lower()}'}},")
def test_docker(client):
......@@ -51,6 +51,7 @@ def test_docker(client):
response = urllib.request.urlopen(f"{_BASE_URL}{_QUERY_BASE}{_EXAMPLE_DATES}")
data = json.loads(response.read())
assert 'description' in data['_embedded'][0]
assert len(data['_embedded']) > 4
response = urllib.request.urlopen(f"{_BASE_URL}{_QUERY_BASE}{_EXAMPLE_RANGE}")
data = json.loads(response.read())
......@@ -59,13 +60,15 @@ def test_docker(client):
res = client.search(a.Instrument("GRIS") & a.sdc.ObsName('gris_20140426_000'))
assert isinstance(res, QueryResponseTable)
assert len(res) == 1
description = res[0]['_embedded'][0].get('description')
description = res[0].get('description')
assert len(description) == 34
assert description['INSTRUMENT'] == 'gris'
assert description['TELESCOPE'] == 'GREGOR'
assert description['BTYPE'] == 'phot.count'
assert description['DATE_BEG']['$date'] == 1398505619000
assert description['DATE_END']['$date'] == 1398506021300
links = res[0]['_embedded'][0].get('links')
file_ids = [ld['$oid'] for ld in links['l1_data']]
file_ids = [ld['$oid'] for ld in res[0]['links']['l1_data']]
assert len(file_ids) == 105
for oid in file_ids[0], file_ids[104]:
meta = json.loads(urllib.request.urlopen(f"{_BASE_URL}gris_l1_data.files/{oid}").read())
......@@ -104,27 +107,27 @@ def test_search(client):
r"Instrument UVES not in registered list"):
client.search(a.Instrument("UVES") & a.Time("2019/01/01", "2021/01/01"))
query = a.Instrument("BBI") & a.Time("2019/01/01", "2021/01/01")
# TODO: Verify returned observation records.
query = a.Instrument("BBI") & a.Time("2017/05/21", "2017/05/22 22:00")
assert client._can_handle_query(query)
if HAS_DOCKERTEST:
res = client.search(query)
assert isinstance(res, QueryResponseTable)
if len(res[0].get('_embedded')) > 0:
assert 'description' in res[0]['_embedded'][0]
assert len(res) > 50
assert 'description' in res.colnames
else:
with pytest.raises(URLError, match=rf"{_dockerexc('bbi')}"
r"{'description.DATE_BEG':{'.lte':{'.date':'2021-01-01T00:00:00.000'}}},"
r"{'description.DATE_END':{'.gte':{'.date':'2019-01-01T00:00:00.000'}}}"
r"{'description.DATE_BEG':{'.lte':{'.date':'2017-05-22T22:00:00.000'}}},"
r"{'description.DATE_END':{'.gte':{'.date':'2017-05-21T00:00:00.000'}}}"
rf".*Confirm that RESTHeart is running on {_BASE_URL} and connected"):
client.search(query)
# Maximum numer of returned observation records defaults to 100 per query.
query = a.Instrument("LARS") & a.sdc.HelioProjLat(-10*u.arcsec, 0.2*u.arcmin)
assert client._can_handle_query(query)
if HAS_DOCKERTEST:
res = client.search(query)
assert len(res[0]['_embedded']) == 100
assert 'description' in res[0]['_embedded'][0]
assert len(res) == 100
assert 'description' in res.colnames
else:
with pytest.raises(URLError, match=rf"{_dockerexc('lars')}"
r"{'description.HPLT_TAN_MIN':{'.lte':12}},"
......@@ -135,8 +138,8 @@ def test_search(client):
assert client._can_handle_query(*query)
if HAS_DOCKERTEST:
res = client.search(*query)
assert len(res[0]['_embedded']) == 100
assert 'description' in res[0]['_embedded'][0]
assert len(res) == 100
assert 'description' in res.colnames
else:
with pytest.raises(URLError, match=rf"{_dockerexc('lars')}"
r"{'description.HPLT_TAN_MIN':{'.lte':12}},"
......@@ -147,14 +150,12 @@ def test_search(client):
assert client._can_handle_query(query)
if HAS_DOCKERTEST:
res = client.search(query)
assert len(res) == 2
assert len(res[0]['_embedded']) == 100
assert 'description' in res[0]['_embedded'][0]
assert 'THETA' in res[0]['_embedded'][0]['description']
assert 'POL_STATES' in res[1]['_embedded'][0]['description']
theta = [obs['description']['THETA'] for obs in res[0]['_embedded']]
assert (min(theta) >= 50) & (max(theta) <= 85)
assert len(res[1]['_embedded']) == 100
assert len(res) == 200
assert 'description' in res.colnames
assert 'THETA' in res[0]['description']
theta = [obs['description']['THETA'] for obs in res]
assert (min(theta[:100]) >= 50) & (max(theta[:100]) <= 85)
assert res[100]['description']['POL_STATES'] == 'IQUV'
else:
# Will raise on first of multi-part OR queries; somehow switches INSTRUMENT and THETA.
with pytest.raises(URLError, match=rf"{_dockerexc('gris')[:80]}") as exc:
......@@ -162,30 +163,47 @@ def test_search(client):
assert "{'description.THETA':{'$gte':50,'$lte':85}}" in str(exc.value)
assert "{'description.POL_STATES':" not in str(exc.value)
query = a.Instrument("LARS") | a.Instrument("GRIS"), a.sdc.Theta(85*u.deg, 3000*u.arcmin)
if HAS_DOCKERTEST:
res = client.search(*query)
assert len(res) == 200
assert 'description' in res.colnames
assert 'THETA' in res[0]['description']
theta = [obs['description']['THETA'] for obs in res]
assert (min(theta) >= 50) & (max(theta) <= 85)
else:
# Will raise on first of multi-part OR queries; somehow switches INSTRUMENT and THETA.
with pytest.raises(URLError, match=rf"{_dockerexc('LARS')[:80]}") as exc:
client.search(*query)
assert "{'description.THETA':{'$gte':50,'$lte':85}}" in str(exc.value)
assert "{'description.INSTRUMENT':'gris'" not in str(exc.value)
def test_fido_search():
"""Test search using the Fido base class with AttrAnd, AttrOr and lists of *args."""
two_inst = a.Instrument("BBI") | a.Instrument("ChroTel")
two_inst = (a.Instrument("LARS") | a.Instrument("GRIS"))
if HAS_DOCKERTEST:
res = Fido.search(a.Instrument("GRIS") & a.sdc.Theta(50*u.deg, 80*u.deg))
assert len(res['kis']) > 0
assert len(res['kis', '_embedded']) > 0
theta = [obs[0]['description']['THETA'] for obs in res['kis', '_embedded']]
assert len(res['kis']) == 100
theta = [obs['description']['THETA'] for obs in res['kis']]
assert (min(theta) >= 50) & (max(theta) <= 80)
res = Fido.search(a.Instrument("GRIS"), a.sdc.Theta(50*u.deg, 80*u.deg))
assert len(res['kis']) > 0
assert len(res['kis', '_embedded']) > 0
theta = [obs[0]['description']['THETA'] for obs in res['kis']['_embedded']]
assert len(res['kis']) == 100
theta = [obs['description']['THETA'] for obs in res['kis']]
assert (min(theta) >= 50) & (max(theta) <= 80)
res = Fido.search(two_inst, a.sdc.Theta(50*u.deg, 80*u.deg))
assert len(res['kis', '_embedded']) == 2
theta = [obs for obs in res['kis', '_embedded']]
assert 'description' in theta[0]['_embedded'][0].keys()
assert len(res['kis']) == 2
theta = [obs['description']['THETA'] for obs in res['kis'][1]]
assert (min(theta) >= 50) & (max(theta) <= 80)
assert len(res['kis'][0]) == 100
assert res['kis'][0][0]['description']['INSTRUMENT'] == 'lars'
theta = [obs['description']['THETA'] for obs in res['kis'][0]]
assert (min(theta) >= 50) & (max(theta) <= 80)
assert res['kis'][1, 0]['description']['INSTRUMENT'] == 'gris'
theta = [obs['description']['THETA'] for obs in res['kis'][1]]
assert (min(theta) >= 50) & (max(theta) <= 80)
assert res['kis', '_embedded'][0]['description']['INSTRUMENT'] == 'bbi'
assert res['kis', '_embedded'][1]['description']['INSTRUMENT'] == 'chrotel'
else:
with pytest.raises(URLError, match=rf"{_dockerexc('gris')}"
r"{'description.THETA':{'.gte':50,'.lte':80}}"):
......@@ -195,10 +213,10 @@ def test_fido_search():
r"{'description.THETA':{'.gte':50,'.lte':80}}"):
Fido.search(a.Instrument("GRIS"), a.sdc.Theta(50*u.deg, 80*u.deg))
with pytest.raises(URLError, match=rf"{_dockerexc('bbi')}") as exc:
with pytest.raises(URLError, match=rf"{_dockerexc('LARS')}") as exc:
Fido.search(two_inst, a.sdc.Theta(50*u.deg, 80*u.deg))
assert "{'description.THETA':{'$gte':50,'$lte':80}}" in str(exc.value)
assert "{'description.INSTRUMENT':'chrotel'" not in str(exc.value)
assert "{'description.INSTRUMENT':'gris'" not in str(exc.value)
@pytest.mark.parametrize("query", ((a.Instrument("GRIS") & a.Level(3)),
......@@ -215,7 +233,7 @@ def test_cant_handle_query(client, query):
@pytest.mark.parametrize("query", (a.Level(1), a.Wavelength(3200*u.AA, 1.6*u.micron),
a.sdc.DataProduct('cube'), a.sdc.ObsName('gris_20140426_000'),
a.sdc.Date('2021/01/31'), a.sdc.Filter('G'),
a.sdc.Date('2021/01/31'), a.sdc.Filter('LOT%233802'),
a.sdc.PolStates('IQUV'), a.sdc.Telescope('VTT'),
a.sdc.Target('Sunspot_22'), a.sdc.AtmosR0(*([1, 20000]*u.mm)),
a.sdc.Theta(20*u.arcmin, 89*u.deg), a.sdc.Mu(0.1, 1),
......@@ -232,8 +250,8 @@ def test_all_queries(client, query):
assert client._can_handle_query(a.Instrument("GRIS"), query)
if HAS_DOCKERTEST:
res = client.search(a.Instrument("GRIS") & query)
if len(res[0].get('_embedded')) > 0:
assert 'description' in res[0]['_embedded'][0]
if len(res) > 0:
assert 'description' in res.colnames
else:
with pytest.raises(URLError, match=rf"{_dockerexc('gris')}"
rf"{{'description.*{query.type_name.upper()}"):
......@@ -249,8 +267,8 @@ def test_range(client):
assert client._can_handle_query(query)
if HAS_DOCKERTEST:
res = client.search(query)
wave_min = [obs['description']['WAVELENGTH_MIN'] for obs in res[0]['_embedded']]
wave_max = [obs['description']['WAVELENGTH_MAX'] for obs in res[0]['_embedded']]
wave_min = [obs['WAVELENGTH_MIN'] for obs in res['description']]
wave_max = [obs['WAVELENGTH_MAX'] for obs in res['description']]
assert max(wave_min) <= 1250
assert min(wave_max) >= 1080
else:
......@@ -265,18 +283,18 @@ def test_full_range(client):
Test 'fullrange' option - 'FIELD_MIN,_MAX' shall completely include `[Attr.min, Attr.max]`.
"""
t = a.Time("2019/01/01", "2019/01/09")
t = a.Time("2014/04/26 09:50", "2014/04/26 09:52")
t.fullrange = True
query = a.Instrument("BBI") & t
query = a.Instrument("GRIS") & t
assert client._can_handle_query(query)
if HAS_DOCKERTEST:
res = client.search(query)
if len(res[0].get('_embedded')) > 0:
assert 'description' in res[0]['_embedded'][0]
assert len(res) == 1
assert 'description' in res.colnames
else:
with pytest.raises(URLError, match=rf"{_dockerexc('bbi')}"
r"{'description.DATE_BEG':{'.lte':{'.date':'2019-01-01T00:00:00.000'}}},"
r"{'description.DATE_END':{'.gte':{'.date':'2019-01-09T00:00:00.000'}"):
with pytest.raises(URLError, match=rf"{_dockerexc('GRIS')}"
r"{'description.DATE_BEG':{'.lte':{'.date':'2014-04-26T09:50:00.000'}}},"
r"{'description.DATE_END':{'.gte':{'.date':'2014-04-26T09:52:00.000'}"):
client.search(query)
# Test with inverted `min`, `max` inputs.
......@@ -284,12 +302,12 @@ def test_full_range(client):
query = a.Instrument("LARS") & hplt
if HAS_DOCKERTEST:
res = client.search(query)
hplt_tan_min = [obs['description']['HPLT_TAN_MIN'] for obs in res[0]['_embedded']]
hplt_tan_max = [obs['description']['HPLT_TAN_MAX'] for obs in res[0]['_embedded']]
hplt_tan_min = [obs['HPLT_TAN_MIN'] for obs in res['description']]
hplt_tan_max = [obs['HPLT_TAN_MAX'] for obs in res['description']]
assert max(hplt_tan_min) <= 6
assert min(hplt_tan_max) >= -2
else:
with pytest.raises(URLError, match=rf"{_dockerexc('lars')}"
with pytest.raises(URLError, match=rf"{_dockerexc('LARS')}"
r"{'description.HPLT_TAN_MIN':{'.lte':6}},"
r"{'description.HPLT_TAN_MAX':{'.gte':-2}}"):
client.search(query)
......@@ -299,12 +317,12 @@ def test_full_range(client):
assert client._can_handle_query(query)
if HAS_DOCKERTEST:
res = client.search(query)
hplt_tan_min = [obs['description']['HPLT_TAN_MIN'] for obs in res[0]['_embedded']]
hplt_tan_max = [obs['description']['HPLT_TAN_MAX'] for obs in res[0]['_embedded']]
hplt_tan_min = [obs['HPLT_TAN_MIN'] for obs in res['description']]
hplt_tan_max = [obs['HPLT_TAN_MAX'] for obs in res['description']]
assert max(hplt_tan_min) <= -2
assert min(hplt_tan_max) >= 6
else:
with pytest.raises(URLError, match=rf"{_dockerexc('lars')}"
with pytest.raises(URLError, match=rf"{_dockerexc('LARS')}"
r"{'description.HPLT_TAN_MIN':{'.lte':-2}},"
r"{'description.HPLT_TAN_MAX':{'.gte':6}}"):
client.search(query)
setup.py
View file @ 4d3675cb
......@@ -23,7 +23,7 @@ setuptools.setup(
url="https://gitlab.leibniz-kis.de/ajl/python-user-api",
packages=setuptools.find_packages(),
include_package_data=True,
python_requires=">=3.6",
python_requires=">=3.7",
# Adds executable scripts to the installer, they can be executed directly from shell in any environment where the package is installed.
scripts=glob("bin/*"),
# Dependencies are listed in 'requirements.txt'
......@@ -35,7 +35,7 @@ setuptools.setup(
"Development Status :: 4 - Beta",
"License :: OSI Approved :: MIT License",
"Programming Language :: Python",
"Programming Language :: Python :: 3.6",
"Programming Language :: Python :: 3.7",
"Topic :: Software Development :: Libraries",
"Topic :: Software Development :: Libraries :: Python Modules",
"Intended Audience :: Developers",
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment