Commit 588d8af7 authored by Derek Homeier's avatar Derek Homeier
Browse files

Test basic queries; fix $lte/$gte filters and sanity check results

parent 89e18371
Pipeline #2463 passed with stage
in 3 minutes and 41 seconds
......@@ -12,6 +12,7 @@ import sdc.attrs as sattrs
import urllib.parse
import urllib.request
from urllib.error import HTTPError, URLError
import json
walker = AttrWalker()
......@@ -71,11 +72,11 @@ def _str_val(value, regex=False):
if isinstance(value, (int, float, complex)):
return f"{value:g}"
elif isinstance(value, Time):
return f"'{value.isot}'"
return f"{{'$date':'{value.isot}'}}"
elif regex:
return f"{{'$regex':'{str.lower(value)}}}'"
return f"{{'$regex':'{str.lower(value)}'}}"
else:
return f"'{str.lower(value)}'"
return f"'{str(value)}'"
def _update_val(dictionary, key, value, block='description', regex=False):
......@@ -83,12 +84,15 @@ def _update_val(dictionary, key, value, block='description', regex=False):
Update dictionary with field_name:value string in format parseable by BSON filter.
"""
strval = _str_val(value, regex)
if key in ('POL_STATES'):
# Some fields are always lower- some uppercase...
if key in ('BTYPE', 'INSTRUMENT', 'OBS_COLLECTION'):
strval = strval.lower()
if key in ('POL_STATES', 'TELESCOPE'):
strval = strval.upper()
# Special case for fields having a _MIN and _MAX form - require MIN <= value <= MAX:
if key.endswith('_'):
query = (f"{{'{block}.{key}MIN':{{'$le':{{{strval}}}}}}},"
f"{{'{block}.{key}MAX':{{'$ge':{{{strval}}}}}}}")
query = (f"{{'{block}.{key}MIN':{{'$lte':{strval}}}}},"
f"{{'{block}.{key}MAX':{{'$gte':{strval}}}}}")
else:
query = f"{{'{block}.{key}':{strval}}}"
return dictionary.update({key: query})
......@@ -103,15 +107,14 @@ def _update_range(dictionary, key, values, block='description', fullrange=False)
if key.endswith('_'):
if fullrange:
# Cover full search range: MIN >= values[:] >= MAX
query = (f"{{'{block}.{key}MIN':{{'$le':{{{strvals[0]}}}}}}},"
f"{{'{block}.{key}MAX':{{'$ge':{{{strvals[1]}}}}}}}")
query = (f"{{'{block}.{key}MIN':{{'$lte':{strvals[0]}}}}},"
f"{{'{block}.{key}MAX':{{'$gte':{strvals[1]}}}}}")
else:
# Default: MIN <= values[1], MAX >= values[0]
query = (f"{{'{block}.{key}MIN':{{'$le':{{{strvals[1]}}}}}}},"
f"{{'{block}.{key}MAX':{{'$ge':{{{strvals[0]}}}}}}}")
query = (f"{{'{block}.{key}MIN':{{'$lte':{strvals[1]}}}}},"
f"{{'{block}.{key}MAX':{{'$gte':{strvals[0]}}}}}")
else:
query = (f"{{'{block}.{key}':{{'$ge':{{{strvals[0]}}},"
f"'$le':{{{strvals[1]}}}}}}}")
query = f"{{'{block}.{key}':{{'$gte':{strvals[0]},'$lte':{strvals[1]}}}}}"
return dictionary.update({key: query})
......@@ -212,6 +215,9 @@ class KISClient(BaseClient):
for query_params in query_dicts:
collection = query_params.pop('OBS_COLLECTION')
instrument = query_params.get('INSTRUMENT')
if not collection.split('_')[0] in instrument.lower():
raise ValueError(f"Mismatch of '{instrument}' in '{collection}.")
# Search for existing dict for same collection (instrument) to update:
# index_obs = [i for i, d in enumerate(params) if collection in d.values()]
# if len(index_obs) > 0:
......@@ -245,8 +251,9 @@ class KISClient(BaseClient):
for query_string in queries:
full_url = f"{self._BASE_URL}{query_string}"
try:
data = urllib.request.urlopen(full_url)
results.append(data)
response = urllib.request.urlopen(full_url)
obs_dict = json.loads(response.read())
results.append(obs_dict)
except(HTTPError, URLError) as exc:
raise URLError(f'Unable to execute search "{full_url}": {exc}. Confirm that '
f'RESTHeart is running on {self._BASE_URL} and connected.')
......@@ -338,14 +345,14 @@ class KISClient(BaseClient):
("BBI", "Broadband Context Imager @ Gregor"),
("GRIS", "Gregor Infrared Spectrograph"),
("GFPI", "Gregor Fast Processes Imaging Spectrograph"),
("HiFi", "High-resolution Fast Imager @ Gregor"),
("M-lite", "M-lite 2M Imagers @ Gregor"),
("ZIMPOL", "Zeeman Imaging Polarimeter @ Gregor"),
("ECHELLE", "Echelle grating Spectrograph @ VTT"),
("HELLRIDE", "HELioseismic Large Region Interferometric Device @ VTT"),
("TESOS", "TESOS/VIP 2D Fabry-Perot interferometric spectrometer @ VTT"),
("TIP-I", "Tenerife Infrared Polarimeter @ VTT"),
("LARS", "Lars is an Absolute Reference Spectrograph @ VTT")],
# ("HiFi", "High-resolution Fast Imager @ Gregor"), # 404 error
# ("TIP-I", "Tenerife Infrared Polarimeter @ VTT"), # name?
# ("ZIMPOL", "Zeeman Imaging Polarimeter @ Gregor"), # 404 error
# description.TELESCOPE - Name of the telescope
a.sdc.Telescope: [("ChroTel", "10 cm Chromospheric Telescope, Observatorio del Teide"),
("Gregor", "150 cm Gregor Solar Telescope, Observatorio del Teide"),
......
import urllib.request
from urllib.error import HTTPError, URLError
import json
import pytest
......@@ -10,9 +11,15 @@ from sdc.client import KISClient
_BASE_URL = "http://dockertest:8083/sdc/"
_QUERY_BASE = "gris_observations?filter="
_EXAMPLE_QUERY = "{'description.OBS_NAME':'gris_20140426_000'}"
_EXAMPLE_RANGE = "{'description.THETA':{'$gt':70.0,'$lt':80}}"
_EXAMPLE_DATES = ("{'$and':[{'description.INSTRUMENT':'gris'},"
"{'description.DATE_BEG':{'$gte':{'$date':'2014-04-26T00:00:00'},"
"'$lte':{'$date':'2014-04-27T00:00:00'}}}]}")
try:
res = urllib.request.urlopen(_BASE_URL)
response = urllib.request.urlopen(f"{_BASE_URL}{_QUERY_BASE}{_EXAMPLE_QUERY}")
HAS_DOCKERTEST = True
except(HTTPError, URLError):
HAS_DOCKERTEST = False
......@@ -23,6 +30,32 @@ def client():
return KISClient()
def test_docker(client):
"""Test example queries on dockertest."""
if not HAS_DOCKERTEST:
pytest.xfail("No dockertest running")
response = urllib.request.urlopen(f"{_BASE_URL}{_QUERY_BASE}{_EXAMPLE_QUERY}")
data = json.loads(response.read())
assert '_embedded' in data.keys()
assert 'description' in data['_embedded'][0]
response = urllib.request.urlopen(f"{_BASE_URL}{_QUERY_BASE}{_EXAMPLE_DATES}")
data = json.loads(response.read())
assert 'description' in data['_embedded'][0]
response = urllib.request.urlopen(f"{_BASE_URL}{_QUERY_BASE}{_EXAMPLE_RANGE}")
data = json.loads(response.read())
assert 'description' in data['_embedded'][0]
res = client.search(a.Instrument("GRIS") & a.sdc.ObsName('gris_20140426_000'))
assert len(res) == 1
description = res[0]['_embedded'][0].get('description')
assert description['INSTRUMENT'] == 'gris'
assert description['TELESCOPE'] == 'GREGOR'
assert description['BTYPE'] == 'phot.count'
def test_search(client):
"""Test conversion of (supported) Attrs to query string."""
......@@ -37,16 +70,18 @@ def test_search(client):
client.search(a.Instrument("UVES") & a.Time("2019/01/01", "2021/01/01"))
query = a.Instrument("BBI") & a.Time("2019/01/01", "2021/01/01")
# TODO: Verify the actual return from the RESTHeart server.
# TODO: Verify returned observation records.
assert client._can_handle_query(query)
if HAS_DOCKERTEST:
res = client.search(query)
if len(res) > 0 and len(res[0].get('_embedded')) > 0:
assert 'description' in res[0]['_embedded'][0]
else:
with pytest.raises(URLError, match=r"Unable to execute search "
r".http://dockertest:8083/sdc/bbi_observations.filter="
r"{'.and':.{'description.INSTRUMENT':'bbi'},"
r"{'description.DATE_BEG':{'.le':{'2021-01-01T00:00:00.000'}}},"
r"{'description.DATE_END':{'.ge':{'2019-01-01T00:00:00.000'}}}.}"
r"{'description.DATE_BEG':{'.lte':{'.date':'2021-01-01T00:00:00.000'}}},"
r"{'description.DATE_END':{'.gte':{'.date':'2019-01-01T00:00:00.000'}}}"
rf".*Confirm that RESTHeart is running on {_BASE_URL} and connected"):
client.search(query)
......@@ -54,24 +89,28 @@ def test_search(client):
assert client._can_handle_query(query)
if HAS_DOCKERTEST:
res = client.search(query)
if len(res) > 0 and len(res[0].get('_embedded')) > 0:
assert 'description' in res[0]['_embedded'][0]
else:
with pytest.raises(URLError, match=r"Unable to execute search "
r".http://dockertest:8083/sdc/lars_observations.filter="
r"{'.and':.{'description.INSTRUMENT':'lars'},"
r"{'description.HPLT_TAN_MIN':{'.le':{12\}}},"
r"{'description.HPLT_TAN_MAX':{'.ge':{-10\}}}"):
r"{'description.HPLT_TAN_MIN':{'.lte':12}},"
r"{'description.HPLT_TAN_MAX':{'.gte':-10}}"):
client.search(query)
query = a.Instrument("Zimpol") & (a.sdc.Theta(85*u.deg, 600*u.arcmin) | a.sdc.PolStates('iq'))
query = a.Instrument("GRIS") & (a.sdc.Theta(85*u.deg, 600*u.arcmin) | a.sdc.PolStates('iq'))
assert client._can_handle_query(query)
if HAS_DOCKERTEST:
res = client.search(query)
if len(res) > 0 and len(res[0].get('_embedded')) > 0:
assert 'description' in res[0]['_embedded'][0]
else:
with pytest.raises(URLError, match=r"Unable to execute search ") as exc:
client.search(query)
# Will raise on first of multi-part OR queries.
assert "http://dockertest:8083/sdc/zimpol_observations?filter=" in str(exc.value)
assert "{'description.THETA':{'$ge':{10},'$le':{85}}}" in str(exc.value)
assert "http://dockertest:8083/sdc/gris_observations?filter=" in str(exc.value)
assert "{'description.THETA':{'$gte':10,'$lte':85}}" in str(exc.value)
assert "{'description.POL_STATES':" not in str(exc.value)
......@@ -88,17 +127,17 @@ def test_cant_handle_query(client, query):
@pytest.mark.parametrize("query", (a.Level(1), a.Wavelength(3200*u.AA, 1.6*u.micron),
a.sdc.DataProduct('cube'), a.sdc.ObsName('gris_211031_12'),
a.sdc.Date('2021/10/31'), a.sdc.Filter('I'),
a.sdc.PolStates('IQUV'), a.sdc.Telescope('Gregor'),
a.sdc.Target('Sunspot_22'), a.sdc.AtmosR0(*([10, 200]*u.cm)),
a.sdc.DataProduct('cube'), a.sdc.ObsName('gris_20140426_000'),
a.sdc.Date('2021/01/31'), a.sdc.Filter('G'),
a.sdc.PolStates('IQUV'), a.sdc.Telescope('VTT'),
a.sdc.Target('Sunspot_22'), a.sdc.AtmosR0(*([1, 20000]*u.mm)),
a.sdc.Theta(20*u.arcmin, 89*u.deg), a.sdc.Mu(0.1, 1),
a.sdc.ExposureTime(*([5, 60]*u.min)),
a.sdc.HelioProjLon(5*u.arcsec), a.sdc.HelioProjLat(9*u.arcsec),
a.sdc.SpatialResolution(0.1*u.arcsec, 0.8*u.arcsec),
a.sdc.SpectralResolution(600, 20000),
a.sdc.SpectralResolution(6000, 200000),
a.sdc.TemporalResolution(2*u.s, 30*u.s),
a.sdc.NDimensions(2, 3), a.sdc.PolXel(2, 4),
a.sdc.NDimensions(1, 2), a.sdc.PolXel(2, 4),
a.sdc.SpatialXel1(200, 3000), a.sdc.SpatialXel2(100, 4000),
a.sdc.SpectralXel(320, 4096), a.sdc.TimeXel(60, 86400)))
def test_all_queries(client, query):
......@@ -106,6 +145,8 @@ def test_all_queries(client, query):
assert client._can_handle_query(a.Instrument("GRIS"), query)
if HAS_DOCKERTEST:
res = client.search(a.Instrument("GRIS") & query)
if len(res) > 0 and len(res[0].get('_embedded')) > 0:
assert 'description' in res[0]['_embedded'][0]
else:
with pytest.raises(URLError, match=r"Unable to execute search "
r".http://dockertest:8083/sdc/gris_observations.filter="
......@@ -125,12 +166,14 @@ def test_full_range(client):
assert client._can_handle_query(query)
if HAS_DOCKERTEST:
res = client.search(query)
if len(res) > 0 and len(res[0].get('_embedded')) > 0:
assert 'description' in res[0]['_embedded'][0]
else:
with pytest.raises(URLError, match=r"Unable to execute search "
r".http://dockertest:8083/sdc/bbi_observations.filter="
r"{'.and':.{'description.INSTRUMENT':'bbi'},"
r"{'description.DATE_BEG':{'.le':{'2019-01-01T00:00:00.000'}}},"
r"{'description.DATE_END':{'.ge':{'2019-01-09T00:00:00.000'}}}.}"):
r"{'description.DATE_BEG':{'.lte':{'.date':'2019-01-01T00:00:00.000'}}},"
r"{'description.DATE_END':{'.gte':{'.date':'2019-01-09T00:00:00.000'}"):
client.search(query)
# Test with inverted `min`, `max` inputs.
......@@ -140,10 +183,12 @@ def test_full_range(client):
assert client._can_handle_query(query)
if HAS_DOCKERTEST:
res = client.search(query)
if len(res) > 0 and len(res[0].get('_embedded')) > 0:
assert 'description' in res[0]['_embedded'][0]
else:
with pytest.raises(URLError, match=r"Unable to execute search "
r".http://dockertest:8083/sdc/lars_observations.filter="
r"{'.and':.{'description.INSTRUMENT':'lars'},"
r"{'description.HPLT_TAN_MIN':{'.le':{-10\}}},"
r"{'description.HPLT_TAN_MAX':{'.ge':{12\}}}"):
r"{'description.HPLT_TAN_MIN':{'.lte':-10}},"
r"{'description.HPLT_TAN_MAX':{'.gte':12\}}"):
client.search(query)
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment