Commit 054c1bd5 authored by Derek Homeier's avatar Derek Homeier
Browse files

Unpack obsrec description into Quantity columns

parent 00357eb4
Pipeline #2756 failed with stage
in 3 minutes and 16 seconds
......@@ -261,6 +261,17 @@ class KISClient(BaseClient):
results : `~sunpy.net.base_client.QueryResponseTable`
A `QueryResponseTable` instance containing the query result.
"""
# Unit converters for 'description' fields
converters = {'ATMOS': lambda x: x * u.cm,
'DATE_': lambda x: Time(x['$date'] * 1e-3, format='unix'),
'EXPTI': lambda x: x * u.s,
'HPLN_': lambda x: x * u.arcsec,
'HPLT_': lambda x: x * u.arcsec,
'S_RES': lambda x: x * u.arcsec,
'THETA': lambda x: x * u.deg,
'T_RES': lambda x: x * u.deg,
'WAVE_': lambda x: x * u.nm}
query = and_(*query)
# Not very extensively tested on hierarchical queries
......@@ -277,6 +288,10 @@ class KISClient(BaseClient):
obs_dict = json.loads(response.read()).get('_embedded', [])
if len(obs_dict) > 0 and 'links' in obs_dict[0]:
l1_data = obs_dict[0]['links'].get('l1_data', [])
obs_dict[0].update(obs_dict[0].pop('description', dict()))
for k in obs_dict[0].keys():
if k[:5] in converters:
obs_dict[0][k] = converters[k[:5]](obs_dict[0][k])
obs_rec = []
for exp in l1_data:
obs_rec.append(copy.deepcopy(obs_dict[0]))
......@@ -316,11 +331,11 @@ class KISClient(BaseClient):
ext = 'json'
for row in query_results:
inst = row['description']['INSTRUMENT']
inst = row['INSTRUMENT']
oid = row['links']['l1_data']['$oid']
# Content-Disposition header default is "{row['_id']['$oid']}/{oid}.{ext}" (no '.json').
# rowpath = row['_id']['$oid']
filepath = os.path.join(row['description']['OBS_NAME'], f"{oid}.{ext}")
filepath = os.path.join(row['OBS_NAME'], f"{oid}.{ext}")
url = f"{self._BASE_URL}{inst}_l1_data.files/{oid}{binfile}"
downloader.enqueue_file(url, filename=str(path).format(file=filepath,
**row.response_block_map))
......
......@@ -69,13 +69,11 @@ def test_docker(client):
res = client.search(a.Instrument("GRIS") & a.sdc.ObsName('gris_20140426_000'))
assert isinstance(res, QueryResponseTable)
assert len(res) == 105
description = res[0].get('description')
assert len(description) == 34
assert description['INSTRUMENT'] == 'gris'
assert description['TELESCOPE'] == 'GREGOR'
assert description['BTYPE'] == 'phot.count'
assert description['DATE_BEG']['$date'] == 1398505619000
assert description['DATE_END']['$date'] == 1398506021300
assert res[0]['INSTRUMENT'] == 'gris'
assert res[0]['TELESCOPE'] == 'GREGOR'
assert res[0]['BTYPE'] == 'phot.count'
assert res[0]['DATE_BEG'].unix == 1398505619.000
assert res[0]['DATE_END'].unix == 1398506021.300
file_ids = [ld['links']['l1_data']['$oid'] for ld in res]
assert len(file_ids) == 105
......@@ -89,7 +87,7 @@ def test_docker(client):
date = a.Time("2014/04/26 01:00", "2014/04/26 22:00")
downloader = parfive.Downloader()
inst = res[0]['description']['INSTRUMENT']
inst = res[0]['INSTRUMENT']
rowpath = f"{res[0]['_id']['$oid']}"
binfile = ''
ext = 'json'
......@@ -166,7 +164,7 @@ def test_search(client):
res = client.search(query)
assert isinstance(res, QueryResponseTable)
assert len(res) == 1
assert 'description' in res.colnames
assert 'INSTRUMENT' in res.colnames
else:
with pytest.raises(URLError, match=rf"{_dockerexc('bbi')}"
r"{'description.DATE_BEG':{'.lte':{'.date':'2017-05-22T22:00:00.000'}}},"
......@@ -180,7 +178,7 @@ def test_search(client):
if HAS_DOCKERTEST:
res = client.search(query)
assert len(res) == 1
assert 'description' in res.colnames
assert 'INSTRUMENT' in res.colnames
else:
with pytest.raises(URLError, match=rf"{_dockerexc('lars')}"
r"{'description.HPLT_TAN_MIN':{'.lte':12}},"
......@@ -192,7 +190,7 @@ def test_search(client):
if HAS_DOCKERTEST:
res = client.search(*query)
assert len(res) == 1
assert 'description' in res.colnames
assert 'INSTRUMENT' in res.colnames
else:
with pytest.raises(URLError, match=rf"{_dockerexc('lars')}"
r"{'description.HPLT_TAN_MIN':{'.lte':12}},"
......@@ -204,11 +202,9 @@ def test_search(client):
if HAS_DOCKERTEST:
res = client.search(query)
assert len(res) == 362
assert 'description' in res.colnames
assert 'THETA' in res[0]['description']
theta = [obs['description']['THETA'] for obs in res]
assert (min(theta[:100]) >= 50) & (max(theta[:100]) <= 85)
assert res[100]['description']['POL_STATES'] == 'IQUV'
assert 'THETA' in res.colnames
assert (min(res['THETA'][:100]) >= 50*u.deg) & (max(res['THETA'][:100]) <= 85*u.deg)
assert res[100]['POL_STATES'] == 'IQUV'
else:
# Will raise on first of multi-part OR queries; somehow switches INSTRUMENT and THETA.
with pytest.raises(URLError, match=rf"{_dockerexc('gris')[:80]}") as exc:
......@@ -220,10 +216,8 @@ def test_search(client):
if HAS_DOCKERTEST:
res = client.search(*query)
assert len(res) == 163
assert 'description' in res.colnames
assert 'THETA' in res[0]['description']
theta = [obs['description']['THETA'] for obs in res]
assert (min(theta) >= 50) & (max(theta) <= 85)
assert 'THETA' in res.colnames
assert (min(res['THETA']) >= 50*u.deg) & (max(res['THETA']) <= 85*u.deg)
else:
# Will raise on first of multi-part OR queries; somehow switches INSTRUMENT and THETA.
with pytest.raises(URLError, match=rf"{_dockerexc('LARS')[:80]}") as exc:
......@@ -238,19 +232,17 @@ def test_fido_search():
if HAS_DOCKERTEST:
res = Fido.search(a.Instrument("GRIS") & a.sdc.Theta(50*u.deg, 80*u.deg))
assert len(res['kis']) == 162
theta = [obs['description']['THETA'] for obs in res['kis']]
assert (min(theta) >= 50) & (max(theta) <= 80)
assert (min(res['kis']['THETA']) >= 50*u.deg) & (max(res['kis']['THETA']) <= 80*u.deg)
res = Fido.search(a.Instrument("GRIS"), a.sdc.Theta(50*u.deg, 80*u.deg))
assert len(res['kis']) == 162
theta = [obs['description']['THETA'] for obs in res['kis']]
assert (min(theta) >= 50) & (max(theta) <= 80)
assert (min(res['kis']['THETA']) >= 50*u.deg) & (max(res['kis']['THETA']) <= 80*u.deg)
date = a.Time("2017/05/12 01:40", "2017/05/16 19:00")
res = Fido.search(a.Instrument("LARS") & date)
assert len(res['kis']) > 0
date_beg = [obs['description']['DATE_BEG']['$date'] for obs in res['kis']]
date_end = [obs['description']['DATE_END']['$date'] for obs in res['kis']]
date_beg = [obs['DATE_BEG'] for obs in res['kis']]
date_end = [obs['DATE_END'] for obs in res['kis']]
assert max(date_beg) < date.end.unix * 1000
assert min(date_end) > date.start.unix * 1000
......@@ -258,17 +250,19 @@ def test_fido_search():
assert len(res['kis']) == 2
assert len(res['kis'][0]) == 1
assert len(res['kis'][1]) == 162
assert res['kis'][0][0]['description']['INSTRUMENT'] == 'lars'
theta = [obs['description']['THETA'] for obs in res['kis'][0]]
assert (min(theta) >= 50) & (max(theta) <= 80)
assert res['kis'][1, 0]['description']['INSTRUMENT'] == 'gris'
theta = [obs['description']['THETA'] for obs in res['kis'][1]]
assert (min(theta) >= 50) & (max(theta) <= 80)
assert res['kis'][0][0]['INSTRUMENT'] == 'lars'
assert res['kis'][1, 0]['INSTRUMENT'] == 'gris'
assert (min(res[0]['kis']['THETA']) >= 50*u.deg) & (max(res[0]['kis']['THETA']) <= 80*u.deg)
assert (min(res[1]['kis']['THETA']) >= 50*u.deg) & (max(res[1]['kis']['THETA']) <= 80*u.deg)
theta = [obs['THETA'] for obs in res['kis'][0]]
assert (min(theta) >= 50*u.deg) & (max(theta) <= 80*u.deg)
theta = [obs['THETA'] for obs in res['kis'][1]]
assert (min(theta) >= 50*u.deg) & (max(theta) <= 80*u.deg)
date = a.Time("2016/08/26 16:25", "2016/08/26 16:45")
res = Fido.search(a.Instrument("GRIS"), a.sdc.PolStates('iquv'), date)
assert len(res['kis']) == 400
assert res['kis'][0]['description']['POL_STATES'] == 'IQUV'
assert res['kis'][0]['POL_STATES'] == 'IQUV'
else:
with pytest.raises(URLError, match=rf"{_dockerexc('gris')}"
r"{'description.THETA':{'.gte':50,'.lte':80}}"):
......@@ -292,10 +286,10 @@ def test_fido_fetch():
date = a.Time("2017/05/22 08:45", "2017/05/22 08:55")
res = Fido.search(a.Instrument("BBI"), date)
assert len(res['kis']) == 10
desc = [r['description'] for r in res['kis']]
desc = res['kis']
assert max([d['DATE_BEG']['$date'] for d in desc]) < date.end.unix * 1000
assert min([d['DATE_END']['$date'] for d in desc]) > date.start.unix * 1000
assert max([d['DATE_BEG'] for d in desc]) < date.end.unix * 1000
assert min([d['DATE_END'] for d in desc]) > date.start.unix * 1000
files = Fido.fetch(res['kis'])
assert len(files) == 10
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment