test_client.py 14.7 KB
Newer Older
1
2
import pytest

3
4
import urllib.request
from urllib.error import HTTPError, URLError
5
import json
6

7
import astropy.units as u
8
from astropy.io import fits
9
from sunpy.net import Fido
10
from sunpy.net import attrs as a
11
from sunpy.net.base_client import QueryResponseTable
12

13
from sdc.client import KISClient
14
15
16


_BASE_URL = "http://dockertest:8083/sdc/"
17
18
19
20
21
22
_QUERY_BASE = "gris_observations?filter="
_EXAMPLE_QUERY = "{'description.OBS_NAME':'gris_20140426_000'}"
_EXAMPLE_RANGE = "{'description.THETA':{'$gt':70.0,'$lt':80}}"
_EXAMPLE_DATES = ("{'$and':[{'description.INSTRUMENT':'gris'},"
                  "{'description.DATE_BEG':{'$gte':{'$date':'2014-04-26T00:00:00'},"
                  "'$lte':{'$date':'2014-04-27T00:00:00'}}}]}")
23
24

try:
25
    response = urllib.request.urlopen(f"{_BASE_URL}{_QUERY_BASE}{_EXAMPLE_QUERY}")
26
27
28
29
30
31
32
    HAS_DOCKERTEST = True
except(HTTPError, URLError):
    HAS_DOCKERTEST = False


@pytest.fixture
def client():
33
    return KISClient()
34
35


36
def _dockerexc(instr):
37
38
    return (rf"Unable to execute search .http://dockertest:8083/sdc/{instr.lower()}_observations."
            rf"filter={{'.and':.{{'description.INSTRUMENT':'{instr.lower()}'}},")
39
40


41
42
43
44
45
46
47
48
49
50
51
52
53
def test_docker(client):
    """Test example queries on dockertest."""
    if not HAS_DOCKERTEST:
        pytest.xfail("No dockertest running")

    response = urllib.request.urlopen(f"{_BASE_URL}{_QUERY_BASE}{_EXAMPLE_QUERY}")
    data = json.loads(response.read())
    assert '_embedded' in data.keys()
    assert 'description' in data['_embedded'][0]

    response = urllib.request.urlopen(f"{_BASE_URL}{_QUERY_BASE}{_EXAMPLE_DATES}")
    data = json.loads(response.read())
    assert 'description' in data['_embedded'][0]
54
    assert len(data['_embedded']) > 4
55
56
57
58
59
60

    response = urllib.request.urlopen(f"{_BASE_URL}{_QUERY_BASE}{_EXAMPLE_RANGE}")
    data = json.loads(response.read())
    assert 'description' in data['_embedded'][0]

    res = client.search(a.Instrument("GRIS") & a.sdc.ObsName('gris_20140426_000'))
61
    assert isinstance(res, QueryResponseTable)
62
    assert len(res) == 1
63
64
    description = res[0].get('description')
    assert len(description) == 34
65
66
67
    assert description['INSTRUMENT'] == 'gris'
    assert description['TELESCOPE'] == 'GREGOR'
    assert description['BTYPE'] == 'phot.count'
68
69
    assert description['DATE_BEG']['$date'] == 1398505619000
    assert description['DATE_END']['$date'] == 1398506021300
70

71
    file_ids = [ld['$oid'] for ld in res[0]['links']['l1_data']]
72
73
74
75
76
77
78
79
    assert len(file_ids) == 105
    for oid in file_ids[0], file_ids[104]:
        meta = json.loads(urllib.request.urlopen(f"{_BASE_URL}gris_l1_data.files/{oid}").read())
        assert meta['_id']['$oid'] == oid
        hdulist = fits.open(f"{_BASE_URL}gris_l1_data.files/{oid}/binary")
        assert hdulist[0].header.get('TELESCOP') == 'GREGOR'
        assert '2014-04-26T' in hdulist[0].header.get('DATE-OBS')
        hdulist.close()
80
81
82
83
84
85
86
87
88
89
90
91
92
93


def test_gridfs(client):
    """Test gridfs access on dockertest."""
    pytest.xfail("No GridFS on gitlab-runner")
    import gridfs
    from kis_tools.generic import get_sdc_connection

    res = client.search(a.Instrument("GRIS") & a.sdc.ObsName('gris_20140426_000'))
    links = res[0]['_embedded'][0].get('links')
    file_ids = [ld['$oid'] for ld in links['l1_data']]
    sdc = get_sdc_connection()
    gfs = gridfs.GridFS(sdc.sdc_test, "gris_l1_data")
    found = gfs.find({"_id": {"$in": file_ids}})
94
    assert(len(found)) == 105
95

96

97
def test_search(client):
98
    """Test conversion of (supported) Attrs to query string."""
99

100
    assert not client._can_handle_query(a.Time("2019/01/01", "2021/01/01"))
101
102
103
104
    with pytest.raises(AttributeError, match=r"Query not possible: "
                       r"No 'Instrument' found in Attributes"):
        client.search(a.Time("2019/01/01", "2021/01/01"))

105
    assert not client._can_handle_query(a.Instrument("UVES"), a.Time("2019/01/01", "2021/01/01"))
106
107
108
109
    with pytest.raises(AttributeError, match=r"Query not possible: "
                       r"Instrument UVES not in registered list"):
        client.search(a.Instrument("UVES") & a.Time("2019/01/01", "2021/01/01"))

110
    query = a.Instrument("BBI") & a.Time("2017/05/21", "2017/05/22 22:00")
111
    assert client._can_handle_query(query)
112
    if HAS_DOCKERTEST:
113
        res = client.search(query)
114
        assert isinstance(res, QueryResponseTable)
115
116
        assert len(res) > 50
        assert 'description' in res.colnames
117
    else:
118
        with pytest.raises(URLError, match=rf"{_dockerexc('bbi')}"
119
120
                           r"{'description.DATE_BEG':{'.lte':{'.date':'2017-05-22T22:00:00.000'}}},"
                           r"{'description.DATE_END':{'.gte':{'.date':'2017-05-21T00:00:00.000'}}}"
121
                           rf".*Confirm that RESTHeart is running on {_BASE_URL} and connected"):
122
123
            client.search(query)

124
    # Maximum numer of returned observation records defaults to 100 per query.
125
126
127
128
    query = a.Instrument("LARS") & a.sdc.HelioProjLat(-10*u.arcsec, 0.2*u.arcmin)
    assert client._can_handle_query(query)
    if HAS_DOCKERTEST:
        res = client.search(query)
129
130
        assert len(res) == 100
        assert 'description' in res.colnames
131
    else:
132
        with pytest.raises(URLError, match=rf"{_dockerexc('lars')}"
133
134
                           r"{'description.HPLT_TAN_MIN':{'.lte':12}},"
                           r"{'description.HPLT_TAN_MAX':{'.gte':-10}}"):
135
136
            client.search(query)

137
138
139
140
    query = a.Instrument("LARS"), a.sdc.HelioProjLat(-10*u.arcsec, 0.2*u.arcmin)
    assert client._can_handle_query(*query)
    if HAS_DOCKERTEST:
        res = client.search(*query)
141
142
        assert len(res) == 100
        assert 'description' in res.colnames
143
144
145
146
147
148
149
    else:
        with pytest.raises(URLError, match=rf"{_dockerexc('lars')}"
                           r"{'description.HPLT_TAN_MIN':{'.lte':12}},"
                           r"{'description.HPLT_TAN_MAX':{'.gte':-10}}"):
            client.search(*query)

    query = a.Instrument("GRIS") & (a.sdc.Theta(85*u.deg, 3000*u.arcmin) | a.sdc.PolStates('iquv'))
150
151
152
    assert client._can_handle_query(query)
    if HAS_DOCKERTEST:
        res = client.search(query)
153
154
155
156
157
158
        assert len(res) == 200
        assert 'description' in res.colnames
        assert 'THETA' in res[0]['description']
        theta = [obs['description']['THETA'] for obs in res]
        assert (min(theta[:100]) >= 50) & (max(theta[:100]) <= 85)
        assert res[100]['description']['POL_STATES'] == 'IQUV'
159
    else:
160
161
        # Will raise on first of multi-part OR queries; somehow switches INSTRUMENT and THETA.
        with pytest.raises(URLError, match=rf"{_dockerexc('gris')[:80]}") as exc:
162
            client.search(query)
163
        assert "{'description.THETA':{'$gte':50,'$lte':85}}" in str(exc.value)
164
165
        assert "{'description.POL_STATES':" not in str(exc.value)

166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
    query = a.Instrument("LARS") | a.Instrument("GRIS"), a.sdc.Theta(85*u.deg, 3000*u.arcmin)
    if HAS_DOCKERTEST:
        res = client.search(*query)
        assert len(res) == 200
        assert 'description' in res.colnames
        assert 'THETA' in res[0]['description']
        theta = [obs['description']['THETA'] for obs in res]
        assert (min(theta) >= 50) & (max(theta) <= 85)
    else:
        # Will raise on first of multi-part OR queries; somehow switches INSTRUMENT and THETA.
        with pytest.raises(URLError, match=rf"{_dockerexc('LARS')[:80]}") as exc:
            client.search(*query)
        assert "{'description.THETA':{'$gte':50,'$lte':85}}" in str(exc.value)
        assert "{'description.INSTRUMENT':'gris'" not in str(exc.value)

181

182
183
def test_fido_search():
    """Test search using the Fido base class with AttrAnd, AttrOr and lists of *args."""
184
    two_inst = (a.Instrument("LARS") | a.Instrument("GRIS"))
185
186
    if HAS_DOCKERTEST:
        res = Fido.search(a.Instrument("GRIS") & a.sdc.Theta(50*u.deg, 80*u.deg))
187
188
        assert len(res['kis']) == 100
        theta = [obs['description']['THETA'] for obs in res['kis']]
189
190
191
        assert (min(theta) >= 50) & (max(theta) <= 80)

        res = Fido.search(a.Instrument("GRIS"), a.sdc.Theta(50*u.deg, 80*u.deg))
192
193
        assert len(res['kis']) == 100
        theta = [obs['description']['THETA'] for obs in res['kis']]
194
195
196
        assert (min(theta) >= 50) & (max(theta) <= 80)

        res = Fido.search(two_inst, a.sdc.Theta(50*u.deg, 80*u.deg))
197
198
199
200
201
202
203
204
205
        assert len(res['kis']) == 2
        theta = [obs['description']['THETA'] for obs in res['kis'][1]]
        assert (min(theta) >= 50) & (max(theta) <= 80)
        assert len(res['kis'][0]) == 100
        assert res['kis'][0][0]['description']['INSTRUMENT'] == 'lars'
        theta = [obs['description']['THETA'] for obs in res['kis'][0]]
        assert (min(theta) >= 50) & (max(theta) <= 80)
        assert res['kis'][1, 0]['description']['INSTRUMENT'] == 'gris'
        theta = [obs['description']['THETA'] for obs in res['kis'][1]]
206
207
208
209
210
211
212
213
214
215
        assert (min(theta) >= 50) & (max(theta) <= 80)
    else:
        with pytest.raises(URLError, match=rf"{_dockerexc('gris')}"
                           r"{'description.THETA':{'.gte':50,'.lte':80}}"):
            Fido.search(a.Instrument("GRIS") & a.sdc.Theta(50*u.deg, 80*u.deg))

        with pytest.raises(URLError, match=rf"{_dockerexc('gris')}"
                           r"{'description.THETA':{'.gte':50,'.lte':80}}"):
            Fido.search(a.Instrument("GRIS"), a.sdc.Theta(50*u.deg, 80*u.deg))

216
        with pytest.raises(URLError, match=rf"{_dockerexc('LARS')}") as exc:
217
218
            Fido.search(two_inst, a.sdc.Theta(50*u.deg, 80*u.deg))
        assert "{'description.THETA':{'$gte':50,'$lte':80}}" in str(exc.value)
219
        assert "{'description.INSTRUMENT':'gris'" not in str(exc.value)
220
221


222
223
224
225
226
227
228
229
230
231
232
233
234
@pytest.mark.parametrize("query", ((a.Instrument("GRIS") & a.Level(3)),
                                   (a.Instrument("ChroTel") & a.Physobs("perspective.vortex")),
                                   (a.Level(0) & a.Instrument("Bob")),
                                   (a.Instrument("LARS") & a.sdc.Telescope("Leviathan"))))
def test_cant_handle_query(client, query):
    """Some examples of invalid queries with exceptions."""
    assert not client._can_handle_query(*query.attrs)
    with pytest.raises(AttributeError, match=r"Query not possible: "
                       rf"[ILPT][a-z]* {query.attrs[1].value} not in [rs]"):
        client.search(query)


@pytest.mark.parametrize("query", (a.Level(1), a.Wavelength(3200*u.AA, 1.6*u.micron),
235
                                   a.sdc.DataProduct('cube'), a.sdc.ObsName('gris_20140426_000'),
236
                                   a.sdc.Date('2021/01/31'), a.sdc.Filter('LOT%233802'),
237
238
                                   a.sdc.PolStates('IQUV'), a.sdc.Telescope('VTT'),
                                   a.sdc.Target('Sunspot_22'), a.sdc.AtmosR0(*([1, 20000]*u.mm)),
239
240
241
242
                                   a.sdc.Theta(20*u.arcmin, 89*u.deg), a.sdc.Mu(0.1, 1),
                                   a.sdc.ExposureTime(*([5, 60]*u.min)),
                                   a.sdc.HelioProjLon(5*u.arcsec), a.sdc.HelioProjLat(9*u.arcsec),
                                   a.sdc.SpatialResolution(0.1*u.arcsec, 0.8*u.arcsec),
243
                                   a.sdc.SpectralResolution(6000, 200000),
244
                                   a.sdc.TemporalResolution(2*u.s, 30*u.s),
245
                                   a.sdc.NDimensions(1, 2), a.sdc.PolXel(2, 4),
246
247
248
                                   a.sdc.SpatialXel1(200, 3000), a.sdc.SpatialXel2(100, 4000),
                                   a.sdc.SpectralXel(320, 4096), a.sdc.TimeXel(60, 86400)))
def test_all_queries(client, query):
249
    """Test an example of all supported query attributes with automatic field names."""
250
251
252
    assert client._can_handle_query(a.Instrument("GRIS"), query)
    if HAS_DOCKERTEST:
        res = client.search(a.Instrument("GRIS") & query)
253
254
        if len(res) > 0:
            assert 'description' in res.colnames
255
    else:
256
        with pytest.raises(URLError, match=rf"{_dockerexc('gris')}"
257
258
                           rf"{{'description.*{query.type_name.upper()}"):
            client.search(a.Instrument("GRIS") & query)
259
260


261
262
263
264
265
266
267
268
269
def test_range(client):
    """
    Test range filter - 'FIELD_MIN,_MAX' shall include at least one of `Attr.min`, `Attr.max`.
    """
    wl = a.Wavelength(10800*u.AA, 1.25*u.micron)
    query = a.Instrument("GRIS") & wl
    assert client._can_handle_query(query)
    if HAS_DOCKERTEST:
        res = client.search(query)
270
271
        wave_min = [obs['WAVELENGTH_MIN'] for obs in res['description']]
        wave_max = [obs['WAVELENGTH_MAX'] for obs in res['description']]
272
273
274
        assert max(wave_min) <= 1250
        assert min(wave_max) >= 1080
    else:
275
        with pytest.raises(URLError, match=rf"{_dockerexc('gris')}"
276
277
278
279
280
                           r"{'description.WAVELENGTH_MIN':{'.lte':1250}},"
                           r"{'description.WAVELENGTH_MAX':{'.gte':1080}}"):
            client.search(query)


281
282
283
284
285
def test_full_range(client):
    """
    Test 'fullrange' option - 'FIELD_MIN,_MAX' shall completely include `[Attr.min, Attr.max]`.
    """

286
    t = a.Time("2014/04/26 09:50", "2014/04/26 09:52")
287
    t.fullrange = True
288
    query = a.Instrument("GRIS") & t
289
290
291
    assert client._can_handle_query(query)
    if HAS_DOCKERTEST:
        res = client.search(query)
292
293
        assert len(res) == 1
        assert 'description' in res.colnames
294
    else:
295
296
297
        with pytest.raises(URLError, match=rf"{_dockerexc('GRIS')}"
                           r"{'description.DATE_BEG':{'.lte':{'.date':'2014-04-26T09:50:00.000'}}},"
                           r"{'description.DATE_END':{'.gte':{'.date':'2014-04-26T09:52:00.000'}"):
298
299
300
            client.search(query)

    # Test with inverted `min`, `max` inputs.
301
302
303
304
    hplt = a.sdc.HelioProjLat(0.1*u.arcmin, -2*u.arcsec)
    query = a.Instrument("LARS") & hplt
    if HAS_DOCKERTEST:
        res = client.search(query)
305
306
        hplt_tan_min = [obs['HPLT_TAN_MIN'] for obs in res['description']]
        hplt_tan_max = [obs['HPLT_TAN_MAX'] for obs in res['description']]
307
308
309
        assert max(hplt_tan_min) <= 6
        assert min(hplt_tan_max) >= -2
    else:
310
        with pytest.raises(URLError, match=rf"{_dockerexc('LARS')}"
311
312
313
314
                           r"{'description.HPLT_TAN_MIN':{'.lte':6}},"
                           r"{'description.HPLT_TAN_MAX':{'.gte':-2}}"):
            client.search(query)

315
316
317
318
319
    hplt.fullrange = True
    query = a.Instrument("LARS") & hplt
    assert client._can_handle_query(query)
    if HAS_DOCKERTEST:
        res = client.search(query)
320
321
        hplt_tan_min = [obs['HPLT_TAN_MIN'] for obs in res['description']]
        hplt_tan_max = [obs['HPLT_TAN_MAX'] for obs in res['description']]
322
323
        assert max(hplt_tan_min) <= -2
        assert min(hplt_tan_max) >= 6
324
    else:
325
        with pytest.raises(URLError, match=rf"{_dockerexc('LARS')}"
326
327
                           r"{'description.HPLT_TAN_MIN':{'.lte':-2}},"
                           r"{'description.HPLT_TAN_MAX':{'.gte':6}}"):
328
            client.search(query)