test_client.py 14 KB
Newer Older
1
2
import pytest

3
4
import urllib.request
from urllib.error import HTTPError, URLError
5
import json
6

7
import astropy.units as u
8
from astropy.io import fits
9
from sunpy.net import Fido
10
from sunpy.net import attrs as a
11
from sunpy.net.base_client import QueryResponseTable
12

13
from sdc.client import KISClient
14
15
16


_BASE_URL = "http://dockertest:8083/sdc/"
17
18
19
20
21
22
_QUERY_BASE = "gris_observations?filter="
_EXAMPLE_QUERY = "{'description.OBS_NAME':'gris_20140426_000'}"
_EXAMPLE_RANGE = "{'description.THETA':{'$gt':70.0,'$lt':80}}"
_EXAMPLE_DATES = ("{'$and':[{'description.INSTRUMENT':'gris'},"
                  "{'description.DATE_BEG':{'$gte':{'$date':'2014-04-26T00:00:00'},"
                  "'$lte':{'$date':'2014-04-27T00:00:00'}}}]}")
23
24

try:
25
    response = urllib.request.urlopen(f"{_BASE_URL}{_QUERY_BASE}{_EXAMPLE_QUERY}")
26
27
28
29
30
31
32
    HAS_DOCKERTEST = True
except(HTTPError, URLError):
    HAS_DOCKERTEST = False


@pytest.fixture
def client():
33
    return KISClient()
34
35


36
37
38
39
40
def _dockerexc(instr):
    return (rf"Unable to execute search .http://dockertest:8083/sdc/{instr}_observations.filter="
            rf"{{'.and':.{{'description.INSTRUMENT':'{instr}'}},")


41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
def test_docker(client):
    """Test example queries on dockertest."""
    if not HAS_DOCKERTEST:
        pytest.xfail("No dockertest running")

    response = urllib.request.urlopen(f"{_BASE_URL}{_QUERY_BASE}{_EXAMPLE_QUERY}")
    data = json.loads(response.read())
    assert '_embedded' in data.keys()
    assert 'description' in data['_embedded'][0]

    response = urllib.request.urlopen(f"{_BASE_URL}{_QUERY_BASE}{_EXAMPLE_DATES}")
    data = json.loads(response.read())
    assert 'description' in data['_embedded'][0]

    response = urllib.request.urlopen(f"{_BASE_URL}{_QUERY_BASE}{_EXAMPLE_RANGE}")
    data = json.loads(response.read())
    assert 'description' in data['_embedded'][0]

    res = client.search(a.Instrument("GRIS") & a.sdc.ObsName('gris_20140426_000'))
60
    assert isinstance(res, QueryResponseTable)
61
62
63
64
65
66
    assert len(res) == 1
    description = res[0]['_embedded'][0].get('description')
    assert description['INSTRUMENT'] == 'gris'
    assert description['TELESCOPE'] == 'GREGOR'
    assert description['BTYPE'] == 'phot.count'

67
    links = res[0]['_embedded'][0].get('links')
68
69
70
71
72
73
74
75
76
    file_ids = [ld['$oid'] for ld in links['l1_data']]
    assert len(file_ids) == 105
    for oid in file_ids[0], file_ids[104]:
        meta = json.loads(urllib.request.urlopen(f"{_BASE_URL}gris_l1_data.files/{oid}").read())
        assert meta['_id']['$oid'] == oid
        hdulist = fits.open(f"{_BASE_URL}gris_l1_data.files/{oid}/binary")
        assert hdulist[0].header.get('TELESCOP') == 'GREGOR'
        assert '2014-04-26T' in hdulist[0].header.get('DATE-OBS')
        hdulist.close()
77
78
79
80
81
82
83
84
85
86
87
88
89
90


def test_gridfs(client):
    """Test gridfs access on dockertest."""
    pytest.xfail("No GridFS on gitlab-runner")
    import gridfs
    from kis_tools.generic import get_sdc_connection

    res = client.search(a.Instrument("GRIS") & a.sdc.ObsName('gris_20140426_000'))
    links = res[0]['_embedded'][0].get('links')
    file_ids = [ld['$oid'] for ld in links['l1_data']]
    sdc = get_sdc_connection()
    gfs = gridfs.GridFS(sdc.sdc_test, "gris_l1_data")
    found = gfs.find({"_id": {"$in": file_ids}})
91
    assert(len(found)) == 105
92

93

94
def test_search(client):
95
    """Test conversion of (supported) Attrs to query string."""
96

97
    assert not client._can_handle_query(a.Time("2019/01/01", "2021/01/01"))
98
99
100
101
    with pytest.raises(AttributeError, match=r"Query not possible: "
                       r"No 'Instrument' found in Attributes"):
        client.search(a.Time("2019/01/01", "2021/01/01"))

102
    assert not client._can_handle_query(a.Instrument("UVES"), a.Time("2019/01/01", "2021/01/01"))
103
104
105
106
    with pytest.raises(AttributeError, match=r"Query not possible: "
                       r"Instrument UVES not in registered list"):
        client.search(a.Instrument("UVES") & a.Time("2019/01/01", "2021/01/01"))

107
    query = a.Instrument("BBI") & a.Time("2019/01/01", "2021/01/01")
108
    # TODO: Verify returned observation records.
109
    assert client._can_handle_query(query)
110
    if HAS_DOCKERTEST:
111
        res = client.search(query)
112
113
        assert isinstance(res, QueryResponseTable)
        if len(res[0].get('_embedded')) > 0:
114
            assert 'description' in res[0]['_embedded'][0]
115
    else:
116
        with pytest.raises(URLError, match=rf"{_dockerexc('bbi')}"
117
118
                           r"{'description.DATE_BEG':{'.lte':{'.date':'2021-01-01T00:00:00.000'}}},"
                           r"{'description.DATE_END':{'.gte':{'.date':'2019-01-01T00:00:00.000'}}}"
119
                           rf".*Confirm that RESTHeart is running on {_BASE_URL} and connected"):
120
121
122
123
124
125
            client.search(query)

    query = a.Instrument("LARS") & a.sdc.HelioProjLat(-10*u.arcsec, 0.2*u.arcmin)
    assert client._can_handle_query(query)
    if HAS_DOCKERTEST:
        res = client.search(query)
126
127
        assert len(res[0]['_embedded']) == 100
        assert 'description' in res[0]['_embedded'][0]
128
    else:
129
        with pytest.raises(URLError, match=rf"{_dockerexc('lars')}"
130
131
                           r"{'description.HPLT_TAN_MIN':{'.lte':12}},"
                           r"{'description.HPLT_TAN_MAX':{'.gte':-10}}"):
132
133
            client.search(query)

134
135
136
137
138
139
140
141
142
143
144
145
146
    query = a.Instrument("LARS"), a.sdc.HelioProjLat(-10*u.arcsec, 0.2*u.arcmin)
    assert client._can_handle_query(*query)
    if HAS_DOCKERTEST:
        res = client.search(*query)
        assert len(res[0]['_embedded']) == 100
        assert 'description' in res[0]['_embedded'][0]
    else:
        with pytest.raises(URLError, match=rf"{_dockerexc('lars')}"
                           r"{'description.HPLT_TAN_MIN':{'.lte':12}},"
                           r"{'description.HPLT_TAN_MAX':{'.gte':-10}}"):
            client.search(*query)

    query = a.Instrument("GRIS") & (a.sdc.Theta(85*u.deg, 3000*u.arcmin) | a.sdc.PolStates('iquv'))
147
148
149
    assert client._can_handle_query(query)
    if HAS_DOCKERTEST:
        res = client.search(query)
150
        assert len(res) == 2
151
152
153
154
155
156
157
        assert len(res[0]['_embedded']) == 100
        assert 'description' in res[0]['_embedded'][0]
        assert 'THETA' in res[0]['_embedded'][0]['description']
        assert 'POL_STATES' in res[1]['_embedded'][0]['description']
        theta = [obs['description']['THETA'] for obs in res[0]['_embedded']]
        assert (min(theta) >= 50) & (max(theta) <= 85)
        assert len(res[1]['_embedded']) == 100
158
    else:
159
160
        # Will raise on first of multi-part OR queries; somehow switches INSTRUMENT and THETA.
        with pytest.raises(URLError, match=rf"{_dockerexc('gris')[:80]}") as exc:
161
            client.search(query)
162
        assert "{'description.THETA':{'$gte':50,'$lte':85}}" in str(exc.value)
163
164
165
        assert "{'description.POL_STATES':" not in str(exc.value)


166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
def test_fido_search():
    """Test search using the Fido base class with AttrAnd, AttrOr and lists of *args."""
    two_inst = a.Instrument("BBI") | a.Instrument("ChroTel")
    if HAS_DOCKERTEST:
        res = Fido.search(a.Instrument("GRIS") & a.sdc.Theta(50*u.deg, 80*u.deg))
        assert len(res['kis']) > 0
        assert len(res['kis', '_embedded']) > 0
        theta = [obs[0]['description']['THETA'] for obs in res['kis', '_embedded']]
        assert (min(theta) >= 50) & (max(theta) <= 80)

        res = Fido.search(a.Instrument("GRIS"), a.sdc.Theta(50*u.deg, 80*u.deg))
        assert len(res['kis']) > 0
        assert len(res['kis', '_embedded']) > 0
        theta = [obs[0]['description']['THETA'] for obs in res['kis']['_embedded']]
        assert (min(theta) >= 50) & (max(theta) <= 80)

        res = Fido.search(two_inst, a.sdc.Theta(50*u.deg, 80*u.deg))
        assert len(res['kis', '_embedded']) == 2
        theta = [obs for obs in res['kis', '_embedded']]
        assert 'description' in theta[0]['_embedded'][0].keys()
        assert (min(theta) >= 50) & (max(theta) <= 80)
        assert res['kis', '_embedded'][0]['description']['INSTRUMENT'] == 'bbi'
        assert res['kis', '_embedded'][1]['description']['INSTRUMENT'] == 'chrotel'
    else:
        with pytest.raises(URLError, match=rf"{_dockerexc('gris')}"
                           r"{'description.THETA':{'.gte':50,'.lte':80}}"):
            Fido.search(a.Instrument("GRIS") & a.sdc.Theta(50*u.deg, 80*u.deg))

        with pytest.raises(URLError, match=rf"{_dockerexc('gris')}"
                           r"{'description.THETA':{'.gte':50,'.lte':80}}"):
            Fido.search(a.Instrument("GRIS"), a.sdc.Theta(50*u.deg, 80*u.deg))

        with pytest.raises(URLError, match=rf"{_dockerexc('bbi')}") as exc:
            Fido.search(two_inst, a.sdc.Theta(50*u.deg, 80*u.deg))
        assert "{'description.THETA':{'$gte':50,'$lte':80}}" in str(exc.value)
        assert "{'description.INSTRUMENT':'chrotel'" not in str(exc.value)


204
205
206
207
208
209
210
211
212
213
214
215
216
@pytest.mark.parametrize("query", ((a.Instrument("GRIS") & a.Level(3)),
                                   (a.Instrument("ChroTel") & a.Physobs("perspective.vortex")),
                                   (a.Level(0) & a.Instrument("Bob")),
                                   (a.Instrument("LARS") & a.sdc.Telescope("Leviathan"))))
def test_cant_handle_query(client, query):
    """Some examples of invalid queries with exceptions."""
    assert not client._can_handle_query(*query.attrs)
    with pytest.raises(AttributeError, match=r"Query not possible: "
                       rf"[ILPT][a-z]* {query.attrs[1].value} not in [rs]"):
        client.search(query)


@pytest.mark.parametrize("query", (a.Level(1), a.Wavelength(3200*u.AA, 1.6*u.micron),
217
218
219
220
                                   a.sdc.DataProduct('cube'), a.sdc.ObsName('gris_20140426_000'),
                                   a.sdc.Date('2021/01/31'), a.sdc.Filter('G'),
                                   a.sdc.PolStates('IQUV'), a.sdc.Telescope('VTT'),
                                   a.sdc.Target('Sunspot_22'), a.sdc.AtmosR0(*([1, 20000]*u.mm)),
221
222
223
224
                                   a.sdc.Theta(20*u.arcmin, 89*u.deg), a.sdc.Mu(0.1, 1),
                                   a.sdc.ExposureTime(*([5, 60]*u.min)),
                                   a.sdc.HelioProjLon(5*u.arcsec), a.sdc.HelioProjLat(9*u.arcsec),
                                   a.sdc.SpatialResolution(0.1*u.arcsec, 0.8*u.arcsec),
225
                                   a.sdc.SpectralResolution(6000, 200000),
226
                                   a.sdc.TemporalResolution(2*u.s, 30*u.s),
227
                                   a.sdc.NDimensions(1, 2), a.sdc.PolXel(2, 4),
228
229
230
                                   a.sdc.SpatialXel1(200, 3000), a.sdc.SpatialXel2(100, 4000),
                                   a.sdc.SpectralXel(320, 4096), a.sdc.TimeXel(60, 86400)))
def test_all_queries(client, query):
231
    """Test an example of all supported query attributes with automatic field names."""
232
233
234
    assert client._can_handle_query(a.Instrument("GRIS"), query)
    if HAS_DOCKERTEST:
        res = client.search(a.Instrument("GRIS") & query)
235
        if len(res[0].get('_embedded')) > 0:
236
            assert 'description' in res[0]['_embedded'][0]
237
    else:
238
        with pytest.raises(URLError, match=rf"{_dockerexc('gris')}"
239
240
                           rf"{{'description.*{query.type_name.upper()}"):
            client.search(a.Instrument("GRIS") & query)
241
242


243
244
245
246
247
248
249
250
251
252
253
254
255
256
def test_range(client):
    """
    Test range filter - 'FIELD_MIN,_MAX' shall include at least one of `Attr.min`, `Attr.max`.
    """
    wl = a.Wavelength(10800*u.AA, 1.25*u.micron)
    query = a.Instrument("GRIS") & wl
    assert client._can_handle_query(query)
    if HAS_DOCKERTEST:
        res = client.search(query)
        wave_min = [obs['description']['WAVELENGTH_MIN'] for obs in res[0]['_embedded']]
        wave_max = [obs['description']['WAVELENGTH_MAX'] for obs in res[0]['_embedded']]
        assert max(wave_min) <= 1250
        assert min(wave_max) >= 1080
    else:
257
        with pytest.raises(URLError, match=rf"{_dockerexc('gris')}"
258
259
260
261
262
                           r"{'description.WAVELENGTH_MIN':{'.lte':1250}},"
                           r"{'description.WAVELENGTH_MAX':{'.gte':1080}}"):
            client.search(query)


263
264
265
266
267
268
269
270
271
272
273
def test_full_range(client):
    """
    Test 'fullrange' option - 'FIELD_MIN,_MAX' shall completely include `[Attr.min, Attr.max]`.
    """

    t = a.Time("2019/01/01", "2019/01/09")
    t.fullrange = True
    query = a.Instrument("BBI") & t
    assert client._can_handle_query(query)
    if HAS_DOCKERTEST:
        res = client.search(query)
274
        if len(res[0].get('_embedded')) > 0:
275
            assert 'description' in res[0]['_embedded'][0]
276
    else:
277
        with pytest.raises(URLError, match=rf"{_dockerexc('bbi')}"
278
279
                           r"{'description.DATE_BEG':{'.lte':{'.date':'2019-01-01T00:00:00.000'}}},"
                           r"{'description.DATE_END':{'.gte':{'.date':'2019-01-09T00:00:00.000'}"):
280
281
282
            client.search(query)

    # Test with inverted `min`, `max` inputs.
283
284
285
286
287
288
289
290
291
    hplt = a.sdc.HelioProjLat(0.1*u.arcmin, -2*u.arcsec)
    query = a.Instrument("LARS") & hplt
    if HAS_DOCKERTEST:
        res = client.search(query)
        hplt_tan_min = [obs['description']['HPLT_TAN_MIN'] for obs in res[0]['_embedded']]
        hplt_tan_max = [obs['description']['HPLT_TAN_MAX'] for obs in res[0]['_embedded']]
        assert max(hplt_tan_min) <= 6
        assert min(hplt_tan_max) >= -2
    else:
292
        with pytest.raises(URLError, match=rf"{_dockerexc('lars')}"
293
294
295
296
                           r"{'description.HPLT_TAN_MIN':{'.lte':6}},"
                           r"{'description.HPLT_TAN_MAX':{'.gte':-2}}"):
            client.search(query)

297
298
299
300
301
    hplt.fullrange = True
    query = a.Instrument("LARS") & hplt
    assert client._can_handle_query(query)
    if HAS_DOCKERTEST:
        res = client.search(query)
302
303
304
305
        hplt_tan_min = [obs['description']['HPLT_TAN_MIN'] for obs in res[0]['_embedded']]
        hplt_tan_max = [obs['description']['HPLT_TAN_MAX'] for obs in res[0]['_embedded']]
        assert max(hplt_tan_min) <= -2
        assert min(hplt_tan_max) >= 6
306
    else:
307
        with pytest.raises(URLError, match=rf"{_dockerexc('lars')}"
308
309
                           r"{'description.HPLT_TAN_MIN':{'.lte':-2}},"
                           r"{'description.HPLT_TAN_MAX':{'.gte':6}}"):
310
            client.search(query)