test_client.py 12 KB
Newer Older
1
2
import pytest

3
4
import urllib.request
from urllib.error import HTTPError, URLError
5
import json
6

7
import astropy.units as u
8
from astropy.io import fits
9
from sunpy.net import attrs as a
10
from sunpy.net.base_client import QueryResponseTable
11

12
from sdc.client import KISClient
13
14
15


_BASE_URL = "http://dockertest:8083/sdc/"
16
17
18
19
20
21
_QUERY_BASE = "gris_observations?filter="
_EXAMPLE_QUERY = "{'description.OBS_NAME':'gris_20140426_000'}"
_EXAMPLE_RANGE = "{'description.THETA':{'$gt':70.0,'$lt':80}}"
_EXAMPLE_DATES = ("{'$and':[{'description.INSTRUMENT':'gris'},"
                  "{'description.DATE_BEG':{'$gte':{'$date':'2014-04-26T00:00:00'},"
                  "'$lte':{'$date':'2014-04-27T00:00:00'}}}]}")
22
23

try:
24
    response = urllib.request.urlopen(f"{_BASE_URL}{_QUERY_BASE}{_EXAMPLE_QUERY}")
25
26
27
28
29
30
31
    HAS_DOCKERTEST = True
except(HTTPError, URLError):
    HAS_DOCKERTEST = False


@pytest.fixture
def client():
32
    return KISClient()
33
34


35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
def test_docker(client):
    """Test example queries on dockertest."""
    if not HAS_DOCKERTEST:
        pytest.xfail("No dockertest running")

    response = urllib.request.urlopen(f"{_BASE_URL}{_QUERY_BASE}{_EXAMPLE_QUERY}")
    data = json.loads(response.read())
    assert '_embedded' in data.keys()
    assert 'description' in data['_embedded'][0]

    response = urllib.request.urlopen(f"{_BASE_URL}{_QUERY_BASE}{_EXAMPLE_DATES}")
    data = json.loads(response.read())
    assert 'description' in data['_embedded'][0]

    response = urllib.request.urlopen(f"{_BASE_URL}{_QUERY_BASE}{_EXAMPLE_RANGE}")
    data = json.loads(response.read())
    assert 'description' in data['_embedded'][0]

    res = client.search(a.Instrument("GRIS") & a.sdc.ObsName('gris_20140426_000'))
54
    assert isinstance(res, QueryResponseTable)
55
56
57
58
59
60
    assert len(res) == 1
    description = res[0]['_embedded'][0].get('description')
    assert description['INSTRUMENT'] == 'gris'
    assert description['TELESCOPE'] == 'GREGOR'
    assert description['BTYPE'] == 'phot.count'

61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
    links = res[0]['_embedded'][0].get('links')
    oid = links['l1_data'][0]['$oid']
    meta = json.loads(urllib.request.urlopen(f"{_BASE_URL}gris_l1_data.files/{oid}").read())
    assert meta['_id']['$oid'] == oid
    hdulist = fits.open(f"{_BASE_URL}gris_l1_data.files/{oid}/binary")
    assert hdulist[0].header.get('TELESCOP') == 'GREGOR'
    assert '2014-04-26T' in hdulist[0].header.get('DATE-OBS')


def test_gridfs(client):
    """Test gridfs access on dockertest."""
    pytest.xfail("No GridFS on gitlab-runner")
    import gridfs
    from kis_tools.generic import get_sdc_connection

    res = client.search(a.Instrument("GRIS") & a.sdc.ObsName('gris_20140426_000'))
    links = res[0]['_embedded'][0].get('links')
    file_ids = [ld['$oid'] for ld in links['l1_data']]
    sdc = get_sdc_connection()
    gfs = gridfs.GridFS(sdc.sdc_test, "gris_l1_data")
    found = gfs.find({"_id": {"$in": file_ids}})
    assert(len(found)) == 50

84

85
def test_search(client):
86
    """Test conversion of (supported) Attrs to query string."""
87

88
    assert not client._can_handle_query(a.Time("2019/01/01", "2021/01/01"))
89
90
91
92
    with pytest.raises(AttributeError, match=r"Query not possible: "
                       r"No 'Instrument' found in Attributes"):
        client.search(a.Time("2019/01/01", "2021/01/01"))

93
    assert not client._can_handle_query(a.Instrument("UVES"), a.Time("2019/01/01", "2021/01/01"))
94
95
96
97
    with pytest.raises(AttributeError, match=r"Query not possible: "
                       r"Instrument UVES not in registered list"):
        client.search(a.Instrument("UVES") & a.Time("2019/01/01", "2021/01/01"))

98
    query = a.Instrument("BBI") & a.Time("2019/01/01", "2021/01/01")
99
    # TODO: Verify returned observation records.
100
    assert client._can_handle_query(query)
101
    if HAS_DOCKERTEST:
102
        res = client.search(query)
103
104
        assert isinstance(res, QueryResponseTable)
        if len(res[0].get('_embedded')) > 0:
105
            assert 'description' in res[0]['_embedded'][0]
106
107
    else:
        with pytest.raises(URLError, match=r"Unable to execute search "
108
109
                           r".http://dockertest:8083/sdc/bbi_observations.filter="
                           r"{'.and':.{'description.INSTRUMENT':'bbi'},"
110
111
                           r"{'description.DATE_BEG':{'.lte':{'.date':'2021-01-01T00:00:00.000'}}},"
                           r"{'description.DATE_END':{'.gte':{'.date':'2019-01-01T00:00:00.000'}}}"
112
                           rf".*Confirm that RESTHeart is running on {_BASE_URL} and connected"):
113
114
115
116
117
118
            client.search(query)

    query = a.Instrument("LARS") & a.sdc.HelioProjLat(-10*u.arcsec, 0.2*u.arcmin)
    assert client._can_handle_query(query)
    if HAS_DOCKERTEST:
        res = client.search(query)
119
        if len(res[0].get('_embedded')) > 0:
120
            assert 'description' in res[0]['_embedded'][0]
121
122
123
124
    else:
        with pytest.raises(URLError, match=r"Unable to execute search "
                           r".http://dockertest:8083/sdc/lars_observations.filter="
                           r"{'.and':.{'description.INSTRUMENT':'lars'},"
125
126
                           r"{'description.HPLT_TAN_MIN':{'.lte':12}},"
                           r"{'description.HPLT_TAN_MAX':{'.gte':-10}}"):
127
128
            client.search(query)

129
    query = a.Instrument("GRIS") & (a.sdc.Theta(85*u.deg, 600*u.arcmin) | a.sdc.PolStates('iq'))
130
131
132
    assert client._can_handle_query(query)
    if HAS_DOCKERTEST:
        res = client.search(query)
133
134
        assert len(res) == 2
        if len(res[0].get('_embedded')) > 0:
135
            assert 'description' in res[0]['_embedded'][0]
136
    else:
Derek Homeier's avatar
Derek Homeier committed
137
        with pytest.raises(URLError, match=r"Unable to execute search ") as exc:
138
139
            client.search(query)
        # Will raise on first of multi-part OR queries.
140
141
        assert "http://dockertest:8083/sdc/gris_observations?filter=" in str(exc.value)
        assert "{'description.THETA':{'$gte':10,'$lte':85}}" in str(exc.value)
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
        assert "{'description.POL_STATES':" not in str(exc.value)


@pytest.mark.parametrize("query", ((a.Instrument("GRIS") & a.Level(3)),
                                   (a.Instrument("ChroTel") & a.Physobs("perspective.vortex")),
                                   (a.Level(0) & a.Instrument("Bob")),
                                   (a.Instrument("LARS") & a.sdc.Telescope("Leviathan"))))
def test_cant_handle_query(client, query):
    """Some examples of invalid queries with exceptions."""
    assert not client._can_handle_query(*query.attrs)
    with pytest.raises(AttributeError, match=r"Query not possible: "
                       rf"[ILPT][a-z]* {query.attrs[1].value} not in [rs]"):
        client.search(query)


@pytest.mark.parametrize("query", (a.Level(1), a.Wavelength(3200*u.AA, 1.6*u.micron),
158
159
160
161
                                   a.sdc.DataProduct('cube'), a.sdc.ObsName('gris_20140426_000'),
                                   a.sdc.Date('2021/01/31'), a.sdc.Filter('G'),
                                   a.sdc.PolStates('IQUV'), a.sdc.Telescope('VTT'),
                                   a.sdc.Target('Sunspot_22'), a.sdc.AtmosR0(*([1, 20000]*u.mm)),
162
163
164
165
                                   a.sdc.Theta(20*u.arcmin, 89*u.deg), a.sdc.Mu(0.1, 1),
                                   a.sdc.ExposureTime(*([5, 60]*u.min)),
                                   a.sdc.HelioProjLon(5*u.arcsec), a.sdc.HelioProjLat(9*u.arcsec),
                                   a.sdc.SpatialResolution(0.1*u.arcsec, 0.8*u.arcsec),
166
                                   a.sdc.SpectralResolution(6000, 200000),
167
                                   a.sdc.TemporalResolution(2*u.s, 30*u.s),
168
                                   a.sdc.NDimensions(1, 2), a.sdc.PolXel(2, 4),
169
170
171
                                   a.sdc.SpatialXel1(200, 3000), a.sdc.SpatialXel2(100, 4000),
                                   a.sdc.SpectralXel(320, 4096), a.sdc.TimeXel(60, 86400)))
def test_all_queries(client, query):
172
    """Test an example of all supported query attributes with automatic field names."""
173
174
175
    assert client._can_handle_query(a.Instrument("GRIS"), query)
    if HAS_DOCKERTEST:
        res = client.search(a.Instrument("GRIS") & query)
176
        if len(res[0].get('_embedded')) > 0:
177
            assert 'description' in res[0]['_embedded'][0]
178
179
180
181
182
183
    else:
        with pytest.raises(URLError, match=r"Unable to execute search "
                           r".http://dockertest:8083/sdc/gris_observations.filter="
                           r"{'.and':.{'description.INSTRUMENT':'gris'},"
                           rf"{{'description.*{query.type_name.upper()}"):
            client.search(a.Instrument("GRIS") & query)
184
185


186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
def test_range(client):
    """
    Test range filter - 'FIELD_MIN,_MAX' shall include at least one of `Attr.min`, `Attr.max`.
    """
    wl = a.Wavelength(10800*u.AA, 1.25*u.micron)
    query = a.Instrument("GRIS") & wl
    assert client._can_handle_query(query)
    if HAS_DOCKERTEST:
        res = client.search(query)
        wave_min = [obs['description']['WAVELENGTH_MIN'] for obs in res[0]['_embedded']]
        wave_max = [obs['description']['WAVELENGTH_MAX'] for obs in res[0]['_embedded']]
        assert max(wave_min) <= 1250
        assert min(wave_max) >= 1080
    else:
        with pytest.raises(URLError, match=r"Unable to execute search "
                           r".http://dockertest:8083/sdc/gris_observations.filter="
                           r"{'.and':.{'description.INSTRUMENT':'gris'},"
                           r"{'description.WAVELENGTH_MIN':{'.lte':1250}},"
                           r"{'description.WAVELENGTH_MAX':{'.gte':1080}}"):
            client.search(query)


208
209
210
211
212
213
214
215
216
217
218
def test_full_range(client):
    """
    Test 'fullrange' option - 'FIELD_MIN,_MAX' shall completely include `[Attr.min, Attr.max]`.
    """

    t = a.Time("2019/01/01", "2019/01/09")
    t.fullrange = True
    query = a.Instrument("BBI") & t
    assert client._can_handle_query(query)
    if HAS_DOCKERTEST:
        res = client.search(query)
219
        if len(res[0].get('_embedded')) > 0:
220
            assert 'description' in res[0]['_embedded'][0]
221
222
223
224
    else:
        with pytest.raises(URLError, match=r"Unable to execute search "
                           r".http://dockertest:8083/sdc/bbi_observations.filter="
                           r"{'.and':.{'description.INSTRUMENT':'bbi'},"
225
226
                           r"{'description.DATE_BEG':{'.lte':{'.date':'2019-01-01T00:00:00.000'}}},"
                           r"{'description.DATE_END':{'.gte':{'.date':'2019-01-09T00:00:00.000'}"):
227
228
229
            client.search(query)

    # Test with inverted `min`, `max` inputs.
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
    hplt = a.sdc.HelioProjLat(0.1*u.arcmin, -2*u.arcsec)
    query = a.Instrument("LARS") & hplt
    if HAS_DOCKERTEST:
        res = client.search(query)
        hplt_tan_min = [obs['description']['HPLT_TAN_MIN'] for obs in res[0]['_embedded']]
        hplt_tan_max = [obs['description']['HPLT_TAN_MAX'] for obs in res[0]['_embedded']]
        assert max(hplt_tan_min) <= 6
        assert min(hplt_tan_max) >= -2
    else:
        with pytest.raises(URLError, match=r"Unable to execute search "
                           r".http://dockertest:8083/sdc/lars_observations.filter="
                           r"{'.and':.{'description.INSTRUMENT':'lars'},"
                           r"{'description.HPLT_TAN_MIN':{'.lte':6}},"
                           r"{'description.HPLT_TAN_MAX':{'.gte':-2}}"):
            client.search(query)

246
247
248
249
250
    hplt.fullrange = True
    query = a.Instrument("LARS") & hplt
    assert client._can_handle_query(query)
    if HAS_DOCKERTEST:
        res = client.search(query)
251
252
253
254
        hplt_tan_min = [obs['description']['HPLT_TAN_MIN'] for obs in res[0]['_embedded']]
        hplt_tan_max = [obs['description']['HPLT_TAN_MAX'] for obs in res[0]['_embedded']]
        assert max(hplt_tan_min) <= -2
        assert min(hplt_tan_max) >= 6
255
256
257
258
    else:
        with pytest.raises(URLError, match=r"Unable to execute search "
                           r".http://dockertest:8083/sdc/lars_observations.filter="
                           r"{'.and':.{'description.INSTRUMENT':'lars'},"
259
260
                           r"{'description.HPLT_TAN_MIN':{'.lte':-2}},"
                           r"{'description.HPLT_TAN_MAX':{'.gte':6}}"):
261
            client.search(query)