import pytest import urllib.request from urllib.error import HTTPError, URLError from pathlib import Path import os import json import parfive import astropy.units as u from astropy.io import fits from astropy.time import Time import sunpy from sunpy.net import Fido from sunpy.net import attrs as a from sunpy.net.base_client import QueryResponseTable from sdc.client import KISClient _BASE_URL = "http://dockertest:8083/sdc/" _QUERY_BASE = "gris_observations?filter=" _EXAMPLE_QUERY = "{'description.OBS_NAME':'gris_20140426_000'}" _EXAMPLE_RANGE = "{'description.THETA':{'$gt':70.0,'$lt':80}}" _EXAMPLE_DATES = ("{'$and':[{'description.INSTRUMENT':'gris'}," "{'description.DATE_BEG':{'$gte':{'$date':'2014-04-26T00:00:00'}," "'$lte':{'$date':'2014-04-27T00:00:00'}}}]}") try: response = urllib.request.urlopen(f"{_BASE_URL}{_QUERY_BASE}{_EXAMPLE_QUERY}") HAS_DOCKERTEST = True except(HTTPError, URLError): HAS_DOCKERTEST = False def dirnames(path): return os.path.dirname(path).split(os.path.sep) @pytest.fixture def client(): return KISClient() def _dockerexc(instr): return (rf"Unable to execute search .http://dockertest:8083/sdc/{instr.lower()}_observations." rf"filter={{'.and':.{{'description.INSTRUMENT':'{instr.lower()}'}},") def test_docker(client): """Test example queries on dockertest.""" if not HAS_DOCKERTEST: pytest.xfail("No dockertest running") response = urllib.request.urlopen(f"{_BASE_URL}{_QUERY_BASE}{_EXAMPLE_QUERY}") data = json.loads(response.read()) assert '_embedded' in data.keys() assert 'description' in data['_embedded'][0] response = urllib.request.urlopen(f"{_BASE_URL}{_QUERY_BASE}{_EXAMPLE_DATES}") data = json.loads(response.read()) assert 'description' in data['_embedded'][0] assert len(data['_embedded']) > 4 response = urllib.request.urlopen(f"{_BASE_URL}{_QUERY_BASE}{_EXAMPLE_RANGE}") data = json.loads(response.read()) assert 'description' in data['_embedded'][0] res = client.search(a.Instrument("GRIS") & a.sdc.ObsName('gris_20140426_000')) assert isinstance(res, QueryResponseTable) assert len(res) == 105 description = res[0].get('description') assert len(description) == 34 assert description['INSTRUMENT'] == 'gris' assert description['TELESCOPE'] == 'GREGOR' assert description['BTYPE'] == 'phot.count' assert description['DATE_BEG']['$date'] == 1398505619000 assert description['DATE_END']['$date'] == 1398506021300 file_ids = [ld['links']['l1_data']['$oid'] for ld in res] assert len(file_ids) == 105 for oid in file_ids[0], file_ids[104]: meta = json.loads(urllib.request.urlopen(f"{_BASE_URL}gris_l1_data.files/{oid}").read()) assert meta['_id']['$oid'] == oid hdulist = fits.open(f"{_BASE_URL}gris_l1_data.files/{oid}/binary") assert hdulist[0].header.get('TELESCOP') == 'GREGOR' assert '2014-04-26T' in hdulist[0].header.get('DATE-OBS') hdulist.close() date = a.Time("2014/04/26 01:00", "2014/04/26 22:00") downloader = parfive.Downloader() inst = res[0]['description']['INSTRUMENT'] rowpath = f"{res[0]['_id']['$oid']}" binfile = '' ext = 'json' for i, ld in enumerate(res[:10]): oid = ld['links']['l1_data']['$oid'] filename = f"{oid}.{ext}" url = f"{_BASE_URL}{inst}_l1_data.files/{oid}{binfile}" assert url == f"{_BASE_URL}gris_l1_data.files/{file_ids[i]}" downloader.enqueue_file(url, filename=os.path.join(rowpath, filename), max_splits=1) binfile = '/binary' ext = 'fits' for ld in res[:2]: oid = ld['links']['l1_data']['$oid'] filename = f"{oid}.{ext}" url = f"{_BASE_URL}{inst}_l1_data.files/{oid}{binfile}" downloader.enqueue_file(url, filename=os.path.join(rowpath, filename), max_splits=1) assert downloader.queued_downloads == 12 assert downloader.http_queue[0].keywords['url'].startswith(_BASE_URL) assert res[0]['links']['l1_data']['$oid'] in downloader.http_queue[0].keywords['url'] assert downloader.http_queue[10].keywords['url'].endswith(binfile) files = downloader.download() assert len(files) == 12 assert os.path.dirname(files[0]) == '5ee0feb97a92554c6de920ab' for filepath in files: if filepath.endswith('.fits'): hdulist = fits.open(filepath) assert hdulist[0].header.get('TELESCOP') == 'GREGOR' assert '2014-04-26T' in hdulist[0].header.get('DATE-OBS') assert date.start < Time(hdulist[0].header['DATE-OBS']) < date.end hdulist.close() else: assert filepath.endswith('.json') meta = json.load(open(filepath)) assert date.start < Time(meta['metadata']['header']['DATE-BEG']) < date.end assert date.start < Time(meta['metadata']['header']['DATE-OBS']) < date.end assert meta['_id']['$oid'] == os.path.splitext(os.path.basename(filepath))[0] def test_gridfs(client): """Test gridfs access on dockertest.""" pytest.xfail("No GridFS on gitlab-runner") import gridfs from kis_tools.generic import get_sdc_connection res = client.search(a.Instrument("GRIS") & a.sdc.ObsName('gris_20140426_000')) links = res[0]['_embedded'][0].get('links') file_ids = [ld['$oid'] for ld in links['l1_data']] sdc = get_sdc_connection() gfs = gridfs.GridFS(sdc.sdc_test, "gris_l1_data") found = gfs.find({"_id": {"$in": file_ids}}) assert(len(found)) == 105 def test_search(client): """Test conversion of (supported) Attrs to query string.""" assert not client._can_handle_query(a.Time("2019/01/01", "2021/01/01")) with pytest.raises(AttributeError, match=r"Query not possible: " r"No 'Instrument' found in Attributes"): client.search(a.Time("2019/01/01", "2021/01/01")) assert not client._can_handle_query(a.Instrument("UVES"), a.Time("2019/01/01", "2021/01/01")) with pytest.raises(AttributeError, match=r"Query not possible: " r"Instrument UVES not in registered list"): client.search(a.Instrument("UVES") & a.Time("2019/01/01", "2021/01/01")) query = a.Instrument("BBI") & a.Time("2017/05/21", "2017/05/22 22:00") assert client._can_handle_query(query) if HAS_DOCKERTEST: res = client.search(query) assert isinstance(res, QueryResponseTable) assert len(res) == 1 assert 'description' in res.colnames else: with pytest.raises(URLError, match=rf"{_dockerexc('bbi')}" r"{'description.DATE_BEG':{'.lte':{'.date':'2017-05-22T22:00:00.000'}}}," r"{'description.DATE_END':{'.gte':{'.date':'2017-05-21T00:00:00.000'}}}" rf".*Confirm that RESTHeart is running on {_BASE_URL} and connected"): client.search(query) # Maximum numer of returned observation records defaults to 100 per query. query = a.Instrument("LARS") & a.sdc.HelioProjLat(-10*u.arcsec, 0.2*u.arcmin) assert client._can_handle_query(query) if HAS_DOCKERTEST: res = client.search(query) assert len(res) == 1 assert 'description' in res.colnames else: with pytest.raises(URLError, match=rf"{_dockerexc('lars')}" r"{'description.HPLT_TAN_MIN':{'.lte':12}}," r"{'description.HPLT_TAN_MAX':{'.gte':-10}}"): client.search(query) query = a.Instrument("LARS"), a.sdc.HelioProjLat(-10*u.arcsec, 0.2*u.arcmin) assert client._can_handle_query(*query) if HAS_DOCKERTEST: res = client.search(*query) assert len(res) == 1 assert 'description' in res.colnames else: with pytest.raises(URLError, match=rf"{_dockerexc('lars')}" r"{'description.HPLT_TAN_MIN':{'.lte':12}}," r"{'description.HPLT_TAN_MAX':{'.gte':-10}}"): client.search(*query) query = a.Instrument("GRIS") & (a.sdc.Theta(85*u.deg, 3000*u.arcmin) | a.sdc.PolStates('iquv')) assert client._can_handle_query(query) if HAS_DOCKERTEST: res = client.search(query) assert len(res) == 362 assert 'description' in res.colnames assert 'THETA' in res[0]['description'] theta = [obs['description']['THETA'] for obs in res] assert (min(theta[:100]) >= 50) & (max(theta[:100]) <= 85) assert res[100]['description']['POL_STATES'] == 'IQUV' else: # Will raise on first of multi-part OR queries; somehow switches INSTRUMENT and THETA. with pytest.raises(URLError, match=rf"{_dockerexc('gris')[:80]}") as exc: client.search(query) assert "{'description.THETA':{'$gte':50,'$lte':85}}" in str(exc.value) assert "{'description.POL_STATES':" not in str(exc.value) query = a.Instrument("LARS") | a.Instrument("GRIS"), a.sdc.Theta(85*u.deg, 3000*u.arcmin) if HAS_DOCKERTEST: res = client.search(*query) assert len(res) == 163 assert 'description' in res.colnames assert 'THETA' in res[0]['description'] theta = [obs['description']['THETA'] for obs in res] assert (min(theta) >= 50) & (max(theta) <= 85) else: # Will raise on first of multi-part OR queries; somehow switches INSTRUMENT and THETA. with pytest.raises(URLError, match=rf"{_dockerexc('LARS')[:80]}") as exc: client.search(*query) assert "{'description.THETA':{'$gte':50,'$lte':85}}" in str(exc.value) assert "{'description.INSTRUMENT':'gris'" not in str(exc.value) def test_fido_search(): """Test search using the Fido base class with AttrAnd, AttrOr and lists of *args.""" two_inst = (a.Instrument("LARS") | a.Instrument("GRIS")) if HAS_DOCKERTEST: res = Fido.search(a.Instrument("GRIS") & a.sdc.Theta(50*u.deg, 80*u.deg)) assert len(res['kis']) == 162 theta = [obs['description']['THETA'] for obs in res['kis']] assert (min(theta) >= 50) & (max(theta) <= 80) res = Fido.search(a.Instrument("GRIS"), a.sdc.Theta(50*u.deg, 80*u.deg)) assert len(res['kis']) == 162 theta = [obs['description']['THETA'] for obs in res['kis']] assert (min(theta) >= 50) & (max(theta) <= 80) date = a.Time("2017/05/12 01:40", "2017/05/16 19:00") res = Fido.search(a.Instrument("LARS") & date) assert len(res['kis']) > 0 date_beg = [obs['description']['DATE_BEG']['$date'] for obs in res['kis']] date_end = [obs['description']['DATE_END']['$date'] for obs in res['kis']] assert max(date_beg) < date.end.unix * 1000 assert min(date_end) > date.start.unix * 1000 res = Fido.search(two_inst, a.sdc.Theta(50*u.deg, 80*u.deg)) assert len(res['kis']) == 2 assert len(res['kis'][0]) == 1 assert len(res['kis'][1]) == 162 assert res['kis'][0][0]['description']['INSTRUMENT'] == 'lars' theta = [obs['description']['THETA'] for obs in res['kis'][0]] assert (min(theta) >= 50) & (max(theta) <= 80) assert res['kis'][1, 0]['description']['INSTRUMENT'] == 'gris' theta = [obs['description']['THETA'] for obs in res['kis'][1]] assert (min(theta) >= 50) & (max(theta) <= 80) date = a.Time("2016/08/26 16:25", "2016/08/26 16:45") res = Fido.search(a.Instrument("GRIS"), a.sdc.PolStates('iquv'), date) assert len(res['kis']) == 400 assert res['kis'][0]['description']['POL_STATES'] == 'IQUV' else: with pytest.raises(URLError, match=rf"{_dockerexc('gris')}" r"{'description.THETA':{'.gte':50,'.lte':80}}"): Fido.search(a.Instrument("GRIS") & a.sdc.Theta(50*u.deg, 80*u.deg)) with pytest.raises(URLError, match=rf"{_dockerexc('gris')}" r"{'description.THETA':{'.gte':50,'.lte':80}}"): Fido.search(a.Instrument("GRIS"), a.sdc.Theta(50*u.deg, 80*u.deg)) with pytest.raises(URLError, match=rf"{_dockerexc('LARS')}") as exc: Fido.search(two_inst, a.sdc.Theta(50*u.deg, 80*u.deg)) assert "{'description.THETA':{'$gte':50,'$lte':80}}" in str(exc.value) assert "{'description.INSTRUMENT':'gris'" not in str(exc.value) def test_fido_fetch(): """Test search and fetch using the Fido interface.""" if not HAS_DOCKERTEST: pytest.xfail("No dockertest running") date = a.Time("2017/05/22 08:45", "2017/05/22 08:55") res = Fido.search(a.Instrument("BBI"), date) assert len(res['kis']) == 10 desc = [r['description'] for r in res['kis']] assert max([d['DATE_BEG']['$date'] for d in desc]) < date.end.unix * 1000 assert min([d['DATE_END']['$date'] for d in desc]) > date.start.unix * 1000 files = Fido.fetch(res['kis']) assert len(files) == 10 for i, filepath in enumerate(files): assert dirnames(filepath)[-1] == desc[::-1][i]['OBS_NAME'] meta = json.load(open(filepath)) assert meta['_id']['$oid'] == os.path.splitext(os.path.basename(filepath))[0] assert date.start.isot[:12] in meta['metadata']['header']['DATE-BEG'] assert date.start < Time(meta['metadata']['header']['DATE-BEG']) < date.end assert desc[i]['OBS_NAME'] in meta['metadata']['header']['FILENAME'] files = Fido.fetch(res['kis'][:3], binary=True) assert len(files) == 3 for i, filepath in enumerate(files): assert dirnames(filepath)[-1] == desc[::-1][i]['OBS_NAME'] hdulist = fits.open(filepath) assert hdulist[0].header.get('TELESCOP') == 'GREGOR' assert hdulist[0].header.get('INSTRUME') == 'BBI' assert date.start.iso[:12] in hdulist[0].header['DATE-OBS'] assert date.start < Time(hdulist[0].header['DATE-OBS']) < date.end def test_fido_fetch_2(): """ Test search and fetch from 2 instruments in time interval using the Fido interface. Assert observations are within some exposure times (10 min) of range. """ date = a.Time("2016/08/26 16:25", "2016/08/26 16:26") if not HAS_DOCKERTEST: with pytest.raises(URLError, match=rf"{_dockerexc('gris')}" rf"{{'description.DATE_BEG':{{'.lte':{{'.date':'{date.end.isot}'}}}}}}," rf"{{'description.DATE_END':{{'.gte':{{'.date':'{date.start.isot}'}}"): res = Fido.search((a.Instrument("GRIS") | a.Instrument("LARS")) & date) else: res = Fido.search((a.Instrument("GRIS") | a.Instrument("LARS")) & date) assert len(res['kis']) == 2 assert len(res['kis'][0]) == 400 assert res['kis'][0][0]['description']['DATE_BEG']['$date'] < date.end.unix * 1000 assert res['kis'][0][0]['description']['DATE_END']['$date'] > date.start.unix * 1000 files = Fido.fetch(res['kis'][0, :100], binary=False) assert len(files) == 100 assert files[0].endswith('.json') for filepath in files: meta = json.load(open(filepath)) assert meta['_id']['$oid'] == os.path.splitext(os.path.basename(filepath))[0] assert date.start.isot[:12] in meta['metadata']['header']['DATE-OBS'] assert date.start < Time(meta['metadata']['header']['DATE-OBS']) files = Fido.fetch(res['kis'][0, :5], binary=True) assert len(files) == 5 assert files[0].endswith('.fits') for filepath in files: hdulist = fits.open(filepath) assert hdulist[0].header['TELESCOP'] in ('GREGOR', 'VTT') assert hdulist[0].header['INSTRUME'] in ('GRIS', 'LARS') assert date.start.iso[:10] in hdulist[0].header['DATE-OBS'] assert Time(hdulist[0].header['DATE-OBS']).mjd < date.end.mjd + 600 assert Time(hdulist[0].header['DATE-OBS']).mjd > date.start.mjd - 600 hdulist.close() date = a.Time("2016/05/13 10:55", "2016/05/13 11:00") if not HAS_DOCKERTEST: with pytest.raises(URLError, match=rf"{_dockerexc('gris')}" rf"{{'description.DATE_BEG':{{'.lte':{{'.date':'{date.end.isot}'}}}}}}," rf"{{'description.DATE_END':{{'.gte':{{'.date':'{date.start.isot}'}}"): res = Fido.search((a.Instrument("GRIS") | a.Instrument("LARS")) & date) return res = Fido.search((a.Instrument("GRIS") | a.Instrument("LARS")) & date) assert len(res['kis']) == 2 assert len(res['kis'][0]) == 300 assert len(res['kis'][1]) == 1 desc = [r['description'] for r in res['kis'][0]] + [r['description'] for r in res['kis'][1]] dirs = [d['OBS_NAME'] for d in desc] assert len(desc) == 301 assert max([d['DATE_BEG']['$date'] for d in desc]) < date.end.unix * 1000 assert min([d['DATE_END']['$date'] for d in desc]) > date.start.unix * 1000 files = Fido.fetch(res['kis'][:, :100], binary=False) assert len(files.errors) == 0 assert len(files) == 101 assert files[0].endswith('.json') for filepath in files: assert dirnames(filepath)[-1] in dirs meta = json.load(open(filepath)) assert meta['_id']['$oid'] == os.path.splitext(os.path.basename(filepath))[0] assert date.start.iso[:10] in meta['metadata']['header']['DATE-OBS'] assert Time(meta['metadata']['header']['DATE-OBS']).mjd < date.end.mjd + 600 files = Fido.fetch(res['kis'][:, :10], binary=True) assert files[0].endswith('.fits') for filepath in files: hdulist = fits.open(filepath) assert hdulist[0].header['TELESCOP'] in ('GREGOR', 'VTT') assert hdulist[0].header['INSTRUME'] in ('GRIS', 'LARS') assert hdulist[0].header['EXTNAME'] in dirnames(filepath)[-1] assert date.start.iso[:10] in hdulist[0].header['DATE-OBS'] assert Time(hdulist[0].header['DATE-OBS']).mjd < date.end.mjd + 600 assert Time(hdulist[0].header['DATE-OBS']).mjd > date.start.mjd - 600 hdulist.close() assert len(files) == 11 @pytest.mark.parametrize("query", ((a.Instrument("GRIS") & a.Level(3)), (a.Instrument("ChroTel") & a.Physobs("perspective.vortex")), (a.Level(0) & a.Instrument("Bob")), (a.Instrument("LARS") & a.sdc.Telescope("Leviathan")))) def test_cant_handle_query(client, query): """Some examples of invalid queries with exceptions.""" assert not client._can_handle_query(*query.attrs) with pytest.raises(AttributeError, match=r"Query not possible: " rf"[ILPT][a-z]* {query.attrs[1].value} not in [rs]"): client.search(query) @pytest.mark.parametrize("query", (a.Level(1), a.Wavelength(3200*u.AA, 1.6*u.micron), a.sdc.DataProduct('cube'), a.sdc.ObsName('gris_20140426_000'), a.sdc.Date('2021/01/31'), a.sdc.Filter('LOT%233802'), a.sdc.PolStates('IQUV'), a.sdc.Telescope('VTT'), a.sdc.Target('Sunspot_22'), a.sdc.AtmosR0(*([1, 20000]*u.mm)), a.sdc.Theta(20*u.arcmin, 89*u.deg), a.sdc.Mu(0.1, 1), a.sdc.ExposureTime(*([5, 60]*u.min)), a.sdc.HelioProjLon(5*u.arcsec), a.sdc.HelioProjLat(9*u.arcsec), a.sdc.SpatialResolution(0.1*u.arcsec, 0.8*u.arcsec), a.sdc.SpectralResolution(6000, 200000), a.sdc.TemporalResolution(2*u.s, 30*u.s), a.sdc.NDimensions(1, 2), a.sdc.PolXel(2, 4), a.sdc.SpatialXel1(200, 3000), a.sdc.SpatialXel2(100, 4000), a.sdc.SpectralXel(320, 4096), a.sdc.TimeXel(60, 86400))) def test_all_queries(client, query): """Test an example of all supported query attributes with automatic field names.""" assert client._can_handle_query(a.Instrument("GRIS"), query) if HAS_DOCKERTEST: res = client.search(a.Instrument("GRIS") & query) if len(res) > 0: assert 'description' in res.colnames else: with pytest.raises(URLError, match=rf"{_dockerexc('gris')}" rf"{{'description.*{query.type_name.upper()}"): client.search(a.Instrument("GRIS") & query) def test_range(client): """ Test range filter - 'FIELD_MIN,_MAX' shall include at least one of `Attr.min`, `Attr.max`. """ wl = a.Wavelength(10800*u.AA, 1.25*u.micron) query = a.Instrument("GRIS") & wl assert client._can_handle_query(query) if HAS_DOCKERTEST: res = client.search(query) wave_min = [obs['WAVELENGTH_MIN'] for obs in res['description']] wave_max = [obs['WAVELENGTH_MAX'] for obs in res['description']] assert max(wave_min) <= 1250 assert min(wave_max) >= 1080 else: with pytest.raises(URLError, match=rf"{_dockerexc('gris')}" r"{'description.WAVELENGTH_MIN':{'.lte':1250}}," r"{'description.WAVELENGTH_MAX':{'.gte':1080}}"): client.search(query) def test_full_range(client): """ Test 'fullrange' option - 'FIELD_MIN,_MAX' shall completely include `[Attr.min, Attr.max]`. """ t = a.Time("2014/04/26 09:50", "2014/04/26 09:52") t.fullrange = True query = a.Instrument("GRIS") & t assert client._can_handle_query(query) if HAS_DOCKERTEST: res = client.search(query) assert len(res) == 105 assert 'description' in res.colnames else: with pytest.raises(URLError, match=rf"{_dockerexc('GRIS')}" r"{'description.DATE_BEG':{'.lte':{'.date':'2014-04-26T09:50:00.000'}}}," r"{'description.DATE_END':{'.gte':{'.date':'2014-04-26T09:52:00.000'}"): client.search(query) # Test with inverted `min`, `max` inputs. hplt = a.sdc.HelioProjLat(0.1*u.arcmin, -2*u.arcsec) query = a.Instrument("LARS") & hplt if HAS_DOCKERTEST: res = client.search(query) hplt_tan_min = [obs['HPLT_TAN_MIN'] for obs in res['description']] hplt_tan_max = [obs['HPLT_TAN_MAX'] for obs in res['description']] assert max(hplt_tan_min) <= 6 assert min(hplt_tan_max) >= -2 else: with pytest.raises(URLError, match=rf"{_dockerexc('LARS')}" r"{'description.HPLT_TAN_MIN':{'.lte':6}}," r"{'description.HPLT_TAN_MAX':{'.gte':-2}}"): client.search(query) hplt.fullrange = True query = a.Instrument("LARS") & hplt assert client._can_handle_query(query) if HAS_DOCKERTEST: res = client.search(query) hplt_tan_min = [obs['HPLT_TAN_MIN'] for obs in res['description']] hplt_tan_max = [obs['HPLT_TAN_MAX'] for obs in res['description']] assert max(hplt_tan_min) <= -2 assert min(hplt_tan_max) >= 6 else: with pytest.raises(URLError, match=rf"{_dockerexc('LARS')}" r"{'description.HPLT_TAN_MIN':{'.lte':-2}}," r"{'description.HPLT_TAN_MAX':{'.gte':6}}"): client.search(query)