test_gris.py 8.16 KB
Newer Older
Carl Schaffer's avatar
Carl Schaffer committed
1
import os
Carl Schaffer's avatar
Carl Schaffer committed
2
import shutil
Carl Schaffer's avatar
Carl Schaffer committed
3
import unittest
Carl Schaffer's avatar
Carl Schaffer committed
4
from glob import glob
5
from os.path import isdir, join
Carl Schaffer's avatar
Carl Schaffer committed
6
from unittest.mock import patch
7

8
import numpy as np
9
from bson.objectid import ObjectId
10
11
from importer_test_data import test_data

12
13
14
15
16
from kis_tools.gris.GrisArchive import GrisArchive
from kis_tools.gris.GrisDay import GrisDay
from kis_tools.gris.GrisFitsFile import GrisFitsFile
from kis_tools.gris.GrisMongoClient import GrisMongoClient
from kis_tools.gris.GrisRun import GrisRun
17
from kis_tools.gris.wcs import reorder_gris_data
Carl Schaffer's avatar
Carl Schaffer committed
18

19
20
_testdata = "/dat/sdc/testing_data_for_importer/gris/"

Carl Schaffer's avatar
Carl Schaffer committed
21

Carl Schaffer's avatar
Carl Schaffer committed
22
class TestGris(unittest.TestCase):
23
    """Class to hold tests for gris importer"""
Carl Schaffer's avatar
Carl Schaffer committed
24
25

    @classmethod
Carl Schaffer's avatar
Carl Schaffer committed
26
    def setUpClass(cls):
27
        """global set up for tests"""
Carl Schaffer's avatar
Carl Schaffer committed
28
        # create instance of mongoclient
Carl Schaffer's avatar
Carl Schaffer committed
29
        cls.server = "localhost:27017"
30
        cls.database = "test_database"
Carl Schaffer's avatar
Carl Schaffer committed
31
32
        cls.errdir = "/tmp/gris_testing/"
        os.makedirs(cls.errdir, exist_ok=True)
Carl Schaffer's avatar
Carl Schaffer committed
33

Carl Schaffer's avatar
Carl Schaffer committed
34
        with patch("pymongo.database.Database.command", lambda *x, **y: True):
Carl Schaffer's avatar
Carl Schaffer committed
35
            gmc = GrisMongoClient(cls.server, cls.database)
Carl Schaffer's avatar
Carl Schaffer committed
36

Carl Schaffer's avatar
Carl Schaffer committed
37
        cls.gmc = gmc
Carl Schaffer's avatar
Carl Schaffer committed
38
39

        collections = [
Carl Schaffer's avatar
Carl Schaffer committed
40
41
42
43
44
45
46
47
48
            "gris_observations",
            "gris_l1_data.files",
            "gris_l1_data.chunks",
            "gris_l1_toti.files",
            "gris_l1_toti.chunks",
            "gris_obs_loc.files",
            "gris_obs_loc.chunks",
            "gris_obs_log.files",
            "gris_obs_log.chunks",
Carl Schaffer's avatar
Carl Schaffer committed
49
        ]
Carl Schaffer's avatar
Carl Schaffer committed
50
        cls.collections = collections
Carl Schaffer's avatar
Carl Schaffer committed
51
52

        # check that testdata directory exists
Carl Schaffer's avatar
Carl Schaffer committed
53
54
        cls.testdata_location = _testdata
        assert isdir(cls.testdata_location)
55
56

        # set up target for day-wise testing
57
58
        pattern = join(cls.testdata_location, "201?????")

Carl Schaffer's avatar
Carl Schaffer committed
59
        daytarget = list(
60
            filter(isdir, glob(pattern))
Carl Schaffer's avatar
Carl Schaffer committed
61
        )[0]
Carl Schaffer's avatar
Carl Schaffer committed
62
        cls.daytarget = os.path.join(cls.testdata_location, daytarget)
Carl Schaffer's avatar
Carl Schaffer committed
63

64
        # setup targets for run-wise testing
Carl Schaffer's avatar
Carl Schaffer committed
65
        files = os.listdir(os.path.join(cls.daytarget, "level1_split"))
66
        run = files[0].split("_")[4]
Carl Schaffer's avatar
Carl Schaffer committed
67
        cls.runfiles = []
68
        for f in files:
69
            if f.split("_")[4] == run:
Carl Schaffer's avatar
Carl Schaffer committed
70
71
                cls.runfiles.append(
                    os.path.join(cls.daytarget, "level1_split", f)
Carl Schaffer's avatar
Carl Schaffer committed
72
                )
Carl Schaffer's avatar
Carl Schaffer committed
73
            if len(cls.runfiles) == 2:
74
                break
Carl Schaffer's avatar
Carl Schaffer committed
75
            # setup target for file-wise-testing
Carl Schaffer's avatar
Carl Schaffer committed
76
            cls.filetarget = cls.runfiles[0]
77

Carl Schaffer's avatar
Carl Schaffer committed
78
    @classmethod
Carl Schaffer's avatar
Carl Schaffer committed
79
    def tearDownClass(cls):
80
        """gobal clean up after all tests """
81
82
83
84
85
        gmc = cls.gmc.db
        assert "sdc" not in gmc.name
        for c in gmc.list_collection_names():
            gmc[c].drop()

Carl Schaffer's avatar
Carl Schaffer committed
86
87
        del cls.gmc
        shutil.rmtree(cls.errdir)
Carl Schaffer's avatar
Carl Schaffer committed
88
89

    def test_day(self):
90
        """test instatiation of GrisDay """
91
        # test instantiation with valid folder
Carl Schaffer's avatar
Carl Schaffer committed
92
        gd = GrisDay(os.path.join(self.testdata_location, self.daytarget))
Carl Schaffer's avatar
Carl Schaffer committed
93
94
95
        self.assertIsInstance(
            gd, GrisDay, "Instantiation of GrisDay class failed!"
        )
Carl Schaffer's avatar
Carl Schaffer committed
96
        # test instantiation with folder like 20140426_1
97
        fakeday = GrisDay("20140426_1")
Carl Schaffer's avatar
Carl Schaffer committed
98
        self.assertIsInstance(
Carl Schaffer's avatar
Carl Schaffer committed
99
100
101
102
            fakeday,
            GrisDay,
            "Instantiation of GrisDay class failed on fake day!",
        )
Carl Schaffer's avatar
Carl Schaffer committed
103
104
105
106
107
108
109
110
        # test printing of object
        self.assertIsInstance(gd.__str__(), str)
        # test repr of object
        self.assertIsInstance(gd.__repr__(), str)
        # test printing of object
        self.assertIsInstance(fakeday.__str__(), str)
        # test repr of object
        self.assertIsInstance(fakeday.__repr__(), str)
Carl Schaffer's avatar
Carl Schaffer committed
111

Carl Schaffer's avatar
Carl Schaffer committed
112
        # test the count method
113
114
        self.assertGreater(gd.count("*"), 0)
        self.assertEqual(fakeday.count("*"), 0)
Carl Schaffer's avatar
Carl Schaffer committed
115

116
    def test_gris_fits_file(self):
117
        """test instantiation of GrisFitsFile """
118
        gff = GrisFitsFile(self.filetarget)
Carl Schaffer's avatar
Carl Schaffer committed
119
120
121
        self.assertIsInstance(
            gff, GrisFitsFile, "Instantiation of GrisFitsFile failed"
        )
122
        # test parse function
123
        gff.parse()
Carl Schaffer's avatar
Carl Schaffer committed
124
        gff.parse()
125

126
127
128
129
130
        # test printing of object
        self.assertIsInstance(gff.__str__(), str)
        # test repr of object
        self.assertIsInstance(gff.__repr__(), str)

Carl Schaffer's avatar
Carl Schaffer committed
131
132
133
        # test make_bson
        self.assertIsInstance(gff.make_bson(), dict)

Carl Schaffer's avatar
Carl Schaffer committed
134
135
136
137
        # check that constructor complains on invalid filename
        with self.assertRaises(AssertionError):
            _ = GrisFitsFile("")

138
139
140
141
        # test repr
        print(repr(gff))
        print(gff)

142
        # test _coords_from_simple_header with file from 2017
143
        gf2 = GrisFitsFile(
144
            test_data.gris.l1_data[0]
145
146
147
148
        )
        gf2.slit_orientation
        gf2.parse()
        _ = gf2.modestring
149
        _ = gf2._coords_from_simple_header
150

151
152
        _ = gff.mu

153
        # test coordinates from WCS
154
155
156
157
158
159
160
161
162
        self.assertEqual(len(gf2._coords_from_wcs.shape), 2)

        # test coordinates from translated and untranslated file
        translated_file = self.filetarget
        raw_file = "/dat/sdc/testing_data_for_importer/gris/raw_split_files/gris_20140503_140538_l1p_012_001_0012.fits"

        for f in [translated_file, raw_file]:
            coords = GrisFitsFile(f).coords
            self.assertTrue(coords)
163

164
    def test_gris_run(self):
165
        """test instantiation of GrisRun """
166
167
        myfiles = [GrisFitsFile(f) for f in self.runfiles]
        gr = GrisRun(myfiles)
168
        self.assertIsInstance(
Carl Schaffer's avatar
Carl Schaffer committed
169
170
            gr, GrisRun, "Instantiation of GrisRun from GrisFitsFiles failed"
        )
Carl Schaffer's avatar
Carl Schaffer committed
171
172

        # run @property functions once to test
Carl Schaffer's avatar
Carl Schaffer committed
173
174
175
176
177
        self.assertIsNotNone(gr.files_l0)
        self.assertIsNotNone(gr.files_l1)
        self.assertIsNotNone(gr.files_l1_split)
        self.assertIsNotNone(gr.files_l2)
        self.assertIsNotNone(gr.map_saves)
Carl Schaffer's avatar
Carl Schaffer committed
178

179
        gr = GrisRun(self.runfiles)
Carl Schaffer's avatar
Carl Schaffer committed
180
181
182
        self.assertIsInstance(
            gr, GrisRun, "Instantiation of GrisRun from Paths failed"
        )
Carl Schaffer's avatar
Carl Schaffer committed
183

184
        gr.parse()
Carl Schaffer's avatar
Carl Schaffer committed
185

186
187
188
189
190
191
        # test printing of object
        self.assertIsInstance(gr.__str__(), str)

        # test repr of object
        self.assertIsInstance(gr.__repr__(), str)

192
193
194
        # test was aborted
        self.assertTrue(gr.was_aborted)

195
        # test observers
Carl Schaffer's avatar
Carl Schaffer committed
196
        obs = gr.observers
197
198
        self.assertTrue(obs)

199
200
201
202
        fig, ax = gr.plot_location()
        self.assertEqual(len(ax.patches), 4)
        return fig, ax

Carl Schaffer's avatar
Carl Schaffer committed
203
    def test_client(self):
204
        """test GrisMongoClient """
Carl Schaffer's avatar
Carl Schaffer committed
205
        # Tests for GrisMongoClient:
Carl Schaffer's avatar
Carl Schaffer committed
206
207
208
        with patch("pymongo.database.Database.command", return_value=True):
            gmc = GrisMongoClient(self.server, self.database)

Carl Schaffer's avatar
Carl Schaffer committed
209
210
211
        self.assertIsInstance(
            gmc, GrisMongoClient, "Could not instantiate mongodb client class"
        )
Carl Schaffer's avatar
Carl Schaffer committed
212
213
214

        # instantiate test run and try writing to gridfs
        test_run = GrisRun(self.runfiles)
Carl Schaffer's avatar
Carl Schaffer committed
215
        # random object id
216
        spoof_id = ObjectId("5ae07acf01b0127188e9da15")
Carl Schaffer's avatar
Carl Schaffer committed
217
        print(spoof_id)
218

Carl Schaffer's avatar
Carl Schaffer committed
219
220
        gmc.write_gris_run(test_run)

Carl Schaffer's avatar
typo    
Carl Schaffer committed
221
        # try to get id from db, fails if no runs have been inserted into db
222
223
        gmc.id_from_run(test_run)

224
    def test_gris_archive(self):
225
        """ test GrisArchive"""
226
        ga = GrisArchive(self.testdata_location)
Carl Schaffer's avatar
Carl Schaffer committed
227
228
229
        self.assertIsInstance(
            ga, GrisArchive, "Could not instantiate gris archive class"
        )
230
231
        df = ga.get_df()
        self.assertGreater(len(df), 0)
Carl Schaffer's avatar
Carl Schaffer committed
232

233
    def test_wcs(self):
234
235

        # Test reordering of data
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
        test_shapes = [
            (4, 166, 1010),  # polarimetric  data
            (493, 1010),  # spectroscopic data
        ]
        for shape in test_shapes:
            data = np.zeros(shape)
            reordered = reorder_gris_data(data, reverse_axes=False)
            reordered_rev = reorder_gris_data(data, reverse_axes=True)

            # check that something has been done
            self.assertNotEqual(data.shape, reordered.shape)

            # check the correct order of spatial axes
            shape = reordered.shape
            self.assertLess(shape[0], shape[1])

252
253
254
            # check that reversed data is the reverse of normal data
            self.assertEqual(reordered.shape[::-1], reordered_rev.shape)

255
256
257
            # check that the maximum (spectral axis) is at third position
            self.assertEqual(2, shape.index(max(shape)))

Carl Schaffer's avatar
Carl Schaffer committed
258

Carl Schaffer's avatar
Carl Schaffer committed
259
if __name__ == "__main__":
260
    with open(os.devnull, "w") as log:
Carl Schaffer's avatar
Carl Schaffer committed
261
        unittest.main()