Merge branch 'setup_fixing' into 'master'

Setup fixing

See merge request !98

MANIFEST.in

+include kis_tools/generic/field_list/obs_base.csv
+include kis_tools/generic/field_list/obs_gris.csv
+include kis_tools/headers/sdc_header_templates.xlsx
+include kis_tools/headers/*.csv
+include kis_tools/headers/*.txt
+include kis_tools/util/*.pkl
+include kis_tools/bbi/*.yaml
+include kis_tools/bbi/*.csv
\ No newline at end of file

bin/translate_header

 #!/usr/bin/env python3
 import sys
 
-from kis_tools.headers.translate_header import run_translate_header
+from kis_tools.gris.wcs import run_translate_header
 
 if __name__ == '__main__':
     run_translate_header(sys.argv[1:])
\ No newline at end of file

kis_tools/__init__.py

-__version__='0.0.0'
+__version__='1.0.0'

kis_tools/generic/SDCMongoClient.py

@@ -318,11 +318,21 @@ class SDCMongoClient(MongoClient):
         # get git_info for all gitfiles and return the most recent set of infos
         git_info_sets = []
         for f in self.gitfiles:
-            git_info = util.get_git_info(f)
+            git_info = util.load_git_info()
             if git_info:
                 git_info_sets.append(git_info)
 
-        most_recent = max(git_info_sets, key=lambda x: x["date"])
+        # TODO: Fix version info, set source repo and version tag as git info
+        default_res = {
+        "date": datetime.datetime.now(),
+        "last_commit": "unknown",
+        "url": "unknown",
+    }
+        try:
+            most_recent = max(git_info_sets, key=lambda x: x["date"])
+        except:
+            print(f'could not determine most recent git version, using default')
+            most_recent = default_res
         return most_recent
 
     @staticmethod

kis_tools/gris/wcs.py

 import math
 import re
 import sys
+from argparse import ArgumentParser
 from glob import glob
 from math import isnan
-from os.path import exists, join, dirname, basename
+from os.path import exists, join, dirname, basename, isdir
 from tempfile import NamedTemporaryFile
+from warnings import filterwarnings
 
 import numpy as np
 import pandas as pd
 from astropy.io.fits import Card, getval
+from astropy.io.fits.verify import VerifyWarning
+from kis_tools.headers.GrisTranslator import NothingToDoError
+from kis_tools.util.util import setup_outdir
 from scipy.io.idl import readsav
+from tqdm import tqdm
+
 from .GrisFitsFile import GrisFitsFile
 from ..headers.GrisTranslator import NothingToDoError
 from ..headers.translate_header import main as translate_header
@@ -290,3 +297,59 @@ def main(infile, outfile):
 if __name__ == '__main__':
     with NamedTemporaryFile() as t:
         main(sys.argv[1], t.name)
+
+
+def run_translate_header(cmd_line_args):
+
+    parser = ArgumentParser()
+
+    parser.add_argument(
+        "folder_in",
+        help="folder to be processed, needs to be a GRIS level1_split folder",
+    )
+    parser.add_argument(
+        "--folder_out",
+        help="path where output is written. Defaults to input folder",
+        default=None,
+    )
+
+    parser.add_argument(
+        "-v",
+        "--verbose",
+        help="don'tsilence warnings",
+        action="store_true",
+        default=False,
+    )
+
+
+    args = parser.parse_args(cmd_line_args)
+    folder_in = args.folder_in
+
+    folder_out = args.folder_out
+    if folder_out:
+        setup_outdir(folder_out, overwrite=True)
+
+    if not isdir(folder_in):
+        raise IOError(f"Directory {folder_in} not found!")
+
+    files = sorted(glob(join(folder_in, "*.fits")))
+
+
+    if not args.verbose:
+        filterwarnings("ignore", ".*", UserWarning)
+        filterwarnings("ignore", ".*", VerifyWarning)
+
+    progress_bar = tqdm(files)
+
+    skipped = 0
+    for f in progress_bar:
+        progress_bar.set_description(f"Modifying {f}")
+        if folder_out:
+            outfile = f.replace(folder_in, folder_out)
+        try:
+            main(f, outfile=f)
+        except NothingToDoError:
+            skipped += 1
+            continue
+    if skipped > 0:
+        print(f"Skipped {skipped} files which were already done.")
\ No newline at end of file

kis_tools/headers/translate_header.py

@@ -9,20 +9,15 @@
 
 # standard library imports
 import re
-from argparse import ArgumentParser
-from glob import glob
 from math import isnan
-from os.path import join, isdir
-from warnings import warn, filterwarnings
+from os.path import join
+from warnings import warn
 
 import numpy as np
 # third party imports
 from astropy.io import fits
 from astropy.io.fits import Header, PrimaryHDU, Card
-from astropy.io.fits.verify import VerifyWarning
-from kis_tools.headers.GrisTranslator import NothingToDoError
-from kis_tools.util.util import setup_outdir
-from tqdm import tqdm
+from kis_tools.util.util import load_git_info
 
 from . import ObservationInfo
 # custom imports
@@ -35,7 +30,6 @@ from .properties import (
 from ..util.util import (
     gen_history_time,
     gen_history_git,
-    get_git_info,
 )
 
 
@@ -204,7 +198,7 @@ def main(infile, outfile=None, wcs_generator=None, **kwargs):  # noqa: C901
     # translation processing step
     old_hist = list(oi._header["HISTORY"])
     hist_main = [gen_history_time("translate_header")]
-    git_info = get_git_info(__file__)
+    git_info = load_git_info()
     hist_git = gen_history_git("translate_header", git_info)
     hist_entries = old_hist + hist_main + hist_git
     for hist_entry in hist_entries:
@@ -270,57 +264,3 @@ def reorder_gris_data(data, reverse_axes=False):
     return trans_data
 
 
-def run_translate_header(cmd_line_args):
-
-    parser = ArgumentParser()
-
-    parser.add_argument(
-        "folder_in",
-        help="folder to be processed, needs to be a GRIS level1_split folder",
-    )
-    parser.add_argument(
-        "--folder_out",
-        help="path where output is written. Defaults to input folder",
-        default=None,
-    )
-
-    parser.add_argument(
-        "-v",
-        "--verbose",
-        help="don'tsilence warnings",
-        action="store_true",
-        default=False,
-    )
-
-
-    args = parser.parse_args(cmd_line_args)
-    folder_in = args.folder_in
-
-    folder_out = args.folder_out
-    if folder_out:
-        setup_outdir(folder_out, overwrite=True)
-
-    if not isdir(folder_in):
-        raise IOError(f"Directory {folder_in} not found!")
-
-    files = sorted(glob(join(folder_in, "*.fits")))
-
-
-    if not args.verbose:
-        filterwarnings("ignore", ".*", UserWarning)
-        filterwarnings("ignore", ".*", VerifyWarning)
-
-    progress_bar = tqdm(files)
-
-    skipped = 0
-    for f in progress_bar:
-        progress_bar.set_description(f"Modifying {f}")
-        if folder_out:
-            outfile = f.replace(folder_in, folder_out)
-        try:
-            main(f, outfile=f)
-        except NothingToDoError:
-            skipped += 1
-            continue
-    if skipped > 0:
-        print(f"Skipped {skipped} files which were already done.")
\ No newline at end of file

kis_tools/tests/test_util.py

@@ -50,7 +50,7 @@ class TestUtil(unittest.TestCase):
         # Non repo file should not return anything
         self.assertTrue(util.get_git_info(self.testfile) == {})
 
-        # Not commited file should return None, None
+        # Not committed file should return None, None
         self.assertTrue(util.get_git_info(self.emptyfile) == {})
 
         # A file that is sure to be in a repo

kis_tools/util/util.py

@@ -7,18 +7,17 @@ import re
 import subprocess
 from collections import Counter
 from glob import iglob
-from os.path import basename, exists, join
+from os.path import basename, exists, join, dirname
 from shutil import rmtree, copyfileobj
 from sys import exc_info
 from warnings import filterwarnings
 
 import astropy
+import astropy.units as u
 import pandas as pd
 from PIL import Image
 from astropy.io.fits import open as fitsopen
 from sunpy.coordinates.sun import angular_radius
-import astropy.units as u
-
 
 
 def get_git_info(path):
@@ -72,6 +71,18 @@ def get_git_info(path):
     }
 
 
+def write_git_info(path_in, path_out=join(dirname(__file__), "git_info.pkl")):
+    info = get_git_info(path_in)
+    with open(path_out, "wb+") as f:
+        pickle.dump(info, f)
+
+
+def load_git_info(path_in=join(dirname(__file__), "git_info.pkl")):
+    with open(path_in, "rb") as f:
+        res = pickle.load(f)
+    return res
+
+
 def get_md5(fname):
     """Calculate md5 has of a given file
 
@@ -597,7 +608,7 @@ def add_history(header: astropy.io.fits.Header, file, name=None):
     """
     if not name:
         name = basename(file)
-    git_info = get_git_info(file)
+    git_info = load_git_info()
     hist_time = gen_history_time(name)
     hist_git = gen_history_git("git_info: " + name, git_info)
     entries = [hist_time] + hist_git
@@ -682,6 +693,7 @@ def get_data_folders(base_folder):
     res = pool.map(has_chrotel_files, folders)
     return [f for f, valid in res if valid]
 
+
 def get_solar_radius(time):
     r = angular_radius(time).to(u.arcsec)
-    return r.value
\ No newline at end of file
+    return r.value

sdc_env.yaml

-name: sdc
-channels:
-  - astropy
-  - conda-forge
-  - defaults
-dependencies:
-  - tqdm
-  - astropy=3.1.2=py37h14c3975_0
-  - ipython=7.4.0=py37h24bf2e0_0
-  - matplotlib=3.0.3=py37_0
-  - numpy=1.16.2=py37h8b7e671_1
-  - pandas=0.24.2=py37hf484d3e_0
-  - pillow=6.0.0=py37he7afcd5_0
-  - pymongo=3.7.2=py37hf484d3e_0
-  - pytest=4.4.1=py37_0
-  - pytest-cov=2.6.1=py_0
-  - python=3.7.3=h5b0a415_0
-  - pyyaml=5.1=py37h14c3975_0
-  - scikit-image=0.15.0=py37he1b5a44_0
-  - setuptools=41.0.0=py37_0
-  - sunpy>=1.0.0
-  - tk=8.6.9=h84994c4_1001
-  - yaml=0.1.7=h14c3975_1001
-  - xlrd >= 1.0.0

setup.py

 """Setup for the kis_tools package."""
 from glob import glob
+from os.path import join
 
 import setuptools
+from kis_tools import __version__
+from kis_tools.util.util import write_git_info
 
 with open('README.md') as f:
     README = f.read()
-from kis_tools import __version__
+info = write_git_info(join("kis_tools", "__init__.py"),join("kis_tools","util", "git_info.pkl"))
 version = __version__
+
 setuptools.setup(
     author="Carl Schaffer",
     author_email="schaffer@leibniz-kis.de",
     name='kis_tools',
     license="MIT",
     description='Tools for interacting with data produced at KIS',
-    version='v'+__version__,
+    version='v' + __version__,
     long_description=README,
     url='https://gitlab.leibniz-kis.de/sdc/kis_tools',
     packages=setuptools.find_packages(),
+    include_package_data=True,
     python_requires=">=3.6",
     install_requires=[
         "pandas",