Commit 93eeecbb authored by Carl Schaffer's avatar Carl Schaffer
Browse files

Merge branch 'setup_fixing' into 'master'

Setup fixing

See merge request !98
parents e6439c50 bc550f95
include kis_tools/generic/field_list/obs_base.csv
include kis_tools/generic/field_list/obs_gris.csv
include kis_tools/headers/sdc_header_templates.xlsx
include kis_tools/headers/*.csv
include kis_tools/headers/*.txt
include kis_tools/util/*.pkl
include kis_tools/bbi/*.yaml
include kis_tools/bbi/*.csv
\ No newline at end of file
#!/usr/bin/env python3
import sys
from kis_tools.headers.translate_header import run_translate_header
from kis_tools.gris.wcs import run_translate_header
if __name__ == '__main__':
run_translate_header(sys.argv[1:])
\ No newline at end of file
__version__='0.0.0'
__version__='1.0.0'
......@@ -318,11 +318,21 @@ class SDCMongoClient(MongoClient):
# get git_info for all gitfiles and return the most recent set of infos
git_info_sets = []
for f in self.gitfiles:
git_info = util.get_git_info(f)
git_info = util.load_git_info()
if git_info:
git_info_sets.append(git_info)
most_recent = max(git_info_sets, key=lambda x: x["date"])
# TODO: Fix version info, set source repo and version tag as git info
default_res = {
"date": datetime.datetime.now(),
"last_commit": "unknown",
"url": "unknown",
}
try:
most_recent = max(git_info_sets, key=lambda x: x["date"])
except:
print(f'could not determine most recent git version, using default')
most_recent = default_res
return most_recent
@staticmethod
......
import math
import re
import sys
from argparse import ArgumentParser
from glob import glob
from math import isnan
from os.path import exists, join, dirname, basename
from os.path import exists, join, dirname, basename, isdir
from tempfile import NamedTemporaryFile
from warnings import filterwarnings
import numpy as np
import pandas as pd
from astropy.io.fits import Card, getval
from astropy.io.fits.verify import VerifyWarning
from kis_tools.headers.GrisTranslator import NothingToDoError
from kis_tools.util.util import setup_outdir
from scipy.io.idl import readsav
from tqdm import tqdm
from .GrisFitsFile import GrisFitsFile
from ..headers.GrisTranslator import NothingToDoError
from ..headers.translate_header import main as translate_header
......@@ -290,3 +297,59 @@ def main(infile, outfile):
if __name__ == '__main__':
with NamedTemporaryFile() as t:
main(sys.argv[1], t.name)
def run_translate_header(cmd_line_args):
parser = ArgumentParser()
parser.add_argument(
"folder_in",
help="folder to be processed, needs to be a GRIS level1_split folder",
)
parser.add_argument(
"--folder_out",
help="path where output is written. Defaults to input folder",
default=None,
)
parser.add_argument(
"-v",
"--verbose",
help="don'tsilence warnings",
action="store_true",
default=False,
)
args = parser.parse_args(cmd_line_args)
folder_in = args.folder_in
folder_out = args.folder_out
if folder_out:
setup_outdir(folder_out, overwrite=True)
if not isdir(folder_in):
raise IOError(f"Directory {folder_in} not found!")
files = sorted(glob(join(folder_in, "*.fits")))
if not args.verbose:
filterwarnings("ignore", ".*", UserWarning)
filterwarnings("ignore", ".*", VerifyWarning)
progress_bar = tqdm(files)
skipped = 0
for f in progress_bar:
progress_bar.set_description(f"Modifying {f}")
if folder_out:
outfile = f.replace(folder_in, folder_out)
try:
main(f, outfile=f)
except NothingToDoError:
skipped += 1
continue
if skipped > 0:
print(f"Skipped {skipped} files which were already done.")
\ No newline at end of file
......@@ -9,20 +9,15 @@
# standard library imports
import re
from argparse import ArgumentParser
from glob import glob
from math import isnan
from os.path import join, isdir
from warnings import warn, filterwarnings
from os.path import join
from warnings import warn
import numpy as np
# third party imports
from astropy.io import fits
from astropy.io.fits import Header, PrimaryHDU, Card
from astropy.io.fits.verify import VerifyWarning
from kis_tools.headers.GrisTranslator import NothingToDoError
from kis_tools.util.util import setup_outdir
from tqdm import tqdm
from kis_tools.util.util import load_git_info
from . import ObservationInfo
# custom imports
......@@ -35,7 +30,6 @@ from .properties import (
from ..util.util import (
gen_history_time,
gen_history_git,
get_git_info,
)
......@@ -204,7 +198,7 @@ def main(infile, outfile=None, wcs_generator=None, **kwargs): # noqa: C901
# translation processing step
old_hist = list(oi._header["HISTORY"])
hist_main = [gen_history_time("translate_header")]
git_info = get_git_info(__file__)
git_info = load_git_info()
hist_git = gen_history_git("translate_header", git_info)
hist_entries = old_hist + hist_main + hist_git
for hist_entry in hist_entries:
......@@ -270,57 +264,3 @@ def reorder_gris_data(data, reverse_axes=False):
return trans_data
def run_translate_header(cmd_line_args):
parser = ArgumentParser()
parser.add_argument(
"folder_in",
help="folder to be processed, needs to be a GRIS level1_split folder",
)
parser.add_argument(
"--folder_out",
help="path where output is written. Defaults to input folder",
default=None,
)
parser.add_argument(
"-v",
"--verbose",
help="don'tsilence warnings",
action="store_true",
default=False,
)
args = parser.parse_args(cmd_line_args)
folder_in = args.folder_in
folder_out = args.folder_out
if folder_out:
setup_outdir(folder_out, overwrite=True)
if not isdir(folder_in):
raise IOError(f"Directory {folder_in} not found!")
files = sorted(glob(join(folder_in, "*.fits")))
if not args.verbose:
filterwarnings("ignore", ".*", UserWarning)
filterwarnings("ignore", ".*", VerifyWarning)
progress_bar = tqdm(files)
skipped = 0
for f in progress_bar:
progress_bar.set_description(f"Modifying {f}")
if folder_out:
outfile = f.replace(folder_in, folder_out)
try:
main(f, outfile=f)
except NothingToDoError:
skipped += 1
continue
if skipped > 0:
print(f"Skipped {skipped} files which were already done.")
\ No newline at end of file
......@@ -50,7 +50,7 @@ class TestUtil(unittest.TestCase):
# Non repo file should not return anything
self.assertTrue(util.get_git_info(self.testfile) == {})
# Not commited file should return None, None
# Not committed file should return None, None
self.assertTrue(util.get_git_info(self.emptyfile) == {})
# A file that is sure to be in a repo
......
......@@ -7,18 +7,17 @@ import re
import subprocess
from collections import Counter
from glob import iglob
from os.path import basename, exists, join
from os.path import basename, exists, join, dirname
from shutil import rmtree, copyfileobj
from sys import exc_info
from warnings import filterwarnings
import astropy
import astropy.units as u
import pandas as pd
from PIL import Image
from astropy.io.fits import open as fitsopen
from sunpy.coordinates.sun import angular_radius
import astropy.units as u
def get_git_info(path):
......@@ -72,6 +71,18 @@ def get_git_info(path):
}
def write_git_info(path_in, path_out=join(dirname(__file__), "git_info.pkl")):
info = get_git_info(path_in)
with open(path_out, "wb+") as f:
pickle.dump(info, f)
def load_git_info(path_in=join(dirname(__file__), "git_info.pkl")):
with open(path_in, "rb") as f:
res = pickle.load(f)
return res
def get_md5(fname):
"""Calculate md5 has of a given file
......@@ -597,7 +608,7 @@ def add_history(header: astropy.io.fits.Header, file, name=None):
"""
if not name:
name = basename(file)
git_info = get_git_info(file)
git_info = load_git_info()
hist_time = gen_history_time(name)
hist_git = gen_history_git("git_info: " + name, git_info)
entries = [hist_time] + hist_git
......@@ -682,6 +693,7 @@ def get_data_folders(base_folder):
res = pool.map(has_chrotel_files, folders)
return [f for f, valid in res if valid]
def get_solar_radius(time):
r = angular_radius(time).to(u.arcsec)
return r.value
\ No newline at end of file
return r.value
name: sdc
channels:
- astropy
- conda-forge
- defaults
dependencies:
- tqdm
- astropy=3.1.2=py37h14c3975_0
- ipython=7.4.0=py37h24bf2e0_0
- matplotlib=3.0.3=py37_0
- numpy=1.16.2=py37h8b7e671_1
- pandas=0.24.2=py37hf484d3e_0
- pillow=6.0.0=py37he7afcd5_0
- pymongo=3.7.2=py37hf484d3e_0
- pytest=4.4.1=py37_0
- pytest-cov=2.6.1=py_0
- python=3.7.3=h5b0a415_0
- pyyaml=5.1=py37h14c3975_0
- scikit-image=0.15.0=py37he1b5a44_0
- setuptools=41.0.0=py37_0
- sunpy>=1.0.0
- tk=8.6.9=h84994c4_1001
- yaml=0.1.7=h14c3975_1001
- xlrd >= 1.0.0
"""Setup for the kis_tools package."""
from glob import glob
from os.path import join
import setuptools
from kis_tools import __version__
from kis_tools.util.util import write_git_info
with open('README.md') as f:
README = f.read()
from kis_tools import __version__
info = write_git_info(join("kis_tools", "__init__.py"),join("kis_tools","util", "git_info.pkl"))
version = __version__
setuptools.setup(
author="Carl Schaffer",
author_email="schaffer@leibniz-kis.de",
name='kis_tools',
license="MIT",
description='Tools for interacting with data produced at KIS',
version='v'+__version__,
version='v' + __version__,
long_description=README,
url='https://gitlab.leibniz-kis.de/sdc/kis_tools',
packages=setuptools.find_packages(),
include_package_data=True,
python_requires=">=3.6",
install_requires=[
"pandas",
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment