clean_data.py 691 Bytes
Newer Older
1
2
3
4
5
6
7
8
#!/usr/bin/env python3
# -*- coding: utf-8 -*-

"""
Created by schaffer at 11/26/19

"""
import pandas as pd
9
10

from kis_tools.gris.study_gris_slit_orientation.get_data import get_raw_data
11
12
13


def get_cleaned_data(*args, **kwargs):
14
    raw_values = get_raw_data(*args, **kwargs)
15
    df = pd.DataFrame(raw_values)
Carl Schaffer's avatar
Carl Schaffer committed
16
    df["year"] = df.date.dt.year
17
18

    # Clean values
Carl Schaffer's avatar
Carl Schaffer committed
19
20
21
22
23
    df["SLITORIE"] = pd.to_numeric(df.SLITORIE.replace(""))
    if "DATE-BEG" in df.columns:
        df["date_beg"] = pd.to_datetime(df["DATE-BEG"])
        del df["DATE-BEG"]
    df["day"] = df.date.dt.dayofyear
24
25
26

    # Try to model the angle correlation if no deroator is installed
    cleaned = df.dropna()
27

28
    return cleaned