#Reads in the raw data from the MetDB office. Puts it into one csv file per year per station. 

import csv
import os
import shutil

rootdir = input("Please input full path of data folder: ") 
datadir = input("Please input full path of output folder: ") 
for subdir, dirs, files in os.walk(rootdir):
    for file in files:
        if file.endswith(".csv"):
            print("processed " + file)
            date = file[len("ukmo-metdb_winpro_"):len(file)-len(".csv")]
            year = date[0:4]
            month = date[4:6]
            day = date[6:8]
            with open(os.path.join(subdir, file)) as csvfile:
                reader = csv.reader(csvfile, delimiter=',', quotechar='|')
                shouldsave = False;
                for row in reader:
                    if len(row) == 1 and row[0].find("end data")==0:
                        shouldsave = False
                        targetfile.close()
                    if shouldsave:
                        rawstr = ','.join(str(e) for e in row)
                        targetfile.writelines(datevalid + "," + avgtimeperiod + "," + pressure + "," + temperature + "," + precipitation + "," + humidity + "," + history + "," + rawstr + "\n")
                    if len(row) == 4 and row[0].find("location") == 0:
                        lat = row[2]
                        lon = row[3]
                    elif len(row) == 3 and row[0].find("date_valid") == 0:
                        datevalid = row[2]
                    elif row[0].find("observation_station") == 0:
                        location = row[2][len("WMO "):]
                    elif row[0].find("height") == 0:
                        height = row[2]
                    elif len(row) == 4 and row[0].find("averaging_time_period") == 0:
                        avgtimeperiod = row[2]
                    elif len(row) == 4 and row[0].find("pressure_at_mean_sea_level") == 0:
                        pressure = row[2]
                    elif len(row) == 4 and row[0].find("air_temperature") == 0:
                        temperature = row[2]
                    elif len(row) == 4 and row[0].find("precipitation_amount") == 0:
                        precipitation = row[2]
                    elif len(row) == 4 and row[0].find("relative_humidity") == 0:
                        humidity = row[2]
                    elif len(row) == 3 and row[0].find("history") == 0:
                        history = row[2][:len("yyyy-mm-dd hh:mm:ss")]
                    elif len(row) == 11 and row[0].find("1")==0 and row[10].find("11")==0:
                        shouldsave = True
                        shouldhead = False
                        outfilename = location + "_" + lat + "_" + lon + "_" + height + ".csv"
                        if not os.path.isfile(os.path.join(datadir, outfilename)):
                            shouldhead = True
                        targetfile = open(os.path.join(datadir, outfilename), "a")
                        if shouldhead:
                            targetfile.writelines("Date Valid,Avg_time_period,Pressure,Temperature,Precipitation,Humidity,History,Level Height,WP mode info,US WP QC info,Euro uv WP QC info,Euro W WP QC info,Wind u,Wind v,Wind w,hrzl wind speed SD,vrtl wind speed SD,SNR\n")