-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy path0_CleanData_BR.py
93 lines (88 loc) · 4.77 KB
/
0_CleanData_BR.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
# Cleaning data for all places of the country SINAN
# only confirmed cases, assign year and week of first symptoms
# years 2000 to 2021
# Code developed by Denise Cammarota
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import os
import sys
import glob
def clean_data_total():
files = glob.glob('./Data/DataBR/*.csv')
files = files[-3:] # years after 2021
for file in files:
# reading data
data_test = pd.read_csv(file,
delimiter = ',',
index_col=False,
parse_dates = ['DT_SIN_PRI','SEM_PRI','DT_NOTIFIC','SEM_NOT'])
# get the year we are working with in question
year = int(file[-8:-4])
if(year < 2007):
# first column is read differently
data_test = data_test.drop(columns = ['Unnamed: 0'])
#last row is weird
data_test = data_test[:-1]
# filtered confirmed cases
names_key = data_test.keys()
confirmed_class = ['1', '3', '4', '2', 1, 2, 3, 4, 10, 11, 12, '10', '11', '12']
if 'CON_CLASSI' in names_key:
data_test = data_test[data_test['CON_CLASSI'].isin(confirmed_class)]
else:
data_test = data_test[data_test['CLASSI_FIN'].isin(confirmed_class)]
# define symptom onset week and year
data_test = data_test[~data_test['SEM_PRI'].str.contains('/', regex=False)]
data_test = data_test[~data_test['SEM_PRI'].str.contains(' ', regex=False)]
data_test = data_test[~data_test['SEM_PRI'].str.contains('.', regex=False)]
data_test = data_test[~data_test['SEM_PRI'].str.contains('\x12008', regex=False)]
data_test = data_test[(data_test['SEM_PRI'].str.len() == 6)]
data_test['SIN_YEAR'] = data_test['SEM_PRI'].str[2:]
data_test['SIN_WEEK'] = data_test['SEM_PRI'].str[0:2]
data_test['SIN_YEAR'] = data_test['SIN_YEAR'].astype('int64')
data_test['SIN_WEEK'] = data_test['SIN_WEEK'].astype('int64')
# filtered weird data with symptoms after notification
filt_df1 = (data_test['DT_NOTIFIC'] >= data_test['DT_SIN_PRI'])
data_filtered_1 = data_test[filt_df1]
# filtered weird records with symptoms before 2000 (many of these)
filt_df2 = (data_filtered_1['SIN_YEAR'] >= 2000)
data_filtered_2 = data_filtered_1[filt_df2]
# save in a different path
if(not(os.path.exists('Data/DataBR_Processed/'))):
os.makedirs('Data/DataBR_Processed/')
path_save = './Data/DataBR_Processed/dengue_BR_'+str(year)+'.csv'
data_filtered_2.to_csv(path_save, sep=';')
else:
# first column is read differently
data_test = data_test.drop(columns = ['Unnamed: 0'])
#last row is weird
data_test = data_test[:-1]
# filtered confirmed cases
names_key = data_test.keys()
confirmed_class = ['1', '3', '4', '2', 1, 2, 3, 4, 10, 11, 12, '10', '11', '12']
if 'CON_CLASSI' in names_key:
data_test = data_test[data_test['CON_CLASSI'].isin(confirmed_class)]
else:
data_test = data_test[data_test['CLASSI_FIN'].isin(confirmed_class)]
# define symptom onset week and year
data_test = data_test[~data_test['SEM_PRI'].str.contains('/', regex=False)]
data_test = data_test[~data_test['SEM_PRI'].str.contains(' ', regex=False)]
data_test = data_test[~data_test['SEM_PRI'].str.contains('.', regex=False)]
data_test = data_test[~data_test['SEM_PRI'].str.contains('\x12008', regex=False)]
data_test = data_test[(data_test['SEM_PRI'].str.len() == 6)]
data_test['SIN_YEAR'] = data_test['SEM_PRI'].str[0:4]
data_test['SIN_WEEK'] = data_test['SEM_PRI'].str[4:]
data_test['SIN_YEAR'] = data_test['SIN_YEAR'].astype('int64')
data_test['SIN_WEEK'] = data_test['SIN_WEEK'].astype('int64')
# filtered weird data with symptoms after notification
filt_df1 = (data_test['DT_NOTIFIC'] >= data_test['DT_SIN_PRI'])
data_filtered_1 = data_test[filt_df1]
# filtered weird records with symptoms before 2000 (many of these)
filt_df2 = (data_filtered_1['SIN_YEAR'] >= 2000)
data_filtered_2 = data_filtered_1[filt_df2]
# save in a different path
if(not(os.path.exists('Data/DataBR_Processed/'))):
os.makedirs('Data/DataBR_Processed/')
path_save = './Data/DataBR_Processed/dengue_BR_'+str(year)+'.csv'
data_filtered_2.to_csv(path_save, sep=';')
clean_data_total()