-
Notifications
You must be signed in to change notification settings - Fork 0
/
3sortdata.py
81 lines (64 loc) · 2.48 KB
/
3sortdata.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
import pandas as pd
import os
from collections import Counter
import numpy as np
# 考虑这里开始分 窗口 把time drop掉重新设time!
def train():
os.chdir('data')
sample = 6000
data = pd.read_csv('raw_data.csv')
# data.sort_values('time', kind='mergesort', inplace=True)
data.drop(['time'], inplace=True, axis=1)
time = pd.DataFrame(np.arange(1, (data.shape[0] + 1) / sample))
time = pd.concat(([time] * sample), axis=1)
time = pd.DataFrame(np.asarray(time).flatten(), columns=["time"], index=None)
data = pd.concat((time, pd.DataFrame(data)), axis=1)
print(data.shape)
def fun(group):
label = Counter(group.label).most_common(1)[0][0]
group = group.loc[lambda s: s.label == label]
return group
#
# # ----------label,顺便改变时间之后就可以设置不同的窗口了
label = data.groupby(['time']).apply(
lambda group: Counter(group.label).most_common(1)[0][0]
)
print(label.shape)
label.to_csv('label_unsorted_60.csv', index=False, header=['label'])
fin = data.groupby(['time']).apply(
lambda group: fun(group)
)
print(fin.shape)
fin.to_csv('data_unsorted_60.csv', index=False, chunksize=6000000)
def dev():
sample = 6000
os.chdir('test')
data = pd.read_csv('raw_data.csv')
# data.sort_values('time', kind='mergesort', inplace=True)
data.drop(['time'], inplace=True, axis=1)
label = data['label']
print(label.shape)
label.to_csv('label_unsorted_all.csv', index=False, header=['label'])
time = pd.DataFrame(np.arange(1, (data.shape[0] + 1) / sample))
time = pd.concat(([time] * sample), axis=1)
time = pd.DataFrame(np.asarray(time).flatten(), columns=["time"], index=None)
data = pd.concat((time, pd.DataFrame(data)), axis=1)
print(data.shape)
def fun(group):
label = Counter(group.label).most_common(1)[0][0]
group = group.loc[lambda s: s.label == label]
group = pd.DataFrame(group)
return group
# ----------label,顺便改变时间之后就可以设置不同的窗口了
label = data.groupby(['time']).apply(
lambda group: Counter(group.label).most_common(1)[0][0]
) #
data = data.groupby(['time']).apply(
lambda group: fun(group)
)
print(label.shape)
label.to_csv('label_unsorted_60.csv', index=False, header=['label'])
data.to_csv('data_unsorted_60.csv', index=False, chunksize=6000000)
if __name__ == '__main__':
train()
# dev()