-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathprocess_data.py
64 lines (54 loc) · 2.26 KB
/
process_data.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
# import os
# import librosa
# # Set directories
# ogg_dir = "data_narrative/ogg/open_images_train_v6_localized_narratives-00000-of-00010"
# image_features_dir = "data_narrative/image_features/open_images_train_v6_localized_narratives-00000-of-00010"
# txt_file = "data_narrative/data_list/open_images_train_v6_localized_narratives-00000-of-00010.txt"
# # Loop through ogg files
# for filename in os.listdir(ogg_dir):
# if filename.endswith(".ogg"):
# # Extract duration
# full_path = os.path.join(ogg_dir, filename)
# duration = librosa.get_duration(filename=full_path)
# # Check duration and remove file and npy file if necessary
# if duration >= 30:
# # os.remove(full_path)
# # npy_path = os.path.join(image_features_dir, f"{filename[:-4]}.npy")
# # if os.path.exists(npy_path):
# # os.remove(npy_path)
# # Remove filename from txt file
# with open(txt_file, 'r') as f:
# lines = f.readlines()
# with open(txt_file, 'w') as f:
# for line in lines:
# if filename not in line:
# f.write(line)
# filename = "path/to/your/file.txt" # Replace with your filename
# with open(filename, "r") as f:
# lines = f.readlines()
# new_lines = []
# for line in lines:
# if len(line) <= 120:
# new_lines.append(line)
# with open(filename, "w") as f:
# f.writelines(new_lines)
import os
import librosa
ogg_dir = "data_narrative/ogg/open_images_train_v6_localized_narratives-00000-of-00010"
filename = "data_narrative/data_list/open_images_train_v6_localized_narratives-00000-of-00010.txt"
with open(filename, "r") as f:
lines = f.readlines()
new_lines = []
for line in lines:
image_id, caption, url = line.split("\t")
image_id=image_id.split('/')[-1]
# Check caption length and ogg file duration
if len(caption) <= 100000:
ogg_file = os.path.join(ogg_dir, f"{image_id}.ogg")
if os.path.exists(ogg_file):
duration = librosa.get_duration(filename=ogg_file)
if duration < 30:
new_lines.append(line)
# Write filtered lines to file
with open(filename, "w") as f:
f.writelines(new_lines)