-
Notifications
You must be signed in to change notification settings - Fork 174
/
pile-europarl-refine.yaml
58 lines (54 loc) · 1.58 KB
/
pile-europarl-refine.yaml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
# global parameters
project_name: 'Data-Juicer-recipes-EuroParl'
dataset_path: '/path/to/your/dataset' # path to your dataset directory or file
export_path: '/path/to/your/dataset.jsonl'
np: 50 # number of subprocess to process your dataset
open_tracer: true
# process schedule
# a list of several process operators with their arguments
process:
- clean_email_mapper:
- clean_links_mapper:
- fix_unicode_mapper:
- punctuation_normalization_mapper:
- whitespace_normalization_mapper:
- alphanumeric_filter:
tokenization: false
min_ratio: 0.75 # <3sigma (0.779)
max_ratio: 0.90 # >3sigma(0.878)
- average_line_length_filter: # for code
max_len: 588 # 3sigma
- character_repetition_filter:
rep_len: 10
max_ratio: 0.16 # >3sigma (0.114)
- flagged_words_filter:
lang: en
tokenization: true
max_ratio: 0.0007 # 3sigma
- language_id_score_filter:
min_score: 0.7
- maximum_line_length_filter: # for code
max_len: 4000 # >3sigma (3104)
- perplexity_filter:
lang: en
max_ppl: 7596 #(3sigma)
- special_characters_filter:
max_ratio: 0.3 # > 3sigma (0.243)
- text_length_filter:
max_len: 2e5
- words_num_filter:
tokenization: true
min_num: 20
max_num: 1e5 # 3sigma
- word_repetition_filter:
lang: en
tokenization: true
rep_len: 10
max_ratio: 0.2 # > 3sigma (0.185)
- document_simhash_deduplicator:
tokenization: space
window_size: 6
lowercase: true
ignore_pattern: '\p{P}'
num_blocks: 6
hamming_distance: 4