Skip to content

Commit

Permalink
Google Translator: Improves newline handling
Browse files Browse the repository at this point in the history
- The newline replacement only worked when extra spaces were added
  by Google Translate API, which is not always happening.
- Adds some more tests to catch this and changes the replacement to
  be closer to replacing of interpolations.
- Fixes #595

This is a frustating bug that has been open for many years:
https://issuetracker.google.com/issues/119256504?pli=1

Maybe one of the other APIs are more consistent.
davidwessman authored and glebm committed Sep 15, 2024

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature. The key has expired.
1 parent 2cba109 commit 7b4eb29
Showing 2 changed files with 46 additions and 23 deletions.
10 changes: 7 additions & 3 deletions lib/i18n/tasks/translators/google_translator.rb
Original file line number Diff line number Diff line change
@@ -21,7 +21,7 @@ def translate_values(list, **options)
EasyTranslate.translate(
replace_newlines_with_placeholder(list, options[:html]),
options,
format: :text
format: options[:html] ? :html : :text
),
options[:html]
)
@@ -69,15 +69,19 @@ def replace_newlines_with_placeholder(list, html)
return list unless html

list.map do |value|
value.gsub("\n", NEWLINE_PLACEHOLDER)
value.gsub(/\n(\s*)/) do
"<Z__#{::Regexp.last_match(1)&.length || 0}>"
end
end
end

def restore_newlines(translations, html)
return translations unless html

translations.map do |translation|
translation.gsub("#{NEWLINE_PLACEHOLDER} ", "\n")
translation.gsub(/<Z__(\d+)>/) do
"\n#{' ' * ::Regexp.last_match(1).to_i}"
end
end
end
end
59 changes: 39 additions & 20 deletions spec/google_translate_spec.rb
Original file line number Diff line number Diff line change
@@ -7,10 +7,26 @@
nil_value_test = ['nil-value-key', nil, nil]
empty_value_test = ['empty-value-key', '', '']
text_test = ['hello', "Hello, %{user} O'Neill!", "¡Hola, %{user} O'Neill!"]
text_test_multiline = ['hello_multiline', "Hello,\n%{user}\nO'Neill!", "Hola,\n%{user}\n¡O'Neill!"]
text_test_multiline = [
'hello_multiline',
"Hello,\n%{user}\nO'Neill!",
"Hola,\n%{user}\nO'Neill!"
]
html_test = ['html-key.html', "Hello, <b>%{user} O'neill</b>", "Hola, <b>%{user} O'neill</b>"]
html_test_plrl = ['html-key.html.one', '<b>Hello %{count}</b>', '<b>Hola %{count}</b>']
html_test_multiline = ['html-key.html.multiline', "<b>Hello</b>\n<b>%{user}</b>", "<b>Hola</b>\n<b>%{user}</b>"]
html_test_multiline = [
'html-key.html.multiline_html',
"<b>Hello</b>\n<b>%{user}</b>",
"<b>Hola</b>\n <b>%{user}</b>"
]
# Google Translate API adds extra spaces before some characters
# https://issuetracker.google.com/issues/119256504?pli=1
# Atleast it should be valid HTML
html_test_multiline_indentation = [
'html-key.html.multiline_indentation_html',
"<p>Hello</p>\n<ul>\n <li>%{user}</li>\n <li>\n %{user2}\n </li>\n <li>Dog</li>\n<ul>\n",
"<p>Hola</p>\n<ul>\n <li> %{user}</li>\n <li>\n %{user2}\n </li>\n <li> Perro</li>\n<ul>\n"
]
array_test = ['array-key', ['Hello.', nil, '', 'Goodbye.'], ['Hola.', nil, '', 'Adiós.']]
fixnum_test = ['numeric-key', 1, 1]
ref_key_test = ['ref-key', :reference, :reference]
@@ -30,27 +46,29 @@
let(:task) { i18n_task }

it 'works' do # rubocop:disable RSpec/MultipleExpectations
skip 'temporarily disabled on JRuby due to https://github.com/jruby/jruby/issues/4802' if RUBY_ENGINE == 'jruby'
skip 'GOOGLE_TRANSLATE_API_KEY env var not set' unless ENV['GOOGLE_TRANSLATE_API_KEY']
skip 'GOOGLE_TRANSLATE_API_KEY env var is empty' if ENV['GOOGLE_TRANSLATE_API_KEY'].empty?
in_test_app_dir do
task.data[:en] = build_tree('en' => {
'common' => {
'a' => 'λ',
'hello' => text_test[1],
'hello_multiline' => text_test_multiline[1],
'hello_html' => html_test[1],
'hello_plural_html' => {
'one' => html_test_plrl[1]
},
'hello_multiline_html' => html_test_multiline[1],
'array_key' => array_test[1],
'nil-value-key' => nil_value_test[1],
'empty-value-key' => empty_value_test[1],
'fixnum-key' => fixnum_test[1],
'ref-key' => ref_key_test[1]
}
})
task.data[:en] = build_tree(
'en' => {
'common' => {
'a' => 'λ',
'hello' => text_test[1],
'hello_multiline' => text_test_multiline[1],
'hello_html' => html_test[1],
'hello_plural_html' => {
'one' => html_test_plrl[1]
},
'hello_multiline_html' => html_test_multiline[1],
'multiline_indentation_html' => html_test_multiline_indentation[1],
'array_key' => array_test[1],
'nil-value-key' => nil_value_test[1],
'empty-value-key' => empty_value_test[1],
'fixnum-key' => fixnum_test[1],
'ref-key' => ref_key_test[1]
}
}
)
task.data[:es] = build_tree('es' => {
'common' => {
'a' => 'λ'
@@ -63,6 +81,7 @@
expect(task.t('common.hello_html', 'es')).to eq(html_test[2])
expect(task.t('common.hello_plural_html.one', 'es')).to eq(html_test_plrl[2])
expect(task.t('common.hello_multiline_html', 'es')).to eq(html_test_multiline[2])
expect(task.t('common.multiline_indentation_html', 'es')).to eq(html_test_multiline_indentation[2])
expect(task.t('common.array_key', 'es')).to eq(array_test[2])
expect(task.t('common.nil-value-key', 'es')).to eq(nil_value_test[2])
expect(task.t('common.empty-value-key', 'es')).to eq(empty_value_test[2])

0 comments on commit 7b4eb29

Please sign in to comment.