From c6d4e8fb7513ceaa254b940ef90edf72180869f2 Mon Sep 17 00:00:00 2001 From: Vincent Nguyen Date: Wed, 18 Dec 2024 18:49:11 +0100 Subject: [PATCH] EuroLLM Gradio (web based) translator 35 languages to 35 languages (#164) --- recipes/eurollm/README.md | 36 +++++++ recipes/eurollm/eole-translator.py | 151 +++++++++++++++++++++++++++++ recipes/eurollm/serve.yaml | 9 ++ 3 files changed, 196 insertions(+) create mode 100644 recipes/eurollm/README.md create mode 100644 recipes/eurollm/eole-translator.py create mode 100644 recipes/eurollm/serve.yaml diff --git a/recipes/eurollm/README.md b/recipes/eurollm/README.md new file mode 100644 index 00000000..81c04ef9 --- /dev/null +++ b/recipes/eurollm/README.md @@ -0,0 +1,36 @@ +# Serving Eurollm in a gradio interface with Eole + +## Retrieve and convert model + +### Set environment variables + +``` +export EOLE_MODEL_DIR= +export HF_TOKEN= +``` + +### Download and convert model + + +``` +eole convert HF --model_dir utter-project/EuroLLM-9B-Instruct --output $EOLE_MODEL_DIR/EuroLLM-9B-Instruct --token $HF_TOKEN +``` + +## Run server with the config file from this folder (you can add options according to your needs) + +``` +eole serve -c serve.yaml +``` + +## Start the gradio based translator + +``` +python eole-translator.py +``` + +You can access the Web based translator from the url given by Gradio (either local or from Gradio proxy with share=True turned on) + + +## Alternatively you can also play with the API + +FastAPI exposes a swagger UI by default. It should be accessible via your browser at `http://localhost:5000/docs`. diff --git a/recipes/eurollm/eole-translator.py b/recipes/eurollm/eole-translator.py new file mode 100644 index 00000000..9429f50f --- /dev/null +++ b/recipes/eurollm/eole-translator.py @@ -0,0 +1,151 @@ +import gradio as gr +import requests + +# List of 35 supported languages +languages = [ + "Bulgarian", + "Croatian", + "Czech", + "Danish", + "Dutch", + "English", + "Estonian", + "Finnish", + "French", + "German", + "Greek", + "Hungarian", + "Irish", + "Italian", + "Latvian", + "Lithuanian", + "Maltese", + "Polish", + "Portuguese", + "Romanian", + "Slovak", + "Slovenian", + "Spanish", + "Swedish", + "Arabic", + "Catalan", + "Chinese", + "Galician", + "Hindi", + "Japanese", + "Korean", + "Norwegian", + "Russian", + "Turkish", + "Ukrainian", +] + + +# Function to call Eole API +def translate_text(source_lang, target_lang, source_text): + # Define the API URL for inference + api_url = "http://localhost:5000/infer" + + # Replace this with the correct model ID from the server's configuration + model_id = "EuroLLM-9B-Instruct" # Adjust this to match the server configuration + + # Construct the payload for the API + payload = { + "model": model_id, + "inputs": ( + f"<|im_start|>system｟newline｠<|im_end|>｟newline｠<|im_start|>user｟newline｠" + f"Translate the following text from {source_lang} into {target_lang}.｟newline｠" + f"{source_lang}: {source_text}｟newline｠" + f"{target_lang}: <|im_end|>｟newline｠<|im_start|>assistant｟newline｠" + ), + # Optional: Add decoding settings (based on `DecodingConfig`) + "temperature": 1, + "max_length": 512, + } + + try: + # Send the POST request + response = requests.post(api_url, json=payload) + response.raise_for_status() # Raise exception for HTTP errors + + # Parse the response + result = response.json() + predictions = result.get("predictions", []) + + # Extract the first translation, if available + if predictions and predictions[0]: + translated_text = predictions[0][0] # First prediction of the first input + else: + translated_text = "Error: No translation returned by the server." + except Exception as e: + translated_text = f"Error: {str(e)}" + + return translated_text + + +# Gradio Interface +def gradio_translation_interface(source_lang, target_lang, source_text): + return translate_text(source_lang, target_lang, source_text) + + +with gr.Blocks(title="EuroLLM-Eole Multilingual Translator") as iface: + # Add Title + with gr.Row(): + gr.Markdown( + "

EuroLLM-Eole Multilingual Translator

" + ) + + # Add side-by-side inputs + with gr.Row(equal_height=True): + with gr.Column(scale=1): + source_lang = gr.Dropdown( + languages, label="Source Language", value="English" + ) + source_text = gr.Textbox( + placeholder="Enter text here...", lines=10, label="Source Text" + ) + with gr.Column(scale=1): + target_lang = gr.Dropdown( + languages, label="Target Language", value="French" + ) + translated_text = gr.Textbox( + placeholder="Translation will appear here...", + lines=10, + label="Translated Text", + interactive=False, + ) + + # Add a narrower Translate button + with gr.Row(): + with gr.Column(scale=1, min_width=200, elem_id="button-container"): + translate_button = gr.Button("Translate") + + # Define interactions + translate_button.click( + gradio_translation_interface, + inputs=[source_lang, target_lang, source_text], + outputs=[translated_text], + ) + +# Add custom CSS for styling +iface.css = """ +#button-container { + display: flex; + justify-content: center; +} +button { + width: 200px; + background-color: #4285F4; + color: white; + font-size: 16px; + border: none; + border-radius: 4px; + padding: 10px; + cursor: pointer; +} +button:hover { + background-color: #357AE8; +} +""" + +iface.launch(share=True) diff --git a/recipes/eurollm/serve.yaml b/recipes/eurollm/serve.yaml new file mode 100644 index 00000000..823345e4 --- /dev/null +++ b/recipes/eurollm/serve.yaml @@ -0,0 +1,9 @@ +models_root: "." # used only for HF downloads for now, but might override $EOLE_MODEL_DIR at some point +models: +# local model +- id: "EuroLLM-9B-Instruct" + path: "${EOLE_MODEL_DIR}/EuroLLM-9B-Instruct" + preload: true +# config: +# quant_layers: ['gate_up_proj', 'down_proj', 'up_proj', 'linear_values', 'linear_query', 'linear_keys', 'final_linear'] +# quant_type: "bnb_NF4"