diff --git a/.github/workflows/snap.yml b/.github/workflows/snap.yml new file mode 100644 index 0000000..b25afe8 --- /dev/null +++ b/.github/workflows/snap.yml @@ -0,0 +1,35 @@ +name: Build snap + +on: + workflow_dispatch: + schedule: + - cron: 30 10 * * 1 # run Monday's at 10:30 + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + build: + strategy: + fail-fast: false + + runs-on: "ubuntu-24.04" + + steps: + - uses: actions/checkout@v4 + - name: Install Snapcraft + uses: samuelmeuli/action-snapcraft@v3 + - name: download genai + run: | + curl -O https://storage.openvinotoolkit.org/repositories/openvino_genai/packages/2024.5/linux/openvino_genai_ubuntu24_2024.5.0.0_x86_64.tar.gz + tar zxf openvino_genai_ubuntu24_2024.5.0.0_x86_64.tar.gz + mv openvino_genai_ubuntu24_2024.5.0.0_x86_64 openvino_genai + - name: ls + run: | + ls + - name: run snapcraft + run: | + sudo usermod -a -G lxd $USER + newgrp lxd + snapcraft diff --git a/README.md b/README.md index fb0f550..4364f43 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ More information: ## About the example The example consists of three files: `CMakeLists.txt` and `openvinodemo.cpp` that make up the sample application, and `snap/snapcraft.yaml` that describes how to build the snap. The -demo application is trivially simple and just runs a few tokens of inference on an LLM. +demo application is trivially simple and just runs a few tokens of inference on an LLM. For a more interesting application, rename openvinodemo_chatbot.cpp to openvinodemo.cpp. This creates a chatbot demo. The chatbot demo doesn't work with all models - models need to have a compatible chat template to work (For example: Llama 3.1 and 3.2 Instruct models). ## Limitations @@ -26,6 +26,13 @@ First install snapcraft: sudo snap install --classic snapcraft ``` +Add your user to the `lxd` group: + +```shell +sudo usermod -a -G lxd $USER +newgrp lxd +``` + Clone this repository and download and uncompress the OpenVINO GenAI archive. This example uses OpenVINO 2024.5. For other OpenVINO versions, change CMakeLists.txt to match. @@ -66,8 +73,8 @@ You can copy this snap to another Ubuntu 24.04 machine, and install and run it t ## Run the demo -``` -openvinodemo modelpath prompt CPU +```shell +openvinodemo modelpath ``` > [!NOTE] @@ -80,7 +87,7 @@ openvinodemo modelpath prompt CPU By default snap allows access to $HOME only. To allow other specific directories, for example `/data/models` you can run the following command on the system where you run the application: -``` +```shell sudo snap set system homedirs=/data/models ``` @@ -101,6 +108,6 @@ cmake --build . --config Release cmake --install . --prefix "openvinodemo_install" ``` -### Snapcraft demo environment +### Snapcraft debug environment Run `snapcraft --debug` instead of just `snapcraft` to launch a shell in the build environment. diff --git a/openvinodemo.cpp b/openvinodemo.cpp index 5b61d5e..800140e 100644 --- a/openvinodemo.cpp +++ b/openvinodemo.cpp @@ -2,12 +2,12 @@ #include // Very basic demo application for testing -// Usage: ./openvinodemo model_path prompt device +// Usage: openvinodemo /path/to/model int main(int argc, char* argv[]) { std::filesystem::path model_path = argv[1]; - std::string prompt = argv[2]; - ov::genai::LLMPipeline pipe(model_path, argv[3]); + std::string prompt = "why is the sky blue?"; + ov::genai::LLMPipeline pipe(model_path, "CPU"); ov::genai::GenerationConfig config = pipe.get_generation_config(); config.max_new_tokens = 32; std::string result = pipe.generate(prompt, config); diff --git a/openvinodemo_chatbot.cpp b/openvinodemo_chatbot.cpp new file mode 100644 index 0000000..d046535 --- /dev/null +++ b/openvinodemo_chatbot.cpp @@ -0,0 +1,58 @@ +// Copyright (C) 2023-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// +// Modified from https://github.com/openvinotoolkit/openvino.genai/tree/master/samples/cpp/chat_sample +// OpenVINO chat_sample WITHOUT HISTORY between questions +// +// Usage: openvinodemo /path/to/model + +#include "openvino/genai/llm_pipeline.hpp" + +std::string start_message = " <|start_header_id|>system<|end_header_id|>\n\n You are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe. Your answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure that your responses are socially unbiased and positive in nature. Make sure that code is concise and correct. If a question does not make any sense or is not factually coherent, explain why instead of answering something not correct. If you don't know the answer to a question, please don't share false information.<|eot_id|>"; + +int main(int argc, char* argv[]) try { + if (2 != argc) { + throw std::runtime_error(std::string{"Usage: "} + argv[0] + " "); + } + std::string prompt; + std::string models_path = argv[1]; + std::string device = "CPU"; + ov::genai::LLMPipeline pipe(models_path, device); + + ov::genai::GenerationConfig config = pipe.get_generation_config(); + config.max_new_tokens = 512; + config.do_sample = false; + config.repetition_penalty = 1.1; + + std::function streamer = [](std::string word) { + std::cout << word << std::flush; + // Return flag corresponds whether generation should be stopped. + // false means continue generation. + return false; + }; + + // Warmup inference for GPU reproducibility + ov::genai::GenerationConfig warmupconfig = pipe.get_generation_config(); + warmupconfig.max_new_tokens = 5; + warmupconfig.do_sample = false; + pipe.generate("say something", warmupconfig); + + pipe.start_chat(start_message); + std::cout << "question:\n"; + while (std::getline(std::cin, prompt)) { + pipe.generate(prompt, config, streamer); + std::cout << "\n----------\n" + "question:\n"; + } + pipe.finish_chat(); +} catch (const std::exception& error) { + try { + std::cerr << error.what() << '\n'; + } catch (const std::ios_base::failure&) {} + return EXIT_FAILURE; +} catch (...) { + try { + std::cerr << "Non-exception object thrown\n"; + } catch (const std::ios_base::failure&) {} + return EXIT_FAILURE; +} diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index 128696a..8b95d67 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -10,6 +10,10 @@ confinement: strict apps: openvinodemo: command: openvinodemo + # To allow the application to find TBB libraries, you can set the + # LD_LIBRARY_PATH to the $SNAP directory, as below, or run + # `patchelf --set-rpath "\$ORIGIN"` for all OpenVINO and TBB libraries + # in the openvino_genai archive before building the snap environment: LD_LIBRARY_PATH: $LD_LIBRARY_PATH:$SNAP plugs: