Update demo, add chatbot example, add workflow

.github/workflows/snap.yml

@@ -0,0 +1,35 @@
+name: Build snap
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: 30 10 * * 1  # run Monday's at 10:30
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+jobs:
+  build:
+    strategy:
+      fail-fast: false
+
+    runs-on: "ubuntu-24.04"
+
+    steps:
+    - uses: actions/checkout@v4
+    - name: Install Snapcraft
+      uses: samuelmeuli/action-snapcraft@v3
+    - name: download genai
+      run: |
+        curl -O https://storage.openvinotoolkit.org/repositories/openvino_genai/packages/2024.5/linux/openvino_genai_ubuntu24_2024.5.0.0_x86_64.tar.gz
+        tar zxf openvino_genai_ubuntu24_2024.5.0.0_x86_64.tar.gz
+        mv openvino_genai_ubuntu24_2024.5.0.0_x86_64 openvino_genai
+    - name: ls
+      run: |
+        ls
+    - name: run snapcraft
+      run: |
+        sudo usermod -a -G lxd $USER
+        newgrp lxd
+        snapcraft

README.md

@@ -9,7 +9,7 @@ More information:
 ## About the example
 
 The example consists of three files: `CMakeLists.txt` and `openvinodemo.cpp` that make up the sample application, and `snap/snapcraft.yaml` that describes how to build the snap. The 
-demo application is trivially simple and just runs a few tokens of inference on an LLM.
+demo application is trivially simple and just runs a few tokens of inference on an LLM. For a more interesting application, rename openvinodemo_chatbot.cpp to openvinodemo.cpp. This creates a chatbot demo. The chatbot demo doesn't work with all models - models need to have a compatible chat template to work (For example: Llama 3.1 and 3.2 Instruct models).
 
 ## Limitations
 
@@ -26,6 +26,13 @@ First install snapcraft:
 sudo snap install --classic snapcraft
 ```
 
+Add your user to the `lxd` group:
+
+```shell
+sudo usermod -a -G lxd $USER
+newgrp lxd
+```
+
 Clone this repository and download and uncompress the OpenVINO GenAI archive. This example uses OpenVINO 2024.5. For other OpenVINO
 versions, change CMakeLists.txt to match.
 
@@ -66,8 +73,8 @@ You can copy this snap to another Ubuntu 24.04 machine, and install and run it t
 
 ## Run the demo
 
-```
-openvinodemo modelpath prompt CPU
+```shell
+openvinodemo modelpath
 ```
 
 > [!NOTE]
@@ -80,7 +87,7 @@ openvinodemo modelpath prompt CPU
 
 By default snap allows access to $HOME only. To allow other specific directories, for example `/data/models` you can run the following command on the system where you run the application:
 
-```
+```shell
 sudo snap set system homedirs=/data/models
 ```
 
@@ -101,6 +108,6 @@ cmake --build . --config Release
 cmake --install . --prefix "openvinodemo_install"
 ```
 
-### Snapcraft demo environment
+### Snapcraft debug environment
 
 Run `snapcraft --debug` instead of just `snapcraft` to launch a shell in the build environment.

openvinodemo.cpp

@@ -2,12 +2,12 @@
 #include <iostream>
 
 // Very basic demo application for testing
-// Usage: ./openvinodemo model_path prompt device
+// Usage: openvinodemo /path/to/model
 
 int main(int argc, char* argv[]) {
    std::filesystem::path model_path = argv[1];
-   std::string prompt = argv[2];
-   ov::genai::LLMPipeline pipe(model_path, argv[3]);
+   std::string prompt = "why is the sky blue?";
+   ov::genai::LLMPipeline pipe(model_path, "CPU");
    ov::genai::GenerationConfig config = pipe.get_generation_config();
    config.max_new_tokens = 32;
    std::string result = pipe.generate(prompt, config);

openvinodemo_chatbot.cpp

@@ -0,0 +1,58 @@
+// Copyright (C) 2023-2024 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+//
+// Modified from https://github.com/openvinotoolkit/openvino.genai/tree/master/samples/cpp/chat_sample
+// OpenVINO chat_sample WITHOUT HISTORY between questions
+//
+// Usage: openvinodemo /path/to/model
+
+#include "openvino/genai/llm_pipeline.hpp"
+
+std::string start_message = " <|start_header_id|>system<|end_header_id|>\n\n You are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe.  Your answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure that your responses are socially unbiased and positive in nature. Make sure that code is concise and correct.  If a question does not make any sense or is not factually coherent, explain why instead of answering something not correct. If you don't know the answer to a question, please don't share false information.<|eot_id|>";
+
+int main(int argc, char* argv[]) try {
+    if (2 != argc) {
+        throw std::runtime_error(std::string{"Usage: "} + argv[0] + " <MODEL_DIR>");
+    }
+    std::string prompt; 
+    std::string models_path = argv[1];
+    std::string device = "CPU";
+    ov::genai::LLMPipeline pipe(models_path, device);
+
+    ov::genai::GenerationConfig config = pipe.get_generation_config();
+    config.max_new_tokens = 512;
+    config.do_sample = false;
+    config.repetition_penalty = 1.1;
+
+    std::function<bool(std::string)> streamer = [](std::string word) { 
+        std::cout << word << std::flush;
+        // Return flag corresponds whether generation should be stopped.
+        // false means continue generation.
+        return false; 
+    };
+
+    // Warmup inference for GPU reproducibility
+    ov::genai::GenerationConfig warmupconfig = pipe.get_generation_config();
+    warmupconfig.max_new_tokens = 5;
+    warmupconfig.do_sample = false;
+    pipe.generate("say something", warmupconfig);
+
+    pipe.start_chat(start_message);
+    std::cout << "question:\n";
+    while (std::getline(std::cin, prompt)) {
+        pipe.generate(prompt, config, streamer);
+        std::cout << "\n----------\n"
+            "question:\n";
+    }
+    pipe.finish_chat();
+} catch (const std::exception& error) {
+    try {
+        std::cerr << error.what() << '\n';
+    } catch (const std::ios_base::failure&) {}
+    return EXIT_FAILURE;
+} catch (...) {
+    try {
+        std::cerr << "Non-exception object thrown\n";
+    } catch (const std::ios_base::failure&) {}
+    return EXIT_FAILURE;
+}

snap/snapcraft.yaml

@@ -10,6 +10,10 @@ confinement: strict
 apps:
   openvinodemo:
     command: openvinodemo 
+    # To allow the application to find TBB libraries, you can set the
+    # LD_LIBRARY_PATH to the $SNAP directory, as below, or run
+    # `patchelf --set-rpath "\$ORIGIN"` for all OpenVINO and TBB libraries
+    # in the openvino_genai archive before building the snap
     environment:
       LD_LIBRARY_PATH: $LD_LIBRARY_PATH:$SNAP
     plugs: