Fixed a small typo in the README

Bump llama-cpp-python[server] from 0.1.54 to 0.1.56

README.md

@@ -1,6 +1,6 @@
 # Eucalyptus Chat
 
-A frontend for [Koala](https://bair.berkeley.edu/blog/2023/04/03/koala/) running on CPU with [llama.cpp](https://github.com/ggerganov/llama.cpp), using the API server library provided by [llama-cpp-python](https://github.com/abetlen/llama-cpp-python).
+A frontend for large language models like [🐨 Koala](https://bair.berkeley.edu/blog/2023/04/03/koala/) or [🦙 Vicuna](https://lmsys.org/blog/2023-03-30-vicuna/) running on CPU with [llama.cpp](https://github.com/ggerganov/llama.cpp), using the API server library provided by [llama-cpp-python](https://github.com/abetlen/llama-cpp-python).
 
 ![](misc/screenshot.png)
 
@@ -8,22 +8,33 @@ A frontend for [Koala](https://bair.berkeley.edu/blog/2023/04/03/koala/) running
 
 - Python 3.10
 - The pip packages listed in `requirements.txt`
-- A Koala model in the ggml format (should be quantized)
+- An AI model in the ggml format (should be quantized)
 
-The 7B-Model, `q4_0`-quantized, requires approx. 5 GB of RAM.
+For memory and disk requirements for the different models, see [llama.cpp - Memory/Disk Requirements](https://github.com/ggerganov/llama.cpp#memorydisk-requirements)
+
+## Supported Models
+
+- [🐨 Koala](https://bair.berkeley.edu/blog/2023/04/03/koala/)
+- [🦙 Vicuna v.0](https://lmsys.org/blog/2023-03-30-vicuna/)
+- [🦙 Vicuna v.1.1](https://lmsys.org/blog/2023-03-30-vicuna/)
+- [🦁 Manticore Chat](https://huggingface.co/openaccess-ai-collective/manticore-13b-chat-pyg)
+
+(see `./profiles/`)
 
 ## Usage
 
 To use Eucalyptus locally, start both the API-Server (`api-server.py`) and the Frontend-Server (`frontend-server.py`).  
 The default URL of the Frontend-Server is http://localhost:8080.
 
+You have to choose the correct profile for the model you use. See [Supported Models](#supported-models) and [Frontend Server CLI Argument](#frontend-server-cli-arguments).
+
 ### API Server CLI Arguments
 
 The following command-line arguments are available:
 
 * `-m` or `--model`: Specifies the path to the model file. This is required and must be provided.
-* `--host`: Specifies the address to listen on. By default, it listens on localhost.
+* `--host`: Specifies the address to listen on. By default, it listens on `localhost`.
-* `--port`: Specifies the port number to listen on. The default value is 7331.
+* `--port`: Specifies the port number to listen on. The default value is `7331`.
 
 ```bash
 python3 api-server.py [-h] -m MODEL [--host HOST] [--port PORT]
@@ -33,12 +44,13 @@ python3 api-server.py [-h] -m MODEL [--host HOST] [--port PORT]
 
 The following command-line options are available:
 
-* `--host`: Specifies the IP address or hostname to listen on. Defaults to "localhost".
+* `--profile`: Path to the profile file for the model.
-* `--port`: Specifies the port number to listen on. Defaults to 8080.
+* `--host`: Specifies the IP address or hostname to listen on. Defaults to `localhost`.
-* `--api`: Specifies the URL of the API server. Defaults to http://localhost:7331.
+* `--port`: Specifies the port number to listen on. Defaults to `8080`.
+* `--api`: Specifies the URL of the API server. Defaults to `http://localhost:7331`.
 
 ```bash
-python3 frontend-server.py [-h] [--host HOST] [--port PORT] [--api API]
+python3 frontend-server.py [-h] [--profile PROFILE] [--host HOST] [--port PORT] [--api API]
 ```
 
 ## Third-Party Licenses

api-server.py

@@ -4,6 +4,8 @@
 from argparse import ArgumentParser
 from os import environ
 
+from llama_cpp.server.app import create_app
+
 import uvicorn
 
 if __name__ == "__main__":
@@ -13,10 +15,7 @@ if __name__ == "__main__":
     ap.add_argument("--host", help="Address to listen on (default: localhost)", type=str, default="localhost")
     ap.add_argument("--port", help="Port to listen on (default: 7331)", type=int, default=7331)
     args = ap.parse_args()
-    # Set environment variable before importing api server
     environ["MODEL"] = args.model
-    # Import api server
-    from llama_cpp.server.app import create_app
     # Run
     app = create_app()
     uvicorn.run(app, host=args.host, port=args.port)

frontend-server.py

@@ -2,6 +2,8 @@
 # Copyright (c) 2023 Julian Müller (ChaoticByte)
 
 from argparse import ArgumentParser
+from json import load
+from pathlib import Path
 
 import uvicorn
 from frontend.app import app
@@ -9,11 +11,31 @@ from frontend.app import app
 if __name__ == "__main__":
     # CLI
     ap = ArgumentParser()
+    ap.add_argument("--profile", help="Path to a profile file that includes settings for a specific model", type=Path, required=True)
     ap.add_argument("--host", help="Address to listen on (default: localhost)", type=str, default="localhost")
     ap.add_argument("--port", help="Port to listen on (default: 8080)", type=int, default=8080)
     ap.add_argument("--api", help="URL of the API Server (default: 'http://localhost:7331')", type=str, default="http://localhost:7331")
     args = ap.parse_args()
+    # Read profile
+    with args.profile.open("r") as pf:
+        profile = load(pf)
+    # Check profile
+    assert "name" in profile
+    assert "conversation_prefix" in profile
+    assert "user_keyword" in profile
+    assert "assistant_keyword" in profile
+    assert "stop_sequences" in profile
     # Pass frontend config to the app
-    app.config.frontend_config = {"api_url": args.api.rstrip("/")}
+    app.config.frontend_config = {
+        "api_url": args.api.rstrip("/"),
+        "profile": {
+            "name": profile["name"],
+            "conversation_prefix": profile["conversation_prefix"],
+            "user_keyword": profile["user_keyword"],
+            "assistant_keyword": profile["assistant_keyword"],
+            "separator": profile["separator"],
+            "stop_sequences": profile["stop_sequences"]
+        }
+    }
     # Run
     uvicorn.run(app, host=args.host, port=args.port)

frontend/static/index.html

@@ -18,9 +18,20 @@
             </button>
         </div>
     </div>
-    <div class="sidepanel flex flex-column">
+    <div class="flex sidebar-container sidebar-hidden" id="sidebar-container">
+        <button id="sidebar-toggle-open" class="icon-button">
+            <svg xmlns="http://www.w3.org/2000/svg" height="48" viewBox="0 96 960 960" width="48"><path d="M433 712V440L297 576l136 136ZM180 936q-24.75 0-42.375-17.625T120 876V276q0-24.75 17.625-42.375T180 216h600q24.75 0 42.375 17.625T840 276v600q0 24.75-17.625 42.375T780 936H180Zm393-60V276H180v600h393Z"/></svg>
+        </button>
+        <button id="sidebar-toggle-close" class="icon-button hidden">
+            <svg xmlns="http://www.w3.org/2000/svg" height="48" viewBox="0 96 960 960" width="48"><path d="M297 440v272l136-136-136-136ZM180 936q-24.75 0-42.375-17.625T120 876V276q0-24.75 17.625-42.375T180 216h600q24.75 0 42.375 17.625T840 276v600q0 24.75-17.625 42.375T780 936H180Zm393-60V276H180v600h393Z"/></svg>
+        </button>
+        <div class="sidebar flex flex-column">
             <div class="max-width">Settings</div>
             <div class="settings flex flex-column">
+                <div class="setting flex">
+                    <div>Assistant</div>
+                    <div id="settings-label-assistant"></div>
+                </div>
                 <div class="setting flex">
                     <div>max_tokens</div>
                     <div><input type="number" id="settings-max-tokens" min="16" value="100"></div>
@@ -33,6 +44,22 @@
                     <div>top_p</div>
                     <div><input type="number" id="settings-top-p" min="0.0" max="1.0" step="0.01" value="0.95"></div>
                 </div>
+                <div class="setting flex">
+                    <div>top_k</div>
+                    <div><input type="number" id="settings-top-k" min="0" step="1" value="40"></div>
+                </div>
+                <div class="setting flex">
+                    <div>repeat_penalty</div>
+                    <div><input type="number" id="settings-repeat-penalty" min="0.0" step="0.01" value="1.1"></div>
+                </div>
+                <div class="setting flex">
+                    <div>presence_penalty</div>
+                    <div><input type="number" id="settings-presence-penalty" min="-2.0" max="2.0" step="0.01" value="0"></div>
+                </div>
+                <div class="setting flex">
+                    <div>frequency_penalty</div>
+                    <div><input type="number" id="settings-frequency-penalty" min="-2.0" max="2.0" step="0.01" value="0"></div>
+                </div>
             </div>
             <div class="flex">
                 <button id="reset-settings-btn" class="icon-button">
@@ -43,6 +70,7 @@
                 </button>
             </div>
         </div>
+    </div>
     <script src="/ui/main.js"></script>
 </body>
 </html>

frontend/static/main.js

@@ -1,47 +1,52 @@
 // Copyright (c) 2023 Julian Müller (ChaoticByte)
 
-(() => {
+const isMobile = /Android|BlackBerry|iPhone|iPod|Opera Mini/i.test(navigator.userAgent);
 
-    // Koala specific keywords
+// Fetch configuration and initialize Eucalyptus Chat Frontend
-    const conversationBeginning = "BEGINNING OF CONVERSATION:";
-    const userKeyword = " USER: ";
-    const assistantKeyword = " GPT:";
-    const koalaStopSequence = "</s>";
 
-    // Get frontend config
+fetch("/config")
-    let frontend_config = null;
+.then(r => {
-    fetch("/config")
-        .then(r => {
     return r.json();
-        })
+}).then(frontend_config => {
-        .then(j => {
-            frontend_config = j;
-        });
 
     // Message Context
-    let conversation = [conversationBeginning];
+    let conversation = [frontend_config.profile.conversation_prefix];
 
     // Elements - Sidebar
+    const sidebarOpenButton = document.getElementById("sidebar-toggle-open");
+    const sidebarCloseButton = document.getElementById("sidebar-toggle-close");
+    const sidebarContainer = document.getElementById("sidebar-container");
+    const settingsLabelAssistantNameElement = document.getElementById("settings-label-assistant");
     const settingsMaxTokensElement = document.getElementById("settings-max-tokens");
     const settingsTemperatureElement = document.getElementById("settings-temperature");
     const settingsTopPElement = document.getElementById("settings-top-p");
-    const resetSettingsButtonElement = document.getElementById("reset-settings-btn");
+    const settingsTopKElement = document.getElementById("settings-top-k");
-    const resetHistoryButtonElement = document.getElementById("reset-history-btn");
+    const settingsRepeatPenaltyElement = document.getElementById("settings-repeat-penalty");
+    const settingsPresencePenaltyElement = document.getElementById("settings-presence-penalty");
+    const settingsFrequencyPenaltyElement = document.getElementById("settings-frequency-penalty");
+    const resetSettingsButton = document.getElementById("reset-settings-btn");
+    const resetHistoryButton = document.getElementById("reset-history-btn");
+
+    settingsLabelAssistantNameElement.innerText = frontend_config.profile.name;
 
     // Elements - Main
     const messageHistoryContainer = document.getElementById("messages");
     const textInputElement = document.getElementById("text-input");
-    const sendButtonElement = document.getElementById("send-btn");
+    const sendButton = document.getElementById("send-btn");
 
     // API requests
 
     async function apiCompletion(prompt, settings) {
         const bodyData = JSON.stringify({
             "prompt": prompt,
-            "stop": [koalaStopSequence],
+            "stop": frontend_config.profile.stop_sequences,
             "max_tokens": settings.max_tokens,
             "temperature": settings.temperature,
-            "top_p": settings.top_p
+            "top_p": settings.top_p,
+            "top_k": settings.top_k,
+            "repeat_penalty": settings.repeat_penalty,
+            "presence_penalty": settings.presence_penalty,
+            "frequency_penalty": settings.frequency_penalty
         });
         const response = await fetch(frontend_config.api_url + "/v1/completions", {
             method: "post",
@@ -60,14 +65,22 @@
     const defaultSettings = {
         max_tokens: 100,
         temperature: 0.8,
-        top_p: 0.95
+        top_p: 0.95,
+        top_k: 40,
+        repeat_penalty: 1.1,
+        presence_penalty: 0.0,
+        frequency_penalty: 0.0
     }
 
     function getSettings() {
         return {
             max_tokens: settingsMaxTokensElement.value,
             temperature: settingsTemperatureElement.value,
-            top_p: settingsTopPElement.value
+            top_p: settingsTopPElement.value,
+            top_k: settingsTopKElement.value,
+            repeat_penalty: settingsRepeatPenaltyElement.value,
+            presence_penalty: settingsPresencePenaltyElement.value,
+            frequency_penalty: settingsFrequencyPenaltyElement.value
         }
     }
 
@@ -75,37 +88,44 @@
         settingsMaxTokensElement.value = defaultSettings.max_tokens;
         settingsTemperatureElement.value = defaultSettings.temperature;
         settingsTopPElement.value = defaultSettings.top_p;
+        settingsTopKElement.value = defaultSettings.top_k;
+        settingsRepeatPenaltyElement.value = defaultSettings.repeat_penalty;
+        settingsPresencePenaltyElement.value = defaultSettings.presence_penalty;
+        settingsFrequencyPenaltyElement.value = defaultSettings.frequency_penalty;
     }
 
     // Chat
 
-    const MessageType = {
+    // Message Roles
+    const Roles = {
         USER: {
             name: "User",
             class: "message-bg-user"
         },
         ASSISTANT: {
-            name: "Koala",
+            name: frontend_config.profile.name,
             class: "message-bg-assistant"
         }
     }
 
-    function addMessage(message, type) {
+    function addMessage(message, role) {
-        if (type == MessageType.USER) {
+        if (role == Roles.USER) {
-            conversation.push(userKeyword + message + assistantKeyword);
+            conversation.push(
+                frontend_config.profile.user_keyword + " "
+                + message + frontend_config.profile.separator + frontend_config.profile.assistant_keyword);
         }
-        else { conversation.push(message); }
+        else { conversation.push(message + frontend_config.profile.separator); }
         // UI
-        let messageTypeElem = document.createElement("div");
+        let messageRoleElem = document.createElement("div");
-        messageTypeElem.classList.add("message-type");
+        messageRoleElem.classList.add("message-type");
-        messageTypeElem.innerText = type.name;
+        messageRoleElem.innerText = role.name;
         let messageTextElem = document.createElement("div");
         messageTextElem.classList.add("message-text");
-        messageTextElem.innerText = message;
+        messageTextElem.innerText = message.trim();
         let messageElem = document.createElement("div");
         messageElem.classList.add("message");
-        messageElem.classList.add(type.class);
+        messageElem.classList.add(role.class);
-        messageElem.appendChild(messageTypeElem);
+        messageElem.appendChild(messageRoleElem);
         messageElem.appendChild(messageTextElem);
         messageHistoryContainer.appendChild(messageElem);
         messageHistoryContainer.scrollTo(0, messageHistoryContainer.scrollHeight);
@@ -122,9 +142,13 @@
         settingsMaxTokensElement.disabled = true;
         settingsTemperatureElement.disabled = true;
         settingsTopPElement.disabled = true;
-        resetSettingsButtonElement.disabled = true;
+        settingsTopKElement.disabled = true;
-        resetHistoryButtonElement.disabled = true;
+        settingsRepeatPenaltyElement.disabled = true;
-        sendButtonElement.disabled = true;
+        settingsPresencePenaltyElement.disabled = true;
+        settingsFrequencyPenaltyElement.disabled = true;
+        resetSettingsButton.disabled = true;
+        resetHistoryButton.disabled = true;
+        sendButton.disabled = true;
         textInputElement.disabled = true;
     }
 
@@ -132,19 +156,19 @@
         settingsMaxTokensElement.disabled = false;
         settingsTemperatureElement.disabled = false;
         settingsTopPElement.disabled = false;
-        resetSettingsButtonElement.disabled = false;
+        settingsTopKElement.disabled = false;
-        resetHistoryButtonElement.disabled = false;
+        settingsRepeatPenaltyElement.disabled = false;
-        sendButtonElement.disabled = false;
+        settingsPresencePenaltyElement.disabled = false;
+        settingsFrequencyPenaltyElement.disabled = false;
+        resetSettingsButton.disabled = false;
+        resetHistoryButton.disabled = false;
+        sendButton.disabled = false;
         textInputElement.disabled = false;
         // focus text input
-        textInputElement.focus();
+        if (!isMobile) textInputElement.focus();
     }
 
     async function chat() {
-        if (frontend_config == null) {
-            console.log("Couldn't fetch frontend configuration.");
-        }
-        else {
         disableInput();
         let input = textInputElement.value.trim();
         if (input == "") {
@@ -153,27 +177,44 @@
         else {
             textInputElement.value = "";
             resizeInputElement();
-                addMessage(input, MessageType.USER);
+            addMessage(input, Roles.USER);
             let prompt = conversation.join("");
             let settings = getSettings();
             apiCompletion(prompt, settings).then(r => {
-                    addMessage(r, MessageType.ASSISTANT);
+                addMessage(r.trim(), Roles.ASSISTANT);
                 enableInput();
             });
         }
     }
-    }
 
     function resetHistory() {
-        conversation = [conversationBeginning];
+        conversation = [frontend_config.profile.conversation_prefix];
         messageHistoryContainer.innerText = "";
     }
 
+    // Sidebar
+
+    function toggleSidebar() {
+        if (sidebarContainer.classList.contains("sidebar-hidden")) {
+            sidebarCloseButton.classList.remove("hidden");
+            sidebarOpenButton.classList.add("hidden");
+            sidebarContainer.classList.remove("sidebar-hidden");
+        }
+        else {
+            sidebarOpenButton.classList.remove("hidden");
+            sidebarCloseButton.classList.add("hidden");
+            sidebarContainer.classList.add("sidebar-hidden");
+        }
+    }
+
     // Event Listeners
 
-    resetSettingsButtonElement.addEventListener("click", resetSettings);
+    sidebarOpenButton.addEventListener("click", toggleSidebar);
-    resetHistoryButtonElement.addEventListener("click", resetHistory);
+    sidebarCloseButton.addEventListener("click", toggleSidebar);
-    sendButtonElement.addEventListener("click", chat);
+
+    resetSettingsButton.addEventListener("click", resetSettings);
+    resetHistoryButton.addEventListener("click", resetHistory);
+    sendButton.addEventListener("click", chat);
 
     textInputElement.addEventListener("keypress", e => {
         // Send via Ctrl+Enter
@@ -183,5 +224,6 @@
     });
 
     textInputElement.addEventListener("input", resizeInputElement);
+    resizeInputElement();
 
-})();
+});

frontend/static/style.css

@@ -5,12 +5,13 @@
     --background2: #303030;
     --background3: #161616;
     --background4: #131313;
+    --background5: #1a1a1a;
     --button-bg: #3b3b3b;
     --button-bg2: #4f4f4f;
     --icon-button-fill: #ffffff;
     --send-icon-button-fill: #29c76d;
     --color: #fafafa;
-    --padding: .5rem;
+    --color2: #bbbbbb;
     --border-radius: .5rem;
 }
 
@@ -21,19 +22,30 @@ body {
     color: var(--color);
     font-family: sans-serif;
     flex-direction: row;
-    min-height: 100vh;
+    height: 100vh;
 }
 
-
 input[type="number"] {
     width: 4rem;
 }
 
-.sidepanel {
+.sidebar-container {
-    gap: var(--padding);
+    padding: .5rem;
+    height: 100%;
+    background-color: var(--background5);
+    box-sizing: border-box;
+}
+
+.sidebar-container.sidebar-hidden > .sidebar {
+    display: none;
+}
+
+.sidebar {
+    margin-left: .5rem;
+    margin-right: .5rem;
+    margin-top: .4rem;
+    gap: .5rem;
     align-items: flex-end;
-    padding: 1rem;
-    padding-left: 0;
     min-width: fit-content;
 }
 
@@ -58,19 +70,38 @@ input[type="number"] {
 }
 
 .messages {
-    gap: 1.1rem;
+    gap: 1rem;
-    padding-bottom: var(--padding);
+    margin-bottom: 1rem;
     overflow-y: scroll;
     max-height: 89vh;
+    align-items: center;
+    flex-grow: 2;
 }
 
 .message {
     display: flex;
-    flex-direction: row;
+    flex-direction: column;
-    gap: var(--padding);
+    flex-wrap: wrap;
-    padding: var(--padding);
+    gap: 1rem;
+}
+
+.message-type {
+    color: var(--color2);
+    text-align: center;
+}
+
+.message-text {
+    white-space: pre-wrap;
+    padding: .5rem .8rem;
     border-radius: var(--border-radius);
-    max-width: fit-content;
+}
+
+.message-bg-assistant > .message-text {
+    background: var(--background2);
+}
+
+.message-bg-user > .message-text {
+    background: var(--background3);
 }
 
 button {
@@ -103,19 +134,6 @@ button:hover {
     width: 100%;
 }
 
-.message-bg-assistant {
-    background: var(--background2);
-}
-
-.message-bg-user {
-    background: var(--background3);
-}
-
-.message-type {
-    min-width: 3.5rem;
-    padding-left: .1rem;
-}
-
 .input-container {
     margin-top: auto;
     flex-direction: row;
@@ -137,6 +155,8 @@ button:hover {
 }
 
 .icon-button {
+    width: fit-content;
+    height: fit-content;
     padding: .2rem;
     display: flex;
     align-items: center;
@@ -149,7 +169,7 @@ button:hover {
 }
 
 .icon-button > svg {
-    height: 1.8rem;
+    height: 1.5rem;
     width: auto;
     fill: var(--icon-button-fill);
 }
@@ -162,3 +182,39 @@ button:hover {
     height: 2.2rem;
     fill: var(--send-icon-button-fill);
 }
+
+.hidden {
+    display: none;
+}
+
+@media only screen and (max-width: 600px) {
+
+    body {
+        height: 100dvh;
+    }
+
+    .sidebar-container {
+        position: absolute;
+        width: 100%;
+        height: 100%;
+        z-index: 1;
+    }
+
+    .sidebar-container.sidebar-hidden {
+        position: absolute;
+        width: unset;
+        height: unset;
+        left: auto;
+        right: 0;
+        background: transparent;
+    }
+
+    .sidebar-container > #sidebar-toggle-close {
+        margin-right: auto;
+    }
+
+    .sidebar {
+        margin-right: auto;
+        padding-right: 2.5rem;
+    }
+}

profiles/empty.json

@@ -0,0 +1,8 @@
+{
+    "name": "None",
+    "conversation_prefix": "",
+    "user_keyword": "",
+    "assistant_keyword": "",
+    "separator": "",
+    "stop_sequences": []
+}

profiles/koala.json

@@ -0,0 +1,8 @@
+{
+    "name": "Koala",
+    "conversation_prefix": "BEGINNING OF CONVERSATION: ",
+    "user_keyword": "USER:",
+    "assistant_keyword": "GPT:",
+    "separator": " ",
+    "stop_sequences": ["</s>"]
+}

profiles/manticore-chat.json

@@ -0,0 +1,8 @@
+{
+    "name": "Manticore",
+    "conversation_prefix": "",
+    "user_keyword": "USER:",
+    "assistant_keyword": "ASSISTANT:",
+    "separator": "\n",
+    "stop_sequences": ["</s>", "<unk>", "### USER:", "USER:"]
+}

profiles/vicuna-v0.json

@@ -0,0 +1,8 @@
+{
+    "name": "Vicuna v0",
+    "conversation_prefix": "A chat between a curious human and a helpful AI assistant.\n\n",
+    "user_keyword": "### Human:",
+    "assistant_keyword": "### Assistant:",
+    "separator": "\n",
+    "stop_sequences": ["### Human:"]
+}

profiles/vicuna-v1.1.json

@@ -0,0 +1,8 @@
+{
+    "name": "Vicuna v1.1",
+    "conversation_prefix": "A chat between a curious user and a helpful AI assistant.\n\n",
+    "user_keyword": "USER:",
+    "assistant_keyword": "ASSISTANT:",
+    "separator": "\n",
+    "stop_sequences": ["</s>"]
+}

requirements.txt

@@ -1,3 +1,3 @@
-llama-cpp-python[server]==0.1.48
+llama-cpp-python[server]==0.1.56
 uvicorn==0.22.0
 sanic==23.3.0