README.md

@@ -1,6 +1,6 @@
 # Eucalyptus Chat
 
-A frontend for large language models like [🐨 Koala](https://bair.berkeley.edu/blog/2023/04/03/koala/) or [🦙 Vicuna](https://lmsys.org/blog/2023-03-30-vicuna/) running on CPU with [llama.cpp](https://github.com/ggerganov/llama.cpp), using the API server library provided by [llama-cpp-python](https://github.com/abetlen/llama-cpp-python).
+A frontend for [Koala](https://bair.berkeley.edu/blog/2023/04/03/koala/) running on CPU with [llama.cpp](https://github.com/ggerganov/llama.cpp), using the API server library provided by [llama-cpp-python](https://github.com/abetlen/llama-cpp-python).
 
 ![](misc/screenshot.png)
 
@@ -8,33 +8,22 @@ A frontend for large language models like [🐨 Koala](https://bair.berkeley.edu
 
 - Python 3.10
 - The pip packages listed in `requirements.txt`
-- An AI model in the ggml format (should be quantized)
+- A Koala model in the ggml format (should be quantized)
 
-For memory and disk requirements for the different models, see [llama.cpp - Memory/Disk Requirements](https://github.com/ggerganov/llama.cpp#memorydisk-requirements)
+The 7B-Model, `q4_0`-quantized, requires approx. 5 GB of RAM.
-
-## Supported Models
-
-- [🐨 Koala](https://bair.berkeley.edu/blog/2023/04/03/koala/)
-- [🦙 Vicuna v.0](https://lmsys.org/blog/2023-03-30-vicuna/)
-- [🦙 Vicuna v.1.1](https://lmsys.org/blog/2023-03-30-vicuna/)
-- [🦁 Manticore Chat](https://huggingface.co/openaccess-ai-collective/manticore-13b-chat-pyg)
-
-(see `./profiles/`)
 
 ## Usage
 
 To use Eucalyptus locally, start both the API-Server (`api-server.py`) and the Frontend-Server (`frontend-server.py`).  
 The default URL of the Frontend-Server is http://localhost:8080.
 
-You have to choose the correct profile for the model you use. See [Supported Models](#supported-models) and [Frontend Server CLI Argument](#frontend-server-cli-arguments).
-
 ### API Server CLI Arguments
 
 The following command-line arguments are available:
 
 * `-m` or `--model`: Specifies the path to the model file. This is required and must be provided.
-* `--host`: Specifies the address to listen on. By default, it listens on `localhost`.
+* `--host`: Specifies the address to listen on. By default, it listens on localhost.
-* `--port`: Specifies the port number to listen on. The default value is `7331`.
+* `--port`: Specifies the port number to listen on. The default value is 7331.
 
 ```bash
 python3 api-server.py [-h] -m MODEL [--host HOST] [--port PORT]
@@ -44,13 +33,12 @@ python3 api-server.py [-h] -m MODEL [--host HOST] [--port PORT]
 
 The following command-line options are available:
 
-* `--profile`: Path to the profile file for the model.
+* `--host`: Specifies the IP address or hostname to listen on. Defaults to "localhost".
-* `--host`: Specifies the IP address or hostname to listen on. Defaults to `localhost`.
+* `--port`: Specifies the port number to listen on. Defaults to 8080.
-* `--port`: Specifies the port number to listen on. Defaults to `8080`.
+* `--api`: Specifies the URL of the API server. Defaults to http://localhost:7331.
-* `--api`: Specifies the URL of the API server. Defaults to `http://localhost:7331`.
 
 ```bash
-python3 frontend-server.py [-h] [--profile PROFILE] [--host HOST] [--port PORT] [--api API]
+python3 frontend-server.py [-h] [--host HOST] [--port PORT] [--api API]
 ```
 
 ## Third-Party Licenses

api-server.py

@@ -4,8 +4,6 @@
 from argparse import ArgumentParser
 from os import environ
 
-from llama_cpp.server.app import create_app
-
 import uvicorn
 
 if __name__ == "__main__":
@@ -15,7 +13,10 @@ if __name__ == "__main__":
     ap.add_argument("--host", help="Address to listen on (default: localhost)", type=str, default="localhost")
     ap.add_argument("--port", help="Port to listen on (default: 7331)", type=int, default=7331)
     args = ap.parse_args()
+    # Set environment variable before importing api server
     environ["MODEL"] = args.model
+    # Import api server
+    from llama_cpp.server.app import create_app
     # Run
     app = create_app()
     uvicorn.run(app, host=args.host, port=args.port)

frontend-server.py

@@ -2,8 +2,6 @@
 # Copyright (c) 2023 Julian Müller (ChaoticByte)
 
 from argparse import ArgumentParser
-from json import load
-from pathlib import Path
 
 import uvicorn
 from frontend.app import app
@@ -11,31 +9,11 @@ from frontend.app import app
 if __name__ == "__main__":
     # CLI
     ap = ArgumentParser()
-    ap.add_argument("--profile", help="Path to a profile file that includes settings for a specific model", type=Path, required=True)
     ap.add_argument("--host", help="Address to listen on (default: localhost)", type=str, default="localhost")
     ap.add_argument("--port", help="Port to listen on (default: 8080)", type=int, default=8080)
     ap.add_argument("--api", help="URL of the API Server (default: 'http://localhost:7331')", type=str, default="http://localhost:7331")
     args = ap.parse_args()
-    # Read profile
-    with args.profile.open("r") as pf:
-        profile = load(pf)
-    # Check profile
-    assert "name" in profile
-    assert "conversation_prefix" in profile
-    assert "user_keyword" in profile
-    assert "assistant_keyword" in profile
-    assert "stop_sequences" in profile
     # Pass frontend config to the app
-    app.config.frontend_config = {
+    app.config.frontend_config = {"api_url": args.api.rstrip("/")}
-        "api_url": args.api.rstrip("/"),
-        "profile": {
-            "name": profile["name"],
-            "conversation_prefix": profile["conversation_prefix"],
-            "user_keyword": profile["user_keyword"],
-            "assistant_keyword": profile["assistant_keyword"],
-            "separator": profile["separator"],
-            "stop_sequences": profile["stop_sequences"]
-        }
-    }
     # Run
     uvicorn.run(app, host=args.host, port=args.port)

frontend/static/index.html

@@ -18,20 +18,9 @@
             </button>
         </div>
     </div>
-    <div class="flex sidebar-container sidebar-hidden" id="sidebar-container">
+    <div class="sidepanel flex flex-column">
-        <button id="sidebar-toggle-open" class="icon-button">
-            <svg xmlns="http://www.w3.org/2000/svg" height="48" viewBox="0 96 960 960" width="48"><path d="M433 712V440L297 576l136 136ZM180 936q-24.75 0-42.375-17.625T120 876V276q0-24.75 17.625-42.375T180 216h600q24.75 0 42.375 17.625T840 276v600q0 24.75-17.625 42.375T780 936H180Zm393-60V276H180v600h393Z"/></svg>
-        </button>
-        <button id="sidebar-toggle-close" class="icon-button hidden">
-            <svg xmlns="http://www.w3.org/2000/svg" height="48" viewBox="0 96 960 960" width="48"><path d="M297 440v272l136-136-136-136ZM180 936q-24.75 0-42.375-17.625T120 876V276q0-24.75 17.625-42.375T180 216h600q24.75 0 42.375 17.625T840 276v600q0 24.75-17.625 42.375T780 936H180Zm393-60V276H180v600h393Z"/></svg>
-        </button>
-        <div class="sidebar flex flex-column">
         <div class="max-width">Settings</div>
         <div class="settings flex flex-column">
-                <div class="setting flex">
-                    <div>Assistant</div>
-                    <div id="settings-label-assistant"></div>
-                </div>
             <div class="setting flex">
                 <div>max_tokens</div>
                 <div><input type="number" id="settings-max-tokens" min="16" value="100"></div>
@@ -44,22 +33,6 @@
                 <div>top_p</div>
                 <div><input type="number" id="settings-top-p" min="0.0" max="1.0" step="0.01" value="0.95"></div>
             </div>    
-                <div class="setting flex">
-                    <div>top_k</div>
-                    <div><input type="number" id="settings-top-k" min="0" step="1" value="40"></div>
-                </div>
-                <div class="setting flex">
-                    <div>repeat_penalty</div>
-                    <div><input type="number" id="settings-repeat-penalty" min="0.0" step="0.01" value="1.1"></div>
-                </div>
-                <div class="setting flex">
-                    <div>presence_penalty</div>
-                    <div><input type="number" id="settings-presence-penalty" min="-2.0" max="2.0" step="0.01" value="0"></div>
-                </div>
-                <div class="setting flex">
-                    <div>frequency_penalty</div>
-                    <div><input type="number" id="settings-frequency-penalty" min="-2.0" max="2.0" step="0.01" value="0"></div>
-                </div>
         </div>
         <div class="flex">
             <button id="reset-settings-btn" class="icon-button">
@@ -70,7 +43,6 @@
             </button>
         </div>
     </div>
-    </div>
     <script src="/ui/main.js"></script>
 </body>
 </html>

frontend/static/main.js

@@ -1,52 +1,47 @@
 // Copyright (c) 2023 Julian Müller (ChaoticByte)
 
-const isMobile = /Android|BlackBerry|iPhone|iPod|Opera Mini/i.test(navigator.userAgent);
+(() => {
 
-// Fetch configuration and initialize Eucalyptus Chat Frontend
+    // Koala specific keywords
+    const conversationBeginning = "BEGINNING OF CONVERSATION:";
+    const userKeyword = " USER: ";
+    const assistantKeyword = " GPT:";
+    const koalaStopSequence = "</s>";
 
+    // Get frontend config
+    let frontend_config = null;
     fetch("/config")
         .then(r => {
             return r.json();
-}).then(frontend_config => {
+        })
+        .then(j => {
+            frontend_config = j;
+        });
 
     // Message Context
-    let conversation = [frontend_config.profile.conversation_prefix];
+    let conversation = [conversationBeginning];
 
     // Elements - Sidebar
-    const sidebarOpenButton = document.getElementById("sidebar-toggle-open");
-    const sidebarCloseButton = document.getElementById("sidebar-toggle-close");
-    const sidebarContainer = document.getElementById("sidebar-container");
-    const settingsLabelAssistantNameElement = document.getElementById("settings-label-assistant");
     const settingsMaxTokensElement = document.getElementById("settings-max-tokens");
     const settingsTemperatureElement = document.getElementById("settings-temperature");
     const settingsTopPElement = document.getElementById("settings-top-p");
-    const settingsTopKElement = document.getElementById("settings-top-k");
+    const resetSettingsButtonElement = document.getElementById("reset-settings-btn");
-    const settingsRepeatPenaltyElement = document.getElementById("settings-repeat-penalty");
+    const resetHistoryButtonElement = document.getElementById("reset-history-btn");
-    const settingsPresencePenaltyElement = document.getElementById("settings-presence-penalty");
-    const settingsFrequencyPenaltyElement = document.getElementById("settings-frequency-penalty");
-    const resetSettingsButton = document.getElementById("reset-settings-btn");
-    const resetHistoryButton = document.getElementById("reset-history-btn");
-
-    settingsLabelAssistantNameElement.innerText = frontend_config.profile.name;
 
     // Elements - Main
     const messageHistoryContainer = document.getElementById("messages");
     const textInputElement = document.getElementById("text-input");
-    const sendButton = document.getElementById("send-btn");
+    const sendButtonElement = document.getElementById("send-btn");
 
     // API requests
 
     async function apiCompletion(prompt, settings) {
         const bodyData = JSON.stringify({
             "prompt": prompt,
-            "stop": frontend_config.profile.stop_sequences,
+            "stop": [koalaStopSequence],
             "max_tokens": settings.max_tokens,
             "temperature": settings.temperature,
-            "top_p": settings.top_p,
+            "top_p": settings.top_p
-            "top_k": settings.top_k,
-            "repeat_penalty": settings.repeat_penalty,
-            "presence_penalty": settings.presence_penalty,
-            "frequency_penalty": settings.frequency_penalty
         });
         const response = await fetch(frontend_config.api_url + "/v1/completions", {
             method: "post",
@@ -65,22 +60,14 @@ fetch("/config")
     const defaultSettings = {
         max_tokens: 100,
         temperature: 0.8,
-        top_p: 0.95,
+        top_p: 0.95
-        top_k: 40,
-        repeat_penalty: 1.1,
-        presence_penalty: 0.0,
-        frequency_penalty: 0.0
     }
 
     function getSettings() {
         return {
             max_tokens: settingsMaxTokensElement.value,
             temperature: settingsTemperatureElement.value,
-            top_p: settingsTopPElement.value,
+            top_p: settingsTopPElement.value
-            top_k: settingsTopKElement.value,
-            repeat_penalty: settingsRepeatPenaltyElement.value,
-            presence_penalty: settingsPresencePenaltyElement.value,
-            frequency_penalty: settingsFrequencyPenaltyElement.value
         }
     }
 
@@ -88,44 +75,37 @@ fetch("/config")
         settingsMaxTokensElement.value = defaultSettings.max_tokens;
         settingsTemperatureElement.value = defaultSettings.temperature;
         settingsTopPElement.value = defaultSettings.top_p;
-        settingsTopKElement.value = defaultSettings.top_k;
-        settingsRepeatPenaltyElement.value = defaultSettings.repeat_penalty;
-        settingsPresencePenaltyElement.value = defaultSettings.presence_penalty;
-        settingsFrequencyPenaltyElement.value = defaultSettings.frequency_penalty;
     }
 
     // Chat
 
-    // Message Roles
+    const MessageType = {
-    const Roles = {
         USER: {
             name: "User",
             class: "message-bg-user"
         },
         ASSISTANT: {
-            name: frontend_config.profile.name,
+            name: "Koala",
             class: "message-bg-assistant"
         }
     }
 
-    function addMessage(message, role) {
+    function addMessage(message, type) {
-        if (role == Roles.USER) {
+        if (type == MessageType.USER) {
-            conversation.push(
+            conversation.push(userKeyword + message + assistantKeyword);
-                frontend_config.profile.user_keyword + " "
-                + message + frontend_config.profile.separator + frontend_config.profile.assistant_keyword);
         }
-        else { conversation.push(message + frontend_config.profile.separator); }
+        else { conversation.push(message); }
         // UI
-        let messageRoleElem = document.createElement("div");
+        let messageTypeElem = document.createElement("div");
-        messageRoleElem.classList.add("message-type");
+        messageTypeElem.classList.add("message-type");
-        messageRoleElem.innerText = role.name;
+        messageTypeElem.innerText = type.name;
         let messageTextElem = document.createElement("div");
         messageTextElem.classList.add("message-text");
-        messageTextElem.innerText = message.trim();
+        messageTextElem.innerText = message;
         let messageElem = document.createElement("div");
         messageElem.classList.add("message");
-        messageElem.classList.add(role.class);
+        messageElem.classList.add(type.class);
-        messageElem.appendChild(messageRoleElem);
+        messageElem.appendChild(messageTypeElem);
         messageElem.appendChild(messageTextElem);
         messageHistoryContainer.appendChild(messageElem);
         messageHistoryContainer.scrollTo(0, messageHistoryContainer.scrollHeight);
@@ -142,13 +122,9 @@ fetch("/config")
         settingsMaxTokensElement.disabled = true;
         settingsTemperatureElement.disabled = true;
         settingsTopPElement.disabled = true;
-        settingsTopKElement.disabled = true;
+        resetSettingsButtonElement.disabled = true;
-        settingsRepeatPenaltyElement.disabled = true;
+        resetHistoryButtonElement.disabled = true;
-        settingsPresencePenaltyElement.disabled = true;
+        sendButtonElement.disabled = true;
-        settingsFrequencyPenaltyElement.disabled = true;
-        resetSettingsButton.disabled = true;
-        resetHistoryButton.disabled = true;
-        sendButton.disabled = true;
         textInputElement.disabled = true;
     }
 
@@ -156,19 +132,19 @@ fetch("/config")
         settingsMaxTokensElement.disabled = false;
         settingsTemperatureElement.disabled = false;
         settingsTopPElement.disabled = false;
-        settingsTopKElement.disabled = false;
+        resetSettingsButtonElement.disabled = false;
-        settingsRepeatPenaltyElement.disabled = false;
+        resetHistoryButtonElement.disabled = false;
-        settingsPresencePenaltyElement.disabled = false;
+        sendButtonElement.disabled = false;
-        settingsFrequencyPenaltyElement.disabled = false;
-        resetSettingsButton.disabled = false;
-        resetHistoryButton.disabled = false;
-        sendButton.disabled = false;
         textInputElement.disabled = false;
         // focus text input
-        if (!isMobile) textInputElement.focus();
+        textInputElement.focus();
     }
 
     async function chat() {
+        if (frontend_config == null) {
+            console.log("Couldn't fetch frontend configuration.");
+        }
+        else {
             disableInput();
             let input = textInputElement.value.trim();
             if (input == "") {
@@ -177,44 +153,27 @@ fetch("/config")
             else {
                 textInputElement.value = "";
                 resizeInputElement();
-            addMessage(input, Roles.USER);
+                addMessage(input, MessageType.USER);
                 let prompt = conversation.join("");
                 let settings = getSettings();
                 apiCompletion(prompt, settings).then(r => {
-                addMessage(r.trim(), Roles.ASSISTANT);
+                    addMessage(r, MessageType.ASSISTANT);
                     enableInput();
                 });
             }
         }
+    }
 
     function resetHistory() {
-        conversation = [frontend_config.profile.conversation_prefix];
+        conversation = [conversationBeginning];
         messageHistoryContainer.innerText = "";
     }
 
-    // Sidebar
-
-    function toggleSidebar() {
-        if (sidebarContainer.classList.contains("sidebar-hidden")) {
-            sidebarCloseButton.classList.remove("hidden");
-            sidebarOpenButton.classList.add("hidden");
-            sidebarContainer.classList.remove("sidebar-hidden");
-        }
-        else {
-            sidebarOpenButton.classList.remove("hidden");
-            sidebarCloseButton.classList.add("hidden");
-            sidebarContainer.classList.add("sidebar-hidden");
-        }
-    }
-
     // Event Listeners
 
-    sidebarOpenButton.addEventListener("click", toggleSidebar);
+    resetSettingsButtonElement.addEventListener("click", resetSettings);
-    sidebarCloseButton.addEventListener("click", toggleSidebar);
+    resetHistoryButtonElement.addEventListener("click", resetHistory);
-
+    sendButtonElement.addEventListener("click", chat);
-    resetSettingsButton.addEventListener("click", resetSettings);
-    resetHistoryButton.addEventListener("click", resetHistory);
-    sendButton.addEventListener("click", chat);
 
     textInputElement.addEventListener("keypress", e => {
         // Send via Ctrl+Enter
@@ -226,4 +185,4 @@ fetch("/config")
     textInputElement.addEventListener("input", resizeInputElement);
     resizeInputElement();
 
-});
+})();

frontend/static/style.css

@@ -5,13 +5,11 @@
     --background2: #303030;
     --background3: #161616;
     --background4: #131313;
-    --background5: #1a1a1a;
     --button-bg: #3b3b3b;
     --button-bg2: #4f4f4f;
     --icon-button-fill: #ffffff;
     --send-icon-button-fill: #29c76d;
     --color: #fafafa;
-    --color2: #bbbbbb;
     --border-radius: .5rem;
 }
 
@@ -29,23 +27,11 @@ input[type="number"] {
     width: 4rem;
 }
 
-.sidebar-container {
+.sidepanel {
-    padding: .5rem;
-    height: 100%;
-    background-color: var(--background5);
-    box-sizing: border-box;
-}
-
-.sidebar-container.sidebar-hidden > .sidebar {
-    display: none;
-}
-
-.sidebar {
-    margin-left: .5rem;
-    margin-right: .5rem;
-    margin-top: .4rem;
     gap: .5rem;
     align-items: flex-end;
+    padding: 1rem;
+    padding-left: 0;
     min-width: fit-content;
 }
 
@@ -70,38 +56,19 @@ input[type="number"] {
 }
 
 .messages {
-    gap: 1rem;
+    gap: 1.1rem;
     margin-bottom: 1rem;
     overflow-y: scroll;
     max-height: 89vh;
-    align-items: center;
-    flex-grow: 2;
 }
 
 .message {
     display: flex;
-    flex-direction: column;
+    flex-direction: row;
-    flex-wrap: wrap;
+    gap: .5rem;
-    gap: 1rem;
+    padding: .5rem;
-}
-
-.message-type {
-    color: var(--color2);
-    text-align: center;
-}
-
-.message-text {
-    white-space: pre-wrap;
-    padding: .5rem .8rem;
     border-radius: var(--border-radius);
-}
+    max-width: fit-content;
-
-.message-bg-assistant > .message-text {
-    background: var(--background2);
-}
-
-.message-bg-user > .message-text {
-    background: var(--background3);
 }
 
 button {
@@ -134,6 +101,19 @@ button:hover {
     width: 100%;
 }
 
+.message-bg-assistant {
+    background: var(--background2);
+}
+
+.message-bg-user {
+    background: var(--background3);
+}
+
+.message-type {
+    min-width: 3.5rem;
+    padding-left: .1rem;
+}
+
 .input-container {
     margin-top: auto;
     flex-direction: row;
@@ -155,8 +135,6 @@ button:hover {
 }
 
 .icon-button {
-    width: fit-content;
-    height: fit-content;
     padding: .2rem;
     display: flex;
     align-items: center;
@@ -182,39 +160,3 @@ button:hover {
     height: 2.2rem;
     fill: var(--send-icon-button-fill);
 }
-
-.hidden {
-    display: none;
-}
-
-@media only screen and (max-width: 600px) {
-
-    body {
-        height: 100dvh;
-    }
-
-    .sidebar-container {
-        position: absolute;
-        width: 100%;
-        height: 100%;
-        z-index: 1;
-    }
-
-    .sidebar-container.sidebar-hidden {
-        position: absolute;
-        width: unset;
-        height: unset;
-        left: auto;
-        right: 0;
-        background: transparent;
-    }
-
-    .sidebar-container > #sidebar-toggle-close {
-        margin-right: auto;
-    }
-
-    .sidebar {
-        margin-right: auto;
-        padding-right: 2.5rem;
-    }
-}

profiles/empty.json

@@ -1,8 +0,0 @@
-{
-    "name": "None",
-    "conversation_prefix": "",
-    "user_keyword": "",
-    "assistant_keyword": "",
-    "separator": "",
-    "stop_sequences": []
-}

profiles/koala.json

@@ -1,8 +0,0 @@
-{
-    "name": "Koala",
-    "conversation_prefix": "BEGINNING OF CONVERSATION: ",
-    "user_keyword": "USER:",
-    "assistant_keyword": "GPT:",
-    "separator": " ",
-    "stop_sequences": ["</s>"]
-}

profiles/manticore-chat.json

@@ -1,8 +0,0 @@
-{
-    "name": "Manticore",
-    "conversation_prefix": "",
-    "user_keyword": "USER:",
-    "assistant_keyword": "ASSISTANT:",
-    "separator": "\n",
-    "stop_sequences": ["</s>", "<unk>", "### USER:", "USER:"]
-}

profiles/vicuna-v0.json

@@ -1,8 +0,0 @@
-{
-    "name": "Vicuna v0",
-    "conversation_prefix": "A chat between a curious human and a helpful AI assistant.\n\n",
-    "user_keyword": "### Human:",
-    "assistant_keyword": "### Assistant:",
-    "separator": "\n",
-    "stop_sequences": ["### Human:"]
-}

profiles/vicuna-v1.1.json

@@ -1,8 +0,0 @@
-{
-    "name": "Vicuna v1.1",
-    "conversation_prefix": "A chat between a curious user and a helpful AI assistant.\n\n",
-    "user_keyword": "USER:",
-    "assistant_keyword": "ASSISTANT:",
-    "separator": "\n",
-    "stop_sequences": ["</s>"]
-}

requirements.txt

@@ -1,3 +1,3 @@
-llama-cpp-python[server]==0.1.56
+llama-cpp-python[server]==0.1.48
 uvicorn==0.22.0
 sanic==23.3.0