README.md

@@ -8,18 +8,16 @@ A frontend for large language models like [🐨 Koala](https://bair.berkeley.edu
 
 - Python 3.10
 - The pip packages listed in `requirements.txt`
-- An AI model in the ggml format (should be quantized)
+- A Koala model in the ggml format (should be quantized)
 
-For memory and disk requirements for the different models, see [llama.cpp - Memory/Disk Requirements](https://github.com/ggerganov/llama.cpp#memorydisk-requirements)
+The 7B-Model of Koala, `q4_0`-quantized, requires approx. 5 GB of RAM.
 
 ## Supported Models
 
 - [🐨 Koala](https://bair.berkeley.edu/blog/2023/04/03/koala/)
-- [🦙 Vicuna v.0](https://lmsys.org/blog/2023-03-30-vicuna/)
+- [🦙 Vicuna](https://lmsys.org/blog/2023-03-30-vicuna/)
-- [🦙 Vicuna v.1.1](https://lmsys.org/blog/2023-03-30-vicuna/)
-- [🦁 Manticore Chat](https://huggingface.co/openaccess-ai-collective/manticore-13b-chat-pyg)
 
-(see `./profiles/`)
+see `./profiles/`
 
 ## Usage
 
@@ -44,7 +42,7 @@ python3 api-server.py [-h] -m MODEL [--host HOST] [--port PORT]
 
 The following command-line options are available:
 
-* `--profile`: Path to the profile file for the model.
+* `--profile`: Path to the profile file for the model. Defaults to `./profiles/koala.json`.
 * `--host`: Specifies the IP address or hostname to listen on. Defaults to `localhost`.
 * `--port`: Specifies the port number to listen on. Defaults to `8080`.
 * `--api`: Specifies the URL of the API server. Defaults to `http://localhost:7331`.

api-server.py

@@ -4,8 +4,6 @@
 from argparse import ArgumentParser
 from os import environ
 
-from llama_cpp.server.app import create_app
-
 import uvicorn
 
 if __name__ == "__main__":
@@ -15,7 +13,10 @@ if __name__ == "__main__":
     ap.add_argument("--host", help="Address to listen on (default: localhost)", type=str, default="localhost")
     ap.add_argument("--port", help="Port to listen on (default: 7331)", type=int, default=7331)
     args = ap.parse_args()
+    # Set environment variable before importing api server
     environ["MODEL"] = args.model
+    # Import api server
+    from llama_cpp.server.app import create_app
     # Run
     app = create_app()
     uvicorn.run(app, host=args.host, port=args.port)

frontend-server.py

@@ -9,9 +9,10 @@ import uvicorn
 from frontend.app import app
 
 if __name__ == "__main__":
+    koala_profile_path = Path(__file__).parent / "profiles" / "koala.json"
     # CLI
     ap = ArgumentParser()
-    ap.add_argument("--profile", help="Path to a profile file that includes settings for a specific model", type=Path, required=True)
+    ap.add_argument("--profile", help="Path to a profile file that includes settings for a specific model (default: ./profiles/koala.json)", type=Path, default=koala_profile_path)
     ap.add_argument("--host", help="Address to listen on (default: localhost)", type=str, default="localhost")
     ap.add_argument("--port", help="Port to listen on (default: 8080)", type=int, default=8080)
     ap.add_argument("--api", help="URL of the API Server (default: 'http://localhost:7331')", type=str, default="http://localhost:7331")
@@ -33,7 +34,6 @@ if __name__ == "__main__":
             "conversation_prefix": profile["conversation_prefix"],
             "user_keyword": profile["user_keyword"],
             "assistant_keyword": profile["assistant_keyword"],
-            "separator": profile["separator"],
             "stop_sequences": profile["stop_sequences"]
         }
     }

frontend/static/index.html

@@ -18,14 +18,7 @@
             </button>
         </div>
     </div>
-    <div class="flex sidebar-container sidebar-hidden" id="sidebar-container">
+    <div class="sidepanel flex flex-column">
-        <button id="sidebar-toggle-open" class="icon-button">
-            <svg xmlns="http://www.w3.org/2000/svg" height="48" viewBox="0 96 960 960" width="48"><path d="M433 712V440L297 576l136 136ZM180 936q-24.75 0-42.375-17.625T120 876V276q0-24.75 17.625-42.375T180 216h600q24.75 0 42.375 17.625T840 276v600q0 24.75-17.625 42.375T780 936H180Zm393-60V276H180v600h393Z"/></svg>
-        </button>
-        <button id="sidebar-toggle-close" class="icon-button hidden">
-            <svg xmlns="http://www.w3.org/2000/svg" height="48" viewBox="0 96 960 960" width="48"><path d="M297 440v272l136-136-136-136ZM180 936q-24.75 0-42.375-17.625T120 876V276q0-24.75 17.625-42.375T180 216h600q24.75 0 42.375 17.625T840 276v600q0 24.75-17.625 42.375T780 936H180Zm393-60V276H180v600h393Z"/></svg>
-        </button>
-        <div class="sidebar flex flex-column">
         <div class="max-width">Settings</div>
         <div class="settings flex flex-column">
             <div class="setting flex">
@@ -70,7 +63,6 @@
             </button>
         </div>
     </div>
-    </div>
     <script src="/ui/main.js"></script>
 </body>
 </html>

frontend/static/main.js

@@ -1,7 +1,5 @@
 // Copyright (c) 2023 Julian Müller (ChaoticByte)
 
-const isMobile = /Android|BlackBerry|iPhone|iPod|Opera Mini/i.test(navigator.userAgent);
-
 // Fetch configuration and initialize Eucalyptus Chat Frontend
 
 fetch("/config")
@@ -13,9 +11,6 @@ fetch("/config")
     let conversation = [frontend_config.profile.conversation_prefix];
 
     // Elements - Sidebar
-    const sidebarOpenButton = document.getElementById("sidebar-toggle-open");
-    const sidebarCloseButton = document.getElementById("sidebar-toggle-close");
-    const sidebarContainer = document.getElementById("sidebar-container");
     const settingsLabelAssistantNameElement = document.getElementById("settings-label-assistant");
     const settingsMaxTokensElement = document.getElementById("settings-max-tokens");
     const settingsTemperatureElement = document.getElementById("settings-temperature");
@@ -24,15 +19,15 @@ fetch("/config")
     const settingsRepeatPenaltyElement = document.getElementById("settings-repeat-penalty");
     const settingsPresencePenaltyElement = document.getElementById("settings-presence-penalty");
     const settingsFrequencyPenaltyElement = document.getElementById("settings-frequency-penalty");
-    const resetSettingsButton = document.getElementById("reset-settings-btn");
+    const resetSettingsButtonElement = document.getElementById("reset-settings-btn");
-    const resetHistoryButton = document.getElementById("reset-history-btn");
+    const resetHistoryButtonElement = document.getElementById("reset-history-btn");
 
     settingsLabelAssistantNameElement.innerText = frontend_config.profile.name;
 
     // Elements - Main
     const messageHistoryContainer = document.getElementById("messages");
     const textInputElement = document.getElementById("text-input");
-    const sendButton = document.getElementById("send-btn");
+    const sendButtonElement = document.getElementById("send-btn");
 
     // API requests
 
@@ -111,17 +106,17 @@ fetch("/config")
     function addMessage(message, role) {
         if (role == Roles.USER) {
             conversation.push(
-                frontend_config.profile.user_keyword + " "
+                " " + frontend_config.profile.user_keyword + " "
-                + message + frontend_config.profile.separator + frontend_config.profile.assistant_keyword);
+                + message + " " + frontend_config.profile.assistant_keyword);
         }
-        else { conversation.push(message + frontend_config.profile.separator); }
+        else { conversation.push(message); }
         // UI
         let messageRoleElem = document.createElement("div");
         messageRoleElem.classList.add("message-type");
         messageRoleElem.innerText = role.name;
         let messageTextElem = document.createElement("div");
         messageTextElem.classList.add("message-text");
-        messageTextElem.innerText = message.trim();
+        messageTextElem.innerText = message;
         let messageElem = document.createElement("div");
         messageElem.classList.add("message");
         messageElem.classList.add(role.class);
@@ -146,9 +141,9 @@ fetch("/config")
         settingsRepeatPenaltyElement.disabled = true;
         settingsPresencePenaltyElement.disabled = true;
         settingsFrequencyPenaltyElement.disabled = true;
-        resetSettingsButton.disabled = true;
+        resetSettingsButtonElement.disabled = true;
-        resetHistoryButton.disabled = true;
+        resetHistoryButtonElement.disabled = true;
-        sendButton.disabled = true;
+        sendButtonElement.disabled = true;
         textInputElement.disabled = true;
     }
 
@@ -160,12 +155,12 @@ fetch("/config")
         settingsRepeatPenaltyElement.disabled = false;
         settingsPresencePenaltyElement.disabled = false;
         settingsFrequencyPenaltyElement.disabled = false;
-        resetSettingsButton.disabled = false;
+        resetSettingsButtonElement.disabled = false;
-        resetHistoryButton.disabled = false;
+        resetHistoryButtonElement.disabled = false;
-        sendButton.disabled = false;
+        sendButtonElement.disabled = false;
         textInputElement.disabled = false;
         // focus text input
-        if (!isMobile) textInputElement.focus();
+        textInputElement.focus();
     }
 
     async function chat() {
@@ -181,7 +176,7 @@ fetch("/config")
             let prompt = conversation.join("");
             let settings = getSettings();
             apiCompletion(prompt, settings).then(r => {
-                addMessage(r.trim(), Roles.ASSISTANT);
+                addMessage(r, Roles.ASSISTANT);
                 enableInput();
             });
         }
@@ -192,29 +187,11 @@ fetch("/config")
         messageHistoryContainer.innerText = "";
     }
 
-    // Sidebar
-
-    function toggleSidebar() {
-        if (sidebarContainer.classList.contains("sidebar-hidden")) {
-            sidebarCloseButton.classList.remove("hidden");
-            sidebarOpenButton.classList.add("hidden");
-            sidebarContainer.classList.remove("sidebar-hidden");
-        }
-        else {
-            sidebarOpenButton.classList.remove("hidden");
-            sidebarCloseButton.classList.add("hidden");
-            sidebarContainer.classList.add("sidebar-hidden");
-        }
-    }
-
     // Event Listeners
 
-    sidebarOpenButton.addEventListener("click", toggleSidebar);
+    resetSettingsButtonElement.addEventListener("click", resetSettings);
-    sidebarCloseButton.addEventListener("click", toggleSidebar);
+    resetHistoryButtonElement.addEventListener("click", resetHistory);
-
+    sendButtonElement.addEventListener("click", chat);
-    resetSettingsButton.addEventListener("click", resetSettings);
-    resetHistoryButton.addEventListener("click", resetHistory);
-    sendButton.addEventListener("click", chat);
 
     textInputElement.addEventListener("keypress", e => {
         // Send via Ctrl+Enter

frontend/static/style.css

@@ -5,13 +5,11 @@
     --background2: #303030;
     --background3: #161616;
     --background4: #131313;
-    --background5: #1a1a1a;
     --button-bg: #3b3b3b;
     --button-bg2: #4f4f4f;
     --icon-button-fill: #ffffff;
     --send-icon-button-fill: #29c76d;
     --color: #fafafa;
-    --color2: #bbbbbb;
     --border-radius: .5rem;
 }
 
@@ -29,23 +27,11 @@ input[type="number"] {
     width: 4rem;
 }
 
-.sidebar-container {
+.sidepanel {
-    padding: .5rem;
-    height: 100%;
-    background-color: var(--background5);
-    box-sizing: border-box;
-}
-
-.sidebar-container.sidebar-hidden > .sidebar {
-    display: none;
-}
-
-.sidebar {
-    margin-left: .5rem;
-    margin-right: .5rem;
-    margin-top: .4rem;
     gap: .5rem;
     align-items: flex-end;
+    padding: 1rem;
+    padding-left: 0;
     min-width: fit-content;
 }
 
@@ -70,38 +56,19 @@ input[type="number"] {
 }
 
 .messages {
-    gap: 1rem;
+    gap: 1.1rem;
     margin-bottom: 1rem;
     overflow-y: scroll;
     max-height: 89vh;
-    align-items: center;
-    flex-grow: 2;
 }
 
 .message {
     display: flex;
-    flex-direction: column;
+    flex-direction: row;
-    flex-wrap: wrap;
+    gap: .5rem;
-    gap: 1rem;
+    padding: .5rem;
-}
-
-.message-type {
-    color: var(--color2);
-    text-align: center;
-}
-
-.message-text {
-    white-space: pre-wrap;
-    padding: .5rem .8rem;
     border-radius: var(--border-radius);
-}
+    max-width: fit-content;
-
-.message-bg-assistant > .message-text {
-    background: var(--background2);
-}
-
-.message-bg-user > .message-text {
-    background: var(--background3);
 }
 
 button {
@@ -134,6 +101,19 @@ button:hover {
     width: 100%;
 }
 
+.message-bg-assistant {
+    background: var(--background2);
+}
+
+.message-bg-user {
+    background: var(--background3);
+}
+
+.message-type {
+    min-width: 3.5rem;
+    padding-left: .1rem;
+}
+
 .input-container {
     margin-top: auto;
     flex-direction: row;
@@ -155,8 +135,6 @@ button:hover {
 }
 
 .icon-button {
-    width: fit-content;
-    height: fit-content;
     padding: .2rem;
     display: flex;
     align-items: center;
@@ -182,39 +160,3 @@ button:hover {
     height: 2.2rem;
     fill: var(--send-icon-button-fill);
 }
-
-.hidden {
-    display: none;
-}
-
-@media only screen and (max-width: 600px) {
-
-    body {
-        height: 100dvh;
-    }
-
-    .sidebar-container {
-        position: absolute;
-        width: 100%;
-        height: 100%;
-        z-index: 1;
-    }
-
-    .sidebar-container.sidebar-hidden {
-        position: absolute;
-        width: unset;
-        height: unset;
-        left: auto;
-        right: 0;
-        background: transparent;
-    }
-
-    .sidebar-container > #sidebar-toggle-close {
-        margin-right: auto;
-    }
-
-    .sidebar {
-        margin-right: auto;
-        padding-right: 2.5rem;
-    }
-}

profiles/empty.json

@@ -1,8 +0,0 @@
-{
-    "name": "None",
-    "conversation_prefix": "",
-    "user_keyword": "",
-    "assistant_keyword": "",
-    "separator": "",
-    "stop_sequences": []
-}

profiles/koala.json

@@ -3,6 +3,5 @@
     "conversation_prefix": "BEGINNING OF CONVERSATION: ",
     "user_keyword": "USER:",
     "assistant_keyword": "GPT:",
-    "separator": " ",
     "stop_sequences": ["</s>"]
 }

profiles/manticore-chat.json

@@ -1,8 +0,0 @@
-{
-    "name": "Manticore",
-    "conversation_prefix": "",
-    "user_keyword": "USER:",
-    "assistant_keyword": "ASSISTANT:",
-    "separator": "\n",
-    "stop_sequences": ["</s>", "<unk>", "### USER:", "USER:"]
-}

profiles/vicuna-v0.json

@@ -1,8 +0,0 @@
-{
-    "name": "Vicuna v0",
-    "conversation_prefix": "A chat between a curious human and a helpful AI assistant.\n\n",
-    "user_keyword": "### Human:",
-    "assistant_keyword": "### Assistant:",
-    "separator": "\n",
-    "stop_sequences": ["### Human:"]
-}

profiles/vicuna-v1.1.json

@@ -1,8 +0,0 @@
-{
-    "name": "Vicuna v1.1",
-    "conversation_prefix": "A chat between a curious user and a helpful AI assistant.\n\n",
-    "user_keyword": "USER:",
-    "assistant_keyword": "ASSISTANT:",
-    "separator": "\n",
-    "stop_sequences": ["</s>"]
-}

profiles/vicuna.json

@@ -0,0 +1,7 @@
+{
+    "name": "Vicuna",
+    "conversation_prefix": "A chat between a curious user and a helpful AI assistant. ",
+    "user_keyword": "### Human:",
+    "assistant_keyword": "### Assistant:",
+    "stop_sequences": ["### Human:"]
+}

requirements.txt

@@ -1,3 +1,3 @@
-llama-cpp-python[server]==0.1.56
+llama-cpp-python[server]==0.1.50
 uvicorn==0.22.0
 sanic==23.3.0