Fixed a small typo in the README

Bump llama-cpp-python[server] from 0.1.54 to 0.1.56

README.md

@@ -8,16 +8,18 @@ A frontend for large language models like [🐨 Koala](https://bair.berkeley.edu
 
 - Python 3.10
 - The pip packages listed in `requirements.txt`
-- A Koala model in the ggml format (should be quantized)
+- An AI model in the ggml format (should be quantized)
 
-The 7B-Model of Koala, `q4_0`-quantized, requires approx. 5 GB of RAM.
+For memory and disk requirements for the different models, see [llama.cpp - Memory/Disk Requirements](https://github.com/ggerganov/llama.cpp#memorydisk-requirements)
 
 ## Supported Models
 
 - [🐨 Koala](https://bair.berkeley.edu/blog/2023/04/03/koala/)
-- [🦙 Vicuna](https://lmsys.org/blog/2023-03-30-vicuna/)
+- [🦙 Vicuna v.0](https://lmsys.org/blog/2023-03-30-vicuna/)
+- [🦙 Vicuna v.1.1](https://lmsys.org/blog/2023-03-30-vicuna/)
+- [🦁 Manticore Chat](https://huggingface.co/openaccess-ai-collective/manticore-13b-chat-pyg)
 
-see `./profiles/`
+(see `./profiles/`)
 
 ## Usage
 
@@ -42,7 +44,7 @@ python3 api-server.py [-h] -m MODEL [--host HOST] [--port PORT]
 
 The following command-line options are available:
 
-* `--profile`: Path to the profile file for the model. Defaults to `./profiles/koala.json`.
+* `--profile`: Path to the profile file for the model.
 * `--host`: Specifies the IP address or hostname to listen on. Defaults to `localhost`.
 * `--port`: Specifies the port number to listen on. Defaults to `8080`.
 * `--api`: Specifies the URL of the API server. Defaults to `http://localhost:7331`.

api-server.py

@@ -4,6 +4,8 @@
 from argparse import ArgumentParser
 from os import environ
 
+from llama_cpp.server.app import create_app
+
 import uvicorn
 
 if __name__ == "__main__":
@@ -13,10 +15,7 @@ if __name__ == "__main__":
     ap.add_argument("--host", help="Address to listen on (default: localhost)", type=str, default="localhost")
     ap.add_argument("--port", help="Port to listen on (default: 7331)", type=int, default=7331)
     args = ap.parse_args()
-    # Set environment variable before importing api server
     environ["MODEL"] = args.model
-    # Import api server
-    from llama_cpp.server.app import create_app
     # Run
     app = create_app()
     uvicorn.run(app, host=args.host, port=args.port)

frontend-server.py

@@ -9,10 +9,9 @@ import uvicorn
 from frontend.app import app
 
 if __name__ == "__main__":
-    koala_profile_path = Path(__file__).parent / "profiles" / "koala.json"
     # CLI
     ap = ArgumentParser()
-    ap.add_argument("--profile", help="Path to a profile file that includes settings for a specific model (default: ./profiles/koala.json)", type=Path, default=koala_profile_path)
+    ap.add_argument("--profile", help="Path to a profile file that includes settings for a specific model", type=Path, required=True)
     ap.add_argument("--host", help="Address to listen on (default: localhost)", type=str, default="localhost")
     ap.add_argument("--port", help="Port to listen on (default: 8080)", type=int, default=8080)
     ap.add_argument("--api", help="URL of the API Server (default: 'http://localhost:7331')", type=str, default="http://localhost:7331")
@@ -34,6 +33,7 @@ if __name__ == "__main__":
             "conversation_prefix": profile["conversation_prefix"],
             "user_keyword": profile["user_keyword"],
             "assistant_keyword": profile["assistant_keyword"],
+            "separator": profile["separator"],
             "stop_sequences": profile["stop_sequences"]
         }
     }

frontend/static/index.html

@@ -18,7 +18,14 @@
             </button>
         </div>
     </div>
-    <div class="sidepanel flex flex-column">
+    <div class="flex sidebar-container sidebar-hidden" id="sidebar-container">
+        <button id="sidebar-toggle-open" class="icon-button">
+            <svg xmlns="http://www.w3.org/2000/svg" height="48" viewBox="0 96 960 960" width="48"><path d="M433 712V440L297 576l136 136ZM180 936q-24.75 0-42.375-17.625T120 876V276q0-24.75 17.625-42.375T180 216h600q24.75 0 42.375 17.625T840 276v600q0 24.75-17.625 42.375T780 936H180Zm393-60V276H180v600h393Z"/></svg>
+        </button>
+        <button id="sidebar-toggle-close" class="icon-button hidden">
+            <svg xmlns="http://www.w3.org/2000/svg" height="48" viewBox="0 96 960 960" width="48"><path d="M297 440v272l136-136-136-136ZM180 936q-24.75 0-42.375-17.625T120 876V276q0-24.75 17.625-42.375T180 216h600q24.75 0 42.375 17.625T840 276v600q0 24.75-17.625 42.375T780 936H180Zm393-60V276H180v600h393Z"/></svg>
+        </button>
+        <div class="sidebar flex flex-column">
             <div class="max-width">Settings</div>
             <div class="settings flex flex-column">
                 <div class="setting flex">
@@ -63,6 +70,7 @@
                 </button>
             </div>
         </div>
+    </div>
     <script src="/ui/main.js"></script>
 </body>
 </html>

frontend/static/main.js

@@ -1,5 +1,7 @@
 // Copyright (c) 2023 Julian Müller (ChaoticByte)
 
+const isMobile = /Android|BlackBerry|iPhone|iPod|Opera Mini/i.test(navigator.userAgent);
+
 // Fetch configuration and initialize Eucalyptus Chat Frontend
 
 fetch("/config")
@@ -11,6 +13,9 @@ fetch("/config")
     let conversation = [frontend_config.profile.conversation_prefix];
 
     // Elements - Sidebar
+    const sidebarOpenButton = document.getElementById("sidebar-toggle-open");
+    const sidebarCloseButton = document.getElementById("sidebar-toggle-close");
+    const sidebarContainer = document.getElementById("sidebar-container");
     const settingsLabelAssistantNameElement = document.getElementById("settings-label-assistant");
     const settingsMaxTokensElement = document.getElementById("settings-max-tokens");
     const settingsTemperatureElement = document.getElementById("settings-temperature");
@@ -19,15 +24,15 @@ fetch("/config")
     const settingsRepeatPenaltyElement = document.getElementById("settings-repeat-penalty");
     const settingsPresencePenaltyElement = document.getElementById("settings-presence-penalty");
     const settingsFrequencyPenaltyElement = document.getElementById("settings-frequency-penalty");
-    const resetSettingsButtonElement = document.getElementById("reset-settings-btn");
+    const resetSettingsButton = document.getElementById("reset-settings-btn");
-    const resetHistoryButtonElement = document.getElementById("reset-history-btn");
+    const resetHistoryButton = document.getElementById("reset-history-btn");
 
     settingsLabelAssistantNameElement.innerText = frontend_config.profile.name;
 
     // Elements - Main
     const messageHistoryContainer = document.getElementById("messages");
     const textInputElement = document.getElementById("text-input");
-    const sendButtonElement = document.getElementById("send-btn");
+    const sendButton = document.getElementById("send-btn");
 
     // API requests
 
@@ -106,17 +111,17 @@ fetch("/config")
     function addMessage(message, role) {
         if (role == Roles.USER) {
             conversation.push(
-                " " + frontend_config.profile.user_keyword + " "
+                frontend_config.profile.user_keyword + " "
-                + message + " " + frontend_config.profile.assistant_keyword);
+                + message + frontend_config.profile.separator + frontend_config.profile.assistant_keyword);
         }
-        else { conversation.push(message); }
+        else { conversation.push(message + frontend_config.profile.separator); }
         // UI
         let messageRoleElem = document.createElement("div");
         messageRoleElem.classList.add("message-type");
         messageRoleElem.innerText = role.name;
         let messageTextElem = document.createElement("div");
         messageTextElem.classList.add("message-text");
-        messageTextElem.innerText = message;
+        messageTextElem.innerText = message.trim();
         let messageElem = document.createElement("div");
         messageElem.classList.add("message");
         messageElem.classList.add(role.class);
@@ -141,9 +146,9 @@ fetch("/config")
         settingsRepeatPenaltyElement.disabled = true;
         settingsPresencePenaltyElement.disabled = true;
         settingsFrequencyPenaltyElement.disabled = true;
-        resetSettingsButtonElement.disabled = true;
+        resetSettingsButton.disabled = true;
-        resetHistoryButtonElement.disabled = true;
+        resetHistoryButton.disabled = true;
-        sendButtonElement.disabled = true;
+        sendButton.disabled = true;
         textInputElement.disabled = true;
     }
 
@@ -155,12 +160,12 @@ fetch("/config")
         settingsRepeatPenaltyElement.disabled = false;
         settingsPresencePenaltyElement.disabled = false;
         settingsFrequencyPenaltyElement.disabled = false;
-        resetSettingsButtonElement.disabled = false;
+        resetSettingsButton.disabled = false;
-        resetHistoryButtonElement.disabled = false;
+        resetHistoryButton.disabled = false;
-        sendButtonElement.disabled = false;
+        sendButton.disabled = false;
         textInputElement.disabled = false;
         // focus text input
-        textInputElement.focus();
+        if (!isMobile) textInputElement.focus();
     }
 
     async function chat() {
@@ -176,7 +181,7 @@ fetch("/config")
             let prompt = conversation.join("");
             let settings = getSettings();
             apiCompletion(prompt, settings).then(r => {
-                addMessage(r, Roles.ASSISTANT);
+                addMessage(r.trim(), Roles.ASSISTANT);
                 enableInput();
             });
         }
@@ -187,11 +192,29 @@ fetch("/config")
         messageHistoryContainer.innerText = "";
     }
 
+    // Sidebar
+
+    function toggleSidebar() {
+        if (sidebarContainer.classList.contains("sidebar-hidden")) {
+            sidebarCloseButton.classList.remove("hidden");
+            sidebarOpenButton.classList.add("hidden");
+            sidebarContainer.classList.remove("sidebar-hidden");
+        }
+        else {
+            sidebarOpenButton.classList.remove("hidden");
+            sidebarCloseButton.classList.add("hidden");
+            sidebarContainer.classList.add("sidebar-hidden");
+        }
+    }
+
     // Event Listeners
 
-    resetSettingsButtonElement.addEventListener("click", resetSettings);
+    sidebarOpenButton.addEventListener("click", toggleSidebar);
-    resetHistoryButtonElement.addEventListener("click", resetHistory);
+    sidebarCloseButton.addEventListener("click", toggleSidebar);
-    sendButtonElement.addEventListener("click", chat);
+
+    resetSettingsButton.addEventListener("click", resetSettings);
+    resetHistoryButton.addEventListener("click", resetHistory);
+    sendButton.addEventListener("click", chat);
 
     textInputElement.addEventListener("keypress", e => {
         // Send via Ctrl+Enter

frontend/static/style.css

@@ -5,11 +5,13 @@
     --background2: #303030;
     --background3: #161616;
     --background4: #131313;
+    --background5: #1a1a1a;
     --button-bg: #3b3b3b;
     --button-bg2: #4f4f4f;
     --icon-button-fill: #ffffff;
     --send-icon-button-fill: #29c76d;
     --color: #fafafa;
+    --color2: #bbbbbb;
     --border-radius: .5rem;
 }
 
@@ -27,11 +29,23 @@ input[type="number"] {
     width: 4rem;
 }
 
-.sidepanel {
+.sidebar-container {
+    padding: .5rem;
+    height: 100%;
+    background-color: var(--background5);
+    box-sizing: border-box;
+}
+
+.sidebar-container.sidebar-hidden > .sidebar {
+    display: none;
+}
+
+.sidebar {
+    margin-left: .5rem;
+    margin-right: .5rem;
+    margin-top: .4rem;
     gap: .5rem;
     align-items: flex-end;
-    padding: 1rem;
-    padding-left: 0;
     min-width: fit-content;
 }
 
@@ -56,19 +70,38 @@ input[type="number"] {
 }
 
 .messages {
-    gap: 1.1rem;
+    gap: 1rem;
     margin-bottom: 1rem;
     overflow-y: scroll;
     max-height: 89vh;
+    align-items: center;
+    flex-grow: 2;
 }
 
 .message {
     display: flex;
-    flex-direction: row;
+    flex-direction: column;
-    gap: .5rem;
+    flex-wrap: wrap;
-    padding: .5rem;
+    gap: 1rem;
+}
+
+.message-type {
+    color: var(--color2);
+    text-align: center;
+}
+
+.message-text {
+    white-space: pre-wrap;
+    padding: .5rem .8rem;
     border-radius: var(--border-radius);
-    max-width: fit-content;
+}
+
+.message-bg-assistant > .message-text {
+    background: var(--background2);
+}
+
+.message-bg-user > .message-text {
+    background: var(--background3);
 }
 
 button {
@@ -101,19 +134,6 @@ button:hover {
     width: 100%;
 }
 
-.message-bg-assistant {
-    background: var(--background2);
-}
-
-.message-bg-user {
-    background: var(--background3);
-}
-
-.message-type {
-    min-width: 3.5rem;
-    padding-left: .1rem;
-}
-
 .input-container {
     margin-top: auto;
     flex-direction: row;
@@ -135,6 +155,8 @@ button:hover {
 }
 
 .icon-button {
+    width: fit-content;
+    height: fit-content;
     padding: .2rem;
     display: flex;
     align-items: center;
@@ -160,3 +182,39 @@ button:hover {
     height: 2.2rem;
     fill: var(--send-icon-button-fill);
 }
+
+.hidden {
+    display: none;
+}
+
+@media only screen and (max-width: 600px) {
+
+    body {
+        height: 100dvh;
+    }
+
+    .sidebar-container {
+        position: absolute;
+        width: 100%;
+        height: 100%;
+        z-index: 1;
+    }
+
+    .sidebar-container.sidebar-hidden {
+        position: absolute;
+        width: unset;
+        height: unset;
+        left: auto;
+        right: 0;
+        background: transparent;
+    }
+
+    .sidebar-container > #sidebar-toggle-close {
+        margin-right: auto;
+    }
+
+    .sidebar {
+        margin-right: auto;
+        padding-right: 2.5rem;
+    }
+}

profiles/empty.json

@@ -0,0 +1,8 @@
+{
+    "name": "None",
+    "conversation_prefix": "",
+    "user_keyword": "",
+    "assistant_keyword": "",
+    "separator": "",
+    "stop_sequences": []
+}

profiles/koala.json

@@ -3,5 +3,6 @@
     "conversation_prefix": "BEGINNING OF CONVERSATION: ",
     "user_keyword": "USER:",
     "assistant_keyword": "GPT:",
+    "separator": " ",
     "stop_sequences": ["</s>"]
 }

profiles/manticore-chat.json

@@ -0,0 +1,8 @@
+{
+    "name": "Manticore",
+    "conversation_prefix": "",
+    "user_keyword": "USER:",
+    "assistant_keyword": "ASSISTANT:",
+    "separator": "\n",
+    "stop_sequences": ["</s>", "<unk>", "### USER:", "USER:"]
+}

profiles/vicuna-v0.json

@@ -0,0 +1,8 @@
+{
+    "name": "Vicuna v0",
+    "conversation_prefix": "A chat between a curious human and a helpful AI assistant.\n\n",
+    "user_keyword": "### Human:",
+    "assistant_keyword": "### Assistant:",
+    "separator": "\n",
+    "stop_sequences": ["### Human:"]
+}

profiles/vicuna-v1.1.json

@@ -0,0 +1,8 @@
+{
+    "name": "Vicuna v1.1",
+    "conversation_prefix": "A chat between a curious user and a helpful AI assistant.\n\n",
+    "user_keyword": "USER:",
+    "assistant_keyword": "ASSISTANT:",
+    "separator": "\n",
+    "stop_sequences": ["</s>"]
+}

profiles/vicuna.json

@@ -1,7 +0,0 @@
-{
-    "name": "Vicuna",
-    "conversation_prefix": "A chat between a curious user and a helpful AI assistant. ",
-    "user_keyword": "### Human:",
-    "assistant_keyword": "### Assistant:",
-    "stop_sequences": ["### Human:"]
-}

requirements.txt

@@ -1,3 +1,3 @@
-llama-cpp-python[server]==0.1.50
+llama-cpp-python[server]==0.1.56
 uvicorn==0.22.0
 sanic==23.3.0