diff --git a/.gitignore b/.gitignore index 68bc17f..13d164d 100644 --- a/.gitignore +++ b/.gitignore @@ -158,3 +158,6 @@ cython_debug/ # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. #.idea/ + +/models/* +!/models/.gitkeep diff --git a/README.md b/README.md index 98cc06a..1546db7 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,214 @@ -# Eucalyptus-Chat -A frontend for [Koala](https://bair.berkeley.edu/blog/2023/04/03/koala/) running on CPU with [llama.cpp](https://github.com/ggerganov/llama.cpp), using the API server provided by [llama-cpp-python](https://github.com/abetlen/llama-cpp-python). +# Eucalyptus Chat + +A frontend for [Koala](https://bair.berkeley.edu/blog/2023/04/03/koala/) running on CPU with [llama.cpp](https://github.com/ggerganov/llama.cpp), using the API server library provided by [llama-cpp-python](https://github.com/abetlen/llama-cpp-python). + +# Third-Party Licenses + +## Material Design Icons + +The file `./frontend/static/index.html` contains symbols from [Material Design Icons](https://github.com/google/material-design-icons). + +``` + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +``` + diff --git a/api-server.py b/api-server.py new file mode 100644 index 0000000..dac561a --- /dev/null +++ b/api-server.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python3 +# Copyright (c) 2023 Julian Müller (ChaoticByte) + +from argparse import ArgumentParser +from os import environ + +import uvicorn + +if __name__ == "__main__": + # CLI + ap = ArgumentParser() + ap.add_argument("-m", "--model", help="Filepath to the model", type=str, required=True) + ap.add_argument("--host", help="Address to listen on (default: localhost)", type=str, default="localhost") + ap.add_argument("--port", help="Port to listen on (default: 7331)", type=int, default=7331) + args = ap.parse_args() + # Set environment variable before importing api server + environ["MODEL"] = args.model + # Import api server + from llama_cpp.server import __main__ as server + # Run + uvicorn.run(server.app, host=args.host, port=args.port) diff --git a/frontend-server.py b/frontend-server.py new file mode 100644 index 0000000..2e1e7c3 --- /dev/null +++ b/frontend-server.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python3 +# Copyright (c) 2023 Julian Müller (ChaoticByte) + +from argparse import ArgumentParser + +import uvicorn +from frontend.app import app + +if __name__ == "__main__": + # CLI + ap = ArgumentParser() + ap.add_argument("--host", help="Address to listen on (default: localhost)", type=str, default="localhost") + ap.add_argument("--port", help="Port to listen on (default: 8080)", type=int, default=8080) + ap.add_argument("--api", help="URL of the API Server (default: 'http://localhost:7331')", type=str, default="http://localhost:7331") + args = ap.parse_args() + # Pass frontend config to the app + app.config.frontend_config = {"api_url": args.api.rstrip("/")} + # Run + uvicorn.run(app, host=args.host, port=args.port) diff --git a/frontend/__init__.py b/frontend/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/frontend/app.py b/frontend/app.py new file mode 100644 index 0000000..5c1dcb5 --- /dev/null +++ b/frontend/app.py @@ -0,0 +1,42 @@ +# Copyright (c) 2023 Julian Müller (ChaoticByte) + +from pathlib import Path + +from sanic import Sanic +from sanic import Request +from sanic import json +from sanic import redirect + +# App + +app = Sanic("Eucalyptus") + +# Static files + +static_dir = Path(__file__).parent / "static" +app.static("/ui", static_dir, index="index.html") + +# Routes + +@app.get("/") +async def redirect_to_ui(request: Request): + return redirect("/ui/") + +@app.get("/config") +async def frontend_config(request: Request): + return json(app.config.frontend_config) + +# Security Headers + +@app.on_response +async def security_headers(_, response): + response.headers.update({ + "X-Frame-Options": "deny", + "X-Content-Type-Options": "nosniff", + "Content-Security-Policy": f"default-src 'self'; style-src 'unsafe-inline' 'self'; connect-src 'self' {app.config.frontend_config['api_url']}", + "X-Permitted-Cross-Domain-Policies": "none", + "Referrer-Policy": "no-referrer", + "Cross-Origin-Embedder-Policy": "require-corp", + "Cross-Origin-Opener-Policy": "same-origin", + "Cross-Origin-Resource-Policy": "same-origin" + }) diff --git a/frontend/static/index.html b/frontend/static/index.html new file mode 100644 index 0000000..5146310 --- /dev/null +++ b/frontend/static/index.html @@ -0,0 +1,45 @@ + + + + + + + + + Eucalyptus + + +
+
+
+ + +
+
+
+
Settings
+
+
+
max_tokens
+
+
+
+
temperature
+
+
+
+
top_p
+
+
+
+
+ +
+
+ + + \ No newline at end of file diff --git a/frontend/static/main.js b/frontend/static/main.js new file mode 100644 index 0000000..64032a1 --- /dev/null +++ b/frontend/static/main.js @@ -0,0 +1,159 @@ +// Copyright (c) 2023 Julian Müller (ChaoticByte) + +(() => { + + // Koala specific keywords + const conversationBeginning = "BEGINNING OF CONVERSATION:"; + const userKeyword = " USER: "; + const assistantKeyword = " GPT:"; + const koalaStopSequence = ""; + + // Get frontend config + let frontend_config = null; + fetch("/config") + .then(r => { + return r.json(); + }) + .then(j => { + frontend_config = j; + }); + + // Message Context + let conversation = [conversationBeginning]; + + // Elements - Sidebar + const settingsMaxTokensElement = document.getElementById("settings-max-tokens"); + const settingsTemperatureElement = document.getElementById("settings-temperature"); + const settingsTopPElement = document.getElementById("settings-top-p"); + const resetHistoryButtonElement = document.getElementById("reset-history-btn"); + + // Elements - Main + const messageHistoryContainer = document.getElementById("messages"); + const textInputElement = document.getElementById("text-input"); + const sendButtonElement = document.getElementById("send-btn"); + + // API requests + + async function apiCompletion(prompt, settings) { + const bodyData = JSON.stringify({ + "prompt": prompt, + "stop": [koalaStopSequence], + "max_tokens": settings.max_tokens, + "temperature": settings.temperature, + "top_p": settings.top_p + }); + const response = await fetch(frontend_config.api_url + "/v1/completions", { + method: "post", + cache: "no-cache", + body: bodyData, + headers: { + "content-type": "application/json" + } + }); + const responseData = await response.json(); + return responseData["choices"][0]["text"]; + } + + // User-defined settings + + function getSettings() { + return { + max_tokens: settingsMaxTokensElement.value, + temperature: settingsTemperatureElement.value, + top_p: settingsTopPElement.value + } + } + + // Chat + + const MessageType = { + USER: { + name: "User", + class: "message-bg-user" + }, + ASSISTANT: { + name: "Koala", + class: "message-bg-assistant" + } + } + + function addMessage(message, type) { + if (type == MessageType.USER) { + conversation.push(userKeyword + message + assistantKeyword); + } + else { conversation.push(message); } + // UI + let messageTypeElem = document.createElement("div"); + messageTypeElem.classList.add("message-type"); + messageTypeElem.innerText = type.name; + let messageTextElem = document.createElement("div"); + messageTextElem.classList.add("message-text"); + messageTextElem.innerText = message; + let messageElem = document.createElement("div"); + messageElem.classList.add("message"); + messageElem.classList.add(type.class); + messageElem.appendChild(messageTypeElem); + messageElem.appendChild(messageTextElem); + messageHistoryContainer.appendChild(messageElem); + } + + function disableInput() { + sendButtonElement.disabled = true; + textInputElement.disabled = true; + } + + function enableInput() { + sendButtonElement.disabled = false; + textInputElement.disabled = false; + // focus text input + textInputElement.focus(); + } + + async function chat() { + if (frontend_config == null) { + console.log("Couldn't fetch frontend configuration."); + } + else { + disableInput(); + let input = textInputElement.value.trim(); + if (input == "") { + enableInput(); + } + else { + textInputElement.value = ""; + addMessage(input, MessageType.USER); + let prompt = conversation.join(""); + let settings = getSettings(); + apiCompletion(prompt, settings).then(r => { + addMessage(r, MessageType.ASSISTANT); + enableInput(); + }); + } + } + } + + function resetHistory() { + conversation = [conversationBeginning]; + messageHistoryContainer.innerText = ""; + } + + // Event Listeners + + resetHistoryButtonElement.addEventListener("click", resetHistory); + sendButtonElement.addEventListener("click", chat); + + textInputElement.addEventListener("keypress", e => { + // Send via Ctrl+Enter + if (e.key == "Enter" && e.ctrlKey) { + chat(); + } + }); + + textInputElement.addEventListener("input", e => { + // Calculate Line height + textInputElement.style.removeProperty("height"); + let newHeight = textInputElement.scrollHeight; + textInputElement.style.height = newHeight.toString() + "px"; + }); + +})(); \ No newline at end of file diff --git a/frontend/static/style.css b/frontend/static/style.css new file mode 100644 index 0000000..69be8d3 --- /dev/null +++ b/frontend/static/style.css @@ -0,0 +1,164 @@ +/* Copyright (c) 2023 Julian Müller (ChaoticByte) */ + +:root { + --background: #1f1f1f; + --background2: #303030; + --background3: #161616; + --background4: #131313; + --button-bg: #3b3b3b; + --button-bg2: #4f4f4f; + --icon-button-fill: #ffffff; + --send-icon-button-fill: #29c76d; + --color: #fafafa; + --padding: .5rem; + --border-radius: .5rem; +} + +body { + margin: 0; + padding: 0; + background: var(--background); + color: var(--color); + font-family: sans-serif; + flex-direction: row; + min-height: 100vh; +} + + +input[type="number"] { + width: 4rem; +} + +.sidepanel { + gap: var(--padding); + align-items: flex-end; + padding: 1rem; + padding-left: 0; + min-width: fit-content; +} + +.settings { + margin-top: 1rem; + margin-bottom: 1rem; + gap: .5rem; +} + +.setting { + gap: .5rem; +} + +.setting > div:first-child { + flex-grow: 1; +} + +.main { + flex-grow: 1; + justify-content: flex-end; + padding: 1rem; +} + +.messages { + gap: 1.1rem; + padding-bottom: var(--padding); + overflow-y: scroll; + max-height: 89vh; +} + +.message { + display: flex; + flex-direction: row; + gap: var(--padding); + padding: var(--padding); + border-radius: var(--border-radius); + max-width: fit-content; +} + +button { + padding: .5rem .7rem; + border: none; + outline: none; + color: var(--color); + background: var(--button-bg); + border-radius: var(--border-radius); + font-size: .9rem; +} + +button:disabled, input:disabled, textarea:disabled { + opacity: 50%; +} + +button:hover { + background: var(--button-bg2); +} + +.flex { + display: flex; +} + +.flex-column { + flex-direction: column; +} + +.max-width { + width: 100%; +} + +.message-bg-assistant { + background: var(--background2); +} + +.message-bg-user { + background: var(--background3); +} + +.message-type { + min-width: 3.5rem; + padding-left: .1rem; +} + +.input-container { + margin-top: auto; + flex-direction: row; + align-items: center; + gap: .5rem; +} + +.text-input { + margin: 0; + flex-grow: 1; + background-color: var(--background4); + border: none; + outline: none; + padding: .8rem 1.1rem; + border-radius: var(--border-radius); + color: var(--color); + resize: none; + font-size: .9rem; +} + +.icon-button { + padding: .2rem; + display: flex; + align-items: center; + justify-content: center; + background: transparent; +} + +.icon-button:hover { + background: transparent; +} + +.icon-button > svg { + height: 1.8rem; + width: auto; + fill: var(--icon-button-fill); +} + +.icon-button:hover > svg { + transform: scale(1.1); +} + +.send-btn > svg { + height: 2.2rem; + fill: var(--send-icon-button-fill); +} diff --git a/models/.gitkeep b/models/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..b6cf0bc --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +llama-cpp-python[server]~=0.1.39 +uvicorn~=0.22.0 +sanic~=23.3.0 \ No newline at end of file