mirror of
https://github.com/kevinthedang/discord-ollama.git
synced 2026-09-23 17:08:58 -04:00
Document llmman and rename OLLAMA_IP/PORT to LLM_ENDPOINT/PORT (#198)
llmman (https://github.com/llmmanorg/llmman) serves the Ollama API on port 17434 and works with the bot as-is, so document it in setup-local and link it from the README. Since the backend is no longer Ollama-only, rename the env vars to LLM_ENDPOINT / LLM_PORT as requested in review. OLLAMA_IP / OLLAMA_PORT are kept as deprecated aliases in keys.ts and docker-compose.yml so existing deployments keep working.
This commit is contained in:
Eric Curtin
+6
-4
@@ -4,11 +4,13 @@ CLIENT_TOKEN = BOT_TOKEN
|
||||
# Default model for new users
|
||||
MODEL = DEFAULT_MODEL
|
||||
|
||||
# ip/port address of docker container, I use 172.18.0.3 for docker, 127.0.0.1 for local
|
||||
OLLAMA_IP = IP_ADDRESS
|
||||
OLLAMA_PORT = PORT
|
||||
# ip/port address of the LLM server (ollama, llmman, ...), I use 172.18.0.3 for docker, 127.0.0.1 for local
|
||||
# ollama listens on 11434, llmman (https://github.com/llmmanorg/llmman) on 17434
|
||||
# OLLAMA_IP / OLLAMA_PORT are still accepted as deprecated aliases
|
||||
LLM_ENDPOINT = IP_ADDRESS
|
||||
LLM_PORT = PORT
|
||||
|
||||
# ip address for discord bot container, I use 172.18.0.2, use different IP than ollama_ip
|
||||
# ip address for discord bot container, I use 172.18.0.2, use different IP than llm_endpoint
|
||||
DISCORD_IP = IP_ADDRESS
|
||||
|
||||
# subnet address, ex. 172.18.0.0 as we use /16.
|
||||
|
||||
@@ -33,8 +33,8 @@ jobs:
|
||||
run: |
|
||||
touch .env
|
||||
echo CLIENT_TOKEN = ${{ secrets.BOT_TOKEN }} >> .env
|
||||
echo OLLAMA_IP = ${{ secrets.OLLAMA_IP }} >> .env
|
||||
echo OLLAMA_PORT = ${{ secrets.OLLAMA_PORT }} >> .env
|
||||
echo LLM_ENDPOINT = ${{ secrets.OLLAMA_IP }} >> .env
|
||||
echo LLM_PORT = ${{ secrets.OLLAMA_PORT }} >> .env
|
||||
echo MODEL = ${{ secrets.MODEL }} >> .env
|
||||
|
||||
# set -e ensures if nohup fails, this section fails
|
||||
@@ -60,8 +60,8 @@ jobs:
|
||||
run: |
|
||||
touch .env
|
||||
echo CLIENT_TOKEN = ${{ secrets.BOT_TOKEN }} >> .env
|
||||
echo OLLAMA_IP = ${{ secrets.OLLAMA_IP }} >> .env
|
||||
echo OLLAMA_PORT = ${{ secrets.OLLAMA_PORT }} >> .env
|
||||
echo LLM_ENDPOINT = ${{ secrets.OLLAMA_IP }} >> .env
|
||||
echo LLM_PORT = ${{ secrets.OLLAMA_PORT }} >> .env
|
||||
echo MODEL = ${{ secrets.MODEL }} >> .env
|
||||
|
||||
- name: Setup Docker Network and Images
|
||||
|
||||
@@ -18,8 +18,8 @@ jobs:
|
||||
run: |
|
||||
touch .env
|
||||
echo CLIENT_TOKEN = ${{ secrets.CLIENT }} >> .env
|
||||
echo OLLAMA_IP = ${{ secrets.OLLAMA_IP }} >> .env
|
||||
echo OLLAMA_PORT = ${{ secrets.OLLAMA_PORT }} >> .env
|
||||
echo LLM_ENDPOINT = ${{ secrets.OLLAMA_IP }} >> .env
|
||||
echo LLM_PORT = ${{ secrets.OLLAMA_PORT }} >> .env
|
||||
echo MODEL = ${{ secrets.MODEL }} >> .env
|
||||
echo DISCORD_IP = ${{ secrets.DISCORD_IP }} >> .env
|
||||
echo SUBNET_ADDRESS = ${{ secrets.SUBNET_ADDRESS }} >> .env
|
||||
|
||||
@@ -42,8 +42,8 @@ jobs:
|
||||
run: |
|
||||
touch .env
|
||||
echo CLIENT_TOKEN = ${{ secrets.BOT_TOKEN }} >> .env
|
||||
echo OLLAMA_IP = ${{ secrets.OLLAMA_IP }} >> .env
|
||||
echo OLLAMA_PORT = ${{ secrets.OLLAMA_PORT }} >> .env
|
||||
echo LLM_ENDPOINT = ${{ secrets.OLLAMA_IP }} >> .env
|
||||
echo LLM_PORT = ${{ secrets.OLLAMA_PORT }} >> .env
|
||||
echo MODEL = ${{ secrets.MODEL }} >> .env
|
||||
|
||||
- name: Test Application
|
||||
|
||||
@@ -48,6 +48,7 @@ These are guides to the features and capabilities of this app.
|
||||
* Clone this repo using `git clone https://github.com/kevinthedang/discord-ollama.git` or just use [GitHub Desktop](https://desktop.github.com/) to clone the repo.
|
||||
* You will need a `.env` file in the root of the project directory with the bot's token. There is a `.env.sample` is provided for you as a reference for what environment variables.
|
||||
* For example, `CLIENT_TOKEN = [Bot Token]`
|
||||
* The LLM server is configured with `LLM_ENDPOINT` / `LLM_PORT` (`OLLAMA_IP` / `OLLAMA_PORT` still work as deprecated aliases).
|
||||
* Please refer to the docs for bot setup.
|
||||
* [Creating a Discord App](./docs/setup-discord-app.md)
|
||||
* [Local Machine Setup](./docs/setup-local.md)
|
||||
@@ -61,6 +62,7 @@ These are guides to the features and capabilities of this app.
|
||||
* This project requires the use of npm version `10.9.0` or above.
|
||||
* [Ollama](https://ollama.com/)
|
||||
* [Ollama Docker Image](https://hub.docker.com/r/ollama/ollama)
|
||||
* [llmman](https://github.com/llmmanorg/llmman) also serves the Ollama API (on port `17434`) and can be used in its place, see [Local Machine Setup](./docs/setup-local.md#using-with-llmman-alternative-to-ollama).
|
||||
* [Discord.js Docs](https://discord.js.org/docs/packages/discord.js/main)
|
||||
* [Setting up Docker (Ubuntu 20.04)](https://www.digitalocean.com/community/tutorials/how-to-install-and-use-docker-on-ubuntu-20-04)
|
||||
* [Setting up Nvidia Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html)
|
||||
|
||||
+4
-4
@@ -10,8 +10,8 @@ services:
|
||||
image: kevinthedang/discord-ollama:0.9.1
|
||||
environment:
|
||||
CLIENT_TOKEN: ${CLIENT_TOKEN}
|
||||
OLLAMA_IP: ${OLLAMA_IP}
|
||||
OLLAMA_PORT: ${OLLAMA_PORT}
|
||||
LLM_ENDPOINT: ${LLM_ENDPOINT:-${OLLAMA_IP}} # OLLAMA_IP is a deprecated alias
|
||||
LLM_PORT: ${LLM_PORT:-${OLLAMA_PORT}} # OLLAMA_PORT is a deprecated alias
|
||||
MODEL: ${MODEL}
|
||||
networks:
|
||||
ollama-net:
|
||||
@@ -26,14 +26,14 @@ services:
|
||||
restart: always
|
||||
networks:
|
||||
ollama-net:
|
||||
ipv4_address: ${OLLAMA_IP}
|
||||
ipv4_address: ${LLM_ENDPOINT:-${OLLAMA_IP}}
|
||||
runtime: nvidia # use Nvidia Container Toolkit for GPU support
|
||||
devices:
|
||||
- /dev/nvidia0
|
||||
volumes:
|
||||
- ollama:/root/.ollama
|
||||
ports:
|
||||
- ${OLLAMA_PORT}:${OLLAMA_PORT}
|
||||
- ${LLM_PORT:-${OLLAMA_PORT}}:${LLM_PORT:-${OLLAMA_PORT}}
|
||||
|
||||
# create a network that supports giving addresses withing a specific subnet
|
||||
networks:
|
||||
|
||||
@@ -43,10 +43,10 @@ sudo systemctl restart docker
|
||||
* [GitHub repository](https://github.com/NVIDIA/nvidia-container-toolkit?tab=readme-ov-file) for Nvidia Container Toolkit
|
||||
|
||||
## To Run (with Docker and Docker Compose)
|
||||
* With the inclusion of subnets in the `docker-compose.yml`, you will need to set the `SUBNET_ADDRESS`, `OLLAMA_IP`, `OLLAMA_PORT`, and `DISCORD_IP`. Here are some default values if you don't care:
|
||||
* With the inclusion of subnets in the `docker-compose.yml`, you will need to set the `SUBNET_ADDRESS`, `LLM_ENDPOINT`, `LLM_PORT`, and `DISCORD_IP`. Here are some default values if you don't care:
|
||||
* `SUBNET_ADDRESS = 172.18.0.0`
|
||||
* `OLLAMA_IP = 172.18.0.2`
|
||||
* `OLLAMA_PORT = 11434`
|
||||
* `LLM_ENDPOINT = 172.18.0.2`
|
||||
* `LLM_PORT = 11434`
|
||||
* `DISCORD_IP = 172.18.0.3`
|
||||
* Don't understand any of this? watch a Networking video to understand subnetting.
|
||||
* You also need all environment variables shown in [`.env.sample`](../.env.sample)
|
||||
|
||||
+10
-3
@@ -14,11 +14,18 @@
|
||||
> [!NOTE]
|
||||
> You can now pull models directly from the Discord client using `/pull-model <model-name>` or `/switch-model <model-name>`. They must exist from your local model library or from the [Ollama Model Library](https://ollama.com/library)
|
||||
|
||||
## Using with llmman (alternative to Ollama)
|
||||
* [llmman](https://github.com/llmmanorg/llmman) is a local model runner that serves the Ollama API on port `17434`. The bot talks to it exactly as it would to Ollama, only the port differs.
|
||||
* Install it with `curl -fsSL https://raw.githubusercontent.com/llmmanorg/llmman/main/install.sh | sh` (Linux/macOS) or `irm https://raw.githubusercontent.com/llmmanorg/llmman/main/install.ps1 | iex` (Windows).
|
||||
* Start the server with `llmman serve` and pull a model with `llmman pull [model name]`, for example `llmman pull gemma4` or `llmman pull hf.co/unsloth/Qwen3.5-0.8B-GGUF`.
|
||||
* Point the bot at it by setting `LLM_ENDPOINT = 127.0.0.1` and `LLM_PORT = 17434` in your `.env`, then set `MODEL` to a model you pulled. `/pull-model`, `/switch-model` and `/delete-model` work the same way.
|
||||
|
||||
## To Run Locally (without Docker)
|
||||
* Run `npm install` to install the npm packages.
|
||||
* Ensure that your [.env](../.env.sample) file's `OLLAMA_IP` is `127.0.0.1` to work properly.
|
||||
* You only need your `CLIENT_TOKEN`, `OLLAMA_IP`, `OLLAMA_PORT`.
|
||||
* The ollama ip and port should just use it's defaults by nature. If not, utilize `OLLAMA_IP = 127.0.0.1` and `OLLAMA_PORT = 11434`.
|
||||
* Ensure that your [.env](../.env.sample) file's `LLM_ENDPOINT` is `127.0.0.1` to work properly.
|
||||
* You only need your `CLIENT_TOKEN`, `LLM_ENDPOINT`, `LLM_PORT`.
|
||||
* The LLM server ip and port should just use it's defaults by nature. If not, utilize `LLM_ENDPOINT = 127.0.0.1` and `LLM_PORT = 11434`.
|
||||
* The older `OLLAMA_IP` / `OLLAMA_PORT` names are still accepted as deprecated aliases.
|
||||
* Now, you can run the bot by running `npm run client` which will build and run the decompiled typescript and run the setup for ollama.
|
||||
* **IMPORTANT**: This must be ran in the wsl/Linux instance to work properly! Using Command Prompt/Powershell/Git Bash/etc. will not work on Windows (at least in my experience).
|
||||
* Refer to the [resources](../README.md#resources) on what node version to use.
|
||||
|
||||
+2
-2
@@ -2,8 +2,8 @@ import { getEnvVar } from './utils/index.js'
|
||||
|
||||
export const Keys = {
|
||||
clientToken: getEnvVar('CLIENT_TOKEN'),
|
||||
ipAddress: getEnvVar('OLLAMA_IP', '127.0.0.1'), // default ollama ip if none
|
||||
portAddress: getEnvVar('OLLAMA_PORT', '11434'), // default ollama port if none
|
||||
ipAddress: getEnvVar('LLM_ENDPOINT', process.env.OLLAMA_IP ?? '127.0.0.1'), // OLLAMA_IP is a deprecated alias
|
||||
portAddress: getEnvVar('LLM_PORT', process.env.OLLAMA_PORT ?? '11434'), // OLLAMA_PORT is a deprecated alias
|
||||
defaultModel: getEnvVar('MODEL', 'llama3.2')
|
||||
} as const // readonly keys
|
||||
|
||||
|
||||
+1
-1
@@ -26,7 +26,7 @@ export function getEnvVar(name: string, fallback?: string): string {
|
||||
This is probably an invalid token unless Discord updated their token policy. Please provide a valid token.`)
|
||||
|
||||
// validate IPv4 address found in environment variables
|
||||
if ((name.endsWith("_IP") || name.endsWith("_ADDRESS")) && !ipValidate.test(value))
|
||||
if ((name.endsWith("_IP") || name.endsWith("_ADDRESS") || name.endsWith("_ENDPOINT")) && !ipValidate.test(value))
|
||||
throw new Error(`Environment variable ${name} does not follow IPv4 formatting.`)
|
||||
|
||||
// return env variable
|
||||
|
||||
@@ -47,4 +47,28 @@ describe('Environment Setup', () => {
|
||||
it('throws an error if key is not found', () => {
|
||||
expect(() => getEnvVar('NON_EXISTENT_KEY')).toThrowError()
|
||||
})
|
||||
|
||||
// test that *_ENDPOINT keys are validated as IPv4 addresses like *_IP keys
|
||||
it('validates *_ENDPOINT keys as IPv4', () => {
|
||||
expect(getEnvVar('UNSET_TEST_ENDPOINT', '127.0.0.1')).toBe('127.0.0.1')
|
||||
expect(() => getEnvVar('UNSET_TEST_ENDPOINT', 'not-an-ip')).toThrowError()
|
||||
})
|
||||
|
||||
// test the deprecated alias lookup used in src/keys.ts
|
||||
it('falls back to the deprecated OLLAMA_IP / OLLAMA_PORT aliases', () => {
|
||||
const saved = { LLM_ENDPOINT: process.env.LLM_ENDPOINT, LLM_PORT: process.env.LLM_PORT }
|
||||
delete process.env.LLM_ENDPOINT
|
||||
delete process.env.LLM_PORT
|
||||
process.env.OLLAMA_IP = '10.0.0.5'
|
||||
process.env.OLLAMA_PORT = '17434'
|
||||
try {
|
||||
expect(getEnvVar('LLM_ENDPOINT', process.env.OLLAMA_IP ?? '127.0.0.1')).toBe('10.0.0.5')
|
||||
expect(getEnvVar('LLM_PORT', process.env.OLLAMA_PORT ?? '11434')).toBe('17434')
|
||||
} finally {
|
||||
delete process.env.OLLAMA_IP
|
||||
delete process.env.OLLAMA_PORT
|
||||
if (saved.LLM_ENDPOINT !== undefined) process.env.LLM_ENDPOINT = saved.LLM_ENDPOINT
|
||||
if (saved.LLM_PORT !== undefined) process.env.LLM_PORT = saved.LLM_PORT
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
Reference in New Issue
Block a user