diff --git a/.env.sample b/.env.sample index 46ebab8..b5ad2c9 100644 --- a/.env.sample +++ b/.env.sample @@ -4,11 +4,13 @@ CLIENT_TOKEN = BOT_TOKEN # Default model for new users MODEL = DEFAULT_MODEL -# ip/port address of docker container, I use 172.18.0.3 for docker, 127.0.0.1 for local -OLLAMA_IP = IP_ADDRESS -OLLAMA_PORT = PORT +# ip/port address of the LLM server (ollama, llmman, ...), I use 172.18.0.3 for docker, 127.0.0.1 for local +# ollama listens on 11434, llmman (https://github.com/llmmanorg/llmman) on 17434 +# OLLAMA_IP / OLLAMA_PORT are still accepted as deprecated aliases +LLM_ENDPOINT = IP_ADDRESS +LLM_PORT = PORT -# ip address for discord bot container, I use 172.18.0.2, use different IP than ollama_ip +# ip address for discord bot container, I use 172.18.0.2, use different IP than llm_endpoint DISCORD_IP = IP_ADDRESS # subnet address, ex. 172.18.0.0 as we use /16. diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4930371..36f29ae 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -33,8 +33,8 @@ jobs: run: | touch .env echo CLIENT_TOKEN = ${{ secrets.BOT_TOKEN }} >> .env - echo OLLAMA_IP = ${{ secrets.OLLAMA_IP }} >> .env - echo OLLAMA_PORT = ${{ secrets.OLLAMA_PORT }} >> .env + echo LLM_ENDPOINT = ${{ secrets.OLLAMA_IP }} >> .env + echo LLM_PORT = ${{ secrets.OLLAMA_PORT }} >> .env echo MODEL = ${{ secrets.MODEL }} >> .env # set -e ensures if nohup fails, this section fails @@ -60,8 +60,8 @@ jobs: run: | touch .env echo CLIENT_TOKEN = ${{ secrets.BOT_TOKEN }} >> .env - echo OLLAMA_IP = ${{ secrets.OLLAMA_IP }} >> .env - echo OLLAMA_PORT = ${{ secrets.OLLAMA_PORT }} >> .env + echo LLM_ENDPOINT = ${{ secrets.OLLAMA_IP }} >> .env + echo LLM_PORT = ${{ secrets.OLLAMA_PORT }} >> .env echo MODEL = ${{ secrets.MODEL }} >> .env - name: Setup Docker Network and Images diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 51cf4c8..f0c4deb 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -18,8 +18,8 @@ jobs: run: | touch .env echo CLIENT_TOKEN = ${{ secrets.CLIENT }} >> .env - echo OLLAMA_IP = ${{ secrets.OLLAMA_IP }} >> .env - echo OLLAMA_PORT = ${{ secrets.OLLAMA_PORT }} >> .env + echo LLM_ENDPOINT = ${{ secrets.OLLAMA_IP }} >> .env + echo LLM_PORT = ${{ secrets.OLLAMA_PORT }} >> .env echo MODEL = ${{ secrets.MODEL }} >> .env echo DISCORD_IP = ${{ secrets.DISCORD_IP }} >> .env echo SUBNET_ADDRESS = ${{ secrets.SUBNET_ADDRESS }} >> .env diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5c057a0..97802b5 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -42,8 +42,8 @@ jobs: run: | touch .env echo CLIENT_TOKEN = ${{ secrets.BOT_TOKEN }} >> .env - echo OLLAMA_IP = ${{ secrets.OLLAMA_IP }} >> .env - echo OLLAMA_PORT = ${{ secrets.OLLAMA_PORT }} >> .env + echo LLM_ENDPOINT = ${{ secrets.OLLAMA_IP }} >> .env + echo LLM_PORT = ${{ secrets.OLLAMA_PORT }} >> .env echo MODEL = ${{ secrets.MODEL }} >> .env - name: Test Application diff --git a/README.md b/README.md index c067405..5b94ad8 100644 --- a/README.md +++ b/README.md @@ -48,6 +48,7 @@ These are guides to the features and capabilities of this app. * Clone this repo using `git clone https://github.com/kevinthedang/discord-ollama.git` or just use [GitHub Desktop](https://desktop.github.com/) to clone the repo. * You will need a `.env` file in the root of the project directory with the bot's token. There is a `.env.sample` is provided for you as a reference for what environment variables. * For example, `CLIENT_TOKEN = [Bot Token]` + * The LLM server is configured with `LLM_ENDPOINT` / `LLM_PORT` (`OLLAMA_IP` / `OLLAMA_PORT` still work as deprecated aliases). * Please refer to the docs for bot setup. * [Creating a Discord App](./docs/setup-discord-app.md) * [Local Machine Setup](./docs/setup-local.md) @@ -61,6 +62,7 @@ These are guides to the features and capabilities of this app. * This project requires the use of npm version `10.9.0` or above. * [Ollama](https://ollama.com/) * [Ollama Docker Image](https://hub.docker.com/r/ollama/ollama) + * [llmman](https://github.com/llmmanorg/llmman) also serves the Ollama API (on port `17434`) and can be used in its place, see [Local Machine Setup](./docs/setup-local.md#using-with-llmman-alternative-to-ollama). * [Discord.js Docs](https://discord.js.org/docs/packages/discord.js/main) * [Setting up Docker (Ubuntu 20.04)](https://www.digitalocean.com/community/tutorials/how-to-install-and-use-docker-on-ubuntu-20-04) * [Setting up Nvidia Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html) diff --git a/docker-compose.yml b/docker-compose.yml index 3d999ae..32c6932 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,8 +10,8 @@ services: image: kevinthedang/discord-ollama:0.9.1 environment: CLIENT_TOKEN: ${CLIENT_TOKEN} - OLLAMA_IP: ${OLLAMA_IP} - OLLAMA_PORT: ${OLLAMA_PORT} + LLM_ENDPOINT: ${LLM_ENDPOINT:-${OLLAMA_IP}} # OLLAMA_IP is a deprecated alias + LLM_PORT: ${LLM_PORT:-${OLLAMA_PORT}} # OLLAMA_PORT is a deprecated alias MODEL: ${MODEL} networks: ollama-net: @@ -26,14 +26,14 @@ services: restart: always networks: ollama-net: - ipv4_address: ${OLLAMA_IP} + ipv4_address: ${LLM_ENDPOINT:-${OLLAMA_IP}} runtime: nvidia # use Nvidia Container Toolkit for GPU support devices: - /dev/nvidia0 volumes: - ollama:/root/.ollama ports: - - ${OLLAMA_PORT}:${OLLAMA_PORT} + - ${LLM_PORT:-${OLLAMA_PORT}}:${LLM_PORT:-${OLLAMA_PORT}} # create a network that supports giving addresses withing a specific subnet networks: diff --git a/docs/setup-docker.md b/docs/setup-docker.md index 22eaad5..48a48f8 100644 --- a/docs/setup-docker.md +++ b/docs/setup-docker.md @@ -43,10 +43,10 @@ sudo systemctl restart docker * [GitHub repository](https://github.com/NVIDIA/nvidia-container-toolkit?tab=readme-ov-file) for Nvidia Container Toolkit ## To Run (with Docker and Docker Compose) -* With the inclusion of subnets in the `docker-compose.yml`, you will need to set the `SUBNET_ADDRESS`, `OLLAMA_IP`, `OLLAMA_PORT`, and `DISCORD_IP`. Here are some default values if you don't care: +* With the inclusion of subnets in the `docker-compose.yml`, you will need to set the `SUBNET_ADDRESS`, `LLM_ENDPOINT`, `LLM_PORT`, and `DISCORD_IP`. Here are some default values if you don't care: * `SUBNET_ADDRESS = 172.18.0.0` - * `OLLAMA_IP = 172.18.0.2` - * `OLLAMA_PORT = 11434` + * `LLM_ENDPOINT = 172.18.0.2` + * `LLM_PORT = 11434` * `DISCORD_IP = 172.18.0.3` * Don't understand any of this? watch a Networking video to understand subnetting. * You also need all environment variables shown in [`.env.sample`](../.env.sample) diff --git a/docs/setup-local.md b/docs/setup-local.md index ab0decf..cd18ab8 100644 --- a/docs/setup-local.md +++ b/docs/setup-local.md @@ -14,11 +14,18 @@ > [!NOTE] > You can now pull models directly from the Discord client using `/pull-model ` or `/switch-model `. They must exist from your local model library or from the [Ollama Model Library](https://ollama.com/library) +## Using with llmman (alternative to Ollama) +* [llmman](https://github.com/llmmanorg/llmman) is a local model runner that serves the Ollama API on port `17434`. The bot talks to it exactly as it would to Ollama, only the port differs. +* Install it with `curl -fsSL https://raw.githubusercontent.com/llmmanorg/llmman/main/install.sh | sh` (Linux/macOS) or `irm https://raw.githubusercontent.com/llmmanorg/llmman/main/install.ps1 | iex` (Windows). +* Start the server with `llmman serve` and pull a model with `llmman pull [model name]`, for example `llmman pull gemma4` or `llmman pull hf.co/unsloth/Qwen3.5-0.8B-GGUF`. +* Point the bot at it by setting `LLM_ENDPOINT = 127.0.0.1` and `LLM_PORT = 17434` in your `.env`, then set `MODEL` to a model you pulled. `/pull-model`, `/switch-model` and `/delete-model` work the same way. + ## To Run Locally (without Docker) * Run `npm install` to install the npm packages. -* Ensure that your [.env](../.env.sample) file's `OLLAMA_IP` is `127.0.0.1` to work properly. - * You only need your `CLIENT_TOKEN`, `OLLAMA_IP`, `OLLAMA_PORT`. - * The ollama ip and port should just use it's defaults by nature. If not, utilize `OLLAMA_IP = 127.0.0.1` and `OLLAMA_PORT = 11434`. +* Ensure that your [.env](../.env.sample) file's `LLM_ENDPOINT` is `127.0.0.1` to work properly. + * You only need your `CLIENT_TOKEN`, `LLM_ENDPOINT`, `LLM_PORT`. + * The LLM server ip and port should just use it's defaults by nature. If not, utilize `LLM_ENDPOINT = 127.0.0.1` and `LLM_PORT = 11434`. + * The older `OLLAMA_IP` / `OLLAMA_PORT` names are still accepted as deprecated aliases. * Now, you can run the bot by running `npm run client` which will build and run the decompiled typescript and run the setup for ollama. * **IMPORTANT**: This must be ran in the wsl/Linux instance to work properly! Using Command Prompt/Powershell/Git Bash/etc. will not work on Windows (at least in my experience). * Refer to the [resources](../README.md#resources) on what node version to use. diff --git a/src/keys.ts b/src/keys.ts index 03423b2..688b001 100644 --- a/src/keys.ts +++ b/src/keys.ts @@ -2,8 +2,8 @@ import { getEnvVar } from './utils/index.js' export const Keys = { clientToken: getEnvVar('CLIENT_TOKEN'), - ipAddress: getEnvVar('OLLAMA_IP', '127.0.0.1'), // default ollama ip if none - portAddress: getEnvVar('OLLAMA_PORT', '11434'), // default ollama port if none + ipAddress: getEnvVar('LLM_ENDPOINT', process.env.OLLAMA_IP ?? '127.0.0.1'), // OLLAMA_IP is a deprecated alias + portAddress: getEnvVar('LLM_PORT', process.env.OLLAMA_PORT ?? '11434'), // OLLAMA_PORT is a deprecated alias defaultModel: getEnvVar('MODEL', 'llama3.2') } as const // readonly keys diff --git a/src/utils/env.ts b/src/utils/env.ts index 158932d..fbe3cca 100644 --- a/src/utils/env.ts +++ b/src/utils/env.ts @@ -26,7 +26,7 @@ export function getEnvVar(name: string, fallback?: string): string { This is probably an invalid token unless Discord updated their token policy. Please provide a valid token.`) // validate IPv4 address found in environment variables - if ((name.endsWith("_IP") || name.endsWith("_ADDRESS")) && !ipValidate.test(value)) + if ((name.endsWith("_IP") || name.endsWith("_ADDRESS") || name.endsWith("_ENDPOINT")) && !ipValidate.test(value)) throw new Error(`Environment variable ${name} does not follow IPv4 formatting.`) // return env variable diff --git a/tests/getEnvVar.test.ts b/tests/getEnvVar.test.ts index 071edea..c0049ee 100644 --- a/tests/getEnvVar.test.ts +++ b/tests/getEnvVar.test.ts @@ -47,4 +47,28 @@ describe('Environment Setup', () => { it('throws an error if key is not found', () => { expect(() => getEnvVar('NON_EXISTENT_KEY')).toThrowError() }) + + // test that *_ENDPOINT keys are validated as IPv4 addresses like *_IP keys + it('validates *_ENDPOINT keys as IPv4', () => { + expect(getEnvVar('UNSET_TEST_ENDPOINT', '127.0.0.1')).toBe('127.0.0.1') + expect(() => getEnvVar('UNSET_TEST_ENDPOINT', 'not-an-ip')).toThrowError() + }) + + // test the deprecated alias lookup used in src/keys.ts + it('falls back to the deprecated OLLAMA_IP / OLLAMA_PORT aliases', () => { + const saved = { LLM_ENDPOINT: process.env.LLM_ENDPOINT, LLM_PORT: process.env.LLM_PORT } + delete process.env.LLM_ENDPOINT + delete process.env.LLM_PORT + process.env.OLLAMA_IP = '10.0.0.5' + process.env.OLLAMA_PORT = '17434' + try { + expect(getEnvVar('LLM_ENDPOINT', process.env.OLLAMA_IP ?? '127.0.0.1')).toBe('10.0.0.5') + expect(getEnvVar('LLM_PORT', process.env.OLLAMA_PORT ?? '11434')).toBe('17434') + } finally { + delete process.env.OLLAMA_IP + delete process.env.OLLAMA_PORT + if (saved.LLM_ENDPOINT !== undefined) process.env.LLM_ENDPOINT = saved.LLM_ENDPOINT + if (saved.LLM_PORT !== undefined) process.env.LLM_PORT = saved.LLM_PORT + } + }) })