diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 93aaf445..69918a57 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -5,8 +5,14 @@ version: 2 updates: - - package-ecosystem: "pip" # See documentation for possible values - directory: "/" # Location of package manifests + - package-ecosystem: "pip" + directory: "/requirements/full/" + target-branch: "dev" + schedule: + interval: "weekly" + + - package-ecosystem: "pip" + directory: "/requirements/portable/" target-branch: "dev" schedule: interval: "weekly" diff --git a/.github/workflows/build-everything-tgw.yml b/.github/workflows/build-everything-tgw.yml new file mode 100644 index 00000000..91ca5f6b --- /dev/null +++ b/.github/workflows/build-everything-tgw.yml @@ -0,0 +1,49 @@ +name: Build Everything TGW + +on: + workflow_dispatch: + inputs: + version: + description: 'Version tag of text-generation-webui to build: v3.0' + default: 'v3.0' + required: true + type: string + +permissions: + contents: write + +jobs: + build_release_cuda_windows: + name: CUDA Windows + uses: ./.github/workflows/build-portable-release-cuda.yml + with: + version: ${{ inputs.version }} + config: 'os:windows-2019' + + build_release_cuda_linux: + name: CUDA Linux + uses: ./.github/workflows/build-portable-release-cuda.yml + with: + version: ${{ inputs.version }} + config: 'os:ubuntu-22.04' + + build_release_cpu_windows: + name: CPU Windows + uses: ./.github/workflows/build-portable-release.yml + with: + version: ${{ inputs.version }} + config: 'os:windows-2019' + + build_release_cpu_linux: + name: CPU Linux + uses: ./.github/workflows/build-portable-release.yml + with: + version: ${{ inputs.version }} + config: 'os:ubuntu-22.04' + + build_release_macos: + name: macOS + uses: ./.github/workflows/build-portable-release.yml + with: + version: ${{ inputs.version }} + config: 'os:macos-13,macos-14' diff --git a/.github/workflows/build-portable-release-cuda.yml b/.github/workflows/build-portable-release-cuda.yml new file mode 100644 index 00000000..b3cc587a --- /dev/null +++ b/.github/workflows/build-portable-release-cuda.yml @@ -0,0 +1,182 @@ +name: Build CUDA + +on: + workflow_dispatch: + inputs: + version: + description: 'Version tag of text-generation-webui to build: v3.0' + default: 'v3.0' + required: true + type: string + config: + description: 'Override configurations to build: key1:item1-1,item1-2;key2:item2-1,item2-2' + default: 'Default' + required: false + type: string + exclude: + description: 'Exclude build configurations: key1-1:item1-1,key1-2:item1-2;key2-1:item2-1,key2-2:item2-2' + default: 'None' + required: false + type: string + workflow_call: + inputs: + version: + description: 'Version tag of text-generation-webui to build: v3.0' + default: 'v3.0' + required: true + type: string + config: + description: 'Configurations to build: key1:item1-1,item1-2;key2:item2-1,item2-2' + default: 'Default' + required: false + type: string + exclude: + description: 'Exclude build configurations: key1-1:item1-1,key1-2:item1-2;key2-1:item2-1,key2-2:item2-2' + default: 'None' + required: false + type: string + +permissions: + contents: write + +jobs: + define_matrix: + name: Define Build Matrix + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + defaults: + run: + shell: pwsh + env: + CONFIGIN: ${{ inputs.config }} + EXCLUDEIN: ${{ inputs.exclude }} + + steps: + - name: Define Job Output + id: set-matrix + run: | + $matrix = @{ + 'os' = @('ubuntu-22.04', 'windows-2019') + 'pyver' = @("3.11") + 'avx' = @("AVX2") + 'cuda' = @("11.7", "12.4") + } + + if ($env:CONFIGIN -ne 'Default') {$env:CONFIGIN.split(';').foreach({$matrix[$_.split(':')[0]] = $_.split(':')[1].split(',')})} + + if ($env:EXCLUDEIN -ne 'None') { + $exclusions = @() + $exclusions += $env:EXCLUDEIN.split(';').replace(':','=').replace(',',"`n") | ConvertFrom-StringData + $matrix['exclude'] = $exclusions + } + + $matrixOut = ConvertTo-Json $matrix -Compress + Write-Output ('matrix=' + $matrixOut) >> $env:GITHUB_OUTPUT + + build_wheels: + name: ${{ matrix.os }} ${{ matrix.pyver }} CPU ${{ matrix.avx }} CUDA ${{ matrix.cuda }} + needs: define_matrix + runs-on: ${{ matrix.os }} + strategy: + matrix: ${{ fromJSON(needs.define_matrix.outputs.matrix) }} + defaults: + run: + shell: pwsh + env: + AVXVER: ${{ matrix.avx }} + PCKGVER: ${{ inputs.version }} + + steps: + - uses: actions/checkout@v4 + with: + repository: 'oobabooga/text-generation-webui' + ref: ${{ inputs.version }} + submodules: 'recursive' + + - uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.pyver }} + + - name: Build Package + shell: bash + run: | + rm -rf .git cmd* update_wizard* start_wsl.bat wsl.sh Colab-TextGen-GPU.ipynb docker + + # Define common variables + CUDA_VERSION="${{ matrix.cuda }}" + AVX_SUPPORT="${{ matrix.avx }}" + VERSION="${{ inputs.version }}" + + # 1. Set platform-specific variables + if [[ "$RUNNER_OS" == "Windows" ]]; then + PLATFORM="windows" + PYTHON_URL="https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.11.12+20250409-x86_64-pc-windows-msvc-install_only.tar.gz" + PIP_PATH="portable_env/python.exe -m pip" + PACKAGES_PATH="portable_env/Lib/site-packages" + ZIP_CMD="powershell -Command \"Compress-Archive -Path text-generation-webui -DestinationPath" + rm start_linux.sh start_macos.sh + else + PLATFORM="linux" + PYTHON_URL="https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.11.12+20250409-x86_64-unknown-linux-gnu-install_only.tar.gz" + PIP_PATH="portable_env/bin/python -m pip" + PACKAGES_PATH="portable_env/lib/python3.11/site-packages" + ZIP_CMD="zip -r" + rm start_macos.sh start_windows.bat + fi + + # 2. Download and extract Python + cd .. + echo "Downloading Python for $PLATFORM..." + curl -L -o python-build.tar.gz "$PYTHON_URL" + tar -xzf python-build.tar.gz + mv python text-generation-webui/portable_env + + # 3. Prepare requirements file based on AVX and CUDA + if [[ "$AVX_SUPPORT" == "AVX2" ]]; then + BASE_REQ_FILE="requirements/portable/requirements.txt" + else + BASE_REQ_FILE="requirements/portable/requirements_noavx2.txt" + fi + + # Create CUDA-specific requirements file if needed + cd text-generation-webui + if [[ "$CUDA_VERSION" == "11.7" ]]; then + echo "Creating CUDA 11.7 specific requirements file" + sed 's/cu124/cu117/g' "$BASE_REQ_FILE" > requirements_cuda_temp.txt + REQ_FILE="requirements_cuda_temp.txt" + else + REQ_FILE="$BASE_REQ_FILE" + fi + + # 4. Install packages + echo "Installing Python packages from $REQ_FILE..." + $PIP_PATH install --target="./$PACKAGES_PATH" -r "$REQ_FILE" + + # 5. Clean up + if [[ "$CUDA_VERSION" == "11.7" ]]; then + rm requirements_cuda_temp.txt + fi + + # 6. Create ZIP file + cd .. + ZIP_NAME="textgen-portable-${VERSION}-${PLATFORM}-cuda${CUDA_VERSION}.zip" + echo "Creating archive: $ZIP_NAME" + + if [[ "$RUNNER_OS" == "Windows" ]]; then + powershell -Command "Compress-Archive -Path text-generation-webui -DestinationPath $ZIP_NAME" + else + zip -r "$ZIP_NAME" text-generation-webui + fi + + - name: Upload files to a GitHub release + id: upload-release + uses: svenstaro/upload-release-action@2.7.0 + continue-on-error: true + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: ../textgen-portable-${{ inputs.version }}*.zip + tag: ${{ inputs.version }} + file_glob: true + make_latest: false + overwrite: true diff --git a/.github/workflows/build-portable-release.yml b/.github/workflows/build-portable-release.yml new file mode 100644 index 00000000..bb61eb8f --- /dev/null +++ b/.github/workflows/build-portable-release.yml @@ -0,0 +1,192 @@ +name: Build CPU and macOS + +on: + workflow_dispatch: + inputs: + version: + description: 'Version tag of text-generation-webui to build: v3.0' + default: 'v3.0' + required: true + type: string + config: + description: 'Override configurations to build: key1:item1-1,item1-2;key2:item2-1,item2-2' + default: 'Default' + required: false + type: string + exclude: + description: 'Exclude build configurations: key1-1:item1-1,key1-2:item1-2;key2-1:item2-1,key2-2:item2-2' + default: 'None' + required: false + type: string + workflow_call: + inputs: + version: + description: 'Version tag of text-generation-webui to build: v3.0' + default: 'v3.0' + required: true + type: string + config: + description: 'Configurations to build: key1:item1-1,item1-2;key2:item2-1,item2-2' + default: 'Default' + required: false + type: string + exclude: + description: 'Exclude build configurations: key1-1:item1-1,key1-2:item1-2;key2-1:item2-1,key2-2:item2-2' + default: 'None' + required: false + type: string + +permissions: + contents: write + +jobs: + define_matrix: + name: Define Build Matrix + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + defaults: + run: + shell: pwsh + env: + CONFIGIN: ${{ inputs.config }} + EXCLUDEIN: ${{ inputs.exclude }} + + steps: + - name: Define Job Output + id: set-matrix + run: | + $matrix = @{ + 'os' = @('ubuntu-22.04', 'windows-2019', 'macos-13', 'macos-14') + 'pyver' = @("3.11") + 'avx' = @("AVX2") + } + + if ($env:CONFIGIN -ne 'Default') {$env:CONFIGIN.split(';').foreach({$matrix[$_.split(':')[0]] = $_.split(':')[1].split(',')})} + + if ($env:EXCLUDEIN -ne 'None') { + $exclusions = @() + $exclusions += $env:EXCLUDEIN.split(';').replace(':','=').replace(',',"`n") | ConvertFrom-StringData + $matrix['exclude'] = $exclusions + } + + $matrixOut = ConvertTo-Json $matrix -Compress + Write-Output ('matrix=' + $matrixOut) >> $env:GITHUB_OUTPUT + + build_wheels: + name: ${{ matrix.os }} ${{ matrix.pyver }} CPU ${{ matrix.avx }} + needs: define_matrix + runs-on: ${{ matrix.os }} + strategy: + matrix: ${{ fromJSON(needs.define_matrix.outputs.matrix) }} + defaults: + run: + shell: pwsh + env: + AVXVER: ${{ matrix.avx }} + PCKGVER: ${{ inputs.version }} + + steps: + - uses: actions/checkout@v4 + with: + repository: 'oobabooga/text-generation-webui' + ref: ${{ inputs.version }} + submodules: 'recursive' + + - uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.pyver }} + + - name: Build Package + shell: bash + run: | + rm -rf .git cmd* update_wizard* start_wsl.bat wsl.sh Colab-TextGen-GPU.ipynb docker + + # Define common variables + AVX_SUPPORT="${{ matrix.avx }}" + VERSION="${{ inputs.version }}" + OS_TYPE="${{ matrix.os }}" + + # 1. Set platform-specific variables + if [[ "$RUNNER_OS" == "Windows" ]]; then + PLATFORM="windows-cpu" + PYTHON_URL="https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.11.12+20250409-x86_64-pc-windows-msvc-install_only.tar.gz" + PIP_PATH="portable_env/python.exe -m pip" + PACKAGES_PATH="portable_env/Lib/site-packages" + rm start_linux.sh start_macos.sh + elif [[ "$RUNNER_OS" == "macOS" ]]; then + if [[ "$OS_TYPE" == "macos-13" ]]; then + PLATFORM="macos-x86_64" + PYTHON_URL="https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.11.12+20250409-x86_64-apple-darwin-install_only.tar.gz" + REQ_TYPE="apple_intel" + else + PLATFORM="macos-arm64" + PYTHON_URL="https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.11.12+20250409-aarch64-apple-darwin-install_only.tar.gz" + REQ_TYPE="apple_silicon" + fi + PIP_PATH="portable_env/bin/python -m pip" + PACKAGES_PATH="portable_env/lib/python3.11/site-packages" + rm start_linux.sh start_windows.bat + else + # Linux case + PLATFORM="linux-cpu" + PYTHON_URL="https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.11.12+20250409-x86_64-unknown-linux-gnu-install_only.tar.gz" + PIP_PATH="portable_env/bin/python -m pip" + PACKAGES_PATH="portable_env/lib/python3.11/site-packages" + rm start_macos.sh start_windows.bat + fi + + # 2. Download and extract Python + echo "Downloading Python for $PLATFORM..." + cd .. + curl -L -o python-build.tar.gz "$PYTHON_URL" + tar -xzf python-build.tar.gz + mv python text-generation-webui/portable_env + + # 3. Prepare requirements file based on platform and AVX + cd text-generation-webui + + # Select requirements file based on platform + if [[ "$RUNNER_OS" == "macOS" ]]; then + if [[ "$OS_TYPE" == "macos-13" ]]; then + REQ_FILE="requirements/portable/requirements_apple_intel.txt" + else + REQ_FILE="requirements/portable/requirements_apple_silicon.txt" + fi + else + # For Windows and Linux, check AVX support + if [[ "$AVX_SUPPORT" == "AVX2" ]]; then + REQ_FILE="requirements/portable/requirements_cpu_only.txt" + else + REQ_FILE="requirements/portable/requirements_cpu_only_noavx2.txt" + fi + fi + + echo "Using requirements file: $REQ_FILE" + + # 4. Install packages + echo "Installing Python packages from $REQ_FILE..." + $PIP_PATH install --target="./$PACKAGES_PATH" -r "$REQ_FILE" + + # 5. Create ZIP file + cd .. + ZIP_NAME="textgen-portable-${VERSION}-${PLATFORM}.zip" + echo "Creating archive: $ZIP_NAME" + + if [[ "$RUNNER_OS" == "Windows" ]]; then + powershell -Command "Compress-Archive -Path text-generation-webui -DestinationPath $ZIP_NAME" + else + zip -r "$ZIP_NAME" text-generation-webui + fi + + - name: Upload files to a GitHub release + id: upload-release + uses: svenstaro/upload-release-action@2.7.0 + continue-on-error: true + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: ../textgen-portable-${{ inputs.version }}*.zip + tag: ${{ inputs.version }} + file_glob: true + make_latest: false + overwrite: true diff --git a/one_click.py b/one_click.py index eff2ed9f..04b729eb 100644 --- a/one_click.py +++ b/one_click.py @@ -356,14 +356,18 @@ def update_requirements(initial_installation=False, pull=True): ) torver = torch_version() + requirements_base = os.path.join("requirements", "full") + if "+rocm" in torver: - requirements_file = "requirements_amd" + ("_noavx2" if not cpu_has_avx2() else "") + ".txt" + file_name = f"requirements_amd{'_noavx2' if not cpu_has_avx2() else ''}.txt" elif "+cpu" in torver or "+cxx11" in torver: - requirements_file = "requirements_cpu_only" + ("_noavx2" if not cpu_has_avx2() else "") + ".txt" + file_name = f"requirements_cpu_only{'_noavx2' if not cpu_has_avx2() else ''}.txt" elif is_macos(): - requirements_file = "requirements_apple_" + ("intel" if is_x86_64() else "silicon") + ".txt" + file_name = f"requirements_apple_{'intel' if is_x86_64() else 'silicon'}.txt" else: - requirements_file = "requirements" + ("_noavx2" if not cpu_has_avx2() else "") + ".txt" + file_name = f"requirements{'_noavx2' if not cpu_has_avx2() else ''}.txt" + + requirements_file = os.path.join(requirements_base, file_name) # Load state from JSON file current_commit = get_current_commit() diff --git a/requirements.txt b/requirements/full/requirements.txt similarity index 86% rename from requirements.txt rename to requirements/full/requirements.txt index 9e55bb33..b9afaa07 100644 --- a/requirements.txt +++ b/requirements/full/requirements.txt @@ -30,8 +30,8 @@ sse-starlette==1.6.5 tiktoken # CUDA wheels -https://github.com/oobabooga/llama-cpp-binaries/releases/download/textgen-webui/llama_cpp_binaries-0.2.0+cu124-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" -https://github.com/oobabooga/llama-cpp-binaries/releases/download/textgen-webui/llama_cpp_binaries-0.2.0+cu124-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.3.0/llama_cpp_binaries-0.3.0+cu124-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.3.0/llama_cpp_binaries-0.3.0+cu124-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" https://github.com/oobabooga/exllamav3/releases/download/v0.0.1a3/exllamav3-0.0.1a3+cu124.torch2.6.0-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" https://github.com/oobabooga/exllamav3/releases/download/v0.0.1a3/exllamav3-0.0.1a3+cu124.torch2.6.0-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" https://github.com/oobabooga/exllamav2/releases/download/v0.2.8/exllamav2-0.2.8+cu124.torch2.6.0-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" diff --git a/requirements_amd.txt b/requirements/full/requirements_amd.txt similarity index 82% rename from requirements_amd.txt rename to requirements/full/requirements_amd.txt index ba8bc80c..96cb299d 100644 --- a/requirements_amd.txt +++ b/requirements/full/requirements_amd.txt @@ -29,6 +29,6 @@ sse-starlette==1.6.5 tiktoken # AMD wheels -https://github.com/oobabooga/llama-cpp-binaries/releases/download/rocm/llama_cpp_binaries-0.2.0+rocm6.1.2-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.3.0/llama_cpp_binaries-0.3.0+rocm6.1.2-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" https://github.com/oobabooga/exllamav2/releases/download/v0.2.8/exllamav2-0.2.8+rocm6.1.torch2.6.0-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" https://github.com/oobabooga/exllamav2/releases/download/v0.2.8/exllamav2-0.2.8-py3-none-any.whl; platform_system != "Darwin" and platform_machine != "x86_64" diff --git a/requirements_amd_noavx2.txt b/requirements/full/requirements_amd_noavx2.txt similarity index 82% rename from requirements_amd_noavx2.txt rename to requirements/full/requirements_amd_noavx2.txt index d921fbb0..0f1a4fc2 100644 --- a/requirements_amd_noavx2.txt +++ b/requirements/full/requirements_amd_noavx2.txt @@ -29,6 +29,6 @@ sse-starlette==1.6.5 tiktoken # AMD wheels -https://github.com/oobabooga/llama-cpp-binaries/releases/download/rocm/llama_cpp_binaries-0.2.0+rocm6.1.2avx-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.3.0/llama_cpp_binaries-0.3.0+rocm6.1.2avx-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" https://github.com/oobabooga/exllamav2/releases/download/v0.2.8/exllamav2-0.2.8+rocm6.1.torch2.6.0-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" https://github.com/oobabooga/exllamav2/releases/download/v0.2.8/exllamav2-0.2.8-py3-none-any.whl; platform_system != "Darwin" and platform_machine != "x86_64" diff --git a/requirements_apple_silicon.txt b/requirements/full/requirements_apple_intel.txt similarity index 51% rename from requirements_apple_silicon.txt rename to requirements/full/requirements_apple_intel.txt index 3e7fcd02..8d1e5294 100644 --- a/requirements_apple_silicon.txt +++ b/requirements/full/requirements_apple_intel.txt @@ -29,8 +29,7 @@ sse-starlette==1.6.5 tiktoken # Mac wheels -https://github.com/oobabooga/llama-cpp-binaries/releases/download/metal/llama_cpp_binaries-0.2.0-cp311-cp311-macosx_15_0_arm64.whl; platform_system == "Darwin" and platform_release >= "24.0.0" and platform_release < "25.0.0" and python_version == "3.11" -https://github.com/oobabooga/llama-cpp-binaries/releases/download/metal/llama_cpp_binaries-0.2.0-cp311-cp311-macosx_14_0_arm64.whl; platform_system == "Darwin" and platform_release >= "23.0.0" and platform_release < "24.0.0" and python_version == "3.11" -https://github.com/oobabooga/llama-cpp-binaries/releases/download/metal/llama_cpp_binaries-0.2.0-cp311-cp311-macosx_13_0_arm64.whl; platform_system == "Darwin" and platform_release >= "22.0.0" and platform_release < "23.0.0" and python_version == "3.11" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.3.0/llama_cpp_binaries-0.3.0-cp311-cp311-macosx_15_0_x86_64.whl; platform_system == "Darwin" and platform_release >= "24.0.0" and platform_release < "25.0.0" and python_version == "3.11" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.3.0/llama_cpp_binaries-0.3.0-cp311-cp311-macosx_14_0_x86_64.whl; platform_system == "Darwin" and platform_release >= "23.0.0" and platform_release < "24.0.0" and python_version == "3.11" https://github.com/oobabooga/exllamav3/releases/download/v0.0.1a3/exllamav3-0.0.1a3-py3-none-any.whl https://github.com/oobabooga/exllamav2/releases/download/v0.2.8/exllamav2-0.2.8-py3-none-any.whl diff --git a/requirements_apple_intel.txt b/requirements/full/requirements_apple_silicon.txt similarity index 70% rename from requirements_apple_intel.txt rename to requirements/full/requirements_apple_silicon.txt index 0b92376a..a44ff3cb 100644 --- a/requirements_apple_intel.txt +++ b/requirements/full/requirements_apple_silicon.txt @@ -29,7 +29,8 @@ sse-starlette==1.6.5 tiktoken # Mac wheels -https://github.com/oobabooga/llama-cpp-binaries/releases/download/metal/llama_cpp_binaries-0.2.0-cp311-cp311-macosx_15_0_x86_64.whl; platform_system == "Darwin" and platform_release >= "24.0.0" and platform_release < "25.0.0" and python_version == "3.11" -https://github.com/oobabooga/llama-cpp-binaries/releases/download/metal/llama_cpp_binaries-0.2.0-cp311-cp311-macosx_14_0_x86_64.whl; platform_system == "Darwin" and platform_release >= "23.0.0" and platform_release < "24.0.0" and python_version == "3.11" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.3.0/llama_cpp_binaries-0.3.0-cp311-cp311-macosx_15_0_arm64.whl; platform_system == "Darwin" and platform_release >= "24.0.0" and platform_release < "25.0.0" and python_version == "3.11" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.3.0/llama_cpp_binaries-0.3.0-cp311-cp311-macosx_14_0_arm64.whl; platform_system == "Darwin" and platform_release >= "23.0.0" and platform_release < "24.0.0" and python_version == "3.11" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.3.0/llama_cpp_binaries-0.3.0-cp311-cp311-macosx_13_0_arm64.whl; platform_system == "Darwin" and platform_release >= "22.0.0" and platform_release < "23.0.0" and python_version == "3.11" https://github.com/oobabooga/exllamav3/releases/download/v0.0.1a3/exllamav3-0.0.1a3-py3-none-any.whl https://github.com/oobabooga/exllamav2/releases/download/v0.2.8/exllamav2-0.2.8-py3-none-any.whl diff --git a/requirements_cpu_only.txt b/requirements/full/requirements_cpu_only.txt similarity index 63% rename from requirements_cpu_only.txt rename to requirements/full/requirements_cpu_only.txt index 43d54534..35855162 100644 --- a/requirements_cpu_only.txt +++ b/requirements/full/requirements_cpu_only.txt @@ -29,5 +29,5 @@ sse-starlette==1.6.5 tiktoken # llama.cpp (CPU only, AVX2) -https://github.com/oobabooga/llama-cpp-binaries/releases/download/cpu/llama_cpp_binaries-0.2.0+cpuavx2-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" -https://github.com/oobabooga/llama-cpp-binaries/releases/download/cpu/llama_cpp_binaries-0.2.0+cpuavx2-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.3.0/llama_cpp_binaries-0.3.0+cpuavx2-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.3.0/llama_cpp_binaries-0.3.0+cpuavx2-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" diff --git a/requirements_cpu_only_noavx2.txt b/requirements/full/requirements_cpu_only_noavx2.txt similarity index 63% rename from requirements_cpu_only_noavx2.txt rename to requirements/full/requirements_cpu_only_noavx2.txt index b4444443..0716455e 100644 --- a/requirements_cpu_only_noavx2.txt +++ b/requirements/full/requirements_cpu_only_noavx2.txt @@ -29,5 +29,5 @@ sse-starlette==1.6.5 tiktoken # llama.cpp (CPU only, no AVX2) -https://github.com/oobabooga/llama-cpp-binaries/releases/download/cpu/llama_cpp_binaries-0.2.0+cpuavx-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" -https://github.com/oobabooga/llama-cpp-binaries/releases/download/cpu/llama_cpp_binaries-0.2.0+cpuavx-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.3.0/llama_cpp_binaries-0.3.0+cpuavx-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.3.0/llama_cpp_binaries-0.3.0+cpuavx-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" diff --git a/requirements_noavx2.txt b/requirements/full/requirements_noavx2.txt similarity index 86% rename from requirements_noavx2.txt rename to requirements/full/requirements_noavx2.txt index 351ea83e..98c43b88 100644 --- a/requirements_noavx2.txt +++ b/requirements/full/requirements_noavx2.txt @@ -30,8 +30,8 @@ sse-starlette==1.6.5 tiktoken # CUDA wheels -https://github.com/oobabooga/llama-cpp-binaries/releases/download/textgen-webui/llama_cpp_binaries-0.2.0+cu124avx-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" -https://github.com/oobabooga/llama-cpp-binaries/releases/download/textgen-webui/llama_cpp_binaries-0.2.0+cu124avx-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.3.0/llama_cpp_binaries-0.3.0+cu124avx-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.3.0/llama_cpp_binaries-0.3.0+cu124avx-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" https://github.com/oobabooga/exllamav3/releases/download/v0.0.1a3/exllamav3-0.0.1a3+cu124.torch2.6.0-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" https://github.com/oobabooga/exllamav3/releases/download/v0.0.1a3/exllamav3-0.0.1a3+cu124.torch2.6.0-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" https://github.com/oobabooga/exllamav2/releases/download/v0.2.8/exllamav2-0.2.8+cu124.torch2.6.0-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" diff --git a/requirements_nowheels.txt b/requirements/full/requirements_nowheels.txt similarity index 100% rename from requirements_nowheels.txt rename to requirements/full/requirements_nowheels.txt diff --git a/requirements/portable/requirements.txt b/requirements/portable/requirements.txt new file mode 100644 index 00000000..c3336fc7 --- /dev/null +++ b/requirements/portable/requirements.txt @@ -0,0 +1,19 @@ +fastapi==0.112.4 +gradio==4.37.* +jinja2==3.1.6 +markdown +numpy==1.26.* +pydantic==2.8.2 +pyyaml +requests +rich +tqdm + +# API +flask_cloudflared==0.0.14 +sse-starlette==1.6.5 +tiktoken + +# CUDA wheels +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.3.0/llama_cpp_binaries-0.3.0+cu124-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.3.0/llama_cpp_binaries-0.3.0+cu124-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" diff --git a/requirements/portable/requirements_amd.txt b/requirements/portable/requirements_amd.txt new file mode 100644 index 00000000..4855225f --- /dev/null +++ b/requirements/portable/requirements_amd.txt @@ -0,0 +1,18 @@ +fastapi==0.112.4 +gradio==4.37.* +jinja2==3.1.6 +markdown +numpy==1.26.* +pydantic==2.8.2 +pyyaml +requests +rich +tqdm + +# API +flask_cloudflared==0.0.14 +sse-starlette==1.6.5 +tiktoken + +# AMD wheels +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.3.0/llama_cpp_binaries-0.3.0+rocm6.1.2-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" diff --git a/requirements/portable/requirements_amd_noavx2.txt b/requirements/portable/requirements_amd_noavx2.txt new file mode 100644 index 00000000..f40daa8a --- /dev/null +++ b/requirements/portable/requirements_amd_noavx2.txt @@ -0,0 +1,18 @@ +fastapi==0.112.4 +gradio==4.37.* +jinja2==3.1.6 +markdown +numpy==1.26.* +pydantic==2.8.2 +pyyaml +requests +rich +tqdm + +# API +flask_cloudflared==0.0.14 +sse-starlette==1.6.5 +tiktoken + +# AMD wheels +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.3.0/llama_cpp_binaries-0.3.0+rocm6.1.2avx-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" diff --git a/requirements/portable/requirements_apple_intel.txt b/requirements/portable/requirements_apple_intel.txt new file mode 100644 index 00000000..1ede251e --- /dev/null +++ b/requirements/portable/requirements_apple_intel.txt @@ -0,0 +1,19 @@ +fastapi==0.112.4 +gradio==4.37.* +jinja2==3.1.6 +markdown +numpy==1.26.* +pydantic==2.8.2 +pyyaml +requests +rich +tqdm + +# API +flask_cloudflared==0.0.14 +sse-starlette==1.6.5 +tiktoken + +# Mac wheels +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.3.0/llama_cpp_binaries-0.3.0-cp311-cp311-macosx_15_0_x86_64.whl; platform_system == "Darwin" and platform_release >= "24.0.0" and platform_release < "25.0.0" and python_version == "3.11" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.3.0/llama_cpp_binaries-0.3.0-cp311-cp311-macosx_14_0_x86_64.whl; platform_system == "Darwin" and platform_release >= "23.0.0" and platform_release < "24.0.0" and python_version == "3.11" diff --git a/requirements/portable/requirements_apple_silicon.txt b/requirements/portable/requirements_apple_silicon.txt new file mode 100644 index 00000000..26b68bff --- /dev/null +++ b/requirements/portable/requirements_apple_silicon.txt @@ -0,0 +1,20 @@ +fastapi==0.112.4 +gradio==4.37.* +jinja2==3.1.6 +markdown +numpy==1.26.* +pydantic==2.8.2 +pyyaml +requests +rich +tqdm + +# API +flask_cloudflared==0.0.14 +sse-starlette==1.6.5 +tiktoken + +# Mac wheels +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.3.0/llama_cpp_binaries-0.3.0-cp311-cp311-macosx_15_0_arm64.whl; platform_system == "Darwin" and platform_release >= "24.0.0" and platform_release < "25.0.0" and python_version == "3.11" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.3.0/llama_cpp_binaries-0.3.0-cp311-cp311-macosx_14_0_arm64.whl; platform_system == "Darwin" and platform_release >= "23.0.0" and platform_release < "24.0.0" and python_version == "3.11" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.3.0/llama_cpp_binaries-0.3.0-cp311-cp311-macosx_13_0_arm64.whl; platform_system == "Darwin" and platform_release >= "22.0.0" and platform_release < "23.0.0" and python_version == "3.11" diff --git a/requirements/portable/requirements_cpu_only.txt b/requirements/portable/requirements_cpu_only.txt new file mode 100644 index 00000000..456a0499 --- /dev/null +++ b/requirements/portable/requirements_cpu_only.txt @@ -0,0 +1,19 @@ +fastapi==0.112.4 +gradio==4.37.* +jinja2==3.1.6 +markdown +numpy==1.26.* +pydantic==2.8.2 +pyyaml +requests +rich +tqdm + +# API +flask_cloudflared==0.0.14 +sse-starlette==1.6.5 +tiktoken + +# llama.cpp (CPU only, AVX2) +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.3.0/llama_cpp_binaries-0.3.0+cpuavx2-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.3.0/llama_cpp_binaries-0.3.0+cpuavx2-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" diff --git a/requirements/portable/requirements_cpu_only_noavx2.txt b/requirements/portable/requirements_cpu_only_noavx2.txt new file mode 100644 index 00000000..7cd2dd34 --- /dev/null +++ b/requirements/portable/requirements_cpu_only_noavx2.txt @@ -0,0 +1,19 @@ +fastapi==0.112.4 +gradio==4.37.* +jinja2==3.1.6 +markdown +numpy==1.26.* +pydantic==2.8.2 +pyyaml +requests +rich +tqdm + +# API +flask_cloudflared==0.0.14 +sse-starlette==1.6.5 +tiktoken + +# llama.cpp (CPU only, no AVX2) +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.3.0/llama_cpp_binaries-0.3.0+cpuavx-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.3.0/llama_cpp_binaries-0.3.0+cpuavx-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" diff --git a/requirements/portable/requirements_noavx2.txt b/requirements/portable/requirements_noavx2.txt new file mode 100644 index 00000000..b47b8bbc --- /dev/null +++ b/requirements/portable/requirements_noavx2.txt @@ -0,0 +1,19 @@ +fastapi==0.112.4 +gradio==4.37.* +jinja2==3.1.6 +markdown +numpy==1.26.* +pydantic==2.8.2 +pyyaml +requests +rich +tqdm + +# API +flask_cloudflared==0.0.14 +sse-starlette==1.6.5 +tiktoken + +# CUDA wheels +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.3.0/llama_cpp_binaries-0.3.0+cu124avx-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11" +https://github.com/oobabooga/llama-cpp-binaries/releases/download/v0.3.0/llama_cpp_binaries-0.3.0+cu124avx-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11" diff --git a/requirements/portable/requirements_nowheels.txt b/requirements/portable/requirements_nowheels.txt new file mode 100644 index 00000000..6f9566ba --- /dev/null +++ b/requirements/portable/requirements_nowheels.txt @@ -0,0 +1,15 @@ +fastapi==0.112.4 +gradio==4.37.* +jinja2==3.1.6 +markdown +numpy==1.26.* +pydantic==2.8.2 +pyyaml +requests +rich +tqdm + +# API +flask_cloudflared==0.0.14 +sse-starlette==1.6.5 +tiktoken diff --git a/start_linux.sh b/start_linux.sh index 256604cb..00082f07 100755 --- a/start_linux.sh +++ b/start_linux.sh @@ -2,6 +2,12 @@ cd "$(dirname "${BASH_SOURCE[0]}")" +# Portable install case +if [ -d "portable_env" ]; then + ./portable_env/bin/python3 server.py --api --auto-launch "$@" + exit $? +fi + if [[ "$(pwd)" =~ " " ]]; then echo This script relies on Miniconda which can not be silently installed under a path with spaces. && exit; fi # deactivate existing conda envs as needed to avoid conflicts diff --git a/start_macos.sh b/start_macos.sh index 02f1011a..628f59cc 100755 --- a/start_macos.sh +++ b/start_macos.sh @@ -2,6 +2,12 @@ cd "$(dirname "${BASH_SOURCE[0]}")" +# Portable install case +if [ -d "portable_env" ]; then + ./portable_env/bin/python3 server.py --api --auto-launch --api-port 5005 "$@" + exit $? +fi + if [[ "$(pwd)" =~ " " ]]; then echo This script relies on Miniconda which can not be silently installed under a path with spaces. && exit; fi # deactivate existing conda envs as needed to avoid conflicts diff --git a/start_windows.bat b/start_windows.bat index 2e42d6fa..451b85e0 100755 --- a/start_windows.bat +++ b/start_windows.bat @@ -3,6 +3,12 @@ setlocal enabledelayedexpansion cd /D "%~dp0" +@rem Portable install case +if exist "portable_env" ( + .\portable_env\python.exe server.py --api --auto-launch %* + exit /b %errorlevel% +) + set PATH=%PATH%;%SystemRoot%\system32 echo "%CD%"| findstr /C:" " >nul && echo This script relies on Miniconda which can not be silently installed under a path with spaces. && goto end