oobabooga/text-generation-webui
1
0
Fork 0
mirror of https://github.com/oobabooga/text-generation-webui.git synced 2026-04-06 23:23:43 +00:00
Code Issues Projects Releases Packages Wiki Activity

Update Colab notebook: new default model, direct GGUF URL support

Browse source
This commit is contained in:
oobabooga 2026-03-06 06:52:49 -08:00
parent e7e0df0101
commit 3bab7fbfd4
2 changed files with 38 additions and 16 deletions
Download patch file Download diff file Expand all files Collapse all files

34
Colab-TextGen-GPU.ipynb
View file

@ -51,7 +51,7 @@
"source": [
"#@title 2. Launch the web UI\n",
"\n",
"#@markdown If unsure about the branch, write \"main\" or leave it blank.\n",
"#@markdown You can provide a direct GGUF link or a Hugging Face model URL.\n",
"\n",
"import os\n",
"from pathlib import Path\n",
@ -72,9 +72,9 @@
" ./start_linux.sh\n",
"\n",
"# Parameters\n",
"model_url = \"https://huggingface.co/turboderp/gemma-2-9b-it-exl2\" #@param {type:\"string\"}\n",
"branch = \"8.0bpw\" #@param {type:\"string\"}\n",
"command_line_flags = \"--n-gpu-layers 128 --load-in-4bit --use_double_quant --no_flash_attn\" #@param {type:\"string\"}\n",
"model_url = \"https://huggingface.co/unsloth/Qwen3.5-9B-GGUF/resolve/main/Qwen3.5-9B-Q4_K_M.gguf\" #@param {type:\"string\"}\n",
"branch = \"\" #@param {type:\"string\"}\n",
"command_line_flags = \"--load-in-4bit --use_double_quant\" #@param {type:\"string\"}\n",
"api = False #@param {type:\"boolean\"}\n",
"\n",
"if api:\n",
@ -83,26 +83,28 @@
" command_line_flags += f\" {param}\"\n",
"\n",
"model_url = model_url.strip()\n",
"model_name = \"\"\n",
"if model_url != \"\":\n",
" if not model_url.startswith('http'):\n",
" model_url = 'https://huggingface.co/' + model_url\n",
"\n",
" # Download the model\n",
" url_parts = model_url.strip('/').strip().split('/')\n",
" output_folder = f\"{url_parts[-2]}_{url_parts[-1]}\"\n",
" branch = branch.strip('\"\\' ')\n",
" if branch.strip() not in ['', 'main']:\n",
" output_folder += f\"_{branch}\"\n",
" !python download-model.py {model_url} --branch {branch}\n",
" else:\n",
" branch = branch.strip()\n",
" if '/resolve/' in model_url:\n",
" model_name = model_url.split('?')[0].split('/')[-1]\n",
" !python download-model.py {model_url}\n",
"else:\n",
" output_folder = \"\"\n",
" else:\n",
" url_parts = model_url.strip('/').split('/')\n",
" model_name = f\"{url_parts[-2]}_{url_parts[-1]}\"\n",
" if branch not in ['', 'main']:\n",
" model_name += f\"_{branch}\"\n",
" !python download-model.py {model_url} --branch {branch}\n",
" else:\n",
" !python download-model.py {model_url}\n",
"\n",
"# Start the web UI\n",
"cmd = f\"./start_linux.sh {command_line_flags} --share\"\n",
"if output_folder != \"\":\n",
" cmd += f\" --model {output_folder}\"\n",
"if model_name != \"\":\n",
" cmd += f\" --model {model_name}\"\n",
"\n",
"!$cmd"
],