textgen/Colab-TextGen-GPU.ipynb

oobabooga/text-generation-webui

After running both cells, a public gradio URL will appear at the bottom in around 10 minutes. You can optionally generate an API link.

• Project page: https://github.com/oobabooga/text-generation-webui
• Gradio server status: https://status.gradio.app/

In [ ]:

#@title 1. Keep this tab alive to prevent Colab from disconnecting you { display-mode: "form" }

#@markdown Press play on the music player that will appear below:
%%html
<audio src="https://oobabooga.github.io/silence.m4a" controls>

In [ ]:

#@title 2. Launch the web UI

#@markdown If unsure about the branch, write "main" or leave it blank.

import os
from pathlib import Path

os.environ.pop('PYTHONPATH', None)
os.environ.pop('MPLBACKEND', None)

if Path.cwd().name != 'text-generation-webui':
  print("\033[1;32;1m\n --> Installing the web UI. This will take a while, but after the initial setup, you can download and test as many models as you like.\033[0;37;0m\n")

  !git clone https://github.com/oobabooga/text-generation-webui
  %cd text-generation-webui

  # Install the project in an isolated environment
  !GPU_CHOICE=A \
  LAUNCH_AFTER_INSTALL=FALSE \
  INSTALL_EXTENSIONS=FALSE \
  ./start_linux.sh

# Parameters
model_url = "https://huggingface.co/turboderp/gemma-2-9b-it-exl2" #@param {type:"string"}
branch = "8.0bpw" #@param {type:"string"}
command_line_flags = "--n-gpu-layers 128 --load-in-4bit --use_double_quant --no_flash_attn" #@param {type:"string"}
api = False #@param {type:"boolean"}

if api:
  for param in ['--api', '--public-api']:
    if param not in command_line_flags:
      command_line_flags += f" {param}"

model_url = model_url.strip()
if model_url != "":
    if not model_url.startswith('http'):
        model_url = 'https://huggingface.co/' + model_url

    # Download the model
    url_parts = model_url.strip('/').strip().split('/')
    output_folder = f"{url_parts[-2]}_{url_parts[-1]}"
    branch = branch.strip('"\' ')
    if branch.strip() not in ['', 'main']:
        output_folder += f"_{branch}"
        !python download-model.py {model_url} --branch {branch}
    else:
        !python download-model.py {model_url}
else:
    output_folder = ""

# Start the web UI
cmd = f"./start_linux.sh {command_line_flags} --share"
if output_folder != "":
    cmd += f" --model {output_folder}"

!$cmd