{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "private_outputs": true,
      "provenance": [],
      "gpuType": "T4"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "# LLaMA 3.3 70B Setup\n",
        "\n",
        "After running both cells, a public gradio URL will appear at the bottom in around 10 minutes."
      ],
      "metadata": {
        "id": "MFQl6-FjSYtY"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#@title 1. Keep Colab Connection Alive\n",
        "%%html\n",
        "<audio src=\"https://oobabooga.github.io/silence.m4a\" controls>"
      ],
      "metadata": {
        "id": "f7TVVj_z4flw",
        "cellView": "form"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#@title 2. Launch LLaMA 3.3\n",
        "import os\n",
        "from pathlib import Path\n",
        "\n",
        "os.environ.pop('PYTHONPATH', None)\n",
        "\n",
        "if Path.cwd().name != 'text-generation-webui':\n",
        "  !git clone https://github.com/oobabooga/text-generation-webui\n",
        "  %cd text-generation-webui\n",
        "  !GPU_CHOICE=A \\\n",
        "  USE_CUDA118=FALSE \\\n",
        "  LAUNCH_AFTER_INSTALL=FALSE \\\n",
        "  INSTALL_EXTENSIONS=FALSE \\\n",
        "  ./start_linux.sh\n",
        "\n",
        "model_url = \"https://huggingface.co/meta-llama/Llama-3.3-70B-Instruct\"\n",
        "branch = \"main\"\n",
        "command_line_flags = \"--n-gpu-layers 256 --load-in-4bit --use_double_quant --no_flash_attn\"\n",
        "api = False\n",
        "\n",
        "if api:\n",
        "  command_line_flags += \" --api --public-api\"\n",
        "\n",
        "if not model_url.startswith('http'):\n",
        "    model_url = 'https://huggingface.co/' + model_url\n",
        "\n",
        "url_parts = model_url.strip('/').split('/')\n",
        "output_folder = f\"{url_parts[-2]}_{url_parts[-1]}\"\n",
        "\n",
        "if branch.strip() not in ['', 'main']:\n",
        "    output_folder += f\"_{branch}\"\n",
        "    !python download-model.py {model_url} --branch {branch}\n",
        "else:\n",
        "    !python download-model.py {model_url}\n",
        "\n",
        "cmd = f\"./start_linux.sh {command_line_flags} --share\"\n",
        "if output_folder:\n",
        "    cmd += f\" --model {output_folder}\"\n",
        "\n",
        "!$cmd"
      ],
      "metadata": {
        "id": "LGQ8BiMuXMDG",
        "cellView": "form" 
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}