vllm_vs_deepspeed.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "machine_shape": "hm",
      "gpuType": "V100",
      "authorship_tag": "ABX9TyMN7lVw3GkokARFSDnx+f5Q",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "da266468f4104a1cb3689258aed4e877": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_663643d6160a42d3bca0d3489260c82c",
              "IPY_MODEL_53fbebe702fb4fb38ab58ed8c76887db",
              "IPY_MODEL_80f4513ff9cd4dff82e66eebfaf83f1c"
            ],
            "layout": "IPY_MODEL_d23b646f770244d18fa56aa652dd0a10"
          }
        },
        "663643d6160a42d3bca0d3489260c82c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_514b4f8937b44c36adc5f4855d220e5f",
            "placeholder": "​",
            "style": "IPY_MODEL_3b3c52e7ad774c3ab32e684c4f573470",
            "value": "Loading checkpoint shards: 100%"
          }
        },
        "53fbebe702fb4fb38ab58ed8c76887db": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_6343f9427bb4429ea88e870f6c4e0d99",
            "max": 2,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_d8cd1a6ff9804070b54cb9a088b0341b",
            "value": 2
          }
        },
        "80f4513ff9cd4dff82e66eebfaf83f1c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_6968d5c590c64e7ab97a35070eb0e234",
            "placeholder": "​",
            "style": "IPY_MODEL_a82f2a45b8df4cc287a48838d1008b4a",
            "value": " 2/2 [00:02&lt;00:00,  1.13s/it]"
          }
        },
        "d23b646f770244d18fa56aa652dd0a10": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "514b4f8937b44c36adc5f4855d220e5f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3b3c52e7ad774c3ab32e684c4f573470": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "6343f9427bb4429ea88e870f6c4e0d99": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d8cd1a6ff9804070b54cb9a088b0341b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "6968d5c590c64e7ab97a35070eb0e234": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a82f2a45b8df4cc287a48838d1008b4a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/jlonge4/gen_ai_utils/blob/main/vllm_vs_deepspeed.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "CCwH3ukJm518"
      },
      "outputs": [],
      "source": [
        "!pip install flash-attn deepspeed vllm"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install transformers"
      ],
      "metadata": {
        "id": "NwdWzWSsNFc8"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#release cuda memory using torch clear cache\n",
        "import torch\n",
        "torch.cuda.empty_cache()"
      ],
      "metadata": {
        "id": "Ky76BnXfquCV"
      },
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "context = \"\"\"Instruction tuning large language models (LLMs) using machine-generated\n",
        "instruction-following data has been shown to improve zero-shot capabilities on\n",
        "new tasks, but the idea is less explored in the multimodal field. We present the\n",
        "first attempt to use language-only GPT-4 to generate multimodal language-image\n",
        "instruction-following data. By instruction tuning on such generated data, we introduce LLaVA: Large Language and Vision Assistant, an end-to-end trained\n",
        "large multimodal model that connects a vision encoder and an LLM for generalpurpose visual and language understanding. To facilitate future research on visual\n",
        "instruction following, we construct two evaluation benchmarks with diverse and\n",
        "challenging application-oriented tasks. Our experiments show that LLaVA demonstrates impressive multimodal chat abilities, sometimes exhibiting the behaviors\n",
        "of multimodal GPT-4 on unseen images/instructions, and yields a 85.1% relative score compared with GPT-4 on a synthetic multimodal instruction-following\n",
        "dataset. When fine-tuned on Science QA, the synergy of LLaVA and GPT-4\n",
        "achieves a new state-of-the-art accuracy of 92.53%. We make GPT-4 generated\n",
        "visual instruction tuning data, our model, and code publicly available.\n",
        "1 Introduction\n",
        "Humans interact with the world through many channels such as vision and language, as each\n",
        "individual channel has a unique advantage in representing and communicating certain concepts, and\n",
        "thus facilitates a better understanding of the world. One of the core aspirations in artificial intelligence\n",
        "is to develop a general-purpose assistant that can effectively follow multi-modal vision-and-language\n",
        "instructions, aligned with human intent to complete various real-world tasks in the wild [4, 27, 26].\n",
        "To this end, the community has witnessed an emergent interest in developing language-augmented\n",
        "foundation vision models [27, 16], with strong capabilities in open-world visual understanding\n",
        "such as classification [40, 21, 57, 54, 39], detection [29, 62, 33], segmentation [25, 63, 58] and\n",
        "captioning [50, 28], as well as visual generation and editing [42, 43, 56, 15, 44, 30]. We refer readers\n",
        "to the Computer Vision in the Wild reading list for a more up-to-date literature compilation [12]. In\n",
        "this line of work, each task is solved independently by one single large vision model, with the task\n",
        "instruction implicitly considered in the model design. Further, language is only utilized to describe\n",
        "the image content. While this allows language to play an important role in mapping visual signals to\n",
        "language semantics—a common channel for human communication, it leads to models that usually\n",
        "have a fixed interface with limited interactivity and adaptability to the user’s instructions.\n",
        "Large language models (LLM), on the other hand, have shown that language can play a wider\n",
        "role: a universal interface for a general-purpose assistant, where various task instructions can be\n",
        "explicitly represented in language and guide the end-to-end trained neural assistant to switch to the\n",
        "task of interest to solve it. For example, the recent success of ChatGPT [35] and GPT-4 [36] have\n",
        "demonstrated the power of aligned LLMs in following human instructions, and have stimulated\n",
        "tremendous interest in developing open-source LLMs. Among them, LLaMA [49] is an opensource LLM that matches the performance of GPT-3. Alpaca [48], Vicuna [9], GPT-4-LLM [38]\n",
        "37th Conference on Neural Information Processing Systems (NeurIPS 2023).\n",
        "arXiv:2304.08485v2 [cs.CV] 11 Dec 2023\n",
        "utilize various machine-generated high-quality instruction-following samples to improve the LLM’s\n",
        "alignment ability, reporting impressive performance compared with proprietary LLMs. Importantly,\n",
        "this line of work is text-only.\n",
        "In this paper, we present visual instruction-tuning, the first attempt to extend instruction-tuning to\n",
        "the language-image multimodal space, to pave the way towards building a general-purpose visual\n",
        "assistant. In particular, our paper makes the following contributions:\n",
        "• Multimodal instruction-following data. One key challenge is the lack of vision-language\n",
        "instruction-following data. We present a data reformation perspective and pipeline to convert\n",
        "image-text pairs into an appropriate instruction-following format, using ChatGPT/GPT-4.\n",
        "• Large multimodal models. We develop a large multimodal model (LMM), by connecting the\n",
        "open-set visual encoder of CLIP [40] with the language decoder Vicuna [9], and fine-tuning\n",
        "end-to-end on our generated instructional vision-language data. Our empirical study validates\n",
        "the effectiveness of using generated data for LMM instruction-tuning, and suggests practical\n",
        "tips for building a general-purpose instruction-following visual agent. When ensembled with\n",
        "GPT-4, our approach achieves SoTA on the Science QA [34] multimodal reasoning dataset.\n",
        "• Multimodal instruction-following benchmark. We present LLaVA-Bench with two challenging\n",
        "benchmarks, with a diverse selection of paired images, instructions and detailed annotations.\n",
        "• Open-source. We release the following assets to the public: the generated multimodal instruction\n",
        "data, the codebase, the model checkpoints, and a visual chat demo.\n",
        "2 Related Work\n",
        "Multimodal Instruction-following Agents. In computer vision, existing works that build instructionfollowing agents can be broadly categorized into two classes: (i) End-to-end trained models, which\n",
        "are separately explored for each specific research topic. For example, the vision-language navigation\n",
        "task [3, 19] and Habitat [47] require the embodied AI agent to follow natural language instructions\n",
        "and take a sequence of actions to complete goals in visual environments. In the image editing domain,\n",
        "given an input image and a written instruction that tells the agent what to do, InstructPix2Pix [6]\n",
        "edits images by following the human instructions. (ii) A system that coordinates various models\n",
        "via LangChain [1] / LLMs [35], such as Visual ChatGPT [53], X-GPT [63], MM-REACT [55],\n",
        "VisProg [18], and ViperGPT [46]. While sharing the same goal in building instruction-following\n",
        "agents, we focus on developing an end-to-end trained language-vision multimodal model for multiple\n",
        "tasks.\n",
        "Instruction Tuning. In the natural language processing (NLP) community, to enable LLMs such\n",
        "as GPT-3 [7], T5 [41], PaLM [10], and OPT [60] to follow natural language instructions and\n",
        "complete real-world tasks, researchers have explored methods for LLM instruction-tuning [37, 52, 51],\n",
        "leading to instruction-tuned counterparts such as InstructGPT [37]/ChatGPT [35], FLAN-T5 [11],\n",
        "FLAN-PaLM [11], and OPT-IML [22], respectively. It turns out that this simple approach can\n",
        "effectively improve the zero- and few-shot generalization abilities of LLMs. It is thus natural\n",
        "to borrow the idea from NLP to computer vision. More broadly, the teacher-student distillation\n",
        "ideas with foundation models have been studied in other topics such as image classification [14].\n",
        "Flamingo [2] can be viewed as the GPT-3 moment in the multimodal domain, due to its strong\n",
        "performance on zero-shot task transfer and in-context-learning. Other LMMs trained on imagetext pairs include BLIP-2 [28], FROMAGe [24], and KOSMOS-1 [20]. PaLM-E [13] is an LMM\n",
        "for embodied AI. Based on the recent “best” open-source LLM LLaMA, OpenFlamingo [5] and\n",
        "LLaMA-Adapter [59] are open-source efforts that enable LLaMA to use image inputs, paving the\n",
        "way to build open-source multimodal LLMs. While these models present promising task transfer\n",
        "generalization performance, they are not explicitly tuned with vision-language instruction data, and\n",
        "their performance in multimodal tasks usually falls short compared to language-only tasks. In this\n",
        "paper, we aim to fill this gap and study its effectiveness. Finally, note that visual instruction tuning\n",
        "is different from visual prompt tuning [23]: the former aims to improve the model’s instructionfollowing abilities, while the latter aims to improve the parameter-efficiency in model adaptation.\n",
        "3 GPT-assisted Visual Instruction Data Generation\n",
        "\"\"\""
      ],
      "metadata": {
        "id": "lActn96PqMfP"
      },
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import torch\n",
        "from transformers import AutoTokenizer, AutoModelForCausalLM\n",
        "from transformers import pipeline\n",
        "from deepspeed.module_inject import HFBertLayerPolicy\n",
        "import deepspeed\n",
        "\n",
        "# Model Repository on huggingface.co\n",
        "model_id='microsoft/phi-2'\n",
        "\n",
        "# load model and tokenizer\n",
        "tokenizer = AutoTokenizer.from_pretrained(model_id)\n",
        "model = AutoModelForCausalLM.from_pretrained(model_id, torch_dtype=\"auto\", trust_remote_code=True)\n",
        "\n",
        "# init deepspeed inference engine\n",
        "ds_model = deepspeed.init_inference(\n",
        "    model=model,      # Transformers models\n",
        "    mp_size=1,        # Number of GPU\n",
        "    dtype=torch.half, # dtype of the weights (fp16)\n",
        "    # injection_policy={\"BertLayer\" : HFBertLayerPolicy}, # replace BertLayer with DS HFBertLayerPolicy\n",
        "    replace_method=\"auto\", # Lets DS autmatically identify the layer to replace\n",
        "    replace_with_kernel_inject=True, # replace the model with the kernel injector\n",
        "    max_tokens=2048,\n",
        ")"
      ],
      "metadata": {
        "id": "Wj-LnUi3neuC",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 156,
          "referenced_widgets": [
            "da266468f4104a1cb3689258aed4e877",
            "663643d6160a42d3bca0d3489260c82c",
            "53fbebe702fb4fb38ab58ed8c76887db",
            "80f4513ff9cd4dff82e66eebfaf83f1c",
            "d23b646f770244d18fa56aa652dd0a10",
            "514b4f8937b44c36adc5f4855d220e5f",
            "3b3c52e7ad774c3ab32e684c4f573470",
            "6343f9427bb4429ea88e870f6c4e0d99",
            "d8cd1a6ff9804070b54cb9a088b0341b",
            "6968d5c590c64e7ab97a35070eb0e234",
            "a82f2a45b8df4cc287a48838d1008b4a"
          ]
        },
        "outputId": "5ca22689-8bda-4cea-9428-f6cd1caad8a7"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "da266468f4104a1cb3689258aed4e877"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[2024-03-30 03:07:14,253] [INFO] [logging.py:96:log_dist] [Rank -1] DeepSpeed info: version=0.14.0, git-hash=unknown, git-branch=unknown\n",
            "[2024-03-30 03:07:14,256] [WARNING] [config_utils.py:69:_process_deprecated_field] Config parameter replace_method is deprecated. This parameter is no longer needed, please remove from your call to DeepSpeed-inference\n",
            "[2024-03-30 03:07:14,257] [WARNING] [config_utils.py:69:_process_deprecated_field] Config parameter mp_size is deprecated use tensor_parallel.tp_size instead\n",
            "[2024-03-30 03:07:14,266] [INFO] [logging.py:96:log_dist] [Rank -1] quantize_bits = 8 mlp_extra_grouping = False, quantize_groups = 1\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "ds_model"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Ua3Ps_Vgq6m2",
        "outputId": "36bac438-ddef-4e1a-8763-d3372d55f8d1"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "InferenceEngine(\n",
              "  (module): PhiForCausalLM(\n",
              "    (model): PhiModel(\n",
              "      (embed_tokens): Embedding(51200, 2560)\n",
              "      (embed_dropout): Dropout(p=0.0, inplace=False)\n",
              "      (layers): ModuleList(\n",
              "        (0-31): 32 x PhiDecoderLayer(\n",
              "          (self_attn): PhiAttention(\n",
              "            (q_proj): Linear(in_features=2560, out_features=2560, bias=True)\n",
              "            (k_proj): Linear(in_features=2560, out_features=2560, bias=True)\n",
              "            (v_proj): Linear(in_features=2560, out_features=2560, bias=True)\n",
              "            (dense): Linear(in_features=2560, out_features=2560, bias=True)\n",
              "            (rotary_emb): PhiRotaryEmbedding()\n",
              "          )\n",
              "          (mlp): PhiMLP(\n",
              "            (activation_fn): NewGELUActivation()\n",
              "            (fc1): Linear(in_features=2560, out_features=10240, bias=True)\n",
              "            (fc2): Linear(in_features=10240, out_features=2560, bias=True)\n",
              "          )\n",
              "          (input_layernorm): LayerNorm((2560,), eps=1e-05, elementwise_affine=True)\n",
              "          (resid_dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "      )\n",
              "      (final_layernorm): LayerNorm((2560,), eps=1e-05, elementwise_affine=True)\n",
              "    )\n",
              "    (lm_head): Linear(in_features=2560, out_features=51200, bias=True)\n",
              "  )\n",
              ")"
            ]
          },
          "metadata": {},
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "question = 'how do Humans interact with the world?'"
      ],
      "metadata": {
        "id": "LgQqQvG27CG2"
      },
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "HF Vanilla Transformers Pipe"
      ],
      "metadata": {
        "id": "pZ9nb67uR9C6"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import pipeline\n",
        "\n",
        "pipe = pipeline(\"text-generation\", model=\"microsoft/phi-2\", trust_remote_code=True, device=0)"
      ],
      "metadata": {
        "id": "bAIUFj42MO7j"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "%%time\n",
        "prompt = f'Instruct:Based on the context {context}, answer the question {question}. Output:'\n",
        "response = pipe(prompt, max_length=2000)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jf85QFniMfgJ",
        "outputId": "99002721-57b1-4696-a918-f36bb58cc7fe"
      },
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "CPU times: user 3.61 s, sys: 133 ms, total: 3.74 s\n",
            "Wall time: 3.73 s\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "13.8GB GPU RAM / 3.73s"
      ],
      "metadata": {
        "id": "l0iOq1iWSPTi"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "response[0]['generated_text'].split('Output:')[1]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 70
        },
        "id": "mzWWs7sXSFbp",
        "outputId": "2ccb2593-45e5-4dda-f1a4-3fa95be5ec00"
      },
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'\\nHumans interact with the world through many channels such as vision and language, as each\\nindividual channel has a unique advantage in representing and communicating certain concepts, and\\nthus facilitates a better understanding of the world.\\n3.1. The Importance of Visual-Language Interaction\\nHumans interact with the world through many channels such as vision'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "DeepSpeed Implementation:"
      ],
      "metadata": {
        "id": "ZfQLOo89R5F2"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "ds_clf = pipeline(\"text-generation\", model=ds_model, tokenizer=tokenizer,device=0, max_length=2000)\n",
        "\n",
        "# Test pipeline\n",
        "example = f'Instruct:Based on the context {context}, answer the question {question}. Output:'"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8mElcsXPrdiE",
        "outputId": "cdbc4019-6457-4545-d219-d2b5396a4f4c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "The model 'InferenceEngine' is not supported for text-generation. Supported models are ['BartForCausalLM', 'BertLMHeadModel', 'BertGenerationDecoder', 'BigBirdForCausalLM', 'BigBirdPegasusForCausalLM', 'BioGptForCausalLM', 'BlenderbotForCausalLM', 'BlenderbotSmallForCausalLM', 'BloomForCausalLM', 'CamembertForCausalLM', 'LlamaForCausalLM', 'CodeGenForCausalLM', 'CpmAntForCausalLM', 'CTRLLMHeadModel', 'Data2VecTextForCausalLM', 'ElectraForCausalLM', 'ErnieForCausalLM', 'FalconForCausalLM', 'FuyuForCausalLM', 'GemmaForCausalLM', 'GitForCausalLM', 'GPT2LMHeadModel', 'GPT2LMHeadModel', 'GPTBigCodeForCausalLM', 'GPTNeoForCausalLM', 'GPTNeoXForCausalLM', 'GPTNeoXJapaneseForCausalLM', 'GPTJForCausalLM', 'LlamaForCausalLM', 'MarianForCausalLM', 'MBartForCausalLM', 'MegaForCausalLM', 'MegatronBertForCausalLM', 'MistralForCausalLM', 'MixtralForCausalLM', 'MptForCausalLM', 'MusicgenForCausalLM', 'MvpForCausalLM', 'OpenLlamaForCausalLM', 'OpenAIGPTLMHeadModel', 'OPTForCausalLM', 'PegasusForCausalLM', 'PersimmonForCausalLM', 'PhiForCausalLM', 'PLBartForCausalLM', 'ProphetNetForCausalLM', 'QDQBertLMHeadModel', 'Qwen2ForCausalLM', 'ReformerModelWithLMHead', 'RemBertForCausalLM', 'RobertaForCausalLM', 'RobertaPreLayerNormForCausalLM', 'RoCBertForCausalLM', 'RoFormerForCausalLM', 'RwkvForCausalLM', 'Speech2Text2ForCausalLM', 'StableLmForCausalLM', 'TransfoXLLMHeadModel', 'TrOCRForCausalLM', 'WhisperForCausalLM', 'XGLMForCausalLM', 'XLMWithLMHeadModel', 'XLMProphetNetForCausalLM', 'XLMRobertaForCausalLM', 'XLMRobertaXLForCausalLM', 'XLNetLMHeadModel', 'XmodForCausalLM', 'PhiForCausalLM'].\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "7.7GB GPU / Inference in 3.14s"
      ],
      "metadata": {
        "id": "uG0K6GpsW0Vo"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "%%time\n",
        "results = ds_clf(example)\n",
        "print(results[0]['generated_text'].split('Output:')[1])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "TLNaZRuTVFNn",
        "outputId": "9c0364ed-7d1b-4f51-c927-4b70bc942c8d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Humans interact with the world through many channels such as vision and language, as each\n",
            "individual channel has a unique advantage in representing and communicating certain concepts, and\n",
            "thus facilitates a better understanding of the world.\n",
            "3.1.\n",
            "Human Interaction with the World\n",
            "Humans interact with the world through many channels such as vision and language\n",
            "CPU times: user 3.13 s, sys: 23.8 ms, total: 3.15 s\n",
            "Wall time: 3.14 s\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "VLLM Implementation:"
      ],
      "metadata": {
        "id": "1Jaz7EgrSCIX"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "sampling_params = SamplingParams(temperature=0.05, top_p=0.95, max_tokens=500)"
      ],
      "metadata": {
        "id": "rwleqi8SQFHy"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from vllm import LLM, SamplingParams\n",
        "llm = LLM(model=\"microsoft/phi-2\")"
      ],
      "metadata": {
        "id": "zJxAdFQN-9BV"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "15.9GB GPU RAM / 1.46s"
      ],
      "metadata": {
        "id": "Pg7IRfY-W8bA"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "completion = llm.generate(f'Instruct:Based on the context {context}, answer the question {question}. Output:', sampling_params)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Y6iuiByC_cSy",
        "outputId": "1b63d3aa-d810-4562-93c6-9409334dadc0"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Processed prompts: 100%|██████████| 1/1 [00:01<00:00,  1.46s/it]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "completion[0].outputs[0].text"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 104
        },
        "id": "X4ky1_VNQrN4",
        "outputId": "2d9c45bd-433e-401f-aadf-31a792262050"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'\\nHumans interact with the world through many channels such as vision and language, as each\\nindividual channel has a unique advantage in representing and communicating certain concepts, and\\nthus facilitates a better understanding of the world.\\n3.1. The Importance of Visual-Language Interaction\\nHumans interact with the world through many channels such as vision and language, as each\\nindividual channel has a unique advantage in representing and communicating certain concepts, and\\nthus facilitates a better understanding of the world.\\n3.2. The Challenge of Visual-Language Interaction\\nHumans interact with the'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 36
        }
      ]
    }
  ]
}