clip_dg.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Y0myB8lzhQ6L"
      },
      "outputs": [],
      "source": [
        "!pip install git+https://github.com/openai/CLIP.git degirum"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!wget https://github.com/DeGirum/clip_degirum/raw/main/dance.jfif"
      ],
      "metadata": {
        "id": "rbjbYzo1d5D_"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "60bWVpn9hQ6Q"
      },
      "outputs": [],
      "source": [
        "import clip\n",
        "import torch\n",
        "import numpy as np\n",
        "import degirum as dg\n",
        "from torch import nn\n",
        "\n",
        "text_prompts = [\"People Running\", \"People Talking\", \"People Laughing\", \"People Dancing\"]\n",
        "model, preprocess = clip.load(\"RN50\", device=\"cpu\")\n",
        "text = clip.tokenize(text_prompts).to(\"cpu\")\n",
        "text_features = model.encode_text(text)\n",
        "\n",
        "zoo = dg.connect(\"@cloud\", \"https://cs.degirum.com/degirum/clip_dg\", token=\"token\")\n",
        "dg_model = zoo.load_model('clip--224x224_float_tensorrt_gpu_1', input_image_format=\"RAW\", input_pad_method=\"crop-last\", image_backend=\"pil\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CBPoTJ2qhQ6F"
      },
      "source": [
        "Clip demo with Torch dependencies"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# torch\n",
        "# Initialize logit scale\n",
        "logit_scale = nn.Parameter(torch.ones([]) * np.log(1 / 0.07)).exp()\n",
        "logit_scale_exp_tensor = torch.tensor(logit_scale.detach().numpy(), dtype=torch.float32)\n",
        "\n",
        "# Get image features\n",
        "final_res = dg_model(\"dance.jfif\").results[0][\"data\"]\n",
        "image_features_tensor = torch.tensor(final_res, dtype=torch.float32).unsqueeze(0)\n",
        "text_features_tensor = torch.squeeze(text_features, dim=1)\n",
        "\n",
        "# Calculate logits\n",
        "logits_per_image = logit_scale_exp_tensor * torch.matmul(image_features_tensor, text_features_tensor.t())\n",
        "logits_per_image_2d = logits_per_image.view(-1, logits_per_image.size(-1))\n",
        "logits_per_text = logits_per_image_2d.t()\n",
        "\n",
        "# Softmax and print results\n",
        "softmax_output = torch.softmax(logits_per_image_2d, dim=-1).detach().cpu().numpy()\n",
        "print(softmax_output)"
      ],
      "metadata": {
        "id": "9Bc2Msgaay6M"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hZP2VMgshQ6T"
      },
      "source": [
        "Clip demo with non Torch dependencies"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Vt0V-wPhhQ6V",
        "outputId": "5d0f8f15-a6ba-4327-ae42-96ba0379c914"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[[6.2032657e-10 3.2358432e-08 1.0768239e-09 1.0000000e+00]]\n"
          ]
        }
      ],
      "source": [
        "# non torch\n",
        "# Initialize logit scale\n",
        "logit_scale = np.exp(np.log(1 / 0.07)).astype(np.float32)\n",
        "logit_scale_exp_array = np.array(logit_scale, dtype=np.float32)\n",
        "\n",
        "# Get image features\n",
        "final_res = dg_model(\"dance.jfif\").results[0][\"data\"]\n",
        "image_features_array = np.array(final_res, dtype=np.float32).reshape(1, -1)\n",
        "text_features_array = text_features.squeeze().detach().numpy()\n",
        "\n",
        "# Calculate logits using numpy\n",
        "logits_per_image = logit_scale_exp_array * np.matmul(image_features_array, text_features_array.T)\n",
        "logits_per_image_2d = logits_per_image.reshape(-1, logits_per_image.shape[-1])\n",
        "logits_per_text = logits_per_image_2d.T\n",
        "\n",
        "# Softmax and print results\n",
        "softmax_output = np.exp(logits_per_image_2d) / np.sum(np.exp(logits_per_image_2d), axis=1, keepdims=True)\n",
        "print(softmax_output)\n"
      ]
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "tenv",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.10.6"
    },
    "colab": {
      "provenance": []
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}