clip_dg_orca.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "8CJ68AV1y4oi"
      },
      "outputs": [],
      "source": [
        "!pip install degirum openvino openvino_tokenizers transformers"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "aC-mBgTYy4oj"
      },
      "outputs": [],
      "source": [
        "import numpy as np\n",
        "import degirum as dg\n",
        "from openvino import compile_model\n",
        "from transformers import AutoTokenizer\n",
        "from openvino_tokenizers import convert_tokenizer\n",
        "\n",
        "text_input = [\"People Running\", \"People Talking\", \"People Laughing\", \"People Dancing\"]\n",
        "all_tokens = compile_model(convert_tokenizer(AutoTokenizer.from_pretrained(\"openai/clip-vit-base-patch32\")))(text_input)[\"input_ids\"]\n",
        "\n",
        "zoo = dg.connect(\"@cloud\", \"https://cs.degirum.com/degirum/clip_dg\",token=\"dg_token\")\n",
        "text_model = zoo.load_model(\"clip_textual--1x77_float_openvino_cpu_3\")\n",
        "\n",
        "padded_tokens = [np.pad(token, (0, 77 - len(token)), mode='constant', constant_values=0).astype(np.int64) for token in all_tokens]\n",
        "text_features = np.array([text_model(np.reshape(token, (1,1,1,77))).results[0][\"data\"][0] for token in padded_tokens], dtype=np.float32)\n",
        "\n",
        "# Initialize logit scale\n",
        "logit_scale = np.exp(np.log(1 / 0.07)).astype(np.float32)\n",
        "logit_scale_exp_array = np.array(logit_scale, dtype=np.float32)\n",
        "\n",
        "# Connect to Degirum and load CLIP model\n",
        "rn50_model = zoo.load_model('clip_RN50--224x224_float_n2x_orca1_2', input_image_format=\"RAW\", input_pad_method=\"crop-last\", image_backend=\"pil\")\n",
        "transformer_model = zoo.load_model('clip_transformer--7x7_float_openvino_cpu_1')\n",
        "\n",
        "# Get image features\n",
        "rn50_res = rn50_model(\"dance.jfif\").results[0][\"data\"]\n",
        "final_res = transformer_model(rn50_res).results[0][\"data\"]\n",
        "image_features_array = np.array(final_res, dtype=np.float32).reshape(1, -1)\n",
        "\n",
        "# Calculate logits using numpy\n",
        "logits_per_image = logit_scale_exp_array * np.matmul(image_features_array, text_features.T)\n",
        "logits_per_image_2d = logits_per_image.reshape(-1, logits_per_image.shape[-1])\n",
        "logits_per_text = logits_per_image_2d.T\n",
        "\n",
        "# Softmax and print results\n",
        "softmax_output = np.exp(logits_per_image_2d) / np.sum(np.exp(logits_per_image_2d), axis=1, keepdims=True)\n",
        "print(softmax_output)\n"
      ]
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "tenv",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.10.6"
    },
    "colab": {
      "provenance": []
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}