LangChain2.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EyjJJpvQYD0_"
      },
      "source": [
        "## Install packages"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QhI2Die_YCMq",
        "outputId": "012262e3-95a5-4900-9ad8-0bd318be195b"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting langchain\n",
            "  Downloading langchain-0.0.125-py3-none-any.whl (443 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m443.2/443.2 KB\u001b[0m \u001b[31m6.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting aiohttp<4.0.0,>=3.8.3\n",
            "  Downloading aiohttp-3.8.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.0 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.0/1.0 MB\u001b[0m \u001b[31m30.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: SQLAlchemy<2,>=1 in /usr/local/lib/python3.9/dist-packages (from langchain) (1.4.47)\n",
            "Requirement already satisfied: pydantic<2,>=1 in /usr/local/lib/python3.9/dist-packages (from langchain) (1.10.7)\n",
            "Requirement already satisfied: PyYAML>=5.4.1 in /usr/local/lib/python3.9/dist-packages (from langchain) (6.0)\n",
            "Collecting pyowm<4.0.0,>=3.3.0\n",
            "  Downloading pyowm-3.3.0-py3-none-any.whl (4.5 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m4.5/4.5 MB\u001b[0m \u001b[31m54.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: requests<3,>=2 in /usr/local/lib/python3.9/dist-packages (from langchain) (2.27.1)\n",
            "Requirement already satisfied: tenacity<9.0.0,>=8.1.0 in /usr/local/lib/python3.9/dist-packages (from langchain) (8.2.2)\n",
            "Requirement already satisfied: numpy<2,>=1 in /usr/local/lib/python3.9/dist-packages (from langchain) (1.22.4)\n",
            "Collecting dataclasses-json<0.6.0,>=0.5.7\n",
            "  Downloading dataclasses_json-0.5.7-py3-none-any.whl (25 kB)\n",
            "Collecting aiosignal>=1.1.2\n",
            "  Downloading aiosignal-1.3.1-py3-none-any.whl (7.6 kB)\n",
            "Collecting frozenlist>=1.1.1\n",
            "  Downloading frozenlist-1.3.3-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (158 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m158.8/158.8 KB\u001b[0m \u001b[31m7.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: charset-normalizer<4.0,>=2.0 in /usr/local/lib/python3.9/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (2.0.12)\n",
            "Collecting async-timeout<5.0,>=4.0.0a3\n",
            "  Downloading async_timeout-4.0.2-py3-none-any.whl (5.8 kB)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.9/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (22.2.0)\n",
            "Collecting multidict<7.0,>=4.5\n",
            "  Downloading multidict-6.0.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (114 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m114.2/114.2 KB\u001b[0m \u001b[31m3.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting yarl<2.0,>=1.0\n",
            "  Downloading yarl-1.8.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (264 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m264.6/264.6 KB\u001b[0m \u001b[31m13.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting typing-inspect>=0.4.0\n",
            "  Downloading typing_inspect-0.8.0-py3-none-any.whl (8.7 kB)\n",
            "Collecting marshmallow<4.0.0,>=3.3.0\n",
            "  Downloading marshmallow-3.19.0-py3-none-any.whl (49 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m49.1/49.1 KB\u001b[0m \u001b[31m1.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting marshmallow-enum<2.0.0,>=1.5.1\n",
            "  Downloading marshmallow_enum-1.5.1-py2.py3-none-any.whl (4.2 kB)\n",
            "Requirement already satisfied: typing-extensions>=4.2.0 in /usr/local/lib/python3.9/dist-packages (from pydantic<2,>=1->langchain) (4.5.0)\n",
            "Requirement already satisfied: PySocks<2,>=1.7.1 in /usr/local/lib/python3.9/dist-packages (from pyowm<4.0.0,>=3.3.0->langchain) (1.7.1)\n",
            "Collecting geojson<3,>=2.3.0\n",
            "  Downloading geojson-2.5.0-py2.py3-none-any.whl (14 kB)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.9/dist-packages (from requests<3,>=2->langchain) (2022.12.7)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.9/dist-packages (from requests<3,>=2->langchain) (3.4)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.9/dist-packages (from requests<3,>=2->langchain) (1.26.15)\n",
            "Requirement already satisfied: greenlet!=0.4.17 in /usr/local/lib/python3.9/dist-packages (from SQLAlchemy<2,>=1->langchain) (2.0.2)\n",
            "Requirement already satisfied: packaging>=17.0 in /usr/local/lib/python3.9/dist-packages (from marshmallow<4.0.0,>=3.3.0->dataclasses-json<0.6.0,>=0.5.7->langchain) (23.0)\n",
            "Collecting mypy-extensions>=0.3.0\n",
            "  Downloading mypy_extensions-1.0.0-py3-none-any.whl (4.7 kB)\n",
            "Installing collected packages: geojson, mypy-extensions, multidict, marshmallow, frozenlist, async-timeout, yarl, typing-inspect, marshmallow-enum, aiosignal, pyowm, dataclasses-json, aiohttp, langchain\n",
            "Successfully installed aiohttp-3.8.4 aiosignal-1.3.1 async-timeout-4.0.2 dataclasses-json-0.5.7 frozenlist-1.3.3 geojson-2.5.0 langchain-0.0.125 marshmallow-3.19.0 marshmallow-enum-1.5.1 multidict-6.0.4 mypy-extensions-1.0.0 pyowm-3.3.0 typing-inspect-0.8.0 yarl-1.8.2\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting openai\n",
            "  Downloading openai-0.27.2-py3-none-any.whl (70 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m70.1/70.1 KB\u001b[0m \u001b[31m2.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: requests>=2.20 in /usr/local/lib/python3.9/dist-packages (from openai) (2.27.1)\n",
            "Requirement already satisfied: aiohttp in /usr/local/lib/python3.9/dist-packages (from openai) (3.8.4)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.9/dist-packages (from openai) (4.65.0)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.9/dist-packages (from requests>=2.20->openai) (1.26.15)\n",
            "Requirement already satisfied: charset-normalizer~=2.0.0 in /usr/local/lib/python3.9/dist-packages (from requests>=2.20->openai) (2.0.12)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.9/dist-packages (from requests>=2.20->openai) (2022.12.7)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.9/dist-packages (from requests>=2.20->openai) (3.4)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.9/dist-packages (from aiohttp->openai) (22.2.0)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.9/dist-packages (from aiohttp->openai) (1.3.3)\n",
            "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.9/dist-packages (from aiohttp->openai) (4.0.2)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.9/dist-packages (from aiohttp->openai) (1.3.1)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.9/dist-packages (from aiohttp->openai) (6.0.4)\n",
            "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.9/dist-packages (from aiohttp->openai) (1.8.2)\n",
            "Installing collected packages: openai\n",
            "Successfully installed openai-0.27.2\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m177.2/177.2 KB\u001b[0m \u001b[31m5.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m12.2/12.2 MB\u001b[0m \u001b[31m44.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m283.7/283.7 KB\u001b[0m \u001b[31m8.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m58.3/58.3 KB\u001b[0m \u001b[31m1.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h"
          ]
        }
      ],
      "source": [
        "!pip install langchain\n",
        "!pip install openai\n",
        "!pip install -qU pinecone-client pandas"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CPDEzSFiZ_IW",
        "outputId": "d165a577-5fba-41ef-d580-c3dfacb5c582"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting unstructured\n",
            "  Downloading unstructured-0.5.7.tar.gz (1.3 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m23.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting argilla\n",
            "  Downloading argilla-1.5.0-py3-none-any.whl (2.0 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.0/2.0 MB\u001b[0m \u001b[31m59.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: lxml in /usr/local/lib/python3.9/dist-packages (from unstructured) (4.9.2)\n",
            "Requirement already satisfied: nltk in /usr/local/lib/python3.9/dist-packages (from unstructured) (3.8.1)\n",
            "Requirement already satisfied: openpyxl in /usr/local/lib/python3.9/dist-packages (from unstructured) (3.0.10)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.9/dist-packages (from unstructured) (1.5.3)\n",
            "Requirement already satisfied: pillow in /usr/local/lib/python3.9/dist-packages (from unstructured) (8.4.0)\n",
            "Collecting pypandoc\n",
            "  Downloading pypandoc-1.11-py3-none-any.whl (20 kB)\n",
            "Collecting python-docx\n",
            "  Downloading python-docx-0.8.11.tar.gz (5.6 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m5.6/5.6 MB\u001b[0m \u001b[31m80.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting python-pptx\n",
            "  Downloading python-pptx-0.6.21.tar.gz (10.1 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m10.1/10.1 MB\u001b[0m \u001b[31m70.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting python-magic\n",
            "  Downloading python_magic-0.4.27-py2.py3-none-any.whl (13 kB)\n",
            "Requirement already satisfied: markdown in /usr/local/lib/python3.9/dist-packages (from unstructured) (3.4.3)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.9/dist-packages (from unstructured) (2.27.1)\n",
            "Requirement already satisfied: certifi>=2022.12.07 in /usr/local/lib/python3.9/dist-packages (from unstructured) (2022.12.7)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.9/dist-packages (from argilla->unstructured) (23.0)\n",
            "Collecting backoff\n",
            "  Downloading backoff-2.2.1-py3-none-any.whl (15 kB)\n",
            "Collecting rich<=13.0.1\n",
            "  Downloading rich-13.0.1-py3-none-any.whl (238 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m238.1/238.1 KB\u001b[0m \u001b[31m21.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting monotonic\n",
            "  Downloading monotonic-1.6-py2.py3-none-any.whl (8.2 kB)\n",
            "Requirement already satisfied: numpy<1.24.0 in /usr/local/lib/python3.9/dist-packages (from argilla->unstructured) (1.22.4)\n",
            "Collecting deprecated~=1.2.0\n",
            "  Downloading Deprecated-1.2.13-py2.py3-none-any.whl (9.6 kB)\n",
            "Collecting httpx<0.24,>=0.15\n",
            "  Downloading httpx-0.23.3-py3-none-any.whl (71 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m71.5/71.5 KB\u001b[0m \u001b[31m8.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: tqdm>=4.27.0 in /usr/local/lib/python3.9/dist-packages (from argilla->unstructured) (4.65.0)\n",
            "Requirement already satisfied: pydantic>=1.7.1 in /usr/local/lib/python3.9/dist-packages (from argilla->unstructured) (1.10.7)\n",
            "Collecting wrapt<1.15,>=1.13\n",
            "  Downloading wrapt-1.14.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (77 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m77.8/77.8 KB\u001b[0m \u001b[31m8.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.9/dist-packages (from pandas->unstructured) (2022.7.1)\n",
            "Requirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.9/dist-packages (from pandas->unstructured) (2.8.2)\n",
            "Requirement already satisfied: importlib-metadata>=4.4 in /usr/local/lib/python3.9/dist-packages (from markdown->unstructured) (6.1.0)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.9/dist-packages (from nltk->unstructured) (1.1.1)\n",
            "Requirement already satisfied: regex>=2021.8.3 in /usr/local/lib/python3.9/dist-packages (from nltk->unstructured) (2022.10.31)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.9/dist-packages (from nltk->unstructured) (8.1.3)\n",
            "Requirement already satisfied: et-xmlfile in /usr/local/lib/python3.9/dist-packages (from openpyxl->unstructured) (1.1.0)\n",
            "Collecting XlsxWriter>=0.5.7\n",
            "  Downloading XlsxWriter-3.0.9-py3-none-any.whl (152 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m152.8/152.8 KB\u001b[0m \u001b[31m16.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: charset-normalizer~=2.0.0 in /usr/local/lib/python3.9/dist-packages (from requests->unstructured) (2.0.12)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.9/dist-packages (from requests->unstructured) (3.4)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.9/dist-packages (from requests->unstructured) (1.26.15)\n",
            "Collecting sniffio\n",
            "  Downloading sniffio-1.3.0-py3-none-any.whl (10 kB)\n",
            "Collecting httpcore<0.17.0,>=0.15.0\n",
            "  Downloading httpcore-0.16.3-py3-none-any.whl (69 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m69.6/69.6 KB\u001b[0m \u001b[31m7.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting rfc3986[idna2008]<2,>=1.3\n",
            "  Downloading rfc3986-1.5.0-py2.py3-none-any.whl (31 kB)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.9/dist-packages (from importlib-metadata>=4.4->markdown->unstructured) (3.15.0)\n",
            "Requirement already satisfied: typing-extensions>=4.2.0 in /usr/local/lib/python3.9/dist-packages (from pydantic>=1.7.1->argilla->unstructured) (4.5.0)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.9/dist-packages (from python-dateutil>=2.8.1->pandas->unstructured) (1.16.0)\n",
            "Collecting commonmark<0.10.0,>=0.9.0\n",
            "  Downloading commonmark-0.9.1-py2.py3-none-any.whl (51 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m51.1/51.1 KB\u001b[0m \u001b[31m5.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: pygments<3.0.0,>=2.6.0 in /usr/local/lib/python3.9/dist-packages (from rich<=13.0.1->argilla->unstructured) (2.14.0)\n",
            "Collecting anyio<5.0,>=3.0\n",
            "  Downloading anyio-3.6.2-py3-none-any.whl (80 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m80.6/80.6 KB\u001b[0m \u001b[31m8.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting h11<0.15,>=0.13\n",
            "  Downloading h11-0.14.0-py3-none-any.whl (58 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m58.3/58.3 KB\u001b[0m \u001b[31m6.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hBuilding wheels for collected packages: unstructured, python-docx, python-pptx\n",
            "  Building wheel for unstructured (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for unstructured: filename=unstructured-0.5.7-py3-none-any.whl size=1316553 sha256=fb2bf559b2099ce2a53c69f63b3a590f35bcb9b1910e41c07b1e00ac12507f9c\n",
            "  Stored in directory: /root/.cache/pip/wheels/f1/86/d8/4f1eb1c30896c9d5527ee29578470fd7c1c1980dcbcb6c3b9d\n",
            "  Building wheel for python-docx (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for python-docx: filename=python_docx-0.8.11-py3-none-any.whl size=184505 sha256=243fc806b2fc1c703d52f35d774a665e8aef82f5b9329613f2e046397790dc9c\n",
            "  Stored in directory: /root/.cache/pip/wheels/83/8b/7c/09ae60c42c7ba4ed2dddaf2b8b9186cb105255856d6ed3dba5\n",
            "  Building wheel for python-pptx (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for python-pptx: filename=python_pptx-0.6.21-py3-none-any.whl size=470949 sha256=7610cec732cf8cce543bb8a120b7859b29173858d0f2a0b0c5f57898311e8fd3\n",
            "  Stored in directory: /root/.cache/pip/wheels/0e/4a/ed/9653bc799915f52dce3f04d14946fbd85cce9c3cdedc9cfa71\n",
            "Successfully built unstructured python-docx python-pptx\n",
            "Installing collected packages: rfc3986, monotonic, commonmark, XlsxWriter, wrapt, sniffio, rich, python-magic, python-docx, pypandoc, h11, backoff, python-pptx, deprecated, anyio, httpcore, httpx, argilla, unstructured\n",
            "  Attempting uninstall: wrapt\n",
            "    Found existing installation: wrapt 1.15.0\n",
            "    Uninstalling wrapt-1.15.0:\n",
            "      Successfully uninstalled wrapt-1.15.0\n",
            "  Attempting uninstall: rich\n",
            "    Found existing installation: rich 13.3.3\n",
            "    Uninstalling rich-13.3.3:\n",
            "      Successfully uninstalled rich-13.3.3\n",
            "Successfully installed XlsxWriter-3.0.9 anyio-3.6.2 argilla-1.5.0 backoff-2.2.1 commonmark-0.9.1 deprecated-1.2.13 h11-0.14.0 httpcore-0.16.3 httpx-0.23.3 monotonic-1.6 pypandoc-1.11 python-docx-0.8.11 python-magic-0.4.27 python-pptx-0.6.21 rfc3986-1.5.0 rich-13.0.1 sniffio-1.3.0 unstructured-0.5.7 wrapt-1.14.1\n"
          ]
        }
      ],
      "source": [
        "!pip install unstructured"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "ygmv-QiGaSK_",
        "outputId": "f141e672-8cdf-4b91-cc5e-d7e8334cd8ad"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: unstructured[local-inference] in /usr/local/lib/python3.9/dist-packages (0.5.7)\n",
            "Requirement already satisfied: pillow in /usr/local/lib/python3.9/dist-packages (from unstructured[local-inference]) (8.4.0)\n",
            "Requirement already satisfied: python-docx in /usr/local/lib/python3.9/dist-packages (from unstructured[local-inference]) (0.8.11)\n",
            "Requirement already satisfied: python-magic in /usr/local/lib/python3.9/dist-packages (from unstructured[local-inference]) (0.4.27)\n",
            "Requirement already satisfied: nltk in /usr/local/lib/python3.9/dist-packages (from unstructured[local-inference]) (3.8.1)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.9/dist-packages (from unstructured[local-inference]) (1.5.3)\n",
            "Requirement already satisfied: openpyxl in /usr/local/lib/python3.9/dist-packages (from unstructured[local-inference]) (3.0.10)\n",
            "Requirement already satisfied: markdown in /usr/local/lib/python3.9/dist-packages (from unstructured[local-inference]) (3.4.3)\n",
            "Requirement already satisfied: certifi>=2022.12.07 in /usr/local/lib/python3.9/dist-packages (from unstructured[local-inference]) (2022.12.7)\n",
            "Requirement already satisfied: argilla in /usr/local/lib/python3.9/dist-packages (from unstructured[local-inference]) (1.5.0)\n",
            "Requirement already satisfied: lxml in /usr/local/lib/python3.9/dist-packages (from unstructured[local-inference]) (4.9.2)\n",
            "Requirement already satisfied: python-pptx in /usr/local/lib/python3.9/dist-packages (from unstructured[local-inference]) (0.6.21)\n",
            "Requirement already satisfied: pypandoc in /usr/local/lib/python3.9/dist-packages (from unstructured[local-inference]) (1.11)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.9/dist-packages (from unstructured[local-inference]) (2.27.1)\n",
            "Collecting unstructured-inference~=0.2.4\n",
            "  Downloading unstructured_inference-0.2.13.tar.gz (30 kB)\n",
            "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting fastapi\n",
            "  Downloading fastapi-0.95.0-py3-none-any.whl (57 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m57.1/57.1 KB\u001b[0m \u001b[31m5.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting layoutparser[layoutmodels,tesseract]\n",
            "  Downloading layoutparser-0.3.4-py3-none-any.whl (19.2 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m19.2/19.2 MB\u001b[0m \u001b[31m62.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting python-multipart\n",
            "  Downloading python_multipart-0.0.6-py3-none-any.whl (45 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m45.7/45.7 KB\u001b[0m \u001b[31m4.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting uvicorn\n",
            "  Downloading uvicorn-0.21.1-py3-none-any.whl (57 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m57.8/57.8 KB\u001b[0m \u001b[31m5.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting huggingface-hub\n",
            "  Downloading huggingface_hub-0.13.3-py3-none-any.whl (199 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m199.8/199.8 KB\u001b[0m \u001b[31m18.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting opencv-python==4.6.0.66\n",
            "  Downloading opencv_python-4.6.0.66-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (60.9 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m60.9/60.9 MB\u001b[0m \u001b[31m10.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting onnxruntime\n",
            "  Downloading onnxruntime-1.14.1-cp39-cp39-manylinux_2_27_x86_64.whl (5.0 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m5.0/5.0 MB\u001b[0m \u001b[31m58.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting transformers\n",
            "  Downloading transformers-4.27.4-py3-none-any.whl (6.8 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m6.8/6.8 MB\u001b[0m \u001b[31m55.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: numpy>=1.17.3 in /usr/local/lib/python3.9/dist-packages (from opencv-python==4.6.0.66->unstructured-inference~=0.2.4->unstructured[local-inference]) (1.22.4)\n",
            "Requirement already satisfied: httpx<0.24,>=0.15 in /usr/local/lib/python3.9/dist-packages (from argilla->unstructured[local-inference]) (0.23.3)\n",
            "Requirement already satisfied: wrapt<1.15,>=1.13 in /usr/local/lib/python3.9/dist-packages (from argilla->unstructured[local-inference]) (1.14.1)\n",
            "Requirement already satisfied: monotonic in /usr/local/lib/python3.9/dist-packages (from argilla->unstructured[local-inference]) (1.6)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.9/dist-packages (from argilla->unstructured[local-inference]) (23.0)\n",
            "Requirement already satisfied: deprecated~=1.2.0 in /usr/local/lib/python3.9/dist-packages (from argilla->unstructured[local-inference]) (1.2.13)\n",
            "Requirement already satisfied: tqdm>=4.27.0 in /usr/local/lib/python3.9/dist-packages (from argilla->unstructured[local-inference]) (4.65.0)\n",
            "Requirement already satisfied: backoff in /usr/local/lib/python3.9/dist-packages (from argilla->unstructured[local-inference]) (2.2.1)\n",
            "Requirement already satisfied: rich<=13.0.1 in /usr/local/lib/python3.9/dist-packages (from argilla->unstructured[local-inference]) (13.0.1)\n",
            "Requirement already satisfied: pydantic>=1.7.1 in /usr/local/lib/python3.9/dist-packages (from argilla->unstructured[local-inference]) (1.10.7)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.9/dist-packages (from pandas->unstructured[local-inference]) (2022.7.1)\n",
            "Requirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.9/dist-packages (from pandas->unstructured[local-inference]) (2.8.2)\n",
            "Requirement already satisfied: importlib-metadata>=4.4 in /usr/local/lib/python3.9/dist-packages (from markdown->unstructured[local-inference]) (6.1.0)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.9/dist-packages (from nltk->unstructured[local-inference]) (1.1.1)\n",
            "Requirement already satisfied: regex>=2021.8.3 in /usr/local/lib/python3.9/dist-packages (from nltk->unstructured[local-inference]) (2022.10.31)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.9/dist-packages (from nltk->unstructured[local-inference]) (8.1.3)\n",
            "Requirement already satisfied: et-xmlfile in /usr/local/lib/python3.9/dist-packages (from openpyxl->unstructured[local-inference]) (1.1.0)\n",
            "Requirement already satisfied: XlsxWriter>=0.5.7 in /usr/local/lib/python3.9/dist-packages (from python-pptx->unstructured[local-inference]) (3.0.9)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.9/dist-packages (from requests->unstructured[local-inference]) (1.26.15)\n",
            "Requirement already satisfied: charset-normalizer~=2.0.0 in /usr/local/lib/python3.9/dist-packages (from requests->unstructured[local-inference]) (2.0.12)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.9/dist-packages (from requests->unstructured[local-inference]) (3.4)\n",
            "Requirement already satisfied: sniffio in /usr/local/lib/python3.9/dist-packages (from httpx<0.24,>=0.15->argilla->unstructured[local-inference]) (1.3.0)\n",
            "Requirement already satisfied: rfc3986[idna2008]<2,>=1.3 in /usr/local/lib/python3.9/dist-packages (from httpx<0.24,>=0.15->argilla->unstructured[local-inference]) (1.5.0)\n",
            "Requirement already satisfied: httpcore<0.17.0,>=0.15.0 in /usr/local/lib/python3.9/dist-packages (from httpx<0.24,>=0.15->argilla->unstructured[local-inference]) (0.16.3)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.9/dist-packages (from importlib-metadata>=4.4->markdown->unstructured[local-inference]) (3.15.0)\n",
            "Requirement already satisfied: typing-extensions>=4.2.0 in /usr/local/lib/python3.9/dist-packages (from pydantic>=1.7.1->argilla->unstructured[local-inference]) (4.5.0)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.9/dist-packages (from python-dateutil>=2.8.1->pandas->unstructured[local-inference]) (1.16.0)\n",
            "Requirement already satisfied: pygments<3.0.0,>=2.6.0 in /usr/local/lib/python3.9/dist-packages (from rich<=13.0.1->argilla->unstructured[local-inference]) (2.14.0)\n",
            "Requirement already satisfied: commonmark<0.10.0,>=0.9.0 in /usr/local/lib/python3.9/dist-packages (from rich<=13.0.1->argilla->unstructured[local-inference]) (0.9.1)\n",
            "Collecting starlette<0.27.0,>=0.26.1\n",
            "  Downloading starlette-0.26.1-py3-none-any.whl (66 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m66.9/66.9 KB\u001b[0m \u001b[31m6.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.9/dist-packages (from huggingface-hub->unstructured-inference~=0.2.4->unstructured[local-inference]) (6.0)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.9/dist-packages (from huggingface-hub->unstructured-inference~=0.2.4->unstructured[local-inference]) (3.10.7)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.9/dist-packages (from layoutparser[layoutmodels,tesseract]->unstructured-inference~=0.2.4->unstructured[local-inference]) (1.10.1)\n",
            "Collecting pdfplumber\n",
            "  Downloading pdfplumber-0.8.0-py3-none-any.whl (43 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m43.7/43.7 KB\u001b[0m \u001b[31m3.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting iopath\n",
            "  Downloading iopath-0.1.10.tar.gz (42 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m42.2/42.2 KB\u001b[0m \u001b[31m2.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting pdf2image\n",
            "  Downloading pdf2image-1.16.3-py3-none-any.whl (11 kB)\n",
            "Requirement already satisfied: torchvision in /usr/local/lib/python3.9/dist-packages (from layoutparser[layoutmodels,tesseract]->unstructured-inference~=0.2.4->unstructured[local-inference]) (0.14.1+cu116)\n",
            "Collecting effdet\n",
            "  Downloading effdet-0.3.0-py3-none-any.whl (112 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m112.2/112.2 KB\u001b[0m \u001b[31m8.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: torch in /usr/local/lib/python3.9/dist-packages (from layoutparser[layoutmodels,tesseract]->unstructured-inference~=0.2.4->unstructured[local-inference]) (1.13.1+cu116)\n",
            "Collecting pytesseract\n",
            "  Downloading pytesseract-0.3.10-py3-none-any.whl (14 kB)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.9/dist-packages (from onnxruntime->unstructured-inference~=0.2.4->unstructured[local-inference]) (1.11.1)\n",
            "Collecting coloredlogs\n",
            "  Downloading coloredlogs-15.0.1-py2.py3-none-any.whl (46 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m46.0/46.0 KB\u001b[0m \u001b[31m3.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: flatbuffers in /usr/local/lib/python3.9/dist-packages (from onnxruntime->unstructured-inference~=0.2.4->unstructured[local-inference]) (23.3.3)\n",
            "Requirement already satisfied: protobuf in /usr/local/lib/python3.9/dist-packages (from onnxruntime->unstructured-inference~=0.2.4->unstructured[local-inference]) (3.19.6)\n",
            "Collecting tokenizers!=0.11.3,<0.14,>=0.11.1\n",
            "  Downloading tokenizers-0.13.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (7.6 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.6/7.6 MB\u001b[0m \u001b[31m46.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: h11>=0.8 in /usr/local/lib/python3.9/dist-packages (from uvicorn->unstructured-inference~=0.2.4->unstructured[local-inference]) (0.14.0)\n",
            "Requirement already satisfied: anyio<5.0,>=3.0 in /usr/local/lib/python3.9/dist-packages (from httpcore<0.17.0,>=0.15.0->httpx<0.24,>=0.15->argilla->unstructured[local-inference]) (3.6.2)\n",
            "Collecting humanfriendly>=9.1\n",
            "  Downloading humanfriendly-10.0-py2.py3-none-any.whl (86 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m86.8/86.8 KB\u001b[0m \u001b[31m7.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting omegaconf>=2.0\n",
            "  Downloading omegaconf-2.3.0-py3-none-any.whl (79 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m79.5/79.5 KB\u001b[0m \u001b[31m7.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting timm>=0.4.12\n",
            "  Downloading timm-0.6.13-py3-none-any.whl (549 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m549.1/549.1 KB\u001b[0m \u001b[31m30.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: pycocotools>=2.0.2 in /usr/local/lib/python3.9/dist-packages (from effdet->layoutparser[layoutmodels,tesseract]->unstructured-inference~=0.2.4->unstructured[local-inference]) (2.0.6)\n",
            "Collecting portalocker\n",
            "  Downloading portalocker-2.7.0-py2.py3-none-any.whl (15 kB)\n",
            "Collecting Wand>=0.6.10\n",
            "  Downloading Wand-0.6.11-py2.py3-none-any.whl (143 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m143.6/143.6 KB\u001b[0m \u001b[31m13.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting pdfminer.six==20221105\n",
            "  Downloading pdfminer.six-20221105-py3-none-any.whl (5.6 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m5.6/5.6 MB\u001b[0m \u001b[31m68.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting pillow\n",
            "  Downloading Pillow-9.4.0-cp39-cp39-manylinux_2_28_x86_64.whl (3.4 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.4/3.4 MB\u001b[0m \u001b[31m78.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: cryptography>=36.0.0 in /usr/local/lib/python3.9/dist-packages (from pdfminer.six==20221105->pdfplumber->layoutparser[layoutmodels,tesseract]->unstructured-inference~=0.2.4->unstructured[local-inference]) (40.0.1)\n",
            "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.9/dist-packages (from sympy->onnxruntime->unstructured-inference~=0.2.4->unstructured[local-inference]) (1.3.0)\n",
            "Collecting antlr4-python3-runtime==4.9.*\n",
            "  Downloading antlr4-python3-runtime-4.9.3.tar.gz (117 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m117.0/117.0 KB\u001b[0m \u001b[31m12.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: matplotlib>=2.1.0 in /usr/local/lib/python3.9/dist-packages (from pycocotools>=2.0.2->effdet->layoutparser[layoutmodels,tesseract]->unstructured-inference~=0.2.4->unstructured[local-inference]) (3.7.1)\n",
            "Requirement already satisfied: cffi>=1.12 in /usr/local/lib/python3.9/dist-packages (from cryptography>=36.0.0->pdfminer.six==20221105->pdfplumber->layoutparser[layoutmodels,tesseract]->unstructured-inference~=0.2.4->unstructured[local-inference]) (1.15.1)\n",
            "Requirement already satisfied: pyparsing>=2.3.1 in /usr/local/lib/python3.9/dist-packages (from matplotlib>=2.1.0->pycocotools>=2.0.2->effdet->layoutparser[layoutmodels,tesseract]->unstructured-inference~=0.2.4->unstructured[local-inference]) (3.0.9)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.9/dist-packages (from matplotlib>=2.1.0->pycocotools>=2.0.2->effdet->layoutparser[layoutmodels,tesseract]->unstructured-inference~=0.2.4->unstructured[local-inference]) (1.4.4)\n",
            "Requirement already satisfied: contourpy>=1.0.1 in /usr/local/lib/python3.9/dist-packages (from matplotlib>=2.1.0->pycocotools>=2.0.2->effdet->layoutparser[layoutmodels,tesseract]->unstructured-inference~=0.2.4->unstructured[local-inference]) (1.0.7)\n",
            "Requirement already satisfied: fonttools>=4.22.0 in /usr/local/lib/python3.9/dist-packages (from matplotlib>=2.1.0->pycocotools>=2.0.2->effdet->layoutparser[layoutmodels,tesseract]->unstructured-inference~=0.2.4->unstructured[local-inference]) (4.39.2)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.9/dist-packages (from matplotlib>=2.1.0->pycocotools>=2.0.2->effdet->layoutparser[layoutmodels,tesseract]->unstructured-inference~=0.2.4->unstructured[local-inference]) (0.11.0)\n",
            "Requirement already satisfied: importlib-resources>=3.2.0 in /usr/local/lib/python3.9/dist-packages (from matplotlib>=2.1.0->pycocotools>=2.0.2->effdet->layoutparser[layoutmodels,tesseract]->unstructured-inference~=0.2.4->unstructured[local-inference]) (5.12.0)\n",
            "Requirement already satisfied: pycparser in /usr/local/lib/python3.9/dist-packages (from cffi>=1.12->cryptography>=36.0.0->pdfminer.six==20221105->pdfplumber->layoutparser[layoutmodels,tesseract]->unstructured-inference~=0.2.4->unstructured[local-inference]) (2.21)\n",
            "Building wheels for collected packages: unstructured-inference, iopath, antlr4-python3-runtime\n",
            "  Building wheel for unstructured-inference (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for unstructured-inference: filename=unstructured_inference-0.2.13-py3-none-any.whl size=33390 sha256=0bd643c4ee24f6fbc3ab3ab3f6cf014523c0772bb21024aefec1ec01d48f9b15\n",
            "  Stored in directory: /root/.cache/pip/wheels/fd/c1/e7/5479c37c3ab2f7eda7dcf8dd02c6074c60e65a37ceaf5cca31\n",
            "  Building wheel for iopath (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for iopath: filename=iopath-0.1.10-py3-none-any.whl size=31547 sha256=b48b832bbdeb438b1a5deffc924f32a218118873b5dea5e2c37cbde4ce2941ea\n",
            "  Stored in directory: /root/.cache/pip/wheels/c1/13/6d/441d8f2af76ee6d2a3e67eebb1d0c556fefcee0a8b32266a8e\n",
            "  Building wheel for antlr4-python3-runtime (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for antlr4-python3-runtime: filename=antlr4_python3_runtime-4.9.3-py3-none-any.whl size=144573 sha256=d295c5b2c974cf49f61a07bcabc84485f345bb1c44ba6b2eb7f2f1aa4be4c30a\n",
            "  Stored in directory: /root/.cache/pip/wheels/23/cf/80/f3efa822e6ab23277902ee9165fe772eeb1dfb8014f359020a\n",
            "Successfully built unstructured-inference iopath antlr4-python3-runtime\n",
            "Installing collected packages: Wand, tokenizers, antlr4-python3-runtime, uvicorn, python-multipart, portalocker, pillow, opencv-python, omegaconf, humanfriendly, starlette, pytesseract, pdf2image, iopath, huggingface-hub, coloredlogs, transformers, timm, pdfminer.six, onnxruntime, fastapi, pdfplumber, effdet, layoutparser, unstructured-inference\n",
            "  Attempting uninstall: pillow\n",
            "    Found existing installation: Pillow 8.4.0\n",
            "    Uninstalling Pillow-8.4.0:\n",
            "      Successfully uninstalled Pillow-8.4.0\n",
            "  Attempting uninstall: opencv-python\n",
            "    Found existing installation: opencv-python 4.7.0.72\n",
            "    Uninstalling opencv-python-4.7.0.72:\n",
            "      Successfully uninstalled opencv-python-4.7.0.72\n",
            "Successfully installed Wand-0.6.11 antlr4-python3-runtime-4.9.3 coloredlogs-15.0.1 effdet-0.3.0 fastapi-0.95.0 huggingface-hub-0.13.3 humanfriendly-10.0 iopath-0.1.10 layoutparser-0.3.4 omegaconf-2.3.0 onnxruntime-1.14.1 opencv-python-4.6.0.66 pdf2image-1.16.3 pdfminer.six-20221105 pdfplumber-0.8.0 pillow-9.4.0 portalocker-2.7.0 pytesseract-0.3.10 python-multipart-0.0.6 starlette-0.26.1 timm-0.6.13 tokenizers-0.13.2 transformers-4.27.4 unstructured-inference-0.2.13 uvicorn-0.21.1\n"
          ]
        },
        {
          "data": {
            "application/vnd.colab-display-data+json": {
              "pip_warning": {
                "packages": [
                  "PIL",
                  "pydevd_plugins"
                ]
              }
            }
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "!pip install unstructured[local-inference]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 27,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5e7a4RTorcFy",
        "outputId": "2d466db8-8ee6-42b0-b85c-5582bfafa22d"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting tiktoken\n",
            "  Downloading tiktoken-0.3.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.7 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.7/1.7 MB\u001b[0m \u001b[31m28.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: regex>=2022.1.18 in /usr/local/lib/python3.9/dist-packages (from tiktoken) (2022.10.31)\n",
            "Requirement already satisfied: requests>=2.26.0 in /usr/local/lib/python3.9/dist-packages (from tiktoken) (2.27.1)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.9/dist-packages (from requests>=2.26.0->tiktoken) (1.26.15)\n",
            "Requirement already satisfied: charset-normalizer~=2.0.0 in /usr/local/lib/python3.9/dist-packages (from requests>=2.26.0->tiktoken) (2.0.12)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.9/dist-packages (from requests>=2.26.0->tiktoken) (2022.12.7)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.9/dist-packages (from requests>=2.26.0->tiktoken) (3.4)\n",
            "Installing collected packages: tiktoken\n",
            "Successfully installed tiktoken-0.3.3\n"
          ]
        }
      ],
      "source": [
        "!pip install tiktoken"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vDPrL8yBYiEf"
      },
      "source": [
        "## Load Data - pdf"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "id": "lGA09kdGZkV-"
      },
      "outputs": [],
      "source": [
        "from langchain.document_loaders import UnstructuredPDFLoader, OnlinePDFLoader\n",
        "from langchain.text_splitter import RecursiveCharacterTextSplitter"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "id": "I-Yy9tsrYa4s"
      },
      "outputs": [],
      "source": [
        "loader = UnstructuredPDFLoader(\"./budget_english_speech_2022.pdf\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-pjZmXYfZhKW",
        "outputId": "b0eaa366-a4b4-4e37-b2cc-a11791427d20"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Unzipping tokenizers/punkt.zip.\n",
            "[nltk_data] Downloading package averaged_perceptron_tagger to\n",
            "[nltk_data]     /root/nltk_data...\n",
            "[nltk_data]   Unzipping taggers/averaged_perceptron_tagger.zip.\n",
            "WARNING:unstructured:detectron2 is not installed. Cannot use the hi_res partitioning strategy. Falling back to partitioning with the fast strategy.\n"
          ]
        }
      ],
      "source": [
        "data = loader.load()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QmDxIff4aJgx",
        "outputId": "de0703df-a80a-478d-f1eb-f474feb57d3b"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "You have 1 document(s) in your data\n",
            "There are 160195 characters in your document\n"
          ]
        }
      ],
      "source": [
        "print (f'You have {len(data)} document(s) in your data')\n",
        "print (f'There are {len(data[0].page_content)} characters in your document')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0xHr23JbawRy"
      },
      "source": [
        "### Chunk your data up into smaller documents"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "id": "q628oAGmapXV"
      },
      "outputs": [],
      "source": [
        "text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=0)\n",
        "texts = text_splitter.split_documents(data)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "BPVcUj2rayrm",
        "outputId": "768094bd-74c5-4e91-9251-fb795bb13bb6"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Now you have 165 documents\n"
          ]
        }
      ],
      "source": [
        "print (f'Now you have {len(texts)} documents')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 156
        },
        "id": "r7-Xtbx4cWx9",
        "outputId": "6a62be50-7f1d-469a-8847-89e9003f6ae5"
      },
      "outputs": [
        {
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'Assam  through  special  assistance  for  capital\\n\\nexpenditure  amounting  to  Rs.  600  Cr.  in  the  current\\n\\nfinancial  year.  For  the  year  2022-23,  substantial\\n\\nallocation is expected from Union Government. A new\\n\\nscheme,  Prime  Minister’s  Development  Initiative  for\\n\\nNorth-East, PM-DevINE, has been announced in Union\\n\\nBudget this year, which will fund infrastructure, in the\\n\\n(9)\\n\\n23.\\n\\nspirit of PM Gati Shakti and social development projects\\n\\nbased on felt needs of the North-East. As this scheme\\n\\nis  over  and  above  the  existing  support  from  Union\\n\\nGovernment,  I  express  my  deep  gratitude  to\\n\\nGovernment of India for unstinted support to the state.\\n\\nSpeaker Sir, financial inclusion is the first step towards\\n\\ninclusive  growth.  And  our  government  is  totally\\n\\ncommitted  to  ensure  quality  banking  services  to  the\\n\\nones who have been historically un-served. A total of\\n\\naround 1.9 Cr. bank accounts have been opened in the'"
            ]
          },
          "execution_count": 11,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "texts[12].page_content"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LVvQjYMqa3ll"
      },
      "source": [
        "## Create embeddings of your documents to get ready for semantic search"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kCYRnPp0a1Su",
        "outputId": "df6b1b95-6e00-4046-c174-35fd3877a651"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.9/dist-packages/pinecone/index.py:4: TqdmExperimentalWarning: Using `tqdm.autonotebook.tqdm` in notebook mode. Use `tqdm.tqdm` instead to force console mode (e.g. in jupyter console)\n",
            "  from tqdm.autonotebook import tqdm\n"
          ]
        }
      ],
      "source": [
        "from langchain.vectorstores import Chroma, Pinecone\n",
        "from langchain.embeddings.openai import OpenAIEmbeddings\n",
        "import pinecone"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "id": "_wFO9Jewa57F"
      },
      "outputs": [],
      "source": [
        "OPENAI_API_KEY = \"   \"\n",
        "PINECONE_API_KEY = '  '\n",
        "PINECONE_API_ENV = 'us-west4-gcp'"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {
        "id": "Kuz8VPe_b1qu"
      },
      "outputs": [],
      "source": [
        "# Index anme : budget-speech\n",
        "# Dimensions\n",
        "\n",
        "embeddings = OpenAIEmbeddings(openai_api_key=OPENAI_API_KEY)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {
        "id": "UR8g1Q3ScKeJ"
      },
      "outputs": [],
      "source": [
        "# initialize pinecone\n",
        "pinecone.init(\n",
        "    api_key=PINECONE_API_KEY,  # find at app.pinecone.io\n",
        "    environment=PINECONE_API_ENV  # next to api key in console\n",
        ")\n",
        "index_name = \"budget-speech\""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 164
        },
        "id": "39OorezDcQet",
        "outputId": "a2bd88d0-008d-49d7-ba8b-be6958a1207d"
      },
      "outputs": [
        {
          "ename": "NameError",
          "evalue": "ignored",
          "output_type": "error",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-12-7214c7f76696>\u001b[0m in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mdocsearch\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mPinecone\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfrom_texts\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpage_content\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mt\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtexts\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0membeddings\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mindex_name\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mindex_name\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m: name 'texts' is not defined"
          ]
        }
      ],
      "source": [
        "docsearch = Pinecone.from_texts([t.page_content for t in texts], embeddings, index_name=index_name)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 13,
      "metadata": {
        "id": "2yuQ1adPcejL"
      },
      "outputs": [],
      "source": [
        "query = \"What are some key highlights of budget?\"\n",
        "docs = docsearch.similarity_search(query, include_metadata=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 14,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "35bdRvhwcqcO",
        "outputId": "aa3351c1-b598-4687-bb88-2eaeb81199ac"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "[Document(page_content='during 2022-23 have been shown in Table-2. Taking into account\\n\\nthe expenditure of Rs.163743.58 Crore under Public Account\\n\\nand Rs.200 Crore under Contingency Fund, the aggregate\\n\\nexpenditure for the year is estimated at Rs. 283494.64 Crore.\\n\\nThus, estimated transactions during the year will result in an\\n\\nestimated surplus of Rs.420.14 Crore. This, together with the\\n\\nopening deficit of Rs.1020.50 Crore will lead to a Budget deficit\\n\\nof Rs.600.36 Crore at the end of the year 2022-23.\\n\\nThe all-round development of the people of Assam is our\\n\\nsoul aim and objective. Sahityarathi Lakshminath Bezbaroa\\n\\nsaid in his poem ‘Been Boragi’\\n\\nGa been ga Axom Sangeet\\n\\nXukane melok Kuhi\\n\\n(113)\\n\\n(Oh veena sing the song of Assam, so that the wilted\\n\\nmay bud to sprout)\\n\\nOptimism is the vitality of our life. I would like to conclude\\n\\nmy budget speech here. At the end, let’s say together in\\n\\nthe same tune of Rupkonwar Jyotiprasad -\\n\\n‘Joyo alokomoy\\n\\nAnadi ananta prasari\\n\\nBiswa jiwana jyoti.', metadata={}),\n",
              " Document(page_content='burden  on  our  people  of Assam. Accordingly  I  place\\n\\nmy Budget before this August House.\\n\\nmy Budget before this August House.\\n\\n(112)\\n\\nBUDGET ESTIMATES\\n\\nThe Budget Estimates of 2022-23 show a receipt of Rs.\\n\\n119027.95 Crore under the Consolidated Fund of the State. Out\\n\\nof this, Rs.96374.59 Crore is on Revenue Account and the\\n\\nremaining Rs. 23673.71 Crore is under Capital Account. I have\\n\\nshown the estimated receipt in the Consolidated Fund from\\n\\ndifferent  sources  at  Table-1.  After  adding  the  receipt  of\\n\\nRs.162886.83 Crore under Public Account and Rs.2000Crore\\n\\nunder Contingency Fund, the aggregate Receipts amount to\\n\\nRs.283914.78 Crore. As against this, total expenditure from the\\n\\nConsolidated Fund of the State in 2022-23 is estimated at\\n\\nRs.119551.06 Crore of which Rs. 96367.05 Crore is on Revenue\\n\\nAccount and Rs.23184.01 Crore is on Capital Account. The\\n\\nestimates of Grant-wise Expenditure from the Consolidated Fund', metadata={}),\n",
              " Document(page_content='At  the  outset,  I  bow  before  this August Assembly  before  I\\n\\npresent the budget for the financial year 2022-23 and seek\\n\\nyour good wishes. With all humility, I would like to recite a\\n\\nstanza  from  a  poem  penned  by  Rupkonwar  Jyotiprasad\\n\\nAgarwalla:\\n\\n“Tore more alokorejatra\\n\\nAbyartha abyartha\\n\\nAmi palo jibonor artha abhinaba\\n\\nSwagata swagata xotirtha’’\\n\\nStrong economic structure is indispensable to bring positive\\n\\nchange in the society. We cannot continue the journey of light\\n\\nwithout robust economic base. I am present here once again\\n\\nin front of you all to place the budget of 2022-23 including all\\n\\nabout income and expenditure.\\n\\nHon’ble Speaker Sir,\\n\\n1.\\n\\nWhile  presenting  the  first  budget  to  the  people  of\\n\\nAssam, with due humility, I said that the Budget is a\\n\\nvision  document  of  the  alliance  government  for  the\\n\\nsucceeding five years. Now at the end of the financial\\n\\nyear, we will introspect how far we have stridden ahead.', metadata={}),\n",
              " Document(page_content='Government of India released Rs. 25,176.39 Cr. during\\n\\n2021-22.\\n\\n17. During last five years, the trend of expenditure has been\\n\\nin ascending order on the budgetary allocation which\\n\\nsignifies  prudent  fiscal  management,  meticulous\\n\\nplanning  of  expenditure  including  large  scale\\n\\nprocurement reforms in the State. Overall expenditure\\n\\nof  the  state  significantly  improved  from  63%  during\\n\\n(6)\\n\\n2015-16  to  83%  during  2019-20.  The  capital\\n\\nexpenditure also grew from Rs. 4919 Cr. In 2015-16 to\\n\\nRs.  14630  Cr.  In  2020-21.  The  details  of  overall\\n\\nexpenditure against budgetary allocation is reflected in\\n\\nthe following table: -\\n\\nBudgetary expenditure (In Rs. Cr.)\\n\\n18.\\n\\nSource: Finance Accounts, AG, Assam\\n\\nIn order to accelerate the pace of economic growth,\\n\\nGovernment  of  Assam  during  2021-22  decided  to\\n\\nundertake several capital-intensive projects. Presently,\\n\\nnew projects to the tune of around Rs. 11,100 Cr. under\\n\\nNABARD  Infrastructure  Development  Assistance', metadata={}),\n",
              " Document(page_content='more than one lakh beneficiaries under SC, ST and OBC\\n\\n(107)\\n\\ncategories have been provided during the year 2021-22.\\n\\nThis department has extended financial incentives of Rs.\\n\\n50,000 for inter-caste marriages per married couple.\\n\\n377. Sir, all such kind of initiatives will continue next year also\\n\\nand I hereby propose Rs. 943.19 Cr.\\n\\nPart-II\\n\\n1.\\n\\nSpeaker Sir, our Government has marked a new era in\\n\\nthe  transformation  of  the  revenue  sector. The  strong\\n\\nguidance  of  our  Hon’ble  Chief  Minister,  Dr  Himanta\\n\\n(108)\\n\\nBiswa Sarma has invigorated us to take the State revenue\\n\\ncollection mechanism to an enviable height.\\n\\n2.\\n\\nNow, I would like to highlight a few achievements of our\\n\\nGovernment while also mentioning some of the initiatives\\n\\nthat we have proposed for the upcoming year.\\n\\nCOMMERCIAL TAX\\n\\n3.\\n\\n4.\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nSpeaker Sir, during the current fiscal of 2021-22 (up to\\n\\nFebruary, 2022), the Tax Department has mopped up\\n\\nan amount of Rs. 14,470 Cr. compared to Rs. 12,101', metadata={})]"
            ]
          },
          "execution_count": 14,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "docs"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OFaFghFZdo72"
      },
      "source": [
        "## Query using Index in Pinecone"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 15,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IZLKJIwlcoTp",
        "outputId": "582cd6ba-4d4a-411e-920b-2a7e5465b69c"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "langchain.vectorstores.pinecone.Pinecone"
            ]
          },
          "execution_count": 15,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "type(docsearch)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 13,
      "metadata": {
        "id": "O0swQdaqg3zQ"
      },
      "outputs": [],
      "source": [
        "from langchain.llms import OpenAI\n",
        "from langchain.chains.question_answering import load_qa_chain"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 25,
      "metadata": {
        "id": "zZBjHBhpg6Ee"
      },
      "outputs": [],
      "source": [
        "llm = OpenAI(temperature=0.9, openai_api_key=OPENAI_API_KEY)\n",
        "chain = load_qa_chain(llm, chain_type=\"map_reduce\") #stuff 0.06"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 15,
      "metadata": {
        "id": "o9AZcZoifMjn"
      },
      "outputs": [],
      "source": [
        "p_vectorstore = Pinecone.from_existing_index(index_name, embeddings)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 16,
      "metadata": {
        "id": "EinT-X_Dgt-3"
      },
      "outputs": [],
      "source": [
        "query = \"What are some key highlights of budget?\"\n",
        "docs = p_vectorstore.similarity_search(query, include_metadata=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 17,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 122
        },
        "id": "ZW-ZYlKBhCZC",
        "outputId": "75eded16-b640-45ba-a5ea-2d95555dc217"
      },
      "outputs": [
        {
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "' The budget estimates of 2022-23 show a receipt of Rs. 119027.95 Crore under the Consolidated Fund of the State. Out of this, Rs.96374.59 Crore is on Revenue Account and the remaining Rs. 23673.71 Crore is under Capital Account. The estimated transactions during the year will result in an estimated surplus of Rs.420.14 Crore. The budget includes initiatives to accelerate the pace of economic growth, such as new projects to the tune of around Rs. 11,100 Cr. under NABARD Infrastructure Development Assistance. The Tax Department has mopped up an amount of Rs. 14,470 Cr. compared to Rs. 12,101 Cr. during the same period last year. Financial incentives of Rs. 50,000 for inter-caste marriages per married couple have been provided during the year 2021-22.'"
            ]
          },
          "execution_count": 17,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "chain.run(input_documents=docs, question=query)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 18,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 37
        },
        "id": "-MPukMLqhD7i",
        "outputId": "5f3cb51e-5c76-4459-ca4c-f56d47be9316"
      },
      "outputs": [
        {
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "' Narendra Modi was not mentioned in the context provided.'"
            ]
          },
          "execution_count": 18,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "query = \"In what context Narendra Modi was mentioned?\"\n",
        "docs = p_vectorstore.similarity_search(query, include_metadata=True)\n",
        "chain.run(input_documents=docs, question=query, verbose=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 21,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 71
        },
        "id": "j7VFWITsocok",
        "outputId": "0a6ddbb6-67bd-4c66-80ae-ad3f4d7f2c69"
      },
      "outputs": [
        {
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "\" Prime Minister was mentioned in the context of the Prime Minister's Development Initiative for North-East (PM-DevINE) scheme, which was announced in the Union Budget and will fund infrastructure and social development projects in the North-East. Prime Minister was also mentioned in the context of the Prime Minister Gati Shakti scheme.\""
            ]
          },
          "execution_count": 21,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "query = \"In what context Prime Minister was mentioned? List few of them\"\n",
        "docs = p_vectorstore.similarity_search(query, include_metadata=True)\n",
        "chain.run(input_documents=docs, question=query, verbose=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 22,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        },
        "id": "Eru-BQ35olO4",
        "outputId": "47038e18-61a7-4907-f4db-a07f7e30a0a2"
      },
      "outputs": [
        {
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "' Prime Minister said during Independence Day that the next 25 years should be a glorious Amrit Kaal with \"new thresholds, aspirations, and dreams\".'"
            ]
          },
          "execution_count": 22,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# with chain_type =\"stuff\"\n",
        "query = \"What did Prime Minister say during Independence Day?\"\n",
        "docs = p_vectorstore.similarity_search(query, include_metadata=True)\n",
        "chain.run(input_documents=docs, question=query, verbose=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 28,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        },
        "id": "zIml2soeo-7t",
        "outputId": "565188b7-8a3d-46b7-c743-41025795ebec"
      },
      "outputs": [
        {
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "' The Prime Minister called for making the next 25 years a glorious Amrit Kaal with “new thresholds, aspirations, and dreams”, and for driving development through various capital-intensive policy steps to turn the North East into a ‘New Engine of Growth’.'"
            ]
          },
          "execution_count": 28,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# with chain_type =\"map_reduce\"\n",
        "query = \"What did Prime Minister say during Independence Day?\"\n",
        "docs = p_vectorstore.similarity_search(query, include_metadata=True)\n",
        "chain.run(input_documents=docs, question=query, verbose=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 29,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 37
        },
        "id": "kiq7D2Frpp9p",
        "outputId": "544f12e6-d15e-4186-aa60-e35f99520611"
      },
      "outputs": [
        {
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "\" I don't know.\""
            ]
          },
          "execution_count": 29,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "query = \"What did Narendra Modi say during Independence Day?\"\n",
        "docs = p_vectorstore.similarity_search(query, include_metadata=True)\n",
        "chain.run(input_documents=docs, question=query, verbose=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 30,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 88
        },
        "id": "wGqt5q1QxeLY",
        "outputId": "c8de78eb-6ce9-418f-994e-c250fb9bdf8e"
      },
      "outputs": [
        {
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "' In Assam, ex-servicemen have been granted a two percent reservation for Grade III and Grade IV posts, with one-fourth of this reservation being reserved for disabled ex-servicemen. The government has also launched KRITAGYATA portal for online processing of pension orders and Pension Seva Kendras are being set up all over the state to facilitate processing of pension applications and its timely sanction.'"
            ]
          },
          "execution_count": 30,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "query = \"What was spoken about veterans or millitary personnels? Did they get any benefit?\"\n",
        "docs = p_vectorstore.similarity_search(query, include_metadata=True)\n",
        "chain.run(input_documents=docs, question=query, verbose=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 31,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 156
        },
        "id": "QL9euPBMyMz1",
        "outputId": "f8ae8e50-bcf2-425a-a772-8110bb0c00a5"
      },
      "outputs": [
        {
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "' In recognition of the service rendered by ex-servicemen to the nation, the Assam ex-Servicemen (Reservation of Posts in States Services and posts of Grade III and Grade IV) Rules, 2021 has been approved which provides for two percent reservation for ex-Servicemen in Grade III and Grade IV posts. It is also worth noting that out of this two percent reservation, one - fourth is reserved for disabled ex-servicemen whose disability arose in service. To reduce the hardship faced by the pensioners, the government has launched KRITAGYATA portal for online processing of pension orders. Pension Seva Kendras are being set up all over the state to facilitate processing of pension applications and its timely sanction. A total amount of Rs. 129.89 Cr. has been earmarked this year towards interest subvention for state government employees under Apun Ghar and Bidyalakshmi scheme at subsidized rate of interest.'"
            ]
          },
          "execution_count": 31,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "query = \"What was spoken about veterans or millitary personnels? Did they get any benefit?\"\n",
        "docs = p_vectorstore.similarity_search(query, include_metadata=True, k=10)\n",
        "chain.run(input_documents=docs, question=query, verbose=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "fQ4VR5xCywQI"
      },
      "outputs": [],
      "source": []
    }
  ],
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}