check_token_length.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cohere.Tokens {\n",
      "\ttokens: [40, 7890, 4928, 1707, 25121, 4880, 1708, 1785, 2448, 1699, 1888, 2427, 4928, 1707, 36367, 71053, 21, 27822, 1809, 33046, 2050, 2112, 1821, 6833, 3295, 19, 1821, 3384, 2060, 4936, 1773, 1685, 7869, 1994, 2070, 1784, 16, 3123, 1821, 5985, 2013, 5493, 1748, 1784, 21, 27822, 3034, 5010, 1675, 2870, 1983, 1707, 2608, 2050, 7354, 3083, 2568, 21, 2586, 1675, 24688, 19, 1821, 5815, 1682, 15789, 6756, 14992, 19, 1988, 27443, 2191, 2568, 1821, 1827, 9178, 1707, 2050, 1821, 17680, 21, 2592, 2330, 1821, 67369, 2514, 2238, 1682, 68440, 15662, 2552, 1707, 6157, 2102, 2765, 19, 2545, 1739, 2120, 1997, 2165, 1675, 1960, 20, 43078, 13925, 1705, 2930, 1829, 1682, 3184, 11220, 14, 4084, 2050, 1739, 2120, 7350, 3653, 21, 27822, 1809, 2608, 4605, 1932, 3034, 2103, 1705, 3389, 1675, 11639, 1829, 1682, 3656, 7597, 1708, 2226, 1675, 32260, 1707, 2583, 75, 1853, 3140, 1682, 2434, 21]\n",
      "\ttoken_strings: ['A', ' proud', ' son', ' of', ' Hey', 'wood', ' and', ' an', ' even', ' p', 'rou', 'der', ' son', ' of', ' Mama', ' Welles', '.', ' Jackie', \"'s\", ' stubborn', ' -', ' when', ' he', ' wants', ' something', ',', ' he', ' won', \"'t\", ' stop', ' (', 'or', ' shut', ' up', ' about', ' it', ')', ' until', ' he', ' gets', ' his', ' hands', ' on', ' it', '.', ' Jackie', ' always', ' wanted', ' a', ' lot', ' out', ' of', ' life', ' -', ' maybe', ' too', ' much', '.', ' As', ' a', ' teenager', ',', ' he', ' joined', ' the', ' Valent', 'inos', ' gang', ',', ' but', ' realizing', ' how', ' much', ' he', ' was', ' capable', ' of', ' -', ' he', ' quit', '.', ' For', ' years', ' he', ' hust', 'led', ' through', ' the', ' unfor', 'giving', ' world', ' of', ' mer', 'ck', 'work', ',', ' because', ' you', ' need', ' more', ' than', ' a', ' go', '-', 'getter', ' attitude', ' to', ' sit', ' at', ' the', ' big', ' boys', \"'\", ' table', ' -', ' you', ' need', ' street', ' cred', '.', ' Jackie', \"'s\", ' life', ' goal', ' has', ' always', ' been', ' to', ' become', ' a', ' legend', ' at', ' the', ' After', 'life', ' and', ' make', ' a', ' heap', ' of', ' ed', 'd', 'ies', ' along', ' the', ' way', '.']\n",
      "\tmeta: {'api_version': {'version': '1'}}\n",
      "}\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "147"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import cohere\n",
    "import random\n",
    "import json\n",
    "\n",
    "# Randomly select an API key\n",
    "selected_key = json.load(open('apikeys.json', 'r'))['api_keys'][random.randint(\n",
    "    0, len(json.load(open('apikeys.json', 'r'))['api_keys'])-1)]\n",
    "\n",
    "co = cohere.Client(selected_key)\n",
    "\n",
    "response = co.tokenize(\n",
    "  text=\"\"\"A proud son of Heywood and an even prouder son of Mama Welles. Jackie's stubborn - when he wants something, he won't stop (or shut up about it) until he gets his hands on it. Jackie always wanted a lot out of life - maybe too much. As a teenager, he joined the Valentinos gang, but realizing how much he was capable of - he quit. For years he hustled through the unforgiving world of merckwork, because you need more than a go-getter attitude to sit at the big boys' table - you need street cred. Jackie's life goal has always been to become a legend at the Afterlife and make a heap of eddies along the way.\"\"\",\n",
    "  model='command' \n",
    ")\n",
    "print(response)\n",
    "len(response.tokens)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.6"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}