explore_results.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !pip install nervaluate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm import tqdm\n",
    "from nervaluate import Evaluator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"experiments/deberta_v3_base/2023-03-21_15-36-47/infer_text_dev.txt\", \"r\") as f:\n",
    "    test_data = f.read()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "labels, preds = test_data.split(\"\\n\")[:2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "labels = labels.split(\"\\t\")[1]\n",
    "preds = preds.split(\"\\t\")[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "labels = labels.split(\" \")\n",
    "preds = preds.split(\" \")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "51362\n",
      "51362\n",
      "O O\n",
      "O O\n",
      "ORG ORG\n",
      "O O\n",
      "O O\n",
      "O O\n",
      "O O\n",
      "O O\n",
      "O O\n",
      "O O\n"
     ]
    }
   ],
   "source": [
    "print(len(labels))\n",
    "print(len(preds))\n",
    "\n",
    "for i in range(10):\n",
    "    print(labels[i], preds[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " Num Errors: 279\n"
     ]
    }
   ],
   "source": [
    "k = 0\n",
    "for i in range(len(preds)):\n",
    "    if labels[i] != preds[i]:\n",
    "        k +=1\n",
    "print(f\" Num Errors: {k}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def read_data(file):\n",
    "    lines = open(file, \"r\").readlines()\n",
    "    data = {\"sentences\": [], \"labels_per_sent\": []}\n",
    "    sentence, labels = [], []\n",
    "\n",
    "    for line in tqdm(lines):\n",
    "        line = line.strip()\n",
    "\n",
    "        if not line:\n",
    "            if sentence and labels:\n",
    "                assert len(sentence) == len(labels)\n",
    "                data[\"sentences\"].append(sentence)\n",
    "                data[\"labels_per_sent\"].append(labels)\n",
    "                sentence, labels = [], []\n",
    "            continue\n",
    "\n",
    "        if line.startswith(\"-DOCSTART-\"):\n",
    "            continue\n",
    "        else:\n",
    "            values = line.split(\" \")\n",
    "            try:\n",
    "                token, _, _, label = values\n",
    "                sentence.append(token)\n",
    "                if label != \"O\":\n",
    "                    labels.append(label.split(\"-\")[-1])\n",
    "                else:\n",
    "                    labels.append(label)\n",
    "\n",
    "            except Exception as e:\n",
    "                print(f\"Error has occur: {e}\")\n",
    "                continue\n",
    "    return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 219554/219554 [00:00<00:00, 676159.30it/s]\n"
     ]
    }
   ],
   "source": [
    "orig_labels = read_data(\"data/eng.testa\")[\"labels_per_sent\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "true_labels = []\n",
    "pred_labels = []\n",
    "\n",
    "def conv(ls, line, k):\n",
    "    return [f\"I-{l}\" if l != \"O\" else l for l in ls[k:k+len(line)]]\n",
    "\n",
    "k = 0\n",
    "for line in orig_labels:\n",
    "    true_labels.append([f\"I-{l}\" if l != \"O\" else l for l in labels[k:k+len(line)]])\n",
    "    pred_labels.append([f\"I-{l}\" if l != \"O\" else l for l in preds[k:k+len(line)]])\n",
    "    k += len(line)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "14041\n",
      "14041\n",
      "['O', 'O', 'I-ORG', 'O', 'O', 'O', 'O', 'O', 'O'] ['O', 'O', 'I-ORG', 'O', 'O', 'O', 'O', 'O', 'O']\n",
      "['O', 'O'] ['O', 'O']\n",
      "['I-LOC', 'O'] ['I-LOC', 'O']\n",
      "['I-MISC', 'I-MISC', 'O', 'I-PER', 'I-PER', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'I-ORG', 'O', 'I-ORG', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O'] ['I-MISC', 'I-MISC', 'O', 'I-PER', 'I-PER', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'I-ORG', 'O', 'I-ORG', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O']\n",
      "['O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'I-ORG', 'O', 'I-ORG', 'O', 'I-ORG', 'O', 'O', 'O', 'O', 'O', 'O', 'I-ORG', 'O'] ['O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'I-ORG', 'O', 'I-ORG', 'O', 'I-ORG', 'O', 'O', 'O', 'O', 'O', 'O', 'I-ORG', 'O']\n",
      "['O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'I-ORG', 'O', 'O', 'O', 'I-ORG', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'I-LOC', 'I-LOC', 'O', 'I-ORG', 'O', 'O', 'O', 'O', 'O', 'O', 'O'] ['O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'I-ORG', 'O', 'O', 'O', 'I-ORG', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'I-LOC', 'I-LOC', 'O', 'I-ORG', 'O', 'O', 'O', 'O', 'O', 'O', 'O']\n",
      "['O', 'O', 'O', 'O', 'O', 'O', 'O', 'I-LOC', 'O', 'I-PER', 'I-PER', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'I-ORG', 'O', 'O', 'O', 'O'] ['O', 'O', 'O', 'O', 'O', 'O', 'O', 'I-LOC', 'O', 'I-PER', 'I-PER', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'I-ORG', 'O', 'O', 'O', 'O']\n",
      "['O', 'O', 'O', 'O', 'O', 'I-PER', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'I-ORG', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'I-PER', 'I-PER', 'O', 'I-PER', 'I-PER', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O'] ['O', 'O', 'O', 'O', 'O', 'I-PER', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'I-ORG', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'I-PER', 'I-PER', 'O', 'I-PER', 'I-PER', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O']\n",
      "['O', 'I-ORG', 'O', 'I-LOC', 'O', 'I-PER', 'O', 'O', 'O', 'O', 'I-LOC', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'I-ORG'] ['O', 'I-ORG', 'O', 'I-LOC', 'O', 'I-PER', 'O', 'O', 'O', 'O', 'I-LOC', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'I-ORG']\n",
      "['O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'I-ORG', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'I-PER', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O'] ['O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'I-ORG', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'I-PER', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O']\n"
     ]
    }
   ],
   "source": [
    "print(len(true_labels))\n",
    "print(len(pred_labels))\n",
    "\n",
    "for i in range(10):\n",
    "    print(true_labels[i], pred_labels[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " Num error lines: 198\n"
     ]
    }
   ],
   "source": [
    "k = 0\n",
    "for i in range(len(pred_labels)):\n",
    "    if true_labels[i] != pred_labels[i]:\n",
    "        k +=1\n",
    "print(f\" Num error lines: {k}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "evaluator = Evaluator(true_labels, pred_labels, tags=[\"LOC\", \"MISC\", \"ORG\", \"PER\"], loader=\"list\")\n",
    "\n",
    "results, results_by_tag = evaluator.evaluate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'ent_type': {'correct': 5943,\n",
       "  'incorrect': 117,\n",
       "  'partial': 0,\n",
       "  'missed': 25,\n",
       "  'spurious': 63,\n",
       "  'possible': 6085,\n",
       "  'actual': 6123,\n",
       "  'precision': 0.9706026457618814,\n",
       "  'recall': 0.9766639276910436,\n",
       "  'f1': 0.9736238532110091},\n",
       " 'partial': {'correct': 5998,\n",
       "  'incorrect': 0,\n",
       "  'partial': 62,\n",
       "  'missed': 25,\n",
       "  'spurious': 63,\n",
       "  'possible': 6085,\n",
       "  'actual': 6123,\n",
       "  'precision': 0.9846480483423159,\n",
       "  'recall': 0.990797041906327,\n",
       "  'f1': 0.9877129750982963},\n",
       " 'strict': {'correct': 5898,\n",
       "  'incorrect': 162,\n",
       "  'partial': 0,\n",
       "  'missed': 25,\n",
       "  'spurious': 63,\n",
       "  'possible': 6085,\n",
       "  'actual': 6123,\n",
       "  'precision': 0.9632533072023518,\n",
       "  'recall': 0.9692686935086278,\n",
       "  'f1': 0.966251638269987},\n",
       " 'exact': {'correct': 5998,\n",
       "  'incorrect': 62,\n",
       "  'partial': 0,\n",
       "  'missed': 25,\n",
       "  'spurious': 63,\n",
       "  'possible': 6085,\n",
       "  'actual': 6123,\n",
       "  'precision': 0.9795851706679732,\n",
       "  'recall': 0.9857025472473295,\n",
       "  'f1': 0.9826343381389253}}"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'LOC': {'ent_type': {'correct': 1812,\n",
       "   'incorrect': 32,\n",
       "   'partial': 0,\n",
       "   'missed': 1,\n",
       "   'spurious': 6,\n",
       "   'possible': 1845,\n",
       "   'actual': 1850,\n",
       "   'precision': 0.9794594594594594,\n",
       "   'recall': 0.9821138211382113,\n",
       "   'f1': 0.980784844384303},\n",
       "  'partial': {'correct': 1837,\n",
       "   'incorrect': 0,\n",
       "   'partial': 7,\n",
       "   'missed': 1,\n",
       "   'spurious': 6,\n",
       "   'possible': 1845,\n",
       "   'actual': 1850,\n",
       "   'precision': 0.9948648648648649,\n",
       "   'recall': 0.9975609756097561,\n",
       "   'f1': 0.9962110960757782},\n",
       "  'strict': {'correct': 1809,\n",
       "   'incorrect': 35,\n",
       "   'partial': 0,\n",
       "   'missed': 1,\n",
       "   'spurious': 6,\n",
       "   'possible': 1845,\n",
       "   'actual': 1850,\n",
       "   'precision': 0.9778378378378378,\n",
       "   'recall': 0.9804878048780488,\n",
       "   'f1': 0.9791610284167794},\n",
       "  'exact': {'correct': 1837,\n",
       "   'incorrect': 7,\n",
       "   'partial': 0,\n",
       "   'missed': 1,\n",
       "   'spurious': 6,\n",
       "   'possible': 1845,\n",
       "   'actual': 1850,\n",
       "   'precision': 0.9929729729729729,\n",
       "   'recall': 0.9956639566395664,\n",
       "   'f1': 0.994316644113667}},\n",
       " 'MISC': {'ent_type': {'correct': 897,\n",
       "   'incorrect': 23,\n",
       "   'partial': 0,\n",
       "   'missed': 12,\n",
       "   'spurious': 29,\n",
       "   'possible': 932,\n",
       "   'actual': 949,\n",
       "   'precision': 0.9452054794520548,\n",
       "   'recall': 0.9624463519313304,\n",
       "   'f1': 0.9537480063795853},\n",
       "  'partial': {'correct': 894,\n",
       "   'incorrect': 0,\n",
       "   'partial': 26,\n",
       "   'missed': 12,\n",
       "   'spurious': 29,\n",
       "   'possible': 932,\n",
       "   'actual': 949,\n",
       "   'precision': 0.9557428872497366,\n",
       "   'recall': 0.973175965665236,\n",
       "   'f1': 0.9643806485911749},\n",
       "  'strict': {'correct': 877,\n",
       "   'incorrect': 43,\n",
       "   'partial': 0,\n",
       "   'missed': 12,\n",
       "   'spurious': 29,\n",
       "   'possible': 932,\n",
       "   'actual': 949,\n",
       "   'precision': 0.9241306638566913,\n",
       "   'recall': 0.9409871244635193,\n",
       "   'f1': 0.9324827219564062},\n",
       "  'exact': {'correct': 894,\n",
       "   'incorrect': 26,\n",
       "   'partial': 0,\n",
       "   'missed': 12,\n",
       "   'spurious': 29,\n",
       "   'possible': 932,\n",
       "   'actual': 949,\n",
       "   'precision': 0.9420442571127503,\n",
       "   'recall': 0.9592274678111588,\n",
       "   'f1': 0.9505582137161085}},\n",
       " 'ORG': {'ent_type': {'correct': 1343,\n",
       "   'incorrect': 43,\n",
       "   'partial': 0,\n",
       "   'missed': 9,\n",
       "   'spurious': 18,\n",
       "   'possible': 1395,\n",
       "   'actual': 1404,\n",
       "   'precision': 0.9565527065527065,\n",
       "   'recall': 0.9627240143369176,\n",
       "   'f1': 0.9596284387281172},\n",
       "  'partial': {'correct': 1365,\n",
       "   'incorrect': 0,\n",
       "   'partial': 21,\n",
       "   'missed': 9,\n",
       "   'spurious': 18,\n",
       "   'possible': 1395,\n",
       "   'actual': 1404,\n",
       "   'precision': 0.9797008547008547,\n",
       "   'recall': 0.986021505376344,\n",
       "   'f1': 0.9828510182207931},\n",
       "  'strict': {'correct': 1329,\n",
       "   'incorrect': 57,\n",
       "   'partial': 0,\n",
       "   'missed': 9,\n",
       "   'spurious': 18,\n",
       "   'possible': 1395,\n",
       "   'actual': 1404,\n",
       "   'precision': 0.9465811965811965,\n",
       "   'recall': 0.9526881720430107,\n",
       "   'f1': 0.9496248660235799},\n",
       "  'exact': {'correct': 1365,\n",
       "   'incorrect': 21,\n",
       "   'partial': 0,\n",
       "   'missed': 9,\n",
       "   'spurious': 18,\n",
       "   'possible': 1395,\n",
       "   'actual': 1404,\n",
       "   'precision': 0.9722222222222222,\n",
       "   'recall': 0.978494623655914,\n",
       "   'f1': 0.9753483386923901}},\n",
       " 'PER': {'ent_type': {'correct': 1891,\n",
       "   'incorrect': 19,\n",
       "   'partial': 0,\n",
       "   'missed': 3,\n",
       "   'spurious': 10,\n",
       "   'possible': 1913,\n",
       "   'actual': 1920,\n",
       "   'precision': 0.9848958333333333,\n",
       "   'recall': 0.9884997386304234,\n",
       "   'f1': 0.9866944951734933},\n",
       "  'partial': {'correct': 1902,\n",
       "   'incorrect': 0,\n",
       "   'partial': 8,\n",
       "   'missed': 3,\n",
       "   'spurious': 10,\n",
       "   'possible': 1913,\n",
       "   'actual': 1920,\n",
       "   'precision': 0.9927083333333333,\n",
       "   'recall': 0.996340825927862,\n",
       "   'f1': 0.9945212627184972},\n",
       "  'strict': {'correct': 1883,\n",
       "   'incorrect': 27,\n",
       "   'partial': 0,\n",
       "   'missed': 3,\n",
       "   'spurious': 10,\n",
       "   'possible': 1913,\n",
       "   'actual': 1920,\n",
       "   'precision': 0.9807291666666667,\n",
       "   'recall': 0.9843178254051228,\n",
       "   'f1': 0.9825202191494912},\n",
       "  'exact': {'correct': 1902,\n",
       "   'incorrect': 8,\n",
       "   'partial': 0,\n",
       "   'missed': 3,\n",
       "   'spurious': 10,\n",
       "   'possible': 1913,\n",
       "   'actual': 1920,\n",
       "   'precision': 0.990625,\n",
       "   'recall': 0.9942498693152118,\n",
       "   'f1': 0.9924341247064962}}}"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results_by_tag"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Источник, если интересно: https://github.com/MantisAI/nervaluate/blob/main/README.md\n",
    "\n",
    "How can we incorporate these described scenarios into evaluation metrics? See the [original blog](http://www.davidsbatista.net/blog/2018/05/09/Named_Entity_Evaluation/) for a great explanation, a summary is included here:\n",
    "\n",
    "We can use the following five metrics to consider difference categories of errors:\n",
    "\n",
    "|Error type|Explanation|\n",
    "|---|---|\n",
    "|Correct (COR)|both are the same|\n",
    "|Incorrect (INC)|the output of a system and the golden annotation don’t match|\n",
    "|Partial (PAR)|system and the golden annotation are somewhat “similar” but not the same|\n",
    "|Missing (MIS)|a golden annotation is not captured by a system|\n",
    "|Spurius (SPU)|system produces a response which doesn’t exit in the golden annotation|\n",
    "\n",
    "These five metrics can be measured in four different ways:\n",
    "\n",
    "|Evaluation schema|Explanation|\n",
    "|---|---|\n",
    "|Strict|exact boundary surface string match and entity type|\n",
    "|Exact|exact boundary match over the surface string, regardless of the type|\n",
    "|Partial|partial boundary match over the surface string, regardless of the type|\n",
    "|Type|some overlap between the system tagged entity and the gold annotation is required|\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Основная проблема с типом тега\n",
    "- И с совмещением типа тега и его границ\n",
    "- Но в целом качество хорошее"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "flair",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}