test_attention_types.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !pip install xformers pandas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from xformers.ops.fmha.attn_bias import (\n",
    "    AttentionBias,\n",
    "    BlockDiagonalCausalMask,\n",
    "    BlockDiagonalCausalWithOffsetPaddedKeysMask,\n",
    "    BlockDiagonalMask,\n",
    ")\n",
    "\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "col_dict = {0.0: '#68A357', float('-inf'): '#C97064'}\n",
    "def colour_cell(val):\n",
    "    if val in col_dict:\n",
    "        return 'Background-color: %s' % col_dict[val]\n",
    "    return ''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_17144/2484427568.py:13: FutureWarning: Styler.applymap has been deprecated. Use Styler.map instead.\n",
      "  df.style.applymap(colour_cell)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<style type=\"text/css\">\n",
       "#T_93866_row0_col0, #T_93866_row1_col0, #T_93866_row1_col1, #T_93866_row2_col0, #T_93866_row2_col1, #T_93866_row2_col2, #T_93866_row3_col1, #T_93866_row3_col2, #T_93866_row3_col3, #T_93866_row4_col2, #T_93866_row4_col3, #T_93866_row4_col4, #T_93866_row5_col3, #T_93866_row5_col4, #T_93866_row5_col5, #T_93866_row6_col4, #T_93866_row6_col5, #T_93866_row6_col6, #T_93866_row7_col7, #T_93866_row8_col7, #T_93866_row8_col8, #T_93866_row9_col7, #T_93866_row9_col8, #T_93866_row9_col9, #T_93866_row10_col8, #T_93866_row10_col9, #T_93866_row10_col10, #T_93866_row11_col9, #T_93866_row11_col10, #T_93866_row11_col11, #T_93866_row12_col12, #T_93866_row13_col12, #T_93866_row13_col13, #T_93866_row14_col12, #T_93866_row14_col13, #T_93866_row14_col14, #T_93866_row15_col13, #T_93866_row15_col14, #T_93866_row15_col15, #T_93866_row16_col14, #T_93866_row16_col15, #T_93866_row16_col16, #T_93866_row17_col15, #T_93866_row17_col16, #T_93866_row17_col17 {\n",
       "  Background-color: #68A357;\n",
       "}\n",
       "#T_93866_row0_col1, #T_93866_row0_col2, #T_93866_row0_col3, #T_93866_row0_col4, #T_93866_row0_col5, #T_93866_row0_col6, #T_93866_row0_col7, #T_93866_row0_col8, #T_93866_row0_col9, #T_93866_row0_col10, #T_93866_row0_col11, #T_93866_row0_col12, #T_93866_row0_col13, #T_93866_row0_col14, #T_93866_row0_col15, #T_93866_row0_col16, #T_93866_row0_col17, #T_93866_row1_col2, #T_93866_row1_col3, #T_93866_row1_col4, #T_93866_row1_col5, #T_93866_row1_col6, #T_93866_row1_col7, #T_93866_row1_col8, #T_93866_row1_col9, #T_93866_row1_col10, #T_93866_row1_col11, #T_93866_row1_col12, #T_93866_row1_col13, #T_93866_row1_col14, #T_93866_row1_col15, #T_93866_row1_col16, #T_93866_row1_col17, #T_93866_row2_col3, #T_93866_row2_col4, #T_93866_row2_col5, #T_93866_row2_col6, #T_93866_row2_col7, #T_93866_row2_col8, #T_93866_row2_col9, #T_93866_row2_col10, #T_93866_row2_col11, #T_93866_row2_col12, #T_93866_row2_col13, #T_93866_row2_col14, #T_93866_row2_col15, #T_93866_row2_col16, #T_93866_row2_col17, #T_93866_row3_col0, #T_93866_row3_col4, #T_93866_row3_col5, #T_93866_row3_col6, #T_93866_row3_col7, #T_93866_row3_col8, #T_93866_row3_col9, #T_93866_row3_col10, #T_93866_row3_col11, #T_93866_row3_col12, #T_93866_row3_col13, #T_93866_row3_col14, #T_93866_row3_col15, #T_93866_row3_col16, #T_93866_row3_col17, #T_93866_row4_col0, #T_93866_row4_col1, #T_93866_row4_col5, #T_93866_row4_col6, #T_93866_row4_col7, #T_93866_row4_col8, #T_93866_row4_col9, #T_93866_row4_col10, #T_93866_row4_col11, #T_93866_row4_col12, #T_93866_row4_col13, #T_93866_row4_col14, #T_93866_row4_col15, #T_93866_row4_col16, #T_93866_row4_col17, #T_93866_row5_col0, #T_93866_row5_col1, #T_93866_row5_col2, #T_93866_row5_col6, #T_93866_row5_col7, #T_93866_row5_col8, #T_93866_row5_col9, #T_93866_row5_col10, #T_93866_row5_col11, #T_93866_row5_col12, #T_93866_row5_col13, #T_93866_row5_col14, #T_93866_row5_col15, #T_93866_row5_col16, #T_93866_row5_col17, #T_93866_row6_col0, #T_93866_row6_col1, #T_93866_row6_col2, #T_93866_row6_col3, #T_93866_row6_col7, #T_93866_row6_col8, #T_93866_row6_col9, #T_93866_row6_col10, #T_93866_row6_col11, #T_93866_row6_col12, #T_93866_row6_col13, #T_93866_row6_col14, #T_93866_row6_col15, #T_93866_row6_col16, #T_93866_row6_col17, #T_93866_row7_col0, #T_93866_row7_col1, #T_93866_row7_col2, #T_93866_row7_col3, #T_93866_row7_col4, #T_93866_row7_col5, #T_93866_row7_col6, #T_93866_row7_col8, #T_93866_row7_col9, #T_93866_row7_col10, #T_93866_row7_col11, #T_93866_row7_col12, #T_93866_row7_col13, #T_93866_row7_col14, #T_93866_row7_col15, #T_93866_row7_col16, #T_93866_row7_col17, #T_93866_row8_col0, #T_93866_row8_col1, #T_93866_row8_col2, #T_93866_row8_col3, #T_93866_row8_col4, #T_93866_row8_col5, #T_93866_row8_col6, #T_93866_row8_col9, #T_93866_row8_col10, #T_93866_row8_col11, #T_93866_row8_col12, #T_93866_row8_col13, #T_93866_row8_col14, #T_93866_row8_col15, #T_93866_row8_col16, #T_93866_row8_col17, #T_93866_row9_col0, #T_93866_row9_col1, #T_93866_row9_col2, #T_93866_row9_col3, #T_93866_row9_col4, #T_93866_row9_col5, #T_93866_row9_col6, #T_93866_row9_col10, #T_93866_row9_col11, #T_93866_row9_col12, #T_93866_row9_col13, #T_93866_row9_col14, #T_93866_row9_col15, #T_93866_row9_col16, #T_93866_row9_col17, #T_93866_row10_col0, #T_93866_row10_col1, #T_93866_row10_col2, #T_93866_row10_col3, #T_93866_row10_col4, #T_93866_row10_col5, #T_93866_row10_col6, #T_93866_row10_col7, #T_93866_row10_col11, #T_93866_row10_col12, #T_93866_row10_col13, #T_93866_row10_col14, #T_93866_row10_col15, #T_93866_row10_col16, #T_93866_row10_col17, #T_93866_row11_col0, #T_93866_row11_col1, #T_93866_row11_col2, #T_93866_row11_col3, #T_93866_row11_col4, #T_93866_row11_col5, #T_93866_row11_col6, #T_93866_row11_col7, #T_93866_row11_col8, #T_93866_row11_col12, #T_93866_row11_col13, #T_93866_row11_col14, #T_93866_row11_col15, #T_93866_row11_col16, #T_93866_row11_col17, #T_93866_row12_col0, #T_93866_row12_col1, #T_93866_row12_col2, #T_93866_row12_col3, #T_93866_row12_col4, #T_93866_row12_col5, #T_93866_row12_col6, #T_93866_row12_col7, #T_93866_row12_col8, #T_93866_row12_col9, #T_93866_row12_col10, #T_93866_row12_col11, #T_93866_row12_col13, #T_93866_row12_col14, #T_93866_row12_col15, #T_93866_row12_col16, #T_93866_row12_col17, #T_93866_row13_col0, #T_93866_row13_col1, #T_93866_row13_col2, #T_93866_row13_col3, #T_93866_row13_col4, #T_93866_row13_col5, #T_93866_row13_col6, #T_93866_row13_col7, #T_93866_row13_col8, #T_93866_row13_col9, #T_93866_row13_col10, #T_93866_row13_col11, #T_93866_row13_col14, #T_93866_row13_col15, #T_93866_row13_col16, #T_93866_row13_col17, #T_93866_row14_col0, #T_93866_row14_col1, #T_93866_row14_col2, #T_93866_row14_col3, #T_93866_row14_col4, #T_93866_row14_col5, #T_93866_row14_col6, #T_93866_row14_col7, #T_93866_row14_col8, #T_93866_row14_col9, #T_93866_row14_col10, #T_93866_row14_col11, #T_93866_row14_col15, #T_93866_row14_col16, #T_93866_row14_col17, #T_93866_row15_col0, #T_93866_row15_col1, #T_93866_row15_col2, #T_93866_row15_col3, #T_93866_row15_col4, #T_93866_row15_col5, #T_93866_row15_col6, #T_93866_row15_col7, #T_93866_row15_col8, #T_93866_row15_col9, #T_93866_row15_col10, #T_93866_row15_col11, #T_93866_row15_col12, #T_93866_row15_col16, #T_93866_row15_col17, #T_93866_row16_col0, #T_93866_row16_col1, #T_93866_row16_col2, #T_93866_row16_col3, #T_93866_row16_col4, #T_93866_row16_col5, #T_93866_row16_col6, #T_93866_row16_col7, #T_93866_row16_col8, #T_93866_row16_col9, #T_93866_row16_col10, #T_93866_row16_col11, #T_93866_row16_col12, #T_93866_row16_col13, #T_93866_row16_col17, #T_93866_row17_col0, #T_93866_row17_col1, #T_93866_row17_col2, #T_93866_row17_col3, #T_93866_row17_col4, #T_93866_row17_col5, #T_93866_row17_col6, #T_93866_row17_col7, #T_93866_row17_col8, #T_93866_row17_col9, #T_93866_row17_col10, #T_93866_row17_col11, #T_93866_row17_col12, #T_93866_row17_col13, #T_93866_row17_col14 {\n",
       "  Background-color: #C97064;\n",
       "}\n",
       "</style>\n",
       "<table id=\"T_93866\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th class=\"blank level0\" >&nbsp;</th>\n",
       "      <th id=\"T_93866_level0_col0\" class=\"col_heading level0 col0\" >0</th>\n",
       "      <th id=\"T_93866_level0_col1\" class=\"col_heading level0 col1\" >1</th>\n",
       "      <th id=\"T_93866_level0_col2\" class=\"col_heading level0 col2\" >2</th>\n",
       "      <th id=\"T_93866_level0_col3\" class=\"col_heading level0 col3\" >3</th>\n",
       "      <th id=\"T_93866_level0_col4\" class=\"col_heading level0 col4\" >4</th>\n",
       "      <th id=\"T_93866_level0_col5\" class=\"col_heading level0 col5\" >5</th>\n",
       "      <th id=\"T_93866_level0_col6\" class=\"col_heading level0 col6\" >6</th>\n",
       "      <th id=\"T_93866_level0_col7\" class=\"col_heading level0 col7\" >7</th>\n",
       "      <th id=\"T_93866_level0_col8\" class=\"col_heading level0 col8\" >8</th>\n",
       "      <th id=\"T_93866_level0_col9\" class=\"col_heading level0 col9\" >9</th>\n",
       "      <th id=\"T_93866_level0_col10\" class=\"col_heading level0 col10\" >10</th>\n",
       "      <th id=\"T_93866_level0_col11\" class=\"col_heading level0 col11\" >11</th>\n",
       "      <th id=\"T_93866_level0_col12\" class=\"col_heading level0 col12\" >12</th>\n",
       "      <th id=\"T_93866_level0_col13\" class=\"col_heading level0 col13\" >13</th>\n",
       "      <th id=\"T_93866_level0_col14\" class=\"col_heading level0 col14\" >14</th>\n",
       "      <th id=\"T_93866_level0_col15\" class=\"col_heading level0 col15\" >15</th>\n",
       "      <th id=\"T_93866_level0_col16\" class=\"col_heading level0 col16\" >16</th>\n",
       "      <th id=\"T_93866_level0_col17\" class=\"col_heading level0 col17\" >17</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th id=\"T_93866_level0_row0\" class=\"row_heading level0 row0\" >0</th>\n",
       "      <td id=\"T_93866_row0_col0\" class=\"data row0 col0\" >0.000000</td>\n",
       "      <td id=\"T_93866_row0_col1\" class=\"data row0 col1\" >-inf</td>\n",
       "      <td id=\"T_93866_row0_col2\" class=\"data row0 col2\" >-inf</td>\n",
       "      <td id=\"T_93866_row0_col3\" class=\"data row0 col3\" >-inf</td>\n",
       "      <td id=\"T_93866_row0_col4\" class=\"data row0 col4\" >-inf</td>\n",
       "      <td id=\"T_93866_row0_col5\" class=\"data row0 col5\" >-inf</td>\n",
       "      <td id=\"T_93866_row0_col6\" class=\"data row0 col6\" >-inf</td>\n",
       "      <td id=\"T_93866_row0_col7\" class=\"data row0 col7\" >-inf</td>\n",
       "      <td id=\"T_93866_row0_col8\" class=\"data row0 col8\" >-inf</td>\n",
       "      <td id=\"T_93866_row0_col9\" class=\"data row0 col9\" >-inf</td>\n",
       "      <td id=\"T_93866_row0_col10\" class=\"data row0 col10\" >-inf</td>\n",
       "      <td id=\"T_93866_row0_col11\" class=\"data row0 col11\" >-inf</td>\n",
       "      <td id=\"T_93866_row0_col12\" class=\"data row0 col12\" >-inf</td>\n",
       "      <td id=\"T_93866_row0_col13\" class=\"data row0 col13\" >-inf</td>\n",
       "      <td id=\"T_93866_row0_col14\" class=\"data row0 col14\" >-inf</td>\n",
       "      <td id=\"T_93866_row0_col15\" class=\"data row0 col15\" >-inf</td>\n",
       "      <td id=\"T_93866_row0_col16\" class=\"data row0 col16\" >-inf</td>\n",
       "      <td id=\"T_93866_row0_col17\" class=\"data row0 col17\" >-inf</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_93866_level0_row1\" class=\"row_heading level0 row1\" >1</th>\n",
       "      <td id=\"T_93866_row1_col0\" class=\"data row1 col0\" >0.000000</td>\n",
       "      <td id=\"T_93866_row1_col1\" class=\"data row1 col1\" >0.000000</td>\n",
       "      <td id=\"T_93866_row1_col2\" class=\"data row1 col2\" >-inf</td>\n",
       "      <td id=\"T_93866_row1_col3\" class=\"data row1 col3\" >-inf</td>\n",
       "      <td id=\"T_93866_row1_col4\" class=\"data row1 col4\" >-inf</td>\n",
       "      <td id=\"T_93866_row1_col5\" class=\"data row1 col5\" >-inf</td>\n",
       "      <td id=\"T_93866_row1_col6\" class=\"data row1 col6\" >-inf</td>\n",
       "      <td id=\"T_93866_row1_col7\" class=\"data row1 col7\" >-inf</td>\n",
       "      <td id=\"T_93866_row1_col8\" class=\"data row1 col8\" >-inf</td>\n",
       "      <td id=\"T_93866_row1_col9\" class=\"data row1 col9\" >-inf</td>\n",
       "      <td id=\"T_93866_row1_col10\" class=\"data row1 col10\" >-inf</td>\n",
       "      <td id=\"T_93866_row1_col11\" class=\"data row1 col11\" >-inf</td>\n",
       "      <td id=\"T_93866_row1_col12\" class=\"data row1 col12\" >-inf</td>\n",
       "      <td id=\"T_93866_row1_col13\" class=\"data row1 col13\" >-inf</td>\n",
       "      <td id=\"T_93866_row1_col14\" class=\"data row1 col14\" >-inf</td>\n",
       "      <td id=\"T_93866_row1_col15\" class=\"data row1 col15\" >-inf</td>\n",
       "      <td id=\"T_93866_row1_col16\" class=\"data row1 col16\" >-inf</td>\n",
       "      <td id=\"T_93866_row1_col17\" class=\"data row1 col17\" >-inf</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_93866_level0_row2\" class=\"row_heading level0 row2\" >2</th>\n",
       "      <td id=\"T_93866_row2_col0\" class=\"data row2 col0\" >0.000000</td>\n",
       "      <td id=\"T_93866_row2_col1\" class=\"data row2 col1\" >0.000000</td>\n",
       "      <td id=\"T_93866_row2_col2\" class=\"data row2 col2\" >0.000000</td>\n",
       "      <td id=\"T_93866_row2_col3\" class=\"data row2 col3\" >-inf</td>\n",
       "      <td id=\"T_93866_row2_col4\" class=\"data row2 col4\" >-inf</td>\n",
       "      <td id=\"T_93866_row2_col5\" class=\"data row2 col5\" >-inf</td>\n",
       "      <td id=\"T_93866_row2_col6\" class=\"data row2 col6\" >-inf</td>\n",
       "      <td id=\"T_93866_row2_col7\" class=\"data row2 col7\" >-inf</td>\n",
       "      <td id=\"T_93866_row2_col8\" class=\"data row2 col8\" >-inf</td>\n",
       "      <td id=\"T_93866_row2_col9\" class=\"data row2 col9\" >-inf</td>\n",
       "      <td id=\"T_93866_row2_col10\" class=\"data row2 col10\" >-inf</td>\n",
       "      <td id=\"T_93866_row2_col11\" class=\"data row2 col11\" >-inf</td>\n",
       "      <td id=\"T_93866_row2_col12\" class=\"data row2 col12\" >-inf</td>\n",
       "      <td id=\"T_93866_row2_col13\" class=\"data row2 col13\" >-inf</td>\n",
       "      <td id=\"T_93866_row2_col14\" class=\"data row2 col14\" >-inf</td>\n",
       "      <td id=\"T_93866_row2_col15\" class=\"data row2 col15\" >-inf</td>\n",
       "      <td id=\"T_93866_row2_col16\" class=\"data row2 col16\" >-inf</td>\n",
       "      <td id=\"T_93866_row2_col17\" class=\"data row2 col17\" >-inf</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_93866_level0_row3\" class=\"row_heading level0 row3\" >3</th>\n",
       "      <td id=\"T_93866_row3_col0\" class=\"data row3 col0\" >-inf</td>\n",
       "      <td id=\"T_93866_row3_col1\" class=\"data row3 col1\" >0.000000</td>\n",
       "      <td id=\"T_93866_row3_col2\" class=\"data row3 col2\" >0.000000</td>\n",
       "      <td id=\"T_93866_row3_col3\" class=\"data row3 col3\" >0.000000</td>\n",
       "      <td id=\"T_93866_row3_col4\" class=\"data row3 col4\" >-inf</td>\n",
       "      <td id=\"T_93866_row3_col5\" class=\"data row3 col5\" >-inf</td>\n",
       "      <td id=\"T_93866_row3_col6\" class=\"data row3 col6\" >-inf</td>\n",
       "      <td id=\"T_93866_row3_col7\" class=\"data row3 col7\" >-inf</td>\n",
       "      <td id=\"T_93866_row3_col8\" class=\"data row3 col8\" >-inf</td>\n",
       "      <td id=\"T_93866_row3_col9\" class=\"data row3 col9\" >-inf</td>\n",
       "      <td id=\"T_93866_row3_col10\" class=\"data row3 col10\" >-inf</td>\n",
       "      <td id=\"T_93866_row3_col11\" class=\"data row3 col11\" >-inf</td>\n",
       "      <td id=\"T_93866_row3_col12\" class=\"data row3 col12\" >-inf</td>\n",
       "      <td id=\"T_93866_row3_col13\" class=\"data row3 col13\" >-inf</td>\n",
       "      <td id=\"T_93866_row3_col14\" class=\"data row3 col14\" >-inf</td>\n",
       "      <td id=\"T_93866_row3_col15\" class=\"data row3 col15\" >-inf</td>\n",
       "      <td id=\"T_93866_row3_col16\" class=\"data row3 col16\" >-inf</td>\n",
       "      <td id=\"T_93866_row3_col17\" class=\"data row3 col17\" >-inf</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_93866_level0_row4\" class=\"row_heading level0 row4\" >4</th>\n",
       "      <td id=\"T_93866_row4_col0\" class=\"data row4 col0\" >-inf</td>\n",
       "      <td id=\"T_93866_row4_col1\" class=\"data row4 col1\" >-inf</td>\n",
       "      <td id=\"T_93866_row4_col2\" class=\"data row4 col2\" >0.000000</td>\n",
       "      <td id=\"T_93866_row4_col3\" class=\"data row4 col3\" >0.000000</td>\n",
       "      <td id=\"T_93866_row4_col4\" class=\"data row4 col4\" >0.000000</td>\n",
       "      <td id=\"T_93866_row4_col5\" class=\"data row4 col5\" >-inf</td>\n",
       "      <td id=\"T_93866_row4_col6\" class=\"data row4 col6\" >-inf</td>\n",
       "      <td id=\"T_93866_row4_col7\" class=\"data row4 col7\" >-inf</td>\n",
       "      <td id=\"T_93866_row4_col8\" class=\"data row4 col8\" >-inf</td>\n",
       "      <td id=\"T_93866_row4_col9\" class=\"data row4 col9\" >-inf</td>\n",
       "      <td id=\"T_93866_row4_col10\" class=\"data row4 col10\" >-inf</td>\n",
       "      <td id=\"T_93866_row4_col11\" class=\"data row4 col11\" >-inf</td>\n",
       "      <td id=\"T_93866_row4_col12\" class=\"data row4 col12\" >-inf</td>\n",
       "      <td id=\"T_93866_row4_col13\" class=\"data row4 col13\" >-inf</td>\n",
       "      <td id=\"T_93866_row4_col14\" class=\"data row4 col14\" >-inf</td>\n",
       "      <td id=\"T_93866_row4_col15\" class=\"data row4 col15\" >-inf</td>\n",
       "      <td id=\"T_93866_row4_col16\" class=\"data row4 col16\" >-inf</td>\n",
       "      <td id=\"T_93866_row4_col17\" class=\"data row4 col17\" >-inf</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_93866_level0_row5\" class=\"row_heading level0 row5\" >5</th>\n",
       "      <td id=\"T_93866_row5_col0\" class=\"data row5 col0\" >-inf</td>\n",
       "      <td id=\"T_93866_row5_col1\" class=\"data row5 col1\" >-inf</td>\n",
       "      <td id=\"T_93866_row5_col2\" class=\"data row5 col2\" >-inf</td>\n",
       "      <td id=\"T_93866_row5_col3\" class=\"data row5 col3\" >0.000000</td>\n",
       "      <td id=\"T_93866_row5_col4\" class=\"data row5 col4\" >0.000000</td>\n",
       "      <td id=\"T_93866_row5_col5\" class=\"data row5 col5\" >0.000000</td>\n",
       "      <td id=\"T_93866_row5_col6\" class=\"data row5 col6\" >-inf</td>\n",
       "      <td id=\"T_93866_row5_col7\" class=\"data row5 col7\" >-inf</td>\n",
       "      <td id=\"T_93866_row5_col8\" class=\"data row5 col8\" >-inf</td>\n",
       "      <td id=\"T_93866_row5_col9\" class=\"data row5 col9\" >-inf</td>\n",
       "      <td id=\"T_93866_row5_col10\" class=\"data row5 col10\" >-inf</td>\n",
       "      <td id=\"T_93866_row5_col11\" class=\"data row5 col11\" >-inf</td>\n",
       "      <td id=\"T_93866_row5_col12\" class=\"data row5 col12\" >-inf</td>\n",
       "      <td id=\"T_93866_row5_col13\" class=\"data row5 col13\" >-inf</td>\n",
       "      <td id=\"T_93866_row5_col14\" class=\"data row5 col14\" >-inf</td>\n",
       "      <td id=\"T_93866_row5_col15\" class=\"data row5 col15\" >-inf</td>\n",
       "      <td id=\"T_93866_row5_col16\" class=\"data row5 col16\" >-inf</td>\n",
       "      <td id=\"T_93866_row5_col17\" class=\"data row5 col17\" >-inf</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_93866_level0_row6\" class=\"row_heading level0 row6\" >6</th>\n",
       "      <td id=\"T_93866_row6_col0\" class=\"data row6 col0\" >-inf</td>\n",
       "      <td id=\"T_93866_row6_col1\" class=\"data row6 col1\" >-inf</td>\n",
       "      <td id=\"T_93866_row6_col2\" class=\"data row6 col2\" >-inf</td>\n",
       "      <td id=\"T_93866_row6_col3\" class=\"data row6 col3\" >-inf</td>\n",
       "      <td id=\"T_93866_row6_col4\" class=\"data row6 col4\" >0.000000</td>\n",
       "      <td id=\"T_93866_row6_col5\" class=\"data row6 col5\" >0.000000</td>\n",
       "      <td id=\"T_93866_row6_col6\" class=\"data row6 col6\" >0.000000</td>\n",
       "      <td id=\"T_93866_row6_col7\" class=\"data row6 col7\" >-inf</td>\n",
       "      <td id=\"T_93866_row6_col8\" class=\"data row6 col8\" >-inf</td>\n",
       "      <td id=\"T_93866_row6_col9\" class=\"data row6 col9\" >-inf</td>\n",
       "      <td id=\"T_93866_row6_col10\" class=\"data row6 col10\" >-inf</td>\n",
       "      <td id=\"T_93866_row6_col11\" class=\"data row6 col11\" >-inf</td>\n",
       "      <td id=\"T_93866_row6_col12\" class=\"data row6 col12\" >-inf</td>\n",
       "      <td id=\"T_93866_row6_col13\" class=\"data row6 col13\" >-inf</td>\n",
       "      <td id=\"T_93866_row6_col14\" class=\"data row6 col14\" >-inf</td>\n",
       "      <td id=\"T_93866_row6_col15\" class=\"data row6 col15\" >-inf</td>\n",
       "      <td id=\"T_93866_row6_col16\" class=\"data row6 col16\" >-inf</td>\n",
       "      <td id=\"T_93866_row6_col17\" class=\"data row6 col17\" >-inf</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_93866_level0_row7\" class=\"row_heading level0 row7\" >7</th>\n",
       "      <td id=\"T_93866_row7_col0\" class=\"data row7 col0\" >-inf</td>\n",
       "      <td id=\"T_93866_row7_col1\" class=\"data row7 col1\" >-inf</td>\n",
       "      <td id=\"T_93866_row7_col2\" class=\"data row7 col2\" >-inf</td>\n",
       "      <td id=\"T_93866_row7_col3\" class=\"data row7 col3\" >-inf</td>\n",
       "      <td id=\"T_93866_row7_col4\" class=\"data row7 col4\" >-inf</td>\n",
       "      <td id=\"T_93866_row7_col5\" class=\"data row7 col5\" >-inf</td>\n",
       "      <td id=\"T_93866_row7_col6\" class=\"data row7 col6\" >-inf</td>\n",
       "      <td id=\"T_93866_row7_col7\" class=\"data row7 col7\" >0.000000</td>\n",
       "      <td id=\"T_93866_row7_col8\" class=\"data row7 col8\" >-inf</td>\n",
       "      <td id=\"T_93866_row7_col9\" class=\"data row7 col9\" >-inf</td>\n",
       "      <td id=\"T_93866_row7_col10\" class=\"data row7 col10\" >-inf</td>\n",
       "      <td id=\"T_93866_row7_col11\" class=\"data row7 col11\" >-inf</td>\n",
       "      <td id=\"T_93866_row7_col12\" class=\"data row7 col12\" >-inf</td>\n",
       "      <td id=\"T_93866_row7_col13\" class=\"data row7 col13\" >-inf</td>\n",
       "      <td id=\"T_93866_row7_col14\" class=\"data row7 col14\" >-inf</td>\n",
       "      <td id=\"T_93866_row7_col15\" class=\"data row7 col15\" >-inf</td>\n",
       "      <td id=\"T_93866_row7_col16\" class=\"data row7 col16\" >-inf</td>\n",
       "      <td id=\"T_93866_row7_col17\" class=\"data row7 col17\" >-inf</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_93866_level0_row8\" class=\"row_heading level0 row8\" >8</th>\n",
       "      <td id=\"T_93866_row8_col0\" class=\"data row8 col0\" >-inf</td>\n",
       "      <td id=\"T_93866_row8_col1\" class=\"data row8 col1\" >-inf</td>\n",
       "      <td id=\"T_93866_row8_col2\" class=\"data row8 col2\" >-inf</td>\n",
       "      <td id=\"T_93866_row8_col3\" class=\"data row8 col3\" >-inf</td>\n",
       "      <td id=\"T_93866_row8_col4\" class=\"data row8 col4\" >-inf</td>\n",
       "      <td id=\"T_93866_row8_col5\" class=\"data row8 col5\" >-inf</td>\n",
       "      <td id=\"T_93866_row8_col6\" class=\"data row8 col6\" >-inf</td>\n",
       "      <td id=\"T_93866_row8_col7\" class=\"data row8 col7\" >0.000000</td>\n",
       "      <td id=\"T_93866_row8_col8\" class=\"data row8 col8\" >0.000000</td>\n",
       "      <td id=\"T_93866_row8_col9\" class=\"data row8 col9\" >-inf</td>\n",
       "      <td id=\"T_93866_row8_col10\" class=\"data row8 col10\" >-inf</td>\n",
       "      <td id=\"T_93866_row8_col11\" class=\"data row8 col11\" >-inf</td>\n",
       "      <td id=\"T_93866_row8_col12\" class=\"data row8 col12\" >-inf</td>\n",
       "      <td id=\"T_93866_row8_col13\" class=\"data row8 col13\" >-inf</td>\n",
       "      <td id=\"T_93866_row8_col14\" class=\"data row8 col14\" >-inf</td>\n",
       "      <td id=\"T_93866_row8_col15\" class=\"data row8 col15\" >-inf</td>\n",
       "      <td id=\"T_93866_row8_col16\" class=\"data row8 col16\" >-inf</td>\n",
       "      <td id=\"T_93866_row8_col17\" class=\"data row8 col17\" >-inf</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_93866_level0_row9\" class=\"row_heading level0 row9\" >9</th>\n",
       "      <td id=\"T_93866_row9_col0\" class=\"data row9 col0\" >-inf</td>\n",
       "      <td id=\"T_93866_row9_col1\" class=\"data row9 col1\" >-inf</td>\n",
       "      <td id=\"T_93866_row9_col2\" class=\"data row9 col2\" >-inf</td>\n",
       "      <td id=\"T_93866_row9_col3\" class=\"data row9 col3\" >-inf</td>\n",
       "      <td id=\"T_93866_row9_col4\" class=\"data row9 col4\" >-inf</td>\n",
       "      <td id=\"T_93866_row9_col5\" class=\"data row9 col5\" >-inf</td>\n",
       "      <td id=\"T_93866_row9_col6\" class=\"data row9 col6\" >-inf</td>\n",
       "      <td id=\"T_93866_row9_col7\" class=\"data row9 col7\" >0.000000</td>\n",
       "      <td id=\"T_93866_row9_col8\" class=\"data row9 col8\" >0.000000</td>\n",
       "      <td id=\"T_93866_row9_col9\" class=\"data row9 col9\" >0.000000</td>\n",
       "      <td id=\"T_93866_row9_col10\" class=\"data row9 col10\" >-inf</td>\n",
       "      <td id=\"T_93866_row9_col11\" class=\"data row9 col11\" >-inf</td>\n",
       "      <td id=\"T_93866_row9_col12\" class=\"data row9 col12\" >-inf</td>\n",
       "      <td id=\"T_93866_row9_col13\" class=\"data row9 col13\" >-inf</td>\n",
       "      <td id=\"T_93866_row9_col14\" class=\"data row9 col14\" >-inf</td>\n",
       "      <td id=\"T_93866_row9_col15\" class=\"data row9 col15\" >-inf</td>\n",
       "      <td id=\"T_93866_row9_col16\" class=\"data row9 col16\" >-inf</td>\n",
       "      <td id=\"T_93866_row9_col17\" class=\"data row9 col17\" >-inf</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_93866_level0_row10\" class=\"row_heading level0 row10\" >10</th>\n",
       "      <td id=\"T_93866_row10_col0\" class=\"data row10 col0\" >-inf</td>\n",
       "      <td id=\"T_93866_row10_col1\" class=\"data row10 col1\" >-inf</td>\n",
       "      <td id=\"T_93866_row10_col2\" class=\"data row10 col2\" >-inf</td>\n",
       "      <td id=\"T_93866_row10_col3\" class=\"data row10 col3\" >-inf</td>\n",
       "      <td id=\"T_93866_row10_col4\" class=\"data row10 col4\" >-inf</td>\n",
       "      <td id=\"T_93866_row10_col5\" class=\"data row10 col5\" >-inf</td>\n",
       "      <td id=\"T_93866_row10_col6\" class=\"data row10 col6\" >-inf</td>\n",
       "      <td id=\"T_93866_row10_col7\" class=\"data row10 col7\" >-inf</td>\n",
       "      <td id=\"T_93866_row10_col8\" class=\"data row10 col8\" >0.000000</td>\n",
       "      <td id=\"T_93866_row10_col9\" class=\"data row10 col9\" >0.000000</td>\n",
       "      <td id=\"T_93866_row10_col10\" class=\"data row10 col10\" >0.000000</td>\n",
       "      <td id=\"T_93866_row10_col11\" class=\"data row10 col11\" >-inf</td>\n",
       "      <td id=\"T_93866_row10_col12\" class=\"data row10 col12\" >-inf</td>\n",
       "      <td id=\"T_93866_row10_col13\" class=\"data row10 col13\" >-inf</td>\n",
       "      <td id=\"T_93866_row10_col14\" class=\"data row10 col14\" >-inf</td>\n",
       "      <td id=\"T_93866_row10_col15\" class=\"data row10 col15\" >-inf</td>\n",
       "      <td id=\"T_93866_row10_col16\" class=\"data row10 col16\" >-inf</td>\n",
       "      <td id=\"T_93866_row10_col17\" class=\"data row10 col17\" >-inf</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_93866_level0_row11\" class=\"row_heading level0 row11\" >11</th>\n",
       "      <td id=\"T_93866_row11_col0\" class=\"data row11 col0\" >-inf</td>\n",
       "      <td id=\"T_93866_row11_col1\" class=\"data row11 col1\" >-inf</td>\n",
       "      <td id=\"T_93866_row11_col2\" class=\"data row11 col2\" >-inf</td>\n",
       "      <td id=\"T_93866_row11_col3\" class=\"data row11 col3\" >-inf</td>\n",
       "      <td id=\"T_93866_row11_col4\" class=\"data row11 col4\" >-inf</td>\n",
       "      <td id=\"T_93866_row11_col5\" class=\"data row11 col5\" >-inf</td>\n",
       "      <td id=\"T_93866_row11_col6\" class=\"data row11 col6\" >-inf</td>\n",
       "      <td id=\"T_93866_row11_col7\" class=\"data row11 col7\" >-inf</td>\n",
       "      <td id=\"T_93866_row11_col8\" class=\"data row11 col8\" >-inf</td>\n",
       "      <td id=\"T_93866_row11_col9\" class=\"data row11 col9\" >0.000000</td>\n",
       "      <td id=\"T_93866_row11_col10\" class=\"data row11 col10\" >0.000000</td>\n",
       "      <td id=\"T_93866_row11_col11\" class=\"data row11 col11\" >0.000000</td>\n",
       "      <td id=\"T_93866_row11_col12\" class=\"data row11 col12\" >-inf</td>\n",
       "      <td id=\"T_93866_row11_col13\" class=\"data row11 col13\" >-inf</td>\n",
       "      <td id=\"T_93866_row11_col14\" class=\"data row11 col14\" >-inf</td>\n",
       "      <td id=\"T_93866_row11_col15\" class=\"data row11 col15\" >-inf</td>\n",
       "      <td id=\"T_93866_row11_col16\" class=\"data row11 col16\" >-inf</td>\n",
       "      <td id=\"T_93866_row11_col17\" class=\"data row11 col17\" >-inf</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_93866_level0_row12\" class=\"row_heading level0 row12\" >12</th>\n",
       "      <td id=\"T_93866_row12_col0\" class=\"data row12 col0\" >-inf</td>\n",
       "      <td id=\"T_93866_row12_col1\" class=\"data row12 col1\" >-inf</td>\n",
       "      <td id=\"T_93866_row12_col2\" class=\"data row12 col2\" >-inf</td>\n",
       "      <td id=\"T_93866_row12_col3\" class=\"data row12 col3\" >-inf</td>\n",
       "      <td id=\"T_93866_row12_col4\" class=\"data row12 col4\" >-inf</td>\n",
       "      <td id=\"T_93866_row12_col5\" class=\"data row12 col5\" >-inf</td>\n",
       "      <td id=\"T_93866_row12_col6\" class=\"data row12 col6\" >-inf</td>\n",
       "      <td id=\"T_93866_row12_col7\" class=\"data row12 col7\" >-inf</td>\n",
       "      <td id=\"T_93866_row12_col8\" class=\"data row12 col8\" >-inf</td>\n",
       "      <td id=\"T_93866_row12_col9\" class=\"data row12 col9\" >-inf</td>\n",
       "      <td id=\"T_93866_row12_col10\" class=\"data row12 col10\" >-inf</td>\n",
       "      <td id=\"T_93866_row12_col11\" class=\"data row12 col11\" >-inf</td>\n",
       "      <td id=\"T_93866_row12_col12\" class=\"data row12 col12\" >0.000000</td>\n",
       "      <td id=\"T_93866_row12_col13\" class=\"data row12 col13\" >-inf</td>\n",
       "      <td id=\"T_93866_row12_col14\" class=\"data row12 col14\" >-inf</td>\n",
       "      <td id=\"T_93866_row12_col15\" class=\"data row12 col15\" >-inf</td>\n",
       "      <td id=\"T_93866_row12_col16\" class=\"data row12 col16\" >-inf</td>\n",
       "      <td id=\"T_93866_row12_col17\" class=\"data row12 col17\" >-inf</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_93866_level0_row13\" class=\"row_heading level0 row13\" >13</th>\n",
       "      <td id=\"T_93866_row13_col0\" class=\"data row13 col0\" >-inf</td>\n",
       "      <td id=\"T_93866_row13_col1\" class=\"data row13 col1\" >-inf</td>\n",
       "      <td id=\"T_93866_row13_col2\" class=\"data row13 col2\" >-inf</td>\n",
       "      <td id=\"T_93866_row13_col3\" class=\"data row13 col3\" >-inf</td>\n",
       "      <td id=\"T_93866_row13_col4\" class=\"data row13 col4\" >-inf</td>\n",
       "      <td id=\"T_93866_row13_col5\" class=\"data row13 col5\" >-inf</td>\n",
       "      <td id=\"T_93866_row13_col6\" class=\"data row13 col6\" >-inf</td>\n",
       "      <td id=\"T_93866_row13_col7\" class=\"data row13 col7\" >-inf</td>\n",
       "      <td id=\"T_93866_row13_col8\" class=\"data row13 col8\" >-inf</td>\n",
       "      <td id=\"T_93866_row13_col9\" class=\"data row13 col9\" >-inf</td>\n",
       "      <td id=\"T_93866_row13_col10\" class=\"data row13 col10\" >-inf</td>\n",
       "      <td id=\"T_93866_row13_col11\" class=\"data row13 col11\" >-inf</td>\n",
       "      <td id=\"T_93866_row13_col12\" class=\"data row13 col12\" >0.000000</td>\n",
       "      <td id=\"T_93866_row13_col13\" class=\"data row13 col13\" >0.000000</td>\n",
       "      <td id=\"T_93866_row13_col14\" class=\"data row13 col14\" >-inf</td>\n",
       "      <td id=\"T_93866_row13_col15\" class=\"data row13 col15\" >-inf</td>\n",
       "      <td id=\"T_93866_row13_col16\" class=\"data row13 col16\" >-inf</td>\n",
       "      <td id=\"T_93866_row13_col17\" class=\"data row13 col17\" >-inf</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_93866_level0_row14\" class=\"row_heading level0 row14\" >14</th>\n",
       "      <td id=\"T_93866_row14_col0\" class=\"data row14 col0\" >-inf</td>\n",
       "      <td id=\"T_93866_row14_col1\" class=\"data row14 col1\" >-inf</td>\n",
       "      <td id=\"T_93866_row14_col2\" class=\"data row14 col2\" >-inf</td>\n",
       "      <td id=\"T_93866_row14_col3\" class=\"data row14 col3\" >-inf</td>\n",
       "      <td id=\"T_93866_row14_col4\" class=\"data row14 col4\" >-inf</td>\n",
       "      <td id=\"T_93866_row14_col5\" class=\"data row14 col5\" >-inf</td>\n",
       "      <td id=\"T_93866_row14_col6\" class=\"data row14 col6\" >-inf</td>\n",
       "      <td id=\"T_93866_row14_col7\" class=\"data row14 col7\" >-inf</td>\n",
       "      <td id=\"T_93866_row14_col8\" class=\"data row14 col8\" >-inf</td>\n",
       "      <td id=\"T_93866_row14_col9\" class=\"data row14 col9\" >-inf</td>\n",
       "      <td id=\"T_93866_row14_col10\" class=\"data row14 col10\" >-inf</td>\n",
       "      <td id=\"T_93866_row14_col11\" class=\"data row14 col11\" >-inf</td>\n",
       "      <td id=\"T_93866_row14_col12\" class=\"data row14 col12\" >0.000000</td>\n",
       "      <td id=\"T_93866_row14_col13\" class=\"data row14 col13\" >0.000000</td>\n",
       "      <td id=\"T_93866_row14_col14\" class=\"data row14 col14\" >0.000000</td>\n",
       "      <td id=\"T_93866_row14_col15\" class=\"data row14 col15\" >-inf</td>\n",
       "      <td id=\"T_93866_row14_col16\" class=\"data row14 col16\" >-inf</td>\n",
       "      <td id=\"T_93866_row14_col17\" class=\"data row14 col17\" >-inf</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_93866_level0_row15\" class=\"row_heading level0 row15\" >15</th>\n",
       "      <td id=\"T_93866_row15_col0\" class=\"data row15 col0\" >-inf</td>\n",
       "      <td id=\"T_93866_row15_col1\" class=\"data row15 col1\" >-inf</td>\n",
       "      <td id=\"T_93866_row15_col2\" class=\"data row15 col2\" >-inf</td>\n",
       "      <td id=\"T_93866_row15_col3\" class=\"data row15 col3\" >-inf</td>\n",
       "      <td id=\"T_93866_row15_col4\" class=\"data row15 col4\" >-inf</td>\n",
       "      <td id=\"T_93866_row15_col5\" class=\"data row15 col5\" >-inf</td>\n",
       "      <td id=\"T_93866_row15_col6\" class=\"data row15 col6\" >-inf</td>\n",
       "      <td id=\"T_93866_row15_col7\" class=\"data row15 col7\" >-inf</td>\n",
       "      <td id=\"T_93866_row15_col8\" class=\"data row15 col8\" >-inf</td>\n",
       "      <td id=\"T_93866_row15_col9\" class=\"data row15 col9\" >-inf</td>\n",
       "      <td id=\"T_93866_row15_col10\" class=\"data row15 col10\" >-inf</td>\n",
       "      <td id=\"T_93866_row15_col11\" class=\"data row15 col11\" >-inf</td>\n",
       "      <td id=\"T_93866_row15_col12\" class=\"data row15 col12\" >-inf</td>\n",
       "      <td id=\"T_93866_row15_col13\" class=\"data row15 col13\" >0.000000</td>\n",
       "      <td id=\"T_93866_row15_col14\" class=\"data row15 col14\" >0.000000</td>\n",
       "      <td id=\"T_93866_row15_col15\" class=\"data row15 col15\" >0.000000</td>\n",
       "      <td id=\"T_93866_row15_col16\" class=\"data row15 col16\" >-inf</td>\n",
       "      <td id=\"T_93866_row15_col17\" class=\"data row15 col17\" >-inf</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_93866_level0_row16\" class=\"row_heading level0 row16\" >16</th>\n",
       "      <td id=\"T_93866_row16_col0\" class=\"data row16 col0\" >-inf</td>\n",
       "      <td id=\"T_93866_row16_col1\" class=\"data row16 col1\" >-inf</td>\n",
       "      <td id=\"T_93866_row16_col2\" class=\"data row16 col2\" >-inf</td>\n",
       "      <td id=\"T_93866_row16_col3\" class=\"data row16 col3\" >-inf</td>\n",
       "      <td id=\"T_93866_row16_col4\" class=\"data row16 col4\" >-inf</td>\n",
       "      <td id=\"T_93866_row16_col5\" class=\"data row16 col5\" >-inf</td>\n",
       "      <td id=\"T_93866_row16_col6\" class=\"data row16 col6\" >-inf</td>\n",
       "      <td id=\"T_93866_row16_col7\" class=\"data row16 col7\" >-inf</td>\n",
       "      <td id=\"T_93866_row16_col8\" class=\"data row16 col8\" >-inf</td>\n",
       "      <td id=\"T_93866_row16_col9\" class=\"data row16 col9\" >-inf</td>\n",
       "      <td id=\"T_93866_row16_col10\" class=\"data row16 col10\" >-inf</td>\n",
       "      <td id=\"T_93866_row16_col11\" class=\"data row16 col11\" >-inf</td>\n",
       "      <td id=\"T_93866_row16_col12\" class=\"data row16 col12\" >-inf</td>\n",
       "      <td id=\"T_93866_row16_col13\" class=\"data row16 col13\" >-inf</td>\n",
       "      <td id=\"T_93866_row16_col14\" class=\"data row16 col14\" >0.000000</td>\n",
       "      <td id=\"T_93866_row16_col15\" class=\"data row16 col15\" >0.000000</td>\n",
       "      <td id=\"T_93866_row16_col16\" class=\"data row16 col16\" >0.000000</td>\n",
       "      <td id=\"T_93866_row16_col17\" class=\"data row16 col17\" >-inf</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_93866_level0_row17\" class=\"row_heading level0 row17\" >17</th>\n",
       "      <td id=\"T_93866_row17_col0\" class=\"data row17 col0\" >-inf</td>\n",
       "      <td id=\"T_93866_row17_col1\" class=\"data row17 col1\" >-inf</td>\n",
       "      <td id=\"T_93866_row17_col2\" class=\"data row17 col2\" >-inf</td>\n",
       "      <td id=\"T_93866_row17_col3\" class=\"data row17 col3\" >-inf</td>\n",
       "      <td id=\"T_93866_row17_col4\" class=\"data row17 col4\" >-inf</td>\n",
       "      <td id=\"T_93866_row17_col5\" class=\"data row17 col5\" >-inf</td>\n",
       "      <td id=\"T_93866_row17_col6\" class=\"data row17 col6\" >-inf</td>\n",
       "      <td id=\"T_93866_row17_col7\" class=\"data row17 col7\" >-inf</td>\n",
       "      <td id=\"T_93866_row17_col8\" class=\"data row17 col8\" >-inf</td>\n",
       "      <td id=\"T_93866_row17_col9\" class=\"data row17 col9\" >-inf</td>\n",
       "      <td id=\"T_93866_row17_col10\" class=\"data row17 col10\" >-inf</td>\n",
       "      <td id=\"T_93866_row17_col11\" class=\"data row17 col11\" >-inf</td>\n",
       "      <td id=\"T_93866_row17_col12\" class=\"data row17 col12\" >-inf</td>\n",
       "      <td id=\"T_93866_row17_col13\" class=\"data row17 col13\" >-inf</td>\n",
       "      <td id=\"T_93866_row17_col14\" class=\"data row17 col14\" >-inf</td>\n",
       "      <td id=\"T_93866_row17_col15\" class=\"data row17 col15\" >0.000000</td>\n",
       "      <td id=\"T_93866_row17_col16\" class=\"data row17 col16\" >0.000000</td>\n",
       "      <td id=\"T_93866_row17_col17\" class=\"data row17 col17\" >0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x7fd7fb37bd90>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## BlockDiagonalCausalMask\n",
    "\n",
    "seqlens = [7, 5, 6]\n",
    "sliding_window_size = 3\n",
    "\n",
    "mask = BlockDiagonalCausalMask.from_seqlens(seqlens).make_local_attention(sliding_window_size)\n",
    "\n",
    "batch_size = 1\n",
    "total_seq_len = sum(seqlens)\n",
    "mask_tensor = mask.materialize((batch_size, total_seq_len, total_seq_len))\n",
    "\n",
    "df = pd.DataFrame(mask_tensor[0, :, :].numpy())\n",
    "df.style.applymap(colour_cell)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_17144/1506970582.py:15: FutureWarning: Styler.applymap has been deprecated. Use Styler.map instead.\n",
      "  df.style.applymap(colour_cell)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<style type=\"text/css\">\n",
       "#T_9171d_row0_col0, #T_9171d_row0_col1, #T_9171d_row0_col2, #T_9171d_row0_col3, #T_9171d_row0_col4, #T_9171d_row0_col8, #T_9171d_row0_col9, #T_9171d_row0_col10, #T_9171d_row0_col11, #T_9171d_row0_col12, #T_9171d_row0_col13, #T_9171d_row0_col14, #T_9171d_row0_col15, #T_9171d_row0_col16, #T_9171d_row0_col17, #T_9171d_row1_col0, #T_9171d_row1_col1, #T_9171d_row1_col2, #T_9171d_row1_col3, #T_9171d_row1_col4, #T_9171d_row1_col5, #T_9171d_row1_col9, #T_9171d_row1_col10, #T_9171d_row1_col11, #T_9171d_row1_col12, #T_9171d_row1_col13, #T_9171d_row1_col14, #T_9171d_row1_col15, #T_9171d_row1_col16, #T_9171d_row1_col17, #T_9171d_row2_col0, #T_9171d_row2_col1, #T_9171d_row2_col2, #T_9171d_row2_col3, #T_9171d_row2_col4, #T_9171d_row2_col5, #T_9171d_row2_col6, #T_9171d_row2_col10, #T_9171d_row2_col11, #T_9171d_row2_col12, #T_9171d_row2_col13, #T_9171d_row2_col14, #T_9171d_row2_col15, #T_9171d_row2_col16, #T_9171d_row2_col17, #T_9171d_row3_col0, #T_9171d_row3_col1, #T_9171d_row3_col2, #T_9171d_row3_col3, #T_9171d_row3_col4, #T_9171d_row3_col5, #T_9171d_row3_col6, #T_9171d_row3_col7, #T_9171d_row3_col8, #T_9171d_row3_col9, #T_9171d_row3_col10, #T_9171d_row3_col14, #T_9171d_row3_col15, #T_9171d_row3_col16, #T_9171d_row3_col17, #T_9171d_row4_col0, #T_9171d_row4_col1, #T_9171d_row4_col2, #T_9171d_row4_col3, #T_9171d_row4_col4, #T_9171d_row4_col5, #T_9171d_row4_col6, #T_9171d_row4_col7, #T_9171d_row4_col8, #T_9171d_row4_col9, #T_9171d_row4_col10, #T_9171d_row4_col11, #T_9171d_row4_col15, #T_9171d_row4_col16, #T_9171d_row4_col17, #T_9171d_row5_col0, #T_9171d_row5_col1, #T_9171d_row5_col2, #T_9171d_row5_col3, #T_9171d_row5_col4, #T_9171d_row5_col5, #T_9171d_row5_col6, #T_9171d_row5_col7, #T_9171d_row5_col8, #T_9171d_row5_col9, #T_9171d_row5_col10, #T_9171d_row5_col11, #T_9171d_row5_col12, #T_9171d_row5_col16, #T_9171d_row5_col17, #T_9171d_row6_col0, #T_9171d_row6_col1, #T_9171d_row6_col2, #T_9171d_row6_col3, #T_9171d_row6_col4, #T_9171d_row6_col5, #T_9171d_row6_col6, #T_9171d_row6_col7, #T_9171d_row6_col8, #T_9171d_row6_col9, #T_9171d_row6_col10, #T_9171d_row6_col11, #T_9171d_row6_col12, #T_9171d_row6_col13, #T_9171d_row6_col17, #T_9171d_row7_col0, #T_9171d_row7_col1, #T_9171d_row7_col2, #T_9171d_row7_col3, #T_9171d_row7_col4, #T_9171d_row7_col5, #T_9171d_row7_col6, #T_9171d_row7_col7, #T_9171d_row7_col8, #T_9171d_row7_col9, #T_9171d_row7_col10, #T_9171d_row7_col11, #T_9171d_row7_col12, #T_9171d_row7_col13, #T_9171d_row7_col14 {\n",
       "  Background-color: #C97064;\n",
       "}\n",
       "#T_9171d_row0_col5, #T_9171d_row0_col6, #T_9171d_row0_col7, #T_9171d_row1_col6, #T_9171d_row1_col7, #T_9171d_row1_col8, #T_9171d_row2_col7, #T_9171d_row2_col8, #T_9171d_row2_col9, #T_9171d_row3_col11, #T_9171d_row3_col12, #T_9171d_row3_col13, #T_9171d_row4_col12, #T_9171d_row4_col13, #T_9171d_row4_col14, #T_9171d_row5_col13, #T_9171d_row5_col14, #T_9171d_row5_col15, #T_9171d_row6_col14, #T_9171d_row6_col15, #T_9171d_row6_col16, #T_9171d_row7_col15, #T_9171d_row7_col16, #T_9171d_row7_col17 {\n",
       "  Background-color: #68A357;\n",
       "}\n",
       "</style>\n",
       "<table id=\"T_9171d\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th class=\"blank level0\" >&nbsp;</th>\n",
       "      <th id=\"T_9171d_level0_col0\" class=\"col_heading level0 col0\" >0</th>\n",
       "      <th id=\"T_9171d_level0_col1\" class=\"col_heading level0 col1\" >1</th>\n",
       "      <th id=\"T_9171d_level0_col2\" class=\"col_heading level0 col2\" >2</th>\n",
       "      <th id=\"T_9171d_level0_col3\" class=\"col_heading level0 col3\" >3</th>\n",
       "      <th id=\"T_9171d_level0_col4\" class=\"col_heading level0 col4\" >4</th>\n",
       "      <th id=\"T_9171d_level0_col5\" class=\"col_heading level0 col5\" >5</th>\n",
       "      <th id=\"T_9171d_level0_col6\" class=\"col_heading level0 col6\" >6</th>\n",
       "      <th id=\"T_9171d_level0_col7\" class=\"col_heading level0 col7\" >7</th>\n",
       "      <th id=\"T_9171d_level0_col8\" class=\"col_heading level0 col8\" >8</th>\n",
       "      <th id=\"T_9171d_level0_col9\" class=\"col_heading level0 col9\" >9</th>\n",
       "      <th id=\"T_9171d_level0_col10\" class=\"col_heading level0 col10\" >10</th>\n",
       "      <th id=\"T_9171d_level0_col11\" class=\"col_heading level0 col11\" >11</th>\n",
       "      <th id=\"T_9171d_level0_col12\" class=\"col_heading level0 col12\" >12</th>\n",
       "      <th id=\"T_9171d_level0_col13\" class=\"col_heading level0 col13\" >13</th>\n",
       "      <th id=\"T_9171d_level0_col14\" class=\"col_heading level0 col14\" >14</th>\n",
       "      <th id=\"T_9171d_level0_col15\" class=\"col_heading level0 col15\" >15</th>\n",
       "      <th id=\"T_9171d_level0_col16\" class=\"col_heading level0 col16\" >16</th>\n",
       "      <th id=\"T_9171d_level0_col17\" class=\"col_heading level0 col17\" >17</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th id=\"T_9171d_level0_row0\" class=\"row_heading level0 row0\" >0</th>\n",
       "      <td id=\"T_9171d_row0_col0\" class=\"data row0 col0\" >-inf</td>\n",
       "      <td id=\"T_9171d_row0_col1\" class=\"data row0 col1\" >-inf</td>\n",
       "      <td id=\"T_9171d_row0_col2\" class=\"data row0 col2\" >-inf</td>\n",
       "      <td id=\"T_9171d_row0_col3\" class=\"data row0 col3\" >-inf</td>\n",
       "      <td id=\"T_9171d_row0_col4\" class=\"data row0 col4\" >-inf</td>\n",
       "      <td id=\"T_9171d_row0_col5\" class=\"data row0 col5\" >0.000000</td>\n",
       "      <td id=\"T_9171d_row0_col6\" class=\"data row0 col6\" >0.000000</td>\n",
       "      <td id=\"T_9171d_row0_col7\" class=\"data row0 col7\" >0.000000</td>\n",
       "      <td id=\"T_9171d_row0_col8\" class=\"data row0 col8\" >-inf</td>\n",
       "      <td id=\"T_9171d_row0_col9\" class=\"data row0 col9\" >-inf</td>\n",
       "      <td id=\"T_9171d_row0_col10\" class=\"data row0 col10\" >-inf</td>\n",
       "      <td id=\"T_9171d_row0_col11\" class=\"data row0 col11\" >-inf</td>\n",
       "      <td id=\"T_9171d_row0_col12\" class=\"data row0 col12\" >-inf</td>\n",
       "      <td id=\"T_9171d_row0_col13\" class=\"data row0 col13\" >-inf</td>\n",
       "      <td id=\"T_9171d_row0_col14\" class=\"data row0 col14\" >-inf</td>\n",
       "      <td id=\"T_9171d_row0_col15\" class=\"data row0 col15\" >-inf</td>\n",
       "      <td id=\"T_9171d_row0_col16\" class=\"data row0 col16\" >-inf</td>\n",
       "      <td id=\"T_9171d_row0_col17\" class=\"data row0 col17\" >-inf</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_9171d_level0_row1\" class=\"row_heading level0 row1\" >1</th>\n",
       "      <td id=\"T_9171d_row1_col0\" class=\"data row1 col0\" >-inf</td>\n",
       "      <td id=\"T_9171d_row1_col1\" class=\"data row1 col1\" >-inf</td>\n",
       "      <td id=\"T_9171d_row1_col2\" class=\"data row1 col2\" >-inf</td>\n",
       "      <td id=\"T_9171d_row1_col3\" class=\"data row1 col3\" >-inf</td>\n",
       "      <td id=\"T_9171d_row1_col4\" class=\"data row1 col4\" >-inf</td>\n",
       "      <td id=\"T_9171d_row1_col5\" class=\"data row1 col5\" >-inf</td>\n",
       "      <td id=\"T_9171d_row1_col6\" class=\"data row1 col6\" >0.000000</td>\n",
       "      <td id=\"T_9171d_row1_col7\" class=\"data row1 col7\" >0.000000</td>\n",
       "      <td id=\"T_9171d_row1_col8\" class=\"data row1 col8\" >0.000000</td>\n",
       "      <td id=\"T_9171d_row1_col9\" class=\"data row1 col9\" >-inf</td>\n",
       "      <td id=\"T_9171d_row1_col10\" class=\"data row1 col10\" >-inf</td>\n",
       "      <td id=\"T_9171d_row1_col11\" class=\"data row1 col11\" >-inf</td>\n",
       "      <td id=\"T_9171d_row1_col12\" class=\"data row1 col12\" >-inf</td>\n",
       "      <td id=\"T_9171d_row1_col13\" class=\"data row1 col13\" >-inf</td>\n",
       "      <td id=\"T_9171d_row1_col14\" class=\"data row1 col14\" >-inf</td>\n",
       "      <td id=\"T_9171d_row1_col15\" class=\"data row1 col15\" >-inf</td>\n",
       "      <td id=\"T_9171d_row1_col16\" class=\"data row1 col16\" >-inf</td>\n",
       "      <td id=\"T_9171d_row1_col17\" class=\"data row1 col17\" >-inf</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_9171d_level0_row2\" class=\"row_heading level0 row2\" >2</th>\n",
       "      <td id=\"T_9171d_row2_col0\" class=\"data row2 col0\" >-inf</td>\n",
       "      <td id=\"T_9171d_row2_col1\" class=\"data row2 col1\" >-inf</td>\n",
       "      <td id=\"T_9171d_row2_col2\" class=\"data row2 col2\" >-inf</td>\n",
       "      <td id=\"T_9171d_row2_col3\" class=\"data row2 col3\" >-inf</td>\n",
       "      <td id=\"T_9171d_row2_col4\" class=\"data row2 col4\" >-inf</td>\n",
       "      <td id=\"T_9171d_row2_col5\" class=\"data row2 col5\" >-inf</td>\n",
       "      <td id=\"T_9171d_row2_col6\" class=\"data row2 col6\" >-inf</td>\n",
       "      <td id=\"T_9171d_row2_col7\" class=\"data row2 col7\" >0.000000</td>\n",
       "      <td id=\"T_9171d_row2_col8\" class=\"data row2 col8\" >0.000000</td>\n",
       "      <td id=\"T_9171d_row2_col9\" class=\"data row2 col9\" >0.000000</td>\n",
       "      <td id=\"T_9171d_row2_col10\" class=\"data row2 col10\" >-inf</td>\n",
       "      <td id=\"T_9171d_row2_col11\" class=\"data row2 col11\" >-inf</td>\n",
       "      <td id=\"T_9171d_row2_col12\" class=\"data row2 col12\" >-inf</td>\n",
       "      <td id=\"T_9171d_row2_col13\" class=\"data row2 col13\" >-inf</td>\n",
       "      <td id=\"T_9171d_row2_col14\" class=\"data row2 col14\" >-inf</td>\n",
       "      <td id=\"T_9171d_row2_col15\" class=\"data row2 col15\" >-inf</td>\n",
       "      <td id=\"T_9171d_row2_col16\" class=\"data row2 col16\" >-inf</td>\n",
       "      <td id=\"T_9171d_row2_col17\" class=\"data row2 col17\" >-inf</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_9171d_level0_row3\" class=\"row_heading level0 row3\" >3</th>\n",
       "      <td id=\"T_9171d_row3_col0\" class=\"data row3 col0\" >-inf</td>\n",
       "      <td id=\"T_9171d_row3_col1\" class=\"data row3 col1\" >-inf</td>\n",
       "      <td id=\"T_9171d_row3_col2\" class=\"data row3 col2\" >-inf</td>\n",
       "      <td id=\"T_9171d_row3_col3\" class=\"data row3 col3\" >-inf</td>\n",
       "      <td id=\"T_9171d_row3_col4\" class=\"data row3 col4\" >-inf</td>\n",
       "      <td id=\"T_9171d_row3_col5\" class=\"data row3 col5\" >-inf</td>\n",
       "      <td id=\"T_9171d_row3_col6\" class=\"data row3 col6\" >-inf</td>\n",
       "      <td id=\"T_9171d_row3_col7\" class=\"data row3 col7\" >-inf</td>\n",
       "      <td id=\"T_9171d_row3_col8\" class=\"data row3 col8\" >-inf</td>\n",
       "      <td id=\"T_9171d_row3_col9\" class=\"data row3 col9\" >-inf</td>\n",
       "      <td id=\"T_9171d_row3_col10\" class=\"data row3 col10\" >-inf</td>\n",
       "      <td id=\"T_9171d_row3_col11\" class=\"data row3 col11\" >0.000000</td>\n",
       "      <td id=\"T_9171d_row3_col12\" class=\"data row3 col12\" >0.000000</td>\n",
       "      <td id=\"T_9171d_row3_col13\" class=\"data row3 col13\" >0.000000</td>\n",
       "      <td id=\"T_9171d_row3_col14\" class=\"data row3 col14\" >-inf</td>\n",
       "      <td id=\"T_9171d_row3_col15\" class=\"data row3 col15\" >-inf</td>\n",
       "      <td id=\"T_9171d_row3_col16\" class=\"data row3 col16\" >-inf</td>\n",
       "      <td id=\"T_9171d_row3_col17\" class=\"data row3 col17\" >-inf</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_9171d_level0_row4\" class=\"row_heading level0 row4\" >4</th>\n",
       "      <td id=\"T_9171d_row4_col0\" class=\"data row4 col0\" >-inf</td>\n",
       "      <td id=\"T_9171d_row4_col1\" class=\"data row4 col1\" >-inf</td>\n",
       "      <td id=\"T_9171d_row4_col2\" class=\"data row4 col2\" >-inf</td>\n",
       "      <td id=\"T_9171d_row4_col3\" class=\"data row4 col3\" >-inf</td>\n",
       "      <td id=\"T_9171d_row4_col4\" class=\"data row4 col4\" >-inf</td>\n",
       "      <td id=\"T_9171d_row4_col5\" class=\"data row4 col5\" >-inf</td>\n",
       "      <td id=\"T_9171d_row4_col6\" class=\"data row4 col6\" >-inf</td>\n",
       "      <td id=\"T_9171d_row4_col7\" class=\"data row4 col7\" >-inf</td>\n",
       "      <td id=\"T_9171d_row4_col8\" class=\"data row4 col8\" >-inf</td>\n",
       "      <td id=\"T_9171d_row4_col9\" class=\"data row4 col9\" >-inf</td>\n",
       "      <td id=\"T_9171d_row4_col10\" class=\"data row4 col10\" >-inf</td>\n",
       "      <td id=\"T_9171d_row4_col11\" class=\"data row4 col11\" >-inf</td>\n",
       "      <td id=\"T_9171d_row4_col12\" class=\"data row4 col12\" >0.000000</td>\n",
       "      <td id=\"T_9171d_row4_col13\" class=\"data row4 col13\" >0.000000</td>\n",
       "      <td id=\"T_9171d_row4_col14\" class=\"data row4 col14\" >0.000000</td>\n",
       "      <td id=\"T_9171d_row4_col15\" class=\"data row4 col15\" >-inf</td>\n",
       "      <td id=\"T_9171d_row4_col16\" class=\"data row4 col16\" >-inf</td>\n",
       "      <td id=\"T_9171d_row4_col17\" class=\"data row4 col17\" >-inf</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_9171d_level0_row5\" class=\"row_heading level0 row5\" >5</th>\n",
       "      <td id=\"T_9171d_row5_col0\" class=\"data row5 col0\" >-inf</td>\n",
       "      <td id=\"T_9171d_row5_col1\" class=\"data row5 col1\" >-inf</td>\n",
       "      <td id=\"T_9171d_row5_col2\" class=\"data row5 col2\" >-inf</td>\n",
       "      <td id=\"T_9171d_row5_col3\" class=\"data row5 col3\" >-inf</td>\n",
       "      <td id=\"T_9171d_row5_col4\" class=\"data row5 col4\" >-inf</td>\n",
       "      <td id=\"T_9171d_row5_col5\" class=\"data row5 col5\" >-inf</td>\n",
       "      <td id=\"T_9171d_row5_col6\" class=\"data row5 col6\" >-inf</td>\n",
       "      <td id=\"T_9171d_row5_col7\" class=\"data row5 col7\" >-inf</td>\n",
       "      <td id=\"T_9171d_row5_col8\" class=\"data row5 col8\" >-inf</td>\n",
       "      <td id=\"T_9171d_row5_col9\" class=\"data row5 col9\" >-inf</td>\n",
       "      <td id=\"T_9171d_row5_col10\" class=\"data row5 col10\" >-inf</td>\n",
       "      <td id=\"T_9171d_row5_col11\" class=\"data row5 col11\" >-inf</td>\n",
       "      <td id=\"T_9171d_row5_col12\" class=\"data row5 col12\" >-inf</td>\n",
       "      <td id=\"T_9171d_row5_col13\" class=\"data row5 col13\" >0.000000</td>\n",
       "      <td id=\"T_9171d_row5_col14\" class=\"data row5 col14\" >0.000000</td>\n",
       "      <td id=\"T_9171d_row5_col15\" class=\"data row5 col15\" >0.000000</td>\n",
       "      <td id=\"T_9171d_row5_col16\" class=\"data row5 col16\" >-inf</td>\n",
       "      <td id=\"T_9171d_row5_col17\" class=\"data row5 col17\" >-inf</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_9171d_level0_row6\" class=\"row_heading level0 row6\" >6</th>\n",
       "      <td id=\"T_9171d_row6_col0\" class=\"data row6 col0\" >-inf</td>\n",
       "      <td id=\"T_9171d_row6_col1\" class=\"data row6 col1\" >-inf</td>\n",
       "      <td id=\"T_9171d_row6_col2\" class=\"data row6 col2\" >-inf</td>\n",
       "      <td id=\"T_9171d_row6_col3\" class=\"data row6 col3\" >-inf</td>\n",
       "      <td id=\"T_9171d_row6_col4\" class=\"data row6 col4\" >-inf</td>\n",
       "      <td id=\"T_9171d_row6_col5\" class=\"data row6 col5\" >-inf</td>\n",
       "      <td id=\"T_9171d_row6_col6\" class=\"data row6 col6\" >-inf</td>\n",
       "      <td id=\"T_9171d_row6_col7\" class=\"data row6 col7\" >-inf</td>\n",
       "      <td id=\"T_9171d_row6_col8\" class=\"data row6 col8\" >-inf</td>\n",
       "      <td id=\"T_9171d_row6_col9\" class=\"data row6 col9\" >-inf</td>\n",
       "      <td id=\"T_9171d_row6_col10\" class=\"data row6 col10\" >-inf</td>\n",
       "      <td id=\"T_9171d_row6_col11\" class=\"data row6 col11\" >-inf</td>\n",
       "      <td id=\"T_9171d_row6_col12\" class=\"data row6 col12\" >-inf</td>\n",
       "      <td id=\"T_9171d_row6_col13\" class=\"data row6 col13\" >-inf</td>\n",
       "      <td id=\"T_9171d_row6_col14\" class=\"data row6 col14\" >0.000000</td>\n",
       "      <td id=\"T_9171d_row6_col15\" class=\"data row6 col15\" >0.000000</td>\n",
       "      <td id=\"T_9171d_row6_col16\" class=\"data row6 col16\" >0.000000</td>\n",
       "      <td id=\"T_9171d_row6_col17\" class=\"data row6 col17\" >-inf</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_9171d_level0_row7\" class=\"row_heading level0 row7\" >7</th>\n",
       "      <td id=\"T_9171d_row7_col0\" class=\"data row7 col0\" >-inf</td>\n",
       "      <td id=\"T_9171d_row7_col1\" class=\"data row7 col1\" >-inf</td>\n",
       "      <td id=\"T_9171d_row7_col2\" class=\"data row7 col2\" >-inf</td>\n",
       "      <td id=\"T_9171d_row7_col3\" class=\"data row7 col3\" >-inf</td>\n",
       "      <td id=\"T_9171d_row7_col4\" class=\"data row7 col4\" >-inf</td>\n",
       "      <td id=\"T_9171d_row7_col5\" class=\"data row7 col5\" >-inf</td>\n",
       "      <td id=\"T_9171d_row7_col6\" class=\"data row7 col6\" >-inf</td>\n",
       "      <td id=\"T_9171d_row7_col7\" class=\"data row7 col7\" >-inf</td>\n",
       "      <td id=\"T_9171d_row7_col8\" class=\"data row7 col8\" >-inf</td>\n",
       "      <td id=\"T_9171d_row7_col9\" class=\"data row7 col9\" >-inf</td>\n",
       "      <td id=\"T_9171d_row7_col10\" class=\"data row7 col10\" >-inf</td>\n",
       "      <td id=\"T_9171d_row7_col11\" class=\"data row7 col11\" >-inf</td>\n",
       "      <td id=\"T_9171d_row7_col12\" class=\"data row7 col12\" >-inf</td>\n",
       "      <td id=\"T_9171d_row7_col13\" class=\"data row7 col13\" >-inf</td>\n",
       "      <td id=\"T_9171d_row7_col14\" class=\"data row7 col14\" >-inf</td>\n",
       "      <td id=\"T_9171d_row7_col15\" class=\"data row7 col15\" >0.000000</td>\n",
       "      <td id=\"T_9171d_row7_col16\" class=\"data row7 col16\" >0.000000</td>\n",
       "      <td id=\"T_9171d_row7_col17\" class=\"data row7 col17\" >0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x7fd7fb601c40>"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## BlockDiagonalMask\n",
    "\n",
    "q_seqlens = [3, 5]\n",
    "kv_seqlens = [10, 8] # (3 + 7, 5 + 3)\n",
    "sliding_window_size = 3\n",
    "\n",
    "mask = BlockDiagonalMask.from_seqlens(q_seqlens, kv_seqlens).make_local_attention_from_bottomright(sliding_window_size)\n",
    "\n",
    "batch_size = 1\n",
    "total_seq_len = sum(q_seqlens)\n",
    "total_kv_seq_len = sum(kv_seqlens)\n",
    "mask_tensor = mask.materialize((batch_size, total_seq_len, total_kv_seq_len))\n",
    "\n",
    "df = pd.DataFrame(mask_tensor[0, :, :].numpy())\n",
    "df.style.applymap(colour_cell)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_17144/2637469656.py:16: FutureWarning: Styler.applymap has been deprecated. Use Styler.map instead.\n",
      "  df.style.applymap(colour_cell)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<style type=\"text/css\">\n",
       "#T_b0453_row0_col0, #T_b0453_row0_col1, #T_b0453_row0_col2, #T_b0453_row1_col6, #T_b0453_row1_col7, #T_b0453_row1_col8, #T_b0453_row1_col9, #T_b0453_row1_col10 {\n",
       "  Background-color: #68A357;\n",
       "}\n",
       "#T_b0453_row0_col3, #T_b0453_row0_col4, #T_b0453_row0_col5, #T_b0453_row0_col6, #T_b0453_row0_col7, #T_b0453_row0_col8, #T_b0453_row0_col9, #T_b0453_row0_col10, #T_b0453_row0_col11, #T_b0453_row1_col0, #T_b0453_row1_col1, #T_b0453_row1_col2, #T_b0453_row1_col3, #T_b0453_row1_col4, #T_b0453_row1_col5, #T_b0453_row1_col11 {\n",
       "  Background-color: #C97064;\n",
       "}\n",
       "</style>\n",
       "<table id=\"T_b0453\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th class=\"blank level0\" >&nbsp;</th>\n",
       "      <th id=\"T_b0453_level0_col0\" class=\"col_heading level0 col0\" >0</th>\n",
       "      <th id=\"T_b0453_level0_col1\" class=\"col_heading level0 col1\" >1</th>\n",
       "      <th id=\"T_b0453_level0_col2\" class=\"col_heading level0 col2\" >2</th>\n",
       "      <th id=\"T_b0453_level0_col3\" class=\"col_heading level0 col3\" >3</th>\n",
       "      <th id=\"T_b0453_level0_col4\" class=\"col_heading level0 col4\" >4</th>\n",
       "      <th id=\"T_b0453_level0_col5\" class=\"col_heading level0 col5\" >5</th>\n",
       "      <th id=\"T_b0453_level0_col6\" class=\"col_heading level0 col6\" >6</th>\n",
       "      <th id=\"T_b0453_level0_col7\" class=\"col_heading level0 col7\" >7</th>\n",
       "      <th id=\"T_b0453_level0_col8\" class=\"col_heading level0 col8\" >8</th>\n",
       "      <th id=\"T_b0453_level0_col9\" class=\"col_heading level0 col9\" >9</th>\n",
       "      <th id=\"T_b0453_level0_col10\" class=\"col_heading level0 col10\" >10</th>\n",
       "      <th id=\"T_b0453_level0_col11\" class=\"col_heading level0 col11\" >11</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th id=\"T_b0453_level0_row0\" class=\"row_heading level0 row0\" >0</th>\n",
       "      <td id=\"T_b0453_row0_col0\" class=\"data row0 col0\" >0.000000</td>\n",
       "      <td id=\"T_b0453_row0_col1\" class=\"data row0 col1\" >0.000000</td>\n",
       "      <td id=\"T_b0453_row0_col2\" class=\"data row0 col2\" >0.000000</td>\n",
       "      <td id=\"T_b0453_row0_col3\" class=\"data row0 col3\" >-inf</td>\n",
       "      <td id=\"T_b0453_row0_col4\" class=\"data row0 col4\" >-inf</td>\n",
       "      <td id=\"T_b0453_row0_col5\" class=\"data row0 col5\" >-inf</td>\n",
       "      <td id=\"T_b0453_row0_col6\" class=\"data row0 col6\" >-inf</td>\n",
       "      <td id=\"T_b0453_row0_col7\" class=\"data row0 col7\" >-inf</td>\n",
       "      <td id=\"T_b0453_row0_col8\" class=\"data row0 col8\" >-inf</td>\n",
       "      <td id=\"T_b0453_row0_col9\" class=\"data row0 col9\" >-inf</td>\n",
       "      <td id=\"T_b0453_row0_col10\" class=\"data row0 col10\" >-inf</td>\n",
       "      <td id=\"T_b0453_row0_col11\" class=\"data row0 col11\" >-inf</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_b0453_level0_row1\" class=\"row_heading level0 row1\" >1</th>\n",
       "      <td id=\"T_b0453_row1_col0\" class=\"data row1 col0\" >-inf</td>\n",
       "      <td id=\"T_b0453_row1_col1\" class=\"data row1 col1\" >-inf</td>\n",
       "      <td id=\"T_b0453_row1_col2\" class=\"data row1 col2\" >-inf</td>\n",
       "      <td id=\"T_b0453_row1_col3\" class=\"data row1 col3\" >-inf</td>\n",
       "      <td id=\"T_b0453_row1_col4\" class=\"data row1 col4\" >-inf</td>\n",
       "      <td id=\"T_b0453_row1_col5\" class=\"data row1 col5\" >-inf</td>\n",
       "      <td id=\"T_b0453_row1_col6\" class=\"data row1 col6\" >0.000000</td>\n",
       "      <td id=\"T_b0453_row1_col7\" class=\"data row1 col7\" >0.000000</td>\n",
       "      <td id=\"T_b0453_row1_col8\" class=\"data row1 col8\" >0.000000</td>\n",
       "      <td id=\"T_b0453_row1_col9\" class=\"data row1 col9\" >0.000000</td>\n",
       "      <td id=\"T_b0453_row1_col10\" class=\"data row1 col10\" >0.000000</td>\n",
       "      <td id=\"T_b0453_row1_col11\" class=\"data row1 col11\" >-inf</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x7fd8cdb35f40>"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## BlockDiagonalCausalWithOffsetPaddedKeysMask\n",
    "\n",
    "# We use this mask with padding because the overall size of the KV-Cache is the same for all the prompts, but for each KV-Cache we may need to use only some of the items.\n",
    "\n",
    "q_seqlen = [1, 1]\n",
    "kv_seq_len = [3, 5]\n",
    "kv_padding = 6\n",
    "\n",
    "mask = BlockDiagonalCausalWithOffsetPaddedKeysMask.from_seqlens(q_seqlen=q_seqlen, kv_padding=kv_padding, kv_seqlen=kv_seq_len)\n",
    "\n",
    "batch_size = 1\n",
    "total_seq_len = sum(q_seqlen)\n",
    "total_kv_seq_len = kv_padding * len(kv_seq_len)\n",
    "\n",
    "mask_tensor = mask.materialize((batch_size, total_seq_len, total_kv_seq_len))\n",
    "\n",
    "df = pd.DataFrame(mask_tensor[0, :, :].numpy())\n",
    "df.style.applymap(colour_cell)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "pytorch-mistral",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.18"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}