cleanml-analysis-cleaning-type.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "73dff9bf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>repair</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>impute_mean_dummy</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>impute_mean_mode</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>impute_median_dummy</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>impute_median_mode</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>impute_mode_dummy</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>impute_mode_mode</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                repair  count\n",
       "0    impute_mean_dummy      9\n",
       "1     impute_mean_mode      5\n",
       "2  impute_median_dummy      7\n",
       "3   impute_median_mode      5\n",
       "4    impute_mode_dummy      6\n",
       "5     impute_mode_mode      5"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import duckdb\n",
    "\n",
    "con = duckdb.connect(database=':memory:')\n",
    "counts = con.execute(\"\"\"\n",
    "    SELECT repair, COUNT(*) as count\n",
    "    FROM 'cleanml.csv' \n",
    "    WHERE fairness_impact = 'positive' and error = 'missing_values'\n",
    "    GROUP BY repair\n",
    "    ORDER BY repair\n",
    "\"\"\").df()\n",
    "\n",
    "counts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "d31d4b8d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>dummy</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>False</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>True</td>\n",
       "      <td>22</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   dummy  count\n",
       "0  False     15\n",
       "1   True     22"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import duckdb\n",
    "\n",
    "con = duckdb.connect(database=':memory:')\n",
    "counts = con.execute(\"\"\"\n",
    "    SELECT repair LIKE '%dummy' as dummy, COUNT(*) as count\n",
    "    FROM 'cleanml.csv' \n",
    "    WHERE fairness_impact = 'positive' and error = 'missing_values'\n",
    "    GROUP BY dummy\n",
    "    ORDER BY dummy\n",
    "\"\"\").df()\n",
    "\n",
    "counts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "5cedd476",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>detection</th>\n",
       "      <th>repair</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>IF</td>\n",
       "      <td>impute__mean_dummy</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>IF</td>\n",
       "      <td>impute__median_dummy</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>IF</td>\n",
       "      <td>impute__mode_dummy</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>IQR</td>\n",
       "      <td>impute__mean_dummy</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>IQR</td>\n",
       "      <td>impute__median_dummy</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>IQR</td>\n",
       "      <td>impute__mode_dummy</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>SD</td>\n",
       "      <td>impute__mean_dummy</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>SD</td>\n",
       "      <td>impute__median_dummy</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>SD</td>\n",
       "      <td>impute__mode_dummy</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  detection                repair  count\n",
       "0        IF    impute__mean_dummy      4\n",
       "1        IF  impute__median_dummy      3\n",
       "2        IF    impute__mode_dummy      4\n",
       "3       IQR    impute__mean_dummy      1\n",
       "4       IQR  impute__median_dummy      2\n",
       "5       IQR    impute__mode_dummy      1\n",
       "6        SD    impute__mean_dummy      5\n",
       "7        SD  impute__median_dummy      4\n",
       "8        SD    impute__mode_dummy      7"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import duckdb\n",
    "\n",
    "con = duckdb.connect(database=':memory:')\n",
    "counts = con.execute(\"\"\"\n",
    "    SELECT detection, repair, COUNT(*) as count\n",
    "    FROM 'cleanml.csv' \n",
    "    WHERE fairness_impact = 'positive' and error = 'outliers'\n",
    "    GROUP BY detection, repair\n",
    "    ORDER BY detection, repair\n",
    "\"\"\").df()\n",
    "\n",
    "counts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "351b0858",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>detection</th>\n",
       "      <th>repair</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>IF</td>\n",
       "      <td>impute__mean_dummy</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>IF</td>\n",
       "      <td>impute__median_dummy</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>IF</td>\n",
       "      <td>impute__mode_dummy</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>IQR</td>\n",
       "      <td>impute__mean_dummy</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>IQR</td>\n",
       "      <td>impute__median_dummy</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>IQR</td>\n",
       "      <td>impute__mode_dummy</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>SD</td>\n",
       "      <td>impute__mean_dummy</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>SD</td>\n",
       "      <td>impute__median_dummy</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>SD</td>\n",
       "      <td>impute__mode_dummy</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  detection                repair  count\n",
       "0        IF    impute__mean_dummy      4\n",
       "1        IF  impute__median_dummy      3\n",
       "2        IF    impute__mode_dummy      4\n",
       "3       IQR    impute__mean_dummy      1\n",
       "4       IQR  impute__median_dummy      2\n",
       "5       IQR    impute__mode_dummy      1\n",
       "6        SD    impute__mean_dummy      5\n",
       "7        SD  impute__median_dummy      4\n",
       "8        SD    impute__mode_dummy      7"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import duckdb\n",
    "\n",
    "con = duckdb.connect(database=':memory:')\n",
    "counts = con.execute(\"\"\"\n",
    "    SELECT detection, repair, COUNT(*) as count\n",
    "    FROM 'cleanml.csv' \n",
    "    WHERE fairness_impact = 'positive' and error = 'outliers'\n",
    "    GROUP BY detection, repair\n",
    "    ORDER BY detection, repair\n",
    "\"\"\").df()\n",
    "\n",
    "counts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "e3e3077c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>detection</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>IF</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>IQR</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>SD</td>\n",
       "      <td>16</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  detection  count\n",
       "0        IF     11\n",
       "1       IQR      4\n",
       "2        SD     16"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import duckdb\n",
    "\n",
    "con = duckdb.connect(database=':memory:')\n",
    "counts = con.execute(\"\"\"\n",
    "    SELECT detection, COUNT(*) as count\n",
    "    FROM 'cleanml.csv' \n",
    "    WHERE fairness_impact = 'positive' and error = 'outliers'\n",
    "    GROUP BY detection\n",
    "    ORDER BY detection\n",
    "\"\"\").df()\n",
    "\n",
    "counts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fd4c390a",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a9d15d58",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}