{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "07463a5f",
   "metadata": {},
   "source": [
    "# ナイーブベイズ分類器\n",
    "\n",
    "ナイーブベイスについては以下を参照してください。\n",
    "[https://scikit-learn.org/stable/modules/naive_bayes.html](https://scikit-learn.org/stable/modules/naive_bayes.html)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "22ae72e6",
   "metadata": {},
   "source": [
    "**データとモジュールのロード**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "4f42c570",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from sklearn import model_selection\n",
    "\n",
    "data = pd.read_csv(\"input/pn_same_judge_preprocessed.csv\")\n",
    "train, test = model_selection.train_test_split(data, test_size=0.1, random_state=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "55fdf55a",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.metrics import ConfusionMatrixDisplay\n",
    "from sklearn.metrics import PrecisionRecallDisplay"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "310d666d",
   "metadata": {},
   "source": [
    "## MultinomialNB"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ea30df74",
   "metadata": {},
   "source": [
    "[sklearn.naive_bayes.MultinomialNB](https://scikit-learn.org/stable/modules/naive_bayes.html)\n",
    "を使います。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "79cc90a0",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.naive_bayes import MultinomialNB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "bb3dbb21",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>#sk-container-id-1 {color: black;background-color: white;}#sk-container-id-1 pre{padding: 0;}#sk-container-id-1 div.sk-toggleable {background-color: white;}#sk-container-id-1 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-1 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-1 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-1 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-1 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-1 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-1 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-1 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-1 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-1 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-1 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-1 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-1 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-1 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-1 div.sk-item {position: relative;z-index: 1;}#sk-container-id-1 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-1 div.sk-item::before, #sk-container-id-1 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-1 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-1 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-1 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-1 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-1 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-1 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-1 div.sk-label-container {text-align: center;}#sk-container-id-1 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-1 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-1\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>Pipeline(steps=[(&#x27;vect&#x27;,\n",
       "                 TfidfVectorizer(tokenizer=&lt;method &#x27;split&#x27; of &#x27;str&#x27; objects&gt;)),\n",
       "                (&#x27;clf&#x27;, MultinomialNB())])</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-1\" type=\"checkbox\" ><label for=\"sk-estimator-id-1\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">Pipeline</label><div class=\"sk-toggleable__content\"><pre>Pipeline(steps=[(&#x27;vect&#x27;,\n",
       "                 TfidfVectorizer(tokenizer=&lt;method &#x27;split&#x27; of &#x27;str&#x27; objects&gt;)),\n",
       "                (&#x27;clf&#x27;, MultinomialNB())])</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-2\" type=\"checkbox\" ><label for=\"sk-estimator-id-2\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">TfidfVectorizer</label><div class=\"sk-toggleable__content\"><pre>TfidfVectorizer(tokenizer=&lt;method &#x27;split&#x27; of &#x27;str&#x27; objects&gt;)</pre></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-3\" type=\"checkbox\" ><label for=\"sk-estimator-id-3\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">MultinomialNB</label><div class=\"sk-toggleable__content\"><pre>MultinomialNB()</pre></div></div></div></div></div></div></div>"
      ],
      "text/plain": [
       "Pipeline(steps=[('vect',\n",
       "                 TfidfVectorizer(tokenizer=<method 'split' of 'str' objects>)),\n",
       "                ('clf', MultinomialNB())])"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pipe_nb = Pipeline([\n",
    "    (\"vect\", TfidfVectorizer(tokenizer=str.split)),\n",
    "    (\"clf\", MultinomialNB())\n",
    "])\n",
    "\n",
    "pipe_nb.fit(train[\"tokens\"], train[\"label_num\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "441429d6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<sklearn.metrics._plot.confusion_matrix.ConfusionMatrixDisplay at 0x7fc05ee5d490>"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAATgAAAEICAYAAADLBejHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAa9klEQVR4nO3df7hVZZ338ffnHA4gqCCCBEihyVhkiV74e/QinRKd5sHmqtSacswZs3T6aT3a88z0w3GupjInJ7WH0hGb0rByoiJ/hHaJlQoakqAo+QsQRUBARX6cs7/PH/s+usFz9lnrsDd778Xn5bWus9e91l7re+Di632ve933rYjAzKyI2hodgJlZvTjBmVlhOcGZWWE5wZlZYTnBmVlhOcGZWWE5wZlZQ0gaLOk+SQ9KWizpK6n8OklPSFqYtsmpXJKukLRM0iJJh/d1jwF1/h1yGTmiPSaM72h0GJbDo4uGNDoEy2EzL7M1tmhnrnHyO4fG2nVdmc69f9GWWyNiWi+HtwAnRsRLkjqAuyX9Oh37fET8ZIfzTwEmpu0o4Or0s1dNleAmjO/gvlvHNzoMy+HksZMbHYLlcG/M3elrrFnXxb237p/p3I4xfx7Z27EojzJ4qfvUtFUbeTAduD597x5JwyWNiYhVvX3BTVQzyynoilKmDRgpaUHFdm7llSS1S1oIrAZuj4h706FLUzP0ckmDUtk4YHnF11eksl41VQ3OzJpfAKWqFa3trImIKb1eK6ILmCxpOHCzpEOAi4FngYHADOB/A1/tT6yuwZlZbqWM/2UVEeuBO4FpEbEqyrYA/wUcmU5bCVQ+w9o/lfXKCc7McgmCbVHKtFUjaVSquSFpD+BdwCOSxqQyAacBD6WvzAY+knpTjwY2VHv+Bm6imllOAXRlb6JWMwaYKamdcmVrVkT8UtIdkkYBAhYC56Xz5wCnAsuATcDZfd3ACc7McsvxDK5XEbEIOKyH8hN7OT+A8/PcwwnOzHIJoKtF5pF0gjOz3LJ3HzSWE5yZ5RJErZ7B1Z0TnJnlEgHbWiO/OcGZWV6ii50azrrLOMGZWS4BlFyDM7Oicg3OzAqp/KKvE5yZFVAA26I1Rnk6wZlZLoHoapFh7E5wZpZbKdxENbMC8jM4Mysw0eVncGZWROUZfZ3gzKyAIsTWaG90GJk4wZlZbiU/gzOzIip3MriJamaF5E4GMysodzKYWaF1+UVfMyuiQGyL1kgdrRGlmTWNVupkaI0ozaxpBKIrsm3VSBos6T5JD0paLOkrqfwASfdKWibpx5IGpvJBaX9ZOj6hr1id4MwstxJtmbY+bAFOjIhDgcnAtLRi/b8Dl0fEQcALwDnp/HOAF1L55em8qpzgzCyXCOiKtkxb9etERMRLabcjbQGcCPwklc8ETkufp6d90vGTJFWtJvoZnJnlUu5kyDxUa6SkBRX7MyJiRveOpHbgfuAg4Ergz8D6iOhMp6wAxqXP44DlABHRKWkDsC+wprebO8GZWW45OhnWRMSU3g5GRBcwWdJw4GbgLTsf3WvcRDWzXAJRimxb5mtGrAfuBI4BhkvqrnztD6xMn1cC4wHS8WHA2mrXdYIzs9y6aMu0VSNpVKq5IWkP4F3Aw5QT3fvSaWcBP0+fZ6d90vE7IqLqAoZuoppZLuV1UWtSNxoDzEzP4dqAWRHxS0lLgBsl/SvwR+CadP41wA8kLQPWAWf0dQMnODPLqTYr20fEIuCwHsofB47soXwz8P4893CCM7NcyssGesJLMyugCNWqiVp3TnBmlpvngzOzQirPB+fpksyskDyjr5kVVPk1EdfgzKyAco5FbSgnODPLzWsymFkhladLchPVzArKz+DMrJDKs4m4iWpmBVQequUEt1vYull87m8PYtvWNro64fi/3sBHPv8s3/z0G1n0h6EM3asEwIX/8TRvPuQVfn/L3lz/jTFI0D4gOO8rKznkqJcb/FtYtylTN3LeJc/Q3hb8+oYRzPrO6EaH1IRcgwNA0jTg20A78P2I+Fo979cIHYOCr9/0Z/YYWqJzG3z2tIkcceJGAP7xn5/h+Pds2O78w45/iWNOXooEjy8ZzKUfm8A18x5pROi2g7a24Px/W8nFZxzImlUd/Oecx7jn1mE8/djgRofWdFplJEPd0nCa4+lK4BRgEnCmpEn1ul+jSLDH0HItrXOb6Nomqi2DscfQ0qvHN29qq3qu7VoHH7aJZ54cyLNPD6JzWxu//flwjjl5Q99f3M1096Lu7LKBu0I965lHAssi4vGI2ArcSHlVnMLp6oKP/9XBnP6OQzjshBd5y+GbALjua2M476SD+e6XxrJ1y2t/2b/79TDOOf4t/PNHDuSz33q6UWHbDvZ9wzaef2bgq/trVnUwcsy2BkbUvErRlmlrtHpG8OoKOEnl6jivknSupAWSFjy/tquO4dRPeztc/Zul/PD+JSxdOIQnHxnM2Rc/w/fnPcIVcx7lxfUDmHXlfq+ef9wpG7hm3iN8+donmPn1MQ2M3Cy/eqzJUC8NT7ERMSMipkTElFH7tsbwj97sOayLQ499ifl37sW+ozuRYOCg4N2nr2PpwiGvO//tR7/Ms08PZMPa1v69i2Ltsx2MGrv11f2RY7axZlVHAyNqTgF0RlumrdHqGcGrK+AklavjFMb6te28tKGcoLa8Ih64ay/GH7SFtc+V+28i4Pe3DGPCwZsBWPnEQLqXyXhs0R5s2yr2HtGaNdeiWbpwCOMO2Mro8VsY0FFi6vT13HPbsEaH1ZRapYlaz17U+cBESQdQTmxnAB+s4/0aYt1zHXzzU2+kVBKlEpzwN+s5+l0b+cL738yGtQOIgDe/7RU++e+rALj7V8P5zU/2YcAAGLRHiS9e/ZQ7GppEqUtc+X/G8W8/epy2drjtxhE89ah7UF+nSZqfWdQtwaWVpy8AbqX8msi1EbG4XvdrlAMnbeaq2x99XfnXb/pzj+effsFqTr9gdb3Dsn6af8fezL9j70aH0dQ84WUSEXOAOfW8h5nteq1Sg2t8I9nMWkr3hJc724sqabykOyUtkbRY0qdS+ZclrZS0MG2nVnznYknLJC2VdHJfsXqolpnlEojOUk3qRp3A5yLiAUl7AfdLuj0duzwivll5choocAbwNmAs8BtJfxERvfbSuQZnZrmVUKatmohYFREPpM8vAg/Tw7uyFaYDN0bEloh4AlhGDwtEV3KCM7N8IlcTdWT3i/xpO7enS0qaQHmV+3tT0QWSFkm6VtI+qSzT4IFKbqKaWS45F51ZExFTqp0gaU/gp8CnI2KjpKuBS9KtLgEuAz7an1id4Mwst1r1okrqoJzcfhgRPwOIiOcqjn8P+GXazT14wE1UM8slEF2ltkxbNZIEXAM8HBHfqiivHKD9XuCh9Hk2cIakQWkAwUTgvmr3cA3OzHKr0Yu+xwEfBv4kaWEq+yLlqdUmU26iPgl8DCAiFkuaBSyh3AN7frUeVHCCM7OcImrTRI2Iu6HHTNnr4ICIuBS4NOs9nODMLLdokZEMTnBmlpMH25tZgbkGZ2aFFAFdJSc4MysoT5dkZoUUuIlqZoXlTgYzK7DudUWanROcmeXmJqqZFVK5F7U1hrE7wZlZbm6imllhuYlqZoUUyAnOzIqrRVqoTnBmllNAeKiWmRWVm6hmVlgt34sq6T+p0tSOiE/WJSIza2pFGYu6YJdFYWatI4BWT3ARMbNyX9KQiNhU/5DMrNm1ShO1z/EWko6RtAR4JO0fKumqukdmZk1KRCnb1mhZBpT9B3AysBYgIh4ETqhjTGbW7CLj1mCZRsxGxPIdiqquRWhmBRblToYsWzWSxku6U9ISSYslfSqVj5B0u6TH0s99UrkkXSFpmaRFkg7vK9QsCW65pGOBkNQh6ULg4QzfM7Oiqk0NrhP4XERMAo4Gzpc0CbgImBsRE4G5aR/gFMqr2U8EzgWu7usGWRLcecD5wDjgGWBy2jez3ZYybr2LiFUR8UD6/CLlitM4YDrQ3ck5EzgtfZ4OXB9l9wDDJY2pdo8+X/SNiDXAh/o6z8x2I6XMZ46UVPnK2YyImLHjSZImAIcB9wKjI2JVOvQsMDp9HgdUPi5bkcpW0Ys+E5ykA4FvU65CBvAH4DMR8Xhf3zWzAsr3HtyaiJhS7QRJewI/BT4dERul164dESGp390VWZqoPwJmAWOAscBNwA39vaGZtb6IbFtfJHVQTm4/jIifpeLnupue6efqVL4SGF/x9f1TWa+yJLghEfGDiOhM238DgzN8z8yKqgadDCpX1a4BHo6Ib1Ucmg2clT6fBfy8ovwjqTf1aGBDRVO2R9XGoo5IH38t6SLgxhTy6cCc6qGbWaHVZqjWccCHgT9JWpjKvgh8DZgl6RzgKeAD6dgc4FRgGbAJOLuvG1R7Bnc/5YTW/Zt8rOJYABdn+hXMrHD6/1TsNRFxN713tZ7Uw/lBzjc4qo1FPSDPhcxsNxGCJhiGlUWm+eAkHQJMouLZW0RcX6+gzKzJNcEwrCyyvCbyJWAq5QQ3h/LbxHcDTnBmu6sWSXBZelHfR7k9/GxEnA0cCgyra1Rm1txaZLB9libqKxFRktQpaW/K76SM7+tLZlZQRZjwssICScOB71HuWX2J8mgGM9tN1aIXdVfIMhb1E+njdyXdAuwdEYvqG5aZNbVWT3DV5lqSdHj3LABmtvspQg3usirHAjixxrHw6OP78u73/32tL2t1pLY/NToEy6NWU9W2+jO4iHjnrgzEzFpEk/SQZuGFn80sPyc4MysqZZ/wsqGc4MwsvxapwWVZF1WS/k7Sv6T9N0o6sv6hmVkzUmTfGi3LUK2rgGOAM9P+i8CVdYvIzJpfKNvWYFmaqEdFxOGS/ggQES9IGljnuMysmTVB7SyLLAlum6R20q8kaRR51tQxs8JphuZnFlkS3BXAzcB+ki6lPLvI/61rVGbWvKJAvagR8UNJ91OeMknAaRHhle3NdmdFqcFJeiPlBR5+UVkWEU/XMzAza2JFSXDAr3ht8ZnBwAHAUuBtdYzLzJpYqzyD6/M1kYh4e0S8I/2cCByJ54MzsxqQdK2k1ZIeqij7sqSVkham7dSKYxdLWiZpqaST+7p+lvfgtpOmSToq7/fMrEBqN2X5dcC0Hsovj4jJaZsDIGkScAbl1uM04Kr0hkevsjyD+2zFbhtwOPBMptDNrHhq2IsaEXdJmpDx9OnAjRGxBXhC0jL6aFFmqcHtVbENovxMbnrGgMysiLLX4EZKWlCxnZvxDhdIWpSasPuksnHA8opzVqSyXlWtwaXq314RcWHGoMys4ESuToY1ETEl5y2uBi6hnCIvoTz57kdzXgOoUoOTNCAiuoDj+nNhMyuwOi4bGBHPRURXRJQoL3bVPbnHSrZf0W//VNarajW4+yg/b1soaTZwE/ByRRA/60fsZtbq6jxTiKQxEbEq7b4X6O5hnQ38SNK3gLHARMp5qldZ3oMbDKylvAZD9/twATjBme2uatTJIOkGYCrlZ3UrgC8BUyVNppxnngQ+BhARiyXNApYAncD5qZXZq2oJbr/Ug/oQryW2bi3ymp+Z1UOtanARcWYPxddUOf9S4NKs16+W4NqBPdk+sb16n6w3MLMCapEMUC3BrYqIr+6ySMysNRRkVa3GT8dpZk2pVcaiVktwJ+2yKMystbR6gouIdbsyEDNrHYWZ8NLMbDsFeQZnZvY6onUe0DvBmVl+rsGZWVEVoRfVzKxnTnBmVkhFWjbQzOx1XIMzs6LyMzgzKy4nODMrKtfgzKyYgppNeFlvTnBmlkvORWcaygnOzPJzgjOzolK0RoZzgjOzfDybiJkVmZ/BmVlhtcpQrV5Xtjcz61WNVraXdK2k1ZIeqigbIel2SY+ln/ukckm6QtIySYskHd7X9Z3gzCyftLJ9li2D64BpO5RdBMyNiInA3LQPcArl1ewnAucCV/d1cSc4M8uvRjW4iLgL2HH9l+nAzPR5JnBaRfn1UXYPMFzSmGrX9zM4M8sl54u+IyUtqNifEREz+vjO6IhYlT4/C4xOn8cByyvOW5HKVtELJzgzy02lzBluTURM6e99IiKk/vfZuolqZvlkbZ72/1WS57qbnunn6lS+Ehhfcd7+qaxXrsHV2NAhW/nsx3/PhPEvECEuu/pYjjxsJcccsZwIWL9hMN+48i9Z98KQRodqO+gYVOKynz5Kx8CgvT2YN2c4P7hsbKPDakp1fk1kNnAW8LX08+cV5RdIuhE4CthQ0ZTtUd0SnKRrgfcAqyPikHrdp9l84uz7mP/HsVxy2VQGDOhi0MAunlo+nJk/PgyA0055mL9734Nc8b1jGhyp7WjbFvGFD0xk86Z22gcE37p5KfPvHMYjDwxtdGjNp0Yv+kq6AZhK+VndCuBLlBPbLEnnAE8BH0inzwFOBZYBm4Cz+7p+PWtw1wHfAa6v4z2aypAhW3n7pOf4xpXHAdDZ2U5nZ/t25wwe1Nkqo1x2Q2LzpvLf14ABQfuAoEWGXO5ytRrJEBFn9nLopB7ODeD8PNevW4KLiLskTajX9ZvRG/Z7ifUbB3Hh+b/jwDe9wGOP78vV/3UEm7d08PdnPsC7TvgzL28ayOe/cnKjQ7VetLUF3/n1I4ydsIVfzBzF0j+69vY6Aa2S+RveySDpXEkLJC3Ytu3lRoezU9rbSkw8YB2/vPVgPvGFv2HzlgGcflr5Be3rbjicD338/dwx70D+17RHGhyp9aZUEp84+a186IhDOHjyy7zp4FcaHVJTUinb1mgNT3ARMSMipkTElI6O1v6/5Zp1Q3l+7RAeWTYKgHl/eBMHHbh2u3Pm3n0Axx/1VCPCsxxe3jiAB3+/F0dM3djoUJpO93twNRrJUFcNT3BF8sL6PXh+7VD2H7sBgMPevoqnVwxn7Bte+0dy7JTlLH9mWKNCtCqGjdjG0L07ARg4uMThx29k+bLBDY6qCUVk3xrMr4nU2JXXHsVFn5zHgAElnn1uT7551XF85rzfM37sRkohVj8/lG9/7+hGh2k9GDF6Gxde/hRt7UGb4K5f7sO9c/0/o540Q+0si3q+JvK67t+IuKZe92sWjz85ggsues92ZZdc9s4GRWN5PPHwEM6f9tZGh9EadvcEV6X718xa3G5fgzOzggqgqzUynBOcmeXmGpyZFVcT9JBm4QRnZrm5BmdmxeRlA82sqATInQxmVlRe2d7MislNVDMrruYYZ5qFE5yZ5eZeVDMrLtfgzKyQwr2oZlZkrZHfnODMLD+/JmJmxeUEZ2aFFECNFpSR9CTwItAFdEbEFEkjgB8DE4AngQ9ExAv9ub7XZDCzXESgyLZl9M6ImBwRU9L+RcDciJgIzE37/eIEZ2b5lUrZtv6ZDsxMn2cCp/X3Qk5wZpZPdxM1y1Zek2VBxXZuD1e7TdL9FcdGR8Sq9PlZYHR/Q/UzODPLLUfzc01F07MnfxkRKyXtB9wuabtV0SMipP6Pm3ANzszyq9G6qBGxMv1cDdwMHAk8J2kMQPq5ur9hOsGZWU61WfhZ0lBJe3V/Bt4NPATMBs5Kp50F/Ly/kbqJamb51G5VrdHAzZKgnIt+FBG3SJoPzJJ0DvAU8IH+3sAJzsxyq8VIhoh4HDi0h/K1wEk7fQOc4MysPzySwcwKKYCSE5yZFZJn9DWzInOCM7NCCqCrRqPt68wJzsxyCggnODMrKjdRzayQ3ItqZoXmGpyZFZYTnJkVUgR0dTU6ikyc4MwsP9fgzKywnODMrJjCvahmVlAB4Rd9zaywPFTLzAopYmeWBNylnODMLD93MphZUYVrcGZWTJ7w0syKyoPtzayoAogWGarlhZ/NLJ9IE15m2fogaZqkpZKWSbqo1qG6BmdmuUUNmqiS2oErgXcBK4D5kmZHxJKdvnjiGpyZ5VebGtyRwLKIeDwitgI3AtNrGaaiiXpDJD0PPNXoOOpgJLCm0UFYLkX9O3tTRIzamQtIuoXyn08Wg4HNFfszImJGus77gGkR8Q9p/8PAURFxwc7EV6mpmqg7+wffrCQtiIgpjY7DsvPfWe8iYlqjY8jKTVQza5SVwPiK/f1TWc04wZlZo8wHJko6QNJA4Axgdi1v0FRN1AKb0egALDf/ndVZRHRKugC4FWgHro2IxbW8R1N1MpiZ1ZKbqGZWWE5wZlZYTnB1VO9hKFZ7kq6VtFrSQ42OxXaeE1ydVAxDOQWYBJwpaVJjo7IMrgNa5j0vq84Jrn7qPgzFai8i7gLWNToOqw0nuPoZByyv2F+RysxsF3GCM7PCcoKrn7oPQzGz6pzg6qfuw1DMrDonuDqJiE6gexjKw8CsWg9DsdqTdAPwB+BgSSskndPomKz/PFTLzArLNTgzKywnODMrLCc4MyssJzgzKywnODMrLCe4FiKpS9JCSQ9JuknSkJ241nVpVSMkfb/aRACSpko6th/3eFLS61Zf6q18h3NeynmvL0u6MG+MVmxOcK3llYiYHBGHAFuB8yoPSurXFPQR8Q99LLY7Fcid4MwazQmudc0DDkq1q3mSZgNLJLVL+oak+ZIWSfoYgMq+k+an+w2wX/eFJP1W0pT0eZqkByQ9KGmupAmUE+lnUu3xeEmjJP003WO+pOPSd/eVdJukxZK+D6ivX0LS/0i6P33n3B2OXZ7K50oalcreLOmW9J15kt5Skz9NKyQvOtOCUk3tFOCWVHQ4cEhEPJGSxIaIOELSIOB3km4DDgMOpjw33WhgCXDtDtcdBXwPOCFda0RErJP0XeCliPhmOu9HwOURcbekN1IerfFW4EvA3RHxVUl/DWQZBfDRdI89gPmSfhoRa4GhwIKI+Iykf0nXvoDyYjDnRcRjko4CrgJO7Mcfo+0GnOBayx6SFqbP84BrKDcd74uIJ1L5u4F3dD9fA4YBE4ETgBsiogt4RtIdPVz/aOCu7mtFRG/zov0VMEl6tYK2t6Q90z3+Nn33V5JeyPA7fVLSe9Pn8SnWtUAJ+HEq/2/gZ+kexwI3Vdx7UIZ72G7KCa61vBIRkysL0j/0lyuLgH+KiFt3OO/UGsbRBhwdEZt7iCUzSVMpJ8tjImKTpN8Cg3s5PdJ91+/4Z2DWGz+DK55bgY9L6gCQ9BeShgJ3AaenZ3RjgHf28N17gBMkHZC+OyKVvwjsVXHebcA/de9Impw+3gV8MJWdAuzTR6zDgBdScnsL5Rpktzaguxb6QcpN343AE5Len+4hSYf2cQ/bjTnBFc/3KT9feyAtnPL/KNfUbwYeS8eupzxjxnYi4nngXMrNwQd5rYn4C+C93Z0MwCeBKakTYwmv9eZ+hXKCXEy5qfp0H7HeAgyQ9DDwNcoJttvLwJHpdzgR+Goq/xBwTopvMZ4G3qrwbCJmVliuwZlZYTnBmVlhOcGZWWE5wZlZYTnBmVlhOcGZWWE5wZlZYf1/s7gTDZ5XIBwAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "pred_nb = pipe_nb.predict(test[\"tokens\"])\n",
    "ConfusionMatrixDisplay.from_predictions(y_true=test[\"label_num\"], y_pred=pred_nb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "dd63ab0c",
   "metadata": {},
   "outputs": [],
   "source": [
    "score_nb = pipe_nb.predict_proba(test[\"tokens\"])[:,1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "2b58a9c2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<sklearn.metrics._plot.precision_recall_curve.PrecisionRecallDisplay at 0x7fc05af28f10>"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAn9ElEQVR4nO3de5xVVf3/8dcbxBAFb6BfAwwySFGRdNRM/XkLL6mQmoLX1JTMS6bW94u/vpla/vLWzTIVlaREwAsFmOlXU1G/ijHkqDHeEA0GKVEM8i74+f2x90yHYWbOHmbOOXPOeT8fj3lw9t7r7P3ZA5zPWWvttZYiAjMzq17dSh2AmZmVlhOBmVmVcyIwM6tyTgRmZlXOicDMrMqtV+oA2qtv374xaNCgUodhZlZW5s2b90ZE9GvpWNklgkGDBlFbW1vqMMzMyoqkv7V2zE1DZmZVzonAzKzKORGYmVU5JwIzsyrnRGBmVuUKlggkTZT0uqS/tnJckq6RtEDSM5J2LlQsZmbWukLWCG4BDm7j+CHAkPRnHHBdAWMxM7NWFGwcQUQ8ImlQG0VGA7+JZB7sOZI2kbRVRCwtRDyXzJpP/Wsr1+m9o0f057jdt+7kiMzMuoZS9hH0BxbnbDek+9YiaZykWkm1y5YtK0pwjeqXrmRG3ZKiXtPMrJjKYmRxREwAJgDU1NSs00o63z98+3W69pgbnlin95mZlYtS1giWAANztgek+8zMrIhKmQhmAielTw99HlhRqP4BMzNrXcGahiRNAfYF+kpqAL4P9ACIiOuBe4AvAQuAd4FTChWLmZm1rpBPDR2b53gAZxXq+mZmlo1HFpuZVTknAjOzKudEYGZW5ZwIzMyqnBOBmVmVcyIwM6tyTgRmZlXOicDMrMo5EZiZVTknAjOzKudEYGZW5ZwIzMyqnBOBmVmVcyIwM6tyTgRmZlXOicDMrMo5EZiZVTknAjOzKlewpSpt3dz25CJm1C1ZY9/oEf05bvetSxSRmVW6gtYIJB0s6QVJCySNb+H4pyT9SdIzkh6WNKCQ8ZSDGXVLqF+6smm7funKtRKDmVlnKliNQFJ34FpgJNAAzJU0MyLqc4pdDfwmIiZJ2h/4EXBioWIqF8O26sO0r+8BwJgbnihxNOXLtSuzbArZNLQbsCAiFgJImgqMBnITwTDg/PT1Q8DvCxhPl9PSB1X90pUM26pPiSIqXy39Lp98ZTkAuw/eDKCpplXKRODkZF1RIRNBf2BxznYDsHuzMk8DRwI/B44AekvaPCLezC0kaRwwDmDrrSvnP0xjM1DuB/+wrfowekT/EkZVHpp/oDb/0G98nfshW8jaVUsf8C3pisnJrNSdxd8GfinpZOARYAmwunmhiJgATACoqamJYgZYaLnNQNayLN/2m3/oFzumlhJRS4qZnMyyKmQiWAIMzNkekO5rEhGvkdQIkLQRcFRE/LOAMZVU8w8PNwOtLcuHfuPrYn3wd8VEZNaZCpkI5gJDJA0mSQBjgeNyC0jqCyyPiI+BC4GJBYyn5Jo3BbkZaN2aeCo9pqzNTE481lkKlggiYpWks4H7gO7AxIiYL+lSoDYiZgL7Aj+SFCRNQ2cVKp6uopRNQaXuqOyK36xLHVPjtXKbiLI0M7lvwTpTQfsIIuIe4J5m+y7KeX0ncGchY7B/a14jKfaHSUud48X84M/6oVvqZp4s13ffgnWmTIlA0hbAnsAngfeAv5J8q/+4gLGVtVI/GtrW9Ys1RqG1PpGu1Dle6g/9vT7TF4BbT2v+QJ1Z8bSZCCTtB4wHNgOeAl4HegJfBraRdCfw44hY2epJqlSpHw0t9vWzNLGUuk+kK37odqVYrHrlqxF8CTg9IhY1PyBpPeAwkpHDdxUgtrJXzG+/pf72Xepmnyz8oWvWsjYTQUR8p41jq6iykcBdWTGfSMrS7GTlo9QPEVjprXNnsaRTIuLXnRmMdUyxPohL3exl665cpuKw4urIU0OXAE4EFL9juNTX87f/8rCug/P8RFL1yddZ/Exrh4AtOz+c8lTsb8ilvp6//ZdeS4/CtlamKz0aa11TvhrBlsBBwFvN9gt4vCARlalif0Mu1PXc/l851vVDv7Uk4wRSufIlgruBjSKirvkBSQ8XIiBbU0v/KQvZDOT2//JQ7Edh3W9Q2fI9NfS1No4d19oxK6zO+mBuK8n423/XVsgE0FKScb9BZSv1NNSWUbE+mP3t3zzeovo4EayDSptO2t/+zaqbE8E6KOZTNI3V9HI7t5mVDyeCdVSsdvRCVtPdBGBmAN2yFpQ0oa1tMzMrT5kTAXBDnm0zMytDmRNBRMxra9vMzMpTvikmZgHR2vGIGNXpEZmZWVHl6yy+uiMnl3Qw8HOSNYtviojLmx3fGpgEbJKWGZ8ub2lmZkWSb2Tx7MbXkjYAto6IF7KcWFJ34FqShWsagLmSZkZEfU6x/wZuj4jrJA0jWd94UPtuwczMOiJTH4Gkw4E64N50e4SkmXnethuwICIWRsSHwFRgdLMyATSOxNoYeC1j3GZm1kmyjiO4mOSD/WGAiKiTNDjPe/oDi3O2G4DmD65fDPyPpHOADYEvZoynaIo96ZuZWbFlfWroo4hY0Wxfq53I7XAscEtEDCBZH/m3ktaKSdI4SbWSapctW9YJl+0Yz8djZpUka41gvqTjgO6ShgDfJP96BEuAgTnbA9J9ub4GHAwQEU9I6gn0BV7PLRQRE4AJADU1NZ2RgNrN8/GYWaXKWiM4B9ge+ACYAqwEvpXnPXOBIZIGS1ofGAs071dYBBwAIGk7oCdQ+q/8ZmZVJFONICLeBb4r6YpkM/6V4T2rJJ0N3EfyaOjEiJgv6VKgNiJmAhcAN0o6j6Sp6eSIKMk3fjOzapUpEUjaFZgI9E63VwCn5htdnI4JuKfZvotyXtcDe7YzZjMz60RZ+whuBs6MiEcBJO0F/BoYXqjAzMysOLL2EaxuTAIAEfEYsKowIZmZWTHlm2to5/TlbEk3kHQUBzCGdEyBmZmVt3xNQz9utv39nNfu1DUzqwD55hrar1iBmJlZaWReqlLSoSRjCXo27ouISwsRlJmZFU/WSeeuJ+kXOAcQcDTwqQLGZWZmRZL1qaEvRMRJwFsRcQmwBzC0cGGZmVmxZG0aei/9811JnwTeBLYqTEhdy16f6VvqEMzMCiprIrhb0ibAVcBfSJ4YuqlQQXUlt57WfOZsM7PKknWuoR+kL++SdDfQs4Vpqc3MrAzlG1B2ZBvHiIjpnR+SmZkVU74aweFtHAvAicDMrMzlG1B2SrECMTOz0sj6+KiZmVUoJwIzsyrnRGBmVuWyTjHRS9L3JN2Ybg+RdFhhQzMzs2LIWiP4NcnC9Xuk20uAHxYkIjMzK6qsI4u3iYgxko6FZDF7Scr3JkkHAz8nWbz+poi4vNnxnwKNU133AraIiE2yBm9mxfHkK8sBGHPDE037Ro/oz3G7b12qkKwTZU0EH0ragHQxGknbkNQQWiWpO3AtMBJoAOZKmpkuWA9ARJyXU/4c4HPtC9/MSqF+6UoAJ4IKkTURXAzcCwyUNBnYEzg5z3t2AxZExEIASVOB0UB9K+WPZc0V0Mysi2icfLFx7q3cmoGVv6xzDf2PpHnA50nWIzg3It7I87b+wOKc7QagxRncJH0KGAw82MrxccA4gK239jcQs2Lz5IuVLetTQ7OAA4GHI+LuDEmgvcYCd0bE6pYORsSEiKiJiJp+/fp18qXNzKpb1qeGrgb2Buol3SnpK5J65nnPEmBgzvaAdF9LxgJTMsZiZmadKFMiiIjZEXEm8GngBuAY4PU8b5sLDJE0WNL6JB/2M5sXkrQtsCngRkczsxLIPLI4fWroKOAMYFdgUlvlI2IVcDZwH/AccHtEzJd0qaRROUXHAlMjItobvJmZdZyyfP5Kup3kKaB7gWnA7Ij4uMCxtaimpiZqa2tLcWkzSw0a/wcAdh+82Rr7Pbag65I0LyJqWjqW9fHRm4FjW+vMNTPz2ILylW+Fsv0j4kFgQ2B088HEXqHMrDo1H1cAHltQzvLVCPYheba/pZXKvEKZWZXyuILKkm+FssaRvpdGxCu5xyQNLlhUZlZ2WpqPCNxvUA6yPjV0Vwv77uzMQMys8tQvXcmMutaGD1lXka+PYFtge2BjSUfmHOoD5BtQZmZVxP0G5StfH8FngcOATVizn+BfwOkFisnMypD7DcpXvj6CGcAMSXtEhFO7mVkFytc09J8RcSVwXOOiNLki4psFi8zMzIoiX9PQc+mfHsprZlah8jUNzUr/bJpXSFI3YKOIWFng2MzMrAgyTTEh6TaSyeZWk8wq2kfSzyPiqkIGZ2blzWsdl4es4wiGpTWALwN/JFlN7MRCBWVmlcnjCrqmrJPO9ZDUgyQR/DIiPpLkaaPNrE1e67g8ZE0ENwCvAk8Dj6RrDLuPwMza5LEF5SHr4vXXANfk7PqbpP0KE5KZmRVT1sXrN5b0E0m16c+PSaamNjOzMpe1s3giybQSx6Q/K4FfFyooMzMrnqx9BNtExFE525dIqitAPGZmVmRZawTvSdqrcUPSnsB7+d4k6WBJL0haIGl8K2WOkVQvaX46XsHMzIooa43gDOA3kjZOt98CvtrWGyR1B64FRgINwFxJMyOiPqfMEOBCYM+IeEvSFu29ATMz65i8iUDSCOAzwFhgCUDG6SV2AxZExML0PFOB0UB9TpnTgWsj4q30vK+3J3gzM+u4NpuGJF0E3A4cBfwBGNOOOYb6A4tzthvSfbmGAkMl/a+kOZIObiWOcY1PLC1btizj5c3MLIt8NYIxwIiIeFfS5sC9wI2dfP0hwL7AAJLBajtGxD9zC0XEBGACQE1NjUc0m5l1onydxR9ExLsAEfFmhvK5lgADc7YHpPtyNQAzI+KjiHgFeJEkMZiZWZHkqxF8WtLM9LWAbXK2iYhRbbx3LjBE0mCSBDAWOK5Zmd8DxwK/ltSXpKloYfbwzcyso/IlgtHNtq/OeuKIWCXpbOA+oDswMSLmS7oUqI2ImemxAyXVk0xx/Z205mFmZkWSb2Ga2R05eUTcA9zTbN9FOa8DOD/9MTOzEsj31NAsSYenU1A3P/ZpSZdKOrVw4ZmZWaHlaxo6neTb+s8kLQeWAT2BQcDLJGsTzChohGZmVlD5mob+Dvwn8J+SBgFbkUwt8WLj00RmZlm1tHQlePnKUss6xQQR8SrJ4jRmZp2mfmkyRtWJoHQyJwIzs45qvnQlePnKrsCJwMyKxktXdk1OBGZWUi31G7jPoLiyLlW5p6T7Jb0oaaGkVyR5BLCZdbr6pSuZUdd8NhorpKw1gpuB84B5JCOAzcw6RfN+A/cZFF/WRLAiIv5Y0EjMrCq536D0siaChyRdBUwHPmjcGRF/KUhUZmZWNFkTQWPKrsnZF8D+nRuOmVk2tz25aK2+BHcyr5tMiSAi9it0IGZmrWnpQ7/xaaPdB28GeGBaR2RKBOmi9d8H/k+6azZwaUSsKFRgZmaNZtQtoX7pSoZt1adp3+6DN1ujBuBO5nWXtWloIvBX4Jh0+0Tg18CRhQjKzKpXS+MKGpPAtK/vUaqwKlrWRLBNRByVs32JpLoCxGNmtpZhW/Vh9Ij+pQ6jYmVNBO9J2isiHoNkgBnJLKRmZp2qpfmIrLCyJoJvAJPSvgIBy4GTCxWUmVWvdU0AnuJ63WV9aqgO2ElSn3R7ZSGDMjPrDE++spwnX1nux0zzaDMRSDohIm6VdH6z/QBExE/yvP9g4Ocki9ffFBGXNzt+MnAV0Pi39MuIuKk9N2BmBi03KQ0a/4e1yvkx07XlqxFsmP7Zu70nltQduBYYCTQAcyXNjIj6ZkWnRcTZ7T2/mVmulpqUvP5BNvmWqrwh/fOSdTj3bsCCiFgIIGkqMBpongjMzArCHc7ZZJ2G+kpJfST1kPQnScsknZDnbf2BxTnbDem+5o6S9IykOyUNbOX64yTVSqpdtmxZlpDNzCyjTIkAODDtID6MZN3izwDf6YTrzwIGRcRw4H5gUkuFImJCRNRERE2/fv064bJmZtYoayJobEI6FLgj49QSS4Dcb/gD+HenMAAR8WZENM5mehOwS8Z4zMysk2RNBHdLep7kg/pPkvoB7+d5z1xgiKTBktYHxgIzcwtI2ipncxTwXMZ4zMysk2QdRzBe0pUkC9SslvQOScdvW+9ZJels4D6Sx0cnRsR8SZcCtRExE/impFHAKjxIzcysJPKNI9g/Ih6UdGTOvtwi09t6f0TcA9zTbN9FOa8vBC5sT8BmZta58tUI9gEeBA5v4ViQJxGYmVnXl28cwffTP08pTjhmZlZsWccR/D9Jm+RsbyrphwWLyszMiibrU0OHRMQ/Gzci4i3gSwWJyMzMiiprIugu6RONG5I2AD7RRnkzMysTWdcjmEwyfuDX6fYptDIK2MzMykvWcQRXSHoa+GK66wcRcV/hwjIzK47bnly01noFUF1rFmStEUAy6ndVRDwgqZek3hHxr0IFZmZWCM1XMmvc3n3wZk1lqm3NgkyJQNLpwDhgM2AbkllErwcOKFxoZmaFt/vgzdb69l9taxZkrRGcRbK+wJMAEfGSpC0KFpWZWYG0tFhNcy2tf1zJTUVZE8EHEfFh4/QSktYjGVlsZlZW1mWxmkpvKsqaCGZL+r/ABpJGAmeSrCVgZlZxmtcaKr2pKGsi+C/gNOBZ4OskE8l5kXkzq0jVtsRl3kSQLkI/PyK2BW4sfEhmZlZMeUcWR8Rq4AVJldk4ZmZW5bI2DW0KzJf0Z+Cdxp0RMaogUZmZWdFkTQTfK2gUZmZWMvlWKOsJnAF8hqSj+OaIWFWMwMzMrDjy1QgmAR8BjwKHAMOAc7OeXNLBwM9J1iy+KSIub6XcUcCdwK4RUZv1/GZmxdDSADOonEFm+RLBsIjYEUDSzcCfs544fdroWmAk0ADMlTQzIuqbletNklyebE/gZmalVEmDzPIlgo8aX0TEqmYL1+ezG7AgIhYCSJoKjAbqm5X7AXAF8J32nNzMrFhampaikgaZ5UsEO0lamb4WycjilenriIg+bby3P7A4Z7sBWGOUhqSdgYER8QdJrSYCSeNIJr1j663LP/uaWXmp9AFm+Rav716oC0vqBvwEODlf2YiYAEwAqKmp8RxHZlZyldRvkHWpynWxBBiYsz0g3deoN7AD8LCkV4HPAzMl1RQwJjOzgqlfurLFRW66uvYsTNNec4EhkgaTJICxwHGNByNiBdC3cVvSw8C3/dSQmZWDSuo3KFgiSDuXzwbuI3l8dGJEzJd0KVAbETMLdW0zs0Jrqd+gXNcxKGSNgIi4h2Sm0tx9F7VSdt9CxmJmVmzl8ohpQROBmVk1Kdd1DJwIzMw6Sbk+ZlrIp4bMzKwMOBGYmVU5Nw2ZmRVIuQw6c43AzKyIuuKgM9cIzMwKpFwGnTkRmJkVSLk8ReSmITOzKudEYGZW5ZwIzMyqnBOBmVmVcyIwM6tyFfHU0EcffURDQwPvv/9+qUOxKtKzZ08GDBhAjx49Sh2KWYdURCJoaGigd+/eDBo0CEmlDseqQETw5ptv0tDQwODBg0sdjpWRrrhmQUU0Db3//vtsvvnmTgJWNJLYfPPNXQu1DusKI40rokYAOAlY0fnfnK2LrrhmQcUkAjOzctAVRxtXRNNQVyCJCy64oGn76quv5uKLL27zPTNnzuTyyy/v8LVvueUW+vXrx4gRI9h+++35yle+wrvvvtvh87bXe++9xz777MPq1aub9v3sZz+jZ8+erFixomnfww8/zMYbb8yIESPYbrvtuOSSSzp87eXLlzNy5EiGDBnCyJEjeeutt9Yq89BDDzFixIimn549e/L73/8eSNr8v/vd7zJ06FC22247rrnmGgDuvvtuLrqoxdVVzSpGQROBpIMlvSBpgaTxLRw/Q9KzkuokPSZpWCHjKaRPfOITTJ8+nTfeeCPze0aNGsX48Wv9WtbJmDFjqKurY/78+ay//vpMmzatU87bHhMnTuTII4+ke/fuTfumTJnCrrvuyvTp09cou/fee1NXV0dtbS233norf/nLXzp07csvv5wDDjiAl156iQMOOKDFBLvffvtRV1dHXV0dDz74IL169eLAAw8EkmS6ePFinn/+eZ577jnGjh0LwKGHHsqsWbNKkljNiqVgTUOSugPXAiOBBmCupJkRUZ9T7LaIuD4tPwr4CXBwR657yaz51L+2siOnWMuwT/bh+4dv32aZ9dZbj3HjxvHTn/6Uyy67bI1js2bN4oc//CEffvghm2++OZMnT2bLLbfklltuoba2lssuu4zhw4fzyiuv0K1bN9555x223XZbFi5cyKJFizjrrLNYtmwZvXr14sYbb2TbbbdtNY5Vq1bxzjvvsOmmm7Z67X79+vHZz36Wxx9/nH79+vHxxx8zdOhQnngiaas844wzWLRoEZB8o99zzz2ZPXs25557LpDUfh555BF69+69xrUnT57Mbbfd1rT98ssv8/bbb/OrX/2Kyy67jFNOOWWteDfccEN22WUXFixYwM4779zm77gtM2bM4OGHHwbgq1/9Kvvuuy9XXHFFq+XvvPNODjnkEHr16gXAddddx2233Ua3bsl3oy222KLpXvfdd1/uvvtujjnmmHWOz6wrK2SNYDdgQUQsjIgPganA6NwCEZH7ib0hEAWMp+DOOussJk+evEYzCMBee+3FnDlzeOqppxg7dixXXnnlGscbm0lmz54NJM0RBx10ED169GDcuHH84he/YN68eVx99dWceeaZLV572rRpjBgxgv79+7N8+XIOP/zwVq/drVs3TjjhBCZPngzAAw88wE477US/fv0499xzOe+885g7dy533XUXp512GpA0dV177bXU1dXx6KOPssEGG6xx/Q8//JCFCxcyaNCgpn1Tp05l7Nix7L333rzwwgv84x//WCvuN998kzlz5rD99msm2n/9619rNOPk/tTX1691nn/84x9stdVWAPzHf/xHi9fKNXXqVI499tim7Zdffplp06ZRU1PDIYccwksvvdR0rKamhkcffbTN85mVs0J2FvcHFudsNwBr9ZJIOgs4H1gf2L+lE0kaB4wD2Hrrtp+1zffNvZD69OnDSSedxDXXXLPGB2VDQwNjxoxh6dKlfPjhhy0+dz5mzBimTZvGfvvtx9SpUznzzDN5++23efzxxzn66KObyn3wwQctXnvMmDH88pe/JCI466yzuOqqqxg/fnyr1z711FMZPXo03/rWt5g4cWLTt/UHHnhgjQ/alStX8vbbb7Pnnnty/vnnc/zxx3PkkUcyYMCANa7/xhtvsMkmm6yxb8qUKfzud7+jW7duHHXUUdxxxx2cffbZADz66KN87nOfo1u3bowfP36tRNC7d2/q6ury/MZbJqnNJ3qWLl3Ks88+y0EHHdS074MPPqBnz57U1tYyffp0Tj311KYP/y222ILXXnttnWIxy6crrGJW8s7iiLg2IrYB/gv471bKTIiImoio6devX3EDbKdvfetb3HzzzbzzzjtN+8455xzOPvtsnn32WW644YYWnz0fNWoU9957L8uXL2fevHnsv//+fPzxx2yyySZN7dp1dXU899xzbV5fEocffjiPPPJIm9ceOHAgW265JQ8++CB//vOfOeSQQwD4+OOPmTNnTtP1lixZwkYbbcT48eO56aabeO+999hzzz15/vnn17juBhtssMZ9Pfvss7z00kuMHDmSQYMGMXXqVKZMmdJ0fO+99+app55i3rx5nHHGGWvdR3trBFtuuSVLly4Fkg/6xqadltx+++0cccQRa4wIHjBgAEceeSQARxxxBM8880zTsffff3+tGpBZIRV7bEEhawRLgIE52wPSfa2ZClxXwHiKYrPNNuOYY47h5ptv5tRTTwVgxYoV9O/fH4BJkya1+L6NNtqIXXfdlXPPPZfDDjuM7t2706dPHwYPHswdd9zB0UcfTUTwzDPPsNNOO7UZw2OPPcY222yT99qnnXYaJ5xwAieeeGJTB++BBx7IL37xC77zne8AUFdXx4gRI3j55ZfZcccd2XHHHZk7dy7PP//8Gn0Vm266KatXr+b999+nZ8+eTJkyhYsvvpgLL7ywqczgwYP529/+lun32N4awahRo5g0aRLjx49n0qRJjB49utWyU6ZM4Uc/+tEa+7785S/z0EMPMXjwYGbPns3QoUObjr344ovssMMOmWMxa4+WVjEbNP4PPPnK8rVqCVn6K9dFIWsEc4EhkgZLWh8YC8zMLSBpSM7mocBLVIALLrhgjaeHLr74Yo4++mh22WUX+vbt2+r7xowZw6233sqYMWOa9k2ePJmbb76ZnXbaie23354ZM2a0+N7GPoLhw4fz1FNP8b3vfS/vtUeNGsXbb7+9RifuNddcQ21tLcOHD2fYsGFcf/31QNJpvMMOOzB8+HB69OjRVIPIdeCBB/LYY48BSRv8EUccscbxI444gqlTp7Z6/x0xfvx47r//foYMGcIDDzzQ9DRWbW1tUz8HwKuvvsrixYvZZ5991nr/XXfdxY477siFF17ITTfd1HTsoYce4tBDDy1I3Ga3nrZ76ccWRETBfoAvAS8CLwPfTfddCoxKX/8cmA/UAQ8B2+c75y677BLN1dfXr7XP8ps7d27stddenXa+efPmxQknnNBp5+sK/v73v8f+++/f6nH/27NCOP7GOXH8jXM69ZxAbbTyuVrQkcURcQ9wT7N9F+W8PreQ17fWXX755Vx33XVNTw51hp133pn99tuP1atXrzGWoJwtWrSIH//4x6UOw6pMsWsIShJF+aipqYna2to19j333HNst912JYrIqpn/7Vm5kDQvImpaOlbyp4Y6S7klNCt//jdnlaIiEkHPnj158803/R/TiibS9Qh69uxZ6lDMOqwiZh8dMGAADQ0NLFu2rNShWBVpXKHMrNxVRCLo0aOHV4kyM1tHFdE0ZGZm686JwMysyjkRmJlVubIbRyBpGZBtwpq19QWyrxxTGXzP1cH3XB06cs+fiogWZ+0su0TQEZJqWxtQUal8z9XB91wdCnXPbhoyM6tyTgRmZlWu2hLBhFIHUAK+5+rge64OBbnnquojMDOztVVbjcDMzJpxIjAzq3IVmQgkHSzpBUkLJI1v4fgnJE1Ljz8paVAJwuxUGe75fEn1kp6R9CdJnypFnJ0p3z3nlDtKUkgq+0cNs9yzpGPSv+v5km4rdoydLcO/7a0lPSTpqfTf95dKEWdnkTRR0uuS/trKcUm6Jv19PCNp5w5ftLWly8r1B+hOsjTmp4H1gaeBYc3KnAlcn74eC0wrddxFuOf9gF7p629Uwz2n5XoDjwBzgJpSx12Ev+chwFPApun2FqWOuwj3PAH4Rvp6GPBqqePu4D3/H2Bn4K+tHP8S8EdAwOeBJzt6zUqsEewGLIiIhRHxITAVGN2szGhgUvr6TuAASSpijJ0t7z1HxEMR8W66OQco9/mTs/w9A/wAuAJ4v5jBFUiWez4duDYi3gKIiNeLHGNny3LPAfRJX28MvFbE+DpdRDwCLG+jyGjgN5GYA2wiaauOXLMSE0F/YHHOdkO6r8UyEbEKWAFsXpToCiPLPef6Gsk3inKW957TKvPAiPhDMQMroCx/z0OBoZL+V9IcSQcXLbrCyHLPFwMnSGogWSP9nOKEVjLt/f+eV0WsR2DZSToBqAH2KXUshSSpG/AT4OQSh1Js65E0D+1LUut7RNKOEfHPUgZVYMcCt0TEjyXtAfxW0g4R8XGpAysXlVgjWAIMzNkekO5rsYyk9Uiqk28WJbrCyHLPSPoi8F1gVER8UKTYCiXfPfcGdgAelvQqSVvqzDLvMM7y99wAzIyIjyLiFeBFksRQrrLc89eA2wEi4gmgJ8nkbJUq0//39qjERDAXGCJpsKT1STqDZzYrMxP4avr6K8CDkfbClKm89yzpc8ANJEmg3NuNIc89R8SKiOgbEYMiYhBJv8ioiKgtTbidIsu/7d+T1AaQ1JekqWhhEWPsbFnueRFwAICk7UgSQSWvWzsTOCl9eujzwIqIWNqRE1Zc01BErJJ0NnAfyRMHEyNivqRLgdqImAncTFJ9XEDSKTO2dBF3XMZ7vgrYCLgj7RdfFBGjShZ0B2W854qS8Z7vAw6UVA+sBr4TEWVb2814zxcAN0o6j6Tj+ORy/mInaQpJMu+b9nt8H+gBEBHXk/SDfAlYALwLnNLha5bx78vMzDpBJTYNmZlZOzgRmJlVOScCM7Mq50RgZlblnAjMzKqcE4EVnKTVkuok/VXSLEmbdPL5X02fmUfS262U2UDSbEndJQ2S9F4aU72k69ORyO25Zo2ka9LX+0r6Qs6xMySd1JF7Ss9zsaRv5ylzi6SvtOOcg1qb1bJZucskLW7++5R0tqRTs17PyoMTgRXDexExIiJ2IBm3cVYJYjgVmB4Rq9PtlyNiBDCcZMbKL7fnZBFRGxHfTDf3Bb6Qc+z6iPhNRwMusVkkE741N5HKn8un6jgRWLE9QTpBlqRtJN0raZ6kRyVtm+7fUtLvJD2d/nwh3f/7tOx8SePaed3jgRnNd6aTDj4OfCb9tvyg/r1mw9bpdY9OazNPS3ok3bevpLuVrGVxBnBeWsPYu/GbvKRtJf258Vrp+Z9NX++S1lDmSbpPeWaPlHS6pLlpDHdJ6pVz+IuSaiW9KOmwtHx3SVel73lG0tfb88uKiDktjVZNZ7B9VVJLScLKlBOBFY2k7iRTATSO+p0AnBMRuwDfBn6V7r8GmB0RO5HMyz4/3X9qWrYG+KakTDPGplMTfDoiXm3hWK80pmeBXwCTImI4MDmNA+Ai4KA0njVGY6fnvB74aVrreTTn2PPA+pIGp7vGANMk9Uiv9ZX0fiYCl+W5jekRsWsaw3Mk8+s0GkTy7f1Q4HpJPdPjKyJiV2BX4PScOBrv/ZOS7slz3ZbUAnuvw/usi6q4KSasS9pAUh1JTeA54H5JG5E0pzROeQHwifTP/YGTANKmnBXp/m9KOiJ9PZBkMrUs0yf0Bf7ZbN82aUwBzIiIP0r6LXBkevy3wJXp6/8FbpF0OzA9w/Vy3U6SAC5P/xwDfJZkQrz703vvDuSbK2YHST8ENiGZKuS+3GukM22+JGkhsC1wIDA8p/9gY5Lf14uNb4qI10imKmiv19NrWIVwIrBieC8iRqTfvu8j6SO4Bfhn2k6fl6R9gS8Ce0TEu5IeJplcLNP1Wyj7ctZrR8QZknYn+cY9T9IuGa8LMI0k2U1PThUvSdoRmB8Re7TjPLcAX46IpyWdTDqxXGOIzUMmWb3qnIjITRioc5Zl7UnyO7UK4aYhK5q0ffmbJJOEvQu8IuloaFqHdae06J9IltNsbOvemOQb7VtpEtiWZFrprNd9C+ieNpm05XH+PQHh8cCjaQzbRMSTEXERyayWA5u9718k0163dO2XSSZ/+x5JUgB4AeinZO58JPWQtH2e2HoDS9NmpeObHTtaUjdJ25As6fgCScL9RloeSUMlbZjnGlkNBfI+eWTlw4nAiioingKeIVlM5Hjga5KeJukHaFyC8Fxgv7RjdR7JUz33AutJeo6kmWVOOy/9P8BeecqcA5wi6RngxDQOgKskPZs+dvk4ybq5uWYBRzR2Frdw3mnACfx7zvwPSaY/vyK99zpynjpqxfeAJ0maqZ5vdmwR8GeSVefOiIj3gZuAeuAvadw30KwFoK0+AklXKpn5spekBkkX5xzeE7g/T7xWRjz7qFUFJctWnhcRJ5Y6lnKmZF2L8/17rCyuEVhViIi/AA+lTy7ZuutLUjuxCuIagZlZlXONwMysyjkRmJlVOScCM7Mq50RgZlblnAjMzKrc/we1mQynsNMUNAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "PrecisionRecallDisplay.from_predictions(\n",
    "    y_true=test[\"label_num\"],\n",
    "    y_pred=score_nb,\n",
    "    name=\"Naive Bayes\",\n",
    ")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}