{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "07463a5f",
   "metadata": {},
   "source": [
    "# ロジスティック回帰\n",
    "\n",
    "ロジスティック回帰全般については以下を参照してください。\n",
    "[https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression](https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "22ae72e6",
   "metadata": {},
   "source": [
    "**データとモジュールのロード**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "4f42c570",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from sklearn import model_selection\n",
    "\n",
    "data = pd.read_csv(\"input/pn_same_judge_preprocessed.csv\")\n",
    "train, test = model_selection.train_test_split(data, test_size=0.1, random_state=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "55fdf55a",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.metrics import ConfusionMatrixDisplay\n",
    "from sklearn.metrics import PrecisionRecallDisplay"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "310d666d",
   "metadata": {},
   "source": [
    "## LogisticRegression"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ea30df74",
   "metadata": {},
   "source": [
    "[sklearn.linear_model.LogisticRegression](https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.LogisticRegression.html)\n",
    "を使います。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "79cc90a0",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "bb3dbb21",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>#sk-container-id-1 {color: black;background-color: white;}#sk-container-id-1 pre{padding: 0;}#sk-container-id-1 div.sk-toggleable {background-color: white;}#sk-container-id-1 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-1 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-1 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-1 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-1 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-1 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-1 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-1 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-1 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-1 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-1 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-1 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-1 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-1 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-1 div.sk-item {position: relative;z-index: 1;}#sk-container-id-1 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-1 div.sk-item::before, #sk-container-id-1 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-1 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-1 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-1 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-1 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-1 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-1 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-1 div.sk-label-container {text-align: center;}#sk-container-id-1 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-1 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-1\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>Pipeline(steps=[(&#x27;vect&#x27;,\n",
       "                 TfidfVectorizer(tokenizer=&lt;method &#x27;split&#x27; of &#x27;str&#x27; objects&gt;)),\n",
       "                (&#x27;clf&#x27;, LogisticRegression())])</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-1\" type=\"checkbox\" ><label for=\"sk-estimator-id-1\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">Pipeline</label><div class=\"sk-toggleable__content\"><pre>Pipeline(steps=[(&#x27;vect&#x27;,\n",
       "                 TfidfVectorizer(tokenizer=&lt;method &#x27;split&#x27; of &#x27;str&#x27; objects&gt;)),\n",
       "                (&#x27;clf&#x27;, LogisticRegression())])</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-2\" type=\"checkbox\" ><label for=\"sk-estimator-id-2\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">TfidfVectorizer</label><div class=\"sk-toggleable__content\"><pre>TfidfVectorizer(tokenizer=&lt;method &#x27;split&#x27; of &#x27;str&#x27; objects&gt;)</pre></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-3\" type=\"checkbox\" ><label for=\"sk-estimator-id-3\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">LogisticRegression</label><div class=\"sk-toggleable__content\"><pre>LogisticRegression()</pre></div></div></div></div></div></div></div>"
      ],
      "text/plain": [
       "Pipeline(steps=[('vect',\n",
       "                 TfidfVectorizer(tokenizer=<method 'split' of 'str' objects>)),\n",
       "                ('clf', LogisticRegression())])"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pipe = Pipeline([\n",
    "    (\"vect\", TfidfVectorizer(tokenizer=str.split)),\n",
    "    (\"clf\", LogisticRegression())\n",
    "])\n",
    "\n",
    "pipe.fit(train[\"tokens\"], train[\"label_num\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "441429d6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<sklearn.metrics._plot.confusion_matrix.ConfusionMatrixDisplay at 0x7f6a74c22d60>"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAATgAAAEGCAYAAADxD4m3AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAZ2ElEQVR4nO3de7QU1Zn38e/vnMNFLoKIEkSMGhElJoDB+8TxfovvQrOcqMkkjHFedKLGJJpZ5rJMonHMJBrfqNGMqNHMqEQnOmJiNIpmecMLGEUBEbwNIIqAgiDXPs/7R9fRBjl9qs7ppruL32etWqdqd3XVc2DxsHftXXsrIjAzy6OmWgdgZlYtTnBmlltOcGaWW05wZpZbTnBmllsttQ6g1MABzbHz0G61DsMyeHl6r1qHYBmsZiVrY426co2jD+0dS5YWUp07bfqa+yPimK7cryvqKsHtPLQbT98/tNZhWAZH7zCq1iFYBk/F5C5fY/HSAk/dv2Oqc7sNfmVgl2/YBXWV4MysEQSFaK11EKk4wZlZJgG00hgvCDjBmVlmrbgGZ2Y5FATr3EQ1szwKoOAmqpnllZ/BmVkuBVBokFmInODMLLPGeALnBGdmGQXhZ3Bmlk8RsK4x8psTnJllJQp06XXWzcYJzswyCaDVNTgzyyvX4Mwsl4oDfZ3gzCyHAlgXjTFXrhOcmWUSiEKDTAbuBGdmmbWGm6hmlkN+BmdmOSYKfgZnZnlUnNHXCc7McihCrI3mWoeRihOcmWXW6mdwZpZHxU4GN1HNLJfcyWBmOeVOBjPLtUKDDPRtjDRsZnUjEOuiJdVWjqSekp6W9LykGZJ+kpTvIukpSXMl/V5S96S8R3I8N/l8545idYIzs0zaOhnSbB1YAxwWESOBUcAxkvYH/h24IiJ2A94FTk/OPx14Nym/IjmvLCc4M8skEIVIt5W9TtGK5LBbsgVwGPDfSfnNwAnJ/tjkmOTzwyWVvYkTnJll1kpTqg0YKGlqyTa+9DqSmiU9BywCHgBeAd6LiPXJKfOBIcn+EGAeQPL5MmDbcnG6k8HMMokgyzCRxRExpv1rRQEYJak/cBewR9cj/IgTnJllUuxkqOyrWhHxnqSHgQOA/pJaklrajsCC5LQFwFBgvqQWoB+wpNx13UQ1s8wq0ckgabuk5oakrYAjgVnAw8BJyWnjgLuT/UnJMcnnD0VE2eVvXIMzs0wCVWrCy8HAzZKaKVa2bo+IP0qaCUyU9FPgb8ANyfk3AP8paS6wFDiloxs4wZlZZpV4FzUipgOjN1H+KrDvJspXA/+Q5R5OcGaWSXFd1MZ4uuUEZ2YZeWV7M8up4rKBnvDSzHIoQm6imll+eT44M8ul4nxwfgZnZrnkGX3NLKeKw0RcgzOzHKrGu6jV4gRnZpl5TQYzy6XidEluoppZTvkZnJnlUnE2ETdRzSyHiq9qOcFtEdauFud9cTfWrW2isB4+/4VlfO27b334+TU/HML9Ewdw99wXAHh7fjd++Z2dWLakhb79C/zrVW+w3Q7rahW+bUJTU3DVfS+zZGE3Lhy3a63DqUONU4OrapSSjpE0O1nH8IJq3qtWuvUIfn7HK/zmwdlc+8Bspv61L7Om9QLg5ee3YsWyDbvTJ1w0hCNOWspvJs/mK99+i99eOrgWYVsZJ/zzYubN6VnrMOpaK0q11VrVElwyS+evgWOBEcCpkkZU6361IsFWvVsBWL9OFNYJCQoFmHDxDpz+wzc3OP+Nl3sw8qDiSmkjD1rBlPv7bfaYrX0DB69l38OX8+dbB9Q6lLrV1ova1WUDN4dq1uD2BeZGxKsRsRaYSHFdw9wpFOBfjhjOyZ/di9EHv88ee3/ApN8O5ICjlrPtoPUbnLvriNU8/udiUnv8z/34YEUzy5c2xqDJLcGZP3mT6386mGit/T/OetYaTam2WqtmBB+uYZgoXd/wQ5LGt62Z+M6SQhXDqZ7mZrj2wdncMm0ms5/rxQtP9ubRe/oz9uvvfOzc8Rcu4IUpffjGkbvzwpQ+DBy8libnt7qw3xHLeW9xC3Nf6FXrUOpa25oMabZaq3knQ0RcB1wHMGZkz7Ir5NS7Pv0KjDxwBc8/3oc3X+/BaQcWW+RrVjXxTwfuyU1PzGLbT6znwhteB2DVyiYeu7cfffo1ZmLPmxH7rGT/o5azz+Ez6d4j6NW32An083M+WevQ6koA6+ugdpZGNRNc2xqGbUrXN8yN95Y009JSTG5rVolnH+nLl85axMTnZ3x4ztjdPsNNT8wCYNmSZvpuU6CpCSZetT1Hnby0VqHbRn576eAPO30+e8AKTjpzkZNbO+qh+ZlGNRPcM8AwSbtQTGynAF+u4v1qYunb3bjs3J1obRWtrXDw/3mP/Y9c3u7506f04cZLd0AKPrPfSs76t/mbMVqzCqiT5mcaVUtwEbFe0tnA/UAzcGNEzOjgaw1n1xGrueaBl8ue0zYGDuDzxy/j88cvq3ZY1kXTp/Rh+pQ+tQ6jLjXShJdVrWdGxL0RsXtEfCoiLqnmvcxs86lEJ4OkoZIeljRT0gxJ5yblP5a0QNJzyXZcyXe+l4yrnS3p6I7irHkng5k1lgpOeLkeOC8inpXUF5gm6YHksysi4rLSk5NxtKcAnwZ2AB6UtHtEtNtL5wRnZpkEYn1rRVa2XwgsTPbflzSLTQwlKzEWmBgRa4DXJM2lON52SntfaIyuEDOrKxle1RrYNs412cZv6nqSdgZGA08lRWdLmi7pRknbJGWpxtaWcg3OzLKJTE3UxRExptwJkvoAfwC+FRHLJV0LXFy8ExcDlwNf70yoTnBmlkklF52R1I1icrslIu4EiIi3Sz6fAPwxOcw8ttZNVDPLrEK9qAJuAGZFxC9Lykun2DkReDHZnwScIqlHMr52GPB0uXu4BmdmmQSiUIFOBuAg4KvAC5KeS8q+T3HmoVEUK4uvA2cARMQMSbcDMyn2wJ5VrgcVnODMrBMqMdA3Ih6DTV7o3jLfuQRIPabWCc7MMolsnQw15QRnZpmFE5yZ5ZNftjezHHMNzsxyKQIKDTKluxOcmWXWKNMlOcGZWSaBm6hmllvuZDCzHIsGWR7KCc7MMnMT1cxyqdiL2hjzdDjBmVlmbqKaWW65iWpmuRTICc7M8qtBWqhOcGaWUUD4VS0zyys3Uc0stxq+F1XSVZRpakfEN6sSkZnVtby8izp1s0VhZo0jgEZPcBFxc+mxpF4R8UH1QzKzetcoTdQO37eQdICkmcBLyfFISddUPTIzq1MiWtNttZbmhbL/BxwNLAGIiOeBg6sYk5nVu0i5lSFpqKSHJc2UNEPSuUn5AEkPSJqT/NwmKZekKyXNlTRd0t4dhZnqjdmImLdRUdnFVs0sx6LYyZBm68B64LyIGAHsD5wlaQRwATA5IoYBk5NjgGMprmY/DBgPXNvRDdIkuHmSDgRCUjdJ5wOzUnzPzPKqAjW4iFgYEc8m++9TzCtDgLFAWx/AzcAJyf5Y4HdR9CTQX9LgcvdIk+DOBM5KbvwmMCo5NrMtllJuDJQ0tWQbv8mrSTsDo4GngEERsTD56C1gULI/BChtTc5PytrV4UDfiFgMfKWj88xsC9Ka+szFETGm3AmS+gB/AL4VEculj5q2ERGSOt1nm6YXdVdJ90h6R9IiSXdL2rWzNzSzBtc2Di7N1gFJ3Sgmt1si4s6k+O22pmfyc1FSvgAYWvL1HZOydqVpot4K3A4MBnYA7gBuS/E9M8upiHRbOSpW1W4AZkXEL0s+mgSMS/bHAXeXlH8t6U3dH1hW0pTdpDQJrldE/GdErE+2/wJ6pviemeVVBToZgIOArwKHSXou2Y4DfgYcKWkOcERyDHAv8CowF5gAfKOjG5R7F3VAsvtnSRcAE5OQT05uZGZbqgq8qhURj0G7K0gfvonzg4wdnOU6GaZRTGhtAZxRei/ge1luZGb50fnH/ptXuXdRd9mcgZhZgwhBHbyGlUaq+eAk7QWMoOTZW0T8rlpBmVmda/QaXBtJPwIOoZjg7qX4usRjgBOc2ZaqQRJcml7Ukyg+8HsrIk4DRgL9qhqVmdW3yvSiVl2aJuqqiGiVtF7S1hQH3Q3t6EtmllN5mPCyxFRJ/SmOO5kGrACmVDMoM6tvDd+L2iYi2gbT/UbSfcDWETG9umGZWV1r9ARXbjI5SXu3TXNiZluePNTgLi/zWQCHVTgW5szuzxf+7oRKX9aqqOUTq2odgmWgxRVaKbTRn8FFxKGbMxAzaxB10kOahhd+NrPsnODMLK+UfsLLmnKCM7PsGqQGl2ZGX0n6R0kXJsc7Sdq3+qGZWT1SpN9qLc2rWtcABwCnJsfvA7+uWkRmVv8qNGV5taVpou4XEXtL+htARLwrqXuV4zKzelYHtbM00iS4dZKaSX4lSduRZU0dM8udemh+ppEmwV0J3AVsL+kSirOL/LCqUZlZ/Yoc9aJGxC2SplGcMknACRHhle3NtmR5qcFJ2gn4ALintCwi/reagZlZHctLggP+xEeLz/QEdgFmA5+uYlxmVsdy8wwuIj5TepzMMtLheoRmZrWWZhzcBpJpkvarQixm1igqNGW5pBslLZL0YknZjyUt2Ggx6LbPvidprqTZko7u6PppnsF9p+SwCdgbeLPj0M0slyrbi3oTcDUfX8Tqioi4rLRA0gjgFIqPx3YAHpS0e0QU2rt4mhpc35KtB8VncmPTRm9mOVShGlxEPAIsTXnXscDEiFgTEa8Bc4Gyr42WrcElA3z7RsT5KQMws5wTmToZBkqaWnJ8XURcl+J7Z0v6GjAVOC8i3gWGAE+WnDM/KWtXuzU4SS1J1e+gFMGY2ZYkfQ1ucUSMKdnSJLdrgU8Bo4CFlJ9dvKxyNbinKT5ve07SJOAOYGXbhxFxZ2dvamYNrMozhUTE2237kiYAf0wOF7DhkqU7JmXtSjMOriewhOIaDG3j4QJwgjPbUlXxVS1JgyNiYXJ4ItDWwzoJuFXSLyl2MgyjWBFrV7kEt33Sg/oiHyW2Ng0yzM/MqqFSNThJtwGHUHxWNx/4EXCIpFEU88zrwBkAETFD0u3ATGA9cFa5HlQon+CagT5smNjaOMGZbckqlAEi4tRNFN9Q5vxLgEvSXr9cglsYERelvZCZbSFysqpW7afjNLO6lId3UQ/fbFGYWWNp9AQXEWlHF5vZFiY3E16amW0gJ8/gzMw+RjTOA3onODPLzjU4M8urPPSimpltmhOcmeVSnpYNNDP7GNfgzCyv/AzOzPLLCc7M8so1ODPLp6CqE15WkhOcmWWScdGZmnKCM7PsnODMLK8UjZHhnODMLBvPJmJmeeZncGaWW35Vy8zyq0FqcE21DsDMGkyysn2arSOSbpS0SNKLJWUDJD0gaU7yc5ukXJKulDRX0nRJe3d0fSc4M8suUm4duwk4ZqOyC4DJETEMmJwcAxxLcTX7YcB44NqOLu4EZ2aZtA30rUQNLiIeATZe4GoscHOyfzNwQkn576LoSaC/pMHlru9ncGaWmVpTP4QbKGlqyfF1EXFdB98ZFBELk/23gEHJ/hBgXsl585OyhbTDCc7Mssk2Dm5xRIzp9K0iQur8oBQnuArq1r3Av1/9GN26t9LcHDz+8A7ccuMejPzcO3z9GzNoagpWrWrhiktGs3BBn1qHa8DAQas576IX6L/tWiLgvjt3ZNJtn2TX3Zdz1g9m0b17K4WCuObSPXl5Rr9ah1s3qjxM5G1JgyNiYdIEXZSULwCGlpy3Y1LWrqolOEk3AscDiyJir2rdp56sW9vE9889iNWrWmhubuUX1z7K1Ke256zzn+fiC/Zj3ht9+cKJr3HKuJe54t867ACyzaBQENdfMZxXXtqarXqt51e3PMnfntyW086dw63/sSvTntiOMQe9w2nnvsz3xu9T63DrR3WHiUwCxgE/S37eXVJ+tqSJwH7AspKm7CZVs5PhJj7eO5JzYvWq4v8ZLS3FWhwBEdCr9zqg+HPJ4p61DNJKvLu4B6+8tDUAqz5oYd5rvdl2+zUE0KtPAYDefdaz9J0eNYyy/lRwmMhtwBRguKT5kk6nmNiOlDQHOCI5BrgXeBWYC0wAvtHR9atWg4uIRyTtXK3r16umpuBXN/yVwUNW8qe7dmH2zAFc+bPR/PgXT7J2TTMfrGzhO2ccXOswbRO2H7yKXYe/z+wX+zHhsuFcdPWznP6t2agJzj9t31qHVz+C4v/albhUxKntfHT4Js4N4Kws16/5MBFJ4yVNlTR1beGDWofTZa2t4pzTDmXcF49m9z3f45O7LOeEk1/hx9/dn3FfPJoH7t2J/3vOix1fyDarnlut5weXPceEy4ezamULx500nwmXD+efjvt7Jlw+nG9dOKPWIdYVtabbaq3mCS4irouIMRExpntzr1qHUzErV3Rj+rMD+dz+b7PLbsuYPXMAAI8+NIQ999p42I/VUnNLK9+/7HkevncwTzxUHJFw+PFv8sRD2wPw2AOD2P3Ty2oZYl2p5Di4aqt5gsuTrfuvoXef4rO27t0LjNpnEfPe6Euv3uvZYegKAEaPeYd5b/StZZi2geDcC2cw77Xe/M8tO39YunRxDz7zuXcBGLnvUt6cl5//fLssIv1WYx4mUkEDtl3Nd37wN5qaAjUFjz00hGee+ARX/XwkP/jp07SGWPF+N3516ehah2qJEaPe4/DjF/LanD5cddsUAG6+ejeuvHgEZ3z3JZqag3Vrmrjqp5+ucaT1pR5qZ2koqpRlk96RQ4CBwNvAjyLihnLf6dfzE3Hgjl+tSjxWJR+sqnUElsETi29n2dpF6so1+vbfMUYffG6qcx+951+ndWWgb1dVsxe1vd4RM2twjVKDcxPVzLIJoNAYGc4Jzswycw3OzPKrDnpI03CCM7PMXIMzs3zysoFmllcC5E4GM8srr2xvZvnkJqqZ5Vd9vGeahhOcmWXmXlQzyy/X4Mwsl8K9qGaWZ42R35zgzCw7DxMxs/xygjOzXAqgDhaUScMJzswyEeEmqpnlWGtlqnCSXgfeBwrA+ogYI2kA8HtgZ+B14EsR8W5nru9Vtcwsm7YmapotnUMjYlTJ2g0XAJMjYhgwOTnuFCc4M8tMEam2ThoL3Jzs3wyc0NkLOcGZWXbp10UdKGlqyTZ+4ysBf5E0reSzQRGxMNl/CxjU2TD9DM7MMsr0sv3iDpYN/LuIWCBpe+ABSS9tcKeIkDr/5qsTnJllU8FVtSJiQfJzkaS7gH2BtyUNjoiFkgYDizp7fTdRzSyzSjyDk9RbUt+2feAo4EVgEjAuOW0ccHdn43QNzsyyq8w4uEHAXZKgmItujYj7JD0D3C7pdOAN4EudvYETnJllE0Br1xNcRLwKjNxE+RLg8C7fACc4M8vMM/qaWZ45wZlZLgVQaIy37Z3gzCyjgHCCM7O8chPVzHKpQr2om4MTnJll5xqcmeWWE5yZ5VIEFAq1jiIVJzgzy841ODPLLSc4M8uncC+qmeVUQHigr5nlll/VMrNciqjYsoHV5gRnZtm5k8HM8ipcgzOzfPKEl2aWV37Z3szyKoDwq1pmlkvhCS/NLMfCTVQzy60GqcEp6qg3RNI7FBd6zZuBwOJaB2GZ5PXv7JMRsV1XLiDpPop/PmksjohjunK/rqirBJdXkqZGxJhax2Hp+e8sH5pqHYCZWbU4wZlZbjnBbR7X1ToAy8x/ZzngZ3BmlluuwZlZbjnBmVluOcFVkaRjJM2WNFfSBbWOxzom6UZJiyS9WOtYrOuc4KpEUjPwa+BYYARwqqQRtY3KUrgJqNnAVKssJ7jq2ReYGxGvRsRaYCIwtsYxWQci4hFgaa3jsMpwgqueIcC8kuP5SZmZbSZOcGaWW05w1bMAGFpyvGNSZmabiRNc9TwDDJO0i6TuwCnApBrHZLZFcYKrkohYD5wN3A/MAm6PiBm1jco6Iuk2YAowXNJ8SafXOibrPL+qZWa55RqcmeWWE5yZ5ZYTnJnllhOcmeWWE5yZ5ZYTXAORVJD0nKQXJd0hqVcXrnWTpJOS/evLTQQg6RBJB3biHq9L+tjqS+2Vb3TOioz3+rGk87PGaPnmBNdYVkXEqIjYC1gLnFn6oaROrXMbEf8cETPLnHIIkDnBmdWaE1zjehTYLaldPSppEjBTUrOkX0h6RtJ0SWcAqOjqZH66B4Ht2y4k6a+SxiT7x0h6VtLzkiZL2pliIv12Unv8vKTtJP0hucczkg5KvrutpL9ImiHpekAd/RKS/kfStOQ74zf67IqkfLKk7ZKyT0m6L/nOo5L2qMifpuWSV7ZvQElN7VjgvqRob2CviHgtSRLLImIfST2AxyX9BRgNDKc4N90gYCZw40bX3Q6YABycXGtARCyV9BtgRURclpx3K3BFRDwmaSeKb2vsCfwIeCwiLpL0BSDNWwBfT+6xFfCMpD9ExBKgNzA1Ir4t6cLk2mdTXAzmzIiYI2k/4BrgsE78MdoWwAmusWwl6blk/1HgBopNx6cj4rWk/Cjgs23P14B+wDDgYOC2iCgAb0p6aBPX3x94pO1aEdHevGhHACOkDytoW0vqk9zji8l3/yTp3RS/0zclnZjsD01iXQK0Ar9Pyv8LuDO5x4HAHSX37pHiHraFcoJrLKsiYlRpQfIPfWVpEXBORNy/0XnHVTCOJmD/iFi9iVhSk3QIxWR5QER8IOmvQM92To/kvu9t/Gdg1h4/g8uf+4F/kdQNQNLuknoDjwAnJ8/oBgOHbuK7TwIHS9ol+e6ApPx9oG/JeX8Bzmk7kDQq2X0E+HJSdiywTQex9gPeTZLbHhRrkG2agLZa6JcpNn2XA69J+ofkHpI0soN72BbMCS5/rqf4fO3ZZOGU/6BYU78LmJN89juKM2ZsICLeAcZTbA4+z0dNxHuAE9s6GYBvAmOSToyZfNSb+xOKCXIGxabq/3YQ631Ai6RZwM8oJtg2K4F9k9/hMOCipPwrwOlJfDPwNPBWhmcTMbPccg3OzHLLCc7McssJzsxyywnOzHLLCc7McssJzsxyywnOzHLr/wM38P8gdoBRgQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "pred = pipe.predict(test[\"tokens\"])\n",
    "ConfusionMatrixDisplay.from_predictions(y_true=test[\"label_num\"], y_pred=pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "dd63ab0c",
   "metadata": {},
   "outputs": [],
   "source": [
    "score = pipe.predict_proba(test[\"tokens\"])[:,1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "2b58a9c2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<sklearn.metrics._plot.precision_recall_curve.PrecisionRecallDisplay at 0x7f6a74b668b0>"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAApQElEQVR4nO3de5xVdb3/8dc7QEFEMIEykEuoGQKSjhIHOV5KIzXJK4Ke1FTUvB1LT/Q7hqKdn1pG5pHzA1O8VN7STFQSPYmXyguDjSgYSUA6QEcOgty8gZ/fH2sNbebCXjPMnj2z9/v5eOzH7HX/rGHYn72+V0UEZmZWvj5R7ADMzKy4nAjMzMqcE4GZWZlzIjAzK3NOBGZmZa59sQNorO7du0e/fv2KHYaZWZsyd+7c/42IHvVta3OJoF+/flRWVhY7DDOzNkXS3xra5qIhM7My50RgZlbmnAjMzMqcE4GZWZlzIjAzK3MFSwSSpkt6W9JrDWyXpJskLZI0T9L+hYrFzMwaVsgngjuAUdvY/lVgr/Q1Hvh/BYzFzMwaULB+BBHxrKR+29hlNHBXJONgvyCpm6TdI2JFIeKZ9Mh8FixfW4hTF8Xoob0YN6xPscMwsxJQzDqCXsBbOcvV6bo6JI2XVCmpcuXKlS0SXGu2YMVaHq5aVuwwzKxEtImexRFxC3ALQEVFRZNm0rnya/s2a0zFNGba88UOwcxKSDGfCJYBe+Qs907XmZlZCypmIpgBfCNtPfRF4N1C1Q+YmVnDClY0JOke4FCgu6Rq4EqgA0BETAVmAkcBi4CNwJmFisXMzBpWyFZDY/NsD+CCQl3fzMyycc9iM7My50RgZlbmnAjMzMqcE4GZWZlzIjAzK3NOBGZmZc6JwMyszLWJsYasuO5+8c06g9x59FOz0uFEYFup70P/xSXvADCs/yeBZPRTwInArEQ4EZSxLB/6Ne9znwA8+qlZaXEiKCO1P/izfOibWelzIihRWb7t+0PfzMCJoGT4276ZNZUTQYl4uGoZC1asZeDuuwD+0Dez7JwI2qCab/u5lbY1SeC+c4cXK6wmcdNUs+JzIigRA3ffhdFDexU7jG1y01Sz1smJoA06eM/uAPzi7GFFjmTbmlJv4aapZi3PiaANao0JwK2UzNqugiYCSaOAnwLtgFsj4rpa2/sC04EewDvAaRFRXciYrDBqV1aDP/jN2opCTl7fDpgCHAFUA3MkzYiIBTm73QDcFRF3SjocuBb4l0LFZM2jlCqrzaywo48eBCyKiMUR8SFwLzC61j4DgafS97Pr2W5tRFuorDaz+hWyaKgX8FbOcjVQu3D7FeB4kuKj44AuknaLiFW5O0kaD4wH6NPHxQzF1lYqq80sm2JXFl8G3CzpDOBZYBmwufZOEXELcAtARUVFtGSAVpcTgFlpKWQiWAbskbPcO123RUQsJ3kiQNLOwAkRsaaAMZmZWS2FTARzgL0k9SdJAKcA43J3kNQdeCciPga+R9KCyKzF1NfstT5u/WSlrGCJICI2SboQmEXSfHR6RMyXdDVQGREzgEOBayUFSdHQBYWKxyzr/Au1ubezlbqC1hFExExgZq11E3PePwA8UMgYrHw114isTe3t3NDThp8urLUpdmWxWcG05IisWZ82/HRhrVGmRCCpJzAC+AzwHvAaSfHOxwWMzSyz+j6IC9nJralPGx5LyVqjbSYCSYcBE4BPAn8C3gY6Al8HBkh6APhxRKwtcJxmW8nyQdxcndzq60ldyHGUXKRkLS3fE8FRwDkR8WbtDZLaA8eQDCHxYAFiMwNa54B2zXU9FylZa7DNRBARl29j2ybgN80dkJW3LN++a9631Ad/c/akrn1/LlKy1qDJlcWSzoyI25szGLP6FHsU00L2pC72vZnB9rUamgQ4EVhBlOoops31dFHIegTXUZSffJXF8xraBHyq+cOxclfzQVmqmpoAshQpNbUeIUvFu+soSlu+J4JPAV8BVtdaL+CPBYnIypoHtMumqfUITa14dx1FacuXCB4Fdo6IqtobJD1diIDMrK6mFCllbZHkegpTRNsa1bmioiIqKyuLHYZZq9NvwmPAPz7kGxpHqSkf+rXP3dTzWPFImhsRFfVt8xATZiWqkN/0XWdQWpwIzEpEIWeOq31u1xmUFhcNmVmj1VdUBC4uas22VTRUyMnrzayMLFixNtMkP9b6ZC4aknRLRIxvaNnMykd9xVAuLmq7GlNHMC3PspmVifrqIeobJwpcXNQWZC4aioi521quj6RRkhZKWiRpQj3b+0iaLelPkuZJOiprPGbW+rm4qG3IN8TEI0CDtckRcew2jm0HTCEZproamCNpRkQsyNntCuD+iPh/kgaSTGvZL3v4ZtZauLio7cpXNHTDdpz7IGBRRCwGkHQvMBrITQQB7JK+7wos347rmVkReXiQtivffATP1LyX1AnoExELM567F/BWznI1UPsv5SrgCUkXAZ2BL9d3IknjgfEAffq4rNHMrDllqiOQ9DWgCng8XR4qaUYzXH8scEdE9CaZDe3nkurEFBG3RERFRFT06NGjGS5rZmY1srYauoqkqOdpgIioktQ/zzHLgD1ylnun63KdBYxKz/m8pI5Ad5K5kc2sjauvJZFbEbU+WVsNfRQR79Zal69L8hxgL0n9Je0AnALUfop4E/gSgKTPAx2BlRljMrM2xq2IWqesTwTzJY0D2knaC7iYPPMRRMQmSRcCs4B2wPSImC/paqAyImYA3wF+JulSksRyRrS1MS/MrEG1WxL1m/AYLy55x30NWpmsieAi4N+BD4B7SD7cr8l3UETMJGkSmrtuYs77BcCIrMGaWduSpSWRRzItvkyJICI2Av8u6fpkMdYVNiwzK0Xua9A6ZUoEkg4EpgNd0uV3gW9m6V1sZlYj69AULipqWVkri28DvhUR/SKiH3ABcHvBojKzsuUK5ZaXtY5gc0Q8V7MQEb+XtKlAMZlZGfGkN8WXb6yh/dO3z0iaRlJRHMAY0j4FZmbbw0NTFF++J4If11q+Mue9m3mamZWAfGMNHdZSgZiZWXE0Zoayo4F9SXr/AhARVxciKDMzazlZB52bSlIvcBEg4CSgbwHjMjOzFpL1ieCfImKIpHkRMUnSj4HfFjIwMytPnvKy5WXtR/Be+nOjpM8AHwG7FyYkM7OtuW9BYWV9InhUUjfgR8DLJC2Gbi1UUGZWvjwMRcvLOtZQzQBzD0p6FOhYz7DUZmbbzf0KWl6+DmXHb2MbEfHr5g/JzMxaUr4ngq9tY1sATgRmZm1cvg5lZ7ZUIGZmVhxZWw2ZmVmJciIwMytzmYeYaApJo4CfksxZfGtEXFdr+0+AmvGMdgJ6RkS3QsZkZm2PO5kVVtYZynYimWi+T0Sck05g/7mIeHQbx7QDpgBHANXAHEkz0nmKAYiIS3P2vwj4QtNuw8zKjec6bj5ZnwhuB+YCw9PlZcCvgAYTAXAQsCgiFgNIuhcYDSxoYP+xbD3MtZkZ4E5mhZY1EQyIiDGSxkIymb0k5TmmF/BWznI1UG9PEUl9gf7AUw1sHw+MB+jTx9nfrNy4k1lhZa0s/lBSJ9LJaCQNAD5oxjhOAR6IiM31bYyIWyKiIiIqevTo0YyXNTOzrE8EVwGPA3tI+iUwAjgjzzHLgD1ylnun6+pzCnBBxljMzKwZZR1r6AlJc4EvksxHcElE/G+ew+YAe0nqT5IATgHG1d5J0j7AroAL/MzMiiBrq6FHgLuBGRGxIcsxEbFJ0oXALJLmo9MjYr6kq4HKiJiR7noKcG9EeA5kM7MiyFo0dAPJDGXXSZoD3As8GhHvb+ugiJgJzKy1bmKt5asyR2tmZs0ua9HQM8Azad+Aw4FzgOnALgWMzczMWkBjJq/vRDIa6Rhgf+DOQgVlZmYtJ2sdwf0kHcQeB24GnomIjwsZmJmZtYysTwS3AWMbaudvZmZtV74Zyg6PiKeAzsDo2p2JPUOZmVnbl++J4BCSYR/qm6nMM5SZmZWAfDOU1QwCd3VELMndlnYUMzOzNi7rWEMP1rPugeYMxMzMiiNfHcE+wL5AV0nH52zaBehYyMDMzKxl5Ksj+BxwDNCNresJ1pF0KjMzaxXufvFNHq6qO66lZzHLL18dwcPAw5KGR4QHhTOzVqP29JU1y8P6f3LLPp7FLJt8RUP/FhE/BMbVTEqTKyIuLlhkZmaNMKz/J+t8+/csZtnkKxp6Pf1ZWehAzMwao77pK7Oorwip3IuP8hUNPZL+3DKukKRPADtHxNoCx2Zm1qCmTl/5cNUyFqxYy8DdkzEzXXyUfayhu4HzgM0kE87sIumnEfGjQgZnZrY9atcjAFuSwH3nDq+zrVxl7UcwMH0C+DrwW5KJ5v+lUEGZmRXKwN13YfTQXsUOo1XJOuhcB0kdSBLBzRHxkSTPKGZmrVpT6xHKTdYngmnAUpLB556V1BfIW0cgaZSkhZIWSZrQwD4nS1ogaX5aBGVm1ix+cfYwJ4EMss5QdhNwU86qv0k6bFvHpLOZTQGOAKqBOZJmRMSCnH32Ar4HjIiI1ZJ6NvYGzMxs+2R6IpDUVdJkSZXp68ckTwfbchCwKCIWR8SHJPMcj661zznAlIhYDRARbzcyfjMz205Zi4amkwwrcXL6WgvcnueYXsBbOcvV6bpcewN7S/qDpBckjcoYj5mZNZOslcUDIuKEnOVJkqqa6fp7AYcCvUnqHwZHxJrcnSSNB8YD9OlTvm19zcwKIesTwXuSDq5ZkDQCeC/PMcuAPXKWe6frclUDMyLio3S+g7+QJIatRMQtEVERERU9evTIGLKZmWWR9YngPOAuSV3T5dXA6XmOmQPslU5gsww4BRhXa5/fAGOB2yV1JykqWpwxJjOz7VZfpzMor2En8iYCSUOBPUk+yJcBZBleIiI2SboQmAW0A6ZHxHxJVwOVETEj3XakpAUkvZYvj4hVTb0ZM7PmUG7DTuQbfXQicBowF/ghcG1E/CzrySNiJjCz1rqJOe8D+Hb6MjNrcfV1Oiu3YSfyPRGMAYZGxEZJuwGPA5kTgZlZa+cOZ/kriz+IiI0AaZFN1splMzNrI/I9EXxW0oz0vYABOctExLEFi8zMzFpEvkRQuyfwDYUKxMzMiiPfxDTPtFQgZmZWHNss85f0iKSvpUNQ1972WUlXS/pm4cIzM7NCy1c0dA5J084bJb0DrAQ6Av2Av5LMTfBwQSM0M2th9XUyK+UOZvmKhv4O/Bvwb5L6AbuTDC3xl5rWRGZmpa7UO5hlHWKCiFhKMjmNmVlZKJd5jTMnAjOzclHT27hcOBGYmdVSbr2N3VPYzKzMZXoiSOcfuAromx4jkjHjPlu40MzMrCVkLRq6DbiUZBTSzYULx8ys9Sn1OQuyJoJ3I+K3BY3EzKwNKaUmpVkTwWxJPwJ+DXxQszIiXi5IVGZmrUipz1mQNRHU3H1FzroADm/ecMzMWp9Sb0WUKRFExGGFDsTMzIojU/NRSV0lTZZUmb5+nDOR/baOGyVpoaRFkibUs/0MSSslVaWvs5tyE2Zm1nRZ+xFMB9YBJ6evtcDt2zpAUjtgCvBVYCAwVtLAena9LyKGpq9bM0duZmbNImsdwYCIOCFneZKkqjzHHAQsiojFAJLuJZnoZkGjozQzs4LJ+kTwnqSDaxbSDmbv5TmmF/BWznJ1uq62EyTNk/SApD3qO5Gk8TXFUitXrswYspmZZZH1ieB84M60XkDAO8AZzXD9R4B7IuIDSecCd1JPS6SIuAW4BaCioiKa4bpmZtullDqZZW01VAXsJ2mXdHlthsOWAbnf8Hun63LPuypn8Vbgh1niMTNrjdpqJ7NtJgJJp0XELyR9u9Z6ACJi8jYOnwPsJak/SQI4BRhX6zy7R8SKdPFY4PXGhW9mVhyl1Mks3xNB5/Rnl8aeOCI2SboQmAW0A6ZHxHxJVwOVETEDuFjSscAmmq+4ycys4Eqpk1m+qSqnpT8nNeXkETETmFlr3cSc998DvteUc5uZWfPI2qHsh5J2kdRB0u/STmCnFTo4MzMrvKytho6MiH+TdBzJvMXHA88CvyhUYGZmbU3tlkQ1y8P6f3Kr/Vpby6Ks/QhqEsbRwK8i4t0CxWNmVtIWrFjLw1XL8u/YgrI+ETwq6c8kncjOl9QDeL9wYZmZtT21WxKdduuLWy1D62xZlLUfwQRJPySZoGazpA0kw0WYmVmqdkuittKyKF8/gsMj4ilJx+esy93l14UKzMzMWka+J4JDgKeAr9WzLXAiMDNr8/L1I7gy/Xlmy4RjZmYtLWs/gv8rqVvO8q6SflCwqMzMrMVkbT761YhYU7MQEauBowoSkZmZtaisiaCdpB1rFiR1Anbcxv5mZtZGZO1H8Evgd5Jqpqc8k2TuADMza+Oy9iO4XtIrwJfTVddExKzChWVmZi0l6xMBJHMFbIqI/5a0k6QuEbGuUIGZmVnLyNpq6BzgAWBauqoX8JsCxWRmZi0oa2XxBcAIYC1ARLwB9CxUUGZm1nKyJoIPIuLDmgVJ7Ul6FpuZWRuXNRE8I+n/AJ0kHQH8Cngk30GSRklaKGmRpAnb2O8ESSGpImM8ZmbWTLImgu8CK4FXgXNJpp+8YlsHSGoHTAG+CgwExkoaWM9+XYBLgBezh21mZs0lb6uh9AN9fkTsA/ysEec+CFgUEYvT89xLMnT1glr7XQNcD1zeiHObmVkzyftEEBGbgYWSGjuvWi/grZzl6nTdFpL2B/aIiMcaeW4zM2smWfsR7ArMl/QSsKFmZUQc29QLS/oEMBk4I8O+44HxAH36tJ55Ps3MSkHWRPD9Jpx7GbBHznLvdF2NLsAg4Ol0sptPAzMkHRsRlbkniohbgFsAKioq3FrJzKwZ5ZuhrCNwHrAnSUXxbRGxKeO55wB7SepPkgBOAcbVbIyId4HuOdd6GrisdhIwM7PCyldHcCdQQZIEvgr8OOuJ04RxITCLZHiK+yNivqSrJTW5SMnMzJpXvqKhgRExGEDSbcBLjTl5RMwkaWqau25iA/se2phzm5lZ88j3RPBRzZtGFAmZmVkbku+JYD9Ja9P3IulZvDZ9HxGxS0GjMzOzgss3eX27lgrEzMyKI+sQE2ZmVqKcCMzMypwTgZlZmXMiMDMrc04EZmZlzonAzKzMORGYmZU5JwIzszLnRGBmVuacCMzMypwTgZlZmXMiMDMrc04EZmZlzonAzKzMORGYmZW5giYCSaMkLZS0SNKEerafJ+lVSVWSfi9pYCHjMTOzuvLNUNZkktoBU4AjgGpgjqQZEbEgZ7e7I2Jquv+xwGRgVKFiMjMrtheXvAPAmGnPb1k3emgvxg3rU6yQCpcIgIOARRGxGEDSvcBoYEsiiIi1Oft3BqIpF/roo4+orq7m/fff345wzdqGjh070rt3bzp06FDsUKwZLFiRfAyWaiLoBbyVs1wNDKu9k6QLgG8DOwCH13ciSeOB8QB9+tT9ZVVXV9OlSxf69euHpO2P3KyVighWrVpFdXU1/fv3L3Y41gQH79kdgF+cnXwc5j4ZFEvRK4sjYkpEDAC+C1zRwD63RERFRFT06NGjzvb333+f3XbbzUnASp4kdtttNz/9tmG/OHvYliTQWhQyESwD9shZ7p2ua8i9wNebejEnASsX/lu35lbIRDAH2EtSf0k7AKcAM3J3kLRXzuLRwBsFjMfMzOpRsEQQEZuAC4FZwOvA/RExX9LVaQshgAslzZdURVJPcHqh4im0nXfeebvPUVlZycUXX9zg9qVLl3L33Xdn3h+gX79+DB48mCFDhnDIIYfwt7/9bbvjbC5Tp07lrrvuapZzrVixgmOOOWardf/6r/9Kr169+Pjjj7esu+OOO+jRowdDhw5l4MCB/OxnP9vuay9ZsoRhw4ax5557MmbMGD788MM6+3z00UecfvrpDB48mM9//vNce+21W7Z985vfpGfPngwaNGirYy677DKeeuqp7Y7PLK+IaFOvAw44IGpbsGBBnXUtrXPnzgW/xuzZs+Poo49u1DF9+/aNlStXRkTExIkT4+yzz97uOD7++OPYvHnzdp+nOV122WXxm9/8Zsvy5s2bo0+fPjFs2LB46qmntqy//fbb44ILLoiIiP/5n/+J7t27x9///vftuvZJJ50U99xzT0REnHvuufFf//Vfdfb55S9/GWPGjImIiA0bNkTfvn1jyZIlERHxzDPPxNy5c2Pffffd6pilS5fGEUccUe81W8PfvDWPk6f+MU6e+seCXweojAY+VwvZaqgoJj0ynwXL1+bfsREGfmYXrvzavo0+rqqqivPOO4+NGzcyYMAApk+fzq677sqcOXM466yz+MQnPsERRxzBb3/7W1577TWefvppbrjhBh599FGeeeYZLrnkEiApE3722WeZMGECr7/+OkOHDuX000/nC1/4wpb9169fz0UXXURlZSWSuPLKKznhhBO2imf48OHcdNNNAKxcuZLzzjuPN998E4Abb7yRESNGsHLlSsaNG8fy5csZPnw4Tz75JHPnzmX9+vV85StfYdiwYcydO5eZM2dy//33c//99/PBBx9w3HHHMWnSJDZs2MDJJ59MdXU1mzdv5vvf/z5jxoxhwoQJzJgxg/bt23PkkUdyww03cNVVV7Hzzjtz2WWXNfi7OvTQQxk2bBizZ89mzZo13HbbbYwcObLO7/rBBx/kBz/4wZblp59+mn333ZcxY8Zwzz33cNhhh9U5pmfPngwYMIC//e1vfOpTn2r0vy8kX6SeeuqpLU9qp59+OldddRXnn3/+VvtJYsOGDWzatIn33nuPHXbYgV122QWAf/7nf2bp0qV1zt23b19WrVrF3//+dz796U83KT6zLIreaqiUfeMb3+D6669n3rx5DB48mEmTJgFw5plnMm3aNKqqqmjXrl29x95www1MmTKFqqoqnnvuOTp16sR1113HyJEjqaqq4tJLL91q/2uuuYauXbvy6quvMm/ePA4/vG5L3Mcff5yvf/3rAFxyySVceumlzJkzhwcffJCzzz4bgEmTJnH44Yczf/58TjzxxC2JAuCNN97gW9/6FvPnz2fhwoW88cYbvPTSS1RVVTF37lyeffZZHn/8cT7zmc/wyiuv8NprrzFq1ChWrVrFQw89xPz585k3bx5XXFG3cVhDvyuATZs28dJLL3HjjTdutb7GkiVL2HXXXdlxxx23rLvnnnsYO3Ysxx13HI899hgfffRRneMWL17M4sWL2XPPPbdav3DhQoYOHVrva82aNVvtu2rVKrp160b79sl3qt69e7NsWd02ESeeeCKdO3dm9913p0+fPlx22WV88pOfrLNfbfvvvz9/+MMf8u5ntj1K7omgKd/cC+Hdd99lzZo1HHLIIUDyTfGkk05izZo1rFu3juHDhwMwbtw4Hn300TrHjxgxgm9/+9uceuqpHH/88fTu3Xub1/vv//5v7r333i3Lu+6665b3hx12GO+88w4777wz11xzzZb9Fyz4RyfvtWvXsn79en7/+9/z0EMPATBq1KitztO3b1+++MUvAvDEE0/wxBNP8IUvfAGA9evX88YbbzBy5Ei+853v8N3vfpdjjjmGkSNHsmnTJjp27MhZZ53FMcccU6csv6HfVY3jjz8egAMOOKDeb84rVqwgt1nxhx9+yMyZM5k8eTJdunRh2LBhzJo1a8t177vvPn7/+9+z4447Mm3atDofyJ/73Oeoqqra1q+70V566SXatWvH8uXLWb16NSNHjuTLX/4yn/3sZ7d5XM+ePVm+fHmzxmJWW8klglIxYcIEjj76aGbOnMmIESOYNWtWk881e/ZsunXrxqmnnsqVV17J5MmT+fjjj3nhhRfo2LFj5vN07tx5y/uI4Hvf+x7nnntunf1efvllZs6cyRVXXMGXvvQlJk6cyEsvvcTvfvc7HnjgAW6++eZGVYLWfNNv164dmzZtqrO9U6dOW7WrnzVrFmvWrGHw4MEAbNy4kU6dOm1JBGPGjOHmm29u8HoLFy5kzJgx9W57+umn6dat25bl3XbbjTVr1rBp0ybat29PdXU1vXr1qnPc3XffzahRo+jQoQM9e/ZkxIgRVFZW5k0E77//Pp06ddrmPmbby0VDBdK1a1d23XVXnnvuOQB+/vOfc8ghh9CtWze6dOnCiy++CLDVt/hcf/3rXxk8eDDf/e53OfDAA/nzn/9Mly5dWLduXb37H3HEEUyZMmXL8urVq7fa3r59e2688Ubuuusu3nnnHY488kj+8z//c8v2mm/AI0aM4P777weSb/21z1PjK1/5CtOnT2f9+vUALFu2jLfffpvly5ez0047cdppp3H55Zfz8ssvs379et59912OOuoofvKTn/DKK69k+l1ltffee2/1pHDPPfdw6623snTpUpYuXcqSJUt48skn2bhxY6bz1TwR1PfKTQKQlP0fdthhPPDAAwDceeedjB49us45+/TpsyX5bdiwgRdeeIF99tknbyx/+ctf6rQmstKzYMVaxkx7fsur34TH6Dfhsa3WjZn2PJMemV+Q6zsRNJONGzfSu3fvLa/Jkydz5513cvnllzNkyBCqqqqYOHEiALfddhvnnHMOQ4cOZcOGDXTt2rXO+W688UYGDRrEkCFD6NChA1/96lcZMmQI7dq1Y7/99uMnP/nJVvtfccUVrF69mkGDBrHffvsxe/bsOufcfffdGTt2LFOmTOGmm26isrKSIUOGMHDgQKZOnQrAlVdeyRNPPMGgQYP41a9+xac//Wm6dOlS51xHHnkk48aNY/jw4QwePJgTTzyRdevW8eqrr3LQQQcxdOhQJk2axBVXXMG6des45phjGDJkCAcffDCTJ0+uc76GfldZdO7cmQEDBrBo0SI2btzI448/ztFHH73V9oMPPphHHnkk8zkb4/rrr2fy5MnsueeerFq1irPOOguAGTNmbLmPCy64gPXr17Pvvvty4IEHcuaZZzJkyBAAxo4dy/Dhw1m4cCG9e/fmtttuA5Imp4sWLaKioqIgcVvrMHpoLwbuvktRY1DSqqjtqKioiMrKyq3Wvf7663z+858vUkSNt379+i39Dq677jpWrFjBT3/60yJHlfjggw9o164d7du35/nnn+f8889v9vLyQnjooYeYO3fuVi2H2rqHHnqIl19+eUu9Tq629jdvjXParUmJQXMORSFpbkTU+63CdQRF8Nhjj3HttdeyadMm+vbtyx133FHskLZ48803Ofnkk/n444/ZYYcdmqXDVUs47rjjWLVqVbHDaFabNm3iO9/5TrHDsCJo6bGI/ERg1gb5b94aa1tPBCVTR9DWEppZU/lv3ZpbSSSCjh07smrVKv8HsZIX6XwEjWn2a5ZPSdQR9O7dm+rqalauXFnsUMwKrmaGMrPmUhKJoEOHDp6tycysiUqiaMjMzJrOicDMrMw5EZiZlbk2149A0kqgqdNsdQf+txnDaQt8z+XB91wetuee+0ZEj/o2tLlEsD0kVTbUoaJU+Z7Lg++5PBTqnl00ZGZW5pwIzMzKXLklgluKHUAR+J7Lg++5PBTknsuqjsDMzOoqtycCMzOrxYnAzKzMlWQikDRK0kJJiyRNqGf7jpLuS7e/KKlfEcJsVhnu+duSFkiaJ+l3kvoWI87mlO+ec/Y7QVJIavNNDbPcs6ST03/r+ZLubukYm1uGv+0+kmZL+lP6931UMeJsLpKmS3pb0msNbJekm9LfxzxJ+2/3RSOipF5AO+CvwGeBHYBXgIG19vkWMDV9fwpwX7HjboF7PgzYKX1/fjncc7pfF+BZ4AWgothxt8C/817An4Bd0+WexY67Be75FuD89P1AYGmx497Oe/5nYH/gtQa2HwX8FhDwReDF7b1mKT4RHAQsiojFEfEhcC8wutY+o4E70/cPAF+SpBaMsbnlveeImB0RG9PFF4C2Po5xln9ngGuA64H3WzK4Aslyz+cAUyJiNUBEvN3CMTa3LPccQM3s712B5S0YX7OLiGeBd7axy2jgrki8AHSTtPv2XLMUE0Ev4K2c5ep0Xb37RMQm4F1gtxaJrjCy3HOus0i+UbRlee85fWTeIyIea8nACijLv/PewN6S/iDpBUmjWiy6wshyz1cBp0mqBmYCF7VMaEXT2P/veZXEfASWnaTTgArgkGLHUkiSPgFMBs4ocigtrT1J8dChJE99z0oaHBFrihlUgY0F7oiIH0saDvxc0qCI+LjYgbUVpfhEsAzYI2e5d7qu3n0ktSd5nFzVItEVRpZ7RtKXgX8Hjo2ID1ootkLJd89dgEHA05KWkpSlzmjjFcZZ/p2rgRkR8VFELAH+QpIY2qos93wWcD9ARDwPdCQZnK1UZfr/3hilmAjmAHtJ6i9pB5LK4Bm19pkBnJ6+PxF4KtJamDYq7z1L+gIwjSQJtPVyY8hzzxHxbkR0j4h+EdGPpF7k2IioLE64zSLL3/ZvSJ4GkNSdpKhocQvG2Nyy3PObwJcAJH2eJBGU8ry1M4BvpK2Hvgi8GxErtueEJVc0FBGbJF0IzCJpcTA9IuZLuhqojIgZwG0kj4+LSCplTilexNsv4z3/CNgZ+FVaL/5mRBxbtKC3U8Z7LikZ73kWcKSkBcBm4PKIaLNPuxnv+TvAzyRdSlJxfEZb/mIn6R6SZN49rfe4EugAEBFTSepBjgIWARuBM7f7mm3492VmZs2gFIuGzMysEZwIzMzKnBOBmVmZcyIwMytzTgRmZmXOicAKTtJmSVWSXpP0iKRuzXz+pWmbeSStb2CfTpKekdROUj9J76UxLZA0Ne2J3JhrVki6KX1/qKR/ytl2nqRvbM89pee5StJlefa5Q9KJjThnv4ZGtay1339Ieqv271PShZK+mfV61jY4EVhLeC8ihkbEIJJ+GxcUIYZvAr+OiM3p8l8jYigwhGTEyq835mQRURkRF6eLhwL/lLNtakTctb0BF9kjJAO+1Tad0h/Lp+w4EVhLe550gCxJAyQ9LmmupOck7ZOu/5SkhyS9kr7+KV3/m3Tf+ZLGN/K6pwIP116ZDjr4R2DP9NvyU/rHnA190uuelD7NvCLp2XTdoZIeVTKXxXnApekTxsiab/KS9pH0Us210vO/mr4/IH1CmStplvKMHinpHElz0hgelLRTzuYvS6qU9BdJx6T7t5P0o/SYeZLObcwvKyJeqK+3ajqC7VJJ9SUJa6OcCKzFSGpHMhRATa/fW4CLIuIA4DLgv9L1NwHPRMR+JOOyz0/XfzPdtwK4WFKmEWPToQk+GxFL69m2UxrTq8B/AndGxBDgl2kcABOBr6TxbNUbOz3nVOAn6VPPcznb/gzsIKl/umoMcJ+kDum1TkzvZzrwH3lu49cRcWAaw+sk4+vU6Efy7f1oYKqkjun2dyPiQOBA4JycOGru/TOSZua5bn0qgZFNOM5aqZIbYsJapU6SqkieBF4HnpS0M0lxSs2QFwA7pj8PB74BkBblvJuuv1jScen7PUgGU8syfEJ3YE2tdQPSmAJ4OCJ+K+nnwPHp9p8DP0zf/wG4Q9L9wK8zXC/X/SQJ4Lr05xjgcyQD4j2Z3ns7IN9YMYMk/QDoRjJUyKzca6Qjbb4haTGwD3AkMCSn/qArye/rLzUHRcRykqEKGuvt9BpWIpwIrCW8FxFD02/fs0jqCO4A1qTl9HlJOhT4MjA8IjZKeppkcLFM169n379mvXZEnCdpGMk37rmSDsh4XYD7SJLdr5NTxRuSBgPzI2J4I85zB/D1iHhF0hmkA8vVhFg7ZJLZqy6KiNyEgZpnWtaOJL9TKxEuGrIWk5YvX0wySNhGYImkk2DLPKz7pbv+jmQ6zZqy7q4k32hXp0lgH5JhpbNedzXQLi0y2ZY/8o8BCE8FnktjGBARL0bERJJRLfeoddw6kmGv67v2X0kGf/s+SVIAWAj0UDJ2PpI6SNo3T2xdgBVpsdKptbadJOkTkgaQTOm4kCThnp/uj6S9JXXOc42s9gbytjyytsOJwFpURPwJmEcymcipwFmSXiGpB6iZgvAS4LC0YnUuSauex4H2kl4nKWZ5oZGXfgI4OM8+FwFnSpoH/EsaB8CPJL2aNrv8I8m8ubkeAY6rqSyu57z3AafxjzHzPyQZ/vz69N6ryGl11IDvAy+SFFP9uda2N4GXSGadOy8i3gduBRYAL6dxT6NWCcC26ggk/VDJyJc7SaqWdFXO5hHAk3nitTbEo49aWVAybeWlEfEvxY6lLVMyr8W3/XssLX4isLIQES8Ds9OWS9Z03UmeTqyE+InAzKzM+YnAzKzMORGYmZU5JwIzszLnRGBmVuacCMzMytz/B64PGVsnHRwqAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "PrecisionRecallDisplay.from_predictions(\n",
    "    y_true=test[\"label_num\"],\n",
    "    y_pred=score,\n",
    "    name=\"LogisticRegression\",\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f527f53b",
   "metadata": {},
   "source": [
    "## 不均衡データに対応する\n",
    "\n",
    "`class_weight` パラメータで不均衡データに対応できます。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "2531e8d1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>#sk-container-id-2 {color: black;background-color: white;}#sk-container-id-2 pre{padding: 0;}#sk-container-id-2 div.sk-toggleable {background-color: white;}#sk-container-id-2 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-2 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-2 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-2 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-2 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-2 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-2 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-2 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-2 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-2 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-2 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-2 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-2 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-2 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-2 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-2 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-2 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-2 div.sk-item {position: relative;z-index: 1;}#sk-container-id-2 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-2 div.sk-item::before, #sk-container-id-2 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-2 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-2 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-2 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-2 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-2 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-2 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-2 div.sk-label-container {text-align: center;}#sk-container-id-2 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-2 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-2\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>Pipeline(steps=[(&#x27;vect&#x27;,\n",
       "                 TfidfVectorizer(tokenizer=&lt;method &#x27;split&#x27; of &#x27;str&#x27; objects&gt;)),\n",
       "                (&#x27;clf&#x27;, LogisticRegression(class_weight=&#x27;balanced&#x27;))])</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-4\" type=\"checkbox\" ><label for=\"sk-estimator-id-4\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">Pipeline</label><div class=\"sk-toggleable__content\"><pre>Pipeline(steps=[(&#x27;vect&#x27;,\n",
       "                 TfidfVectorizer(tokenizer=&lt;method &#x27;split&#x27; of &#x27;str&#x27; objects&gt;)),\n",
       "                (&#x27;clf&#x27;, LogisticRegression(class_weight=&#x27;balanced&#x27;))])</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-5\" type=\"checkbox\" ><label for=\"sk-estimator-id-5\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">TfidfVectorizer</label><div class=\"sk-toggleable__content\"><pre>TfidfVectorizer(tokenizer=&lt;method &#x27;split&#x27; of &#x27;str&#x27; objects&gt;)</pre></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-6\" type=\"checkbox\" ><label for=\"sk-estimator-id-6\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">LogisticRegression</label><div class=\"sk-toggleable__content\"><pre>LogisticRegression(class_weight=&#x27;balanced&#x27;)</pre></div></div></div></div></div></div></div>"
      ],
      "text/plain": [
       "Pipeline(steps=[('vect',\n",
       "                 TfidfVectorizer(tokenizer=<method 'split' of 'str' objects>)),\n",
       "                ('clf', LogisticRegression(class_weight='balanced'))])"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pipe_weight = Pipeline([\n",
    "    (\"vect\", TfidfVectorizer(tokenizer=str.split)),\n",
    "    (\"clf\", LogisticRegression(class_weight=\"balanced\"))\n",
    "])\n",
    "\n",
    "pipe_weight.fit(train[\"tokens\"], train[\"label_num\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "ea8517c4",
   "metadata": {},
   "outputs": [],
   "source": [
    "score_weight = pipe_weight.predict_proba(test[\"tokens\"])[:,1]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "196f6141",
   "metadata": {},
   "source": [
    "class_weightオプションを付けないモデルと比較します。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "5938278d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAz2UlEQVR4nO3deXxU5fX48c8xrEIEZFHKFkBEloQgQaSAgIqiWClIRdCiuKBWwa9bTb+lCNpWrIjoV/oTVBTbCghKRUVwA0QFIdEIBqUsogSwYpAlIpDA+f1xb8bJZJK5SWYymcl5v17zytxl7pybwJy593me84iqYowxpvo6KdoBGGOMiS5LBMYYU81ZIjDGmGrOEoExxlRzlgiMMaaaqxHtAMqqSZMmmpSUFO0wjDEmpmRmZn6vqk2DbYu5RJCUlERGRka0wzDGmJgiIl+XtM1uDRljTDVnicAYY6o5SwTGGFPNWSIwxphqzhKBMcZUcxFLBCIyR0S+E5HPS9guIvKEiGwVkQ0icnakYjHGGFOySF4RPA8MLmX7JUAH9zEO+H8RjMUYY0wJIjaOQFXfF5GkUnYZCrygTh3stSLSUESaq+qeSMSz9u83kbj/i0gcOiqa1K/NaYl1St8peQSkja2cgIwxMSuabQQtgJ1+yznuumJEZJyIZIhIxt69eysluKrs8LHjfJ93tPSdvt0IGxdVTkDGmJgWEyOLVXU2MBsgLS2tXDPpnPu7p8MaUzSNnLUGgAVje5e803NDKikaY0ysi+YVwS6gld9yS3edMcaYShTNRLAEGOP2HjoXOBCp9gFjjDEli9itIRGZBwwAmohIDnA/UBNAVZ8ClgKXAluBw4C1ahpjTBREstfQqBDbFbgtUu9vjDHGGxtZbIwx1ZwlAmOMqeYsERhjTDVnicAYY6o5SwTGGFPNWSIwxphqLiZKTJgYkPFc8NpGVvjOmCrPEoEJ6eOFj1J/y+Ii64pVP/36A+dnm74/r/t2o/PTEoExVZolAlNEsA/9XsecD/TsWsnAz9VPiySCNn2Lf/u3wnfGxARLBNWYlw/9wud5HYbR6zd3Ax6rnxpjYoYlgmok8IPfy4e+MSb+WSKIV+49++y//nzPPvCD3z70jTFgiSBuvPjxN7ya9fN0DguC7GMf/MaYYCwRxIlXs3axac9BOjc/BYANtbrTNLE2XcYvj3JkYWBdU42JKEsEMejjr/YBPzfaAr4ksODmwgbclZUfWDlY11Rjos8SQZzo3PwUhqa2iHYYpbKuqcZUTZYIYlDfM5oA8M8be0U5khACGqyta6oxVZMlghhUFRNAYGM1FG+wtsZqY6qmiCYCERkMPA4kAM+o6tSA7W2AOUBTYB9wjarmRDImExmBjdUQZw3WxsSxSE5enwDMBAYBOcB6EVmiqpv8dpsGvKCqc0XkfOAh4LeRismEh7fGaoh6g3Ww3kbW08iYYiJ5RXAOsFVVtwOIyHxgKOCfCDoDd7nPVwD/jmA8JoKi3lgd7EM/sLeR9TQyJqhIJoIWwE6/5Rwg8Ob2Z8BwnNtHw4BEEWmsqrn+O4nIOGAcQOvWrSMWsPGmSjRWF37IF/YeCtbFNLC3kfU0MiaoaDcW3wM8KSLXAe8Du4DjgTup6mxgNkBaWppWZoCmuKrYWB20i6kxxpNIJoJdQCu/5ZbuOh9V3Y1zRYCI1AeuUNX9EYzJxIt2A5yfY16NahjGxINIJoL1QAcRaYuTAK4CRvvvICJNgH2qegL4A04PImNCC1MCCDbILZhK7fZqJTVMJYtYIlDVAhG5HViO0310jqpmi8gDQIaqLgEGAA+JiOLcGrotUvEY46UiazCtjm1j55bFQAQSgZdGbrCGbhNREW0jUNWlwNKAdZP8ni8Cgnz1MabiwlWR1T9xlEVJVxtFail5aeQGa+g2ERXtxmJjIqYyK7J6ne2tWC0la+Q2VYCnRCAizYA+wC+An4DPcW7vnIhgbMZ4FqzERSQrspZ3treYraVUUrtFIEtqManURCAiA4F04FTgU+A7oA7wa6C9iCwCHlXVgxGO05giAj/4C0c792p7qm9duAa5dSn8kK+k2d6CJbVJuQecW0oVProHXtstAlk7RswKdUVwKXCTqn4TuEFEagCX4ZSQeDkCsRkDBP9gDPzg79X2VIamtmB0r8oZcBiuD34v5wZ+t5Qq9G54+2bvtd0ikLVjxKxSE4Gq3lvKtgKsJIQJs2B1jIJ9MFbmB//G2t0BSP7DygofK/D8vJ5b9l8Tyv5m5f1mb+0W1U65G4tFZKyqPhfOYIwJprK/7QcKRwIoSVjPLfCDv7zf7E21U5FeQ1MASwQmIopWMY0f4arTFKyXUmFbhu+Dv5wf+sFuVwFRTcYmskI1Fm8oaRNUTruVqV4KPyjjVXkTQGCDdbBeSuvpzKcNLmTc2AfLdGwvDe+b9jj9QSwRxKdQVwSnARcDPwSsF+CjiERkqrUqWdCuCiqta+q4Ul5X3oZ3/zYbE39CJYLXgfqqmhW4QURWRiIgY0xx5Wmw9tojKdptMCb6QvUauqGUbaNL2maMCS8vCaC8PZK8CNaby5JH/LASE8bEqUh+07c2g/hiicCYOBHJmeMCj21tBvHFEoExcSKSDe2Bxw52q6hSy2CYsDop2gEYY+JDYRkME3s8XxGIyGxVHVfSsjGm+gh2G6pcZTBMlVCWW0OzQiwbY6qJYLehDh0pAIq3H1jvoqrP860hVc0sbTkYERksIptFZKuIpAfZ3lpEVojIpyKyQUQu9RqPMabq27TnYNByFaZqCVVi4jVAS9quqpeX8toEYCZOmeocYL2ILFHVTX67TQReUtX/JyKdcaa1TPIevjGmqmhQpyZQtE6U9S6KDaFuDU2rwLHPAbaq6nYAEZkPDAX8E4ECp7jPGwC7K/B+xpgo6nTMLU3mNy/BpNwDfFh3IBCfRQTjRaiRxasKn4tIXaC1qm72eOwWwE6/5Rwg8MbiZOAtERkP1AMuDHYgERmHW0KldWu712hMrEjK3x7tEIwHntoIRORXQBawzF1OFZElYXj/UcDzqtoSZza0f4hIsZhUdbaqpqlqWtOmTcPwtsaYsGs3wHmMfcP32FGzXbSjMh547TU0GedWz0oAVc0SkbYhXrMLaOW33NJd5+8GYLB7zDUiUgdogjM3sjEmlox5tdiqYD2JrBdR1eO111C+qh4IWFdiI7JrPdBBRNqKSC3gKiDwKuIb4AIAEekE1AH2eozJGBNjrBdR1eT1iiBbREYDCSLSAZhAiPkIVLVARG4HlgMJwBxVzRaRB4AMVV0C3A08LSJ34iSW61Q1VIIxxsSIwJ5ESelv8PFX+2ysQRXjNRGMB/4IHAXm4Xy4h5wGSVWX4nQJ9V83ye/5JqCP12CNMbGlU/NTQu5jlUyjz1MiUNXDwB9F5GFnUQ9FNixjTDwKVprCxhpEn6dEICI9gTlAort8ALjey+hiY4wpFKw0RbtvFjI04SOy//rzx1HgNJwmsrzeGnoW+J2qrgYQkb7Ac0BKpAIzxsShjOdg46Iiqx6q+QEA2SQD0OrYNnZuWYzThGgqg9dEcLwwCQCo6gciUhChmIwx8eTbjT+PNv7a+dCnTV/f5i9qpfDRyQO54X8ecFZMbkCXYxuLjFAGIHkEpI2thICrn1C1hs52n64SkVk4DcUKjMQdU2CMMSVKHlF0uU3fYh/ondxHqb7d6Py0RBARoa4IHg1Yvt/vuXXzNMaULm1smT+8N9TqDkDK2Dd+Xhl4dWDCKlStoYGVFYgxxgD8pfFDACyIchzVSVlmKBsCdMEZ/QuAqj4QiaCMMcZUHq9F557CaRcYDwjwG6BNBOMyxhhTSbxeEfxSVVNEZIOqThGRR4E3IxmYMaZ6+virfUDRgWaTcg/QpH5tTotWUHHOa9G5n9yfh0XkF0A+0DwyIRljTFGHjx3n+7yj0Q4jbnm9InhdRBoCjwCf4PQYeiZSQRljqq9gZSiy/5oQrXCqBa+1hgoLzL0sIq8DdYKUpTbGmAoLVobCRFaoAWXDS9mGqr4S/pCMMaaoLsfcAWU22jgiQl0R/KqUbQpYIjDGRIeNNg6bUAPK7DdsjIk6G20cWZ4HlBljTLTYaOPIskRgjIlNhZVM/a8MrM2gXCKaCERkMPA4zpzFz6jq1IDtjwGF9YxOBpqpasNIxmSMiT3BBpkVuzqwNoNy8zpD2ck4s0S0VtWb3AnsO6rq66W8JgGYCQwCcoD1IrLEnacYAFW902//8UD38p2GMaa6WUMy9WrV+LndwNoMys3rFcFzQCbQ213eBSwESkwEwDnAVlXdDiAi84GhwKYS9h9F0TLXxhgDlDTX8SOAtRuEg9dE0F5VR4rIKHAmsxcRCfGaFsBOv+UcIOhIERFpA7QF3ith+zhgHEDr1q09hmyMiRc2yCyyvCaCYyJSF3cyGhFpD4Sz8MdVwCJVPR5so6rOBmYDpKWl2YQ4xhhvgsyRDFijcgCviWAysAxoJSL/AvoA14V4zS6gld9yS3ddMFcBt3mMxRhjigvWiyjIHMnWqFyc11pDb4lIJnAuznwEd6jq9yFeth7oICJtcRLAVcDowJ1E5CygEbAmcJsxxlRIkDmSrVG5OK+9hl4DXgSWqOqPXl6jqgUicjuwHKf76BxVzRaRB4AMVV3i7noVMF9V7ZaPMab82g1wfo55NaphxCKvt4am4cxQNlVE1gPzgddV9UhpL1LVpcDSgHWTApYne47WGGNKYgmg3LzeGloFrHLHBpwP3ATMAU6JYGzGGGMqQVkmr6+LU410JHA2MDdSQRljTMRYaYpivLYRvIQzQGwZ8CSwSlVPRDIwY4ypFNaLyPMVwbPAqJL6+RtjTMwIbFS2XkQhZyg7X1XfA+oBQwMHE9sMZcaYmGONysWEuiLoj1P2IdhMZTZDmTHGxIFQM5QVFoF7QFW/8t/mDhQzxpjYFqzxGKpVA/JJHvd7Oci6IAU8jDEmDny7MXiNojgVqo3gLKAL0EBEhvttOgWoE8nAjDGmUgQbkVzNGpBDtRF0BC4DGlK0neAQzqAyY4ypEl78+BtezSpe13JoagtG9yqlfL01HodsI3gVeFVEequqFYUzxlQZgdNXFi73anuqb59New4ClJ4ITMhbQ79X1b8BowsnpfGnqhMiFpkxxpRBr7anFvv27z/HsSlZqFtDX7g/MyIdiDHGlEWw6Su9CHYLKeTtozgX6tbQa+5PX10hETkJqK+qByMcmzHGlKi801e+mrWLTXsO0rm5UzPTbh95rzX0InALcBxnwplTRORxVX0kksEZY0xFBLYjAL4ksODm3sW2+VSzwnReaw11VtWDInI18CaQDmQClgiMMTGlc/NTGJraomwv+voD5xE4tiBOkoPXRFBTRGoCvwaeVNV8EbEZxYwxVVp52xGKjS2Y3KD4PnFUtdRrIpgF7AA+A94XkTZAyDYCERkMPI4zVeUzqjo1yD5XApNxahd9pqrF5jU2xpjyKG87QrGxBXE+6MzrDGVPAE/4rfpaRAaW9hp3NrOZwCAgB1gvIktUdZPfPh2APwB9VPUHEWlW1hMwxpiIi/NBZ55qDYlIAxGZLiIZ7uNRnNLUpTkH2Kqq21X1GM48x0MD9rkJmKmqPwCo6ndljN8YY0wFeS06NwenrMSV7uMg8FyI17QAdvot57jr/J0JnCkiH4rIWvdWkjHGmErktY2gvape4bc8RUSywvT+HYABQEuc9odkVd3vv5OIjAPGAbRuXX37+hpjTCR4vSL4SUT6Fi6ISB/gpxCv2QW08ltu6a7zlwMsUdV8d76D/+AkhiJUdbaqpqlqWtOmTT2GbIwxxguvVwS3AC+ISGEfqh+Aa0O8Zj3QwZ3AZhdwFRDYI+jfwCjgORFpgnOraLvHmIwxpsKCDTqD6lV2ImQiEJFU4AycD/JdAF7KS6hqgYjcDizH6T46R1WzReQBIENVl7jbLhKRTTijlu9V1dzynowxxoRDdSs7Ear66CTgGpxRxH8DHlLVp70eXFWXAksD1k3ye67AXe7DGGMqXbBBZ56qlgYrQ1G4rk3fn9fFwOjjUFcEI4FUVT0sIo2BZYDnRGCMMVVduQedeREjo49DJYKjqnoYQFVz3cqjxhhjgo02fmFo0XUxMvo4VCJoJyJL3OcCtPdbRlUvj1hkxhhTlQUbbRyjI5BDJYLAkcDTIhWIMcaY6Ag1Mc2qygrEGGNMdJR6z19EXhORX7klqAO3tRORB0Tk+siFZ4wxJtJC3Rq6Cadr5wwR2QfsBeoAScA2nLkJYvOmmDHGlCDYILN4HmAW6tbQt8Dvgd+LSBLQHKe0xH8KexMZY0y8i/cBZl5LTKCqO3AmpzHGmGqh1HmN44jnRGCMMdVF4Wjj6sISgTHGBIjoaOMqyEYKG2NMNefpisCdf2Ay0MZ9jeDUjGsXudCMMcZUBq+3hp4F7sSpQno8cuEYY0zVE+9zFnhNBAdU9c2IRmKMMTEknrqUek0EK0TkEeAV4GjhSlX9JCJRGWNMFVLuOQtihNdEUHj2aX7rFDg/vOEYY0zVE++9iDwlAlUdGOlAjDHGRIen7qMi0kBEpotIhvt41G8i+9JeN1hENovIVhFJD7L9OhHZKyJZ7uPG8pyEMcaY8vM6jmAOcAi40n0cBJ4r7QUikgDMBC4BOgOjRKRzkF0XqGqq+3jGc+TGGGPCwmsbQXtVvcJveYqIZIV4zTnAVlXdDiAi83EmutlU5iiNMcZEjNcrgp9EpG/hgjvA7KcQr2kB7PRbznHXBbpCRDaIyCIRaRXsQCIyrvC21N69ez2GbIwxxguvVwS3AnPddgEB9gHXheH9XwPmqepREbkZmEuQnkiqOhuYDZCWlqZheF9jjKkQT4PMvv7A+Rk4iX3yCEgbG+kQPfPaaygL6CYip7jLBz28bBfg/w2/pbvO/7i5fovPAH/zEo8xxlRFngaZfbvR+RkriUBErlHVf4rIXQHrAVDV6aW8fD3QQUTa4iSAq4DRAcdprqp73MXLgS/KFr4xxkSHp0Fm7QY4P8f4TeQYeHVQBYS6Iqjn/kws64FVtUBEbgeWAwnAHFXNFpEHgAxVXQJMEJHLgQLCd7vJGGMiztMgszGxMZNvqKkqZ7k/p5Tn4Kq6FFgasG6S3/M/AH8oz7GNMcaEh9cBZX8TkVNEpKaIvOsOArsm0sEZY4yJPK+9hi5S1d+LyDCceYuHA+8D/4xUYMYYE2sCexIVLvdqe6pvn0m5B2hSvzanVX54JfI6jqAwYQwBFqrqgQjFY4wxce3wseN8n3c09I6VyOsVwesi8iXOILJbRaQpcCRyYRljTOwJ7El0zTMfF1kGyP5rQuUHFoLXcQTpIvI3nAlqjovIjzjlIowxxrgCexLFSvnqUOMIzlfV90RkuN86/11eiVRgxhhjKkeoK4L+wHvAr4JsUywRGGNMzAs1juB+92fVGQttjDEmrLyOI/iriDT0W24kIn+OWFTGGGMqjdfuo5eo6v7CBVX9Abg0IhEZY4ypVF4TQYKI1C5cEJG6QO1S9jfGGBMjvI4j+BfwrogUTk85FmfuAGOMMTHO6ziCh0XkM+BCd9WDqro8cmEZY4ypLF6vCMCZK6BAVd8RkZNFJFFVD0UqMGOMMZXDa6+hm4BFwCx3VQvg3xGKyRhjTCXy2lh8G9AHOAigqluAZpEKyhhjTOXxmgiOquqxwgURqYEzstgYY0yM85oIVonI/wJ1RWQQsBB4LdSLRGSwiGwWka0ikl7KfleIiIpImsd4jDHGhInXRHAfsBfYCNyMM/3kxNJeICIJwEzgEqAzMEpEOgfZLxG4A/jYe9jGGGPCJWSvIfcDPVtVzwKeLsOxzwG2qup29zjzcUpXbwrY70HgYeDeMhzbGGNMmIS8IlDV48BmEWldxmO3AHb6Lee463xE5Gyglaq+UcZjG2OMCROv4wgaAdkisg74sXClql5e3jcWkZOA6cB1HvYdB4wDaN26rPnIGGNMabwmgj+V49i7gFZ+yy3ddYUSga7ASneym9OBJSJyuapm+B9IVWcDswHS0tKst5IxxoRRqBnK6gC3AGfgNBQ/q6oFHo+9HuggIm1xEsBVwOjCjap6AGji914rgXsCk4AxxpjICtVGMBdIw0kClwCPej2wmzBuB5bjlKd4SVWzReQBESn3LSVjjDHhFerWUGdVTQYQkWeBdWU5uKouxelq6r9uUgn7DijLsY0xxoRHqCuC/MInZbglZIwxJoaEuiLoJiIH3eeCM7L4oPtcVfWUiEZnjDEm4kJNXp9QWYEYY4yJDq8lJowxxsQpSwTGGFPNlWWGsiorPz+fnJwcjhw5Eu1QjDGuOnXq0LJlS2rWrBntUEwIcZEIcnJySExMJCkpCXeUsjEmilSV3NxccnJyaNu2bbTDMSHERSI4cuSIJQFjqhARoXHjxuzduzfaoVQ5XY5tdJ48N+TnlckjIG1sdAIijtoILAkYU7XY/0mPvt0IGxdFNYS4uCIwxphYsaFWdwBSxrrV9/2vDKIkbq4Ioq1+/foVPkZGRgYTJkwocfuOHTt48cUXPe8PkJSURHJyMikpKfTv35+vv/66wnGGy1NPPcULL7wQlmPt2bOHyy67rMi6//mf/6FFixacOHHCt+7555+nadOmpKam0rlzZ55+uixzLQX31Vdf0atXL8444wxGjhzJsWPHiu2Tn5/PtddeS3JyMp06deKhhx7ybbv++utp1qwZXbt2LfKae+65h/fee6/C8Zmq5S+NH+IvjR8KvWMlskRQhaSlpfHEE0+UuD0wEYTav9CKFSvYsGEDAwYM4M9//nOF41TVIh+u5XXLLbcwZsyYCh8HYPr06dx0002+5RMnTrB48WJatWrFqlWriuw7cuRIsrKyWLlyJf/7v//Lf//73wq993333cedd97J1q1badSoEc8++2yxfRYuXMjRo0fZuHEjmZmZzJo1ix07dgBw3XXXsWzZsmKvGT9+PFOnTq1QbMZ4EXe3hqa8ls2m3QdD71gGnX9xCvf/qkuZX5eVlcUtt9zC4cOHad++PXPmzKFRo0asX7+eG264gZNOOolBgwbx5ptv8vnnn7Ny5UqmTZvG66+/zqpVq7jjjjsA517r+++/T3p6Ol988QWpqalce+21dO/e3bd/Xl4e48ePJyMjAxHh/vvv54orrigST+/evX2JY+/evdxyyy188803AMyYMYM+ffqwd+9eRo8eze7du+nduzdvv/02mZmZ5OXlcfHFF9OrVy8yMzNZunQpL730Ei+99BJHjx5l2LBhTJkyhR9//JErr7ySnJwcjh8/zp/+9CdGjhxJeno6S5YsoUaNGlx00UVMmzaNyZMnU79+fe65554Sf1cDBgygV69erFixgv379/Pss8/Sr1+/Yr/rl19+uUiSW7lyJV26dGHkyJHMmzePgQMHFntNs2bNaN++PV9//TWnnXZamf++4CTF9957z5egr732WiZPnsytt95aZD8R4ccff6SgoICffvqJWrVqccopToWW8847z5cU/LVp04bc3Fy+/fZbTj/99HLFZ4wXdkUQQWPGjOHhhx9mw4YNJCcnM2XKFADGjh3LrFmzyMrKIiEheBWPadOmMXPmTLKysli9ejV169Zl6tSp9OvXj6ysLO68884i+z/44IM0aNCAjRs3smHDBs4///xix1y2bBm//vWvAbjjjju48847Wb9+PS+//DI33ngjAFOmTOH8888nOzubESNG+BIFwJYtW/jd735HdnY2mzdvZsuWLaxbt46srCwyMzN5//33WbZsGb/4xS/47LPP+Pzzzxk8eDC5ubksXryY7OxsNmzYwMSJEz3/rgAKCgpYt24dM2bMKLK+0FdffUWjRo2oXbu2b928efMYNWoUw4YN44033iA/P7/Y67Zv38727ds544wziqzfvHkzqampQR/79+8vsm9ubi4NGzakRg3nO1XLli3ZtWsXgUaMGEG9evVo3rw5rVu35p577uHUU08ttl+gs88+mw8//DDkfsZURNxdEZTnm3skHDhwgP3799O/f3/A+ab4m9/8hv3793Po0CF69+4NwOjRo3n99deLvb5Pnz7cddddXH311QwfPpyWLVuW+n7vvPMO8+fP9y03atTI93zgwIHs27eP+vXr8+CDD/r237Rpk2+fgwcPkpeXxwcffMDixYsBGDx4cJHjtGnThnPPPReAt956i7feeovu3Z2Gr7y8PLZs2UK/fv24++67ue+++7jsssvo168fBQUF1KlThxtuuIHLLrus2L38kn5XhYYPHw5Ajx49gn5z3rNnD02bNvUtHzt2jKVLlzJ9+nQSExPp1asXy5cv973vggUL+OCDD6hduzazZs0q9oHcsWNHsrKySvt1l9m6detISEhg9+7d/PDDD/Tr148LL7yQdu3alfq6Zs2asXv37rDGYkyguEsE8SI9PZ0hQ4awdOlS+vTpw/Lly8t9rBUrVtCwYUOuvvpq7r//fqZPn86JEydYu3YtderU8XycevXq+Z6rKn/4wx+4+eabi+33ySefsHTpUiZOnMgFF1zApEmTWLduHe+++y6LFi3iySefLFMjaOE3/YSEBAoKildDr1u3bpFR5cuXL2f//v0kJycDcPjwYerWretLBCNHjuTJJ58s8f02b97MyJEjg25buXIlDRs29C03btyY/fv3U1BQQI0aNcjJyaFFixbFXvfiiy8yePBgatasSbNmzejTpw8ZGRkhE8GRI0eoW7duqfsYU1F2ayhCGjRoQKNGjVi9ejUA//jHP+jfvz8NGzYkMTGRjz/+GKDIt3h/27ZtIzk5mfvuu4+ePXvy5ZdfkpiYyKFDh4LuP2jQIGbOnOlb/uGHH4psr1GjBjNmzOCFF15g3759XHTRRfzf//2fb3vhN+A+ffrw0ksvAc63/sDjFLr44ouZM2cOeXl5AOzatYvvvvuO3bt3c/LJJ3PNNddw77338sknn5CXl8eBAwe49NJLeeyxx/jss888/a68OvPMM4tcKcybN49nnnmGHTt2sGPHDr766ivefvttDh8+7Ol4hVcEwR7+SQCce/8DBw5k0SKnH/jcuXMZOnRosWO2bt3al/x+/PFH1q5dy1lnnRUylv/85z/FehMZE26WCMLk8OHDtGzZ0veYPn06c+fO5d577yUlJYWsrCwmTXImZ3v22We56aabSE1N5ccff6RBgwbFjjdjxgy6du1KSkoKNWvW5JJLLiElJYWEhAS6devGY489VmT/iRMn8sMPP9C1a1e6devGihUrih2zefPmjBo1ipkzZ/LEE0+QkZFBSkoKnTt35qmnngLg/vvv56233qJr164sXLiQ008/ncTExGLHuuiiixg9ejS9e/cmOTmZESNGcOjQITZu3Mg555xDamoqU6ZMYeLEiRw6dIjLLruMlJQU+vbty/Tp04sdr6TflRf16tWjffv2bN26lcOHD7Ns2TKGDBlSZHvfvn157bXXPB+zLB5++GGmT5/OGWecQW5uLjfccAMAS5Ys8Z3HbbfdRl5eHl26dKFnz56MHTuWlJQUAEaNGkXv3r3ZvHkzLVu29PU6ys/PZ+vWraSlpUUkbmMKiapG7uAig4HHgQTgGVWdGrD9FuA24DiQB4xT1U3FDuQnLS1NMzKKzm//xRdf0KlTp3CGHlF5eXm+cQdTp05lz549PP7441GOynH06FESEhKoUaMGa9as4dZbbw37/fJIWLx4MZmZmWHpHltVLF68mE8++cTXrhOLYu3/ZmUYOWsNAAtudtoJfQPKCgeYRYiIZKpq0G8VEWsjEJEEYCYwCMgB1ovIkoAP+hdV9Sl3/8uB6cDgSMVUVbzxxhs89NBDFBQU0KZNG55//vloh+TzzTffcOWVV3LixAlq1aoVlgFXlWHYsGHk5uZGO4ywKigo4O677452GCbMPv5qH+CXEPZ84GwIHGFcifWHItlYfA6wVVW3A4jIfGAo4EsEqurf4b8eELnLkypk5MiRJTZGRluHDh349NNPox1GuRR2gY0X/j2nTDXzrVuYLg4SQQtgp99yDtArcCcRuQ24C6gFFO/87uwzDhgHTqObMcbEqr5nNAHgnzc6H4cb/hpQewgqvf5Q1LuPqupMYKaIjAYmAtcG2Wc2MBucNoLKjdAYY8KnMAEUKqw7tCAawbgi2WtoF9DKb7mlu64k84FfRzAeY4wxQUQyEawHOohIWxGpBVwFLPHfQUQ6+C0OAbZEMB5jjDFBRCwRqGoBcDuwHPgCeElVs0XkAbeHEMDtIpItIlk47QTFbgvFCitDXXbhLEMdTpMmTeKdd94pdZ/Jkyczbdq0Yuv379/P3//+9xJf99NPP9G/f3+OHz/uWzdjxgzq1KnDgQMHfOtWrlxJgwYNSE1NpVOnTkFrLJXVvn37GDRoEB06dGDQoEElDhb8/e9/T5cuXejUqRMTJkwgsIv55ZdfXmSQm5XLjn0RHVCmqktV9UxVba+qf3HXTVLVJe7zO1S1i6qmqupAVc2OZDxVnZWhLp/JkyeHtQvuAw88wIUXXliu14ZKBHPmzGH48OFFig3OmzePnj178sorrxTZt7DAYEZGBv/85z/55JNPyhVToalTp3LBBRewZcsWLrjggqAlrj/66CM+/PBDNmzYwOeff8769euLlPF+5ZVXin3psXLZsS/+Rha/me60uIfz8WZ6uULJysri3HPPJSUlhWHDhvm+ga1fv56UlBRSU1O59957fd+uVq5c6auHs2rVKl/Fy+7du3Po0CHS09NZvXo1qampPPbYY0X2z8vLY+zYsb5v/y+//HKxeHr37u2rjLl3716uuOIKevbsSc+ePX0VLvfu3cugQYPo0qULN954I23atOH7779nx44ddOzYkTFjxtC1a1d27tzJI488Qs+ePUlJSeH+++8HnPIJQ4YMoVu3bnTt2pUFC5wmsPT0dDp37kxKSgr33HMPUPRbdUm/qwEDBnDfffdxzjnncOaZZ/rKUISycOFC7rrrLgAef/xxX02f7du306dPHwAyMzPp378/PXr04OKLL2bPnj2AMz9AYcmIpUuXctZZZ9GjRw8mTJhQpGDepk2bGDBgAO3atfMl5PT0dLZt2+b72wb617/+VaQExbZt28jLy+PPf/4z8+bNC3ou9erVo0ePHmzdutXTuZfk1Vdf5dprnYvua6+9ln//+9/F9hERjhw5wrFjxzh69Cj5+fm+Et15eXlMnz69WPVY/3LZJoy+3Ri2z6JQ4i8RVCFWhrpyylAH069fP1/SWL16NY0bN2bXrl2sXr2a8847j/z8fMaPH8+iRYvIzMzk+uuv549//GORYxw5coSbb76ZN998k8zMzGITsX/55ZcsX76cdevWMWXKFPLz85k6dSrt27cnKyuLRx55pMj+x44dY/v27SQlJfnWzZ8/n6uuuop+/fqxefPmoJPk5ObmsnbtWrp0KVpZ99ChQyWWy/avLFvov//9L82bNwfg9NNPD/pevXv3ZuDAgTRv3pzmzZtz8cUX+0YG/+lPf+Luu+/m5JNPLvY6K5cdZskj4PTkSnu7qHcfDbtLqsYlqpWhjmwZ6o0bN/Lb3/4WgG+//ZZatWoxY8YMAN59911OP/108vLyOHToEDt37mT06NG8//77rF69muHDh7N582Y+//xzBg0aBMDx48d9H5KFvvzyS9q1a0fbtm0BpybQ7NmzfduHDBlC7dq1qV27Ns2aNQs509n3339frGjdvHnzWLx4MSeddBJXXHEFCxcu5PbbbwecBNa9e3dOOukk0tPTiyWCxMTEcpf/EJGgk8tv3bqVL774gpycHMApZrh69WoSExPZtm0bjz32WNBS4FYuO8zSxlbaYDKIx0QQJ6wM9c+ClaFOTk72fQhOnjyZpKQkrrvuuiKv++Uvf8lzzz1Hx44d6devH3PmzGHNmjU8+uijfPPNN3Tp0oU1a9Z4jqOkuAJjK0lgueyNGzeyZcsWXzI6duwYbdu29SWCfv36Bf2SUOjQoUNBZ2sDp+x1586di6w77bTT2LNnD82bN2fPnj00a9as2OsWL17Mueee62sHuOSSS1izZg2JiYlkZGSQlJREQUEB3333HQMGDGDlypWAlcuOdXZrKEKsDHXllaEuSb9+/Zg2bRrnnXce3bt3Z8WKFdSuXZsGDRrQsWNH9u7d60sE+fn5ZGcX7avQsWNHtm/f7vsGXNjeUZrS/kaNGjXi+PHjvmQwb948Jk+e7CuXvXv3bnbv3u25Z1fhFUGwR2ASAKe3z9y5c4HSy2WvWrWKgoIC8vPzWbVqFZ06deLWW29l9+7d7Nixgw8++IAzzzzTlwTAymVX1KY9Bxk5a43vkZT+BknpbxRZN3LWGqa8Fpn+NJYIwsTKUEevDHVJ+vXrx86dOznvvPNISEigVatW9O3bF4BatWqxaNEi7rvvPrp160ZqaiofffRRkdfXrVuXv//97wwePJgePXqQmJgY9G/lr3HjxvTp04euXbsGbSy+6KKL+OADp8jY/PnzGTZsWJHtw4YNK/HLQUWlp6fz9ttv06FDB9555x3S052Gx4yMDF8b0YgRI2jfvj3Jycl069aNbt268atf/arU41q57IoZmtqCzs1PiW4QqhpTjx49emigTZs2FVtXlR06dMj3/KGHHtIJEyZEMZqijhw5ovn5+aqq+tFHH2m3bt2iG1CUFf6tTpw4obfeeqtOnz69QsfLzMzUa665JhyhVRmvvPKKTpw4Mei2WPu/WVVc/fRavfrptWE9JpChJXyuWhtBFFgZ6tjx9NNPM3fuXI4dO0b37t2DtomUxdlnn83AgQM5fvx4iT3GYo2Vyw6/wHpEkRbRiWkiIR4mpjGmurD/m1VHaRPTxE0bQawlNGPinf2fjB1xkQjq1KlDbm6u/cMzpopQVXJzc8vUPdlET1y0EbRs2ZKcnJxiIz+NMdFTp06dkAMhTdUQF4mgZs2avtGfxhhjyiYubg0ZY4wpP0sExhhTzVkiMMaYai7mxhGIyF6gvNNsNQG+D2M4scDOuXqwc64eKnLObVS1abANMZcIKkJEMkoaUBGv7JyrBzvn6iFS52y3howxppqzRGCMMdVcdUsEs0PvEnfsnKsHO+fqISLnXK3aCIwxxhRX3a4IjDHGBLBEYIwx1VxcJgIRGSwim0Vkq4ikB9leW0QWuNs/FpGkKIQZVh7O+S4R2SQiG0TkXRFpE404wynUOfvtd4WIqIjEfFdDL+csIle6f+tsEXmxsmMMNw//tluLyAoR+dT9931pNOIMFxGZIyLficjnJWwXEXnC/X1sEJGzK/ymJU1dFqsPIAHYBrQDagGfAZ0D9vkd8JT7/CpgQbTjroRzHgic7D6/tTqcs7tfIvA+sBZIi3bclfB37gB8CjRyl5tFO+5KOOfZwK3u887AjmjHXcFzPg84G/i8hO2XAm8CApwLfFzR94zHK4JzgK2qul1VjwHzgaEB+wwF5rrPFwEXiIhUYozhFvKcVXWFqh52F9cCsV4f2MvfGeBB4GHgSGUGFyFezvkmYKaq/gCgqt9Vcozh5uWcFSic/b0BsLsS4ws7VX0f2FfKLkOBF9SxFmgoIs0r8p7xmAhaADv9lnPcdUH3UdUC4ADQuFKiiwwv5+zvBpxvFLEs5Dm7l8ytVPWNygwsgrz8nc8EzhSRD0VkrYgMrrToIsPLOU8GrhGRHGApML5yQouasv5/Dyku5iMw3onINUAa0D/asUSSiJwETAeui3Iola0Gzu2hAThXfe+LSLKq7o9mUBE2CnheVR8Vkd7AP0Skq6qeiHZgsSIerwh2Aa38llu664LuIyI1cC4ncyslusjwcs6IyIXAH4HLVfVoJcUWKaHOORHoCqwUkR0491KXxHiDsZe/cw6wRFXzVfUr4D84iSFWeTnnG4CXAFR1DVAHpzhbvPL0/70s4jERrAc6iEhbEamF0xi8JGCfJcC17vMRwHvqtsLEqJDnLCLdgVk4SSDW7xtDiHNW1QOq2kRVk1Q1Cadd5HJVzYhOuGHh5d/2v3GuBhCRJji3irZXYozh5uWcvwEuABCRTjiJIJ7nrV0CjHF7D50LHFDVPRU5YNzdGlLVAhG5HViO0+Ngjqpmi8gDQIaqLgGexbl83IrTKHNV9CKuOI/n/AhQH1jotot/o6qXRy3oCvJ4znHF4zkvBy4SkU3AceBeVY3Zq12P53w38LSI3InTcHxdLH+xE5F5OMm8idvucT9QE0BVn8JpB7kU2AocBsZW+D1j+PdljDEmDOLx1pAxxpgysERgjDHVnCUCY4yp5iwRGGNMNWeJwBhjqjlLBCbiROS4iGSJyOci8pqINAzz8Xe4feYRkbwS9qkrIqtEJEFEkkTkJzemTSLylDsSuSzvmSYiT7jPB4jIL/223SIiYypyTu5xJovIPSH2eV5ERpThmEklVbUM2O8vIrIz8PcpIreLyPVe38/EBksEpjL8pKqpqtoVZ9zGbVGI4XrgFVU97i5vU9VUIAWnYuWvy3IwVc1Q1Qnu4gDgl37bnlLVFyoacJS9hlPwLdAc4r+WT7VjicBUtjW4BbJEpL2ILBORTBFZLSJnuetPE5HFIvKZ+/ilu/7f7r7ZIjKujO97NfBq4Eq36OBHwBnut+X35Oc5G1q77/sb92rmMxF53103QEReF2cui1uAO90rjH6F3+RF5CwRWVf4Xu7xN7rPe7hXKJkislxCVI8UkZtEZL0bw8sicrLf5gtFJENE/iMil7n7J4jII+5rNojIzWX5Zanq2mCjVd0KtjtEJFiSMDHKEoGpNCKSgFMKoHDU72xgvKr2AO4B/u6ufwJYpardcOqyZ7vrr3f3TQMmiIinirFuaYJ2qrojyLaT3Zg2Av8HzFXVFOBfbhwAk4CL3XiKjMZ2j/kU8Jh71bPab9uXQC0RaeuuGgksEJGa7nuNcM9nDvCXEKfxiqr2dGP4Aqe+TqEknG/vQ4CnRKSOu/2AqvYEegI3+cVReO6/EJGlId43mAygXzleZ6qouCsxYaqkuiKShXMl8AXwtojUx7mdUljyAqC2+/N8YAyAeyvngLt+gogMc5+3wimm5qV8QhNgf8C69m5MCryqqm+KyD+A4e72fwB/c59/CDwvIi8Br3h4P38v4SSAqe7PkUBHnIJ4b7vnngCEqhXTVUT+DDTEKRWy3P893EqbW0RkO3AWcBGQ4td+0ADn9/Wfwhep6m6cUgVl9Z37HiZOWCIwleEnVU11v30vx2kjeB7Y796nD0lEBgAXAr1V9bCIrMQpLubp/YPsu83re6vqLSLSC+cbd6aI9PD4vgALcJLdK86hdIuIJAPZqtq7DMd5Hvi1qn4mItfhFpYrDDEwZJzZq8arqn/CQMIzLWsdnN+piRN2a8hUGvf+8gScImGHga9E5Dfgm4e1m7vruzjTaRbe626A8432BzcJnIVTVtrr+/4AJLi3TErzET8XILwaWO3G0F5VP1bVSThVLVsFvO4QTtnrYO+9Daf4259wkgLAZqCpOLXzEZGaItIlRGyJwB73ttLVAdt+IyIniUh7nCkdN+Mk3Fvd/RGRM0WkXoj38OpMIGTPIxM7LBGYSqWqnwIbcCYTuRq4QUQ+w2kHKJyC8A5goNuwmonTq2cZUENEvsC5zbK2jG/9FtA3xD7jgbEisgH4rRsHwCMistHtdvkRzry5/l4DhhU2Fgc57gLgGn6umX8Mp/z5w+65Z+HX66gEfwI+xrlN9WXAtm+AdTizzt2iqkeAZ4BNwCdu3LMIuANQWhuBiPxNnMqXJ4tIjohM9tvcB3g7RLwmhlj1UVMtiDNt5Z2q+ttoxxLLxJnX4i77PcYXuyIw1YKqfgKscHsumfJrgnN1YuKIXREYY0w1Z1cExhhTzVkiMMaYas4SgTHGVHOWCIwxppqzRGCMMdXc/wfRWFadG57f2QAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import  matplotlib.pyplot as plt\n",
    "\n",
    "_, ax = plt.subplots()\n",
    "for name, pred in [\n",
    "    (\"LogisticRegression\", score),\n",
    "    (\"LogisticRegression+weight\", score_weight),\n",
    "]:\n",
    "    PrecisionRecallDisplay.from_predictions(ax=ax, y_true=test[\"label_num\"], y_pred=pred, name=name)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}