test_resnet101.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import torch\n",
    "from tqdm import tqdm\n",
    "\n",
    "\n",
    "from preprocessor import AgeRecognitionPreprocessor\n",
    "from dataset import AgeRecognitionDataset\n",
    "from models import resent101_age_recogniser"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "IMAGE_DIR = './Cleaned/'\n",
    "TEST_PAIRINGS = './test_data.csv'\n",
    "VARIANT = 'resnet101'\n",
    "DEVICE = 'cuda'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = resent101_age_recogniser().to(DEVICE)\n",
    "preprocessor = AgeRecognitionPreprocessor()\n",
    "dataset = AgeRecognitionDataset(triplet_csv_path=TEST_PAIRINGS, image_dir=IMAGE_DIR, preprocessor=preprocessor, kfolds=1, device=DEVICE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading best ver...\n"
     ]
    }
   ],
   "source": [
    "if os.path.exists(f'./Checkpoint/{VARIANT}/best.pt'):\n",
    "    print(\"Loading best ver...\")\n",
    "    best_state = torch.load(f'./Checkpoint/{VARIANT}/best.pt')\n",
    "    model.load_state_dict(best_state['model_state_dict'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 49828/49828 [41:54<00:00, 19.82it/s]  \n"
     ]
    }
   ],
   "source": [
    "from torch import cosine_similarity\n",
    "\n",
    "result = []\n",
    "model.eval()    \n",
    "total_loss = 0\n",
    "for item in tqdm(dataset):\n",
    "    # Batch shape: (N, Anchor-Positive-Negative, C, H, W)\n",
    "    predictions = model.forward_features(item)\n",
    "    anchor_pos = (cosine_similarity(predictions[0].unsqueeze(0), predictions[1].unsqueeze(0)).cpu().detach().item() + 1) / 2\n",
    "    anchor_neg = (cosine_similarity(predictions[0].unsqueeze(0), predictions[2].unsqueeze(0)).cpu().detach().item() + 1) / 2\n",
    "    pos_neg = (cosine_similarity(predictions[1].unsqueeze(0), predictions[2].unsqueeze(0)).cpu().detach().item() + 1) / 2\n",
    "    result.append([anchor_pos, anchor_neg, pos_neg])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>anchor_pos</th>\n",
       "      <th>anchor_neg</th>\n",
       "      <th>pos_neg</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.333707</td>\n",
       "      <td>0.377810</td>\n",
       "      <td>0.398726</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.861512</td>\n",
       "      <td>0.470617</td>\n",
       "      <td>0.412320</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.970144</td>\n",
       "      <td>0.404495</td>\n",
       "      <td>0.332855</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.961106</td>\n",
       "      <td>0.461787</td>\n",
       "      <td>0.376850</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.928912</td>\n",
       "      <td>0.288522</td>\n",
       "      <td>0.324836</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49823</th>\n",
       "      <td>0.832842</td>\n",
       "      <td>0.539848</td>\n",
       "      <td>0.483050</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49824</th>\n",
       "      <td>0.843990</td>\n",
       "      <td>0.589229</td>\n",
       "      <td>0.552761</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49825</th>\n",
       "      <td>0.928485</td>\n",
       "      <td>0.431824</td>\n",
       "      <td>0.462893</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49826</th>\n",
       "      <td>0.642524</td>\n",
       "      <td>0.917398</td>\n",
       "      <td>0.576163</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49827</th>\n",
       "      <td>0.843394</td>\n",
       "      <td>0.707247</td>\n",
       "      <td>0.501309</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>49828 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       anchor_pos  anchor_neg   pos_neg\n",
       "0        0.333707    0.377810  0.398726\n",
       "1        0.861512    0.470617  0.412320\n",
       "2        0.970144    0.404495  0.332855\n",
       "3        0.961106    0.461787  0.376850\n",
       "4        0.928912    0.288522  0.324836\n",
       "...           ...         ...       ...\n",
       "49823    0.832842    0.539848  0.483050\n",
       "49824    0.843990    0.589229  0.552761\n",
       "49825    0.928485    0.431824  0.462893\n",
       "49826    0.642524    0.917398  0.576163\n",
       "49827    0.843394    0.707247  0.501309\n",
       "\n",
       "[49828 rows x 3 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "similarities = pd.DataFrame(result, columns=['anchor_pos', 'anchor_neg', 'pos_neg'])\n",
    "similarities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "similarities.to_csv('./test_results_resnet101.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Accuracy computation under given threshold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>anchor_pos</th>\n",
       "      <th>anchor_neg</th>\n",
       "      <th>pos_neg</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.333707</td>\n",
       "      <td>0.377810</td>\n",
       "      <td>0.398726</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.861512</td>\n",
       "      <td>0.470617</td>\n",
       "      <td>0.412320</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.970144</td>\n",
       "      <td>0.404495</td>\n",
       "      <td>0.332855</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.961106</td>\n",
       "      <td>0.461787</td>\n",
       "      <td>0.376850</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.928912</td>\n",
       "      <td>0.288522</td>\n",
       "      <td>0.324836</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49823</th>\n",
       "      <td>0.832842</td>\n",
       "      <td>0.539848</td>\n",
       "      <td>0.483050</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49824</th>\n",
       "      <td>0.843990</td>\n",
       "      <td>0.589229</td>\n",
       "      <td>0.552761</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49825</th>\n",
       "      <td>0.928485</td>\n",
       "      <td>0.431824</td>\n",
       "      <td>0.462893</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49826</th>\n",
       "      <td>0.642524</td>\n",
       "      <td>0.917398</td>\n",
       "      <td>0.576163</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49827</th>\n",
       "      <td>0.843394</td>\n",
       "      <td>0.707247</td>\n",
       "      <td>0.501309</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>49828 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       anchor_pos  anchor_neg   pos_neg\n",
       "0        0.333707    0.377810  0.398726\n",
       "1        0.861512    0.470617  0.412320\n",
       "2        0.970144    0.404495  0.332855\n",
       "3        0.961106    0.461787  0.376850\n",
       "4        0.928912    0.288522  0.324836\n",
       "...           ...         ...       ...\n",
       "49823    0.832842    0.539848  0.483050\n",
       "49824    0.843990    0.589229  0.552761\n",
       "49825    0.928485    0.431824  0.462893\n",
       "49826    0.642524    0.917398  0.576163\n",
       "49827    0.843394    0.707247  0.501309\n",
       "\n",
       "[49828 rows x 3 columns]"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "similarities = pd.read_csv('./test_results_resnet101.csv', index_col=0)\n",
    "similarities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "acc_report = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "THRESHOLD = 0.65"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "anchor_pos_np = similarities['anchor_pos'].to_numpy()\n",
    "anchor_neg_np = similarities['anchor_neg'].to_numpy()\n",
    "pos_neg_np = similarities['pos_neg'].to_numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_acc(np_arr, higher, lower):\n",
    "    '''\n",
    "    Assigns higher to values above threshold and lower to values below threshold.\n",
    "    So for positive pairings: 1, 0; negative pairings: 0, 1\n",
    "    '''\n",
    "    correct = np.where(np_arr >= THRESHOLD, higher, lower).sum()\n",
    "    wrong = len(np_arr) - correct\n",
    "    acc = correct / len(np_arr)\n",
    "    print(correct, wrong, acc)\n",
    "    return correct, wrong, acc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "39990 9838 0.8025608091835915\n"
     ]
    }
   ],
   "source": [
    "acc_report.append(compute_acc(anchor_pos_np, 1, 0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "40751 9077 0.8178333467126917\n"
     ]
    }
   ],
   "source": [
    "acc_report.append(compute_acc(anchor_neg_np, 0, 1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "40726 9102 0.8173316207754676\n"
     ]
    }
   ],
   "source": [
    "acc_report.append(compute_acc(pos_neg_np, 0, 1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>correct</th>\n",
       "      <th>wrong</th>\n",
       "      <th>accuracy</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>39990</td>\n",
       "      <td>9838</td>\n",
       "      <td>0.802561</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>40751</td>\n",
       "      <td>9077</td>\n",
       "      <td>0.817833</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>40726</td>\n",
       "      <td>9102</td>\n",
       "      <td>0.817332</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   correct  wrong  accuracy\n",
       "0    39990   9838  0.802561\n",
       "1    40751   9077  0.817833\n",
       "2    40726   9102  0.817332"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "acc_report_df = pd.DataFrame(acc_report, columns=['correct', 'wrong', 'accuracy'])\n",
    "acc_report_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8125752588905836"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "overall_acc = acc_report_df.correct.sum() / (acc_report_df.correct.sum() + acc_report_df.wrong.sum())\n",
    "overall_acc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(81477.0, 18179.0, 9838.0, 39990.0)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tp = acc_report_df.iloc[0].correct\n",
    "tn = acc_report_df.iloc[1].correct + acc_report_df.iloc[2].correct\n",
    "fp = acc_report_df.iloc[1].wrong + acc_report_df.iloc[2].wrong\n",
    "fn = acc_report_df.iloc[0].wrong\n",
    "tn, fp, fn, tp"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Using sklearn classification report"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import classification_report, confusion_matrix\n",
    "\n",
    "arr = np.where(anchor_pos_np >= THRESHOLD, 1, 0)\n",
    "arr = np.append(arr, np.where(pos_neg_np >= THRESHOLD, 1, 0))\n",
    "y_predict = np.append(arr, np.where(anchor_neg_np >= THRESHOLD, 1, 0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_true = np.array([1 if i < len(anchor_pos_np) else 0 for i in range(len(anchor_pos_np) * 3)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 1, 1, ..., 0, 1, 1])"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_predict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1, 1, 1, ..., 0, 0, 0])"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_true"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.89      0.82      0.85     99656\n",
      "           1       0.69      0.80      0.74     49828\n",
      "\n",
      "    accuracy                           0.81    149484\n",
      "   macro avg       0.79      0.81      0.80    149484\n",
      "weighted avg       0.82      0.81      0.82    149484\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(y_true, y_predict))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[81477 18179]\n",
      " [ 9838 39990]]\n"
     ]
    }
   ],
   "source": [
    "print(confusion_matrix(y_true, y_predict))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Triplet Matching Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8062334430440716"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "max_indices = similarities.to_numpy().argmax(axis=1)\n",
    "correct = np.where(max_indices == 0, 1, 0).sum() # 0th index is the anchor-positive pair, which is the desired matching\n",
    "acc = correct / len(max_indices)\n",
    "acc"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "agerec",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}