train_model_2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "2ff03fd9-4e69-4d76-9579-dc5b77cc256d",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-08-28 20:36:18.829874: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n",
      "2024-08-28 20:36:18.829960: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n",
      "2024-08-28 20:36:18.831707: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n",
      "2024-08-28 20:36:18.840313: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
      "To enable the following instructions: AVX2 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "2024-08-28 20:36:20.114406: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow.keras import datasets, layers, models\n",
    "from tensorflow.keras.optimizers import Adam\n",
    "import keras\n",
    "from keras.models import Sequential, Model\n",
    "from keras.layers import *\n",
    "from keras.utils import Sequence\n",
    "from keras.layers import Conv2D, MaxPooling2D\n",
    "from qkeras import *\n",
    "\n",
    "from keras.utils import Sequence\n",
    "from keras.callbacks import CSVLogger\n",
    "from keras.callbacks import EarlyStopping\n",
    "\n",
    "import os\n",
    "import random\n",
    "\n",
    "pi = 3.14159265359\n",
    "\n",
    "maxval=1e9\n",
    "minval=1e-9\n",
    "\n",
    "import OptimizedDataGenerator as DG\n",
    "from loss import *\n",
    "from models import *\n",
    "import utils"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cae1ba4a-aca2-4fe4-99be-63fd2a88e4de",
   "metadata": {},
   "source": [
    "Read from exisiting tf records"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "5c49bef9-640b-4de7-824c-91e969c2ab09",
   "metadata": {},
   "outputs": [],
   "source": [
    "tfrecords_dir_train = \"/depot/cms/users/kuang14/Smart_Pixel/tf_records/tfrecords_2t_dataset18_bs5000_train3\"\n",
    "tfrecords_dir_validation = \"/depot/cms/users/kuang14/Smart_Pixel/tf_records/tfrecords_2t_dataset18_bs5000_val3\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "529373e7-a40e-48b9-8552-eac10c9f602b",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:root:Quantization is True in data generator. This may affect model performance.\n"
     ]
    }
   ],
   "source": [
    "validation_generator = DG.OptimizedDataGenerator(\n",
    "    load_from_tfrecords_dir = tfrecords_dir_validation, \n",
    "    shuffle = True,\n",
    "    seed = 13,\n",
    "    quantize = True\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "d77f5294-fc4f-42e7-93ff-a9fd124df564",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:root:Quantization is True in data generator. This may affect model performance.\n"
     ]
    }
   ],
   "source": [
    "training_generator =DG.OptimizedDataGenerator(\n",
    "    load_from_tfrecords_dir = tfrecords_dir_validation, \n",
    "    shuffle = True,\n",
    "    seed = 13,\n",
    "    quantize = True\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "62f84e11-bc0e-4ab7-a905-a9fc23f6a720",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "80d0ddbb-39f2-40ea-836b-5e5693dfe2f7",
   "metadata": {},
   "source": [
    "Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "923367f7-dcf5-408b-ab85-bf589d8af05d",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "b11bfa30-ad16-43db-a7de-27c5027988ea",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model\"\n",
      "_________________________________________________________________\n",
      " Layer (type)                Output Shape              Param #   \n",
      "=================================================================\n",
      " input_1 (InputLayer)        [(None, 13, 21, 2)]       0         \n",
      "                                                                 \n",
      " q_separable_conv2d (QSepar  (None, 11, 19, 5)         33        \n",
      " ableConv2D)                                                     \n",
      "                                                                 \n",
      " q_activation (QActivation)  (None, 11, 19, 5)         0         \n",
      "                                                                 \n",
      " q_conv2d (QConv2D)          (None, 11, 19, 5)         30        \n",
      "                                                                 \n",
      " q_activation_1 (QActivatio  (None, 11, 19, 5)         0         \n",
      " n)                                                              \n",
      "                                                                 \n",
      " average_pooling2d (Average  (None, 3, 6, 5)           0         \n",
      " Pooling2D)                                                      \n",
      "                                                                 \n",
      " q_activation_2 (QActivatio  (None, 3, 6, 5)           0         \n",
      " n)                                                              \n",
      "                                                                 \n",
      " flatten (Flatten)           (None, 90)                0         \n",
      "                                                                 \n",
      " q_dense (QDense)            (None, 16)                1456      \n",
      "                                                                 \n",
      " q_activation_3 (QActivatio  (None, 16)                0         \n",
      " n)                                                              \n",
      "                                                                 \n",
      " q_dense_1 (QDense)          (None, 16)                272       \n",
      "                                                                 \n",
      " q_activation_4 (QActivatio  (None, 16)                0         \n",
      " n)                                                              \n",
      "                                                                 \n",
      " q_dense_2 (QDense)          (None, 14)                238       \n",
      "                                                                 \n",
      "=================================================================\n",
      "Total params: 2029 (7.93 KB)\n",
      "Trainable params: 2029 (7.93 KB)\n",
      "Non-trainable params: 0 (0.00 Byte)\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "input_shape = (13, 21, 2)\n",
    "model = CreateModel(input_shape, n_filters=5, pool_size=3)\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "bb1c0e94-6520-4d80-9be6-314264bb6cad",
   "metadata": {},
   "outputs": [],
   "source": [
    "decay_steps = 90*training_generator.__len__()\n",
    "alpha = 0.01\n",
    "initial_learning_rate = 1e-3\n",
    "warmup_target = 1e-1\n",
    "warmup_steps = 10*training_generator.__len__()\n",
    "\n",
    "lr_schedule = tf.keras.optimizers.schedules.CosineDecay(\n",
    "    initial_learning_rate=initial_learning_rate,\n",
    "    decay_steps=decay_steps,\n",
    "    alpha=alpha,\n",
    "    warmup_target = warmup_target,\n",
    "    warmup_steps = warmup_steps\n",
    ")\n",
    "\n",
    "\n",
    "model.compile(\n",
    "    optimizer=tf.keras.optimizers.Nadam(learning_rate=1e-3),\n",
    "    loss=custom_loss\n",
    ")\n",
    "\n",
    "# model.compile(\n",
    "#     optimizer=tf.keras.optimizers.Nadam(learning_rate=lr_schedule),\n",
    "#     loss=custom_loss\n",
    "# )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "c92355d2-0285-4ba3-be6f-76f80db9a4e8",
   "metadata": {},
   "outputs": [],
   "source": [
    "fingerprint = '%08x' % random.randrange(16**8)\n",
    "os.makedirs(\"trained_models\", exist_ok=True)\n",
    "base_dir = f'./trained_models/model-{fingerprint}-checkpoints'\n",
    "os.makedirs(base_dir, exist_ok=True)  \n",
    "checkpoint_filepath = base_dir + '/weights.{epoch:02d}-t{loss:.2f}-v{val_loss:.2f}.hdf5'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "8965396f-3528-4a48-a35c-4e78549a5df1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'./trained_models/model-73e33061-checkpoints/weights.{epoch:02d}-t{loss:.2f}-v{val_loss:.2f}.hdf5'"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "checkpoint_filepath"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "8ebca30e-d757-41b8-acd1-68c73f0d87aa",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "73e33061\n"
     ]
    }
   ],
   "source": [
    "print(fingerprint)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "ec59c08e-cb2d-4a3c-991c-1d786605d509",
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.keras.callbacks import CSVLogger, EarlyStopping, ModelCheckpoint, Callback\n",
    "\n",
    "early_stopping_patience = 50\n",
    "\n",
    "class CustomModelCheckpoint(ModelCheckpoint):\n",
    "    def on_epoch_end(self, epoch, logs=None):\n",
    "        super().on_epoch_end(epoch, logs)\n",
    "        checkpoints = [f for f in os.listdir(base_dir) if f.startswith('weights')]\n",
    "        if len(checkpoints) > 1:\n",
    "            checkpoints.sort()\n",
    "            for checkpoint in checkpoints[:-1]:\n",
    "                os.remove(os.path.join(base_dir, checkpoint))\n",
    "\n",
    "es = EarlyStopping(patience=early_stopping_patience, restore_best_weights=True)\n",
    "\n",
    "mcp = CustomModelCheckpoint(\n",
    "    filepath=checkpoint_filepath,\n",
    "    save_weights_only=True,\n",
    "    monitor='val_loss',\n",
    "    save_best_only=True,\n",
    "    save_freq='epoch',\n",
    "    verbose=1\n",
    ")\n",
    "\n",
    "csv_logger = CSVLogger(f'{base_dir}/training_log.csv', append=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7d4e6ffd-0af9-43fc-b46c-d034434313ad",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/1000\n",
      "1028/1028 [==============================] - ETA: 0s - loss: 20404.6602\n",
      "Epoch 1: val_loss improved from inf to 13933.39062, saving model to ./trained_models/model-73e33061-checkpoints/weights.01-t20404.66-v13933.39.hdf5\n",
      "1028/1028 [==============================] - 325s 314ms/step - loss: 20404.6602 - val_loss: 13933.3906\n",
      "Epoch 2/1000\n",
      "1028/1028 [==============================] - ETA: 0s - loss: 12658.1104\n",
      "Epoch 2: val_loss improved from 13933.39062 to 9955.94531, saving model to ./trained_models/model-73e33061-checkpoints/weights.02-t12658.11-v9955.95.hdf5\n",
      "1028/1028 [==============================] - 332s 323ms/step - loss: 12658.1104 - val_loss: 9955.9453\n",
      "Epoch 3/1000\n",
      "1028/1028 [==============================] - ETA: 0s - loss: 9026.3477\n",
      "Epoch 3: val_loss did not improve from 9955.94531\n",
      "1028/1028 [==============================] - 340s 331ms/step - loss: 9026.3477 - val_loss: 11500.9854\n",
      "Epoch 4/1000\n",
      "1028/1028 [==============================] - ETA: 0s - loss: 6378.6309\n",
      "Epoch 4: val_loss improved from 9955.94531 to 5405.64258, saving model to ./trained_models/model-73e33061-checkpoints/weights.04-t6378.63-v5405.64.hdf5\n",
      "1028/1028 [==============================] - 340s 330ms/step - loss: 6378.6309 - val_loss: 5405.6426\n",
      "Epoch 5/1000\n",
      "1028/1028 [==============================] - ETA: 0s - loss: 3130.2925\n",
      "Epoch 5: val_loss improved from 5405.64258 to 2141.40674, saving model to ./trained_models/model-73e33061-checkpoints/weights.05-t3130.29-v2141.41.hdf5\n",
      "1028/1028 [==============================] - 350s 341ms/step - loss: 3130.2925 - val_loss: 2141.4067\n",
      "Epoch 6/1000\n",
      "1028/1028 [==============================] - ETA: 0s - loss: 3684.8760\n",
      "Epoch 6: val_loss did not improve from 2141.40674\n",
      "1028/1028 [==============================] - 337s 328ms/step - loss: 3684.8760 - val_loss: 3762.6338\n",
      "Epoch 7/1000\n",
      "1028/1028 [==============================] - ETA: 0s - loss: 2347.0017\n",
      "Epoch 7: val_loss improved from 2141.40674 to 1457.01379, saving model to ./trained_models/model-73e33061-checkpoints/weights.07-t2347.00-v1457.01.hdf5\n",
      "1028/1028 [==============================] - 334s 325ms/step - loss: 2347.0017 - val_loss: 1457.0138\n",
      "Epoch 8/1000\n",
      "1028/1028 [==============================] - ETA: 0s - loss: 2033.6012\n",
      "Epoch 8: val_loss did not improve from 1457.01379\n",
      "1028/1028 [==============================] - 318s 310ms/step - loss: 2033.6012 - val_loss: 1485.9369\n",
      "Epoch 9/1000\n",
      "1028/1028 [==============================] - ETA: 0s - loss: 1278.9170\n",
      "Epoch 9: val_loss did not improve from 1457.01379\n",
      "1028/1028 [==============================] - 311s 303ms/step - loss: 1278.9170 - val_loss: 1972.5414\n",
      "Epoch 10/1000\n",
      "1028/1028 [==============================] - ETA: 0s - loss: 1326.1830\n",
      "Epoch 10: val_loss did not improve from 1457.01379\n",
      "1028/1028 [==============================] - 318s 310ms/step - loss: 1326.1830 - val_loss: 1591.8705\n",
      "Epoch 11/1000\n",
      "1028/1028 [==============================] - ETA: 0s - loss: 973.1439\n",
      "Epoch 11: val_loss improved from 1457.01379 to 321.90671, saving model to ./trained_models/model-73e33061-checkpoints/weights.11-t973.14-v321.91.hdf5\n",
      "1028/1028 [==============================] - 322s 313ms/step - loss: 973.1439 - val_loss: 321.9067\n",
      "Epoch 12/1000\n",
      "1028/1028 [==============================] - ETA: 0s - loss: 950.2963\n",
      "Epoch 12: val_loss did not improve from 321.90671\n",
      "1028/1028 [==============================] - 322s 313ms/step - loss: 950.2963 - val_loss: 624.6049\n",
      "Epoch 13/1000\n",
      "1028/1028 [==============================] - ETA: 0s - loss: 760.5010\n",
      "Epoch 13: val_loss did not improve from 321.90671\n",
      "1028/1028 [==============================] - 318s 310ms/step - loss: 760.5010 - val_loss: 961.7076\n",
      "Epoch 14/1000\n",
      "1028/1028 [==============================] - ETA: 0s - loss: 408.5257\n",
      "Epoch 14: val_loss improved from 321.90671 to -329.80090, saving model to ./trained_models/model-73e33061-checkpoints/weights.14-t408.53-v-329.80.hdf5\n",
      "1028/1028 [==============================] - 318s 310ms/step - loss: 408.5257 - val_loss: -329.8009\n",
      "Epoch 15/1000\n",
      "1028/1028 [==============================] - ETA: 0s - loss: 185.5054\n",
      "Epoch 15: val_loss did not improve from -329.80090\n",
      "1028/1028 [==============================] - 318s 310ms/step - loss: 185.5054 - val_loss: 1039.2205\n",
      "Epoch 16/1000\n",
      "1028/1028 [==============================] - ETA: 0s - loss: 5.7977\n",
      "Epoch 16: val_loss did not improve from -329.80090\n",
      "1028/1028 [==============================] - 340s 331ms/step - loss: 5.7977 - val_loss: 33.6915\n",
      "Epoch 17/1000\n",
      "1028/1028 [==============================] - ETA: 0s - loss: 159.1429\n",
      "Epoch 17: val_loss did not improve from -329.80090\n",
      "1028/1028 [==============================] - 389s 378ms/step - loss: 159.1429 - val_loss: -95.7802\n",
      "Epoch 18/1000\n",
      "1028/1028 [==============================] - ETA: 0s - loss: 208.0903\n",
      "Epoch 18: val_loss did not improve from -329.80090\n",
      "1028/1028 [==============================] - 383s 372ms/step - loss: 208.0903 - val_loss: 81.4274\n",
      "Epoch 19/1000\n",
      "1028/1028 [==============================] - ETA: 0s - loss: 104.7689\n",
      "Epoch 19: val_loss improved from -329.80090 to -628.54437, saving model to ./trained_models/model-73e33061-checkpoints/weights.19-t104.77-v-628.54.hdf5\n",
      "1028/1028 [==============================] - 393s 383ms/step - loss: 104.7689 - val_loss: -628.5444\n",
      "Epoch 20/1000\n",
      "1028/1028 [==============================] - ETA: 0s - loss: -212.3896\n",
      "Epoch 20: val_loss did not improve from -628.54437\n",
      "1028/1028 [==============================] - 332s 323ms/step - loss: -212.3896 - val_loss: 806.5480\n",
      "Epoch 21/1000\n",
      "1028/1028 [==============================] - ETA: 0s - loss: -383.2819\n",
      "Epoch 21: val_loss did not improve from -628.54437\n",
      "1028/1028 [==============================] - 322s 314ms/step - loss: -383.2819 - val_loss: -228.5548\n",
      "Epoch 22/1000\n",
      "1028/1028 [==============================] - ETA: 0s - loss: -526.8115\n",
      "Epoch 22: val_loss improved from -628.54437 to -952.19904, saving model to ./trained_models/model-73e33061-checkpoints/weights.22-t-526.81-v-952.20.hdf5\n",
      "1028/1028 [==============================] - 318s 309ms/step - loss: -526.8115 - val_loss: -952.1990\n",
      "Epoch 23/1000\n",
      "1028/1028 [==============================] - ETA: 0s - loss: -813.5906\n",
      "Epoch 23: val_loss did not improve from -952.19904\n",
      "1028/1028 [==============================] - 330s 321ms/step - loss: -813.5906 - val_loss: -542.0620\n",
      "Epoch 24/1000\n",
      "1028/1028 [==============================] - ETA: 0s - loss: -1069.4459\n",
      "Epoch 24: val_loss improved from -952.19904 to -1575.26257, saving model to ./trained_models/model-73e33061-checkpoints/weights.24-t-1069.45-v-1575.26.hdf5\n",
      "1028/1028 [==============================] - 319s 310ms/step - loss: -1069.4459 - val_loss: -1575.2626\n",
      "Epoch 25/1000\n",
      "1028/1028 [==============================] - ETA: 0s - loss: -1262.2716\n",
      "Epoch 25: val_loss did not improve from -1575.26257\n",
      "1028/1028 [==============================] - 325s 316ms/step - loss: -1262.2716 - val_loss: -850.1249\n",
      "Epoch 26/1000\n",
      "1028/1028 [==============================] - ETA: 0s - loss: -1353.0651\n",
      "Epoch 26: val_loss improved from -1575.26257 to -1620.77295, saving model to ./trained_models/model-73e33061-checkpoints/weights.26-t-1353.07-v-1620.77.hdf5\n",
      "1028/1028 [==============================] - 323s 314ms/step - loss: -1353.0651 - val_loss: -1620.7729\n",
      "Epoch 27/1000\n",
      "1028/1028 [==============================] - ETA: 0s - loss: -1361.3528\n",
      "Epoch 27: val_loss did not improve from -1620.77295\n",
      "1028/1028 [==============================] - 321s 312ms/step - loss: -1361.3528 - val_loss: -1490.1154\n",
      "Epoch 28/1000\n",
      "1028/1028 [==============================] - ETA: 0s - loss: -1699.5557\n",
      "Epoch 28: val_loss improved from -1620.77295 to -2320.38672, saving model to ./trained_models/model-73e33061-checkpoints/weights.28-t-1699.56-v-2320.39.hdf5\n",
      "1028/1028 [==============================] - 322s 313ms/step - loss: -1699.5557 - val_loss: -2320.3867\n",
      "Epoch 29/1000\n",
      "1028/1028 [==============================] - ETA: 0s - loss: -1565.9587\n",
      "Epoch 29: val_loss did not improve from -2320.38672\n",
      "1028/1028 [==============================] - 324s 315ms/step - loss: -1565.9587 - val_loss: -1647.0944\n",
      "Epoch 30/1000\n",
      "1028/1028 [==============================] - ETA: 0s - loss: -1816.1023\n",
      "Epoch 30: val_loss did not improve from -2320.38672\n",
      "1028/1028 [==============================] - 325s 316ms/step - loss: -1816.1023 - val_loss: -1513.6600\n",
      "Epoch 31/1000\n",
      "1028/1028 [==============================] - ETA: 0s - loss: -1814.3978\n",
      "Epoch 31: val_loss did not improve from -2320.38672\n",
      "1028/1028 [==============================] - 339s 330ms/step - loss: -1814.3978 - val_loss: -1909.1520\n",
      "Epoch 32/1000\n",
      "1028/1028 [==============================] - ETA: 0s - loss: -2031.3030\n",
      "Epoch 32: val_loss did not improve from -2320.38672\n",
      "1028/1028 [==============================] - 345s 336ms/step - loss: -2031.3030 - val_loss: -1951.0846\n",
      "Epoch 33/1000\n",
      "1028/1028 [==============================] - ETA: 0s - loss: -2258.8115\n",
      "Epoch 33: val_loss did not improve from -2320.38672\n",
      "1028/1028 [==============================] - 352s 343ms/step - loss: -2258.8115 - val_loss: -1612.2986\n",
      "Epoch 34/1000\n",
      "1028/1028 [==============================] - ETA: 0s - loss: -2031.9351\n",
      "Epoch 34: val_loss did not improve from -2320.38672\n",
      "1028/1028 [==============================] - 358s 348ms/step - loss: -2031.9351 - val_loss: -1812.3447\n",
      "Epoch 35/1000\n",
      "1028/1028 [==============================] - ETA: 0s - loss: -2275.2573\n",
      "Epoch 35: val_loss did not improve from -2320.38672\n",
      "1028/1028 [==============================] - 327s 318ms/step - loss: -2275.2573 - val_loss: -2167.8281\n",
      "Epoch 36/1000\n",
      "1028/1028 [==============================] - ETA: 0s - loss: -2496.8137\n",
      "Epoch 36: val_loss improved from -2320.38672 to -3054.27051, saving model to ./trained_models/model-73e33061-checkpoints/weights.36-t-2496.81-v-3054.27.hdf5\n",
      "1028/1028 [==============================] - 326s 317ms/step - loss: -2496.8137 - val_loss: -3054.2705\n",
      "Epoch 37/1000\n",
      "1028/1028 [==============================] - ETA: 0s - loss: -2533.4368\n",
      "Epoch 37: val_loss did not improve from -3054.27051\n",
      "1028/1028 [==============================] - 328s 320ms/step - loss: -2533.4368 - val_loss: -2498.6492\n",
      "Epoch 38/1000\n",
      "1028/1028 [==============================] - ETA: 0s - loss: -2576.9492\n",
      "Epoch 38: val_loss improved from -3054.27051 to -3136.33154, saving model to ./trained_models/model-73e33061-checkpoints/weights.38-t-2576.95-v-3136.33.hdf5\n",
      "1028/1028 [==============================] - 327s 318ms/step - loss: -2576.9492 - val_loss: -3136.3315\n",
      "Epoch 39/1000\n",
      "1028/1028 [==============================] - ETA: 0s - loss: -2747.5896\n",
      "Epoch 39: val_loss did not improve from -3136.33154\n",
      "1028/1028 [==============================] - 331s 322ms/step - loss: -2747.5896 - val_loss: -2757.7996\n",
      "Epoch 40/1000\n",
      "1028/1028 [==============================] - ETA: 0s - loss: -2832.7478\n",
      "Epoch 40: val_loss did not improve from -3136.33154\n",
      "1028/1028 [==============================] - 329s 320ms/step - loss: -2832.7478 - val_loss: -2954.1313\n",
      "Epoch 41/1000\n",
      "1028/1028 [==============================] - ETA: 0s - loss: -2800.5801\n",
      "Epoch 41: val_loss did not improve from -3136.33154\n",
      "1028/1028 [==============================] - 327s 318ms/step - loss: -2800.5801 - val_loss: -2800.9692\n",
      "Epoch 42/1000\n",
      "1028/1028 [==============================] - ETA: 0s - loss: -3076.4001\n",
      "Epoch 42: val_loss improved from -3136.33154 to -3632.82837, saving model to ./trained_models/model-73e33061-checkpoints/weights.42-t-3076.40-v-3632.83.hdf5\n",
      "1028/1028 [==============================] - 335s 326ms/step - loss: -3076.4001 - val_loss: -3632.8284\n",
      "Epoch 43/1000\n",
      "1028/1028 [==============================] - ETA: 0s - loss: -3267.2993\n",
      "Epoch 43: val_loss did not improve from -3632.82837\n",
      "1028/1028 [==============================] - 336s 327ms/step - loss: -3267.2993 - val_loss: -3406.8613\n",
      "Epoch 44/1000\n",
      "1028/1028 [==============================] - ETA: 0s - loss: -3397.3123\n",
      "Epoch 44: val_loss improved from -3632.82837 to -3689.83521, saving model to ./trained_models/model-73e33061-checkpoints/weights.44-t-3397.31-v-3689.84.hdf5\n",
      "1028/1028 [==============================] - 341s 332ms/step - loss: -3397.3123 - val_loss: -3689.8352\n",
      "Epoch 45/1000\n",
      "1028/1028 [==============================] - ETA: 0s - loss: -3248.0547\n",
      "Epoch 45: val_loss did not improve from -3689.83521\n",
      "1028/1028 [==============================] - 329s 320ms/step - loss: -3248.0547 - val_loss: -3078.3945\n",
      "Epoch 46/1000\n",
      "1028/1028 [==============================] - ETA: 0s - loss: -3163.1978\n",
      "Epoch 46: val_loss did not improve from -3689.83521\n",
      "1028/1028 [==============================] - 339s 330ms/step - loss: -3163.1978 - val_loss: -2610.3682\n",
      "Epoch 47/1000\n",
      "1028/1028 [==============================] - ETA: 0s - loss: -3252.8264\n",
      "Epoch 47: val_loss did not improve from -3689.83521\n",
      "1028/1028 [==============================] - 351s 341ms/step - loss: -3252.8264 - val_loss: -2555.3311\n",
      "Epoch 48/1000\n",
      " 490/1028 [=============>................] - ETA: 1:41 - loss: -3260.8528"
     ]
    }
   ],
   "source": [
    "model.fit(\n",
    "    x=training_generator,\n",
    "    validation_data=validation_generator,\n",
    "    callbacks=[es, mcp, csv_logger],\n",
    "    epochs=1000,\n",
    "    shuffle=False,\n",
    "    verbose=1\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "11be62da-d5cc-43b2-807c-12873f93d036",
   "metadata": {},
   "outputs": [],
   "source": [
    "1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6e36b8ba-8f72-4e9d-b186-045a024936fd",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(fingerprint)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7df4ec3a-c08d-4fd4-bc5a-66d5e7882f46",
   "metadata": {},
   "outputs": [],
   "source": [
    "# # clean up tf records\n",
    "# utils.safe_remove_directory(tfrecords_dir_train)\n",
    "# utils.safe_remove_directory(tfrecords_dir_validation)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python3 kernel (default)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}