train.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a4c96cc7-349f-4425-8c01-228742c8c6fd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "a\n",
      "Experimental settings are: \n",
      "{'data_path': './data/', 'filename': 'wtbdata_245days.csv', 'task': 'MS', 'target': 'Patv', 'checkpoints': './checkpoints/', 'input_len': 144, 'output_len': 288, 'start_col': 3, 'in_var': 10, 'out_var': 1, 'day_len': 144, 'train_size': 153, 'val_size': 16, 'test_size': 15, 'total_size': 184, 'lstm_layer': 2, 'dropout': 0.05, 'num_workers': 5, 'train_epochs': 10, 'batch_size': 32, 'patience': 3, 'lr': 0.0001, 'lr_adjust': 'type1', 'capacity': 134, 'turbine_id': 0, 'pred_file': './predict.py', 'stride': 1, 'is_debug': False}\n",
      ">>>>>>> Training Turbine   0 >>>>>>>>>>>>>>>>>>>>>>>>>>\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ma-user/work/common.py:197: UserWarning: torch.nn.utils.clip_grad_norm is now deprecated in favor of torch.nn.utils.clip_grad_norm_.\n",
      "  clip = torch.nn.utils.clip_grad_norm(parameters=self.model.parameters(), max_norm=50.0)\n",
      " 30%|███       | 3/10 [01:23<03:14, 27.84s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Early stopped! \n",
      ">>>>>>> Training Turbine   1 >>>>>>>>>>>>>>>>>>>>>>>>>>\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 70%|███████   | 7/10 [02:46<01:11, 23.72s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Early stopped! \n",
      ">>>>>>> Training Turbine   2 >>>>>>>>>>>>>>>>>>>>>>>>>>\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 40%|████      | 4/10 [01:43<02:35, 25.92s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Early stopped! \n",
      ">>>>>>> Training Turbine   3 >>>>>>>>>>>>>>>>>>>>>>>>>>\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 60%|██████    | 6/10 [02:25<01:37, 24.31s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Early stopped! \n",
      ">>>>>>> Training Turbine   4 >>>>>>>>>>>>>>>>>>>>>>>>>>\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 70%|███████   | 7/10 [02:45<01:10, 23.66s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Early stopped! \n",
      ">>>>>>> Training Turbine   5 >>>>>>>>>>>>>>>>>>>>>>>>>>\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 60%|██████    | 6/10 [02:25<01:37, 24.29s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Early stopped! \n",
      ">>>>>>> Training Turbine   6 >>>>>>>>>>>>>>>>>>>>>>>>>>\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "100%|██████████| 10/10 [03:29<00:00, 20.90s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ">>>>>>> Training Turbine   7 >>>>>>>>>>>>>>>>>>>>>>>>>>\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 30%|███       | 3/10 [01:24<03:16, 28.11s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Early stopped! \n",
      ">>>>>>> Training Turbine   8 >>>>>>>>>>>>>>>>>>>>>>>>>>\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 30%|███       | 3/10 [01:23<03:14, 27.80s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Early stopped! \n",
      ">>>>>>> Training Turbine   9 >>>>>>>>>>>>>>>>>>>>>>>>>>\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 80%|████████  | 8/10 [03:18<00:49, 24.79s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Early stopped! \n",
      ">>>>>>> Training Turbine  27 >>>>>>>>>>>>>>>>>>>>>>>>>>\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 60%|██████    | 6/10 [02:34<01:42, 25.70s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Early stopped! \n",
      ">>>>>>> Training Turbine  28 >>>>>>>>>>>>>>>>>>>>>>>>>>\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 40%|████      | 4/10 [01:50<02:45, 27.56s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Early stopped! \n",
      ">>>>>>> Training Turbine  29 >>>>>>>>>>>>>>>>>>>>>>>>>>\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "100%|██████████| 10/10 [03:40<00:00, 22.09s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ">>>>>>> Training Turbine  30 >>>>>>>>>>>>>>>>>>>>>>>>>>\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 30%|███       | 3/10 [01:28<03:26, 29.52s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Early stopped! \n",
      ">>>>>>> Training Turbine  31 >>>>>>>>>>>>>>>>>>>>>>>>>>\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 70%|███████   | 7/10 [02:54<01:14, 24.94s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Early stopped! \n",
      ">>>>>>> Training Turbine  32 >>>>>>>>>>>>>>>>>>>>>>>>>>\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 60%|██████    | 6/10 [02:32<01:41, 25.40s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Early stopped! \n",
      ">>>>>>> Training Turbine  33 >>>>>>>>>>>>>>>>>>>>>>>>>>\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 50%|█████     | 5/10 [02:12<02:12, 26.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Early stopped! \n",
      ">>>>>>> Training Turbine  34 >>>>>>>>>>>>>>>>>>>>>>>>>>\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 30%|███       | 3/10 [01:27<03:24, 29.19s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Early stopped! \n",
      ">>>>>>> Training Turbine  35 >>>>>>>>>>>>>>>>>>>>>>>>>>\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 40%|████      | 4/10 [01:47<02:40, 26.79s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Early stopped! \n",
      ">>>>>>> Training Turbine  36 >>>>>>>>>>>>>>>>>>>>>>>>>>\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 40%|████      | 4/10 [01:46<02:40, 26.72s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Early stopped! \n",
      ">>>>>>> Training Turbine  37 >>>>>>>>>>>>>>>>>>>>>>>>>>\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 90%|█████████ | 9/10 [03:35<00:23, 23.98s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Early stopped! \n",
      ">>>>>>> Training Turbine  84 >>>>>>>>>>>>>>>>>>>>>>>>>>\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 30%|███       | 3/10 [01:26<03:22, 28.87s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Early stopped! \n",
      ">>>>>>> Training Turbine  85 >>>>>>>>>>>>>>>>>>>>>>>>>>\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 50%|█████     | 5/10 [02:09<02:09, 25.84s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Early stopped! \n",
      ">>>>>>> Training Turbine  86 >>>>>>>>>>>>>>>>>>>>>>>>>>\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 40%|████      | 4/10 [01:48<02:42, 27.08s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Early stopped! \n",
      ">>>>>>> Training Turbine  87 >>>>>>>>>>>>>>>>>>>>>>>>>>\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 80%|████████  | 8/10 [03:11<00:47, 23.89s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Early stopped! \n",
      ">>>>>>> Training Turbine  88 >>>>>>>>>>>>>>>>>>>>>>>>>>\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 70%|███████   | 7/10 [02:50<01:13, 24.35s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Early stopped! \n",
      ">>>>>>> Training Turbine  89 >>>>>>>>>>>>>>>>>>>>>>>>>>\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 50%|█████     | 5/10 [02:09<02:09, 26.00s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Early stopped! \n",
      ">>>>>>> Training Turbine  90 >>>>>>>>>>>>>>>>>>>>>>>>>>\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 90%|█████████ | 9/10 [03:34<00:23, 23.86s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Early stopped! \n",
      ">>>>>>> Training Turbine  91 >>>>>>>>>>>>>>>>>>>>>>>>>>\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 70%|███████   | 7/10 [02:50<01:13, 24.43s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Early stopped! \n",
      ">>>>>>> Training Turbine  92 >>>>>>>>>>>>>>>>>>>>>>>>>>\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 30%|███       | 3/10 [01:26<03:21, 28.74s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Early stopped! \n",
      ">>>>>>> Training Turbine  93 >>>>>>>>>>>>>>>>>>>>>>>>>>\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 30%|███       | 3/10 [01:26<03:20, 28.68s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Early stopped! \n",
      ">>>>>>> Training Turbine  94 >>>>>>>>>>>>>>>>>>>>>>>>>>\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 50%|█████     | 5/10 [02:10<02:10, 26.09s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Early stopped! \n",
      ">>>>>>> Training Turbine  95 >>>>>>>>>>>>>>>>>>>>>>>>>>\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "100%|██████████| 10/10 [03:35<00:00, 21.55s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ">>>>>>> Training Turbine  96 >>>>>>>>>>>>>>>>>>>>>>>>>>\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 50%|█████     | 5/10 [02:09<02:09, 25.94s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Early stopped! \n",
      ">>>>>>> Training Turbine  97 >>>>>>>>>>>>>>>>>>>>>>>>>>\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 30%|███       | 3/10 [01:26<03:22, 28.90s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Early stopped! \n",
      ">>>>>>> Training Turbine  98 >>>>>>>>>>>>>>>>>>>>>>>>>>\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 40%|████      | 4/10 [01:48<02:42, 27.07s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Early stopped! \n",
      ">>>>>>> Training Turbine  99 >>>>>>>>>>>>>>>>>>>>>>>>>>\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 50%|█████     | 5/10 [02:09<02:09, 25.89s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Early stopped! \n",
      ">>>>>>> Training Turbine 100 >>>>>>>>>>>>>>>>>>>>>>>>>>\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 40%|████      | 4/10 [01:46<02:40, 26.73s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Early stopped! \n",
      ">>>>>>> Training Turbine 101 >>>>>>>>>>>>>>>>>>>>>>>>>>\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 70%|███████   | 7/10 [02:51<01:13, 24.53s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Early stopped! \n",
      ">>>>>>> Training Turbine 102 >>>>>>>>>>>>>>>>>>>>>>>>>>\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 50%|█████     | 5/10 [02:09<02:09, 25.84s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Early stopped! \n",
      ">>>>>>> Training Turbine 103 >>>>>>>>>>>>>>>>>>>>>>>>>>\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 60%|██████    | 6/10 [02:30<01:40, 25.03s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Early stopped! \n",
      ">>>>>>> Training Turbine 104 >>>>>>>>>>>>>>>>>>>>>>>>>>\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 30%|███       | 3/10 [01:26<03:22, 28.96s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Early stopped! \n",
      ">>>>>>> Training Turbine 105 >>>>>>>>>>>>>>>>>>>>>>>>>>\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 40%|████      | 4/10 [01:45<02:38, 26.45s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Early stopped! \n",
      ">>>>>>> Training Turbine 106 >>>>>>>>>>>>>>>>>>>>>>>>>>\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 60%|██████    | 6/10 [02:30<01:40, 25.05s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Early stopped! \n",
      ">>>>>>> Training Turbine 107 >>>>>>>>>>>>>>>>>>>>>>>>>>\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 50%|█████     | 5/10 [02:08<02:08, 25.68s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Early stopped! \n",
      ">>>>>>> Training Turbine 108 >>>>>>>>>>>>>>>>>>>>>>>>>>\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 50%|█████     | 5/10 [02:08<02:08, 25.66s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Early stopped! \n",
      ">>>>>>> Training Turbine 109 >>>>>>>>>>>>>>>>>>>>>>>>>>\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 50%|█████     | 5/10 [02:09<02:09, 25.87s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Early stopped! \n",
      ">>>>>>> Training Turbine 110 >>>>>>>>>>>>>>>>>>>>>>>>>>\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 50%|█████     | 5/10 [02:08<02:08, 25.72s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Early stopped! \n",
      ">>>>>>> Training Turbine 111 >>>>>>>>>>>>>>>>>>>>>>>>>>\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 30%|███       | 3/10 [01:26<03:21, 28.80s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Early stopped! \n",
      ">>>>>>> Training Turbine 112 >>>>>>>>>>>>>>>>>>>>>>>>>>\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 40%|████      | 4/10 [01:47<02:41, 26.91s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Early stopped! \n",
      ">>>>>>> Training Turbine 113 >>>>>>>>>>>>>>>>>>>>>>>>>>\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 40%|████      | 4/10 [01:46<02:39, 26.50s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Early stopped! \n",
      ">>>>>>> Training Turbine 114 >>>>>>>>>>>>>>>>>>>>>>>>>>\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 70%|███████   | 7/10 [02:48<01:12, 24.05s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Early stopped! \n",
      ">>>>>>> Training Turbine 115 >>>>>>>>>>>>>>>>>>>>>>>>>>\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 50%|█████     | 5/10 [02:07<02:07, 25.48s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Early stopped! \n",
      ">>>>>>> Training Turbine 116 >>>>>>>>>>>>>>>>>>>>>>>>>>\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 40%|████      | 4/10 [01:47<02:41, 26.92s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Early stopped! \n",
      ">>>>>>> Training Turbine 117 >>>>>>>>>>>>>>>>>>>>>>>>>>\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 40%|████      | 4/10 [01:47<02:41, 26.89s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Early stopped! \n",
      ">>>>>>> Training Turbine 118 >>>>>>>>>>>>>>>>>>>>>>>>>>\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 30%|███       | 3/10 [01:25<03:18, 28.37s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Early stopped! \n",
      ">>>>>>> Training Turbine 119 >>>>>>>>>>>>>>>>>>>>>>>>>>\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 30%|███       | 3/10 [01:25<03:20, 28.59s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Early stopped! \n",
      ">>>>>>> Training Turbine 120 >>>>>>>>>>>>>>>>>>>>>>>>>>\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 50%|█████     | 5/10 [02:06<02:06, 25.39s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Early stopped! \n",
      ">>>>>>> Training Turbine 121 >>>>>>>>>>>>>>>>>>>>>>>>>>\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 50%|█████     | 5/10 [02:06<02:06, 25.40s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Early stopped! \n",
      ">>>>>>> Training Turbine 122 >>>>>>>>>>>>>>>>>>>>>>>>>>\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 60%|██████    | 6/10 [02:29<01:39, 24.85s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Early stopped! \n",
      ">>>>>>> Training Turbine 123 >>>>>>>>>>>>>>>>>>>>>>>>>>\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 40%|████      | 4/10 [01:46<02:40, 26.68s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Early stopped! \n",
      ">>>>>>> Training Turbine 124 >>>>>>>>>>>>>>>>>>>>>>>>>>\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 50%|█████     | 5/10 [02:07<02:07, 25.59s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Early stopped! \n",
      ">>>>>>> Training Turbine 125 >>>>>>>>>>>>>>>>>>>>>>>>>>\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 60%|██████    | 6/10 [02:27<01:38, 24.57s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Early stopped! \n",
      ">>>>>>> Training Turbine 126 >>>>>>>>>>>>>>>>>>>>>>>>>>\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 60%|██████    | 6/10 [02:28<01:39, 24.79s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Early stopped! \n",
      ">>>>>>> Training Turbine 127 >>>>>>>>>>>>>>>>>>>>>>>>>>\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 40%|████      | 4/10 [01:46<02:39, 26.52s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Early stopped! \n",
      ">>>>>>> Training Turbine 128 >>>>>>>>>>>>>>>>>>>>>>>>>>\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 90%|█████████ | 9/10 [03:32<00:23, 23.62s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Early stopped! \n",
      ">>>>>>> Training Turbine 129 >>>>>>>>>>>>>>>>>>>>>>>>>>\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 30%|███       | 3/10 [01:24<03:18, 28.29s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Early stopped! \n",
      ">>>>>>> Training Turbine 130 >>>>>>>>>>>>>>>>>>>>>>>>>>\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 30%|███       | 3/10 [01:26<03:20, 28.71s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Early stopped! \n",
      ">>>>>>> Training Turbine 131 >>>>>>>>>>>>>>>>>>>>>>>>>>\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 40%|████      | 4/10 [01:47<02:40, 26.76s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Early stopped! \n",
      ">>>>>>> Training Turbine 132 >>>>>>>>>>>>>>>>>>>>>>>>>>\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 60%|██████    | 6/10 [02:30<01:40, 25.12s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Early stopped! \n",
      ">>>>>>> Training Turbine 133 >>>>>>>>>>>>>>>>>>>>>>>>>>\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 70%|███████   | 7/10 [02:49<01:12, 24.15s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Early stopped! \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# -*-Encoding: utf-8 -*-\n",
    "################################################################################\n",
    "#\n",
    "# Copyright (c) 2022 Baidu.com, Inc. All Rights Reserved\n",
    "#\n",
    "################################################################################\n",
    "\"\"\"\n",
    "Description: Training and Validation\n",
    "Authors: Lu,Xinjiang (luxinjiang@baidu.com)\n",
    "Date:    2022/03/10\n",
    "\"\"\"\n",
    "import os\n",
    "import time\n",
    "import numpy as np\n",
    "from tqdm import tqdm\n",
    "import torch.nn as nn\n",
    "from typing import Callable\n",
    "from torch.utils.data import DataLoader\n",
    "from common import EarlyStopping\n",
    "from common import adjust_learning_rate\n",
    "from common import Experiment\n",
    "from common import traverse_wind_farm\n",
    "from prepare import prep_env\n",
    "\n",
    "\n",
    "def val(experiment, data_loader, criterion):\n",
    "    # type: (Experiment, DataLoader, Callable) -> np.array\n",
    "    \"\"\"\n",
    "    Desc:\n",
    "        Validation function\n",
    "    Args:\n",
    "        experiment:\n",
    "        data_loader:\n",
    "        criterion:\n",
    "    Returns:\n",
    "        The validation loss\n",
    "    \"\"\"\n",
    "    validation_loss = []\n",
    "    for i, (batch_x, batch_y) in enumerate(data_loader):\n",
    "        sample, true = experiment.process_one_batch(batch_x, batch_y)\n",
    "        loss = criterion(sample, true)\n",
    "        validation_loss.append(loss.item())\n",
    "    validation_loss = np.average(validation_loss)\n",
    "    return validation_loss\n",
    "\n",
    "\n",
    "def train_and_val(experiment, model_folder, is_debug=False):\n",
    "    # type: (Experiment, str, bool) -> None\n",
    "    \"\"\"\n",
    "    Desc:\n",
    "        Training and validation\n",
    "    Args:\n",
    "        experiment:\n",
    "        model_folder: folder name of the model\n",
    "        is_debug:\n",
    "    Returns:\n",
    "        None\n",
    "    \"\"\"\n",
    "    args = experiment.get_args()\n",
    "    model = experiment.get_model()\n",
    "    train_data, train_loader = experiment.get_data(flag='train')\n",
    "    val_data, val_loader = experiment.get_data(flag='val')\n",
    "\n",
    "    path_to_model = os.path.join(args[\"checkpoints\"], model_folder)\n",
    "    if not os.path.exists(path_to_model):\n",
    "        os.makedirs(path_to_model)\n",
    "\n",
    "    time_now = time.time()\n",
    "    early_stopping = EarlyStopping(patience=args[\"patience\"], verbose=True)\n",
    "    model_optim = experiment.get_optimizer()\n",
    "    criterion = Experiment.get_criterion()\n",
    "\n",
    "    epoch_start_time = time.time()\n",
    "    model.cuda()\n",
    "    for epoch in tqdm(range(args[\"train_epochs\"])):\n",
    "        iter_count = 0\n",
    "        train_loss = []\n",
    "        model.train()\n",
    "        for i, (batch_x, batch_y) in enumerate(train_loader):\n",
    "            # batch_x.cuda()\n",
    "            # batch_y.cuda()\n",
    "            iter_count += 1\n",
    "            sample, truth = experiment.process_one_batch(batch_x, batch_y)\n",
    "            loss = criterion(sample, truth)\n",
    "            train_loss.append(loss.item())\n",
    "            loss.backward()\n",
    "            # model_optim.minimize(loss)\n",
    "            nn.utils.clip_grad_norm_(model.parameters(), max_norm=50.0, norm_type=2)\n",
    "            model_optim.step()\n",
    "        val_loss = val(experiment, val_loader, criterion)\n",
    "\n",
    "        if is_debug:\n",
    "            train_loss = np.average(train_loss)\n",
    "            epoch_end_time = time.time()\n",
    "            print(\"Epoch: {}, \\nTrain Loss: {}, \\nValidation Loss: {}\".format(epoch, train_loss, val_loss))\n",
    "            print(\"Elapsed time for epoch-{}: {}\".format(epoch, epoch_end_time - epoch_start_time))\n",
    "            epoch_start_time = epoch_end_time\n",
    "\n",
    "        # Early Stopping if needed\n",
    "        early_stopping(val_loss, model, path_to_model, args[\"turbine_id\"])\n",
    "        if early_stopping.early_stop:\n",
    "            print(\"Early stopped! \")\n",
    "            break\n",
    "        adjust_learning_rate(model_optim, epoch + 1, args)\n",
    "\n",
    "print('a')\n",
    "# if __name__ == \"__main__\":\n",
    "#     settings = prep_env()\n",
    "#     #\n",
    "#     # Set up the initial environment\n",
    "#     # Current settings for the model\n",
    "#     cur_setup = '{}_t{}_i{}_o{}_ls{}_train{}_val{}'.format(\n",
    "#         settings[\"filename\"], settings[\"task\"], settings[\"input_len\"], settings[\"output_len\"], settings[\"lstm_layer\"],\n",
    "#         settings[\"train_size\"], settings[\"val_size\"]\n",
    "#     )\n",
    "#     traverse_wind_farm(train_and_val, settings, cur_setup)\n",
    "\n",
    "settings = prep_env()\n",
    "    #\n",
    "    # Set up the initial environment\n",
    "    # Current settings for the model\n",
    "cur_setup = '{}_t{}_i{}_o{}_ls{}_train{}_val{}'.format(\n",
    "        settings[\"filename\"], settings[\"task\"], settings[\"input_len\"], settings[\"output_len\"], settings[\"lstm_layer\"],\n",
    "        settings[\"train_size\"], settings[\"val_size\"]\n",
    "    )\n",
    "traverse_wind_farm(train_and_val, settings, cur_setup)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1a68dc56-c898-4d2b-a7d3-6c6e7d7c7d7e",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "PyTorch-1.8",
   "language": "python",
   "name": "pytorch-1.8"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}