CrowdAnalytics.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Intelligent Crowd Analytics for Robust and Ubiquitous Surveillance - ICARUS\n",
    "### Intelligent Sensing Systems Practice Module 2022\n",
    "\n",
    "- Analytics engine capable of providing persistent monitoring on both indoor and outdoor static CCTVs to provide higher-order insights on crowd social behaviour\n",
    "- Modules include:\n",
    "    - Social Distancing Monitoring (COVID-19)\n",
    "    - Crowd Management (Crowd Counting)\n",
    "    - Crowd Anomaly (Flow Estimation/Stampede Alert)\n",
    "    - Fall Detection (Medical Emergencies)\n",
    "    - Violent Behaviour Detection (Fighting, assualt)\n",
    "\n",
    "Team: Adriel Kuek, Chua Hao Zi, KC lim & Yap Pow Look"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import\n",
    "import torch\n",
    "from IPython.display import Image, clear_output\n",
    "\n",
    "clear_output()\n",
    "print(f\"Setup complete. Using torch {torch.__version__} ({torch.cuda.get_device_properties(0).name if torch.cuda.is_available() else 'CPU'})\")\n",
    "\n",
    "# limit the number of cpus used by high performance libraries\n",
    "import os\n",
    "os.environ[\"OMP_NUM_THREADS\"] = \"1\"\n",
    "os.environ[\"OPENBLAS_NUM_THREADS\"] = \"1\"\n",
    "os.environ[\"MKL_NUM_THREADS\"] = \"1\"\n",
    "os.environ[\"VECLIB_MAXIMUM_THREADS\"] = \"1\"\n",
    "os.environ[\"NUMEXPR_NUM_THREADS\"] = \"1\"\n",
    "\n",
    "import sys\n",
    "sys.path.insert(0, './yolov5')\n",
    "\n",
    "import argparse\n",
    "import os\n",
    "import platform\n",
    "import shutil\n",
    "import time\n",
    "from pathlib import Path\n",
    "import cv2\n",
    "import torch\n",
    "import torch.backends.cudnn as cudnn\n",
    "\n",
    "from yolov5.models.experimental import attempt_load\n",
    "from yolov5.utils.downloads import attempt_download\n",
    "from yolov5.models.common import DetectMultiBackend\n",
    "from yolov5.utils.datasets import LoadImages, LoadStreams\n",
    "from yolov5.utils.general import (LOGGER, check_img_size, non_max_suppression, scale_coords, \n",
    "                                  check_imshow, xyxy2xywh, increment_path)\n",
    "from yolov5.utils.torch_utils import select_device, time_sync\n",
    "from yolov5.utils.plots import Annotator, colors\n",
    "from deep_sort.utils.parser import get_config\n",
    "from deep_sort.deep_sort import DeepSort\n",
    "\n",
    "# FILE = Path(__file__).resolve()\n",
    "ROOT = os.path.abspath('')\n",
    "if str(ROOT) not in sys.path:\n",
    "    sys.path.append(str(ROOT))  # add ROOT to PATH\n",
    "ROOT = Path(os.path.relpath(ROOT, Path.cwd()))  # relative\n",
    "\n",
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "\n",
    "print(sys.path)\n",
    "print(f'Compute Device: {device}')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Initialise Models\n",
    "## YOLOv5 + DeepSORT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "evaluate = True\n",
    "half = True\n",
    "output = 'output/'\n",
    "imgsz = [640, 640]\n",
    "device = select_device(device)\n",
    "\n",
    "# Initialise DeepSORT\n",
    "cfg = get_config()\n",
    "cfg.merge_from_file('deep_sort/configs/deep_sort.yaml')\n",
    "deepsort = DeepSort('osnet_x0_25',\n",
    "                    device,\n",
    "                    max_dist=cfg.DEEPSORT.MAX_DIST,\n",
    "                    max_iou_distance=cfg.DEEPSORT.MAX_IOU_DISTANCE,\n",
    "                    max_age=cfg.DEEPSORT.MAX_AGE,\n",
    "                    n_init=cfg.DEEPSORT.N_INIT,\n",
    "                    nn_budget=cfg.DEEPSORT.NN_BUDGET)\n",
    "\n",
    "# Initialise half precision - Only for CUDA enabled devices\n",
    "half &= device.type != 'cpu'\n",
    "\n",
    "# For MOT16 evaluation - Run multiple inference streams in parallel\n",
    "if not evaluate:\n",
    "    if os.path.exists(output):\n",
    "        pass\n",
    "        shutil.rmtree(output)\n",
    "    os.makedirs(output)\n",
    "\n",
    "# Directories\n",
    "save_dir = increment_path(Path(ROOT) / 'exp', exist_ok=True)\n",
    "save_dir.mkdir(parents=True, exist_ok=True)  # make dir\n",
    "\n",
    "# Load YOLO Model\n",
    "model_yolo = DetectMultiBackend('yolov5/models/crowdhuman_yolov5m.pt',\n",
    "                            device=device,\n",
    "                            dnn=True)\n",
    "stride, names, pt, jit, _ = model_yolo.stride, model_yolo.names, model_yolo.pt, model_yolo.jit, model_yolo.onnx\n",
    "imgsz = check_img_size(imgsz, s=stride)\n",
    "\n",
    "half &= pt and device.type != 'cpu'\n",
    "if pt:\n",
    "    model_yolo.model.half() if half else model_yolo.model.float()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load Functions for Violence Detection\n",
    "### Framework uses C3D with 5 secs worth of video stream"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow.keras.layers import Activation, Conv3D, Dense, Dropout, Flatten, MaxPooling3D\n",
    "from tensorflow.keras.losses import categorical_crossentropy, binary_crossentropy\n",
    "from tensorflow.keras.models import Sequential, Model\n",
    "from tensorflow.keras.models import load_model\n",
    "\n",
    "# Default Parameters\n",
    "c3d_width = 32\n",
    "c3d_height = 32\n",
    "c3d_depth = 10\n",
    "\n",
    "class Datato3D:\n",
    "\n",
    "    def __init__(self, depth, c3d_model):\n",
    "\n",
    "        self.depth = depth\n",
    "        self.c3d_model = c3d_model\n",
    "\n",
    "    def videoto3D(self, videoarray):\n",
    "        '''\n",
    "        input : video numpy data in [nframes, 32,32]\n",
    "        output: vid3D file in [10,32,32] format\n",
    "        '''\n",
    "\n",
    "        vid3D = []\n",
    "        nframes = videoarray.shape[0]\n",
    "\n",
    "        if (nframes >= self.depth):\n",
    "            frames = [x * nframes / self.depth for x in range(self.depth)]\n",
    "        else:\n",
    "            frames = [x for x in range(int(nframes))]\n",
    "\n",
    "        for i in range(len(frames)):\n",
    "            vid3D.append(videoarray[int(frames[i]), :, :])\n",
    "\n",
    "        return np.array(vid3D)\n",
    "\n",
    "    def inference(self, video_frames):\n",
    "\n",
    "        category = {0: \"No Violence\",\n",
    "                    1: \"Violence\"}\n",
    "\n",
    "        # c3D_model = load_model(os.path.join(os.getcwd(), self.model_path))\n",
    "        # c3D_model.load_weights(os.path.join(os.getcwd(), self.weights_path))\n",
    "        vid3D = self.videoto3D(video_frames)\n",
    "        vid3D = np.expand_dims(vid3D, axis=3)\n",
    "        vid3D = vid3D.transpose((1, 2, 0, 3))\n",
    "        vid3D = np.expand_dims(vid3D, axis=0)\n",
    "\n",
    "        y_pred = self.c3d_model.predict(vid3D)\n",
    "\n",
    "        # return category[np.argmax(y_pred)]\n",
    "        return np.argmax(y_pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Initialise C3D model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "c3d_model_path = 'ViolenceDetection_Models/c3d_model.h5'\n",
    "c3d_model_weights = 'ViolenceDetection_Models/c3d_model_120f_weights.h5'\n",
    "\n",
    "c3d_model = load_model(c3d_model_path)\n",
    "c3d_model.load_weights(c3d_model_weights)\n",
    "\n",
    "datato3D = Datato3D(c3d_depth, c3d_model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Video Extraction Module\n",
    "## Extract video segment\n",
    "## Convert video to image frames"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "show_vid = True\n",
    "\n",
    "# Video Source\n",
    "source = 'SampleVideos/TownCentre_short.mp4'\n",
    "\n",
    "# Set Dataloader\n",
    "vid_path, vid_writer = None, None\n",
    "\n",
    "# Check if environment supports image displays\n",
    "if show_vid:\n",
    "    show_vid = check_imshow()\n",
    "    print(f'show_vid: {show_vid}')\n",
    "\n",
    "# Dataloader\n",
    "dataset = LoadImages(source, img_size=imgsz, stride=stride, auto=pt and not jit)\n",
    "bs = 1  # batch_size\n",
    "vid_path, vid_writer = [None] * bs, [None] * bs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get Names and Colours\n",
    "names = model_yolo.module.names if hasattr(model_yolo, 'module') else model_yolo.names\n",
    "\n",
    "# extract filename\n",
    "txt_file_name = source.split('/')[-1].split('.')[0]\n",
    "txt_path = str(Path(save_dir)) + '/' + txt_file_name + '.txt'\n",
    "\n",
    "if pt and device.type != 'cpu':\n",
    "    model_yolo(torch.zeros(1, 3, *imgsz).to(device).type_as(next(model_yolo.model.parameters())))  # warmup\n",
    "dt, seen = [0.0, 0.0, 0.0, 0.0], 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if txt_file_name == 'TownCentre_short':\n",
    "    bool_towncenter = True\n",
    "else:\n",
    "    bool_towncenter = False\n",
    "\n",
    "if txt_file_name == 'AppleStoreShooting_short1':\n",
    "    bool_applestore = True\n",
    "else:\n",
    "    bool_applestore = False"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Configure Detector parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Configuration\n",
    "augment = True          # Augmented inference\n",
    "visual = False          # Visualisation function - Set this to false as default\n",
    "conf_thres = 0.3        # Object confidence threshold\n",
    "iou_thres = 0.5         # IOU Threshold for NMS\n",
    "classes = 0             # Filter for class 0 - Person\n",
    "agnostic_nms = True     # Class agnostic NMS\n",
    "max_det = 1000          # Max number of detections per image\n",
    "save_txt = False\n",
    "save_vid = True"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Social Distance Metrics\n",
    "## SD Violations version 1 - using tracking box centroid framework.\n",
    "## Initialise hyperparameters for social distance model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Skip this cell\n",
    "%script false\n",
    "\n",
    "import socialdistance as sd\n",
    "\n",
    "# Configuration\n",
    "dist_thres = 1.0\n",
    "violation_thres = 5\n",
    "DepthControlFactor = 0.3\n",
    "height_control = 1.7\n",
    "cam = 1\n",
    "\n",
    "# Global Dictionaries\n",
    "socialdistance_dict = {}\n",
    "printed_tracks = []\n",
    "track_pair_list = []\n",
    "count = 0\n",
    "# filter inaccurate yolo bboxs\n",
    "seen_tracks = {}\n",
    "\n",
    "# Initialise Model with input configuration\n",
    "sd = sd.socialdistance(dist_thres, violation_thres, DepthControlFactor, height_control, socialdistance_dict, seen_tracks, count, track_pair_list, printed_tracks)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Iterate across images stored in pytorch dataloader and perform detection and tracking\n",
    "# - Compute Social Distance Violation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Skip this cell\n",
    "%script false\n",
    "\n",
    "with torch.no_grad():\n",
    "    for frame_idx, (path, img, im0s, vid_cap, s) in enumerate(dataset):\n",
    "\n",
    "        # Initialise every frame for socialdistance model\n",
    "        img_centroid = []\n",
    "        trackid_bbox_centroid = []\n",
    "\n",
    "        t1 = time_sync()\n",
    "        img = torch.from_numpy(img).to(device)\n",
    "        img = img.half() if half else img.float()  # uint8 to fp16/32\n",
    "        img /= 255.0  # 0 - 255 to 0.0 - 1.0\n",
    "        if img.ndimension() == 3:\n",
    "            img = img.unsqueeze(0)\n",
    "        t2 = time_sync()\n",
    "        dt[0] += t2 - t1\n",
    "\n",
    "        # Inference\n",
    "        visualize = increment_path(save_dir / Path(path).stem, mkdir=True) if visual else False\n",
    "        pred = model_yolo(img, augment=augment, visualize=visualize)\n",
    "        t3 = time_sync()\n",
    "        dt[1] += t3 - t2\n",
    "\n",
    "        # Apply NMS\n",
    "        pred = non_max_suppression(pred, conf_thres, iou_thres, classes, agnostic_nms, max_det=max_det)\n",
    "        dt[2] += time_sync() - t3\n",
    "\n",
    "        # Process detections\n",
    "        for i, det in enumerate(pred):  # detections per image\n",
    "            seen += 1\n",
    "            p, im0, _ = path, im0s.copy(), getattr(dataset, 'frame', 0)\n",
    "\n",
    "            p = Path(p)  # to Path\n",
    "            save_path = str(save_dir / p.name) # im.jpg, vid.mp4, ...\n",
    "            s += '%gx%g ' % img.shape[2:]  # print string\n",
    "\n",
    "            # annotator = Annotator(im0, line_width=2, pil=not ascii)\n",
    "\n",
    "            if det is not None and len(det):\n",
    "                # Rescale boxes from img_size to im0 size\n",
    "                det[:, :4] = scale_coords(\n",
    "                    img.shape[2:], det[:, :4], im0.shape).round()\n",
    "\n",
    "                # Print results\n",
    "                for c in det[:, -1].unique():\n",
    "                    n = (det[:, -1] == c).sum()  # detections per class\n",
    "                    s += f\"{n} {names[int(c)]}{'s' * (n > 1)}, \"  # add to string\n",
    "\n",
    "                xywhs = xyxy2xywh(det[:, 0:4])\n",
    "                confs = det[:, 4]\n",
    "                clss = det[:, 5]\n",
    "\n",
    "                # pass detections to deepsort\n",
    "                t4 = time_sync()\n",
    "                outputs = deepsort.update(xywhs.cpu(), confs.cpu(), clss.cpu(), im0)\n",
    "                t5 = time_sync()\n",
    "                dt[3] += t5 - t4\n",
    "\n",
    "                # draw boxes for visualization\n",
    "                if len(outputs) > 0:\n",
    "                    for j, (output, conf) in enumerate(zip(outputs, confs)):\n",
    "\n",
    "                        bboxes = output[0:4]\n",
    "                        id = output[4]\n",
    "                        cls = output[5]\n",
    "\n",
    "                        # Compute box centroid - Social Distance\n",
    "                        img_centroid = [int(output[0] + (output[2] - output[0])/2), int(output[1]+ (output[3] - output[1])/2)]\n",
    "                        img_height = output[3] - output[1]\n",
    "\n",
    "                        # Box definitions\n",
    "                        x = int(output[0])\n",
    "                        y = int(output[1])\n",
    "                        w = int(output[2] - output[0])\n",
    "                        h = int(output[3] - output[1])\n",
    "                        bbox_int = x,y,w,h\n",
    "\n",
    "                        trackid_bbox_centroid.append(['{}'.format(id), '{}'.format(cam), bbox_int, img_centroid, img_height])\n",
    "\n",
    "                        c = int(cls)  # integer class\n",
    "                        label = f'ID:{id} ({conf:.2f})'\n",
    "\n",
    "                        # Comment out this portion and draw our own boxes\n",
    "                        # annotator.box_label(bboxes, label, color=colors(c, True))\n",
    "\n",
    "                        if save_txt:\n",
    "                            # to MOT format\n",
    "                            bbox_left = output[0]\n",
    "                            bbox_top = output[1]\n",
    "                            bbox_w = output[2] - output[0]\n",
    "                            bbox_h = output[3] - output[1]\n",
    "                            # Write MOT compliant results to file\n",
    "                            with open(txt_path, 'a') as f:\n",
    "                                f.write(('%g ' * 10 + '\\n') % (frame_idx + 1, id, bbox_left,  # MOT format\n",
    "                                                                bbox_top, bbox_w, bbox_h, -1, -1, -1, -1))\n",
    "\n",
    "                LOGGER.info(f'{s}Done. YOLO:({t3 - t2:.3f}s), DeepSort:({t5 - t4:.3f}s)')\n",
    "\n",
    "            else:\n",
    "                deepsort.increment_ages()\n",
    "                LOGGER.info('No detections')\n",
    "\n",
    "            # # Stream results\n",
    "            # im0 = annotator.result()\n",
    "            # if show_vid:\n",
    "            #     cv2.imshow(str(p), im0)\n",
    "            #     if cv2.waitKey(1) == ord('q'):  # q to quit\n",
    "            #         raise StopIteration\n",
    "\n",
    "            # Save results (image with detections)\n",
    "            # if save_vid:\n",
    "            #     if vid_path != save_path:  # new video\n",
    "            #         vid_path = save_path\n",
    "            #         if isinstance(vid_writer, cv2.VideoWriter):\n",
    "            #             vid_writer.release()  # release previous video writer\n",
    "            #         if vid_cap:  # video\n",
    "            #             fps = vid_cap.get(cv2.CAP_PROP_FPS)\n",
    "            #             w = int(vid_cap.get(cv2.CAP_PROP_FRAME_WIDTH))\n",
    "            #             h = int(vid_cap.get(cv2.CAP_PROP_FRAME_HEIGHT))\n",
    "            #         else:  # stream\n",
    "            #             fps, w, h = 30, im0.shape[1], im0.shape[0]\n",
    "\n",
    "            #         vid_writer = cv2.VideoWriter(save_path, cv2.VideoWriter_fourcc(*'mp4v'), fps, (w, h))\n",
    "            #     vid_writer.write(im0)\n",
    "\n",
    "        img_out = sd.proximity_evaluation(trackid_bbox_centroid, im0, cam)\n",
    "\n",
    "        # Plot results\n",
    "        cv2.imshow(str(p), img_out)\n",
    "        if cv2.waitKey(1) == ord('q'):  # q to quit\n",
    "            raise StopIteration\n",
    "\n",
    "        # Save Video\n",
    "        if save_vid:\n",
    "            if vid_path != save_path:  # new video\n",
    "                vid_path = save_path\n",
    "                if isinstance(vid_writer, cv2.VideoWriter):\n",
    "                    vid_writer.release()  # release previous video writer\n",
    "                if vid_cap:  # video\n",
    "                    fps = vid_cap.get(cv2.CAP_PROP_FPS)\n",
    "                    w = int(vid_cap.get(cv2.CAP_PROP_FRAME_WIDTH))\n",
    "                    h = int(vid_cap.get(cv2.CAP_PROP_FRAME_HEIGHT))\n",
    "                else:  # stream\n",
    "                    fps, w, h = 30, img_out.shape[1], img_out.shape[0]\n",
    "\n",
    "                vid_writer = cv2.VideoWriter(save_path, cv2.VideoWriter_fourcc(*'mp4v'), fps, (w, h))\n",
    "            vid_writer.write(img_out)\n",
    "    \n",
    "    vid_cap.release()\n",
    "    vid_writer.release()\n",
    "    cv2.destroyAllWindows()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Automated Calibration-free BEV Projection\n",
    "- Manual Labour\n",
    "- Automated Bird's eye view through CNN\n",
    "- Intrinsic camera parameters from Camcalib (SPEC)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import sys\n",
    "import torch\n",
    "import joblib\n",
    "import argparse\n",
    "import numpy as np\n",
    "from tqdm import tqdm\n",
    "from loguru import logger\n",
    "from skimage.io import imsave\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import cv2\n",
    "\n",
    "from scipy.special import softmax\n",
    "\n",
    "sys.path.append('')\n",
    "from camcalib.vis_utils import show_horizon_line\n",
    "from camcalib.model import CameraRegressorNetwork\n",
    "from camcalib.pano_dataset import CameraRegressorDataset, ImageFolder\n",
    "from camcalib.cam_utils import bins2vfov, bins2pitch, bins2roll, convert_preds_to_angles\n",
    "\n",
    "from pare.utils.image_utils import denormalize_images\n",
    "from pare.utils.train_utils import load_pretrained_model\n",
    "\n",
    "CKPT = 'spec_data/camcalib/checkpoints/camcalib_sa_biased_l2.ckpt'\n",
    "\n",
    "screenshot_dir = 'screenshots/'\n",
    "if not os.path.exists(screenshot_dir):\n",
    "    os.mkdir(screenshot_dir)\n",
    "\n",
    "# Extract 1st frame from video and save it to perform camera parameters prediction\n",
    "vidcap = cv2.VideoCapture(source)\n",
    "success,image = vidcap.read()\n",
    "\n",
    "# Save as jpg\n",
    "filename_base = os.path.basename(source)\n",
    "filename = os.path.splitext(filename_base)[0] + '.jpg'\n",
    "\n",
    "if success:\n",
    "    cv2.imwrite(os.path.join(screenshot_dir,filename), image)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch.utils.data import Dataset, DataLoader\n",
    "import torchvision.transforms as transforms\n",
    "\n",
    "class ImageFolder(Dataset):\n",
    "    def __init__(\n",
    "            self,\n",
    "            image_list,\n",
    "            min_size=600,\n",
    "            max_size=1000,\n",
    "    ):\n",
    "        self.image_filenames = image_list\n",
    "\n",
    "        self.data_transform = transforms.Compose([\n",
    "            transforms.Resize(min_size),\n",
    "            # transforms.CenterCrop(224),\n",
    "            transforms.ToTensor(),\n",
    "            transforms.Normalize(mean=[0.485, 0.456, 0.406],\n",
    "                                 std=[0.229, 0.224, 0.225])\n",
    "        ])\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.image_filenames)\n",
    "\n",
    "    def __getitem__(self, index):\n",
    "        item = {}\n",
    "\n",
    "        imgname = os.path.join(self.image_filenames[index])\n",
    "\n",
    "        pil_img = Image.open(imgname).convert('RGB')\n",
    "        orig_img_shape = pil_img.size\n",
    "        norm_img = self.data_transform(pil_img)\n",
    "\n",
    "        item['img'] = norm_img\n",
    "        item['imgname'] = imgname\n",
    "\n",
    "        item['orig_shape'] = orig_img_shape\n",
    "\n",
    "        return item"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prediction of camera intrinsic and extrinsic parameters\n",
    "-vfov, focal length, tilt and roll"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from PIL import Image\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "loss_type = 'softargmax_l2'\n",
    "\n",
    "with torch.no_grad():\n",
    "\n",
    "    # Create dataloader list\n",
    "    dataloader = []\n",
    "\n",
    "    # Load image into dataloader\n",
    "    image_list = [str(os.path.join(screenshot_dir,filename))]\n",
    "\n",
    "    val_dataset = ImageFolder(image_list)\n",
    "\n",
    "    model = CameraRegressorNetwork(\n",
    "        backbone='resnet50',\n",
    "        num_fc_layers=1,\n",
    "        num_fc_channels=1024,\n",
    "    ).to(device)\n",
    "\n",
    "    camcalib_ckpt = torch.load(CKPT)\n",
    "    camcalib_model = load_pretrained_model(model, camcalib_ckpt['state_dict'], remove_lightning=True, strict=True)\n",
    "\n",
    "    logger.info('Loaded pretrained model')\n",
    "    model.eval()\n",
    "\n",
    "    focal_length = []\n",
    "\n",
    "    logger.info('Running CamCalib')\n",
    "\n",
    "    # Extract camera parameters across batch images (if multiple images)\n",
    "    for idx, batch in enumerate(tqdm(val_dataset)):\n",
    "\n",
    "        img_fname = batch['imgname']\n",
    "\n",
    "        image_spec = batch['img'].unsqueeze(0).to(device).float()\n",
    "\n",
    "        preds = camcalib_model(image_spec)\n",
    "        preds_distributions = preds\n",
    "\n",
    "        batch_img = image_spec\n",
    "        batch_img = denormalize_images(batch_img) * 255\n",
    "        batch_img = np.transpose(batch_img.cpu().numpy(), (0,2,3,1))\n",
    "\n",
    "        extract = lambda x:x.detach().cpu().numpy().squeeze()\n",
    "        img = batch_img[0].copy()\n",
    "\n",
    "        if loss_type in ('kl', 'ce'):\n",
    "            pred_vfov, pred_pitch, pred_roll = map(extract, preds)\n",
    "            pred_vfov, pred_pitch, pred_roll = convert_preds_to_angles(\n",
    "                                                pred_vfov, pred_pitch, pred_roll, loss_type=loss_type,\n",
    "                                                return_type='np',)\n",
    "        else:\n",
    "            preds = convert_preds_to_angles(*preds, loss_type=loss_type,)\n",
    "            pred_vfov = extract(preds[0])\n",
    "            pred_pitch = extract(preds[1])\n",
    "            pred_roll = extract(preds[2])\n",
    "        \n",
    "        orig_img_w, orig_img_h = batch['orig_shape']\n",
    "\n",
    "        pred_f_pix = orig_img_h / 2. / np.tan(pred_vfov/2.)\n",
    "\n",
    "        pitch = np.degrees(pred_pitch)\n",
    "        roll = np.degrees(pred_roll)\n",
    "        vfov = np.degrees(pred_vfov)\n",
    "\n",
    "        # Save results into dictionary\n",
    "        results = {}\n",
    "        results['vfov'] = float(pred_vfov)\n",
    "        results['f_pix'] = float(pred_f_pix)\n",
    "        results['pitch'] = float(pred_pitch)\n",
    "        results['roll'] = float(pred_roll)\n",
    "\n",
    "        print(\"VFOV: {} rad\".format(results['vfov']))\n",
    "        print(\"Focal Length: {} px\".format(results['f_pix']))\n",
    "        print(\"Cam Pitch: {} rad\".format(results['pitch']))\n",
    "        print(\"Cam Roll: {} rad\".format(results['roll']))\n",
    "       \n",
    "        focal_length.append(pred_f_pix)\n",
    "\n",
    "        # Display horizontal line\n",
    "        img, _ = show_horizon_line(img.copy(), pred_vfov, pred_pitch, pred_roll, focal_length=pred_f_pix,\n",
    "                                   debug=True, color=(255, 0, 0), width=3, GT=False)\n",
    "        \n",
    "        plt.figure(figsize=[15,15])\n",
    "        plt.imshow(img)\n",
    "        plt.show()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Geometric Projection Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from utils_geometry.transformations import rotation_matrix\n",
    "\n",
    "def modified_matrices_calculate_range_output_without_translation(height, width, overhead_hmatrix,\n",
    "                                                                 verbose=False):\n",
    "    range_u = np.array([np.inf, -np.inf])\n",
    "    range_v = np.array([np.inf, -np.inf])\n",
    "\n",
    "    i = 0\n",
    "    j = 0\n",
    "    u, v, w = np.dot(overhead_hmatrix, [j, i, 1])\n",
    "    u = u / w\n",
    "    v = v / w\n",
    "    out_upperpixel = v\n",
    "    if verbose:\n",
    "        print(u, v)\n",
    "    range_u[0] = min(u, range_u[0])\n",
    "    range_v[0] = min(v, range_v[0])\n",
    "    range_u[1] = max(u, range_u[1])\n",
    "    range_v[1] = max(v, range_v[1])\n",
    "    i = height - 1\n",
    "    j = 0\n",
    "    u, v, w = np.dot(overhead_hmatrix, [j, i, 1])\n",
    "    u = u / w\n",
    "    v = v / w\n",
    "    out_lowerpixel = v\n",
    "    if verbose:\n",
    "        print(u, v)\n",
    "    range_u[0] = min(u, range_u[0])\n",
    "    range_v[0] = min(v, range_v[0])\n",
    "    range_u[1] = max(u, range_u[1])\n",
    "    range_v[1] = max(v, range_v[1])\n",
    "    i = 0\n",
    "    j = width - 1\n",
    "    u, v, w = np.dot(overhead_hmatrix, [j, i, 1])\n",
    "    u = u / w\n",
    "    v = v / w\n",
    "    if verbose:\n",
    "        print(u, v)\n",
    "    range_u[0] = min(u, range_u[0])\n",
    "    range_v[0] = min(v, range_v[0])\n",
    "    range_u[1] = max(u, range_u[1])\n",
    "    range_v[1] = max(v, range_v[1])\n",
    "    i = height - 1\n",
    "    j = width - 1\n",
    "    u, v, w = np.dot(overhead_hmatrix, [j, i, 1])\n",
    "    u = u / w\n",
    "    v = v / w\n",
    "    if verbose:\n",
    "        print(u, v)\n",
    "    range_u[0] = min(u, range_u[0])\n",
    "    range_v[0] = min(v, range_v[0])\n",
    "    range_u[1] = max(u, range_u[1])\n",
    "    range_v[1] = max(v, range_v[1])\n",
    "\n",
    "    range_u = np.array(range_u, dtype=np.int)\n",
    "    range_v = np.array(range_v, dtype=np.int)\n",
    "\n",
    "    # it means that while transforming, after some bottom lower image was transformed,\n",
    "    # upper output pixels got greater than lower\n",
    "    if out_upperpixel > out_lowerpixel:\n",
    "\n",
    "        # range_v needs to be updated\n",
    "        max_height = height * 3\n",
    "        upper_range = out_lowerpixel\n",
    "        best_lower = upper_range  # since out_lowerpixel was lower value than out_upperpixel\n",
    "        #                           i.e. above in image than out_lowerpixel\n",
    "        x_best_lower = np.inf\n",
    "        x_best_upper = -np.inf\n",
    "\n",
    "        for steps_h in range(2, height):\n",
    "            temp = np.dot(overhead_hmatrix, np.vstack(\n",
    "                (np.arange(0, width), np.ones((1, width)) * (height - steps_h), np.ones((1, width)))))\n",
    "            temp = temp / temp[2, :]\n",
    "\n",
    "            lower_range = temp.min(axis=1)[1]\n",
    "            x_lower_range = temp.min(axis=1)[0]\n",
    "            x_upper_range = temp.max(axis=1)[0]\n",
    "            if x_lower_range < x_best_lower:\n",
    "                x_best_lower = x_lower_range\n",
    "            if x_upper_range > x_best_upper:\n",
    "                x_best_upper = x_upper_range\n",
    "\n",
    "            if (upper_range - lower_range) > max_height:  # enforcing max_height of destination image\n",
    "                lower_range = upper_range - max_height\n",
    "                break\n",
    "            if lower_range > upper_range:\n",
    "                lower_range = best_lower\n",
    "                break\n",
    "            if lower_range < best_lower:\n",
    "                best_lower = lower_range\n",
    "            if verbose:\n",
    "                print(steps_h, lower_range, x_best_lower, x_best_upper)\n",
    "        range_v = np.array([lower_range, upper_range], dtype=np.int)\n",
    "\n",
    "        # for testing\n",
    "        range_u = np.array([x_best_lower, x_best_upper], dtype=np.int)\n",
    "\n",
    "    return range_u, range_v\n",
    "    \n",
    "def get_overhead_hmatrix_from_4cameraparams(fx, fy, my_tilt, my_roll, img_dims, verbose=False):\n",
    "    width, height = img_dims\n",
    "\n",
    "    origin, xaxis, yaxis, zaxis = [0, 0, 0], [1, 0, 0], [0, 1, 0], [0, 0, 1]\n",
    "    K3x3 = np.array([[fx, 0, width / 2],\n",
    "                     [0, fy, height / 2],\n",
    "                     [0, 0, 1]])\n",
    "\n",
    "    inv_K3x3 = np.linalg.inv(K3x3)\n",
    "    if verbose:\n",
    "        print(\"K3x3:\\n\", K3x3)\n",
    "\n",
    "    R_overhead = np.array([[1, 0, 0], [0, -1, 0], [0, 0, -1]])\n",
    "    if verbose:\n",
    "        print(\"R_overhead:\\n\", R_overhead)\n",
    "\n",
    "    R_slant = rotation_matrix((pi / 2) + my_tilt, xaxis)[:3, :3]\n",
    "    if verbose:\n",
    "        print(\"R_slant:\\n\", R_slant)\n",
    "\n",
    "    R_roll = rotation_matrix(my_roll, zaxis)[:3, :3]\n",
    "\n",
    "    middle_rotation = np.dot(R_overhead, np.dot(np.linalg.inv(R_slant), R_roll))\n",
    "\n",
    "    overhead_hmatrix = np.dot(K3x3, np.dot(middle_rotation, inv_K3x3))\n",
    "    est_range_u, est_range_v = modified_matrices_calculate_range_output_without_translation(height, width,\n",
    "                                                                                            overhead_hmatrix,\n",
    "                                                                                            verbose=False)\n",
    "\n",
    "    if verbose:\n",
    "        print(\"Estimated destination range: u=\", est_range_u, \"v=\", est_range_v)\n",
    "\n",
    "    moveup_camera = np.array([[1, 0, -est_range_u[0]], [0, 1, -est_range_v[0]], [0, 0, 1]])\n",
    "    if verbose:\n",
    "        print(\"moveup_camera:\\n\", moveup_camera)\n",
    "\n",
    "    overhead_hmatrix = np.dot(moveup_camera, np.dot(K3x3, np.dot(middle_rotation, inv_K3x3)))\n",
    "    if verbose:\n",
    "        print(\"overhead_hmatrix:\\n\", overhead_hmatrix)\n",
    "\n",
    "    return overhead_hmatrix, est_range_u, est_range_v\n",
    "    \n",
    "def get_scaled_homography(H, target_height, estimated_xrange, estimated_yrange):\n",
    "    # if don't want to scale image, then pass target_height = np.inf\n",
    "\n",
    "    current_height = estimated_yrange[1] - estimated_yrange[0]\n",
    "    target_height = min(target_height, current_height)\n",
    "    (tw, th) = int(np.round((estimated_xrange[1] - estimated_xrange[0]))), int(\n",
    "        np.round((estimated_yrange[1] - estimated_yrange[0])))\n",
    "\n",
    "    tr = target_height / float(th)\n",
    "    target_dim = (int(tw * tr), target_height)\n",
    "\n",
    "    scaling_matrix = np.array([[tr, 0, 0], [0, tr, 0], [0, 0, 1]])\n",
    "    scaled_H = np.dot(scaling_matrix, H)\n",
    "\n",
    "    return scaled_H, target_dim\n",
    "\n",
    "# Resize image function\n",
    "def hconcat_resize_min(im_list, interpolation=cv2.INTER_CUBIC):\n",
    "    h_min = min(im.shape[0] for im in im_list)\n",
    "    im_list_resize = [cv2.resize(im, (int(im.shape[1] * h_min / im.shape[0]), h_min), interpolation=interpolation)\n",
    "                      for im in im_list]\n",
    "    return cv2.hconcat(im_list_resize)\n",
    "\n",
    "# Resize image function\n",
    "def vconcat_resize_min(im_list, interpolation=cv2.INTER_CUBIC):\n",
    "    w_min = min(im.shape[1] for im in im_list)\n",
    "    im_list_resize = [cv2.resize(im, (w_min, int(im.shape[0] * w_min / im.shape[1])), interpolation=interpolation)\n",
    "                      for im in im_list]\n",
    "    return cv2.vconcat(im_list_resize)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from math import radians, pi\n",
    "import numpy as np\n",
    "\n",
    "# Test with different parameters\n",
    "# fx = 2349.595\n",
    "# fy = 1662.2         #1700.688\n",
    "# roll_from_horizon = 0 #-0.4013\n",
    "# my_tilt = 0.628319 #0.43695\n",
    "# orig_width = 1920\n",
    "# orig_height = 1080\n",
    "\n",
    "# Obtain overhead birds-eye-view projection.\n",
    "overhead_hmatrix, est_range_u, est_range_v = get_overhead_hmatrix_from_4cameraparams(fx=results['f_pix'], fy=results['f_pix'],\n",
    "                                                                                         my_tilt=results['vfov'],\n",
    "                                                                                         my_roll=-radians(\n",
    "                                                                                             results['roll']),\n",
    "                                                                                         img_dims=(orig_img_w,\n",
    "                                                                                                   orig_img_h),\n",
    "                                                                                         verbose=True)\n",
    "\n",
    "scaled_overhead_hmatrix, target_dim = get_scaled_homography(overhead_hmatrix, orig_img_h*2, est_range_u, est_range_v)\n",
    "print(f'Target Dimensions: {target_dim}')\n",
    "output_dim = (orig_img_w, orig_img_h)\n",
    "print(f'scaled overhead hmatrix:\\n{scaled_overhead_hmatrix}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import cv2\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# 4 point pairs - p1, p2, p3, p4\n",
    "points_birdseye = [[400,695],[828,770],[325,1181],[716,1263]]\n",
    "points_image = [[634,411],[1316,469],[231,717],[1031,847]]\n",
    "\n",
    "# Get homography matrix through manual annotation\n",
    "h_mat, status = cv2.findHomography(np.array(points_image), np.array(points_birdseye))\n",
    "\n",
    "# Read in image screenshot\n",
    "# img_path = 'homography/OxfordTownCenter.png'\n",
    "img_path = os.path.join(screenshot_dir,filename)\n",
    "print(img_path)\n",
    "img_cv = cv2.imread(img_path)\n",
    "img_cv = cv2.cvtColor(img_cv, cv2.COLOR_BGR2RGB)\n",
    "\n",
    "# Perform perspective warping\n",
    "manual_dim = (1080,1465)\n",
    "overhead_dim = (2662,2160)\n",
    "overhead_dim2 = (2987,1440)\n",
    "overhead_dim3 = (1508,2160)\n",
    "\n",
    "print(f'Homography Matrix: {h_mat}')\n",
    "print(f'Auto Homography Matrix: {scaled_overhead_hmatrix}')\n",
    "\n",
    "warped_image = cv2.warpPerspective(img_cv, scaled_overhead_hmatrix, dsize=target_dim, flags=cv2.INTER_CUBIC)\n",
    "plt.figure(figsize=[10,10])\n",
    "plt.imshow(warped_image)\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Run Social Force Model plots\n",
    "Only for TownCenter_Short video for now<br>\n",
    "Pipeline is not fully real-time yet as social force model requires computation of BOW across entire file stream"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "if bool_towncenter is True:\n",
    "\n",
    "    # Load Force Flows\n",
    "\tforce_flow_path = 'TownCentre_flow/video_flowvector_TownCentre_0.3.npy'\n",
    "\tforce_flow = np.load(force_flow_path)\n",
    "\n",
    "\t# Load likelihoods\n",
    "\txdata = np.load('TownCentre_flow/TownCentre_liks_xdata.npy')\n",
    "\tydata = np.load('TownCentre_flow/TownCentre_liks_ydata.npy')\n",
    "\n",
    "if bool_applestore is True:\n",
    "\n",
    "    # Load Force Flows\n",
    "\tforce_flow_path = 'AppleStore_flow/video_flowvector_AppleStoreShooting_01_0.5.npy'\n",
    "\tforce_flow = np.load(force_flow_path)\n",
    "\n",
    "\t# Load likelihoods\n",
    "\txdata = np.load('AppleStore_flow/AppleStoreShooting_01_liks_xdata.npy')\n",
    "\tydata = np.load('AppleStore_flow/AppleStoreShooting_01_liks_ydata.npy')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Run Analytics Pipeline\n",
    "- Detector and Tracker\n",
    "- Homography transformation to birds eye view\n",
    "- Social Distancing violation based on BEV projection\n",
    "- Crowd Violence Inference\n",
    "- Anomalies log-likelihood analysis through LDA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\", category=DeprecationWarning)\n",
    "\n",
    "import cv2\n",
    "import pyshine as ps\n",
    "import scipy\n",
    "import numpy as np\n",
    "from collections import deque\n",
    "import itertools\n",
    "import math\n",
    "\n",
    "# Test LOF algorithm\n",
    "from sklearn.neighbors import LocalOutlierFactor\n",
    "\n",
    "COLOR_RED = (0, 0, 255)\n",
    "COLOR_GREEN = (0, 255, 0)\n",
    "COLOR_BLUE = (255, 0, 0)\n",
    "\n",
    "distance_minimum = 50\n",
    "\n",
    "# Use Auto h_mat\n",
    "# height = 2160\n",
    "# width = 2662\n",
    "height = target_dim[1]\n",
    "width = target_dim[0]\n",
    "scale_resize = 40\n",
    "\n",
    "# Create a deque buffer to store 4 secs worth of frames (120 frames for 30fps) to input into c3d\n",
    "buffer_len = 120\n",
    "c3d_videoInput = deque(maxlen=buffer_len)\n",
    "\n",
    "# buffering parameters to reduce output instability\n",
    "consec_action = 0\n",
    "prev_action = 0\n",
    "\n",
    "# Initialise action output\n",
    "final_action = 0\n",
    "# Initialise SMM Violation\n",
    "smm_volation = 0\n",
    "# Initialise Anomaly Detection\n",
    "anomaly_detected = False\n",
    "liks_thres = -40000\n",
    "\n",
    "fig = plt.figure(figsize=(10,10), dpi=300)\n",
    "plt.grid()\n",
    "plt.ylabel('Scores')\n",
    "plt.xlabel('Frame_idx')\n",
    "plt.title('LDA Log-Likelihood Scores Across Frames')\n",
    "logo = plt.imread('icarus_logo.jpg')\n",
    "fig.figimage(logo, 1100, 1000, alpha=.30, zorder=1)\n",
    "line, = plt.plot(xdata, ydata, color='red')\n",
    "xdata_fig = list()\n",
    "ydata_fig = list()\n",
    "\n",
    "with torch.no_grad():\n",
    "    for frame_idx, (path, img, im0s, vid_cap, s) in enumerate(dataset):\n",
    "        \n",
    "        print(f'FRAME_INDEX: {frame_idx}')\n",
    "        # Copy img frame for c3d processing\n",
    "        img_c3d = img.copy()\n",
    "\n",
    "        # Initialise Violation count back to 0 every new frame\n",
    "        smm_volation = 0\n",
    "\n",
    "        # collect 120 frames for c3d\n",
    "        # if frame_idx <= buffer_len - 1:\n",
    "        if (frame_idx+1) % buffer_len != 0:\n",
    "            img_c3d = img_c3d.transpose((1,2,0))\n",
    "            # resize image frame and convert to grayscale\n",
    "            img_c3d = cv2.resize(img_c3d, (c3d_height, c3d_width))\n",
    "            c3d_videoInput.append(cv2.cvtColor(img_c3d, cv2.COLOR_BGR2GRAY))\n",
    "        else:\n",
    "            startc3d_time = time.time()\n",
    "            # Convert video segment to 3D input\n",
    "            curr_action = datato3D.inference(np.array(c3d_videoInput))\n",
    "            print(f'c3d_compute_time: {time.time() - startc3d_time} secs')\n",
    "            print(f'Immediate_output: {curr_action}')\n",
    "\n",
    "            # 5 consecutive agreeing output before we decide that VD has occurred\n",
    "            # This is to remove frame-by-frame prediction flickering instability\n",
    "            if curr_action == prev_action:\n",
    "                consec_action += 1\n",
    "                action = curr_action\n",
    "            else:\n",
    "                # Reset counter here\n",
    "                consec_action = 0\n",
    "\n",
    "            if consec_action == 3:\n",
    "                final_action = action\n",
    "                # Reset counter here\n",
    "                consec_action = 0\n",
    "                \n",
    "            # Assign to prev state\n",
    "            prev_action = curr_action\n",
    "            img_c3d = img_c3d.transpose((1,2,0))\n",
    "            # resize image frame and convert to grayscale\n",
    "            img_c3d = cv2.resize(img_c3d, (c3d_height, c3d_width))\n",
    "            c3d_videoInput.append(cv2.cvtColor(img_c3d, cv2.COLOR_BGR2GRAY))\n",
    "\n",
    "        # Create a blank map canvas every frame\n",
    "        map_image = np.zeros((height,width,3), np.uint8)\n",
    "        map_pt_array = []\n",
    "        bbox_pt_array = []   \n",
    "\n",
    "        t1 = time_sync()\n",
    "        img = torch.from_numpy(img).to(device)\n",
    "        img = img.half() if half else img.float()  # uint8 to fp16/32\n",
    "        img /= 255.0  # 0 - 255 to 0.0 - 1.0\n",
    "        if img.ndimension() == 3:\n",
    "            img = img.unsqueeze(0)\n",
    "        t2 = time_sync()\n",
    "        dt[0] += t2 - t1\n",
    "\n",
    "        # Inference\n",
    "        visualize = increment_path(save_dir / Path(path).stem, mkdir=True) if visual else False\n",
    "        pred = model_yolo(img, augment=augment, visualize=visualize)\n",
    "        t3 = time_sync()\n",
    "        dt[1] += t3 - t2\n",
    "\n",
    "        # Apply NMS\n",
    "        pred = non_max_suppression(pred, conf_thres, iou_thres, classes, agnostic_nms, max_det=max_det)\n",
    "        dt[2] += time_sync() - t3\n",
    "\n",
    "        # Process detections\n",
    "        for i, det in enumerate(pred):  # detections per image\n",
    "            seen += 1\n",
    "            p, im0, _ = path, im0s.copy(), getattr(dataset, 'frame', 0)\n",
    "\n",
    "            p = Path(p)  # to Path\n",
    "            save_path = str(save_dir / p.name)  # im.jpg, vid.mp4, ...\n",
    "            s += '%gx%g ' % img.shape[2:]  # print string\n",
    "\n",
    "            # annotator = Annotator(im0, line_width=2, pil=not ascii)      \n",
    "\n",
    "            if det is not None and len(det):\n",
    "                # Rescale boxes from img_size to im0 size\n",
    "                det[:, :4] = scale_coords(\n",
    "                    img.shape[2:], det[:, :4], im0.shape).round()\n",
    "\n",
    "                # Print results\n",
    "                for c in det[:, -1].unique():\n",
    "                    n = (det[:, -1] == c).sum()  # detections per class\n",
    "                    s += f\"{n} {names[int(c)]}{'s' * (n > 1)}, \"  # add to string\n",
    "\n",
    "                xywhs = xyxy2xywh(det[:, 0:4])\n",
    "                confs = det[:, 4]\n",
    "                clss = det[:, 5]\n",
    "\n",
    "                # pass detections to deepsort\n",
    "                t4 = time_sync()\n",
    "                outputs = deepsort.update(xywhs.cpu(), confs.cpu(), clss.cpu(), im0)\n",
    "                num_persons = len(outputs)\n",
    "                t5 = time_sync()\n",
    "                dt[3] += t5 - t4\n",
    "\n",
    "                # draw boxes for visualization\n",
    "                if len(outputs) > 0:\n",
    "                    for j, (output, conf) in enumerate(zip(outputs, confs)):\n",
    "\n",
    "                        bboxes = output[0:4]\n",
    "                        id = output[4]\n",
    "                        cls = output[5]\n",
    "\n",
    "                        # Box definitions\n",
    "                        x = int(output[0])\n",
    "                        y = int(output[1])\n",
    "                        w = int(output[2] - output[0])\n",
    "                        h = int(output[3] - output[1])\n",
    "                        bbox_int = x,y,w,h\n",
    "\n",
    "                        c = int(cls)  # integer class\n",
    "                        label = f'ID:{id} ({conf:.2f})'\n",
    "                        # print(im0.shape)\n",
    "                        # Write Track ID on target base\n",
    "                        cv2.putText(im0,str(id),(x,y+h+20), 0, 400*1e-3*2, [255,255,255], 2)\n",
    "\n",
    "                        # Comment out this portion and draw our own boxes\n",
    "                        # annotator.box_label(bboxes, label, color=colors(c, True))\n",
    "\n",
    "                        # Compute box base - Homography transformation\n",
    "                        bbox_pt = np.array([(int(output[0] + (output[2] - output[0])/2), int(output[1]))], dtype='float32')\n",
    "                        bbox_pt = np.array([bbox_pt], dtype='float32')\n",
    "                        bbox_pt_array.append((int(output[0]), int(output[1]), int(output[2]), int(output[3])))\n",
    "\n",
    "                        # Obtain corresponding birdseye point transformation\n",
    "                        # - manual h_mat\n",
    "                        # - auto h_mat (overhead_hmat)\n",
    "                        # birdseye_pt = cv2.perspectiveTransform(bbox_pt, h_mat)\n",
    "                        birdseye_pt = cv2.perspectiveTransform(bbox_pt, scaled_overhead_hmatrix)                        \n",
    "                        map_pt = (int(birdseye_pt[0][0][0]), int(birdseye_pt[0][0][1]))\n",
    "                        map_pt_array.append(map_pt)\n",
    "\n",
    "                        # Draw point on 2d plane\n",
    "                        cv2.circle(map_image, map_pt, 20, COLOR_GREEN, -2)\n",
    "                        # box_pt = (int(bbox_pt[0][0][0]), int(bbox_pt[0][0][1]))\n",
    "                        # cv2.circle(map_image, box_pt, 20, (0,255,0), -2)\n",
    "                        \n",
    "\n",
    "                        if save_txt:\n",
    "                            # to MOT format\n",
    "                            bbox_left = output[0]\n",
    "                            bbox_top = output[1]\n",
    "                            bbox_w = output[2] - output[0]\n",
    "                            bbox_h = output[3] - output[1]\n",
    "                            # Write MOT compliant results to file\n",
    "                            with open(txt_path, 'a') as f:\n",
    "                                f.write(('%g ' * 10 + '\\n') % (frame_idx + 1, id, bbox_left,  # MOT format\n",
    "                                                                bbox_top, bbox_w, bbox_h, -1, -1, -1, -1))\n",
    "\n",
    "                LOGGER.info(f'{s}Done. YOLO:({t3 - t2:.3f}s), DeepSort:({t5 - t4:.3f}s)')\n",
    "\n",
    "                #************** SMM Violations based on BEV Projections **************\n",
    "                # Iterate over every possible 2 by 2 between the points combinations \n",
    "                list_indexes = list(itertools.combinations(range(len(map_pt_array)), 2))\n",
    "                for i,pair in enumerate(itertools.combinations(map_pt_array, r=2)):\n",
    "\n",
    "                    # Check if the distance between each combination of points is less than the minimum distance chosen\n",
    "                    if math.sqrt( (pair[0][0] - pair[1][0])**2 + (pair[0][1] - pair[1][1])**2 ) < int(distance_minimum):\n",
    "                        \n",
    "                        # Change the colors of the points that are too close from each other to red\n",
    "                        if not (pair[0][0] > width or pair[0][0] < 0 or pair[0][1] > height+200  or pair[0][1] < 0 or pair[1][0] > width or pair[1][0] < 0 or pair[1][1] > height+200  or pair[1][1] < 0):\n",
    "                            \n",
    "                            # For every violation pair, we add a count to the SMM violation\n",
    "                            smm_volation += 1\n",
    "\n",
    "                            # Change colours to RED\n",
    "                            cv2.circle(map_image, (pair[0][0],pair[0][1]), 60, COLOR_RED, 2)\n",
    "                            cv2.circle(map_image, (pair[0][0],pair[0][1]), 20, COLOR_RED, -2)\n",
    "                            cv2.circle(map_image, (pair[1][0],pair[1][1]), 60, COLOR_RED, 2)\n",
    "                            cv2.circle(map_image, (pair[1][0],pair[1][1]), 20, COLOR_RED, -2)\n",
    "\n",
    "                            # Get the equivalent indexes of these points in the original frame and change the color to red\n",
    "                            index_pt1 = list_indexes[i][0]\n",
    "                            index_pt2 = list_indexes[i][1]\n",
    "                            \n",
    "                            # Highlight violations in red rectangle boxes\n",
    "                            cv2.rectangle(im0, (bbox_pt_array[index_pt1][0],bbox_pt_array[index_pt1][1]),(bbox_pt_array[index_pt1][2],bbox_pt_array[index_pt1][3]), COLOR_RED, 2)\n",
    "                            cv2.rectangle(im0, (bbox_pt_array[index_pt2][0],bbox_pt_array[index_pt2][1]),(bbox_pt_array[index_pt2][2],bbox_pt_array[index_pt2][3]), COLOR_RED, 2)\n",
    "\n",
    "                #************** LOCAL OUTLIER FACTOR **************\n",
    "                # For testing only\n",
    "\n",
    "                # Fit model for LOF detection\n",
    "                # if len(map_pt_array) != 0:\n",
    "                #     clf = LocalOutlierFactor(n_neighbors=4, metric='euclidean', contamination=0.1)\n",
    "                #     y_pred = clf.fit_predict(map_pt_array)\n",
    "                #     x_scores = clf.negative_outlier_factor_\n",
    "                #     for i in range(len(x_scores)):\n",
    "                #         if x_scores[i] > -1.0:\n",
    "                #             cv2.circle(map_image, map_pt_array[i], 20, (0,0,255), -2)\n",
    "                    \n",
    "                    # Plot circles with radius proportional to the outlier score\n",
    "                    # radius = (x_scores.max() - x_scores) / (x_scores.max() - x_scores.min())\n",
    "                    # radius = (x_scores - x_scores.min()) / (x_scores.max() - x_scores.min())\n",
    "                    # print(f'radius: {radius}')\n",
    "                    # for i in range(len(radius)):\n",
    "                    #     cv2.circle(map_image, map_pt_array[i], int(100*radius[i]), (255,255,255), 5)\n",
    "            else:\n",
    "                deepsort.increment_ages()\n",
    "                LOGGER.info('No detections')\n",
    "            \n",
    "            # Display Output for Crowd Size\n",
    "            text = f'Crowd Size: {num_persons}'\n",
    "            image = ps.putBText(im0,text,text_offset_x=im0.shape[1]-420,text_offset_y=im0.shape[0]-170,\\\n",
    "                                vspace=10,hspace=10, font_scale=1.0,background_RGB=(255,255,255),\\\n",
    "                                text_RGB=(0,0,0))\n",
    "            \n",
    "            # Display output for Social Distancing\n",
    "            text = f'SMM Violations: {smm_volation}'\n",
    "            image = ps.putBText(image,text,text_offset_x=im0.shape[1]-420,text_offset_y=im0.shape[0]-128,\\\n",
    "                                    vspace=10,hspace=10, font_scale=1.0,background_RGB=(255,255,255),\\\n",
    "                                    text_RGB=(0,0,0))\n",
    "\n",
    "            if bool_towncenter is True or bool_applestore is True:\n",
    "                if ydata[frame_idx] <= liks_thres:\n",
    "                    anomaly_detected = True\n",
    "                else:\n",
    "                    anomaly_detected = False\n",
    "\n",
    "            if anomaly_detected == True:\n",
    "                # Display output for Anomaly Detection\n",
    "                text = f'Crowd Anomaly Detected'\n",
    "                image = ps.putBText(image,text,text_offset_x=im0.shape[1]-420,text_offset_y=im0.shape[0]-86,\\\n",
    "                                        vspace=10,hspace=10,font_scale=1.0,background_RGB=(255,255,255),\\\n",
    "                                        text_RGB=(255,0,0))\n",
    "            else:\n",
    "                text = f'No Anomaly Detected'\n",
    "                image = ps.putBText(image,text,text_offset_x=im0.shape[1]-420,text_offset_y=im0.shape[0]-86,\\\n",
    "                                        vspace=10,hspace=10,font_scale=1.0,background_RGB=(255,255,255),\\\n",
    "                                        text_RGB=(0,0,0))                \n",
    "\n",
    "            # Display output for Violence Detection\n",
    "            if final_action == 1:\n",
    "                text  =  'Violence Detected'\n",
    "                image = ps.putBText(image,text,text_offset_x=im0.shape[1]-420,text_offset_y=im0.shape[0]-44,\\\n",
    "                                    vspace=10,hspace=10,font_scale=1.0,background_RGB=(255,255,255),\\\n",
    "                                    text_RGB=(255,0,0))\n",
    "            else:\n",
    "                text  =  'No Violence Detected'\n",
    "                image = ps.putBText(image,text,text_offset_x=im0.shape[1]-420,text_offset_y=im0.shape[0]-44,\\\n",
    "                                    vspace=10,hspace=10,font_scale=1.0,background_RGB=(255,255,255),\\\n",
    "                                    text_RGB=(0,0,0))\n",
    "            \n",
    "            # Plot results by joining the images together\n",
    "            # scale and resize the image smaller for display\n",
    "            # image = im0.copy()  # For testing only - to remove\n",
    "            final_image_joined = hconcat_resize_min([image, map_image])\n",
    "            top_resize_w = int(final_image_joined.shape[1] * scale_resize/100)\n",
    "            top_resize_h = int(final_image_joined.shape[0] * scale_resize/100)\n",
    "            top_resize_dim = (top_resize_w, top_resize_h)\n",
    "            top_resized_image = cv2.resize(final_image_joined, top_resize_dim, interpolation=cv2.INTER_AREA)\n",
    "\n",
    "            if bool_towncenter is True or bool_applestore is True:\n",
    "                #**********************************************************\n",
    "                # Plot and output force flow anomaly detection here\n",
    "                #**********************************************************\n",
    "                # 1st: need to resize force flow vector to be same as img\n",
    "                force_flow_resize = cv2.resize(force_flow[frame_idx], (im0s.shape[1],im0s.shape[0]))\n",
    "                \n",
    "                # 2nd: Create graph to plot likelihoods\n",
    "                xdata_fig.append(frame_idx)\n",
    "                ydata_fig.append(ydata[frame_idx])\n",
    "                line.set_data(xdata_fig, ydata_fig)\n",
    "\n",
    "                # redraw the canvas\n",
    "                fig.canvas.draw()\n",
    "                # # convert canvas to image\n",
    "                graphimg = np.fromstring(fig.canvas.tostring_rgb(), dtype=np.uint8,\n",
    "                        sep='')\n",
    "                graphimg  = graphimg.reshape(fig.canvas.get_width_height()[::-1] + (3,))\n",
    "                # img is rgb, convert to opencv's default bgr\n",
    "                graphimg = cv2.cvtColor(graphimg,cv2.COLOR_RGB2BGR)\n",
    "                graphimg = cv2.resize(graphimg, (width, height))\n",
    "\n",
    "                force_image_joined = hconcat_resize_min([force_flow_resize, graphimg])\n",
    "\n",
    "                bottom_resize_w = int(force_image_joined.shape[1] * scale_resize/100)\n",
    "                bottom_resize_h = int(force_image_joined.shape[0] * scale_resize/100)\n",
    "                bottom_resize_dim = (bottom_resize_w, bottom_resize_h)\n",
    "                bottom_resized_image = cv2.resize(force_image_joined, bottom_resize_dim, interpolation=cv2.INTER_AREA)\n",
    "\n",
    "                # Concat top and bottom together\n",
    "                final_display = vconcat_resize_min([top_resized_image, bottom_resized_image])\n",
    "            else:\n",
    "                final_display = top_resized_image\n",
    "\n",
    "            cv2.imshow(str(p), final_display)\n",
    "            if cv2.waitKey(1) == ord('q'):  # q to quit\n",
    "                raise StopIteration\n",
    "\n",
    "            # # Stream results\n",
    "            # im0 = annotator.result()\n",
    "            # if show_vid:\n",
    "            #     cv2.imshow(str(p), im0)\n",
    "            #     if cv2.waitKey(1) == ord('q'):  # q to quit\n",
    "            #         raise StopIteration\n",
    "        \n",
    "        # img_out = sd.proximity_evaluation(trackid_bbox_centroid, im0, cam)\n",
    "\n",
    "        # # Plot results\n",
    "        # cv2.imshow(str(p), output_img)\n",
    "        # if cv2.waitKey(1) == ord('q'):  # q to quit\n",
    "        #     raise StopIteration\n",
    "\n",
    "            # Save results (image with detections)\n",
    "            if save_vid:\n",
    "                if vid_path != save_path:  # new video\n",
    "                    vid_path = save_path\n",
    "                    if isinstance(vid_writer, cv2.VideoWriter):\n",
    "                        vid_writer.release()  # release previous video writer\n",
    "                    # if vid_cap:  # video\n",
    "                    #     fps = vid_cap.get(cv2.CAP_PROP_FPS)\n",
    "                    #     w = int(vid_cap.get(cv2.CAP_PROP_FRAME_WIDTH))\n",
    "                    #     h = int(vid_cap.get(cv2.CAP_PROP_FRAME_HEIGHT))\n",
    "                    # else:  # stream\n",
    "                    fps, w, h = 30, final_display.shape[1], final_display.shape[0]\n",
    "\n",
    "                    # fourcc = cv2.VideoWriter_fourcc('m', 'p', '4', 'v')\n",
    "                    vid_writer = cv2.VideoWriter(save_path, cv2.VideoWriter_fourcc(*'mp4v'), fps, (w, h), True)\n",
    "                vid_writer.write(final_display)\n",
    "\n",
    "        if bool_towncenter is True or bool_applestore is True:\n",
    "            if frame_idx == len(xdata)-1:\n",
    "                break\n",
    "    \n",
    "    vid_cap.release()\n",
    "    vid_writer.release()\n",
    "    cv2.destroyAllWindows()"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "5309b996a778008cb63ffe24657c166d11607c2f27e123efebc8da9280bc2070"
  },
  "kernelspec": {
   "display_name": "Python 3.7.0 64-bit ('itss': conda)",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.11"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}