real_time_object_detection-v2.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "V8-yl-s-WKMG"
   },
   "source": [
    "# Real-time Multi-object detection and tracking"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "kFSqkTCdWKMI"
   },
   "source": [
    "# Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "id": "hV4P5gyTWKMI"
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import os\n",
    "import six.moves.urllib as urllib\n",
    "import sys\n",
    "import tarfile\n",
    "import tensorflow as tf\n",
    "import zipfile\n",
    "import cv2\n",
    "import copy\n",
    "import time\n",
    "import datetime\n",
    "from multiprocessing import Queue, Pool\n",
    "from distutils.version import StrictVersion\n",
    "from collections import defaultdict\n",
    "from io import StringIO\n",
    "from matplotlib import pyplot as plt\n",
    "from PIL import Image\n",
    "from tensorflow.core.framework import graph_pb2\n",
    "\n",
    "if StrictVersion(tf.__version__) < StrictVersion('1.9.0'):\n",
    "  raise ImportError('Please upgrade your TensorFlow installation to v1.9.* or later!')\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "r5FNuiRPWKMN"
   },
   "source": [
    "## Object detection imports\n",
    "Here are the imports from the object detection module."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "id": "bm0_uNRnWKMN",
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "utils/visualization_utils.py:26: UserWarning: \n",
      "This call to matplotlib.use() has no effect because the backend has already\n",
      "been chosen; matplotlib.use() must be called *before* pylab, matplotlib.pyplot,\n",
      "or matplotlib.backends is imported for the first time.\n",
      "\n",
      "The backend was *originally* set to 'module://ipykernel.pylab.backend_inline' by the following code:\n",
      "  File \"/usr/lib/python2.7/runpy.py\", line 174, in _run_module_as_main\n",
      "    \"__main__\", fname, loader, pkg_name)\n",
      "  File \"/usr/lib/python2.7/runpy.py\", line 72, in _run_code\n",
      "    exec code in run_globals\n",
      "  File \"/home/nvidia/.local/lib/python2.7/site-packages/ipykernel_launcher.py\", line 16, in <module>\n",
      "    app.launch_new_instance()\n",
      "  File \"/home/nvidia/.local/lib/python2.7/site-packages/traitlets/config/application.py\", line 658, in launch_instance\n",
      "    app.start()\n",
      "  File \"/home/nvidia/.local/lib/python2.7/site-packages/ipykernel/kernelapp.py\", line 499, in start\n",
      "    self.io_loop.start()\n",
      "  File \"/home/nvidia/.local/lib/python2.7/site-packages/tornado/ioloop.py\", line 1073, in start\n",
      "    handler_func(fd_obj, events)\n",
      "  File \"/home/nvidia/.local/lib/python2.7/site-packages/tornado/stack_context.py\", line 300, in null_wrapper\n",
      "    return fn(*args, **kwargs)\n",
      "  File \"/home/nvidia/.local/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py\", line 456, in _handle_events\n",
      "    self._handle_recv()\n",
      "  File \"/home/nvidia/.local/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py\", line 486, in _handle_recv\n",
      "    self._run_callback(callback, msg)\n",
      "  File \"/home/nvidia/.local/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py\", line 438, in _run_callback\n",
      "    callback(*args, **kwargs)\n",
      "  File \"/home/nvidia/.local/lib/python2.7/site-packages/tornado/stack_context.py\", line 300, in null_wrapper\n",
      "    return fn(*args, **kwargs)\n",
      "  File \"/home/nvidia/.local/lib/python2.7/site-packages/ipykernel/kernelbase.py\", line 283, in dispatcher\n",
      "    return self.dispatch_shell(stream, msg)\n",
      "  File \"/home/nvidia/.local/lib/python2.7/site-packages/ipykernel/kernelbase.py\", line 233, in dispatch_shell\n",
      "    handler(stream, idents, msg)\n",
      "  File \"/home/nvidia/.local/lib/python2.7/site-packages/ipykernel/kernelbase.py\", line 399, in execute_request\n",
      "    user_expressions, allow_stdin)\n",
      "  File \"/home/nvidia/.local/lib/python2.7/site-packages/ipykernel/ipkernel.py\", line 208, in do_execute\n",
      "    res = shell.run_cell(code, store_history=store_history, silent=silent)\n",
      "  File \"/home/nvidia/.local/lib/python2.7/site-packages/ipykernel/zmqshell.py\", line 537, in run_cell\n",
      "    return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)\n",
      "  File \"/home/nvidia/.local/lib/python2.7/site-packages/IPython/core/interactiveshell.py\", line 2724, in run_cell\n",
      "    self.events.trigger('post_run_cell')\n",
      "  File \"/home/nvidia/.local/lib/python2.7/site-packages/IPython/core/events.py\", line 74, in trigger\n",
      "    func(*args, **kwargs)\n",
      "  File \"/home/nvidia/.local/lib/python2.7/site-packages/ipykernel/pylab/backend_inline.py\", line 164, in configure_once\n",
      "    activate_matplotlib(backend)\n",
      "  File \"/home/nvidia/.local/lib/python2.7/site-packages/IPython/core/pylabtools.py\", line 315, in activate_matplotlib\n",
      "    matplotlib.pyplot.switch_backend(backend)\n",
      "  File \"/home/nvidia/.local/lib/python2.7/site-packages/matplotlib/pyplot.py\", line 231, in switch_backend\n",
      "    matplotlib.use(newbackend, warn=False, force=True)\n",
      "  File \"/home/nvidia/.local/lib/python2.7/site-packages/matplotlib/__init__.py\", line 1422, in use\n",
      "    reload(sys.modules['matplotlib.backends'])\n",
      "  File \"/home/nvidia/.local/lib/python2.7/site-packages/matplotlib/backends/__init__.py\", line 17, in <module>\n",
      "    line for line in traceback.format_stack()\n",
      "\n",
      "\n",
      "  import matplotlib; matplotlib.use('Agg')  # pylint: disable=multiple-statements\n"
     ]
    }
   ],
   "source": [
    "from utils import label_map_util\n",
    "\n",
    "from utils import visualization_utils as vis_util\n",
    "#from imutils.video import WebcamVideoStream\n",
    "from imutils.video import FPS"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "cfn_tRFOWKMO"
   },
   "source": [
    "# Model preparation "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "X_sEBLpVWKMQ"
   },
   "source": [
    "## Variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "id": "VyPz_t8WWKMQ"
   },
   "outputs": [],
   "source": [
    "#MODEL_NAME = 'ssd_mobilenet_v1_coco_2017_11_17'\n",
    "MODEL_NAME = 'ssd_mobilenet_v11'\n",
    "# Path to frozen detection graph. This is the actual model that is used for the object detection.\n",
    "PATH_TO_FROZEN_GRAPH = MODEL_NAME + '/frozen_inference_graph.pb'\n",
    "\n",
    "# List of the strings that is used to add correct label for each box.\n",
    "PATH_TO_LABELS = os.path.join('data', 'mscoco_label_map.pbtxt')\n",
    "\n",
    "NUM_CLASSES = 90"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Session Config: allow seperate GPU/CPU adressing and limit memory allocation\n",
    "_tf_config = tf.ConfigProto(allow_soft_placement=True)\n",
    "_tf_config.gpu_options.allow_growth=True"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "YBcB9QHLWKMU"
   },
   "source": [
    "## Load a (frozen) Tensorflow model into memory."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "id": "KezjCRVvWKMV"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading frozen model into memory...\n"
     ]
    }
   ],
   "source": [
    "def load_frozenmodel():\n",
    "    \n",
    "    print('Loading frozen model into memory...')\n",
    "    \n",
    "    def _node_name(n):\n",
    "        if n.startswith(\"^\"):\n",
    "            return n[1:]\n",
    "        else:\n",
    "            return n.split(\":\")[0]\n",
    "  \n",
    "    # load a frozen Model and split it into GPU and CPU graphs\n",
    "    # this is Hardcoded for ssd_mobilenet only\n",
    "    input_graph = tf.Graph()\n",
    "    with tf.Session(graph=input_graph):\n",
    "        if ssd_shape == 600:\n",
    "            shape = 7326\n",
    "        else:\n",
    "            shape = 1917\n",
    "            \n",
    "        score = tf.placeholder(tf.float32, shape=(None, shape, NUM_CLASSES), name=\"Postprocessor/convert_scores\")\n",
    "        expand = tf.placeholder(tf.float32, shape=(None, shape, 1, 4), name=\"Postprocessor/ExpandDims_1\")\n",
    "        \n",
    "        for node in input_graph.as_graph_def().node:\n",
    "            if node.name == \"Postprocessor/convert_scores\":\n",
    "                score_def = node\n",
    "            if node.name == \"Postprocessor/ExpandDims_1\":\n",
    "                expand_def = node\n",
    "\n",
    "    detection_graph = tf.Graph()\n",
    "    with detection_graph.as_default():\n",
    "        od_graph_def = tf.GraphDef()\n",
    "        with tf.gfile.GFile(PATH_TO_FROZEN_GRAPH, 'rb') as fid:\n",
    "            serialized_graph = fid.read()\n",
    "            od_graph_def.ParseFromString(serialized_graph)\n",
    "            dest_nodes = ['Postprocessor/convert_scores','Postprocessor/ExpandDims_1']\n",
    "\n",
    "            edges = {}\n",
    "            name_to_node_map = {}\n",
    "            node_seq = {}\n",
    "            seq = 0\n",
    "            for node in od_graph_def.node:\n",
    "                n = _node_name(node.name)\n",
    "                name_to_node_map[n] = node\n",
    "                edges[n] = [_node_name(x) for x in node.input]\n",
    "                node_seq[n] = seq\n",
    "                seq += 1\n",
    "            for d in dest_nodes:\n",
    "                assert d in name_to_node_map, \"%s is not in graph\" % d\n",
    "\n",
    "            nodes_to_keep = set()\n",
    "            next_to_visit = dest_nodes[:]\n",
    "\n",
    "            while next_to_visit:\n",
    "                n = next_to_visit[0]\n",
    "                del next_to_visit[0]\n",
    "                if n in nodes_to_keep: continue\n",
    "                nodes_to_keep.add(n)\n",
    "                next_to_visit += edges[n]\n",
    "\n",
    "            nodes_to_keep_list = sorted(list(nodes_to_keep), key=lambda n: node_seq[n])\n",
    "            nodes_to_remove = set()\n",
    "\n",
    "            for n in node_seq:\n",
    "                if n in nodes_to_keep_list: continue\n",
    "                nodes_to_remove.add(n)\n",
    "            nodes_to_remove_list = sorted(list(nodes_to_remove), key=lambda n: node_seq[n])\n",
    "\n",
    "            keep = graph_pb2.GraphDef()\n",
    "            for n in nodes_to_keep_list:\n",
    "                keep.node.extend([copy.deepcopy(name_to_node_map[n])])\n",
    "\n",
    "            remove = graph_pb2.GraphDef()\n",
    "            remove.node.extend([score_def])\n",
    "            remove.node.extend([expand_def])\n",
    "            for n in nodes_to_remove_list:\n",
    "                remove.node.extend([copy.deepcopy(name_to_node_map[n])])\n",
    "\n",
    "            with tf.device('/gpu:0'):\n",
    "                tf.import_graph_def(keep, name='')\n",
    "            with tf.device('/cpu:0'):\n",
    "                tf.import_graph_def(remove, name='')\n",
    "\n",
    "        return detection_graph, score, expand\n",
    "\n",
    "ssd_shape = 300\n",
    "graph, score, expand = load_frozenmodel()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "_1MVVTcLWKMW"
   },
   "source": [
    "## Loading label map"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "label_map = label_map_util.load_labelmap(PATH_TO_LABELS)\n",
    "categories = label_map_util.convert_label_map_to_categories(label_map, max_num_classes=NUM_CLASSES, use_display_name=True)\n",
    "category_index = label_map_util.create_category_index(categories)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Helper functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "class FPS2:\n",
    "    def __init__(self, interval):\n",
    "        self._glob_start = None\n",
    "        self._glob_end = None\n",
    "        self._glob_numFrames = 0\n",
    "        self._local_start = None\n",
    "        self._local_numFrames = 0\n",
    "        self._interval = interval\n",
    "        self.curr_local_elapsed = None\n",
    "        self.first = False\n",
    "\n",
    "    def start(self):\n",
    "        self._glob_start = datetime.datetime.now()\n",
    "        self._local_start = self._glob_start\n",
    "        return self\n",
    "\n",
    "    def stop(self):\n",
    "        self._glob_end = datetime.datetime.now()\n",
    "\n",
    "    def update(self):\n",
    "        self.first = True\n",
    "        curr_time = datetime.datetime.now()\n",
    "        self.curr_local_elapsed = (curr_time - self._local_start).total_seconds()\n",
    "        self._glob_numFrames += 1\n",
    "        self._local_numFrames += 1\n",
    "        if self.curr_local_elapsed > self._interval:\n",
    "          print(\"> FPS: {}\".format(self.fps_local()))\n",
    "          self._local_numFrames = 0\n",
    "          self._local_start = curr_time\n",
    "\n",
    "    def elapsed(self):\n",
    "        return (self._glob_end - self._glob_start).total_seconds()\n",
    "\n",
    "    def fps(self):\n",
    "        return self._glob_numFrames / self.elapsed()\n",
    "    \n",
    "    def fps_local(self):\n",
    "        if self.first:\n",
    "            return round(self._local_numFrames / self.curr_local_elapsed,1)\n",
    "        else:\n",
    "            return 0.0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Detection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Building the Graph..\n",
      "> Start video stream with shape: 640,480\n",
      "(Press 'q' to Exit)\n",
      "Starting the Detection..\n",
      "> FPS: 0.0\n",
      "> FPS: 18.4\n",
      "> FPS: 17.2\n",
      "> FPS: 18.0\n",
      "> FPS: 17.7\n",
      "> FPS: 17.6\n",
      "> FPS: 17.9\n",
      "> [INFO] elapsed time (total): 55.87\n",
      "> [INFO] approx. FPS: 10.42\n"
     ]
    }
   ],
   "source": [
    "from session_worker import SessionWorker\n",
    "from webcam_stream import WebcamVideoStream\n",
    "import time\n",
    "\n",
    "#Params\n",
    "video_input         = 0\n",
    "visualize           = True\n",
    "vis_text            = True\n",
    "max_frames          = 200\n",
    "width               = 600\n",
    "height              = 600\n",
    "fps_interval        = 5\n",
    "det_interval        = max_frames//10\n",
    "det_th              = 0.5\n",
    "\n",
    "\n",
    "def detection(detection_graph, category_index, score, expand):\n",
    "    print(\"Building the Graph..\")\n",
    "    \n",
    "   \n",
    "    cur_frames = 0\n",
    "    with detection_graph.as_default():\n",
    "        with tf.Session(graph=detection_graph,config=_tf_config) as sess:\n",
    "            # Define Input and Ouput tensors\n",
    "            image_tensor = detection_graph.get_tensor_by_name('image_tensor:0')\n",
    "            detection_boxes = detection_graph.get_tensor_by_name('detection_boxes:0')\n",
    "            detection_scores = detection_graph.get_tensor_by_name('detection_scores:0')\n",
    "            detection_classes = detection_graph.get_tensor_by_name('detection_classes:0')\n",
    "            num_detections = detection_graph.get_tensor_by_name('num_detections:0')\n",
    "            \n",
    "            score_out = detection_graph.get_tensor_by_name('Postprocessor/convert_scores:0')\n",
    "            expand_out = detection_graph.get_tensor_by_name('Postprocessor/ExpandDims_1:0')\n",
    "            score_in = detection_graph.get_tensor_by_name('Postprocessor/convert_scores_1:0')\n",
    "            expand_in = detection_graph.get_tensor_by_name('Postprocessor/ExpandDims_1_1:0')\n",
    "            \n",
    "            # Threading\n",
    "            gpu_worker = SessionWorker(\"GPU\",detection_graph,_tf_config)\n",
    "            cpu_worker = SessionWorker(\"CPU\",detection_graph,_tf_config)\n",
    "            gpu_opts = [score_out, expand_out]\n",
    "            cpu_opts = [detection_boxes, detection_scores, detection_classes, num_detections]\n",
    "            gpu_counter = 0\n",
    "            cpu_counter = 0\n",
    "            \n",
    "            # Start Video Stream and FPS calculation\n",
    "            fps = FPS2(fps_interval).start()\n",
    "            video_stream = WebcamVideoStream(video_input,width,height).start()\n",
    "            cur_frames = 0\n",
    "            print(\"(Press 'q' to Exit)\")\n",
    "            print(\"Starting the Detection..\")\n",
    "            \n",
    "            while video_stream.isActive():\n",
    "               \n",
    "                # split model in seperate gpu and cpu session threads\n",
    "                if gpu_worker.is_sess_empty():\n",
    "                    # read video frame, expand dimensions and convert to rgb\n",
    "                    image = video_stream.read()\n",
    "                    \n",
    "                    image_expanded = np.expand_dims(image, axis=0)\n",
    "                    # put new queue\n",
    "                    gpu_feeds = {image_tensor: image_expanded}\n",
    "                    if visualize:\n",
    "                        gpu_extras = image # for visualization frame\n",
    "                    else:\n",
    "                        gpu_extras = None\n",
    "                    gpu_worker.put_sess_queue(gpu_opts,gpu_feeds,gpu_extras)\n",
    "\n",
    "                g = gpu_worker.get_result_queue()\n",
    "                \n",
    "                if g is None:\n",
    "                    # gpu thread has no output queue. ok skip, let's check cpu thread.\n",
    "                    gpu_counter += 1\n",
    "                \n",
    "                else:\n",
    "                    # gpu thread has output queue.\n",
    "                    gpu_counter = 0\n",
    "                    score,expand,image = g[\"results\"][0],g[\"results\"][1],g[\"extras\"]\n",
    "\n",
    "                    if cpu_worker.is_sess_empty():\n",
    "                        # When cpu thread has no next queue, put new queue.\n",
    "                        # else, drop gpu queue.\n",
    "                        cpu_feeds = {score_in: score, expand_in: expand}\n",
    "                        cpu_extras = image\n",
    "                        cpu_worker.put_sess_queue(cpu_opts,cpu_feeds,cpu_extras)\n",
    "\n",
    "                c = cpu_worker.get_result_queue()\n",
    "                if c is None:\n",
    "                    # cpu thread has no output queue. ok, nothing to do. continue\n",
    "                    cpu_counter += 1\n",
    "                    time.sleep(0.005)\n",
    "                    continue # If CPU RESULT has not been set yet, no fps update\n",
    "                \n",
    "                else:\n",
    "                    cpu_counter = 0\n",
    "                    boxes, scores, classes, num, image = c[\"results\"][0],c[\"results\"][1],c[\"results\"][2],c[\"results\"][3],c[\"extras\"]\n",
    "\n",
    "                # Visualization of the results of a detection.\n",
    "                if visualize:\n",
    "                    vis_util.visualize_boxes_and_labels_on_image_array(\n",
    "                        image,\n",
    "                        np.squeeze(boxes),\n",
    "                        np.squeeze(classes).astype(np.int32),\n",
    "                        np.squeeze(scores),\n",
    "                        category_index,\n",
    "                        use_normalized_coordinates=True,\n",
    "                        line_thickness=8)\n",
    "                    if vis_text:\n",
    "                        cv2.putText(image,\"fps: {}\".format(fps.fps_local()), (10,30),\n",
    "                                    cv2.FONT_HERSHEY_SIMPLEX, 0.75, (77, 255, 9), 2)\n",
    "                    cv2.imshow('object_detection', image)\n",
    "                    # Exit Option\n",
    "                    if cv2.waitKey(1) & 0xFF == ord('q'):\n",
    "                        break\n",
    "                else:\n",
    "                    cur_frames += 1\n",
    "                    # Exit after max frames if no visualization\n",
    "                    for box, score, _class in zip(np.squeeze(boxes), np.squeeze(scores), np.squeeze(classes)):\n",
    "                        if cur_frames%det_interval==0 and score > det_th:\n",
    "                            label = category_index[_class]['name']\n",
    "                            print(\"> label: {}\\nscore: {}\\nbox: {}\".format(label, score, box))\n",
    "                    if cur_frames >= max_frames:\n",
    "                        break\n",
    "                fps.update()\n",
    "\n",
    "   \n",
    "    gpu_worker.stop()\n",
    "    cpu_worker.stop()\n",
    "    fps.stop()\n",
    "    video_stream.stop()\n",
    "    cv2.destroyAllWindows()\n",
    "    print('> [INFO] elapsed time (total): {:.2f}'.format(fps.elapsed()))\n",
    "    print('> [INFO] approx. FPS: {:.2f}'.format(fps.fps()))\n",
    "\n",
    "\n",
    "detection(graph, category_index, score, expand)\n"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "default_view": {},
   "name": "object_detection_tutorial.ipynb?workspaceId=ronnyvotel:python_inference::citc",
   "provenance": [],
   "version": "0.3.2",
   "views": {}
  },
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}