Object Detection with TensorFlow

Google's open source intelligent object detection and recognition system makes image search and street view services more accurate.

Related articles:
TensorFlow's entry experience
TensorFlow's handwritten digit recognition MNIST
TensorFlow's object detection
TensorFlow's construction of a character recognition system

(1) Install Protobuf
TensorFlow uses Protocol Buffers internally, and object detection needs to be specially installed.
# yum info protobuf protobuf-compiler
2.5.0 <- version too low requires protobuf 2.6.1 or later
# yum -y install autoconf automake libtool curl make g++ unzip
# cd /usr/local/src/
# wget https://github.com/google/protobuf/archive/v3.3.1.tar.gz -O protobuf-3.3.1.tar.gz
# tar -zxvf protobuf-3.3.1.tar.gz
# cd protobuf-3.3.1
# ./autogen.sh
# ./configure --prefix=/usr/local/protobuf
# make
# make install
# ldconfig
# export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/protobuf/lib
# export LIBRARY_PATH=$LIBRARY_PATH:/usr/local/protobuf/lib
# export PATH=$PATH:/usr/local/protobuf/bin
# protoc --version
libprotoc 3.3.1


(2) Configure the Tensorflow object detection API
# source /usr/local/tensorflow2/bin/activate
# cd /usr/local/tensorflow2/tensorflow-models


Install dependencies
# pip install pillow
# pip install lxml
# pip install jupyter
# pip install matplotlib


Protobuf compilation
# protoc object_detection/protos/*.proto --python_out=.


Set environment variables
# export PYTHONPATH=$PYTHONPATH:`pwd`:`pwd`/slim
# ldconfig


test
# python object_detection/builders/model_builder_test.py

The output OK indicates that the setting is complete

(3) Check the documentation and run the Demo
to use the pre-trained model to detect objects in the image. The official jupyter-based tutorial is provided.
# source /usr/local/tensorflow2/bin/activate
# cd /usr/local/tensorflow2/tensorflow-models/object_detection/
# jupyter notebook --generate-config --allow-root
# python -c 'from notebook.auth import passwd;print(passwd())'
Enter password:123456
Verify password:123456
sha1:7d026454901a:009ae34a09296674d4a13521b80b8527999fd3da
# vi /root/.jupyter/jupyter_notebook_config.py
c.NotebookApp.password = 'sha1:7d026454901a:009ae34a09296674d4a13521b80b8527999fd3da'
# jupyter notebook --ip=127.0.0.1 --allow-root


Visit: http://127.0.0.1:8888/ Open object_detection_tutorial.ipynb.
http://127.0.0.1:8888/notebooks/object_detection_tutorial.ipynb


is to process image1.jpg and image2.jpg in the object_detection/test_images folder by default. If you want to identify other images, you can modify the code of the penultimate Cell:
# TEST_IMAGE_PATHS = [ os.path.join(PATH_TO_TEST_IMAGES_DIR, 'image{}.jpg'.format(i)) for i in range(1, 3) ]
TEST_IMAGE_PATHS = ['<your image path>']


Add 2 lines of code to the last cell:
plt.figure(figsize=IMAGE_SIZE)
plt.imshow(image_np)

->
print(image_path.split('.')[0]+'_labeled.jpg') # Add
plt.figure(figsize=IMAGE_SIZE, dpi=300) # Modify
plt.imshow(image_np)
plt.savefig(image_path.split('.')[0] + '_labeled.jpg') # Add



Then execute each Cell one by one from beginning to end and wait for the result. (Download Model part of the code needs to download files from the Internet is slow!)


After the execution is completed, you can see the result image in the object_detection/test_images folder.
image1_labeled.jpg
image2_labeled.jpg



Compare the official test result map, it can be seen that it has a lot to do with the machine:



(4)

Take a picture of 2008_004037.jpg from ImageNet for the encoding test image, and then change the code in object_detection_tutorial.ipynb to to run the code directly

# vi object_detect_demo.py
# python object_detect_demo.py


import numpy as np
import them
import six.moves.urllib as urllib
import sys
import tarfile
import tensorflow as tf
import zipfile
import matplotlib

# Matplotlib chooses Xwindows backend by default.
matplotlib.use('Agg')

from collections import defaultdict
from io import StringIO
from matplotlib import pyplot as plt
from PIL import Image
from utils import label_map_util
from utils import visualization_utils as vis_util

##################### Download Model
# What model to download.
MODEL_NAME = 'ssd_mobilenet_v1_coco_11_06_2017'
MODEL_FILE = MODEL_NAME + '.tar.gz'
DOWNLOAD_BASE = 'http://download.tensorflow.org/models/object_detection/'

# Path to frozen detection graph. This is the actual model that is used for the object detection.
PATH_TO_CKPT = MODEL_NAME + '/frozen_inference_graph.pb'

# List of the strings that is used to add correct label for each box.
PATH_TO_LABELS = os.path.join('data', 'mscoco_label_map.pbtxt')

NUM_CLASSES = 90

# Download model if not already downloaded
if not os.path.exists(PATH_TO_CKPT):
    print('Downloading model... (This may take over 5 minutes)')
    opener = urllib.request.URLopener()
    opener.retrieve(DOWNLOAD_BASE + MODEL_FILE, MODEL_FILE)
    print('Extracting...')
    tar_file = tarfile.open(MODEL_FILE)
    for file in tar_file.getmembers():
        file_name = os.path.basename(file.name)
        if 'frozen_inference_graph.pb' in file_name:
            tar_file.extract(file, os.getcwd())
else:
    print('Model already downloaded.')

##################### Load a (frozen) Tensorflow model into memory.
print('Loading model...')
detection_graph = tf.Graph()

with detection_graph.as_default():
    od_graph_def = tf.GraphDef()
    with tf.gfile.GFile(PATH_TO_CKPT, 'rb') as fid:
        serialized_graph = fid.read()
        od_graph_def.ParseFromString(serialized_graph)
        tf.import_graph_def(od_graph_def, name='')

##################### Loading label map
print('Loading label map...')
label_map = label_map_util.load_labelmap(PATH_TO_LABELS)
categories = label_map_util.convert_label_map_to_categories(label_map, max_num_classes=NUM_CLASSES, use_display_name=True)
category_index = label_map_util.create_category_index(categories)

##################### Helper code
def load_image_into_numpy_array(image):
  (im_width, im_height) = image.size
  return np.array(image.getdata()).reshape(
      (im_height, im_width, 3)).astype(np.uint8)

##################### Detection
# Path to test image
TEST_IMAGE_PATH = 'test_images/2008_004037.jpg'

# Size, in inches, of the output images.
IMAGE_SIZE = (12, 8)

print('Detecting...')
with detection_graph.as_default():
  with tf.Session(graph=detection_graph) as sess:
    print(TEST_IMAGE_PATH)
    image = Image.open(TEST_IMAGE_PATH)
    image_np = load_image_into_numpy_array(image)
    image_np_expanded = np.expand_dims(image_np, axis=0)
    image_tensor = detection_graph.get_tensor_by_name('image_tensor:0')
    boxes = detection_graph.get_tensor_by_name('detection_boxes:0')
    scores = detection_graph.get_tensor_by_name('detection_scores:0')
    classes = detection_graph.get_tensor_by_name('detection_classes:0')
    num_detections = detection_graph.get_tensor_by_name('num_detections:0')
    # Actual detection.
    (boxes, scores, classes, num_detections) = sess.run(
        [boxes, scores, classes, num_detections],
        feed_dict={image_tensor: image_np_expanded})
    # Print the results of a detection.
    print(scores)
    print(classes)
    print(category_index)
    # Visualization of the results of a detection.
    vis_util.visualize_boxes_and_labels_on_image_array(
        image_np,
        np.squeeze(boxes),
        np.squeeze(classes).astype(np.int32),
        np.squeeze(scores),
        category_index,
        use_normalized_coordinates=True,
        line_thickness=8)
    print(TEST_IMAGE_PATH.split('.')[0]+'_labeled.jpg')
    plt.figure(figsize=IMAGE_SIZE, dpi=300)
    plt.imshow(image_np)
    plt.savefig(TEST_IMAGE_PATH.split('.')[0] + '_labeled.jpg')


The detection results (scores, classes, category_index) are as follows:
quote
[[ 0.91731095  0.80875194  0.67557526  0.67192227  0.3568708   0.23992854
   0.21897335  0.21443138  0.17383011  0.15901341  0.15674619  0.1558814
   0.15265906  0.1489363   0.14805503  0.13470834  0.132047    0.12655555
   0.12086334  0.11752894  0.10897312  0.10791111  0.10386674  0.10181901
   0.09687284  0.09644313  0.0929096   0.09187065  0.08420605  0.08250966
   0.08131051  0.07928694  0.07632151  0.07570603  0.0749495   0.07267584
   0.07258119  0.07075463  0.06964011  0.06901822  0.06894562  0.06892171
   0.06805679  0.06769397  0.06536105  0.06501643  0.06417865  0.06416738
   0.06377003  0.0634084   0.06247949  0.06245064  0.06173467  0.06126672
   0.06037482  0.05930964  0.05813492  0.05751488  0.05747007  0.05746768
   0.05737954  0.05694786  0.05581251  0.05559204  0.05539726  0.054422
   0.05410738  0.05389332  0.05359224  0.05349119  0.05328105  0.05284562
   0.0527565   0.05231072  0.05224103  0.05190464  0.05123441  0.05110639
   0.05002856  0.04982324  0.04956287  0.04943769  0.04906119  0.04891028
   0.04835404  0.04812568  0.0470486   0.04596276  0.04592303  0.04565331
   0.04564101  0.04550403  0.04531116  0.04507401  0.04495776  0.04489629
   0.04475424  0.0447024   0.04434219  0.04395287]]
[[  1.   1.  44.  44.  44.  44.  44.  75.  44.  44.  44.  82.  44.  88.
   79.  44.  44.  44.  88.  44.  88.  79.  44.  82.   1.  47.  88.  67.
   44.  70.  47.  79.  67.  67.  67.  67.  79.  72.  47.   1.  44.  44.
   44.   1.  67.  75.  72.  62.   1.   1.  44.  82.  79.  47.  79.  67.
   44.   1.  51.  75.  79.  51.  79.  62.  67.  44.  82.  82.  79.  82.
   79.  75.  72.  82.   1.   1.  46.  88.  82.  82.  82.  44.  67.  62.
   82.  79.  62.   1.  67.   1.  82.   1.  67.   1.  44.  88.  79.  51.
   44.  82.]]
{1: {'id': 1, 'name': u'person'}, 2: {'id': 2, 'name': u'bicycle'}, 3: {'id': 3, 'name': u'car'}, 4: {'id': 4, 'name': u'motorcycle'}, 5: {'id': 5, 'name': u'airplane'}, 6: {'id': 6, 'name': u'bus'}, 7: {'id': 7, 'name': u'train'}, 8: {'id': 8, 'name': u'truck'}, 9: {'id': 9, 'name': u'boat'}, 10: {'id': 10, 'name': u'traffic light'}, 11: {'id': 11, 'name': u'fire hydrant'}, 13: {'id': 13, 'name': u'stop sign'}, 14: {'id': 14, 'name': u'parking meter'}, 15: {'id': 15, 'name': u'bench'}, 16: {'id': 16, 'name': u'bird'}, 17: {'id': 17, 'name': u'cat'}, 18: {'id': 18, 'name': u'dog'}, 19: {'id': 19, 'name': u'horse'}, 20: {'id': 20, 'name': u'sheep'}, 21: {'id': 21, 'name': u'cow'}, 22: {'id': 22, 'name': u'elephant'}, 23: {'id': 23, 'name': u'bear'}, 24: {'id': 24, 'name': u'zebra'}, 25: {'id': 25, 'name': u'giraffe'}, 27: {'id': 27, 'name': u'backpack'}, 28: {'id': 28, 'name': u'umbrella'}, 31: {'id': 31, 'name': u'handbag'}, 32: {'id': 32, 'name': u'tie'}, 33: {'id': 33, 'name': u'suitcase'}, 34: {'id': 34, 'name': u'frisbee'}, 35: {'id': 35, 'name': u'skis'}, 36: {'id': 36, 'name': u'snowboard'}, 37: {'id': 37, 'name': u'sports ball'}, 38: {'id': 38, 'name': u'kite'}, 39: {'id': 39, 'name': u'baseball bat'}, 40: {'id': 40, 'name': u'baseball glove'}, 41: {'id': 41, 'name': u'skateboard'}, 42: {'id': 42, 'name': u'surfboard'}, 43: {'id': 43, 'name': u'tennis racket'}, 44: {'id': 44, 'name': u'bottle'}, 46: {'id': 46, 'name': u'wine glass'}, 47: {'id': 47, 'name': u'cup'}, 48: {'id': 48, 'name': u'fork'}, 49: {'id': 49, 'name': u'knife'}, 50: {'id': 50, 'name': u'spoon'}, 51: {'id': 51, 'name': u'bowl'}, 52: {'id': 52, 'name': u'banana'}, 53: {'id': 53, 'name': u'apple'}, 54: {'id': 54, 'name': u'sandwich'}, 55: {'id': 55, 'name': u'orange'}, 56: {'id': 56, 'name': u'broccoli'}, 57: {'id': 57, 'name': u'carrot'}, 58: {'id': 58, 'name': u'hot dog'}, 59: {'id': 59, 'name': u'pizza'}, 60: {'id': 60, 'name': u'donut'}, 61: {'id': 61, 'name': u'cake'}, 62: {'id': 62, 'name': u'chair'}, 63: {'id': 63, 'name': u'couch'}, 64: {'id': 64, 'name': u'potted plant'}, 65: {'id': 65, 'name': u'bed'}, 67: {'id': 67, 'name': u'dining table'}, 70: {'id': 70, 'name': u'toilet'}, 72: {'id': 72, 'name': u'tv'}, 73: {'id': 73, 'name': u'laptop'}, 74: {'id': 74, 'name': u'mouse'}, 75: {'id': 75, 'name': u'remote'}, 76: {'id': 76, 'name': u'keyboard'}, 77: {'id': 77, 'name': u'cell phone'}, 78: {'id': 78, 'name': u'microwave'}, 79: {'id': 79, 'name': u'oven'}, 80: {'id': 80, 'name': u'toaster'}, 81: {'id': 81, 'name': u'sink'}, 82: {'id': 82, 'name': u'refrigerator'}, 84: {'id': 84, 'name': u'book'}, 85: {'id': 85, 'name': u'clock'}, 86: {'id': 86, 'name': u'vase'}, 87: {'id': 87, 'name': u'scissors'}, 88: {'id': 88, 'name': u'teddy bear'}, 89: {'id': 89, 'name': u'hair drier'}, 90: {'id': 90, 'name': u'toothbrush'}}name': u'hair drier'}, 90: {'id': 90, 'name': u'toothbrush'}}name': u'hair drier'}, 90: {'id': 90, 'name': u'toothbrush'}}


The results of obtaining the first four objects higher than 50% are as follows:
quote
scores - 0.91731095  0.80875194  0.67557526  0.67192227
classes - 1.   1.  44.  44.
category_index - 1: {'id': 1, 'name': u'person'} 44: {'id': 44, 'name': u'bottle'}

The four objects [91%person, 80%person, 67%bottle, 67%bottle] are also marked in the figure:


(4) Local operation

1) Generate TFRecord
to convert jpg image data into TFRecord data.
# cd /usr/local/tensorflow2/tensorflow-models/object_detection
# wget http://www.robots.ox.ac.uk/~vgg/data/pets/data/images.tar.gz
# wget http://www.robots.ox.ac.uk/~vgg/data/pets/data/annotations.tar.gz
# tar -zxvf annotations.tar.gz
# tar -zxvf images.tar.gz
# python create_pet_tf_record.py --data_dir = `pwd` --output_dir =` pwd`

The images are all marked jpg images. After the execution is completed, two files will be generated in the current directory: pet_train.record and pet_val.record.

2) The configuration pipeline
has channel configurations of various models under object_detection/samples, copy a copy for use.
# wget http://download.tensorflow.org/models/object_detection/faster_rcnn_resnet101_coco_11_06_2017.tar.gz
# tar -zxvf faster_rcnn_resnet101_coco_11_06_2017.tar.gz
# cp samples/configs/faster_rcnn_resnet101_pets.config mypet.config
# vi mypet.config

Modify the PATH_TO_BE_CONFIGURED section as follows:
quote
fine_tune_checkpoint: "/usr/local/tensorflow2/tensorflow-models/object_detection/faster_rcnn_resnet101_coco_11_06_2017/model.ckpt"
from_detection_checkpoint: true

train_input_reader: {
  tf_record_input_reader {
    input_path: "/usr/local/tensorflow2/tensorflow-models/object_detection/pet_train.record"
  }
  label_map_path: "/usr/local/tensorflow2/tensorflow-models/object_detection/data/pet_label_map.pbtxt"
}

eval_input_reader: {
  tf_record_input_reader {
    input_path: "/usr/local/tensorflow2/tensorflow-models/object_detection/pet_val.record"
  }
  label_map_path: "/usr/local/tensorflow2/tensorflow-models/object_detection/data/pet_label_map.pbtxt"
}

from_detection_checkpoint is set to true, fine_tune_checkpoint needs to set the path of the checkpoint. Using checkpoints trained by others can reduce training time.
Checkpoint download address reference:
https://github.com/tensorflow/models/blob/master/object_detection/g3doc/detection_model_zoo.md

3) Training evaluation
# mkdir -p /usr/local/tensorflow2/tensorflow-models/object_detection/model/train
# mkdir -p /usr/local/tensorflow2/tensorflow-models/object_detection/model/eval


-- Training--
# python object_detection/train.py \
     --logtostderr \
     --pipeline_config_path='/usr/local/tensorflow2/tensorflow-models/object_detection/mypet.config' \
     --train_dir='/usr/local/tensorflow2/tensorflow-models/object_detection/model/train'

quote
INFO:tensorflow:Starting Session.
INFO:tensorflow:Saving checkpoint to path /usr/local/tensorflow2/tensorflow-models/object_detection/model/train/model.ckpt
INFO:tensorflow:Starting Queues.
INFO:tensorflow:global_step/sec: 0
INFO:tensorflow:Recording summary at step 0.


-- evaluate--
# python object_detection/eval.py \
    --logtostderr \
    --pipeline_config_path='/usr/local/tensorflow2/tensorflow-models/object_detection/mypet.config' \
    --checkpoint_dir='/usr/local/tensorflow2/tensorflow-models/object_detection/model/train' \
    --eval_dir='/usr/local/tensorflow2/tensorflow-models/object_detection/model/eval'

The following files will be generated in the eval folder, one file corresponds to one image:
events.out.tfevents.1499152949.localhost.localdomain
events.out.tfevents.1499152964.localhost.localdomain
events.out.tfevents.1499152980.localhost.localdomain

-- View Results--
# tensorboard --logdir=/usr/local/tensorflow/tensorflow-models/object_detection/model/


*** After train and eval are executed, it will run until the termination command.
*** Training, evaluation, and viewing can open 3 terminals and run them simultaneously

. The tensorflow-models-master.zip downloaded before June 20 is compatible with Python3. Lots of issues:
https://github.com/tensorflow/models/issues/1597
https://github.com/tensorflow/models/pull/1614/files For
example:

quote
Traceback (most recent call last):
  File "create_pet_tf_record.py", line 213, in <module>
    tf.app.run()
  File "/usr/local/tensorflow/lib/python3.6/site-packages/tensorflow/python/platform/app.py", line 48, in run
    _sys.exit(main(_sys.argv[:1] + flags_passthrough))
  File "create_pet_tf_record.py", line 208, in main
    image_dir, train_examples)
  File "create_pet_tf_record.py", line 177, in create_tf_record
    tf_example = dict_to_tf_example(data, label_map_dict, image_dir)
  File "create_pet_tf_record.py", line 131, in dict_to_tf_example
    'image/filename': dataset_util.bytes_feature(data['filename']),
  File "/usr/local/tensorflow/tensorflow-models/object_detection/utils/dataset_util.py", line 30, in bytes_feature
    return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))
TypeError: 'leonberger_185.jpg' has type str, but expected one of: bytes


quote
Traceback (most recent call last):
  File "object_detection/train.py", line 198, in <module>
    tf.app.run()
  File "/usr/local/tensorflow/lib/python3.6/site-packages/tensorflow/python/platform/app.py", line 48, in run
    _sys.exit(main(_sys.argv[:1] + flags_passthrough))
  File "object_detection/train.py", line 194, in main
    worker_job_name, is_chief, FLAGS.train_dir)
  File "/usr/local/tensorflow/tensorflow-models/object_detection/trainer.py", line 184, in train
    data_augmentation_options)
  File "/usr/local/tensorflow/tensorflow-models/object_detection/trainer.py", line 77,in _create_input_queue
    prefetch_queue_capacity=prefetch_queue_capacity)
  File "/usr/local/tensorflow/tensorflow-models/object_detection/core/batcher.py", line 81, in __init__
    {key: tensor.get_shape() for key, tensor in tensor_dict.iteritems()})
AttributeError: 'dict' object has no attribute 'iteritems'


quote
Traceback (most recent call last):
  File "object_detection/train.py", line 198, in <module>
    tf.app.run()
  File "/usr/local/tensorflow/lib/python3.6/site-packages/tensorflow/python/platform/app.py", line 48, in run
    _sys.exit(main(_sys.argv[:1] + flags_passthrough))
  File "object_detection/train.py", line 194, in main
    worker_job_name, is_chief, FLAGS.train_dir)
  File "/usr/local/tensorflow/tensorflow-models/object_detection/trainer.py", line 184, in train
    data_augmentation_options)
  File "/usr/local/tensorflow/tensorflow-models/object_detection/trainer.py", line 77,in _create_input_queue
    prefetch_queue_capacity=prefetch_queue_capacity)
  File "/usr/local/tensorflow/tensorflow-models/object_detection/core/batcher.py", line 93, in __init__
    num_threads=num_batch_queue_threads)
  File "/usr/local/tensorflow/lib/python3.6/site-packages/tensorflow/python/training/input.py", line 919, in batch
    name=name)
  File "/usr/local/tensorflow/lib/python3.6/site-packages/tensorflow/python/training/input.py", line 697, in _batch
    tensor_list = _as_tensor_list(tensors)
  File "/usr/local/tensorflow/lib/python3.6/site-packages/tensorflow/python/training/input.py", line 385, in _as_tensor_list
    return [tensors[k] for k in sorted(tensors)]
TypeError: '<' not supported between instances of 'tuple' and 'str'

etc

Guess you like

Origin http://43.154.161.224:23101/article/api/json?id=326311062&siteId=291194637