File size: 10,317 Bytes
9375c9a |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 |
// The contents of this file are in the public domain. See LICENSE_FOR_EXAMPLE_PROGRAMS.txt
/*
This is an example illustrating the use of the deep learning tools from the dlib C++
Library. In it, we will show how to do face recognition. This example uses the
pretrained dlib_face_recognition_resnet_model_v1 model which is freely available from
the dlib web site. This model has a 99.38% accuracy on the standard LFW face
recognition benchmark, which is comparable to other state-of-the-art methods for face
recognition as of February 2017.
In this example, we will use dlib to do face clustering. Included in the examples
folder is an image, bald_guys.jpg, which contains a bunch of photos of action movie
stars Vin Diesel, The Rock, Jason Statham, and Bruce Willis. We will use dlib to
automatically find their faces in the image and then to automatically determine how
many people there are (4 in this case) as well as which faces belong to each person.
Finally, this example uses a network with the loss_metric loss. Therefore, if you want
to learn how to train your own models, or to get a general introduction to this loss
layer, you should read the dnn_metric_learning_ex.cpp and
dnn_metric_learning_on_images_ex.cpp examples.
*/
#include <dlib/dnn.h>
#include <dlib/gui_widgets.h>
#include <dlib/clustering.h>
#include <dlib/string.h>
#include <dlib/image_io.h>
#include <dlib/image_processing/frontal_face_detector.h>
using namespace dlib;
using namespace std;
// ----------------------------------------------------------------------------------------
// The next bit of code defines a ResNet network. It's basically copied
// and pasted from the dnn_imagenet_ex.cpp example, except we replaced the loss
// layer with loss_metric and made the network somewhat smaller. Go read the introductory
// dlib DNN examples to learn what all this stuff means.
//
// Also, the dnn_metric_learning_on_images_ex.cpp example shows how to train this network.
// The dlib_face_recognition_resnet_model_v1 model used by this example was trained using
// essentially the code shown in dnn_metric_learning_on_images_ex.cpp except the
// mini-batches were made larger (35x15 instead of 5x5), the iterations without progress
// was set to 10000, and the training dataset consisted of about 3 million images instead of
// 55. Also, the input layer was locked to images of size 150.
template <template <int,template<typename>class,int,typename> class block, int N, template<typename>class BN, typename SUBNET>
using residual = add_prev1<block<N,BN,1,tag1<SUBNET>>>;
template <template <int,template<typename>class,int,typename> class block, int N, template<typename>class BN, typename SUBNET>
using residual_down = add_prev2<avg_pool<2,2,2,2,skip1<tag2<block<N,BN,2,tag1<SUBNET>>>>>>;
template <int N, template <typename> class BN, int stride, typename SUBNET>
using block = BN<con<N,3,3,1,1,relu<BN<con<N,3,3,stride,stride,SUBNET>>>>>;
template <int N, typename SUBNET> using ares = relu<residual<block,N,affine,SUBNET>>;
template <int N, typename SUBNET> using ares_down = relu<residual_down<block,N,affine,SUBNET>>;
template <typename SUBNET> using alevel0 = ares_down<256,SUBNET>;
template <typename SUBNET> using alevel1 = ares<256,ares<256,ares_down<256,SUBNET>>>;
template <typename SUBNET> using alevel2 = ares<128,ares<128,ares_down<128,SUBNET>>>;
template <typename SUBNET> using alevel3 = ares<64,ares<64,ares<64,ares_down<64,SUBNET>>>>;
template <typename SUBNET> using alevel4 = ares<32,ares<32,ares<32,SUBNET>>>;
using anet_type = loss_metric<fc_no_bias<128,avg_pool_everything<
alevel0<
alevel1<
alevel2<
alevel3<
alevel4<
max_pool<3,3,2,2,relu<affine<con<32,7,7,2,2,
input_rgb_image_sized<150>
>>>>>>>>>>>>;
// ----------------------------------------------------------------------------------------
std::vector<matrix<rgb_pixel>> jitter_image(
const matrix<rgb_pixel>& img
);
// ----------------------------------------------------------------------------------------
int main(int argc, char** argv) try
{
if (argc != 2)
{
cout << "Run this example by invoking it like this: " << endl;
cout << " ./dnn_face_recognition_ex faces/bald_guys.jpg" << endl;
cout << endl;
cout << "You will also need to get the face landmarking model file as well as " << endl;
cout << "the face recognition model file. Download and then decompress these files from: " << endl;
cout << "http://dlib.net/files/shape_predictor_5_face_landmarks.dat.bz2" << endl;
cout << "http://dlib.net/files/dlib_face_recognition_resnet_model_v1.dat.bz2" << endl;
cout << endl;
return 1;
}
// The first thing we are going to do is load all our models. First, since we need to
// find faces in the image we will need a face detector:
frontal_face_detector detector = get_frontal_face_detector();
// We will also use a face landmarking model to align faces to a standard pose: (see face_landmark_detection_ex.cpp for an introduction)
shape_predictor sp;
deserialize("shape_predictor_5_face_landmarks.dat") >> sp;
// And finally we load the DNN responsible for face recognition.
anet_type net;
deserialize("dlib_face_recognition_resnet_model_v1.dat") >> net;
matrix<rgb_pixel> img;
load_image(img, argv[1]);
// Display the raw image on the screen
image_window win(img);
// Run the face detector on the image of our action heroes, and for each face extract a
// copy that has been normalized to 150x150 pixels in size and appropriately rotated
// and centered.
std::vector<matrix<rgb_pixel>> faces;
for (auto face : detector(img))
{
auto shape = sp(img, face);
matrix<rgb_pixel> face_chip;
extract_image_chip(img, get_face_chip_details(shape,150,0.25), face_chip);
faces.push_back(move(face_chip));
// Also put some boxes on the faces so we can see that the detector is finding
// them.
win.add_overlay(face);
}
if (faces.size() == 0)
{
cout << "No faces found in image!" << endl;
return 1;
}
// This call asks the DNN to convert each face image in faces into a 128D vector.
// In this 128D vector space, images from the same person will be close to each other
// but vectors from different people will be far apart. So we can use these vectors to
// identify if a pair of images are from the same person or from different people.
std::vector<matrix<float,0,1>> face_descriptors = net(faces);
// In particular, one simple thing we can do is face clustering. This next bit of code
// creates a graph of connected faces and then uses the Chinese whispers graph clustering
// algorithm to identify how many people there are and which faces belong to whom.
std::vector<sample_pair> edges;
for (size_t i = 0; i < face_descriptors.size(); ++i)
{
for (size_t j = i; j < face_descriptors.size(); ++j)
{
// Faces are connected in the graph if they are close enough. Here we check if
// the distance between two face descriptors is less than 0.6, which is the
// decision threshold the network was trained to use. Although you can
// certainly use any other threshold you find useful.
if (length(face_descriptors[i]-face_descriptors[j]) < 0.6)
edges.push_back(sample_pair(i,j));
}
}
std::vector<unsigned long> labels;
const auto num_clusters = chinese_whispers(edges, labels);
// This will correctly indicate that there are 4 people in the image.
cout << "number of people found in the image: "<< num_clusters << endl;
// Now let's display the face clustering results on the screen. You will see that it
// correctly grouped all the faces.
std::vector<image_window> win_clusters(num_clusters);
for (size_t cluster_id = 0; cluster_id < num_clusters; ++cluster_id)
{
std::vector<matrix<rgb_pixel>> temp;
for (size_t j = 0; j < labels.size(); ++j)
{
if (cluster_id == labels[j])
temp.push_back(faces[j]);
}
win_clusters[cluster_id].set_title("face cluster " + cast_to_string(cluster_id));
win_clusters[cluster_id].set_image(tile_images(temp));
}
// Finally, let's print one of the face descriptors to the screen.
cout << "face descriptor for one face: " << trans(face_descriptors[0]) << endl;
// It should also be noted that face recognition accuracy can be improved if jittering
// is used when creating face descriptors. In particular, to get 99.38% on the LFW
// benchmark you need to use the jitter_image() routine to compute the descriptors,
// like so:
matrix<float,0,1> face_descriptor = mean(mat(net(jitter_image(faces[0]))));
cout << "jittered face descriptor for one face: " << trans(face_descriptor) << endl;
// If you use the model without jittering, as we did when clustering the bald guys, it
// gets an accuracy of 99.13% on the LFW benchmark. So jittering makes the whole
// procedure a little more accurate but makes face descriptor calculation slower.
cout << "hit enter to terminate" << endl;
cin.get();
}
catch (std::exception& e)
{
cout << e.what() << endl;
}
// ----------------------------------------------------------------------------------------
std::vector<matrix<rgb_pixel>> jitter_image(
const matrix<rgb_pixel>& img
)
{
// All this function does is make 100 copies of img, all slightly jittered by being
// zoomed, rotated, and translated a little bit differently. They are also randomly
// mirrored left to right.
thread_local dlib::rand rnd;
std::vector<matrix<rgb_pixel>> crops;
for (int i = 0; i < 100; ++i)
crops.push_back(jitter_image(img,rnd));
return crops;
}
// ----------------------------------------------------------------------------------------
|