<html><!-- Created using the cpp_pretty_printer from the dlib C++ library. See http://dlib.net for updates. --><head><title>dlib C++ Library - cpu_dlib.cpp</title></head><body bgcolor='white'><pre>
<font color='#009900'>// Copyright (C) 2015 Davis E. King (davis@dlib.net)
</font><font color='#009900'>// License: Boost Software License See LICENSE.txt for the full license.
</font><font color='#0000FF'>#ifndef</font> DLIB_DNN_CPU_cPP_
<font color='#0000FF'>#define</font> DLIB_DNN_CPU_cPP_
<font color='#009900'>// This file contains CPU implementations of the GPU based functions in cuda_dlib.h
</font>
<font color='#0000FF'>#include</font> "<a style='text-decoration:none' href='cpu_dlib.h.html'>cpu_dlib.h</a>"
<font color='#0000FF'>#include</font> "<a style='text-decoration:none' href='tensor_tools.h.html'>tensor_tools.h</a>"
<font color='#0000FF'>#include</font> "<a style='text-decoration:none' href='../image_transforms/interpolation.h.html'>../image_transforms/interpolation.h</a>"
<font color='#0000FF'>#include</font> "<a style='text-decoration:none' href='../threads.h.html'>../threads.h</a>"
<font color='#0000FF'>namespace</font> dlib
<b>{</b>
<font color='#0000FF'>namespace</font> cpu
<b>{</b>
<font color='#009900'>// -----------------------------------------------------------------------------------
</font>
<font color='#0000FF'><u>void</u></font> <b><a name='multiply'></a>multiply</b> <font face='Lucida Console'>(</font>
<font color='#0000FF'><u>bool</u></font> add_to,
tensor<font color='#5555FF'>&</font> dest,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> src1,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> src2
<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>dest.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> src1.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&</font><font color='#5555FF'>&</font> src1.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> src2.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&</font><font color='#5555FF'>&</font>
dest.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> src1.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&</font><font color='#5555FF'>&</font> src1.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> src2.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&</font><font color='#5555FF'>&</font>
dest.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> src1.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&</font><font color='#5555FF'>&</font> src1.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> src2.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font face='Lucida Console'>)</font>;
<font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> MD <font color='#5555FF'>=</font> std::<font color='#BB00BB'>max</font><font face='Lucida Console'>(</font>std::<font color='#BB00BB'>max</font><font face='Lucida Console'>(</font>dest.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,src1.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>,src2.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font><font face='Lucida Console'>(</font>dest.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>=</font><font color='#5555FF'>=</font><font color='#979000'>1</font> <font color='#5555FF'>|</font><font color='#5555FF'>|</font> dest.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>=</font><font color='#5555FF'>=</font>MD<font face='Lucida Console'>)</font> <font color='#5555FF'>&</font><font color='#5555FF'>&</font>
<font face='Lucida Console'>(</font>src1.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>=</font><font color='#5555FF'>=</font><font color='#979000'>1</font> <font color='#5555FF'>|</font><font color='#5555FF'>|</font> src1.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>=</font><font color='#5555FF'>=</font>MD<font face='Lucida Console'>)</font> <font color='#5555FF'>&</font><font color='#5555FF'>&</font>
<font face='Lucida Console'>(</font>src2.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>=</font><font color='#5555FF'>=</font><font color='#979000'>1</font> <font color='#5555FF'>|</font><font color='#5555FF'>|</font> src2.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>=</font><font color='#5555FF'>=</font>MD<font face='Lucida Console'>)</font> <font face='Lucida Console'>)</font>;
<font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>dest.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>
<font color='#0000FF'>return</font>;
<font color='#0000FF'>const</font> <font color='#0000FF'><u>size_t</u></font> max_size <font color='#5555FF'>=</font> std::<font color='#BB00BB'>max</font><font face='Lucida Console'>(</font>std::<font color='#BB00BB'>max</font><font face='Lucida Console'>(</font>dest.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,src1.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>,src2.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> d <font color='#5555FF'>=</font> dest.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> s1 <font color='#5555FF'>=</font> src1.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> s2 <font color='#5555FF'>=</font> src2.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>dest.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> src1.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&</font><font color='#5555FF'>&</font> src1.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> src2.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>add_to<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'><</font> src1.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
d[i] <font color='#5555FF'>+</font><font color='#5555FF'>=</font> s1[i]<font color='#5555FF'>*</font>s2[i];
<b>}</b>
<font color='#0000FF'>else</font>
<b>{</b>
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'><</font> src1.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
d[i] <font color='#5555FF'>=</font> s1[i]<font color='#5555FF'>*</font>s2[i];
<b>}</b>
<b>}</b>
<font color='#0000FF'>else</font> <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>dest.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#5555FF'>!</font>add_to<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'><</font> dest.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
d[i] <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
<b>}</b>
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'><</font> max_size; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
d[i<font color='#5555FF'>%</font>dest.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>] <font color='#5555FF'>+</font><font color='#5555FF'>=</font> s1[i<font color='#5555FF'>%</font>src1.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>]<font color='#5555FF'>*</font>s2[i<font color='#5555FF'>%</font>src2.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>];
<b>}</b>
<font color='#0000FF'>else</font>
<b>{</b>
<font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>add_to<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'><</font> max_size; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
d[i] <font color='#5555FF'>+</font><font color='#5555FF'>=</font> s1[i<font color='#5555FF'>%</font>src1.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>]<font color='#5555FF'>*</font>s2[i<font color='#5555FF'>%</font>src2.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>];
<b>}</b>
<font color='#0000FF'>else</font>
<b>{</b>
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'><</font> max_size; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
d[i] <font color='#5555FF'>=</font> s1[i<font color='#5555FF'>%</font>src1.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>]<font color='#5555FF'>*</font>s2[i<font color='#5555FF'>%</font>src2.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>];
<b>}</b>
<b>}</b>
<b>}</b>
<font color='#009900'>// ------------------------------------------------------------------------------------
</font>
<font color='#0000FF'><u>void</u></font> <b><a name='multiply_conv'></a>multiply_conv</b> <font face='Lucida Console'>(</font>
<font color='#0000FF'><u>bool</u></font> add_to,
tensor<font color='#5555FF'>&</font> dest,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> src1,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> src2
<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>auto</font> d <font color='#5555FF'>=</font> dest.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>auto</font> s1 <font color='#5555FF'>=</font> src1.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>auto</font> s2 <font color='#5555FF'>=</font> src2.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#BB00BB'>have_same_dimensions</font><font face='Lucida Console'>(</font>dest,src1<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
<b>{</b>
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>src2.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font> <font color='#5555FF'>&</font><font color='#5555FF'>&</font> src2.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font> <font color='#5555FF'>&</font><font color='#5555FF'>&</font> src2.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font> <font color='#5555FF'>&</font><font color='#5555FF'>&</font> src2.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> src1.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>add_to<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> n <font color='#5555FF'>=</font> <font color='#979000'>0</font>; n <font color='#5555FF'><</font> dest.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>n<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> k <font color='#5555FF'>=</font> <font color='#979000'>0</font>; k <font color='#5555FF'><</font> dest.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>k<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> r <font color='#5555FF'>=</font> <font color='#979000'>0</font>; r <font color='#5555FF'><</font> dest.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>r<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> c <font color='#5555FF'>=</font> <font color='#979000'>0</font>; c <font color='#5555FF'><</font> dest.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>c<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#5555FF'>*</font>d<font color='#5555FF'>+</font><font color='#5555FF'>+</font> <font color='#5555FF'>+</font><font color='#5555FF'>=</font> <font face='Lucida Console'>(</font><font color='#5555FF'>*</font>s1<font color='#5555FF'>+</font><font color='#5555FF'>+</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>s2[k];
<b>}</b>
<b>}</b>
<b>}</b>
<b>}</b>
<b>}</b>
<font color='#0000FF'>else</font>
<b>{</b>
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> n <font color='#5555FF'>=</font> <font color='#979000'>0</font>; n <font color='#5555FF'><</font> dest.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>n<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> k <font color='#5555FF'>=</font> <font color='#979000'>0</font>; k <font color='#5555FF'><</font> dest.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>k<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> r <font color='#5555FF'>=</font> <font color='#979000'>0</font>; r <font color='#5555FF'><</font> dest.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>r<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> c <font color='#5555FF'>=</font> <font color='#979000'>0</font>; c <font color='#5555FF'><</font> dest.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>c<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#5555FF'>*</font>d<font color='#5555FF'>+</font><font color='#5555FF'>+</font> <font color='#5555FF'>=</font> <font face='Lucida Console'>(</font><font color='#5555FF'>*</font>s1<font color='#5555FF'>+</font><font color='#5555FF'>+</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>s2[k];
<b>}</b>
<b>}</b>
<b>}</b>
<b>}</b>
<b>}</b>
<b>}</b>
<font color='#0000FF'>else</font>
<b>{</b>
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font><font color='#BB00BB'>have_same_dimensions</font><font face='Lucida Console'>(</font>src1,src2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>dest.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font> <font color='#5555FF'>&</font><font color='#5555FF'>&</font> dest.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font> <font color='#5555FF'>&</font><font color='#5555FF'>&</font> dest.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font> <font color='#5555FF'>&</font><font color='#5555FF'>&</font> dest.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> src1.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#5555FF'>!</font>add_to<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> k <font color='#5555FF'>=</font> <font color='#979000'>0</font>; k <font color='#5555FF'><</font> src1.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>k<font face='Lucida Console'>)</font>
d[k] <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
<b>}</b>
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> n <font color='#5555FF'>=</font> <font color='#979000'>0</font>; n <font color='#5555FF'><</font> src1.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>n<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> k <font color='#5555FF'>=</font> <font color='#979000'>0</font>; k <font color='#5555FF'><</font> src1.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>k<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> r <font color='#5555FF'>=</font> <font color='#979000'>0</font>; r <font color='#5555FF'><</font> src1.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>r<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> c <font color='#5555FF'>=</font> <font color='#979000'>0</font>; c <font color='#5555FF'><</font> src1.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>c<font face='Lucida Console'>)</font>
<b>{</b>
d[k] <font color='#5555FF'>+</font><font color='#5555FF'>=</font> <font face='Lucida Console'>(</font><font color='#5555FF'>*</font>s1<font color='#5555FF'>+</font><font color='#5555FF'>+</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font><font face='Lucida Console'>(</font><font color='#5555FF'>*</font>s2<font color='#5555FF'>+</font><font color='#5555FF'>+</font><font face='Lucida Console'>)</font>;
<b>}</b>
<b>}</b>
<b>}</b>
<b>}</b>
<b>}</b>
<b>}</b>
<font color='#009900'>// ------------------------------------------------------------------------------------
</font>
<font color='#0000FF'><u>void</u></font> <b><a name='scale_channels'></a>scale_channels</b> <font face='Lucida Console'>(</font>
<font color='#0000FF'><u>bool</u></font> add_to,
tensor<font color='#5555FF'>&</font> dest,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> src,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> scales
<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font><font color='#BB00BB'>have_same_dimensions</font><font face='Lucida Console'>(</font>dest,src<font face='Lucida Console'>)</font> <font color='#5555FF'>&</font><font color='#5555FF'>&</font>
scales.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&</font><font color='#5555FF'>&</font>
scales.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> src.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&</font><font color='#5555FF'>&</font>
scales.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font> <font color='#5555FF'>&</font><font color='#5555FF'>&</font>
scales.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font> <font face='Lucida Console'>)</font>;
<font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>dest.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>
<font color='#0000FF'>return</font>;
<font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>add_to<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>auto</font> d <font color='#5555FF'>=</font> dest.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>auto</font> s <font color='#5555FF'>=</font> src.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>auto</font> scal <font color='#5555FF'>=</font> scales.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> n <font color='#5555FF'>=</font> <font color='#979000'>0</font>; n <font color='#5555FF'><</font> src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>n<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> k <font color='#5555FF'>=</font> <font color='#979000'>0</font>; k <font color='#5555FF'><</font> src.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>k<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> scale <font color='#5555FF'>=</font> scal[n<font color='#5555FF'>*</font>scales.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> k];
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> r <font color='#5555FF'>=</font> <font color='#979000'>0</font>; r <font color='#5555FF'><</font> src.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>r<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> c <font color='#5555FF'>=</font> <font color='#979000'>0</font>; c <font color='#5555FF'><</font> src.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>c<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#5555FF'>*</font>d<font color='#5555FF'>+</font><font color='#5555FF'>+</font> <font color='#5555FF'>+</font><font color='#5555FF'>=</font> <font face='Lucida Console'>(</font><font color='#5555FF'>*</font>s<font color='#5555FF'>+</font><font color='#5555FF'>+</font><font face='Lucida Console'>)</font> <font color='#5555FF'>*</font> scale;
<b>}</b>
<b>}</b>
<b>}</b>
<b>}</b>
<b>}</b>
<font color='#0000FF'>else</font>
<b>{</b>
<font color='#0000FF'>auto</font> d <font color='#5555FF'>=</font> dest.<font color='#BB00BB'>host_write_only</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>auto</font> s <font color='#5555FF'>=</font> src.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>auto</font> scal <font color='#5555FF'>=</font> scales.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> n <font color='#5555FF'>=</font> <font color='#979000'>0</font>; n <font color='#5555FF'><</font> src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>n<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> k <font color='#5555FF'>=</font> <font color='#979000'>0</font>; k <font color='#5555FF'><</font> src.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>k<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> scale <font color='#5555FF'>=</font> scal[n<font color='#5555FF'>*</font>scales.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> k];
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> r <font color='#5555FF'>=</font> <font color='#979000'>0</font>; r <font color='#5555FF'><</font> src.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>r<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> c <font color='#5555FF'>=</font> <font color='#979000'>0</font>; c <font color='#5555FF'><</font> src.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>c<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#5555FF'>*</font>d<font color='#5555FF'>+</font><font color='#5555FF'>+</font> <font color='#5555FF'>=</font> <font face='Lucida Console'>(</font><font color='#5555FF'>*</font>s<font color='#5555FF'>+</font><font color='#5555FF'>+</font><font face='Lucida Console'>)</font> <font color='#5555FF'>*</font> scale;
<b>}</b>
<b>}</b>
<b>}</b>
<b>}</b>
<b>}</b>
<b>}</b>
<font color='#009900'>// ------------------------------------------------------------------------------------
</font>
<font color='#0000FF'><u>void</u></font> <b><a name='add'></a>add</b><font face='Lucida Console'>(</font>
<font color='#0000FF'><u>float</u></font> beta,
tensor<font color='#5555FF'>&</font> dest,
<font color='#0000FF'><u>float</u></font> alpha,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> src
<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>
<font face='Lucida Console'>(</font><font color='#BB00BB'>have_same_dimensions</font><font face='Lucida Console'>(</font>src, dest<font face='Lucida Console'>)</font> <font color='#5555FF'>|</font><font color='#5555FF'>|</font>
<font face='Lucida Console'>(</font>src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>=</font><font color='#5555FF'>=</font><font color='#979000'>1</font> <font color='#5555FF'>&</font><font color='#5555FF'>&</font> src.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>=</font><font color='#5555FF'>=</font>dest.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&</font><font color='#5555FF'>&</font> src.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>=</font><font color='#5555FF'>=</font><font color='#979000'>1</font> <font color='#5555FF'>&</font><font color='#5555FF'>&</font> src.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>=</font><font color='#5555FF'>=</font><font color='#979000'>1</font><font face='Lucida Console'>)</font> <font color='#5555FF'>|</font><font color='#5555FF'>|</font>
<font face='Lucida Console'>(</font>src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>=</font><font color='#5555FF'>=</font><font color='#979000'>1</font> <font color='#5555FF'>&</font><font color='#5555FF'>&</font> src.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>=</font><font color='#5555FF'>=</font>dest.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&</font><font color='#5555FF'>&</font> src.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>=</font><font color='#5555FF'>=</font>dest.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&</font><font color='#5555FF'>&</font> src.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>=</font><font color='#5555FF'>=</font>dest.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>|</font><font color='#5555FF'>|</font>
<font face='Lucida Console'>(</font>src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>=</font><font color='#5555FF'>=</font><font color='#979000'>1</font> <font color='#5555FF'>&</font><font color='#5555FF'>&</font> src.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>=</font><font color='#5555FF'>=</font><font color='#979000'>1</font> <font color='#5555FF'>&</font><font color='#5555FF'>&</font> src.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>=</font><font color='#5555FF'>=</font>dest.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&</font><font color='#5555FF'>&</font> src.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>=</font><font color='#5555FF'>=</font>dest.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>|</font><font color='#5555FF'>|</font>
<font face='Lucida Console'>(</font>src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>=</font><font color='#5555FF'>=</font>dest.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&</font><font color='#5555FF'>&</font> src.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>=</font><font color='#5555FF'>=</font><font color='#979000'>1</font> <font color='#5555FF'>&</font><font color='#5555FF'>&</font> src.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>=</font><font color='#5555FF'>=</font><font color='#979000'>1</font> <font color='#5555FF'>&</font><font color='#5555FF'>&</font> src.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>=</font><font color='#5555FF'>=</font><font color='#979000'>1</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&</font><font color='#5555FF'>&</font>
<font color='#BB00BB'>is_same_object</font><font face='Lucida Console'>(</font>src,dest<font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>false</font> ,
"<font color='#CC0000'>\n\t dest.num_samples(): </font>" <font color='#5555FF'><</font><font color='#5555FF'><</font> dest.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
<font color='#5555FF'><</font><font color='#5555FF'><</font>"<font color='#CC0000'>\n\t dest.k(): </font>" <font color='#5555FF'><</font><font color='#5555FF'><</font> dest.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
<font color='#5555FF'><</font><font color='#5555FF'><</font>"<font color='#CC0000'>\n\t dest.nr(): </font>" <font color='#5555FF'><</font><font color='#5555FF'><</font> dest.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
<font color='#5555FF'><</font><font color='#5555FF'><</font>"<font color='#CC0000'>\n\t dest.nc(): </font>" <font color='#5555FF'><</font><font color='#5555FF'><</font> dest.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
<font color='#5555FF'><</font><font color='#5555FF'><</font>"<font color='#CC0000'>\n\t src.num_samples(): </font>" <font color='#5555FF'><</font><font color='#5555FF'><</font> src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
<font color='#5555FF'><</font><font color='#5555FF'><</font>"<font color='#CC0000'>\n\t src.k(): </font>" <font color='#5555FF'><</font><font color='#5555FF'><</font> src.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
<font color='#5555FF'><</font><font color='#5555FF'><</font>"<font color='#CC0000'>\n\t src.nr(): </font>" <font color='#5555FF'><</font><font color='#5555FF'><</font> src.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
<font color='#5555FF'><</font><font color='#5555FF'><</font>"<font color='#CC0000'>\n\t src.nc(): </font>" <font color='#5555FF'><</font><font color='#5555FF'><</font> src.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
<font face='Lucida Console'>)</font>;
<font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>beta <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>0</font> <font color='#5555FF'>&</font><font color='#5555FF'>&</font> alpha <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>
<b>{</b>
dest <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
<font color='#0000FF'>return</font>;
<b>}</b>
<font color='#0000FF'>auto</font> d <font color='#5555FF'>=</font> dest.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>auto</font> s <font color='#5555FF'>=</font> src.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> n <font color='#5555FF'>=</font> <font color='#979000'>0</font>; n <font color='#5555FF'><</font> dest.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>n<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> sn <font color='#5555FF'>=</font> src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>=</font><font color='#5555FF'>=</font><font color='#979000'>1</font> ? <font color='#979000'>0</font>:n;
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> k <font color='#5555FF'>=</font> <font color='#979000'>0</font>; k <font color='#5555FF'><</font> dest.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>k<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> sk <font color='#5555FF'>=</font> src.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>=</font><font color='#5555FF'>=</font><font color='#979000'>1</font> ? <font color='#979000'>0</font>:k;
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> r <font color='#5555FF'>=</font> <font color='#979000'>0</font>; r <font color='#5555FF'><</font> dest.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>r<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> sr <font color='#5555FF'>=</font> src.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>=</font><font color='#5555FF'>=</font><font color='#979000'>1</font> ? <font color='#979000'>0</font>:r;
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> c <font color='#5555FF'>=</font> <font color='#979000'>0</font>; c <font color='#5555FF'><</font> dest.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>c<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> sc <font color='#5555FF'>=</font> src.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>=</font><font color='#5555FF'>=</font><font color='#979000'>1</font> ? <font color='#979000'>0</font>:c;
<font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> s_idx <font color='#5555FF'>=</font> <font face='Lucida Console'>(</font><font face='Lucida Console'>(</font>sn<font color='#5555FF'>*</font>src.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> sk<font face='Lucida Console'>)</font><font color='#5555FF'>*</font>src.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> sr<font face='Lucida Console'>)</font><font color='#5555FF'>*</font>src.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> sc;
<font color='#5555FF'>*</font>d <font color='#5555FF'>=</font> beta<font color='#5555FF'>*</font><font face='Lucida Console'>(</font><font color='#5555FF'>*</font>d<font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> alpha<font color='#5555FF'>*</font>s[s_idx];
<font color='#5555FF'>+</font><font color='#5555FF'>+</font>d;
<b>}</b>
<b>}</b>
<b>}</b>
<b>}</b>
<b>}</b>
<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
<font color='#0000FF'><u>void</u></font> <b><a name='add'></a>add</b> <font face='Lucida Console'>(</font>
tensor<font color='#5555FF'>&</font> dest,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> src1,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> src2
<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>auto</font> d <font color='#5555FF'>=</font> dest.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>auto</font> s1 <font color='#5555FF'>=</font> src1.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>auto</font> s2 <font color='#5555FF'>=</font> src2.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#009900'>// Do the simple and fast version if everything has the same dimensions
</font> <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#BB00BB'>have_same_dimensions</font><font face='Lucida Console'>(</font>dest, src1<font face='Lucida Console'>)</font> <font color='#5555FF'>&</font><font color='#5555FF'>&</font>
<font color='#BB00BB'>have_same_dimensions</font><font face='Lucida Console'>(</font>dest, src2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'><</font> dest.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
d[i] <font color='#5555FF'>=</font> s1[i] <font color='#5555FF'>+</font> s2[i];
<font color='#0000FF'>return</font>;
<b>}</b>
<font color='#009900'>// Otherwise, do the more complex version with bounds checking.
</font> <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> n <font color='#5555FF'>=</font> <font color='#979000'>0</font>; n <font color='#5555FF'><</font> dest.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>n<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> k <font color='#5555FF'>=</font> <font color='#979000'>0</font>; k <font color='#5555FF'><</font> dest.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>k<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> r <font color='#5555FF'>=</font> <font color='#979000'>0</font>; r <font color='#5555FF'><</font> dest.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>r<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> c <font color='#5555FF'>=</font> <font color='#979000'>0</font>; c <font color='#5555FF'><</font> dest.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>c<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'><u>float</u></font> v1 <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
<font color='#0000FF'><u>float</u></font> v2 <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
<font color='#009900'>// if this index is inside src1
</font> <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>n <font color='#5555FF'><</font> src1.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&</font><font color='#5555FF'>&</font>
k <font color='#5555FF'><</font> src1.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&</font><font color='#5555FF'>&</font>
r <font color='#5555FF'><</font> src1.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&</font><font color='#5555FF'>&</font>
c <font color='#5555FF'><</font> src1.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> s_idx <font color='#5555FF'>=</font> <font face='Lucida Console'>(</font><font face='Lucida Console'>(</font>n<font color='#5555FF'>*</font>src1.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> k<font face='Lucida Console'>)</font><font color='#5555FF'>*</font>src1.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> r<font face='Lucida Console'>)</font><font color='#5555FF'>*</font>src1.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> c;
v1 <font color='#5555FF'>=</font> s1[s_idx];
<b>}</b>
<font color='#009900'>// if this index is inside src2
</font> <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>n <font color='#5555FF'><</font> src2.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&</font><font color='#5555FF'>&</font>
k <font color='#5555FF'><</font> src2.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&</font><font color='#5555FF'>&</font>
r <font color='#5555FF'><</font> src2.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&</font><font color='#5555FF'>&</font>
c <font color='#5555FF'><</font> src2.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> s_idx <font color='#5555FF'>=</font> <font face='Lucida Console'>(</font><font face='Lucida Console'>(</font>n<font color='#5555FF'>*</font>src2.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> k<font face='Lucida Console'>)</font><font color='#5555FF'>*</font>src2.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> r<font face='Lucida Console'>)</font><font color='#5555FF'>*</font>src2.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> c;
v2 <font color='#5555FF'>=</font> s2[s_idx];
<b>}</b>
<font color='#5555FF'>*</font>d <font color='#5555FF'>=</font> v1 <font color='#5555FF'>+</font> v2;
<font color='#5555FF'>+</font><font color='#5555FF'>+</font>d;
<b>}</b>
<b>}</b>
<b>}</b>
<b>}</b>
<b>}</b>
<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
<font color='#0000FF'><u>void</u></font> <b><a name='multiply_zero_padded'></a>multiply_zero_padded</b> <font face='Lucida Console'>(</font>
<font color='#0000FF'><u>bool</u></font> add_to,
tensor<font color='#5555FF'>&</font> dest,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> src1,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> src2
<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>auto</font> d <font color='#5555FF'>=</font> dest.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>auto</font> s1 <font color='#5555FF'>=</font> src1.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>auto</font> s2 <font color='#5555FF'>=</font> src2.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#009900'>// Do the simple and fast version if everything has the same dimensions
</font> <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#BB00BB'>have_same_dimensions</font><font face='Lucida Console'>(</font>dest, src1<font face='Lucida Console'>)</font> <font color='#5555FF'>&</font><font color='#5555FF'>&</font>
<font color='#BB00BB'>have_same_dimensions</font><font face='Lucida Console'>(</font>dest, src2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>add_to<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'><</font> dest.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
d[i] <font color='#5555FF'>+</font><font color='#5555FF'>=</font> s1[i] <font color='#5555FF'>*</font> s2[i];
<b>}</b>
<font color='#0000FF'>else</font>
<b>{</b>
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'><</font> dest.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
d[i] <font color='#5555FF'>=</font> s1[i] <font color='#5555FF'>*</font> s2[i];
<b>}</b>
<font color='#0000FF'>return</font>;
<b>}</b>
<font color='#009900'>// Otherwise, do the more complex version with bounds checking.
</font> <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> n <font color='#5555FF'>=</font> <font color='#979000'>0</font>; n <font color='#5555FF'><</font> dest.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>n<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> k <font color='#5555FF'>=</font> <font color='#979000'>0</font>; k <font color='#5555FF'><</font> dest.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>k<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> r <font color='#5555FF'>=</font> <font color='#979000'>0</font>; r <font color='#5555FF'><</font> dest.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>r<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> c <font color='#5555FF'>=</font> <font color='#979000'>0</font>; c <font color='#5555FF'><</font> dest.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>c<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'><u>float</u></font> v1 <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
<font color='#0000FF'><u>float</u></font> v2 <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
<font color='#009900'>// if this index is inside src1
</font> <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>n <font color='#5555FF'><</font> src1.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&</font><font color='#5555FF'>&</font>
k <font color='#5555FF'><</font> src1.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&</font><font color='#5555FF'>&</font>
r <font color='#5555FF'><</font> src1.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&</font><font color='#5555FF'>&</font>
c <font color='#5555FF'><</font> src1.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> s_idx <font color='#5555FF'>=</font> <font face='Lucida Console'>(</font><font face='Lucida Console'>(</font>n<font color='#5555FF'>*</font>src1.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> k<font face='Lucida Console'>)</font><font color='#5555FF'>*</font>src1.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> r<font face='Lucida Console'>)</font><font color='#5555FF'>*</font>src1.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> c;
v1 <font color='#5555FF'>=</font> s1[s_idx];
<b>}</b>
<font color='#009900'>// if this index is inside src2
</font> <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>n <font color='#5555FF'><</font> src2.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&</font><font color='#5555FF'>&</font>
k <font color='#5555FF'><</font> src2.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&</font><font color='#5555FF'>&</font>
r <font color='#5555FF'><</font> src2.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&</font><font color='#5555FF'>&</font>
c <font color='#5555FF'><</font> src2.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> s_idx <font color='#5555FF'>=</font> <font face='Lucida Console'>(</font><font face='Lucida Console'>(</font>n<font color='#5555FF'>*</font>src2.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> k<font face='Lucida Console'>)</font><font color='#5555FF'>*</font>src2.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> r<font face='Lucida Console'>)</font><font color='#5555FF'>*</font>src2.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> c;
v2 <font color='#5555FF'>=</font> s2[s_idx];
<b>}</b>
<font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>add_to<font face='Lucida Console'>)</font>
<font color='#5555FF'>*</font>d <font color='#5555FF'>+</font><font color='#5555FF'>=</font> v1 <font color='#5555FF'>*</font> v2;
<font color='#0000FF'>else</font>
<font color='#5555FF'>*</font>d <font color='#5555FF'>=</font> v1 <font color='#5555FF'>*</font> v2;
<font color='#5555FF'>+</font><font color='#5555FF'>+</font>d;
<b>}</b>
<b>}</b>
<b>}</b>
<b>}</b>
<b>}</b>
<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
<font color='#0000FF'><u>void</u></font> <b><a name='assign_bias_gradient'></a>assign_bias_gradient</b> <font face='Lucida Console'>(</font>
tensor<font color='#5555FF'>&</font> grad,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> gradient_input
<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>
grad.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font> <font color='#5555FF'>&</font><font color='#5555FF'>&</font>
gradient_input.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> grad.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&</font><font color='#5555FF'>&</font>
gradient_input.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> grad.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&</font><font color='#5555FF'>&</font>
gradient_input.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> grad.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&</font><font color='#5555FF'>&</font>
gradient_input.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>></font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>auto</font> out <font color='#5555FF'>=</font> grad.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>auto</font> in <font color='#5555FF'>=</font> gradient_input.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'><</font> grad.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
out[i] <font color='#5555FF'>=</font> <font color='#5555FF'>*</font>in<font color='#5555FF'>+</font><font color='#5555FF'>+</font>;
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> j <font color='#5555FF'>=</font> <font color='#979000'>1</font>; j <font color='#5555FF'><</font> gradient_input.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>j<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'><</font> grad.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
out[i] <font color='#5555FF'>+</font><font color='#5555FF'>=</font> <font color='#5555FF'>*</font>in<font color='#5555FF'>+</font><font color='#5555FF'>+</font>;
<b>}</b>
<b>}</b>
<font color='#009900'>// ------------------------------------------------------------------------------------
</font>
<font color='#0000FF'><u>void</u></font> <b><a name='assign_conv_bias_gradient'></a>assign_conv_bias_gradient</b> <font face='Lucida Console'>(</font>
tensor<font color='#5555FF'>&</font> grad,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> gradient_input
<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>
grad.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font> <font color='#5555FF'>&</font><font color='#5555FF'>&</font>
grad.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>></font><font color='#5555FF'>=</font> <font color='#979000'>1</font> <font color='#5555FF'>&</font><font color='#5555FF'>&</font>
grad.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font> <font color='#5555FF'>&</font><font color='#5555FF'>&</font>
grad.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font> <font color='#5555FF'>&</font><font color='#5555FF'>&</font>
gradient_input.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> grad.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&</font><font color='#5555FF'>&</font>
gradient_input.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>></font> <font color='#979000'>0</font> <font color='#5555FF'>&</font><font color='#5555FF'>&</font>
<font color='#BB00BB'>is_same_object</font><font face='Lucida Console'>(</font>grad,gradient_input<font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>false</font>
<font face='Lucida Console'>)</font>;
<font color='#0000FF'>auto</font> g <font color='#5555FF'>=</font> grad.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>auto</font> gi <font color='#5555FF'>=</font> gradient_input.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> k <font color='#5555FF'>=</font> <font color='#979000'>0</font>; k <font color='#5555FF'><</font> gradient_input.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>k<font face='Lucida Console'>)</font>
g[k] <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> n <font color='#5555FF'>=</font> <font color='#979000'>0</font>; n <font color='#5555FF'><</font> gradient_input.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>n<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> k <font color='#5555FF'>=</font> <font color='#979000'>0</font>; k <font color='#5555FF'><</font> gradient_input.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>k<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> r <font color='#5555FF'>=</font> <font color='#979000'>0</font>; r <font color='#5555FF'><</font> gradient_input.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>r<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> c <font color='#5555FF'>=</font> <font color='#979000'>0</font>; c <font color='#5555FF'><</font> gradient_input.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>c<font face='Lucida Console'>)</font>
<b>{</b>
g[k] <font color='#5555FF'>+</font><font color='#5555FF'>=</font> <font face='Lucida Console'>(</font><font color='#5555FF'>*</font>gi<font color='#5555FF'>+</font><font color='#5555FF'>+</font><font face='Lucida Console'>)</font>;
<b>}</b>
<b>}</b>
<b>}</b>
<b>}</b>
<b>}</b>
<font color='#009900'>// -----------------------------------------------------------------------------------
</font>
<font color='#0000FF'><u>void</u></font> <b><a name='affine_transform'></a>affine_transform</b><font face='Lucida Console'>(</font>
tensor<font color='#5555FF'>&</font> dest,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> src,
<font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> A,
<font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> B
<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>dest.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>=</font><font color='#5555FF'>=</font>src.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> d <font color='#5555FF'>=</font> dest.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> s <font color='#5555FF'>=</font> src.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'><</font> src.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
d[i] <font color='#5555FF'>=</font> A<font color='#5555FF'>*</font>s[i] <font color='#5555FF'>+</font> B;
<b>}</b>
<font color='#0000FF'><u>void</u></font> <b><a name='affine_transform'></a>affine_transform</b><font face='Lucida Console'>(</font>
tensor<font color='#5555FF'>&</font> dest,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> src1,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> src2,
<font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> A,
<font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> B,
<font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> C
<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>dest.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>=</font><font color='#5555FF'>=</font>src1.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>dest.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>=</font><font color='#5555FF'>=</font>src2.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> d <font color='#5555FF'>=</font> dest.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> s1 <font color='#5555FF'>=</font> src1.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> s2 <font color='#5555FF'>=</font> src2.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'><</font> src1.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
d[i] <font color='#5555FF'>=</font> A<font color='#5555FF'>*</font>s1[i] <font color='#5555FF'>+</font> B<font color='#5555FF'>*</font>s2[i] <font color='#5555FF'>+</font> C;
<b>}</b>
<font color='#0000FF'><u>void</u></font> <b><a name='affine_transform'></a>affine_transform</b><font face='Lucida Console'>(</font>
tensor<font color='#5555FF'>&</font> dest,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> src1,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> src2,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> src3,
<font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> A,
<font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> B,
<font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> C,
<font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> D
<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>dest.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>=</font><font color='#5555FF'>=</font>src1.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>dest.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>=</font><font color='#5555FF'>=</font>src2.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>dest.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>=</font><font color='#5555FF'>=</font>src3.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> d <font color='#5555FF'>=</font> dest.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> s1 <font color='#5555FF'>=</font> src1.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> s2 <font color='#5555FF'>=</font> src2.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> s3 <font color='#5555FF'>=</font> src3.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'><</font> src1.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
d[i] <font color='#5555FF'>=</font> A<font color='#5555FF'>*</font>s1[i] <font color='#5555FF'>+</font> B<font color='#5555FF'>*</font>s2[i] <font color='#5555FF'>+</font> C<font color='#5555FF'>*</font>s3[i] <font color='#5555FF'>+</font> D;
<b>}</b>
<font color='#0000FF'><u>void</u></font> <b><a name='affine_transform_range'></a>affine_transform_range</b><font face='Lucida Console'>(</font>
<font color='#0000FF'><u>size_t</u></font> begin,
<font color='#0000FF'><u>size_t</u></font> end,
tensor<font color='#5555FF'>&</font> dest,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> src1,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> src2,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> src3,
<font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> A,
<font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> B,
<font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> C
<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>dest.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>=</font><font color='#5555FF'>=</font>src1.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>dest.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>=</font><font color='#5555FF'>=</font>src2.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>dest.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>=</font><font color='#5555FF'>=</font>src3.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>begin <font color='#5555FF'><</font><font color='#5555FF'>=</font> end <font color='#5555FF'>&</font><font color='#5555FF'>&</font> end <font color='#5555FF'><</font><font color='#5555FF'>=</font> dest.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> d <font color='#5555FF'>=</font> dest.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> s1 <font color='#5555FF'>=</font> src1.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> s2 <font color='#5555FF'>=</font> src2.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> s3 <font color='#5555FF'>=</font> src3.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> i <font color='#5555FF'>=</font> begin; i <font color='#5555FF'><</font> end; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
d[i] <font color='#5555FF'>=</font> A<font color='#5555FF'>*</font>s1[i] <font color='#5555FF'>+</font> B<font color='#5555FF'>*</font>s2[i] <font color='#5555FF'>+</font> C<font color='#5555FF'>*</font>s3[i];
<b>}</b>
<font color='#009900'>// -----------------------------------------------------------------------------------
</font>
<font color='#0000FF'><u>void</u></font> <b><a name='affine_transform'></a>affine_transform</b><font face='Lucida Console'>(</font>
tensor<font color='#5555FF'>&</font> dest,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> src,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> A,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> B
<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font><font color='#BB00BB'>have_same_dimensions</font><font face='Lucida Console'>(</font>dest,src<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>
<font face='Lucida Console'>(</font><font face='Lucida Console'>(</font>A.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>=</font><font color='#5555FF'>=</font><font color='#979000'>1</font> <font color='#5555FF'>&</font><font color='#5555FF'>&</font> B.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>=</font><font color='#5555FF'>=</font><font color='#979000'>1</font><font face='Lucida Console'>)</font> <font color='#5555FF'>|</font><font color='#5555FF'>|</font>
<font face='Lucida Console'>(</font>A.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>=</font><font color='#5555FF'>=</font>src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&</font><font color='#5555FF'>&</font> B.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>=</font><font color='#5555FF'>=</font>src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&</font><font color='#5555FF'>&</font>
A.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>=</font><font color='#5555FF'>=</font>B.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&</font><font color='#5555FF'>&</font> B.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>=</font><font color='#5555FF'>=</font>src.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&</font><font color='#5555FF'>&</font>
A.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>=</font><font color='#5555FF'>=</font>B.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&</font><font color='#5555FF'>&</font> B.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>=</font><font color='#5555FF'>=</font>src.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&</font><font color='#5555FF'>&</font>
A.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font>B.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&</font><font color='#5555FF'>&</font> B.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>=</font><font color='#5555FF'>=</font>src.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>auto</font> d <font color='#5555FF'>=</font> dest.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>auto</font> s <font color='#5555FF'>=</font> src.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> a <font color='#5555FF'>=</font> A.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> b <font color='#5555FF'>=</font> B.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>A.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> num <font color='#5555FF'>=</font> src.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font>src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'><</font> src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> j <font color='#5555FF'>=</font> <font color='#979000'>0</font>; j <font color='#5555FF'><</font> num; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>j<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#5555FF'>*</font>d <font color='#5555FF'>=</font> a[j]<font color='#5555FF'>*</font><font face='Lucida Console'>(</font><font color='#5555FF'>*</font>s<font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> b[j];
d<font color='#5555FF'>+</font><font color='#5555FF'>+</font>;
s<font color='#5555FF'>+</font><font color='#5555FF'>+</font>;
<b>}</b>
<b>}</b>
<b>}</b>
<font color='#0000FF'>else</font>
<b>{</b>
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'><</font> src.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
d[i] <font color='#5555FF'>=</font> a[i]<font color='#5555FF'>*</font>s[i] <font color='#5555FF'>+</font> b[i];
<b>}</b>
<b>}</b>
<font color='#009900'>// -----------------------------------------------------------------------------------
</font>
<font color='#0000FF'><u>void</u></font> <b><a name='affine_transform_conv'></a>affine_transform_conv</b><font face='Lucida Console'>(</font>
tensor<font color='#5555FF'>&</font> dest,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> src,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> A,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> B
<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font><font color='#BB00BB'>have_same_dimensions</font><font face='Lucida Console'>(</font>dest,src<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font><font color='#BB00BB'>have_same_dimensions</font><font face='Lucida Console'>(</font>A,B<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>A.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font> <font color='#5555FF'>&</font><font color='#5555FF'>&</font>
A.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font> <font color='#5555FF'>&</font><font color='#5555FF'>&</font>
A.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font> <font color='#5555FF'>&</font><font color='#5555FF'>&</font>
A.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> src.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>auto</font> d <font color='#5555FF'>=</font> dest.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>auto</font> s <font color='#5555FF'>=</font> src.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> a <font color='#5555FF'>=</font> A.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> b <font color='#5555FF'>=</font> B.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> n <font color='#5555FF'>=</font> <font color='#979000'>0</font>; n <font color='#5555FF'><</font> dest.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>n<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> k <font color='#5555FF'>=</font> <font color='#979000'>0</font>; k <font color='#5555FF'><</font> dest.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>k<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> r <font color='#5555FF'>=</font> <font color='#979000'>0</font>; r <font color='#5555FF'><</font> dest.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>r<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> c <font color='#5555FF'>=</font> <font color='#979000'>0</font>; c <font color='#5555FF'><</font> dest.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>c<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#5555FF'>*</font>d<font color='#5555FF'>+</font><font color='#5555FF'>+</font> <font color='#5555FF'>=</font> a[k]<font color='#5555FF'>*</font><font face='Lucida Console'>(</font><font color='#5555FF'>*</font>s<font color='#5555FF'>+</font><font color='#5555FF'>+</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> b[k];
<b>}</b>
<b>}</b>
<b>}</b>
<b>}</b>
<b>}</b>
<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
<font color='#0000FF'><u>void</u></font> <b><a name='affine_transform'></a>affine_transform</b><font face='Lucida Console'>(</font>
<font color='#0000FF'>const</font> rectangle<font color='#5555FF'>&</font> rect,
tensor<font color='#5555FF'>&</font> dest,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> src1,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> src2,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> src3,
<font color='#0000FF'><u>float</u></font> A,
<font color='#0000FF'><u>float</u></font> B,
<font color='#0000FF'><u>float</u></font> C
<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>dest.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> src1.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>dest.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> src2.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>dest.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> src3.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>dest.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> src1.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>dest.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> src2.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>dest.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> src3.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font><font color='#BB00BB'>rectangle</font><font face='Lucida Console'>(</font><font color='#979000'>0</font>,<font color='#979000'>0</font>, dest.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font>dest.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#979000'>1</font>, dest.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#979000'>1</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>contains</font><font face='Lucida Console'>(</font>rect<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>auto</font> d <font color='#5555FF'>=</font> dest.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>auto</font> s1 <font color='#5555FF'>=</font> src1.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>auto</font> s2 <font color='#5555FF'>=</font> src2.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>auto</font> s3 <font color='#5555FF'>=</font> src3.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> nc <font color='#5555FF'>=</font> dest.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font>dest.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> r <font color='#5555FF'>=</font> rect.<font color='#BB00BB'>top</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; r <font color='#5555FF'><</font><font color='#5555FF'>=</font> rect.<font color='#BB00BB'>bottom</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>r<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> c <font color='#5555FF'>=</font> rect.<font color='#BB00BB'>left</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; c <font color='#5555FF'><</font><font color='#5555FF'>=</font> rect.<font color='#BB00BB'>right</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>c<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>auto</font> idx <font color='#5555FF'>=</font> r<font color='#5555FF'>*</font>nc <font color='#5555FF'>+</font> c;
d[idx] <font color='#5555FF'>=</font> s1[idx]<font color='#5555FF'>*</font>A <font color='#5555FF'>+</font> s2[idx]<font color='#5555FF'>*</font>B <font color='#5555FF'>+</font> s3[idx]<font color='#5555FF'>*</font>C;
<b>}</b>
<b>}</b>
<b>}</b>
<font color='#009900'>// -----------------------------------------------------------------------------------
</font>
<font color='#0000FF'><u>void</u></font> <b><a name='compute_adam_update'></a>compute_adam_update</b> <font face='Lucida Console'>(</font>
<font color='#0000FF'><u>size_t</u></font> begin,
<font color='#0000FF'><u>size_t</u></font> end,
tensor<font color='#5555FF'>&</font> s,
tensor<font color='#5555FF'>&</font> m,
tensor<font color='#5555FF'>&</font> v,
<font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> t,
<font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> learning_rate,
<font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> weight_decay,
<font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> momentum1,
<font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> momentum2,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> params,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> params_grad
<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>s.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> m.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&</font><font color='#5555FF'>&</font>
s.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> v.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&</font><font color='#5555FF'>&</font>
s.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> params.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&</font><font color='#5555FF'>&</font>
s.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> params_grad.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>begin <font color='#5555FF'><</font><font color='#5555FF'>=</font> end <font color='#5555FF'>&</font><font color='#5555FF'>&</font> end <font color='#5555FF'><</font><font color='#5555FF'>=</font> params.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> eps <font color='#5555FF'>=</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>8</font>;
<font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> alpha <font color='#5555FF'>=</font> learning_rate<font color='#5555FF'>*</font>std::<font color='#BB00BB'>sqrt</font><font face='Lucida Console'>(</font><font color='#979000'>1</font><font color='#5555FF'>-</font>std::<font color='#BB00BB'>pow</font><font face='Lucida Console'>(</font>momentum2,t<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font><font face='Lucida Console'>(</font><font color='#979000'>1</font><font color='#5555FF'>-</font>std::<font color='#BB00BB'>pow</font><font face='Lucida Console'>(</font>momentum1, t<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
<font color='#009900'>// The loop is equivalent to doing this:
</font> <font color='#009900'>// m = momentum1*m + (1-momentum1) * (weight_decay*params + params_grad);
</font> <font color='#009900'>// v = momentum2*v + (1-momentum2)*squared(weight_decay*params + params_grad);
</font> <font color='#009900'>// s = -alpha*m/(sqrt(v) + eps);
</font> <font color='#0000FF'>auto</font> pm <font color='#5555FF'>=</font> m.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>auto</font> pv <font color='#5555FF'>=</font> v.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>auto</font> ps <font color='#5555FF'>=</font> s.<font color='#BB00BB'>host_write_only</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>auto</font> pparams <font color='#5555FF'>=</font> params.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>auto</font> ppgrad <font color='#5555FF'>=</font> params_grad.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> i <font color='#5555FF'>=</font> begin; i <font color='#5555FF'><</font> end; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'><u>float</u></font> g <font color='#5555FF'>=</font> weight_decay<font color='#5555FF'>*</font>pparams[i] <font color='#5555FF'>+</font> ppgrad[i];
pm[i] <font color='#5555FF'>=</font> momentum1<font color='#5555FF'>*</font>pm[i] <font color='#5555FF'>+</font> <font face='Lucida Console'>(</font><font color='#979000'>1</font><font color='#5555FF'>-</font>momentum1<font face='Lucida Console'>)</font><font color='#5555FF'>*</font>g;
pv[i] <font color='#5555FF'>=</font> momentum2<font color='#5555FF'>*</font>pv[i] <font color='#5555FF'>+</font> <font face='Lucida Console'>(</font><font color='#979000'>1</font><font color='#5555FF'>-</font>momentum2<font face='Lucida Console'>)</font><font color='#5555FF'>*</font>g<font color='#5555FF'>*</font>g;
ps[i] <font color='#5555FF'>=</font> <font color='#5555FF'>-</font>alpha<font color='#5555FF'>*</font>pm[i]<font color='#5555FF'>/</font><font face='Lucida Console'>(</font>std::<font color='#BB00BB'>sqrt</font><font face='Lucida Console'>(</font>pv[i]<font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> eps<font face='Lucida Console'>)</font>;
<b>}</b>
<b>}</b>
<font color='#009900'>// -----------------------------------------------------------------------------------
</font>
<font color='#0000FF'><u>void</u></font> <b><a name='batch_normalize_inference'></a>batch_normalize_inference</b> <font face='Lucida Console'>(</font>
<font color='#0000FF'>const</font> <font color='#0000FF'><u>double</u></font> eps,
resizable_tensor<font color='#5555FF'>&</font> dest,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> src,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> gamma,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> beta,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> running_means,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> running_variances
<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>
gamma.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font> <font color='#5555FF'>&</font><font color='#5555FF'>&</font>
gamma.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> src.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&</font><font color='#5555FF'>&</font>
gamma.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> src.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&</font><font color='#5555FF'>&</font>
gamma.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> src.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&</font><font color='#5555FF'>&</font>
<font color='#BB00BB'>have_same_dimensions</font><font face='Lucida Console'>(</font>gamma, beta<font face='Lucida Console'>)</font> <font color='#5555FF'>&</font><font color='#5555FF'>&</font>
<font color='#BB00BB'>have_same_dimensions</font><font face='Lucida Console'>(</font>gamma, running_means<font face='Lucida Console'>)</font> <font color='#5555FF'>&</font><font color='#5555FF'>&</font>
<font color='#BB00BB'>have_same_dimensions</font><font face='Lucida Console'>(</font>gamma, running_variances<font face='Lucida Console'>)</font> <font color='#5555FF'>&</font><font color='#5555FF'>&</font>
eps <font color='#5555FF'>></font> <font color='#979000'>0</font>,
"<font color='#CC0000'>\ngamma.num_samples(): </font>" <font color='#5555FF'><</font><font color='#5555FF'><</font> gamma.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'><</font><font color='#5555FF'><</font>
"<font color='#CC0000'>\ngamma.k(): </font>" <font color='#5555FF'><</font><font color='#5555FF'><</font> gamma.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'><</font><font color='#5555FF'><</font>
"<font color='#CC0000'>\ngamma.nr(): </font>" <font color='#5555FF'><</font><font color='#5555FF'><</font> gamma.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'><</font><font color='#5555FF'><</font>
"<font color='#CC0000'>\ngamma.nc(): </font>" <font color='#5555FF'><</font><font color='#5555FF'><</font> gamma.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'><</font><font color='#5555FF'><</font>
"<font color='#CC0000'>\nbeta.num_samples(): </font>" <font color='#5555FF'><</font><font color='#5555FF'><</font> beta.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'><</font><font color='#5555FF'><</font>
"<font color='#CC0000'>\nbeta.k(): </font>" <font color='#5555FF'><</font><font color='#5555FF'><</font> beta.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'><</font><font color='#5555FF'><</font>
"<font color='#CC0000'>\nbeta.nr(): </font>" <font color='#5555FF'><</font><font color='#5555FF'><</font> beta.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'><</font><font color='#5555FF'><</font>
"<font color='#CC0000'>\nbeta.nc(): </font>" <font color='#5555FF'><</font><font color='#5555FF'><</font> beta.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'><</font><font color='#5555FF'><</font>
"<font color='#CC0000'>\nrunning_means.num_samples(): </font>" <font color='#5555FF'><</font><font color='#5555FF'><</font> running_means.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'><</font><font color='#5555FF'><</font>
"<font color='#CC0000'>\nrunning_means.k(): </font>" <font color='#5555FF'><</font><font color='#5555FF'><</font> running_means.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'><</font><font color='#5555FF'><</font>
"<font color='#CC0000'>\nrunning_means.nr(): </font>" <font color='#5555FF'><</font><font color='#5555FF'><</font> running_means.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'><</font><font color='#5555FF'><</font>
"<font color='#CC0000'>\nrunning_means.nc(): </font>" <font color='#5555FF'><</font><font color='#5555FF'><</font> running_means.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'><</font><font color='#5555FF'><</font>
"<font color='#CC0000'>\nrunning_variances.num_samples(): </font>" <font color='#5555FF'><</font><font color='#5555FF'><</font> running_variances.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'><</font><font color='#5555FF'><</font>
"<font color='#CC0000'>\nrunning_variances.k(): </font>" <font color='#5555FF'><</font><font color='#5555FF'><</font> running_variances.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'><</font><font color='#5555FF'><</font>
"<font color='#CC0000'>\nrunning_variances.nr(): </font>" <font color='#5555FF'><</font><font color='#5555FF'><</font> running_variances.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'><</font><font color='#5555FF'><</font>
"<font color='#CC0000'>\nrunning_variances.nc(): </font>" <font color='#5555FF'><</font><font color='#5555FF'><</font> running_variances.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'><</font><font color='#5555FF'><</font>
"<font color='#CC0000'>\nsrc.k(): </font>" <font color='#5555FF'><</font><font color='#5555FF'><</font> src.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'><</font><font color='#5555FF'><</font>
"<font color='#CC0000'>\nsrc.nr(): </font>" <font color='#5555FF'><</font><font color='#5555FF'><</font> src.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'><</font><font color='#5555FF'><</font>
"<font color='#CC0000'>\nsrc.nc(): </font>" <font color='#5555FF'><</font><font color='#5555FF'><</font> src.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'><</font><font color='#5555FF'><</font>
"<font color='#CC0000'>\neps: </font>" <font color='#5555FF'><</font><font color='#5555FF'><</font> eps
<font face='Lucida Console'>)</font>;
dest.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>src<font face='Lucida Console'>)</font>;
<font color='#0000FF'>auto</font> d <font color='#5555FF'>=</font> dest.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>auto</font> s <font color='#5555FF'>=</font> src.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>auto</font> g <font color='#5555FF'>=</font> gamma.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>auto</font> b <font color='#5555FF'>=</font> beta.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>auto</font> m <font color='#5555FF'>=</font> running_means.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>auto</font> v <font color='#5555FF'>=</font> running_variances.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> num <font color='#5555FF'>=</font> src.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>src.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>src.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> n <font color='#5555FF'>=</font> <font color='#979000'>0</font>; n <font color='#5555FF'><</font> src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>n<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> k <font color='#5555FF'>=</font> <font color='#979000'>0</font>; k <font color='#5555FF'><</font> num; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>k<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#5555FF'>*</font>d <font color='#5555FF'>=</font> g[k]<font color='#5555FF'>*</font><font face='Lucida Console'>(</font><font color='#5555FF'>*</font>s <font color='#5555FF'>-</font> m[k]<font face='Lucida Console'>)</font><font color='#5555FF'>/</font>std::<font color='#BB00BB'>sqrt</font><font face='Lucida Console'>(</font>v[k]<font color='#5555FF'>+</font>eps<font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> b[k];
<font color='#5555FF'>+</font><font color='#5555FF'>+</font>d;
<font color='#5555FF'>+</font><font color='#5555FF'>+</font>s;
<b>}</b>
<b>}</b>
<b>}</b>
<font color='#0000FF'><u>void</u></font> <b><a name='batch_normalize'></a>batch_normalize</b> <font face='Lucida Console'>(</font>
<font color='#0000FF'>const</font> <font color='#0000FF'><u>double</u></font> eps,
resizable_tensor<font color='#5555FF'>&</font> dest,
resizable_tensor<font color='#5555FF'>&</font> means,
resizable_tensor<font color='#5555FF'>&</font> invstds,
<font color='#0000FF'>const</font> <font color='#0000FF'><u>double</u></font> averaging_factor,
resizable_tensor<font color='#5555FF'>&</font> running_means,
resizable_tensor<font color='#5555FF'>&</font> running_variances,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> src,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> gamma,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> beta
<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font><font color='#979000'>0</font> <font color='#5555FF'><</font><font color='#5555FF'>=</font> averaging_factor <font color='#5555FF'>&</font><font color='#5555FF'>&</font> averaging_factor <font color='#5555FF'><</font><font color='#5555FF'>=</font> <font color='#979000'>1</font>, "<font color='#CC0000'>averaging_factor: </font>" <font color='#5555FF'><</font><font color='#5555FF'><</font> averaging_factor<font face='Lucida Console'>)</font>;
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>averaging_factor<font color='#5555FF'>=</font><font color='#5555FF'>=</font><font color='#979000'>1</font> <font color='#5555FF'>|</font><font color='#5555FF'>|</font> <font color='#BB00BB'>have_same_dimensions</font><font face='Lucida Console'>(</font>running_means,means<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>averaging_factor<font color='#5555FF'>=</font><font color='#5555FF'>=</font><font color='#979000'>1</font> <font color='#5555FF'>|</font><font color='#5555FF'>|</font> <font color='#BB00BB'>have_same_dimensions</font><font face='Lucida Console'>(</font>running_variances,invstds<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>
src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>></font> <font color='#979000'>1</font> <font color='#5555FF'>&</font><font color='#5555FF'>&</font>
gamma.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font> <font color='#5555FF'>&</font><font color='#5555FF'>&</font>
beta.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font> <font color='#5555FF'>&</font><font color='#5555FF'>&</font>
gamma.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> beta.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&</font><font color='#5555FF'>&</font> beta.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> src.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&</font><font color='#5555FF'>&</font>
gamma.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> beta.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&</font><font color='#5555FF'>&</font> beta.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> src.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&</font><font color='#5555FF'>&</font>
gamma.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> beta.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&</font><font color='#5555FF'>&</font> beta.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> src.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&</font><font color='#5555FF'>&</font>
eps <font color='#5555FF'>></font> <font color='#979000'>0</font>,
"<font color='#CC0000'>\ngamma.num_samples(): </font>" <font color='#5555FF'><</font><font color='#5555FF'><</font> gamma.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'><</font><font color='#5555FF'><</font>
"<font color='#CC0000'>\ngamma.k(): </font>" <font color='#5555FF'><</font><font color='#5555FF'><</font> gamma.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'><</font><font color='#5555FF'><</font>
"<font color='#CC0000'>\ngamma.nr(): </font>" <font color='#5555FF'><</font><font color='#5555FF'><</font> gamma.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'><</font><font color='#5555FF'><</font>
"<font color='#CC0000'>\ngamma.nc(): </font>" <font color='#5555FF'><</font><font color='#5555FF'><</font> gamma.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'><</font><font color='#5555FF'><</font>
"<font color='#CC0000'>\nbeta.num_samples(): </font>" <font color='#5555FF'><</font><font color='#5555FF'><</font> beta.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'><</font><font color='#5555FF'><</font>
"<font color='#CC0000'>\nbeta.k(): </font>" <font color='#5555FF'><</font><font color='#5555FF'><</font> beta.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'><</font><font color='#5555FF'><</font>
"<font color='#CC0000'>\nbeta.nr(): </font>" <font color='#5555FF'><</font><font color='#5555FF'><</font> beta.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'><</font><font color='#5555FF'><</font>
"<font color='#CC0000'>\nbeta.nc(): </font>" <font color='#5555FF'><</font><font color='#5555FF'><</font> beta.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'><</font><font color='#5555FF'><</font>
"<font color='#CC0000'>\nsrc.k(): </font>" <font color='#5555FF'><</font><font color='#5555FF'><</font> src.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'><</font><font color='#5555FF'><</font>
"<font color='#CC0000'>\nsrc.nr(): </font>" <font color='#5555FF'><</font><font color='#5555FF'><</font> src.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'><</font><font color='#5555FF'><</font>
"<font color='#CC0000'>\nsrc.nc(): </font>" <font color='#5555FF'><</font><font color='#5555FF'><</font> src.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'><</font><font color='#5555FF'><</font>
"<font color='#CC0000'>\neps: </font>" <font color='#5555FF'><</font><font color='#5555FF'><</font> eps
<font face='Lucida Console'>)</font>;
dest.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>src<font face='Lucida Console'>)</font>;
means.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>, src.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, src.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, src.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
invstds.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>, src.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, src.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, src.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
<font color='#009900'>// first compute means and invstds
</font> means <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
invstds <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
<font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> p_invstds <font color='#5555FF'>=</font> invstds.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> p_means <font color='#5555FF'>=</font> means.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>auto</font> p_src <font color='#5555FF'>=</font> src.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> num <font color='#5555FF'>=</font> src.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>src.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>src.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#009900'>// compute means, and sum of squares
</font> <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'><</font> num; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> n <font color='#5555FF'>=</font> <font color='#979000'>0</font>; n <font color='#5555FF'><</font> src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>n<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'><u>float</u></font> val <font color='#5555FF'>=</font> p_src[n<font color='#5555FF'>*</font>num<font color='#5555FF'>+</font>i];
p_means[i] <font color='#5555FF'>+</font><font color='#5555FF'>=</font> val;
p_invstds[i] <font color='#5555FF'>+</font><font color='#5555FF'>=</font> val<font color='#5555FF'>*</font>val;
<b>}</b>
<b>}</b>
means <font color='#5555FF'>/</font><font color='#5555FF'>=</font> src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
invstds <font color='#5555FF'>/</font><font color='#5555FF'>=</font> src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#009900'>// copy data back to host
</font> invstds.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; means.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#009900'>// compute variances
</font> running_variances.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>invstds<font face='Lucida Console'>)</font>;
<font color='#0000FF'>auto</font> rvar <font color='#5555FF'>=</font> running_variances.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#009900'>// This scale makes the running variances unbiased.
</font> <font color='#0000FF'>const</font> <font color='#0000FF'><u>double</u></font> scale <font color='#5555FF'>=</font> <font face='Lucida Console'>(</font>src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font><font face='Lucida Console'>(</font>src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#979000'>1.0</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'><</font> num; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>auto</font> actual_var <font color='#5555FF'>=</font> p_invstds[i] <font color='#5555FF'>-</font> p_means[i]<font color='#5555FF'>*</font>p_means[i];
<font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>averaging_factor <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>
rvar[i] <font color='#5555FF'>=</font> scale<font color='#5555FF'>*</font>actual_var;
<font color='#0000FF'>else</font>
rvar[i] <font color='#5555FF'>=</font> <font face='Lucida Console'>(</font><font color='#979000'>1</font><font color='#5555FF'>-</font>averaging_factor<font face='Lucida Console'>)</font><font color='#5555FF'>*</font>rvar[i] <font color='#5555FF'>+</font> scale<font color='#5555FF'>*</font>averaging_factor<font color='#5555FF'>*</font>actual_var;
p_invstds[i] <font color='#5555FF'>=</font> <font color='#979000'>1.0f</font><font color='#5555FF'>/</font>std::<font color='#BB00BB'>sqrt</font><font face='Lucida Console'>(</font>actual_var <font color='#5555FF'>+</font> eps<font face='Lucida Console'>)</font>;
<b>}</b>
p_src <font color='#5555FF'>=</font> src.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>auto</font> p_dest <font color='#5555FF'>=</font> dest.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> p_gamma <font color='#5555FF'>=</font> gamma.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> p_beta <font color='#5555FF'>=</font> beta.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> n <font color='#5555FF'>=</font> <font color='#979000'>0</font>; n <font color='#5555FF'><</font> src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>n<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'><</font> num; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#5555FF'>*</font>p_dest <font color='#5555FF'>=</font> <font face='Lucida Console'>(</font><font color='#5555FF'>*</font>p_src <font color='#5555FF'>-</font> p_means[i]<font face='Lucida Console'>)</font><font color='#5555FF'>*</font>p_invstds[i];
<font color='#5555FF'>*</font>p_dest <font color='#5555FF'>=</font> <font face='Lucida Console'>(</font><font color='#5555FF'>*</font>p_dest<font face='Lucida Console'>)</font><font color='#5555FF'>*</font>p_gamma[i] <font color='#5555FF'>+</font> p_beta[i];
<font color='#5555FF'>+</font><font color='#5555FF'>+</font>p_src;
<font color='#5555FF'>+</font><font color='#5555FF'>+</font>p_dest;
<b>}</b>
<b>}</b>
<font color='#009900'>// now keep track of the running means
</font> running_means.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>means<font face='Lucida Console'>)</font>;
<font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>averaging_factor <font color='#5555FF'>!</font><font color='#5555FF'>=</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>
running_means <font color='#5555FF'>=</font> <font face='Lucida Console'>(</font><font color='#979000'>1</font><font color='#5555FF'>-</font>averaging_factor<font face='Lucida Console'>)</font><font color='#5555FF'>*</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>running_means<font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> averaging_factor<font color='#5555FF'>*</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>means<font face='Lucida Console'>)</font>;
<font color='#0000FF'>else</font>
running_means <font color='#5555FF'>=</font> means;
<b>}</b>
<font color='#0000FF'><u>void</u></font> <b><a name='batch_normalize_gradient'></a>batch_normalize_gradient</b> <font face='Lucida Console'>(</font>
<font color='#0000FF'>const</font> <font color='#0000FF'><u>double</u></font> eps,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> gradient_input,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> means,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> invstds,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> src,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> gamma,
tensor<font color='#5555FF'>&</font> src_grad,
tensor<font color='#5555FF'>&</font> gamma_grad,
tensor<font color='#5555FF'>&</font> beta_grad
<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> num <font color='#5555FF'>=</font> src.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>src.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>src.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>></font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>;
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>num <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font><font face='Lucida Console'>)</font>means.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>num <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font><font face='Lucida Console'>)</font>invstds.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>num <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font><font face='Lucida Console'>)</font>gamma.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>num <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font><font face='Lucida Console'>)</font>gamma_grad.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>num <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font><font face='Lucida Console'>)</font>beta_grad.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font><font color='#BB00BB'>have_same_dimensions</font><font face='Lucida Console'>(</font>gradient_input, src<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font><font color='#BB00BB'>have_same_dimensions</font><font face='Lucida Console'>(</font>gradient_input, src_grad<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>eps <font color='#5555FF'>></font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
beta_grad <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
gamma_grad <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
<font color='#0000FF'>auto</font> p_grad <font color='#5555FF'>=</font> gradient_input.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>auto</font> p_src <font color='#5555FF'>=</font> src.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> p_gamma <font color='#5555FF'>=</font> gamma.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> p_gamma_grad <font color='#5555FF'>=</font> gamma_grad.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> p_beta_grad <font color='#5555FF'>=</font> beta_grad.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> p_invstds <font color='#5555FF'>=</font> invstds.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> p_means <font color='#5555FF'>=</font> means.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
resizable_tensor dvars, dmeans;
dvars.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>invstds<font face='Lucida Console'>)</font>;
dmeans.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>means<font face='Lucida Console'>)</font>;
dvars <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
dmeans <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
<font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> p_dvars <font color='#5555FF'>=</font> dvars.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> p_dmeans <font color='#5555FF'>=</font> dmeans.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> n <font color='#5555FF'>=</font> <font color='#979000'>0</font>; n <font color='#5555FF'><</font> src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>n<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'><</font> num; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> x_hat <font color='#5555FF'>=</font> <font face='Lucida Console'>(</font><font color='#5555FF'>*</font>p_src <font color='#5555FF'>-</font> p_means[i]<font face='Lucida Console'>)</font><font color='#5555FF'>*</font>p_invstds[i];
p_beta_grad[i] <font color='#5555FF'>+</font><font color='#5555FF'>=</font> <font color='#5555FF'>*</font>p_grad;
p_gamma_grad[i] <font color='#5555FF'>+</font><font color='#5555FF'>=</font> <font face='Lucida Console'>(</font><font color='#5555FF'>*</font>p_grad<font face='Lucida Console'>)</font><font color='#5555FF'>*</font>x_hat;
<font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> dx <font color='#5555FF'>=</font> <font color='#5555FF'>*</font>p_grad <font color='#5555FF'>*</font> p_gamma[i];
p_dvars[i] <font color='#5555FF'>+</font><font color='#5555FF'>=</font> dx<font color='#5555FF'>*</font><font face='Lucida Console'>(</font><font color='#5555FF'>*</font>p_src <font color='#5555FF'>-</font> p_means[i]<font face='Lucida Console'>)</font><font color='#5555FF'>*</font><font color='#5555FF'>-</font><font color='#979000'>0.5</font><font color='#5555FF'>*</font>std::<font color='#BB00BB'>pow</font><font face='Lucida Console'>(</font>p_invstds[i], <font color='#979000'>3.0f</font><font face='Lucida Console'>)</font>;
<font color='#5555FF'>+</font><font color='#5555FF'>+</font>p_grad;
<font color='#5555FF'>+</font><font color='#5555FF'>+</font>p_src;
<b>}</b>
<b>}</b>
<font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> invnum <font color='#5555FF'>=</font> <font color='#979000'>1.0f</font><font color='#5555FF'>/</font>src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
p_grad <font color='#5555FF'>=</font> gradient_input.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
p_src <font color='#5555FF'>=</font> src.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> n <font color='#5555FF'>=</font> <font color='#979000'>0</font>; n <font color='#5555FF'><</font> src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>n<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'><</font> num; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> dx <font color='#5555FF'>=</font> <font color='#5555FF'>*</font>p_grad <font color='#5555FF'>*</font> p_gamma[i];
p_dmeans[i] <font color='#5555FF'>+</font><font color='#5555FF'>=</font> dx<font color='#5555FF'>*</font><font color='#5555FF'>-</font>p_invstds[i] <font color='#5555FF'>+</font> p_dvars[i] <font color='#5555FF'>*</font> <font color='#5555FF'>-</font><font color='#979000'>2</font><font color='#5555FF'>*</font><font face='Lucida Console'>(</font><font color='#5555FF'>*</font>p_src <font color='#5555FF'>-</font> p_means[i]<font face='Lucida Console'>)</font><font color='#5555FF'>*</font>invnum;
<font color='#5555FF'>+</font><font color='#5555FF'>+</font>p_grad;
<font color='#5555FF'>+</font><font color='#5555FF'>+</font>p_src;
<b>}</b>
<b>}</b>
p_grad <font color='#5555FF'>=</font> gradient_input.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
p_src <font color='#5555FF'>=</font> src.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>auto</font> p_src_grad <font color='#5555FF'>=</font> src_grad.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> n <font color='#5555FF'>=</font> <font color='#979000'>0</font>; n <font color='#5555FF'><</font> src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>n<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'><</font> num; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> dx <font color='#5555FF'>=</font> <font color='#5555FF'>*</font>p_grad <font color='#5555FF'>*</font> p_gamma[i];
<font color='#5555FF'>*</font>p_src_grad <font color='#5555FF'>+</font><font color='#5555FF'>=</font> dx<font color='#5555FF'>*</font>p_invstds[i] <font color='#5555FF'>+</font>
p_dvars[i] <font color='#5555FF'>*</font><font color='#979000'>2</font><font color='#5555FF'>*</font><font face='Lucida Console'>(</font><font color='#5555FF'>*</font>p_src <font color='#5555FF'>-</font> p_means[i]<font face='Lucida Console'>)</font><font color='#5555FF'>*</font>invnum <font color='#5555FF'>+</font>
p_dmeans[i]<font color='#5555FF'>*</font>invnum;
<font color='#5555FF'>+</font><font color='#5555FF'>+</font>p_grad;
<font color='#5555FF'>+</font><font color='#5555FF'>+</font>p_src;
<font color='#5555FF'>+</font><font color='#5555FF'>+</font>p_src_grad;
<b>}</b>
<b>}</b>
<b>}</b>
<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
<font color='#0000FF'><u>void</u></font> <b><a name='batch_normalize_conv_inference'></a>batch_normalize_conv_inference</b> <font face='Lucida Console'>(</font>
<font color='#0000FF'>const</font> <font color='#0000FF'><u>double</u></font> eps,
resizable_tensor<font color='#5555FF'>&</font> dest,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> src,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> gamma,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> beta,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> running_means,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> running_variances
<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>
gamma.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font> <font color='#5555FF'>&</font><font color='#5555FF'>&</font>
gamma.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font> <font color='#5555FF'>&</font><font color='#5555FF'>&</font>
gamma.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font> <font color='#5555FF'>&</font><font color='#5555FF'>&</font>
gamma.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> src.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&</font><font color='#5555FF'>&</font>
<font color='#BB00BB'>have_same_dimensions</font><font face='Lucida Console'>(</font>gamma, beta<font face='Lucida Console'>)</font> <font color='#5555FF'>&</font><font color='#5555FF'>&</font>
<font color='#BB00BB'>have_same_dimensions</font><font face='Lucida Console'>(</font>gamma, running_means<font face='Lucida Console'>)</font> <font color='#5555FF'>&</font><font color='#5555FF'>&</font>
<font color='#BB00BB'>have_same_dimensions</font><font face='Lucida Console'>(</font>gamma, running_variances<font face='Lucida Console'>)</font> <font color='#5555FF'>&</font><font color='#5555FF'>&</font>
eps <font color='#5555FF'>></font> <font color='#979000'>0</font>,
"<font color='#CC0000'>\ngamma.num_samples(): </font>" <font color='#5555FF'><</font><font color='#5555FF'><</font> gamma.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'><</font><font color='#5555FF'><</font>
"<font color='#CC0000'>\ngamma.k(): </font>" <font color='#5555FF'><</font><font color='#5555FF'><</font> gamma.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'><</font><font color='#5555FF'><</font>
"<font color='#CC0000'>\ngamma.nr(): </font>" <font color='#5555FF'><</font><font color='#5555FF'><</font> gamma.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'><</font><font color='#5555FF'><</font>
"<font color='#CC0000'>\ngamma.nc(): </font>" <font color='#5555FF'><</font><font color='#5555FF'><</font> gamma.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'><</font><font color='#5555FF'><</font>
"<font color='#CC0000'>\nbeta.num_samples(): </font>" <font color='#5555FF'><</font><font color='#5555FF'><</font> beta.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'><</font><font color='#5555FF'><</font>
"<font color='#CC0000'>\nbeta.k(): </font>" <font color='#5555FF'><</font><font color='#5555FF'><</font> beta.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'><</font><font color='#5555FF'><</font>
"<font color='#CC0000'>\nbeta.nr(): </font>" <font color='#5555FF'><</font><font color='#5555FF'><</font> beta.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'><</font><font color='#5555FF'><</font>
"<font color='#CC0000'>\nbeta.nc(): </font>" <font color='#5555FF'><</font><font color='#5555FF'><</font> beta.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'><</font><font color='#5555FF'><</font>
"<font color='#CC0000'>\nrunning_means.num_samples(): </font>" <font color='#5555FF'><</font><font color='#5555FF'><</font> running_means.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'><</font><font color='#5555FF'><</font>
"<font color='#CC0000'>\nrunning_means.k(): </font>" <font color='#5555FF'><</font><font color='#5555FF'><</font> running_means.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'><</font><font color='#5555FF'><</font>
"<font color='#CC0000'>\nrunning_means.nr(): </font>" <font color='#5555FF'><</font><font color='#5555FF'><</font> running_means.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'><</font><font color='#5555FF'><</font>
"<font color='#CC0000'>\nrunning_means.nc(): </font>" <font color='#5555FF'><</font><font color='#5555FF'><</font> running_means.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'><</font><font color='#5555FF'><</font>
"<font color='#CC0000'>\nrunning_variances.num_samples(): </font>" <font color='#5555FF'><</font><font color='#5555FF'><</font> running_variances.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'><</font><font color='#5555FF'><</font>
"<font color='#CC0000'>\nrunning_variances.k(): </font>" <font color='#5555FF'><</font><font color='#5555FF'><</font> running_variances.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'><</font><font color='#5555FF'><</font>
"<font color='#CC0000'>\nrunning_variances.nr(): </font>" <font color='#5555FF'><</font><font color='#5555FF'><</font> running_variances.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'><</font><font color='#5555FF'><</font>
"<font color='#CC0000'>\nrunning_variances.nc(): </font>" <font color='#5555FF'><</font><font color='#5555FF'><</font> running_variances.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'><</font><font color='#5555FF'><</font>
"<font color='#CC0000'>\nsrc.k(): </font>" <font color='#5555FF'><</font><font color='#5555FF'><</font> src.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'><</font><font color='#5555FF'><</font>
"<font color='#CC0000'>\nsrc.nr(): </font>" <font color='#5555FF'><</font><font color='#5555FF'><</font> src.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'><</font><font color='#5555FF'><</font>
"<font color='#CC0000'>\nsrc.nc(): </font>" <font color='#5555FF'><</font><font color='#5555FF'><</font> src.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'><</font><font color='#5555FF'><</font>
"<font color='#CC0000'>\neps: </font>" <font color='#5555FF'><</font><font color='#5555FF'><</font> eps
<font face='Lucida Console'>)</font>;
dest.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>src<font face='Lucida Console'>)</font>;
<font color='#0000FF'>auto</font> d <font color='#5555FF'>=</font> dest.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>auto</font> s <font color='#5555FF'>=</font> src.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>auto</font> g <font color='#5555FF'>=</font> gamma.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>auto</font> b <font color='#5555FF'>=</font> beta.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>auto</font> m <font color='#5555FF'>=</font> running_means.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>auto</font> v <font color='#5555FF'>=</font> running_variances.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> num <font color='#5555FF'>=</font> src.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>src.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> n <font color='#5555FF'>=</font> <font color='#979000'>0</font>; n <font color='#5555FF'><</font> src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>n<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> k <font color='#5555FF'>=</font> <font color='#979000'>0</font>; k <font color='#5555FF'><</font> src.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>k<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> invstd <font color='#5555FF'>=</font> <font color='#979000'>1.0f</font><font color='#5555FF'>/</font>std::<font color='#BB00BB'>sqrt</font><font face='Lucida Console'>(</font>v[k] <font color='#5555FF'>+</font> eps<font face='Lucida Console'>)</font>;
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> j <font color='#5555FF'>=</font> <font color='#979000'>0</font>; j <font color='#5555FF'><</font> num; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>j<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#5555FF'>*</font>d <font color='#5555FF'>=</font> g[k]<font color='#5555FF'>*</font><font face='Lucida Console'>(</font><font color='#5555FF'>*</font>s <font color='#5555FF'>-</font> m[k]<font face='Lucida Console'>)</font><font color='#5555FF'>*</font>invstd <font color='#5555FF'>+</font> b[k];
<font color='#5555FF'>+</font><font color='#5555FF'>+</font>d;
<font color='#5555FF'>+</font><font color='#5555FF'>+</font>s;
<b>}</b>
<b>}</b>
<b>}</b>
<b>}</b>
<font color='#0000FF'><u>void</u></font> <b><a name='batch_normalize_conv'></a>batch_normalize_conv</b> <font face='Lucida Console'>(</font>
<font color='#0000FF'>const</font> <font color='#0000FF'><u>double</u></font> eps,
resizable_tensor<font color='#5555FF'>&</font> dest,
resizable_tensor<font color='#5555FF'>&</font> means,
resizable_tensor<font color='#5555FF'>&</font> invstds,
<font color='#0000FF'>const</font> <font color='#0000FF'><u>double</u></font> averaging_factor,
resizable_tensor<font color='#5555FF'>&</font> running_means,
resizable_tensor<font color='#5555FF'>&</font> running_variances,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> src,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> gamma,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> beta
<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font><font color='#979000'>0</font> <font color='#5555FF'><</font><font color='#5555FF'>=</font> averaging_factor <font color='#5555FF'>&</font><font color='#5555FF'>&</font> averaging_factor <font color='#5555FF'><</font><font color='#5555FF'>=</font> <font color='#979000'>1</font>, "<font color='#CC0000'>averaging_factor: </font>" <font color='#5555FF'><</font><font color='#5555FF'><</font> averaging_factor<font face='Lucida Console'>)</font>;
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>averaging_factor<font color='#5555FF'>=</font><font color='#5555FF'>=</font><font color='#979000'>1</font> <font color='#5555FF'>|</font><font color='#5555FF'>|</font> <font color='#BB00BB'>have_same_dimensions</font><font face='Lucida Console'>(</font>running_means,means<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>averaging_factor<font color='#5555FF'>=</font><font color='#5555FF'>=</font><font color='#979000'>1</font> <font color='#5555FF'>|</font><font color='#5555FF'>|</font> <font color='#BB00BB'>have_same_dimensions</font><font face='Lucida Console'>(</font>running_variances,invstds<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>
src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>></font> <font color='#979000'>1</font> <font color='#5555FF'>&</font><font color='#5555FF'>&</font>
gamma.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font> <font color='#5555FF'>&</font><font color='#5555FF'>&</font>
beta.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font> <font color='#5555FF'>&</font><font color='#5555FF'>&</font>
gamma.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font> <font color='#5555FF'>&</font><font color='#5555FF'>&</font>
beta.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font> <font color='#5555FF'>&</font><font color='#5555FF'>&</font>
gamma.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font> <font color='#5555FF'>&</font><font color='#5555FF'>&</font>
beta.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font> <font color='#5555FF'>&</font><font color='#5555FF'>&</font>
gamma.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> beta.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&</font><font color='#5555FF'>&</font> beta.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> src.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&</font><font color='#5555FF'>&</font>
eps <font color='#5555FF'>></font> <font color='#979000'>0</font>,
"<font color='#CC0000'>\ngamma.num_samples(): </font>" <font color='#5555FF'><</font><font color='#5555FF'><</font> gamma.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'><</font><font color='#5555FF'><</font>
"<font color='#CC0000'>\ngamma.k(): </font>" <font color='#5555FF'><</font><font color='#5555FF'><</font> gamma.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'><</font><font color='#5555FF'><</font>
"<font color='#CC0000'>\ngamma.nr(): </font>" <font color='#5555FF'><</font><font color='#5555FF'><</font> gamma.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'><</font><font color='#5555FF'><</font>
"<font color='#CC0000'>\ngamma.nc(): </font>" <font color='#5555FF'><</font><font color='#5555FF'><</font> gamma.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'><</font><font color='#5555FF'><</font>
"<font color='#CC0000'>\nbeta.num_samples(): </font>" <font color='#5555FF'><</font><font color='#5555FF'><</font> beta.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'><</font><font color='#5555FF'><</font>
"<font color='#CC0000'>\nbeta.k(): </font>" <font color='#5555FF'><</font><font color='#5555FF'><</font> beta.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'><</font><font color='#5555FF'><</font>
"<font color='#CC0000'>\nbeta.nr(): </font>" <font color='#5555FF'><</font><font color='#5555FF'><</font> beta.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'><</font><font color='#5555FF'><</font>
"<font color='#CC0000'>\nbeta.nc(): </font>" <font color='#5555FF'><</font><font color='#5555FF'><</font> beta.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'><</font><font color='#5555FF'><</font>
"<font color='#CC0000'>\nsrc.k(): </font>" <font color='#5555FF'><</font><font color='#5555FF'><</font> src.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'><</font><font color='#5555FF'><</font>
"<font color='#CC0000'>\nsrc.nr(): </font>" <font color='#5555FF'><</font><font color='#5555FF'><</font> src.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'><</font><font color='#5555FF'><</font>
"<font color='#CC0000'>\nsrc.nc(): </font>" <font color='#5555FF'><</font><font color='#5555FF'><</font> src.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'><</font><font color='#5555FF'><</font>
"<font color='#CC0000'>\neps: </font>" <font color='#5555FF'><</font><font color='#5555FF'><</font> eps
<font face='Lucida Console'>)</font>;
dest.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>src<font face='Lucida Console'>)</font>;
means.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>, src.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
invstds.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>, src.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
<font color='#009900'>// first compute means and invstds
</font> means <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
invstds <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
<font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> p_invstds <font color='#5555FF'>=</font> invstds.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> p_means <font color='#5555FF'>=</font> means.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> p_gamma <font color='#5555FF'>=</font> gamma.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> p_beta <font color='#5555FF'>=</font> beta.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>auto</font> p_src <font color='#5555FF'>=</font> src.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> num <font color='#5555FF'>=</font> src.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>src.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#009900'>// compute means, and sum of squares
</font> <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> n <font color='#5555FF'>=</font> <font color='#979000'>0</font>; n <font color='#5555FF'><</font> src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>n<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> k <font color='#5555FF'>=</font> <font color='#979000'>0</font>; k <font color='#5555FF'><</font> src.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>k<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'><</font> num; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
<b>{</b>
p_means[k] <font color='#5555FF'>+</font><font color='#5555FF'>=</font> <font color='#5555FF'>*</font>p_src;
p_invstds[k] <font color='#5555FF'>+</font><font color='#5555FF'>=</font> <font face='Lucida Console'>(</font><font color='#5555FF'>*</font>p_src<font face='Lucida Console'>)</font><font color='#5555FF'>*</font><font face='Lucida Console'>(</font><font color='#5555FF'>*</font>p_src<font face='Lucida Console'>)</font>;
<font color='#5555FF'>+</font><font color='#5555FF'>+</font>p_src;
<b>}</b>
<b>}</b>
<b>}</b>
means <font color='#5555FF'>/</font><font color='#5555FF'>=</font> src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>num;
invstds <font color='#5555FF'>/</font><font color='#5555FF'>=</font> src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>num;
<font color='#009900'>// copy data back to host
</font> invstds.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; means.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
p_src <font color='#5555FF'>=</font> src.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#009900'>// compute variances
</font> running_variances.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>invstds<font face='Lucida Console'>)</font>;
<font color='#0000FF'>auto</font> rvar <font color='#5555FF'>=</font> running_variances.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#009900'>// This scale makes the running variances unbiased.
</font> <font color='#0000FF'>const</font> <font color='#0000FF'><u>double</u></font> scale <font color='#5555FF'>=</font> <font face='Lucida Console'>(</font>src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>num<font face='Lucida Console'>)</font><font color='#5555FF'>/</font><font face='Lucida Console'>(</font>src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>num<font color='#5555FF'>-</font><font color='#979000'>1.0</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> k <font color='#5555FF'>=</font> <font color='#979000'>0</font>; k <font color='#5555FF'><</font> src.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>k<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'><u>float</u></font> actual_var <font color='#5555FF'>=</font> p_invstds[k] <font color='#5555FF'>-</font> p_means[k]<font color='#5555FF'>*</font>p_means[k];
<font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>averaging_factor <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>
rvar[k] <font color='#5555FF'>=</font> scale<font color='#5555FF'>*</font>actual_var;
<font color='#0000FF'>else</font>
rvar[k] <font color='#5555FF'>=</font> <font face='Lucida Console'>(</font><font color='#979000'>1</font><font color='#5555FF'>-</font>averaging_factor<font face='Lucida Console'>)</font><font color='#5555FF'>*</font>rvar[k] <font color='#5555FF'>+</font> scale<font color='#5555FF'>*</font>averaging_factor<font color='#5555FF'>*</font>actual_var;
p_invstds[k] <font color='#5555FF'>=</font> <font color='#979000'>1.0f</font><font color='#5555FF'>/</font>std::<font color='#BB00BB'>sqrt</font><font face='Lucida Console'>(</font>actual_var <font color='#5555FF'>+</font> eps<font face='Lucida Console'>)</font>;
<b>}</b>
p_src <font color='#5555FF'>=</font> src.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>auto</font> p_dest <font color='#5555FF'>=</font> dest.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> n <font color='#5555FF'>=</font> <font color='#979000'>0</font>; n <font color='#5555FF'><</font> src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>n<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> k <font color='#5555FF'>=</font> <font color='#979000'>0</font>; k <font color='#5555FF'><</font> src.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>k<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'><</font> num; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#5555FF'>*</font>p_dest <font color='#5555FF'>=</font> <font face='Lucida Console'>(</font><font color='#5555FF'>*</font>p_src <font color='#5555FF'>-</font> p_means[k]<font face='Lucida Console'>)</font><font color='#5555FF'>*</font>p_invstds[k];
<font color='#5555FF'>*</font>p_dest <font color='#5555FF'>=</font> <font face='Lucida Console'>(</font><font color='#5555FF'>*</font>p_dest<font face='Lucida Console'>)</font><font color='#5555FF'>*</font>p_gamma[k] <font color='#5555FF'>+</font> p_beta[k];
<font color='#5555FF'>+</font><font color='#5555FF'>+</font>p_src;
<font color='#5555FF'>+</font><font color='#5555FF'>+</font>p_dest;
<b>}</b>
<b>}</b>
<b>}</b>
<font color='#009900'>// now keep track of the running means
</font> running_means.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>means<font face='Lucida Console'>)</font>;
<font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>averaging_factor <font color='#5555FF'>!</font><font color='#5555FF'>=</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>
running_means <font color='#5555FF'>=</font> <font face='Lucida Console'>(</font><font color='#979000'>1</font><font color='#5555FF'>-</font>averaging_factor<font face='Lucida Console'>)</font><font color='#5555FF'>*</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>running_means<font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> averaging_factor<font color='#5555FF'>*</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>means<font face='Lucida Console'>)</font>;
<font color='#0000FF'>else</font>
running_means <font color='#5555FF'>=</font> means;
<b>}</b>
<font color='#0000FF'><u>void</u></font> <b><a name='batch_normalize_conv_gradient'></a>batch_normalize_conv_gradient</b><font face='Lucida Console'>(</font>
<font color='#0000FF'>const</font> <font color='#0000FF'><u>double</u></font> eps,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> gradient_input,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> means,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> invstds,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> src,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> gamma,
tensor<font color='#5555FF'>&</font> src_grad,
tensor<font color='#5555FF'>&</font> gamma_grad,
tensor<font color='#5555FF'>&</font> beta_grad
<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> num <font color='#5555FF'>=</font> src.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>src.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>></font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>;
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>src.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font><font face='Lucida Console'>)</font>means.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>src.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font><font face='Lucida Console'>)</font>invstds.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>src.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font><font face='Lucida Console'>)</font>gamma.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>src.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font><font face='Lucida Console'>)</font>gamma_grad.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>src.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font><font face='Lucida Console'>)</font>beta_grad.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font><font color='#BB00BB'>have_same_dimensions</font><font face='Lucida Console'>(</font>gradient_input, src<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font><font color='#BB00BB'>have_same_dimensions</font><font face='Lucida Console'>(</font>gradient_input, src_grad<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>eps <font color='#5555FF'>></font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
beta_grad <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
gamma_grad <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
<font color='#0000FF'>auto</font> p_grad <font color='#5555FF'>=</font> gradient_input.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>auto</font> p_src <font color='#5555FF'>=</font> src.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> p_gamma <font color='#5555FF'>=</font> gamma.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> p_gamma_grad <font color='#5555FF'>=</font> gamma_grad.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> p_beta_grad <font color='#5555FF'>=</font> beta_grad.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> p_invstds <font color='#5555FF'>=</font> invstds.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> p_means <font color='#5555FF'>=</font> means.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
resizable_tensor dvars, dmeans;
dvars.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>invstds<font face='Lucida Console'>)</font>;
dmeans.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>means<font face='Lucida Console'>)</font>;
dvars <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
dmeans <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
<font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> p_dvars <font color='#5555FF'>=</font> dvars.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> p_dmeans <font color='#5555FF'>=</font> dmeans.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> n <font color='#5555FF'>=</font> <font color='#979000'>0</font>; n <font color='#5555FF'><</font> src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>n<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> k <font color='#5555FF'>=</font> <font color='#979000'>0</font>; k <font color='#5555FF'><</font> src.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>k<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> invstd_pow <font color='#5555FF'>=</font> <font color='#5555FF'>-</font><font color='#979000'>0.5</font><font color='#5555FF'>*</font>std::<font color='#BB00BB'>pow</font><font face='Lucida Console'>(</font>p_invstds[k], <font color='#979000'>3.0f</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'><</font> num; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> x_hat <font color='#5555FF'>=</font> <font face='Lucida Console'>(</font><font color='#5555FF'>*</font>p_src <font color='#5555FF'>-</font> p_means[k]<font face='Lucida Console'>)</font><font color='#5555FF'>*</font>p_invstds[k];
p_beta_grad[k] <font color='#5555FF'>+</font><font color='#5555FF'>=</font> <font color='#5555FF'>*</font>p_grad;
p_gamma_grad[k] <font color='#5555FF'>+</font><font color='#5555FF'>=</font> <font face='Lucida Console'>(</font><font color='#5555FF'>*</font>p_grad<font face='Lucida Console'>)</font><font color='#5555FF'>*</font>x_hat;
<font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> dx <font color='#5555FF'>=</font> <font color='#5555FF'>*</font>p_grad <font color='#5555FF'>*</font> p_gamma[k];
p_dvars[k] <font color='#5555FF'>+</font><font color='#5555FF'>=</font> dx<font color='#5555FF'>*</font><font face='Lucida Console'>(</font><font color='#5555FF'>*</font>p_src <font color='#5555FF'>-</font> p_means[k]<font face='Lucida Console'>)</font><font color='#5555FF'>*</font>invstd_pow;
<font color='#5555FF'>+</font><font color='#5555FF'>+</font>p_grad;
<font color='#5555FF'>+</font><font color='#5555FF'>+</font>p_src;
<b>}</b>
<b>}</b>
<b>}</b>
p_grad <font color='#5555FF'>=</font> gradient_input.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
p_src <font color='#5555FF'>=</font> src.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> invnum <font color='#5555FF'>=</font> <font color='#979000'>1.0f</font><font color='#5555FF'>/</font><font face='Lucida Console'>(</font>src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>num<font face='Lucida Console'>)</font>;
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> n <font color='#5555FF'>=</font> <font color='#979000'>0</font>; n <font color='#5555FF'><</font> src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>n<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> k <font color='#5555FF'>=</font> <font color='#979000'>0</font>; k <font color='#5555FF'><</font> src.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>k<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'><</font> num; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> dx <font color='#5555FF'>=</font> <font color='#5555FF'>*</font>p_grad <font color='#5555FF'>*</font> p_gamma[k];
p_dmeans[k] <font color='#5555FF'>+</font><font color='#5555FF'>=</font> <font color='#5555FF'>-</font>dx<font color='#5555FF'>*</font>p_invstds[k] <font color='#5555FF'>+</font> p_dvars[k] <font color='#5555FF'>*</font> <font color='#5555FF'>-</font><font color='#979000'>2</font><font color='#5555FF'>*</font><font face='Lucida Console'>(</font><font color='#5555FF'>*</font>p_src <font color='#5555FF'>-</font> p_means[k]<font face='Lucida Console'>)</font><font color='#5555FF'>*</font>invnum;
<font color='#5555FF'>+</font><font color='#5555FF'>+</font>p_grad;
<font color='#5555FF'>+</font><font color='#5555FF'>+</font>p_src;
<b>}</b>
<b>}</b>
<b>}</b>
p_grad <font color='#5555FF'>=</font> gradient_input.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
p_src <font color='#5555FF'>=</font> src.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>auto</font> p_src_grad <font color='#5555FF'>=</font> src_grad.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> n <font color='#5555FF'>=</font> <font color='#979000'>0</font>; n <font color='#5555FF'><</font> src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>n<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> k <font color='#5555FF'>=</font> <font color='#979000'>0</font>; k <font color='#5555FF'><</font> src.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>k<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'><</font> num; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> dx <font color='#5555FF'>=</font> <font color='#5555FF'>*</font>p_grad <font color='#5555FF'>*</font> p_gamma[k];
<font color='#5555FF'>*</font>p_src_grad <font color='#5555FF'>+</font><font color='#5555FF'>=</font> dx<font color='#5555FF'>*</font>p_invstds[k] <font color='#5555FF'>+</font>
p_dvars[k]<font color='#5555FF'>*</font><font color='#979000'>2</font><font color='#5555FF'>*</font><font face='Lucida Console'>(</font><font color='#5555FF'>*</font>p_src <font color='#5555FF'>-</font> p_means[k]<font face='Lucida Console'>)</font><font color='#5555FF'>*</font>invnum <font color='#5555FF'>+</font>
p_dmeans[k]<font color='#5555FF'>*</font>invnum;
<font color='#5555FF'>+</font><font color='#5555FF'>+</font>p_grad;
<font color='#5555FF'>+</font><font color='#5555FF'>+</font>p_src;
<font color='#5555FF'>+</font><font color='#5555FF'>+</font>p_src_grad;
<b>}</b>
<b>}</b>
<b>}</b>
<b>}</b>
<font color='#009900'>// -----------------------------------------------------------------------------------
</font>
<font color='#0000FF'><u>void</u></font> <b><a name='layer_normalize'></a>layer_normalize</b> <font face='Lucida Console'>(</font>
<font color='#0000FF'>const</font> <font color='#0000FF'><u>double</u></font> eps,
resizable_tensor<font color='#5555FF'>&</font> dest,
resizable_tensor<font color='#5555FF'>&</font> means,
resizable_tensor<font color='#5555FF'>&</font> invstds,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> src,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> gamma,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> beta
<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> num <font color='#5555FF'>=</font> src.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>*</font> src.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>*</font> src.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>
<font color='#BB00BB'>have_same_dimensions</font><font face='Lucida Console'>(</font>gamma, beta<font face='Lucida Console'>)</font> <font color='#5555FF'>&</font><font color='#5555FF'>&</font>
src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> gamma.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&</font><font color='#5555FF'>&</font>
src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> beta.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&</font><font color='#5555FF'>&</font>
eps <font color='#5555FF'>></font> <font color='#979000'>0</font>,
"<font color='#CC0000'>\ngamma.k(): </font>" <font color='#5555FF'><</font><font color='#5555FF'><</font> gamma.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'><</font><font color='#5555FF'><</font>
"<font color='#CC0000'>\ngamma.nr(): </font>" <font color='#5555FF'><</font><font color='#5555FF'><</font> gamma.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'><</font><font color='#5555FF'><</font>
"<font color='#CC0000'>\ngamma.nc(): </font>" <font color='#5555FF'><</font><font color='#5555FF'><</font> gamma.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'><</font><font color='#5555FF'><</font>
"<font color='#CC0000'>\nbeta.k(): </font>" <font color='#5555FF'><</font><font color='#5555FF'><</font> beta.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'><</font><font color='#5555FF'><</font>
"<font color='#CC0000'>\nbeta.nr(): </font>" <font color='#5555FF'><</font><font color='#5555FF'><</font> beta.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'><</font><font color='#5555FF'><</font>
"<font color='#CC0000'>\nbeta.nc(): </font>" <font color='#5555FF'><</font><font color='#5555FF'><</font> beta.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'><</font><font color='#5555FF'><</font>
"<font color='#CC0000'>\nsrc.k(): </font>" <font color='#5555FF'><</font><font color='#5555FF'><</font> src.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'><</font><font color='#5555FF'><</font>
"<font color='#CC0000'>\nsrc.nr(): </font>" <font color='#5555FF'><</font><font color='#5555FF'><</font> src.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'><</font><font color='#5555FF'><</font>
"<font color='#CC0000'>\nsrc.nc(): </font>" <font color='#5555FF'><</font><font color='#5555FF'><</font> src.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'><</font><font color='#5555FF'><</font>
"<font color='#CC0000'>\neps: </font>" <font color='#5555FF'><</font><font color='#5555FF'><</font> eps
<font face='Lucida Console'>)</font>;
dest.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>src<font face='Lucida Console'>)</font>;
means.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font>src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
invstds.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font>src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
<font color='#009900'>// first compute means and invstds
</font> means <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
invstds <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
<font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> p_invstds <font color='#5555FF'>=</font> invstds.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> p_means <font color='#5555FF'>=</font> means.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>auto</font> p_src <font color='#5555FF'>=</font> src.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#009900'>// compute means, and sum of squares
</font> <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> n <font color='#5555FF'>=</font> <font color='#979000'>0</font>; n <font color='#5555FF'><</font> src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>n<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'><</font> num; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'><u>float</u></font> val <font color='#5555FF'>=</font> p_src[n<font color='#5555FF'>*</font>num<font color='#5555FF'>+</font>i];
p_means[n] <font color='#5555FF'>+</font><font color='#5555FF'>=</font> val;
p_invstds[n] <font color='#5555FF'>+</font><font color='#5555FF'>=</font> val<font color='#5555FF'>*</font>val;
<b>}</b>
<b>}</b>
means <font color='#5555FF'>/</font><font color='#5555FF'>=</font> num;
invstds <font color='#5555FF'>/</font><font color='#5555FF'>=</font> num;
<font color='#009900'>// copy data back to host
</font> invstds.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; means.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#009900'>// compute variances
</font> <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> n <font color='#5555FF'>=</font> <font color='#979000'>0</font>; n <font color='#5555FF'><</font> src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>n<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>auto</font> var <font color='#5555FF'>=</font> p_invstds[n] <font color='#5555FF'>-</font> p_means[n] <font color='#5555FF'>*</font> p_means[n];
p_invstds[n] <font color='#5555FF'>=</font> <font color='#979000'>1.0f</font> <font color='#5555FF'>/</font> std::<font color='#BB00BB'>sqrt</font><font face='Lucida Console'>(</font>var <font color='#5555FF'>+</font> eps<font face='Lucida Console'>)</font>;
<b>}</b>
p_src <font color='#5555FF'>=</font> src.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>auto</font> p_dest <font color='#5555FF'>=</font> dest.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>auto</font> p_gamma <font color='#5555FF'>=</font> gamma.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>auto</font> p_beta <font color='#5555FF'>=</font> beta.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> n <font color='#5555FF'>=</font> <font color='#979000'>0</font>; n <font color='#5555FF'><</font> src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>n<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'><</font> num; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#5555FF'>*</font>p_dest <font color='#5555FF'>=</font> <font face='Lucida Console'>(</font><font color='#5555FF'>*</font>p_src <font color='#5555FF'>-</font> p_means[n]<font face='Lucida Console'>)</font><font color='#5555FF'>*</font>p_invstds[n];
<font color='#5555FF'>*</font>p_dest <font color='#5555FF'>=</font> <font face='Lucida Console'>(</font><font color='#5555FF'>*</font>p_dest<font face='Lucida Console'>)</font><font color='#5555FF'>*</font>p_gamma[n] <font color='#5555FF'>+</font> p_beta[n];
<font color='#5555FF'>+</font><font color='#5555FF'>+</font>p_src;
<font color='#5555FF'>+</font><font color='#5555FF'>+</font>p_dest;
<b>}</b>
<b>}</b>
<b>}</b>
<font color='#0000FF'><u>void</u></font> <b><a name='layer_normalize_gradient'></a>layer_normalize_gradient</b> <font face='Lucida Console'>(</font>
<font color='#0000FF'>const</font> <font color='#0000FF'><u>double</u></font> eps,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> gradient_input,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> means,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> invstds,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> src,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> gamma,
tensor<font color='#5555FF'>&</font> src_grad,
tensor<font color='#5555FF'>&</font> gamma_grad,
tensor<font color='#5555FF'>&</font> beta_grad
<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> num <font color='#5555FF'>=</font> src.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>*</font> src.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>*</font> src.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> means.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> invstds.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> gamma.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> gamma_grad.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> beta_grad.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font><font color='#BB00BB'>have_same_dimensions</font><font face='Lucida Console'>(</font>gradient_input, src<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font><font color='#BB00BB'>have_same_dimensions</font><font face='Lucida Console'>(</font>gradient_input, src_grad<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>eps <font color='#5555FF'>></font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
beta_grad <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
gamma_grad <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
<font color='#0000FF'>auto</font> p_grad <font color='#5555FF'>=</font> gradient_input.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>auto</font> p_src <font color='#5555FF'>=</font> src.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> p_gamma <font color='#5555FF'>=</font> gamma.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> p_gamma_grad <font color='#5555FF'>=</font> gamma_grad.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> p_beta_grad <font color='#5555FF'>=</font> beta_grad.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> p_invstds <font color='#5555FF'>=</font> invstds.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> p_means <font color='#5555FF'>=</font> means.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
resizable_tensor dvars, dmeans;
dvars.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>invstds<font face='Lucida Console'>)</font>;
dmeans.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>means<font face='Lucida Console'>)</font>;
dvars <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
dmeans <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
<font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> p_dvars <font color='#5555FF'>=</font> dvars.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> p_dmeans <font color='#5555FF'>=</font> dmeans.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> n <font color='#5555FF'>=</font> <font color='#979000'>0</font>; n <font color='#5555FF'><</font> src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>n<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'><</font> num; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> x_hat <font color='#5555FF'>=</font> <font face='Lucida Console'>(</font><font color='#5555FF'>*</font>p_src <font color='#5555FF'>-</font> p_means[n]<font face='Lucida Console'>)</font><font color='#5555FF'>*</font>p_invstds[n];
p_beta_grad[n] <font color='#5555FF'>+</font><font color='#5555FF'>=</font> <font color='#5555FF'>*</font>p_grad;
p_gamma_grad[n] <font color='#5555FF'>+</font><font color='#5555FF'>=</font> <font face='Lucida Console'>(</font><font color='#5555FF'>*</font>p_grad<font face='Lucida Console'>)</font><font color='#5555FF'>*</font>x_hat;
<font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> dx <font color='#5555FF'>=</font> <font color='#5555FF'>*</font>p_grad <font color='#5555FF'>*</font> p_gamma[n];
p_dvars[n] <font color='#5555FF'>+</font><font color='#5555FF'>=</font> dx<font color='#5555FF'>*</font><font face='Lucida Console'>(</font><font color='#5555FF'>*</font>p_src <font color='#5555FF'>-</font> p_means[n]<font face='Lucida Console'>)</font><font color='#5555FF'>*</font><font color='#5555FF'>-</font><font color='#979000'>0.5</font><font color='#5555FF'>*</font>std::<font color='#BB00BB'>pow</font><font face='Lucida Console'>(</font>p_invstds[n], <font color='#979000'>3.0f</font><font face='Lucida Console'>)</font>;
<font color='#5555FF'>+</font><font color='#5555FF'>+</font>p_grad;
<font color='#5555FF'>+</font><font color='#5555FF'>+</font>p_src;
<b>}</b>
<b>}</b>
<font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> invnum <font color='#5555FF'>=</font> <font color='#979000'>1.0f</font><font color='#5555FF'>/</font>num;
p_grad <font color='#5555FF'>=</font> gradient_input.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
p_src <font color='#5555FF'>=</font> src.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> n <font color='#5555FF'>=</font> <font color='#979000'>0</font>; n <font color='#5555FF'><</font> src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>n<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'><</font> num; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> dx <font color='#5555FF'>=</font> <font color='#5555FF'>*</font>p_grad <font color='#5555FF'>*</font> p_gamma[n];
p_dmeans[n] <font color='#5555FF'>+</font><font color='#5555FF'>=</font> dx<font color='#5555FF'>*</font><font color='#5555FF'>-</font>p_invstds[n] <font color='#5555FF'>+</font> p_dvars[n] <font color='#5555FF'>*</font> <font color='#5555FF'>-</font><font color='#979000'>2</font><font color='#5555FF'>*</font><font face='Lucida Console'>(</font><font color='#5555FF'>*</font>p_src <font color='#5555FF'>-</font> p_means[n]<font face='Lucida Console'>)</font><font color='#5555FF'>*</font>invnum;
<font color='#5555FF'>+</font><font color='#5555FF'>+</font>p_grad;
<font color='#5555FF'>+</font><font color='#5555FF'>+</font>p_src;
<b>}</b>
<b>}</b>
p_grad <font color='#5555FF'>=</font> gradient_input.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
p_src <font color='#5555FF'>=</font> src.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>auto</font> p_src_grad <font color='#5555FF'>=</font> src_grad.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> n <font color='#5555FF'>=</font> <font color='#979000'>0</font>; n <font color='#5555FF'><</font> src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>n<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'><</font> num; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> dx <font color='#5555FF'>=</font> <font color='#5555FF'>*</font>p_grad <font color='#5555FF'>*</font> p_gamma[n];
<font color='#5555FF'>*</font>p_src_grad <font color='#5555FF'>+</font><font color='#5555FF'>=</font> dx<font color='#5555FF'>*</font>p_invstds[n] <font color='#5555FF'>+</font>
p_dvars[n] <font color='#5555FF'>*</font><font color='#979000'>2</font><font color='#5555FF'>*</font><font face='Lucida Console'>(</font><font color='#5555FF'>*</font>p_src <font color='#5555FF'>-</font> p_means[n]<font face='Lucida Console'>)</font><font color='#5555FF'>*</font>invnum <font color='#5555FF'>+</font>
p_dmeans[n]<font color='#5555FF'>*</font>invnum;
<font color='#5555FF'>+</font><font color='#5555FF'>+</font>p_grad;
<font color='#5555FF'>+</font><font color='#5555FF'>+</font>p_src;
<font color='#5555FF'>+</font><font color='#5555FF'>+</font>p_src_grad;
<b>}</b>
<b>}</b>
<b>}</b>
<font color='#009900'>// -----------------------------------------------------------------------------------
</font>
<font color='#0000FF'><u>void</u></font> <b><a name='threshold'></a>threshold</b> <font face='Lucida Console'>(</font>
tensor<font color='#5555FF'>&</font> data,
<font color='#0000FF'><u>float</u></font> thresh
<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> d <font color='#5555FF'>=</font> data.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'><</font> data.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
d[i] <font color='#5555FF'>=</font> d[i]<font color='#5555FF'>></font>thresh ? <font color='#979000'>1</font>:<font color='#979000'>0</font>;
<b>}</b>
<font color='#0000FF'><u>void</u></font> <b><a name='dot'></a>dot</b> <font face='Lucida Console'>(</font>
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> a,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> b,
tensor<font color='#5555FF'>&</font> result,
<font color='#0000FF'><u>size_t</u></font> idx
<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>a.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> b.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>idx <font color='#5555FF'><</font> result.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> aa <font color='#5555FF'>=</font> a.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> bb <font color='#5555FF'>=</font> b.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>auto</font> r <font color='#5555FF'>=</font> result.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'><</font> a.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
r[idx] <font color='#5555FF'>+</font><font color='#5555FF'>=</font> aa[i]<font color='#5555FF'>*</font>bb[i];
<b>}</b>
<font color='#009900'>// -----------------------------------------------------------------------------------
</font> <font color='#009900'>// -----------------------------------------------------------------------------------
</font> <font color='#009900'>// -----------------------------------------------------------------------------------
</font>
<font color='#0000FF'>namespace</font> ttimpl
<b>{</b>
<font color='#0000FF'><u>void</u></font> <b><a name='softmax'></a>softmax</b> <font face='Lucida Console'>(</font>
<font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> num_locations,
<font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> num_channels,
tensor<font color='#5555FF'>&</font> dest,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> src
<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#BB00BB'>DLIB_ASSERT</font><font face='Lucida Console'>(</font>num_channels<font color='#5555FF'>*</font>num_locations <font color='#5555FF'>=</font><font color='#5555FF'>=</font> src.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>src.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>src.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font><font color='#BB00BB'>have_same_dimensions</font><font face='Lucida Console'>(</font>dest,src<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> d <font color='#5555FF'>=</font> dest.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> s <font color='#5555FF'>=</font> src.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#009900'>// Note that we subtract out the max values in each channel before applying
</font> <font color='#009900'>// exp() to avoid numeric overflow in the subsequent computations. Doing this
</font> <font color='#009900'>// doesn't change the resulting output, it just makes it more numerically
</font> <font color='#009900'>// stable.
</font> <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> n <font color='#5555FF'>=</font> <font color='#979000'>0</font>; n <font color='#5555FF'><</font> src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>n<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>auto</font> ss <font color='#5555FF'>=</font> s <font color='#5555FF'>+</font> num_locations<font color='#5555FF'>*</font>num_channels<font color='#5555FF'>*</font>n;
<font color='#0000FF'>auto</font> dd <font color='#5555FF'>=</font> d <font color='#5555FF'>+</font> num_locations<font color='#5555FF'>*</font>num_channels<font color='#5555FF'>*</font>n;
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'><</font> num_locations; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'><u>float</u></font> max_val <font color='#5555FF'>=</font> <font color='#5555FF'>-</font>std::numeric_limits<font color='#5555FF'><</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>></font>::<font color='#BB00BB'>infinity</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> k <font color='#5555FF'>=</font> <font color='#979000'>0</font>; k <font color='#5555FF'><</font> num_channels; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>k<font face='Lucida Console'>)</font>
max_val <font color='#5555FF'>=</font> std::<font color='#BB00BB'>max</font><font face='Lucida Console'>(</font>max_val, ss[k<font color='#5555FF'>*</font>num_locations]<font face='Lucida Console'>)</font>;
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> k <font color='#5555FF'>=</font> <font color='#979000'>0</font>; k <font color='#5555FF'><</font> num_channels; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>k<font face='Lucida Console'>)</font>
dd[k<font color='#5555FF'>*</font>num_locations] <font color='#5555FF'>=</font> std::<font color='#BB00BB'>exp</font><font face='Lucida Console'>(</font>ss[k<font color='#5555FF'>*</font>num_locations]<font color='#5555FF'>-</font>max_val<font face='Lucida Console'>)</font>;
<font color='#5555FF'>+</font><font color='#5555FF'>+</font>ss;
<font color='#5555FF'>+</font><font color='#5555FF'>+</font>dd;
<b>}</b>
<b>}</b>
<font color='#009900'>// Now normalize each channel so they sum to 1.
</font> <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> n <font color='#5555FF'>=</font> <font color='#979000'>0</font>; n <font color='#5555FF'><</font> src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>n<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> dd <font color='#5555FF'>=</font> d <font color='#5555FF'>+</font> num_locations<font color='#5555FF'>*</font>num_channels<font color='#5555FF'>*</font>n;
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'><</font> num_locations; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> ddd <font color='#5555FF'>=</font> dd<font color='#5555FF'>+</font>i;
<font color='#0000FF'><u>float</u></font> temp <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> k <font color='#5555FF'>=</font> <font color='#979000'>0</font>; k <font color='#5555FF'><</font> num_channels; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>k<font face='Lucida Console'>)</font>
temp <font color='#5555FF'>+</font><font color='#5555FF'>=</font> ddd[k<font color='#5555FF'>*</font>num_locations];
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> k <font color='#5555FF'>=</font> <font color='#979000'>0</font>; k <font color='#5555FF'><</font> num_channels; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>k<font face='Lucida Console'>)</font>
ddd[k<font color='#5555FF'>*</font>num_locations] <font color='#5555FF'>/</font><font color='#5555FF'>=</font> temp;
<b>}</b>
<b>}</b>
<b>}</b>
<font color='#0000FF'><u>void</u></font> <b><a name='softmax_gradient'></a>softmax_gradient</b> <font face='Lucida Console'>(</font>
<font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> num_locations,
<font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> num_channels,
tensor<font color='#5555FF'>&</font> grad,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> dest,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> gradient_input
<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#BB00BB'>DLIB_ASSERT</font><font face='Lucida Console'>(</font>num_channels<font color='#5555FF'>*</font>num_locations <font color='#5555FF'>=</font><font color='#5555FF'>=</font> grad.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>grad.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>grad.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font><font color='#BB00BB'>have_same_dimensions</font><font face='Lucida Console'>(</font>grad,dest<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font><font color='#BB00BB'>have_same_dimensions</font><font face='Lucida Console'>(</font>grad,gradient_input<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> d <font color='#5555FF'>=</font> dest.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> g <font color='#5555FF'>=</font> grad.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> in <font color='#5555FF'>=</font> gradient_input.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> n <font color='#5555FF'>=</font> <font color='#979000'>0</font>; n <font color='#5555FF'><</font> grad.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>n<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> d2 <font color='#5555FF'>=</font> d <font color='#5555FF'>+</font> num_locations<font color='#5555FF'>*</font>num_channels<font color='#5555FF'>*</font>n;
<font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> g2 <font color='#5555FF'>=</font> g <font color='#5555FF'>+</font> num_locations<font color='#5555FF'>*</font>num_channels<font color='#5555FF'>*</font>n;
<font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> in2 <font color='#5555FF'>=</font> in <font color='#5555FF'>+</font> num_locations<font color='#5555FF'>*</font>num_channels<font color='#5555FF'>*</font>n;
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'><</font> num_locations; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> d3 <font color='#5555FF'>=</font> d2<font color='#5555FF'>+</font>i;
<font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> g3 <font color='#5555FF'>=</font> g2<font color='#5555FF'>+</font>i;
<font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> in3 <font color='#5555FF'>=</font> in2<font color='#5555FF'>+</font>i;
<font color='#0000FF'><u>float</u></font> temp <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> k <font color='#5555FF'>=</font> <font color='#979000'>0</font>; k <font color='#5555FF'><</font> num_channels; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>k<font face='Lucida Console'>)</font>
temp <font color='#5555FF'>+</font><font color='#5555FF'>=</font> <font color='#5555FF'>-</font>d3[k<font color='#5555FF'>*</font>num_locations]<font color='#5555FF'>*</font>in3[k<font color='#5555FF'>*</font>num_locations];
<font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#BB00BB'>is_same_object</font><font face='Lucida Console'>(</font>gradient_input, grad<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> k <font color='#5555FF'>=</font> <font color='#979000'>0</font>; k <font color='#5555FF'><</font> num_channels; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>k<font face='Lucida Console'>)</font>
g3[k<font color='#5555FF'>*</font>num_locations] <font color='#5555FF'>=</font> d3[k<font color='#5555FF'>*</font>num_locations]<font color='#5555FF'>*</font><font face='Lucida Console'>(</font>temp<font color='#5555FF'>+</font>in3[k<font color='#5555FF'>*</font>num_locations]<font face='Lucida Console'>)</font>;
<b>}</b>
<font color='#0000FF'>else</font>
<b>{</b>
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> k <font color='#5555FF'>=</font> <font color='#979000'>0</font>; k <font color='#5555FF'><</font> num_channels; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>k<font face='Lucida Console'>)</font>
g3[k<font color='#5555FF'>*</font>num_locations] <font color='#5555FF'>+</font><font color='#5555FF'>=</font> d3[k<font color='#5555FF'>*</font>num_locations]<font color='#5555FF'>*</font><font face='Lucida Console'>(</font>temp<font color='#5555FF'>+</font>in3[k<font color='#5555FF'>*</font>num_locations]<font face='Lucida Console'>)</font>;
<b>}</b>
<b>}</b>
<b>}</b>
<b>}</b>
<b>}</b>
<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
<font color='#0000FF'><u>void</u></font> <b><a name='softmax'></a>softmax</b> <font face='Lucida Console'>(</font>
tensor<font color='#5555FF'>&</font> dest,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> src
<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font><font color='#BB00BB'>have_same_dimensions</font><font face='Lucida Console'>(</font>dest,src<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
ttimpl::<font color='#BB00BB'>softmax</font><font face='Lucida Console'>(</font>src.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>src.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, src.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, dest, src<font face='Lucida Console'>)</font>;
<b>}</b>
<font color='#0000FF'><u>void</u></font> <b><a name='softmax_gradient'></a>softmax_gradient</b> <font face='Lucida Console'>(</font>
tensor<font color='#5555FF'>&</font> grad,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> dest,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> gradient_input
<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font><font color='#BB00BB'>have_same_dimensions</font><font face='Lucida Console'>(</font>grad,dest<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font><font color='#BB00BB'>have_same_dimensions</font><font face='Lucida Console'>(</font>grad,gradient_input<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
ttimpl::<font color='#BB00BB'>softmax_gradient</font><font face='Lucida Console'>(</font>grad.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>grad.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, grad.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, grad, dest, gradient_input<font face='Lucida Console'>)</font>;
<b>}</b>
<font color='#009900'>// ------------------------------------------------------------------------------------
</font>
<font color='#0000FF'><u>void</u></font> <b><a name='softmax_all'></a>softmax_all</b> <font face='Lucida Console'>(</font>
tensor<font color='#5555FF'>&</font> dest,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> src
<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font><font color='#BB00BB'>have_same_dimensions</font><font face='Lucida Console'>(</font>dest,src<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
ttimpl::<font color='#BB00BB'>softmax</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>, src.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>src.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>src.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, dest, src<font face='Lucida Console'>)</font>;
<b>}</b>
<font color='#0000FF'><u>void</u></font> <b><a name='softmax_all_gradient'></a>softmax_all_gradient</b> <font face='Lucida Console'>(</font>
tensor<font color='#5555FF'>&</font> grad,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> dest,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> gradient_input
<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font><font color='#BB00BB'>have_same_dimensions</font><font face='Lucida Console'>(</font>grad,dest<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font><font color='#BB00BB'>have_same_dimensions</font><font face='Lucida Console'>(</font>grad,gradient_input<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
ttimpl::<font color='#BB00BB'>softmax_gradient</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>, grad.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>grad.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>grad.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, grad, dest, gradient_input<font face='Lucida Console'>)</font>;
<b>}</b>
<font color='#009900'>// ------------------------------------------------------------------------------------
</font>
<font color='#0000FF'><u>void</u></font> <b><a name='sigmoid'></a>sigmoid</b> <font face='Lucida Console'>(</font>
tensor<font color='#5555FF'>&</font> dest,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> src
<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> d <font color='#5555FF'>=</font> dest.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> s <font color='#5555FF'>=</font> src.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'><</font> src.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
d[i] <font color='#5555FF'>=</font> <font color='#979000'>1</font><font color='#5555FF'>/</font><font face='Lucida Console'>(</font><font color='#979000'>1</font><font color='#5555FF'>+</font>std::<font color='#BB00BB'>exp</font><font face='Lucida Console'>(</font><font color='#5555FF'>-</font>s[i]<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
<b>}</b>
<font color='#0000FF'><u>void</u></font> <b><a name='sigmoid_gradient'></a>sigmoid_gradient</b> <font face='Lucida Console'>(</font>
tensor<font color='#5555FF'>&</font> grad,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> dest,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> gradient_input
<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> g <font color='#5555FF'>=</font> grad.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> d <font color='#5555FF'>=</font> dest.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> in <font color='#5555FF'>=</font> gradient_input.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#BB00BB'>is_same_object</font><font face='Lucida Console'>(</font>gradient_input, grad<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'><</font> dest.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
g[i] <font color='#5555FF'>=</font> in[i]<font color='#5555FF'>*</font>d[i]<font color='#5555FF'>*</font><font face='Lucida Console'>(</font><font color='#979000'>1</font><font color='#5555FF'>-</font>d[i]<font face='Lucida Console'>)</font>;
<b>}</b>
<font color='#0000FF'>else</font>
<b>{</b>
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'><</font> dest.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
g[i] <font color='#5555FF'>+</font><font color='#5555FF'>=</font> in[i]<font color='#5555FF'>*</font>d[i]<font color='#5555FF'>*</font><font face='Lucida Console'>(</font><font color='#979000'>1</font><font color='#5555FF'>-</font>d[i]<font face='Lucida Console'>)</font>;
<b>}</b>
<b>}</b>
<font color='#009900'>// ------------------------------------------------------------------------------------
</font>
<font color='#0000FF'><u>void</u></font> <b><a name='mish'></a>mish</b> <font face='Lucida Console'>(</font>
tensor<font color='#5555FF'>&</font> dest,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> src
<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> d <font color='#5555FF'>=</font> dest.<font color='#BB00BB'>host_write_only</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> s <font color='#5555FF'>=</font> src.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'><</font> src.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> e <font color='#5555FF'>=</font> std::<font color='#BB00BB'>exp</font><font face='Lucida Console'>(</font>s[i]<font face='Lucida Console'>)</font>;
<font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> delta <font color='#5555FF'>=</font> <font color='#979000'>2</font><font color='#5555FF'>*</font>e <font color='#5555FF'>+</font> e<font color='#5555FF'>*</font>e <font color='#5555FF'>+</font> <font color='#979000'>2</font>;
d[i] <font color='#5555FF'>=</font> s[i] <font color='#5555FF'>-</font> <font color='#979000'>2</font><font color='#5555FF'>*</font>s[i]<font color='#5555FF'>/</font>delta;
<b>}</b>
<b>}</b>
<font color='#0000FF'><u>void</u></font> <b><a name='mish_gradient'></a>mish_gradient</b><font face='Lucida Console'>(</font>
tensor<font color='#5555FF'>&</font> grad,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> src,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> gradient_input
<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> g <font color='#5555FF'>=</font> grad.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> s <font color='#5555FF'>=</font> src.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> in <font color='#5555FF'>=</font> gradient_input.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> calculate_gradient <font color='#5555FF'>=</font> []<font face='Lucida Console'>(</font><font color='#0000FF'><u>float</u></font> x<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>x <font color='#5555FF'>></font><font color='#5555FF'>=</font> <font color='#979000'>8</font><font face='Lucida Console'>)</font>
<font color='#0000FF'>return</font> <font color='#979000'>1.f</font>;
<font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>x <font color='#5555FF'><</font><font color='#5555FF'>=</font> <font color='#5555FF'>-</font><font color='#979000'>8</font><font face='Lucida Console'>)</font>
<font color='#0000FF'>return</font> <font color='#979000'>0.f</font>;
<font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> e <font color='#5555FF'>=</font> std::<font color='#BB00BB'>exp</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font>;
<font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> delta <font color='#5555FF'>=</font> <font color='#979000'>2</font><font color='#5555FF'>*</font>e <font color='#5555FF'>+</font> e<font color='#5555FF'>*</font>e <font color='#5555FF'>+</font> <font color='#979000'>2</font>;
<font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> omega <font color='#5555FF'>=</font> <font color='#979000'>4</font><font color='#5555FF'>*</font><font face='Lucida Console'>(</font>x <font color='#5555FF'>+</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> <font color='#979000'>4</font><font color='#5555FF'>*</font>e<font color='#5555FF'>*</font>e <font color='#5555FF'>+</font> e<font color='#5555FF'>*</font>e<font color='#5555FF'>*</font>e <font color='#5555FF'>+</font> e<font color='#5555FF'>*</font><font face='Lucida Console'>(</font><font color='#979000'>4</font><font color='#5555FF'>*</font>x <font color='#5555FF'>+</font> <font color='#979000'>6</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>return</font> e<font color='#5555FF'>*</font>omega<font color='#5555FF'>/</font><font face='Lucida Console'>(</font>delta<font color='#5555FF'>*</font>delta<font face='Lucida Console'>)</font>;
<b>}</b>;
<font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#BB00BB'>is_same_object</font><font face='Lucida Console'>(</font>gradient_input, grad<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'><</font> src.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
g[i] <font color='#5555FF'>=</font> in[i]<font color='#5555FF'>*</font><font color='#BB00BB'>calculate_gradient</font><font face='Lucida Console'>(</font>s[i]<font face='Lucida Console'>)</font>;
<b>}</b>
<font color='#0000FF'>else</font>
<b>{</b>
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'><</font> src.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
g[i] <font color='#5555FF'>+</font><font color='#5555FF'>=</font> in[i]<font color='#5555FF'>*</font><font color='#BB00BB'>calculate_gradient</font><font face='Lucida Console'>(</font>s[i]<font face='Lucida Console'>)</font>;
<b>}</b>
<b>}</b>
<font color='#009900'>// ------------------------------------------------------------------------------------
</font>
<font color='#0000FF'><u>void</u></font> <b><a name='relu'></a>relu</b> <font face='Lucida Console'>(</font>
tensor<font color='#5555FF'>&</font> dest,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> src
<font face='Lucida Console'>)</font>
<b>{</b>
dest <font color='#5555FF'>=</font> <font color='#BB00BB'>lowerbound</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>src<font face='Lucida Console'>)</font>, <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
<b>}</b>
<font color='#0000FF'><u>void</u></font> <b><a name='relu_gradient'></a>relu_gradient</b> <font face='Lucida Console'>(</font>
tensor<font color='#5555FF'>&</font> grad,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> dest,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> gradient_input
<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font><font color='#5555FF'>*</font> gi <font color='#5555FF'>=</font> gradient_input.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font><font color='#5555FF'>*</font> in <font color='#5555FF'>=</font> dest.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'><u>float</u></font><font color='#5555FF'>*</font> out <font color='#5555FF'>=</font> grad.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#BB00BB'>is_same_object</font><font face='Lucida Console'>(</font>grad, gradient_input<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'><</font> dest.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>in[i] <font color='#5555FF'>></font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>
out[i] <font color='#5555FF'>=</font> gi[i];
<font color='#0000FF'>else</font>
out[i] <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
<b>}</b>
<b>}</b>
<font color='#0000FF'>else</font>
<b>{</b>
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'><</font> dest.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>in[i] <font color='#5555FF'>></font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>
out[i] <font color='#5555FF'>+</font><font color='#5555FF'>=</font> gi[i];
<b>}</b>
<b>}</b>
<b>}</b>
<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
<font color='#0000FF'><u>void</u></font> <b><a name='prelu'></a>prelu</b> <font face='Lucida Console'>(</font>
tensor<font color='#5555FF'>&</font> dest,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> src,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> param
<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> p <font color='#5555FF'>=</font> param.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>[<font color='#979000'>0</font>];
<font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font><font color='#5555FF'>*</font> s <font color='#5555FF'>=</font> src.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'><u>float</u></font><font color='#5555FF'>*</font> d <font color='#5555FF'>=</font> dest.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'><</font> dest.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>s[i] <font color='#5555FF'>></font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>
d[i] <font color='#5555FF'>=</font> s[i];
<font color='#0000FF'>else</font>
d[i] <font color='#5555FF'>=</font> p<font color='#5555FF'>*</font>s[i];
<b>}</b>
<b>}</b>
<font color='#0000FF'><u>void</u></font> <b><a name='prelu_gradient'></a>prelu_gradient</b> <font face='Lucida Console'>(</font>
tensor<font color='#5555FF'>&</font> grad,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> src,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> gradient_input,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> param,
tensor<font color='#5555FF'>&</font> params_grad
<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font><font color='#BB00BB'>is_same_object</font><font face='Lucida Console'>(</font>grad, gradient_input<font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>false</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> p <font color='#5555FF'>=</font> param.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>[<font color='#979000'>0</font>];
<font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font><font color='#5555FF'>*</font> gi <font color='#5555FF'>=</font> gradient_input.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font><font color='#5555FF'>*</font> s <font color='#5555FF'>=</font> src.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'><u>float</u></font><font color='#5555FF'>*</font> out <font color='#5555FF'>=</font> grad.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'><u>float</u></font> pgrad <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'><</font> src.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>s[i] <font color='#5555FF'>></font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>
<b>{</b>
out[i] <font color='#5555FF'>+</font><font color='#5555FF'>=</font> gi[i];
<b>}</b>
<font color='#0000FF'>else</font>
<b>{</b>
out[i] <font color='#5555FF'>+</font><font color='#5555FF'>=</font> p<font color='#5555FF'>*</font>gi[i];
pgrad <font color='#5555FF'>+</font><font color='#5555FF'>=</font> gi[i]<font color='#5555FF'>*</font>s[i];
<b>}</b>
<b>}</b>
params_grad.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>[<font color='#979000'>0</font>] <font color='#5555FF'>=</font> pgrad;
<b>}</b>
<font color='#009900'>// ------------------------------------------------------------------------------------
</font>
<font color='#0000FF'><u>void</u></font> <b><a name='leaky_relu'></a>leaky_relu</b> <font face='Lucida Console'>(</font>
tensor<font color='#5555FF'>&</font> dest,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> src,
<font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> alpha
<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font><font color='#5555FF'>*</font> s <font color='#5555FF'>=</font> src.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'><u>float</u></font><font color='#5555FF'>*</font> d <font color='#5555FF'>=</font> dest.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'><</font> dest.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>s[i] <font color='#5555FF'>></font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>
d[i] <font color='#5555FF'>=</font> s[i];
<font color='#0000FF'>else</font>
d[i] <font color='#5555FF'>=</font> alpha <font color='#5555FF'>*</font> s[i];
<b>}</b>
<b>}</b>
<font color='#0000FF'><u>void</u></font> <b><a name='leaky_relu_gradient'></a>leaky_relu_gradient</b> <font face='Lucida Console'>(</font>
tensor<font color='#5555FF'>&</font> grad,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> dest,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> gradient_input,
<font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> alpha
<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font><font color='#5555FF'>*</font> gi <font color='#5555FF'>=</font> gradient_input.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font><font color='#5555FF'>*</font> in <font color='#5555FF'>=</font> dest.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'><u>float</u></font><font color='#5555FF'>*</font> out <font color='#5555FF'>=</font> grad.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#BB00BB'>is_same_object</font><font face='Lucida Console'>(</font>grad, gradient_input<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'><</font> dest.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>in[i] <font color='#5555FF'>></font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>
out[i] <font color='#5555FF'>=</font> gi[i];
<font color='#0000FF'>else</font>
out[i] <font color='#5555FF'>=</font> alpha <font color='#5555FF'>*</font> gi[i];
<b>}</b>
<b>}</b>
<font color='#0000FF'>else</font>
<b>{</b>
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'><</font> dest.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>in[i] <font color='#5555FF'>></font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>
out[i] <font color='#5555FF'>+</font><font color='#5555FF'>=</font> gi[i];
<font color='#0000FF'>else</font>
out[i] <font color='#5555FF'>+</font><font color='#5555FF'>=</font> alpha <font color='#5555FF'>*</font> gi[i];
<b>}</b>
<b>}</b>
<b>}</b>
<font color='#009900'>// ------------------------------------------------------------------------------------
</font>
<font color='#0000FF'><u>void</u></font> <b><a name='tanh'></a>tanh</b> <font face='Lucida Console'>(</font>
tensor<font color='#5555FF'>&</font> dest,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> src
<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> d <font color='#5555FF'>=</font> dest.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> s <font color='#5555FF'>=</font> src.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'><</font> src.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
d[i] <font color='#5555FF'>=</font> std::<font color='#BB00BB'>tanh</font><font face='Lucida Console'>(</font>s[i]<font face='Lucida Console'>)</font>;
<b>}</b>
<font color='#0000FF'><u>void</u></font> <b><a name='tanh_gradient'></a>tanh_gradient</b> <font face='Lucida Console'>(</font>
tensor<font color='#5555FF'>&</font> grad,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> dest,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> gradient_input
<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> g <font color='#5555FF'>=</font> grad.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> d <font color='#5555FF'>=</font> dest.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> in <font color='#5555FF'>=</font> gradient_input.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#BB00BB'>is_same_object</font><font face='Lucida Console'>(</font>grad, gradient_input<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'><</font> dest.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
g[i] <font color='#5555FF'>=</font> in[i]<font color='#5555FF'>*</font><font face='Lucida Console'>(</font><font color='#979000'>1</font><font color='#5555FF'>-</font>d[i]<font color='#5555FF'>*</font>d[i]<font face='Lucida Console'>)</font>;
<b>}</b>
<font color='#0000FF'>else</font>
<b>{</b>
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'><</font> dest.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
g[i] <font color='#5555FF'>+</font><font color='#5555FF'>=</font> in[i]<font color='#5555FF'>*</font><font face='Lucida Console'>(</font><font color='#979000'>1</font><font color='#5555FF'>-</font>d[i]<font color='#5555FF'>*</font>d[i]<font face='Lucida Console'>)</font>;
<b>}</b>
<b>}</b>
<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
<font color='#0000FF'><u>void</u></font> <b><a name='gelu'></a>gelu</b> <font face='Lucida Console'>(</font>
tensor<font color='#5555FF'>&</font> dest,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> src
<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> d <font color='#5555FF'>=</font> dest.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> s <font color='#5555FF'>=</font> src.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'><</font> src.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
d[i] <font color='#5555FF'>=</font> <font color='#979000'>0.5f</font><font color='#5555FF'>*</font>s[i]<font color='#5555FF'>*</font><font face='Lucida Console'>(</font><font color='#979000'>1.0f</font> <font color='#5555FF'>+</font> std::<font color='#BB00BB'>erf</font><font face='Lucida Console'>(</font>s[i]<font color='#5555FF'>/</font>sqrt_2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
<b>}</b>
<font color='#0000FF'><u>void</u></font> <b><a name='gelu_gradient'></a>gelu_gradient</b> <font face='Lucida Console'>(</font>
tensor<font color='#5555FF'>&</font> grad,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> src,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> gradient_input
<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> beta <font color='#5555FF'>=</font> <font color='#979000'>1.0f</font> <font color='#5555FF'>/</font> std::<font color='#BB00BB'>sqrt</font><font face='Lucida Console'>(</font><font color='#979000'>2.0f</font> <font color='#5555FF'>*</font> pi<font face='Lucida Console'>)</font>;
<font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> compute_gradient <font color='#5555FF'>=</font> [beta]<font face='Lucida Console'>(</font><font color='#0000FF'><u>float</u></font> x<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> cdf <font color='#5555FF'>=</font> <font color='#979000'>0.5f</font><font color='#5555FF'>*</font><font face='Lucida Console'>(</font><font color='#979000'>1.0f</font> <font color='#5555FF'>+</font> std::<font color='#BB00BB'>erf</font><font face='Lucida Console'>(</font>x<font color='#5555FF'>/</font>sqrt_2<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> pdf <font color='#5555FF'>=</font> beta<font color='#5555FF'>*</font>std::<font color='#BB00BB'>exp</font><font face='Lucida Console'>(</font><font color='#5555FF'>-</font><font color='#979000'>0.5f</font><font color='#5555FF'>*</font>x<font color='#5555FF'>*</font>x<font face='Lucida Console'>)</font>;
<font color='#0000FF'>return</font> cdf <font color='#5555FF'>+</font> x <font color='#5555FF'>*</font> pdf;
<b>}</b>;
<font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> g <font color='#5555FF'>=</font> grad.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> s <font color='#5555FF'>=</font> src.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> in <font color='#5555FF'>=</font> gradient_input.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#BB00BB'>is_same_object</font><font face='Lucida Console'>(</font>grad, gradient_input<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'><</font> src.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
g[i] <font color='#5555FF'>=</font> in[i]<font color='#5555FF'>*</font><font color='#BB00BB'>compute_gradient</font><font face='Lucida Console'>(</font>s[i]<font face='Lucida Console'>)</font>;
<b>}</b>
<font color='#0000FF'>else</font>
<b>{</b>
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'><</font> src.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
g[i] <font color='#5555FF'>+</font><font color='#5555FF'>=</font> in[i]<font color='#5555FF'>*</font><font color='#BB00BB'>compute_gradient</font><font face='Lucida Console'>(</font>s[i]<font face='Lucida Console'>)</font>;
<b>}</b>
<b>}</b>
<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
<font color='#0000FF'><u>void</u></font> <b><a name='resize_bilinear'></a>resize_bilinear</b> <font face='Lucida Console'>(</font>
tensor<font color='#5555FF'>&</font> dest,
<font color='#0000FF'><u>long</u></font> dest_row_stride,
<font color='#0000FF'><u>long</u></font> dest_channel_stride,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> src,
<font color='#0000FF'><u>long</u></font> src_row_stride,
<font color='#0000FF'><u>long</u></font> src_channel_stride
<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font><font color='#BB00BB'>is_same_object</font><font face='Lucida Console'>(</font>dest, src<font face='Lucida Console'>)</font><font color='#5555FF'>=</font><font color='#5555FF'>=</font><font color='#979000'>false</font><font face='Lucida Console'>)</font>;
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>dest.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>dest.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> src.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>dest.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>0</font> <font color='#5555FF'>|</font><font color='#5555FF'>|</font> src.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>
<font color='#0000FF'>return</font>;
<font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font><font color='#5555FF'>*</font> s <font color='#5555FF'>=</font> src.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'><u>float</u></font><font color='#5555FF'>*</font> d <font color='#5555FF'>=</font> dest.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#BB00BB'>parallel_for</font><font face='Lucida Console'>(</font><font color='#979000'>0</font>, dest.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>dest.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, [<font color='#5555FF'>&</font>]<font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> i<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>auto</font> simg <font color='#5555FF'>=</font> <font color='#BB00BB'>sub_image</font><font face='Lucida Console'>(</font>s<font color='#5555FF'>+</font>i<font color='#5555FF'>*</font>src_channel_stride, src.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, src.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, src_row_stride<font face='Lucida Console'>)</font>;
<font color='#0000FF'>auto</font> dimg <font color='#5555FF'>=</font> <font color='#BB00BB'>sub_image</font><font face='Lucida Console'>(</font>d<font color='#5555FF'>+</font>i<font color='#5555FF'>*</font>dest_channel_stride, dest.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, dest.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, dest_row_stride<font face='Lucida Console'>)</font>;
<font color='#BB00BB'>resize_image</font><font face='Lucida Console'>(</font>simg, dimg<font face='Lucida Console'>)</font>;
<b>}</b><font face='Lucida Console'>)</font>;
<b>}</b>
<font color='#0000FF'><u>void</u></font> <b><a name='resize_bilinear_gradient'></a>resize_bilinear_gradient</b> <font face='Lucida Console'>(</font>
tensor<font color='#5555FF'>&</font> grad,
<font color='#0000FF'><u>long</u></font> grad_row_stride,
<font color='#0000FF'><u>long</u></font> grad_channel_stride,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> gradient_input,
<font color='#0000FF'><u>long</u></font> gradient_input_row_stride,
<font color='#0000FF'><u>long</u></font> gradient_input_channel_stride
<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font><font color='#BB00BB'>is_same_object</font><font face='Lucida Console'>(</font>grad, gradient_input<font face='Lucida Console'>)</font><font color='#5555FF'>=</font><font color='#5555FF'>=</font><font color='#979000'>false</font><font face='Lucida Console'>)</font>;
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>gradient_input.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> grad.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>gradient_input.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> grad.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>gradient_input.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>0</font> <font color='#5555FF'>|</font><font color='#5555FF'>|</font> grad.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>
<font color='#0000FF'>return</font>;
<font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font><font color='#5555FF'>*</font> gi <font color='#5555FF'>=</font> gradient_input.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'><u>float</u></font><font color='#5555FF'>*</font> g <font color='#5555FF'>=</font> grad.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> x_scale <font color='#5555FF'>=</font> <font face='Lucida Console'>(</font>grad.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#979000'>1</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font><font face='Lucida Console'>(</font><font color='#0000FF'><u>float</u></font><font face='Lucida Console'>)</font>std::max<font color='#5555FF'><</font><font color='#0000FF'><u>long</u></font><font color='#5555FF'>></font><font face='Lucida Console'>(</font><font face='Lucida Console'>(</font>gradient_input.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#979000'>1</font><font face='Lucida Console'>)</font>,<font color='#979000'>1</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> y_scale <font color='#5555FF'>=</font> <font face='Lucida Console'>(</font>grad.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#979000'>1</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font><font face='Lucida Console'>(</font><font color='#0000FF'><u>float</u></font><font face='Lucida Console'>)</font>std::max<font color='#5555FF'><</font><font color='#0000FF'><u>long</u></font><font color='#5555FF'>></font><font face='Lucida Console'>(</font><font face='Lucida Console'>(</font>gradient_input.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#979000'>1</font><font face='Lucida Console'>)</font>,<font color='#979000'>1</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> <font color='#0000FF'><u>long</u></font> samp <font color='#5555FF'>=</font> <font color='#979000'>0</font>; samp <font color='#5555FF'><</font> gradient_input.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>samp<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> <font color='#0000FF'><u>long</u></font> k <font color='#5555FF'>=</font> <font color='#979000'>0</font>; k <font color='#5555FF'><</font> gradient_input.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>k<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> <font color='#0000FF'><u>long</u></font> r <font color='#5555FF'>=</font> <font color='#979000'>0</font>; r <font color='#5555FF'><</font> gradient_input.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>r<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> y <font color='#5555FF'>=</font> r<font color='#5555FF'>*</font>y_scale;
<font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> <font color='#0000FF'><u>long</u></font> top <font color='#5555FF'>=</font> <font color='#0000FF'>static_cast</font><font color='#5555FF'><</font><font color='#0000FF'><u>long</u></font> <font color='#0000FF'><u>long</u></font><font color='#5555FF'>></font><font face='Lucida Console'>(</font>std::<font color='#BB00BB'>floor</font><font face='Lucida Console'>(</font>y<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> <font color='#0000FF'><u>long</u></font> bottom <font color='#5555FF'>=</font> std::<font color='#BB00BB'>min</font><font face='Lucida Console'>(</font>top<font color='#5555FF'>+</font><font color='#979000'>1</font>, grad.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#979000'>1</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> tb_frac <font color='#5555FF'>=</font> y <font color='#5555FF'>-</font> top;
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> <font color='#0000FF'><u>long</u></font> c <font color='#5555FF'>=</font> <font color='#979000'>0</font>; c <font color='#5555FF'><</font> gradient_input.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>c<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> x <font color='#5555FF'>=</font> c<font color='#5555FF'>*</font>x_scale;
<font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> <font color='#0000FF'><u>long</u></font> left <font color='#5555FF'>=</font> <font color='#0000FF'>static_cast</font><font color='#5555FF'><</font><font color='#0000FF'><u>long</u></font> <font color='#0000FF'><u>long</u></font><font color='#5555FF'>></font><font face='Lucida Console'>(</font>std::<font color='#BB00BB'>floor</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> <font color='#0000FF'><u>long</u></font> right <font color='#5555FF'>=</font> std::<font color='#BB00BB'>min</font><font face='Lucida Console'>(</font>left<font color='#5555FF'>+</font><font color='#979000'>1</font>, grad.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#979000'>1</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> lr_frac <font color='#5555FF'>=</font> x <font color='#5555FF'>-</font> left;
<font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> tmp <font color='#5555FF'>=</font> gi[r<font color='#5555FF'>*</font>gradient_input_row_stride<font color='#5555FF'>+</font>c];
g[top<font color='#5555FF'>*</font>grad_row_stride<font color='#5555FF'>+</font>left] <font color='#5555FF'>+</font><font color='#5555FF'>=</font> tmp<font color='#5555FF'>*</font><font face='Lucida Console'>(</font><font color='#979000'>1</font><font color='#5555FF'>-</font>tb_frac<font face='Lucida Console'>)</font><font color='#5555FF'>*</font><font face='Lucida Console'>(</font><font color='#979000'>1</font><font color='#5555FF'>-</font>lr_frac<font face='Lucida Console'>)</font>;
g[top<font color='#5555FF'>*</font>grad_row_stride<font color='#5555FF'>+</font>right] <font color='#5555FF'>+</font><font color='#5555FF'>=</font> tmp<font color='#5555FF'>*</font><font face='Lucida Console'>(</font><font color='#979000'>1</font><font color='#5555FF'>-</font>tb_frac<font face='Lucida Console'>)</font><font color='#5555FF'>*</font><font face='Lucida Console'>(</font>lr_frac<font face='Lucida Console'>)</font>;
g[bottom<font color='#5555FF'>*</font>grad_row_stride<font color='#5555FF'>+</font>left] <font color='#5555FF'>+</font><font color='#5555FF'>=</font> tmp<font color='#5555FF'>*</font><font face='Lucida Console'>(</font>tb_frac<font face='Lucida Console'>)</font><font color='#5555FF'>*</font><font face='Lucida Console'>(</font><font color='#979000'>1</font><font color='#5555FF'>-</font>lr_frac<font face='Lucida Console'>)</font>;
g[bottom<font color='#5555FF'>*</font>grad_row_stride<font color='#5555FF'>+</font>right] <font color='#5555FF'>+</font><font color='#5555FF'>=</font> tmp<font color='#5555FF'>*</font><font face='Lucida Console'>(</font>tb_frac<font face='Lucida Console'>)</font><font color='#5555FF'>*</font><font face='Lucida Console'>(</font>lr_frac<font face='Lucida Console'>)</font>;
<b>}</b>
<b>}</b>
g <font color='#5555FF'>+</font><font color='#5555FF'>=</font> grad_channel_stride;
gi <font color='#5555FF'>+</font><font color='#5555FF'>=</font> gradient_input_channel_stride;
<b>}</b>
<b>}</b>
<b>}</b>
<font color='#009900'>// ------------------------------------------------------------------------------------
</font> <font color='#009900'>// ------------------------------------------------------------------------------------
</font> <font color='#009900'>// ------------------------------------------------------------------------------------
</font>
pooling::<b><a name='pooling'></a>pooling</b> <font face='Lucida Console'>(</font>
<font face='Lucida Console'>)</font> : window_height<font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font>,window_width<font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font>,stride_y<font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font>,stride_x<font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font>,padding_y<font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font>,padding_x<font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font>,do_max_pooling<font face='Lucida Console'>(</font><font color='#979000'>true</font><font face='Lucida Console'>)</font>
<b>{</b>
<b>}</b>
<font color='#0000FF'><u>void</u></font> pooling::
<b><a name='clear'></a>clear</b><font face='Lucida Console'>(</font>
<font face='Lucida Console'>)</font>
<b>{</b>
window_height <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
window_width <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
stride_y <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
stride_x <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
padding_y <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
padding_x <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
<b>}</b>
<font color='#0000FF'><u>void</u></font> pooling::
<b><a name='setup_max_pooling'></a>setup_max_pooling</b><font face='Lucida Console'>(</font>
<font color='#0000FF'><u>int</u></font> window_height_,
<font color='#0000FF'><u>int</u></font> window_width_,
<font color='#0000FF'><u>int</u></font> stride_y_,
<font color='#0000FF'><u>int</u></font> stride_x_,
<font color='#0000FF'><u>int</u></font> padding_y_,
<font color='#0000FF'><u>int</u></font> padding_x_
<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>window_width_ <font color='#5555FF'>></font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>window_height_ <font color='#5555FF'>></font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>stride_y_ <font color='#5555FF'>></font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>stride_x_ <font color='#5555FF'>></font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font><font color='#979000'>0</font> <font color='#5555FF'><</font><font color='#5555FF'>=</font> padding_y_ <font color='#5555FF'>&</font><font color='#5555FF'>&</font> padding_y_ <font color='#5555FF'><</font> window_height_<font face='Lucida Console'>)</font>;
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font><font color='#979000'>0</font> <font color='#5555FF'><</font><font color='#5555FF'>=</font> padding_x_ <font color='#5555FF'>&</font><font color='#5555FF'>&</font> padding_x_ <font color='#5555FF'><</font> window_width_<font face='Lucida Console'>)</font>;
window_height <font color='#5555FF'>=</font> window_height_;
window_width <font color='#5555FF'>=</font> window_width_;
stride_y <font color='#5555FF'>=</font> stride_y_;
stride_x <font color='#5555FF'>=</font> stride_x_;
padding_y <font color='#5555FF'>=</font> padding_y_;
padding_x <font color='#5555FF'>=</font> padding_x_;
do_max_pooling <font color='#5555FF'>=</font> <font color='#979000'>true</font>;
<b>}</b>
<font color='#0000FF'><u>void</u></font> pooling::
<b><a name='setup_avg_pooling'></a>setup_avg_pooling</b><font face='Lucida Console'>(</font>
<font color='#0000FF'><u>int</u></font> window_height_,
<font color='#0000FF'><u>int</u></font> window_width_,
<font color='#0000FF'><u>int</u></font> stride_y_,
<font color='#0000FF'><u>int</u></font> stride_x_,
<font color='#0000FF'><u>int</u></font> padding_y_,
<font color='#0000FF'><u>int</u></font> padding_x_
<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>window_width_ <font color='#5555FF'>></font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>window_height_ <font color='#5555FF'>></font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>stride_y_ <font color='#5555FF'>></font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>stride_x_ <font color='#5555FF'>></font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font><font color='#979000'>0</font> <font color='#5555FF'><</font><font color='#5555FF'>=</font> padding_y_ <font color='#5555FF'>&</font><font color='#5555FF'>&</font> padding_y_ <font color='#5555FF'><</font> window_height_<font face='Lucida Console'>)</font>;
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font><font color='#979000'>0</font> <font color='#5555FF'><</font><font color='#5555FF'>=</font> padding_x_ <font color='#5555FF'>&</font><font color='#5555FF'>&</font> padding_x_ <font color='#5555FF'><</font> window_width_<font face='Lucida Console'>)</font>;
window_height <font color='#5555FF'>=</font> window_height_;
window_width <font color='#5555FF'>=</font> window_width_;
stride_y <font color='#5555FF'>=</font> stride_y_;
stride_x <font color='#5555FF'>=</font> stride_x_;
padding_y <font color='#5555FF'>=</font> padding_y_;
padding_x <font color='#5555FF'>=</font> padding_x_;
do_max_pooling <font color='#5555FF'>=</font> <font color='#979000'>false</font>;
<b>}</b>
<font color='#0000FF'><u>void</u></font> pooling::
<b><a name='operator'></a>operator</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font face='Lucida Console'>(</font>
resizable_tensor<font color='#5555FF'>&</font> dest,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> src
<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>window_width <font color='#5555FF'>></font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>window_height <font color='#5555FF'>></font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>stride_y <font color='#5555FF'>></font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>stride_x <font color='#5555FF'>></font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font><font color='#979000'>0</font> <font color='#5555FF'><</font><font color='#5555FF'>=</font> padding_y <font color='#5555FF'>&</font><font color='#5555FF'>&</font> padding_y <font color='#5555FF'><</font> window_height<font face='Lucida Console'>)</font>;
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font><font color='#979000'>0</font> <font color='#5555FF'><</font><font color='#5555FF'>=</font> padding_x <font color='#5555FF'>&</font><font color='#5555FF'>&</font> padding_x <font color='#5555FF'><</font> window_width<font face='Lucida Console'>)</font>;
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>window_width <font color='#5555FF'><</font><font color='#5555FF'>=</font> src.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> <font color='#979000'>2</font><font color='#5555FF'>*</font>padding_x,
"<font color='#CC0000'>Pooling windows must be small enough to fit into the padded image.</font>"<font face='Lucida Console'>)</font>;
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>window_height <font color='#5555FF'><</font><font color='#5555FF'>=</font> src.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> <font color='#979000'>2</font><font color='#5555FF'>*</font>padding_y,
"<font color='#CC0000'>Pooling windows must be small enough to fit into the padded image.</font>"<font face='Lucida Console'>)</font>;
dest.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font>
src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,
src.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,
<font color='#979000'>1</font><font color='#5555FF'>+</font><font face='Lucida Console'>(</font>src.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>+</font><font color='#979000'>2</font><font color='#5555FF'>*</font>padding_y<font color='#5555FF'>-</font>window_height<font face='Lucida Console'>)</font><font color='#5555FF'>/</font>stride_y,
<font color='#979000'>1</font><font color='#5555FF'>+</font><font face='Lucida Console'>(</font>src.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>+</font><font color='#979000'>2</font><font color='#5555FF'>*</font>padding_x<font color='#5555FF'>-</font>window_width<font face='Lucida Console'>)</font><font color='#5555FF'>/</font>stride_x
<font face='Lucida Console'>)</font>;
<font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>src.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>
<b>{</b>
dest <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
<font color='#0000FF'>return</font>;
<b>}</b>
<font color='#0000FF'>auto</font> d <font color='#5555FF'>=</font> dest.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> x_offset <font color='#5555FF'>=</font> window_width<font color='#5555FF'>/</font><font color='#979000'>2</font> <font color='#5555FF'>-</font> padding_x;
<font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> y_offset <font color='#5555FF'>=</font> window_height<font color='#5555FF'>/</font><font color='#979000'>2</font> <font color='#5555FF'>-</font> padding_y;
<font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#BB00BB'>does_max_pooling</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> n <font color='#5555FF'>=</font> <font color='#979000'>0</font>; n <font color='#5555FF'><</font> dest.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>n<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> k <font color='#5555FF'>=</font> <font color='#979000'>0</font>; k <font color='#5555FF'><</font> dest.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>k<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>auto</font> simg <font color='#5555FF'>=</font> <font color='#BB00BB'>image_plane</font><font face='Lucida Console'>(</font>src,n,k<font face='Lucida Console'>)</font>;
<font color='#0000FF'>auto</font> dimg <font color='#5555FF'>=</font> d <font color='#5555FF'>+</font> <font face='Lucida Console'>(</font>n<font color='#5555FF'>*</font>dest.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> k<font face='Lucida Console'>)</font><font color='#5555FF'>*</font>dest.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>dest.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> r <font color='#5555FF'>=</font> <font color='#979000'>0</font>; r <font color='#5555FF'><</font> dest.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>r<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> c <font color='#5555FF'>=</font> <font color='#979000'>0</font>; c <font color='#5555FF'><</font> dest.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>c<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>auto</font> win <font color='#5555FF'>=</font> <font color='#BB00BB'>centered_rect</font><font face='Lucida Console'>(</font>c<font color='#5555FF'>*</font>stride_x<font color='#5555FF'>+</font>x_offset,
r<font color='#5555FF'>*</font>stride_y<font color='#5555FF'>+</font>y_offset,
window_width,
window_height<font face='Lucida Console'>)</font>;
dimg[r<font color='#5555FF'>*</font>dest.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> c] <font color='#5555FF'>=</font> <font color='#BB00BB'>max</font><font face='Lucida Console'>(</font><font color='#BB00BB'>subm_clipped</font><font face='Lucida Console'>(</font>simg,win<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
<b>}</b>
<b>}</b>
<b>}</b>
<b>}</b>
<b>}</b>
<font color='#0000FF'>else</font>
<b>{</b>
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> n <font color='#5555FF'>=</font> <font color='#979000'>0</font>; n <font color='#5555FF'><</font> dest.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>n<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> k <font color='#5555FF'>=</font> <font color='#979000'>0</font>; k <font color='#5555FF'><</font> dest.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>k<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>auto</font> simg <font color='#5555FF'>=</font> <font color='#BB00BB'>image_plane</font><font face='Lucida Console'>(</font>src,n,k<font face='Lucida Console'>)</font>;
<font color='#0000FF'>auto</font> dimg <font color='#5555FF'>=</font> d <font color='#5555FF'>+</font> <font face='Lucida Console'>(</font>n<font color='#5555FF'>*</font>dest.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> k<font face='Lucida Console'>)</font><font color='#5555FF'>*</font>dest.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>dest.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> r <font color='#5555FF'>=</font> <font color='#979000'>0</font>; r <font color='#5555FF'><</font> dest.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>r<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> c <font color='#5555FF'>=</font> <font color='#979000'>0</font>; c <font color='#5555FF'><</font> dest.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>c<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>auto</font> win <font color='#5555FF'>=</font> <font color='#BB00BB'>centered_rect</font><font face='Lucida Console'>(</font>c<font color='#5555FF'>*</font>stride_x<font color='#5555FF'>+</font>x_offset,
r<font color='#5555FF'>*</font>stride_y<font color='#5555FF'>+</font>y_offset,
window_width,
window_height<font face='Lucida Console'>)</font>;
dimg[r<font color='#5555FF'>*</font>dest.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> c] <font color='#5555FF'>=</font> <font color='#BB00BB'>mean</font><font face='Lucida Console'>(</font><font color='#BB00BB'>subm_clipped</font><font face='Lucida Console'>(</font>simg,win<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
<b>}</b>
<b>}</b>
<b>}</b>
<b>}</b>
<b>}</b>
<b>}</b>
<font color='#0000FF'><u>void</u></font> pooling::<b><a name='get_gradient'></a>get_gradient</b><font face='Lucida Console'>(</font>
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> gradient_input,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> dest,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> src,
tensor<font color='#5555FF'>&</font> grad
<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font><font color='#BB00BB'>have_same_dimensions</font><font face='Lucida Console'>(</font>gradient_input,dest<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font><font color='#BB00BB'>have_same_dimensions</font><font face='Lucida Console'>(</font>src,grad<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>src.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>return</font>;
<b>}</b>
<font color='#0000FF'>auto</font> gi <font color='#5555FF'>=</font> gradient_input.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>auto</font> g <font color='#5555FF'>=</font> grad.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> x_offset <font color='#5555FF'>=</font> window_width<font color='#5555FF'>/</font><font color='#979000'>2</font> <font color='#5555FF'>-</font> padding_x;
<font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> y_offset <font color='#5555FF'>=</font> window_height<font color='#5555FF'>/</font><font color='#979000'>2</font> <font color='#5555FF'>-</font> padding_y;
<font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#BB00BB'>does_max_pooling</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> n <font color='#5555FF'>=</font> <font color='#979000'>0</font>; n <font color='#5555FF'><</font> dest.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>n<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> k <font color='#5555FF'>=</font> <font color='#979000'>0</font>; k <font color='#5555FF'><</font> dest.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>k<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>auto</font> simg <font color='#5555FF'>=</font> <font color='#BB00BB'>image_plane</font><font face='Lucida Console'>(</font>src,n,k<font face='Lucida Console'>)</font>;
<font color='#0000FF'>auto</font> gimg <font color='#5555FF'>=</font> g <font color='#5555FF'>+</font> <font face='Lucida Console'>(</font>n<font color='#5555FF'>*</font>grad.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> k<font face='Lucida Console'>)</font><font color='#5555FF'>*</font>grad.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>grad.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>auto</font> giimg <font color='#5555FF'>=</font> gi <font color='#5555FF'>+</font> <font face='Lucida Console'>(</font>n<font color='#5555FF'>*</font>dest.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> k<font face='Lucida Console'>)</font><font color='#5555FF'>*</font>dest.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>dest.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>auto</font> imgbox <font color='#5555FF'>=</font> <font color='#BB00BB'>get_rect</font><font face='Lucida Console'>(</font>simg<font face='Lucida Console'>)</font>;
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> r <font color='#5555FF'>=</font> <font color='#979000'>0</font>; r <font color='#5555FF'><</font> dest.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>r<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> c <font color='#5555FF'>=</font> <font color='#979000'>0</font>; c <font color='#5555FF'><</font> dest.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>c<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>auto</font> win <font color='#5555FF'>=</font> <font color='#BB00BB'>centered_rect</font><font face='Lucida Console'>(</font>c<font color='#5555FF'>*</font>stride_x<font color='#5555FF'>+</font>x_offset,
r<font color='#5555FF'>*</font>stride_y<font color='#5555FF'>+</font>y_offset,
window_width,
window_height<font face='Lucida Console'>)</font>.<font color='#BB00BB'>intersect</font><font face='Lucida Console'>(</font>imgbox<font face='Lucida Console'>)</font>;
<font color='#0000FF'>auto</font> p <font color='#5555FF'>=</font> <font color='#BB00BB'>max_point</font><font face='Lucida Console'>(</font><font color='#BB00BB'>subm</font><font face='Lucida Console'>(</font>simg,win<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font color='#5555FF'>+</font>win.<font color='#BB00BB'>tl_corner</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
gimg[p.<font color='#BB00BB'>y</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>grad.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>+</font>p.<font color='#BB00BB'>x</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>] <font color='#5555FF'>+</font><font color='#5555FF'>=</font> giimg[r<font color='#5555FF'>*</font>dest.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>+</font>c];
<b>}</b>
<b>}</b>
<b>}</b>
<b>}</b>
<b>}</b>
<font color='#0000FF'>else</font>
<b>{</b>
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> n <font color='#5555FF'>=</font> <font color='#979000'>0</font>; n <font color='#5555FF'><</font> dest.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>n<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> k <font color='#5555FF'>=</font> <font color='#979000'>0</font>; k <font color='#5555FF'><</font> dest.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>k<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>auto</font> simg <font color='#5555FF'>=</font> <font color='#BB00BB'>image_plane</font><font face='Lucida Console'>(</font>src,n,k<font face='Lucida Console'>)</font>;
<font color='#0000FF'>auto</font> gimg <font color='#5555FF'>=</font> g <font color='#5555FF'>+</font> <font face='Lucida Console'>(</font>n<font color='#5555FF'>*</font>grad.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> k<font face='Lucida Console'>)</font><font color='#5555FF'>*</font>grad.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>grad.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>auto</font> giimg <font color='#5555FF'>=</font> gi <font color='#5555FF'>+</font> <font face='Lucida Console'>(</font>n<font color='#5555FF'>*</font>dest.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> k<font face='Lucida Console'>)</font><font color='#5555FF'>*</font>dest.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>dest.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>auto</font> imgbox <font color='#5555FF'>=</font> <font color='#BB00BB'>get_rect</font><font face='Lucida Console'>(</font>simg<font face='Lucida Console'>)</font>;
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> r <font color='#5555FF'>=</font> <font color='#979000'>0</font>; r <font color='#5555FF'><</font> dest.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>r<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> c <font color='#5555FF'>=</font> <font color='#979000'>0</font>; c <font color='#5555FF'><</font> dest.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>c<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>auto</font> win <font color='#5555FF'>=</font> <font color='#BB00BB'>centered_rect</font><font face='Lucida Console'>(</font>c<font color='#5555FF'>*</font>stride_x<font color='#5555FF'>+</font>x_offset,
r<font color='#5555FF'>*</font>stride_y<font color='#5555FF'>+</font>y_offset,
window_width,
window_height<font face='Lucida Console'>)</font>.<font color='#BB00BB'>intersect</font><font face='Lucida Console'>(</font>imgbox<font face='Lucida Console'>)</font>;
<font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> delta <font color='#5555FF'>=</font> giimg[r<font color='#5555FF'>*</font>dest.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>+</font>c]<font color='#5555FF'>/</font>win.<font color='#BB00BB'>area</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> y <font color='#5555FF'>=</font> win.<font color='#BB00BB'>top</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; y <font color='#5555FF'><</font><font color='#5555FF'>=</font> win.<font color='#BB00BB'>bottom</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>y<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> x <font color='#5555FF'>=</font> win.<font color='#BB00BB'>left</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; x <font color='#5555FF'><</font><font color='#5555FF'>=</font> win.<font color='#BB00BB'>right</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>x<font face='Lucida Console'>)</font>
<b>{</b>
gimg[y<font color='#5555FF'>*</font>grad.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>+</font>x] <font color='#5555FF'>+</font><font color='#5555FF'>=</font> delta;
<b>}</b>
<b>}</b>
<b>}</b>
<b>}</b>
<b>}</b>
<b>}</b>
<b>}</b>
<b>}</b>
<font color='#009900'>// ------------------------------------------------------------------------------------
</font> <font color='#009900'>// ------------------------------------------------------------------------------------
</font> <font color='#009900'>// ------------------------------------------------------------------------------------
</font>
<font color='#0000FF'><u>void</u></font> <b><a name='img2col'></a>img2col</b><font face='Lucida Console'>(</font>
matrix<font color='#5555FF'><</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>></font><font color='#5555FF'>&</font> output,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> data,
<font color='#0000FF'><u>long</u></font> n,
<font color='#0000FF'><u>long</u></font> filter_nr,
<font color='#0000FF'><u>long</u></font> filter_nc,
<font color='#0000FF'><u>long</u></font> stride_y,
<font color='#0000FF'><u>long</u></font> stride_x,
<font color='#0000FF'><u>long</u></font> padding_y,
<font color='#0000FF'><u>long</u></font> padding_x
<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> d <font color='#5555FF'>=</font> data.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> data.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>data.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>data.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>n;
<font color='#0000FF'>const</font> rectangle boundary <font color='#5555FF'>=</font> <font color='#BB00BB'>get_rect</font><font face='Lucida Console'>(</font>data<font face='Lucida Console'>)</font>;
<font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> out_nr <font color='#5555FF'>=</font> <font color='#979000'>1</font><font color='#5555FF'>+</font><font face='Lucida Console'>(</font>data.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>+</font><font color='#979000'>2</font><font color='#5555FF'>*</font>padding_y<font color='#5555FF'>-</font>filter_nr<font face='Lucida Console'>)</font><font color='#5555FF'>/</font>stride_y;
<font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> out_nc <font color='#5555FF'>=</font> <font color='#979000'>1</font><font color='#5555FF'>+</font><font face='Lucida Console'>(</font>data.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>+</font><font color='#979000'>2</font><font color='#5555FF'>*</font>padding_x<font color='#5555FF'>-</font>filter_nc<font face='Lucida Console'>)</font><font color='#5555FF'>/</font>stride_x;
output.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font>out_nr<font color='#5555FF'>*</font>out_nc,
data.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>filter_nr<font color='#5555FF'>*</font>filter_nc<font face='Lucida Console'>)</font>;
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>output.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>!</font><font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'><u>float</u></font><font color='#5555FF'>*</font> t <font color='#5555FF'>=</font> <font color='#5555FF'>&</font><font color='#BB00BB'>output</font><font face='Lucida Console'>(</font><font color='#979000'>0</font>,<font color='#979000'>0</font><font face='Lucida Console'>)</font>;
<font color='#009900'>// now fill in the Toeplitz output matrix for the n-th sample in data.
</font> <font color='#0000FF'><u>size_t</u></font> cnt <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
<font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> max_r <font color='#5555FF'>=</font> data.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> padding_y<font color='#5555FF'>-</font><font face='Lucida Console'>(</font>filter_nr<font color='#5555FF'>-</font><font color='#979000'>1</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> max_c <font color='#5555FF'>=</font> data.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> padding_x<font color='#5555FF'>-</font><font face='Lucida Console'>(</font>filter_nc<font color='#5555FF'>-</font><font color='#979000'>1</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> r <font color='#5555FF'>=</font> <font color='#5555FF'>-</font>padding_y; r <font color='#5555FF'><</font> max_r; r<font color='#5555FF'>+</font><font color='#5555FF'>=</font>stride_y<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> c <font color='#5555FF'>=</font> <font color='#5555FF'>-</font>padding_x; c <font color='#5555FF'><</font> max_c; c<font color='#5555FF'>+</font><font color='#5555FF'>=</font>stride_x<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> k <font color='#5555FF'>=</font> <font color='#979000'>0</font>; k <font color='#5555FF'><</font> data.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>k<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> y <font color='#5555FF'>=</font> <font color='#979000'>0</font>; y <font color='#5555FF'><</font> filter_nr; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>y<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> x <font color='#5555FF'>=</font> <font color='#979000'>0</font>; x <font color='#5555FF'><</font> filter_nc; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>x<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#BB00BB'>DLIB_ASSERT</font><font face='Lucida Console'>(</font>cnt <font color='#5555FF'><</font> output.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'><u>long</u></font> xx <font color='#5555FF'>=</font> c<font color='#5555FF'>+</font>x;
<font color='#0000FF'><u>long</u></font> yy <font color='#5555FF'>=</font> r<font color='#5555FF'>+</font>y;
<font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>boundary.<font color='#BB00BB'>contains</font><font face='Lucida Console'>(</font>xx,yy<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
<font color='#5555FF'>*</font>t <font color='#5555FF'>=</font> d[<font face='Lucida Console'>(</font>k<font color='#5555FF'>*</font>data.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> yy<font face='Lucida Console'>)</font><font color='#5555FF'>*</font>data.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> xx];
<font color='#0000FF'>else</font>
<font color='#5555FF'>*</font>t <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
<font color='#5555FF'>+</font><font color='#5555FF'>+</font>t;
<font color='#5555FF'>+</font><font color='#5555FF'>+</font>cnt;
<b>}</b>
<b>}</b>
<b>}</b>
<b>}</b>
<b>}</b>
<b>}</b>
<font color='#0000FF'><u>void</u></font> <b><a name='col2img'></a>col2img</b><font face='Lucida Console'>(</font>
<font color='#0000FF'>const</font> matrix<font color='#5555FF'><</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>></font><font color='#5555FF'>&</font> output,
tensor<font color='#5555FF'>&</font> data,
<font color='#0000FF'><u>long</u></font> n,
<font color='#0000FF'><u>long</u></font> filter_nr,
<font color='#0000FF'><u>long</u></font> filter_nc,
<font color='#0000FF'><u>long</u></font> stride_y,
<font color='#0000FF'><u>long</u></font> stride_x,
<font color='#0000FF'><u>long</u></font> padding_y,
<font color='#0000FF'><u>long</u></font> padding_x
<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>const</font> <font color='#0000FF'>auto</font> d <font color='#5555FF'>=</font> data.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> data.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>data.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>data.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>n;
<font color='#0000FF'>const</font> rectangle boundary <font color='#5555FF'>=</font> <font color='#BB00BB'>get_rect</font><font face='Lucida Console'>(</font>data<font face='Lucida Console'>)</font>;
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>output.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>!</font><font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font><font color='#5555FF'>*</font> t <font color='#5555FF'>=</font> <font color='#5555FF'>&</font><font color='#BB00BB'>output</font><font face='Lucida Console'>(</font><font color='#979000'>0</font>,<font color='#979000'>0</font><font face='Lucida Console'>)</font>;
<font color='#009900'>// now fill in the Toeplitz output matrix for the n-th sample in data.
</font> <font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> max_r <font color='#5555FF'>=</font> data.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> padding_y<font color='#5555FF'>-</font><font face='Lucida Console'>(</font>filter_nr<font color='#5555FF'>-</font><font color='#979000'>1</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> max_c <font color='#5555FF'>=</font> data.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> padding_x<font color='#5555FF'>-</font><font face='Lucida Console'>(</font>filter_nc<font color='#5555FF'>-</font><font color='#979000'>1</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> r <font color='#5555FF'>=</font> <font color='#5555FF'>-</font>padding_y; r <font color='#5555FF'><</font> max_r; r<font color='#5555FF'>+</font><font color='#5555FF'>=</font>stride_y<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> c <font color='#5555FF'>=</font> <font color='#5555FF'>-</font>padding_x; c <font color='#5555FF'><</font> max_c; c<font color='#5555FF'>+</font><font color='#5555FF'>=</font>stride_x<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> k <font color='#5555FF'>=</font> <font color='#979000'>0</font>; k <font color='#5555FF'><</font> data.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>k<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> y <font color='#5555FF'>=</font> <font color='#979000'>0</font>; y <font color='#5555FF'><</font> filter_nr; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>y<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> x <font color='#5555FF'>=</font> <font color='#979000'>0</font>; x <font color='#5555FF'><</font> filter_nc; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>x<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'><u>long</u></font> xx <font color='#5555FF'>=</font> c<font color='#5555FF'>+</font>x;
<font color='#0000FF'><u>long</u></font> yy <font color='#5555FF'>=</font> r<font color='#5555FF'>+</font>y;
<font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>boundary.<font color='#BB00BB'>contains</font><font face='Lucida Console'>(</font>xx,yy<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
d[<font face='Lucida Console'>(</font>k<font color='#5555FF'>*</font>data.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> yy<font face='Lucida Console'>)</font><font color='#5555FF'>*</font>data.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> xx] <font color='#5555FF'>+</font><font color='#5555FF'>=</font> <font color='#5555FF'>*</font>t;
<font color='#5555FF'>+</font><font color='#5555FF'>+</font>t;
<b>}</b>
<b>}</b>
<b>}</b>
<b>}</b>
<b>}</b>
<b>}</b>
<font color='#0000FF'><u>void</u></font> tensor_conv::<b><a name='operator'></a>operator</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font face='Lucida Console'>(</font>
<font color='#0000FF'>const</font> <font color='#0000FF'><u>bool</u></font> add_to_output,
resizable_tensor<font color='#5555FF'>&</font> output,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> data,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> filters
<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>last_stride_y <font color='#5555FF'>></font> <font color='#979000'>0</font> <font color='#5555FF'>&</font><font color='#5555FF'>&</font> last_stride_x <font color='#5555FF'>></font> <font color='#979000'>0</font>, "<font color='#CC0000'>You must call setup() before calling this function.</font>"<font face='Lucida Console'>)</font>;
output.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font>data.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,
filters.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,
<font color='#979000'>1</font><font color='#5555FF'>+</font><font face='Lucida Console'>(</font>data.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>+</font><font color='#979000'>2</font><font color='#5555FF'>*</font>last_padding_y<font color='#5555FF'>-</font>filters.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font>last_stride_y,
<font color='#979000'>1</font><font color='#5555FF'>+</font><font face='Lucida Console'>(</font>data.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>+</font><font color='#979000'>2</font><font color='#5555FF'>*</font>last_padding_x<font color='#5555FF'>-</font>filters.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font>last_stride_x<font face='Lucida Console'>)</font>;
<font face='Lucida Console'>(</font><font color='#5555FF'>*</font><font color='#0000FF'>this</font><font face='Lucida Console'>)</font><font face='Lucida Console'>(</font>add_to_output, <font color='#0000FF'>static_cast</font><font color='#5555FF'><</font>tensor<font color='#5555FF'>&</font><font color='#5555FF'>></font><font face='Lucida Console'>(</font>output<font face='Lucida Console'>)</font>,data,filters<font face='Lucida Console'>)</font>;
<b>}</b>
<font color='#0000FF'><u>void</u></font> tensor_conv::<b><a name='operator'></a>operator</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font face='Lucida Console'>(</font>
<font color='#0000FF'>const</font> <font color='#0000FF'><u>bool</u></font> add_to_output,
tensor<font color='#5555FF'>&</font> output,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> data,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> filters
<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font><font color='#BB00BB'>is_same_object</font><font face='Lucida Console'>(</font>output,data<font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>false</font><font face='Lucida Console'>)</font>;
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font><font color='#BB00BB'>is_same_object</font><font face='Lucida Console'>(</font>output,filters<font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>false</font><font face='Lucida Console'>)</font>;
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>filters.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> data.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>last_stride_y <font color='#5555FF'>></font> <font color='#979000'>0</font> <font color='#5555FF'>&</font><font color='#5555FF'>&</font> last_stride_x <font color='#5555FF'>></font> <font color='#979000'>0</font>, "<font color='#CC0000'>You must call setup() before calling this function.</font>"<font face='Lucida Console'>)</font>;
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>filters.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'><</font><font color='#5555FF'>=</font> data.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> <font color='#979000'>2</font><font color='#5555FF'>*</font>last_padding_y,
"<font color='#CC0000'>Filter windows must be small enough to fit into the padded image.</font>"<font face='Lucida Console'>)</font>;
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>filters.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'><</font><font color='#5555FF'>=</font> data.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> <font color='#979000'>2</font><font color='#5555FF'>*</font>last_padding_x,
"<font color='#CC0000'>Filter windows must be small enough to fit into the padded image.</font>"<font face='Lucida Console'>)</font>;
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>output.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> data.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>output.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> filters.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>output.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font><font color='#5555FF'>+</font><font face='Lucida Console'>(</font>data.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>+</font><font color='#979000'>2</font><font color='#5555FF'>*</font>last_padding_y<font color='#5555FF'>-</font>filters.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font>last_stride_y<font face='Lucida Console'>)</font>;
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>output.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font><font color='#5555FF'>+</font><font face='Lucida Console'>(</font>data.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>+</font><font color='#979000'>2</font><font color='#5555FF'>*</font>last_padding_x<font color='#5555FF'>-</font>filters.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font>last_stride_x<font face='Lucida Console'>)</font>;
matrix<font color='#5555FF'><</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>></font> temp;
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> n <font color='#5555FF'>=</font> <font color='#979000'>0</font>; n <font color='#5555FF'><</font> data.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>n<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#BB00BB'>img2col</font><font face='Lucida Console'>(</font>temp, data, n, filters.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, filters.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, last_stride_y, last_stride_x, last_padding_y, last_padding_x<font face='Lucida Console'>)</font>;
<font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>add_to_output<font face='Lucida Console'>)</font>
output.<font color='#BB00BB'>add_to_sample</font><font face='Lucida Console'>(</font>n, <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>filters<font face='Lucida Console'>)</font><font color='#5555FF'>*</font><font color='#BB00BB'>trans</font><font face='Lucida Console'>(</font>temp<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>else</font>
output.<font color='#BB00BB'>set_sample</font><font face='Lucida Console'>(</font>n, <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>filters<font face='Lucida Console'>)</font><font color='#5555FF'>*</font><font color='#BB00BB'>trans</font><font face='Lucida Console'>(</font>temp<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
<b>}</b>
<b>}</b>
<font color='#009900'>// ------------------------------------------------------------------------------------
</font>
<font color='#0000FF'><u>void</u></font> tensor_conv::
<b><a name='get_gradient_for_data'></a>get_gradient_for_data</b> <font face='Lucida Console'>(</font>
<font color='#0000FF'>const</font> <font color='#0000FF'><u>bool</u></font> add_to_output,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> gradient_input,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> filters,
tensor<font color='#5555FF'>&</font> data_gradient
<font face='Lucida Console'>)</font>
<b>{</b>
matrix<font color='#5555FF'><</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>></font> temp;
<font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#5555FF'>!</font>add_to_output<font face='Lucida Console'>)</font>
data_gradient <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> n <font color='#5555FF'>=</font> <font color='#979000'>0</font>; n <font color='#5555FF'><</font> gradient_input.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>n<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>auto</font> gi <font color='#5555FF'>=</font> <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>gradient_input.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>+</font>gradient_input.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>gradient_input.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>gradient_input.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>n,
gradient_input.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,
gradient_input.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>gradient_input.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
temp <font color='#5555FF'>=</font> <font color='#BB00BB'>trans</font><font face='Lucida Console'>(</font>gi<font face='Lucida Console'>)</font><font color='#5555FF'>*</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>filters<font face='Lucida Console'>)</font>;
<font color='#BB00BB'>col2img</font><font face='Lucida Console'>(</font>temp, data_gradient, n, filters.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, filters.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, last_stride_y, last_stride_x, last_padding_y, last_padding_x<font face='Lucida Console'>)</font>;
<b>}</b>
<b>}</b>
<font color='#009900'>// ------------------------------------------------------------------------------------
</font>
<font color='#0000FF'><u>void</u></font> tensor_conv::
<b><a name='get_gradient_for_filters'></a>get_gradient_for_filters</b> <font face='Lucida Console'>(</font>
<font color='#0000FF'>const</font> <font color='#0000FF'><u>bool</u></font> add_to_output,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> gradient_input,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> data,
tensor<font color='#5555FF'>&</font> filters_gradient
<font face='Lucida Console'>)</font>
<b>{</b>
matrix<font color='#5555FF'><</font><font color='#0000FF'><u>float</u></font><font color='#5555FF'>></font> temp;
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> n <font color='#5555FF'>=</font> <font color='#979000'>0</font>; n <font color='#5555FF'><</font> gradient_input.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>n<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>auto</font> gi <font color='#5555FF'>=</font> <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>gradient_input.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>+</font>gradient_input.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>gradient_input.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>gradient_input.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>n,
gradient_input.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,
gradient_input.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>gradient_input.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
<font color='#BB00BB'>img2col</font><font face='Lucida Console'>(</font>temp, data, n, filters_gradient.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, filters_gradient.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, last_stride_y, last_stride_x, last_padding_y, last_padding_x<font face='Lucida Console'>)</font>;
<font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>n <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>add_to_output<font face='Lucida Console'>)</font>
filters_gradient <font color='#5555FF'>+</font><font color='#5555FF'>=</font> gi<font color='#5555FF'>*</font>temp;
<font color='#0000FF'>else</font>
filters_gradient <font color='#5555FF'>=</font> gi<font color='#5555FF'>*</font>temp;
<b>}</b>
<font color='#0000FF'>else</font>
<b>{</b>
filters_gradient <font color='#5555FF'>+</font><font color='#5555FF'>=</font> gi<font color='#5555FF'>*</font>temp;
<b>}</b>
<b>}</b>
<b>}</b>
<font color='#009900'>// ------------------------------------------------------------------------------------
</font>
<font color='#0000FF'><u>void</u></font> <b><a name='copy_tensor'></a>copy_tensor</b><font face='Lucida Console'>(</font>
<font color='#0000FF'><u>bool</u></font> add_to,
tensor<font color='#5555FF'>&</font> dest,
<font color='#0000FF'><u>size_t</u></font> dest_k_offset,
<font color='#0000FF'>const</font> tensor<font color='#5555FF'>&</font> src,
<font color='#0000FF'><u>size_t</u></font> src_k_offset,
<font color='#0000FF'><u>size_t</u></font> count_k
<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>const</font> <font color='#0000FF'><u>size_t</u></font> dest_sample_size <font color='#5555FF'>=</font> <font color='#0000FF'>static_cast</font><font color='#5555FF'><</font><font color='#0000FF'><u>size_t</u></font><font color='#5555FF'>></font><font face='Lucida Console'>(</font>dest.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>*</font> dest.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>*</font> dest.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>const</font> <font color='#0000FF'><u>size_t</u></font> src_sample_size <font color='#5555FF'>=</font> <font color='#0000FF'>static_cast</font><font color='#5555FF'><</font><font color='#0000FF'><u>size_t</u></font><font color='#5555FF'>></font><font face='Lucida Console'>(</font>src.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>*</font> src.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>*</font> src.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>const</font> <font color='#0000FF'><u>size_t</u></font> block_size <font color='#5555FF'>=</font> count_k <font color='#5555FF'>*</font> dest.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>*</font> dest.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>dest.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&</font><font color='#5555FF'>&</font>
dest.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> src.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&</font><font color='#5555FF'>&</font> dest.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> src.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, "<font color='#CC0000'>All sources should fit into dest tensor size</font>"<font face='Lucida Console'>)</font>;
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>dest.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> dest_k_offset <font color='#5555FF'>></font><font color='#5555FF'>=</font> count_k, "<font color='#CC0000'>Not enough space in dest tensor</font>"<font face='Lucida Console'>)</font>;
<font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>src.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> src_k_offset <font color='#5555FF'>></font><font color='#5555FF'>=</font> count_k, "<font color='#CC0000'>Not enough space in src tensor</font>"<font face='Lucida Console'>)</font>;
<font color='#0000FF'><u>float</u></font><font color='#5555FF'>*</font> dest_p <font color='#5555FF'>=</font> dest.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> dest_k_offset <font color='#5555FF'>*</font> dest.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>*</font> dest.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font><font color='#5555FF'>*</font> src_p <font color='#5555FF'>=</font> src.<font color='#BB00BB'>host</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> src_k_offset <font color='#5555FF'>*</font> src.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>*</font> src.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> i <font color='#5555FF'>=</font> <font color='#979000'>0</font>; i <font color='#5555FF'><</font> src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>i<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>add_to<font face='Lucida Console'>)</font>
<b>{</b>
<font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>size_t</u></font> j <font color='#5555FF'>=</font> <font color='#979000'>0</font>; j <font color='#5555FF'><</font> block_size; <font color='#5555FF'>+</font><font color='#5555FF'>+</font>j<font face='Lucida Console'>)</font>
dest_p[j] <font color='#5555FF'>+</font><font color='#5555FF'>=</font> src_p[j];
<b>}</b>
<font color='#0000FF'>else</font>
<b>{</b>
::<font color='#BB00BB'>memcpy</font><font face='Lucida Console'>(</font>dest_p, src_p, block_size <font color='#5555FF'>*</font> <font color='#0000FF'>sizeof</font><font face='Lucida Console'>(</font><font color='#0000FF'><u>float</u></font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
<b>}</b>
dest_p <font color='#5555FF'>+</font><font color='#5555FF'>=</font> dest_sample_size;
src_p <font color='#5555FF'>+</font><font color='#5555FF'>=</font> src_sample_size;
<b>}</b>
<b>}</b>
<font color='#009900'>// ------------------------------------------------------------------------------------
</font> <font color='#009900'>// ------------------------------------------------------------------------------------
</font> <font color='#009900'>// ------------------------------------------------------------------------------------
</font>
<b>}</b>
<b>}</b>
<font color='#0000FF'>#endif</font> <font color='#009900'>// DLIB_DNN_CPU_cPP_
</font>
</pre></body></html>