Upgraded to cuDNN v4.0. This means changing the binding to max_pool a little

since that's a little different in cuDNN.  I also removed my CUDA code for
doing batch normalization and replaced it with cuDNN's new batch normalization
methods.

Finally, I forgot to add a convolutional option to the bn_ object.  Now it has
one so you can set the mode however you like, either BATCH_NORM_FC or
BATCH_NORM_CONV.

dlib/CMakeLists.txt

@@ -476,7 +476,7 @@ if (NOT TARGET dlib)
             set(DLIB_USE_CUDA OFF CACHE STRING ${DLIB_USE_BLAS_STR} FORCE )
             toggle_preprocessor_switch(DLIB_USE_CUDA)
             if (NOT cudnn OR NOT cudnn_include)
-               message(STATUS "***cuDNN NOT FOUND.  DLIB WILL NOT USE CUDA.***")
+               message(STATUS "***cuDNN V4.0 OR GREATER NOT FOUND.  DLIB WILL NOT USE CUDA.***")
             endif()
             if (NOT COMPILER_CAN_DO_CPP_11)
                message(STATUS "***Dlib CUDA support requires C++11 but your compiler doesn't support it.***")

dlib/dnn/cpu_dlib.h

@@ -13,6 +13,10 @@ namespace dlib
     namespace cpu 
     {
 
+    // ----------------------------------------------------------------------------------------
+
+        const double BATCH_NORM_EPS = 0.00001;
+
     // -----------------------------------------------------------------------------------
 
         void multiply (
@@ -73,19 +77,28 @@ namespace dlib
 
     // -----------------------------------------------------------------------------------
 
+        void batch_normalize_inference (
+            resizable_tensor& dest,
+            const tensor& src,
+            const tensor& gamma, 
+            const tensor& beta,
+            const tensor& running_means,
+            const tensor& running_invstds
+        );
+
         void batch_normalize (
             resizable_tensor& dest,
             resizable_tensor& means,
             resizable_tensor& invstds,
+            const double averaging_factor,
+            resizable_tensor& running_means,
+            resizable_tensor& running_invstds,
             const tensor& src,
             const tensor& gamma, 
             const tensor& beta 
         );
 
-        class batch_normalize_gradient
-        {
-        public:
-            void operator() (
+        void batch_normalize_gradient (
             const tensor& gradient_input,
             const tensor& means,
             const tensor& invstds,
@@ -95,23 +108,29 @@ namespace dlib
             tensor& gamma_grad, 
             tensor& beta_grad 
         );
-        private:
-            resizable_tensor dvars, dmeans;
-        };
+
+        void batch_normalize_conv_inference (
+            resizable_tensor& dest,
+            const tensor& src,
+            const tensor& gamma, 
+            const tensor& beta,
+            const tensor& running_means,
+            const tensor& running_invstds
+        );
 
         void batch_normalize_conv (
             resizable_tensor& dest,
             resizable_tensor& means,
             resizable_tensor& invstds,
+            const double averaging_factor,
+            resizable_tensor& running_means,
+            resizable_tensor& running_invstds,
             const tensor& src,
             const tensor& gamma, 
             const tensor& beta 
         );
 
-        class batch_normalize_conv_gradient
-        {
-        public:
-            void operator() (
+        void batch_normalize_conv_gradient (
             const tensor& gradient_input,
             const tensor& means,
             const tensor& invstds,
@@ -121,9 +140,6 @@ namespace dlib
             tensor& gamma_grad, 
             tensor& beta_grad 
         );
-        private:
-            resizable_tensor dvars, dmeans;
-        };
 
     // -----------------------------------------------------------------------------------
 

dlib/dnn/cuda_dlib.h

@@ -76,60 +76,6 @@ namespace dlib
             const tensor& gradient_input
         );
 
-    // -----------------------------------------------------------------------------------
-
-        void batch_normalize (
-            resizable_tensor& dest,
-            resizable_tensor& means,
-            resizable_tensor& invstds,
-            const tensor& src,
-            const tensor& gamma, 
-            const tensor& beta 
-        );
-
-        class batch_normalize_gradient
-        {
-        public:
-            void operator() (
-                const tensor& gradient_input,
-                const tensor& means,
-                const tensor& invstds,
-                const tensor& src,
-                const tensor& gamma,
-                tensor& src_grad,
-                tensor& gamma_grad, 
-                tensor& beta_grad 
-            );
-        private:
-            resizable_tensor dvars, dmeans;
-        };
-
-        void batch_normalize_conv (
-            resizable_tensor& dest,
-            resizable_tensor& means,
-            resizable_tensor& invstds,
-            const tensor& src,
-            const tensor& gamma, 
-            const tensor& beta 
-        );
-
-        class batch_normalize_conv_gradient
-        {
-        public:
-            void operator() (
-                const tensor& gradient_input,
-                const tensor& means,
-                const tensor& invstds,
-                const tensor& src,
-                const tensor& gamma,
-                tensor& src_grad,
-                tensor& gamma_grad, 
-                tensor& beta_grad 
-            );
-        private:
-            resizable_tensor dvars, dmeans;
-        };
-
     // -----------------------------------------------------------------------------------
 
         void threshold (

dlib/dnn/cudnn_dlibapi.h

@@ -132,6 +132,74 @@ namespace dlib
                   assigns it to grad.
         !*/
 
+    // ------------------------------------------------------------------------------------
+
+        void batch_normalize_inference (
+            resizable_tensor& dest,
+            const tensor& src,
+            const tensor& gamma, 
+            const tensor& beta,
+            const tensor& running_means,
+            const tensor& running_invstds
+        );
+
+        void batch_normalize (
+            resizable_tensor& dest,
+            resizable_tensor& means,
+            resizable_tensor& invstds,
+            const double averaging_factor,
+            resizable_tensor& running_means,
+            resizable_tensor& running_invstds,
+            const tensor& src,
+            const tensor& gamma, 
+            const tensor& beta 
+        );
+
+        void batch_normalize_gradient(
+            const tensor& gradient_input,
+            const tensor& means,
+            const tensor& invstds,
+            const tensor& src,
+            const tensor& gamma,
+            tensor& src_grad,
+            tensor& gamma_grad, 
+            tensor& beta_grad 
+        );
+
+    // ------------------------------------------------------------------------------------
+
+        void batch_normalize_conv_inference (
+            resizable_tensor& dest,
+            const tensor& src,
+            const tensor& gamma, 
+            const tensor& beta,
+            const tensor& running_means,
+            const tensor& running_invstds
+        );
+
+        void batch_normalize_conv (
+            resizable_tensor& dest,
+            resizable_tensor& means,
+            resizable_tensor& invstds,
+            const double averaging_factor,
+            resizable_tensor& running_means,
+            resizable_tensor& running_invstds,
+            const tensor& src,
+            const tensor& gamma, 
+            const tensor& beta 
+        );
+
+        void batch_normalize_conv_gradient(
+            const tensor& gradient_input,
+            const tensor& means,
+            const tensor& invstds,
+            const tensor& src,
+            const tensor& gamma,
+            tensor& src_grad,
+            tensor& gamma_grad, 
+            tensor& beta_grad 
+        );
+
     // ------------------------------------------------------------------------------------
 
         class tensor_conv
@@ -295,14 +363,14 @@ namespace dlib
                 ensures
                     - #dest.num_samples() == src.num_samples()
                     - #dest.k() == src.k()
-                    - #dest.nr() == src.nr()/stride_y
-                    - #dest.nc() == src.nc()/stride_x
+                    - #dest.nr() == 1+(src.nr()-window_height%2)/stride_y
+                    - #dest.nc() == 1+(src.nc()-window_width%2)/stride_x
                     - for all valid s, k, r, and c:
                         - image_plane(#dest,s,k)(r,c) == max(subm_clipped(image_plane(src,s,k),
+                                                                          centered_rect(c*stride_x,
                                                                                         r*stride_y,
-                                                                          c*stride_x,
-                                                                          window_height,
-                                                                          window_width))
+                                                                                        window_width,
+                                                                                        window_height)))
             !*/
 
             void get_gradient(

dlib/dnn/layers.h

@@ -298,19 +298,37 @@ namespace dlib
 
 // ----------------------------------------------------------------------------------------
 
+    enum batch_normalization_mode
+    {
+        BATCH_NORM_CONV = 0,
+        BATCH_NORM_FC = 1
+    };
+
     class bn_
     {
     public:
-        bn_() : num_updates(0), running_stats_window_size(1000), running_nim_out_of_date(true)
+        bn_() : num_updates(0), running_stats_window_size(1000), mode(BATCH_NORM_FC)
+        {}
+
+        explicit bn_(batch_normalization_mode mode_) : num_updates(0), running_stats_window_size(1000), mode(mode_)
         {}
 
+        batch_normalization_mode get_mode() const { return mode; }
+
         template <typename SUBNET>
         void setup (const SUBNET& sub)
+        {
+            if (mode == BATCH_NORM_FC)
             {
                 gamma = alias_tensor(1,
                                 sub.get_output().k(),
                                 sub.get_output().nr(),
                                 sub.get_output().nc());
+            }
+            else
+            {
+                gamma = alias_tensor(1, sub.get_output().k());
+            }
             beta = gamma;
 
             params.set_size(gamma.size()+beta.size());
@@ -318,15 +336,11 @@ namespace dlib
             gamma(params,0) = 1;
             beta(params,gamma.size()) = 0;
 
-            running_means.set_size(1,
-                            sub.get_output().k(),
-                            sub.get_output().nr(),
-                            sub.get_output().nc());
-            running_invstds.copy_size(running_means);
+            running_means.copy_size(gamma(params,0));
+            running_invstds.copy_size(gamma(params,0));
             running_means = 0;
             running_invstds = 1;
             num_updates = 0;
-            running_nim_out_of_date = true;
         }
 
         template <typename SUBNET>
@@ -336,27 +350,20 @@ namespace dlib
             auto b = beta(params,gamma.size());
             if (sub.get_output().num_samples() > 1)
             {
-                tt::batch_normalize(output, means, invstds, sub.get_output(), g, b); 
-
-                const double decay = num_updates/(num_updates+1.0);
+                const double decay = 1.0 - num_updates/(num_updates+1.0);
                 if (num_updates <running_stats_window_size)
                     ++num_updates;
-                tt::affine_transform(running_means, running_means, means, decay, 1-decay, 0);
-                tt::affine_transform(running_invstds, running_invstds, invstds, decay, 1-decay, 0);
-                running_nim_out_of_date = true;
+                if (mode == BATCH_NORM_FC)
+                    tt::batch_normalize(output, means, invstds, decay, running_means, running_invstds, sub.get_output(), g, b);
+                else 
+                    tt::batch_normalize_conv(output, means, invstds, decay, running_means, running_invstds, sub.get_output(), g, b);
             }
             else // we are running in testing mode so we just linearly scale the input tensor.
             {
-                if (running_nim_out_of_date)
-                {
-                    running_nim_out_of_date = false;
-                    running_nim.copy_size(running_means);
-                    tt::multiply(running_nim, running_means, running_invstds);
-                    running_nim *= -1;
-                }
-                output.copy_size(sub.get_output());
-                tt::affine_transform(output, sub.get_output(), running_invstds, running_nim);
-                tt::affine_transform(output, output, g, b);
+                if (mode == BATCH_NORM_FC)
+                    tt::batch_normalize_inference(output, sub.get_output(), g, b, running_means, running_invstds);
+                else
+                    tt::batch_normalize_conv_inference(output, sub.get_output(), g, b, running_means, running_invstds);
             }
         } 
 
@@ -366,7 +373,10 @@ namespace dlib
             auto g = gamma(params,0);
             auto g_grad = gamma(params_grad, 0);
             auto b_grad = beta(params_grad, gamma.size());
-            bng(gradient_input, means, invstds, sub.get_output(), g, sub.get_gradient_input(), g_grad, b_grad);
+            if (mode == BATCH_NORM_FC)
+                tt::batch_normalize_gradient(gradient_input, means, invstds, sub.get_output(), g, sub.get_gradient_input(), g_grad, b_grad );
+            else
+                tt::batch_normalize_conv_gradient(gradient_input, means, invstds, sub.get_output(), g, sub.get_gradient_input(), g_grad, b_grad );
         }
 
         const tensor& get_layer_params() const { return params; }
@@ -384,6 +394,7 @@ namespace dlib
             serialize(item.running_invstds, out);
             serialize(item.num_updates, out);
             serialize(item.running_stats_window_size, out);
+            serialize((int)item.mode, out);
         }
 
         friend void deserialize(bn_& item, std::istream& in)
@@ -401,21 +412,20 @@ namespace dlib
             deserialize(item.running_invstds, in);
             deserialize(item.num_updates, in);
             deserialize(item.running_stats_window_size, in);
-            item.running_nim_out_of_date = true;
+            int mode;
+            deserialize(mode, in);
+            item.mode = (batch_normalization_mode)mode;
         }
 
     private:
 
-        tt::batch_normalize_gradient bng;
         resizable_tensor params;
         alias_tensor gamma, beta;
         resizable_tensor means, running_means;
         resizable_tensor invstds, running_invstds;
         unsigned long num_updates;
         unsigned long running_stats_window_size;
-        
-        bool running_nim_out_of_date;
-        resizable_tensor running_nim;
+        batch_normalization_mode mode;
     };
 
     template <typename SUBNET>

dlib/dnn/layers_abstract.h

@@ -563,6 +563,12 @@ namespace dlib
 
 // ----------------------------------------------------------------------------------------
 
+    enum batch_normalization_mode
+    {
+        BATCH_NORM_CONV = 0,
+        BATCH_NORM_FC = 1
+    };
+
     class bn_
     {
         /*!
@@ -581,6 +587,34 @@ namespace dlib
     public:
         bn_(
         );
+        /*!
+            ensures
+                - #get_mode() == BATCH_NORM_FC
+        !*/
+
+        explicit bn_(
+            batch_normalization_mode mode
+        );
+        /*!
+            ensures
+                - #get_mode() == mode 
+        !*/
+
+        batch_normalization_mode get_mode(
+        ) const; 
+        /*!
+            ensures
+                - returns the mode of this layer, either BATCH_NORM_CONV or BATCH_NORM_FC.
+                  If the mode is BATCH_NORM_FC then the normalization is applied across the
+                  samples in a tensor (i.e. k()*nr()*nc() different things will be
+                  normalized).  Otherwise, normalization is applied across everything
+                  except for the k() dimension, resulting in there being only k()
+                  normalization equations that are applied spatially over the tensor.
+
+                  Therefore, if you are putting batch normalization after a fully connected
+                  layer you should use BATCH_NORM_FC.  Otherwise, if you are putting batch
+                  normalization after a convolutional layer you should use BATCH_NORM_CONV.
+        !*/
 
         template <typename SUBNET> void setup (const SUBNET& sub);
         template <typename SUBNET> void forward(const SUBNET& sub, resizable_tensor& output);

dlib/dnn/tensor_tools.cpp

@@ -183,37 +183,113 @@ namespace dlib { namespace tt
 
 // ----------------------------------------------------------------------------------------
 
+    void batch_normalize_inference (
+        resizable_tensor& dest,
+        const tensor& src,
+        const tensor& gamma, 
+        const tensor& beta,
+        const tensor& running_means,
+        const tensor& running_invstds
+    )
+    {
+#ifdef DLIB_USE_CUDA
+        cuda::batch_normalize_inference(dest,src,gamma,beta,running_means,running_invstds);
+#else
+        cpu::batch_normalize_inference(dest,src,gamma,beta,running_means,running_invstds);
+#endif
+    }
+
     void batch_normalize (
         resizable_tensor& dest,
         resizable_tensor& means,
         resizable_tensor& vars,
+        const double averaging_factor,
+        resizable_tensor& running_means,
+        resizable_tensor& running_invstds,
         const tensor& src,
         const tensor& gamma, 
         const tensor& beta 
     )
     {
 #ifdef DLIB_USE_CUDA
-        cuda::batch_normalize(dest,means,vars,src,gamma,beta);
+        cuda::batch_normalize(dest,means,vars,averaging_factor,running_means,running_invstds,src,gamma,beta);
+#else
+        cpu::batch_normalize(dest,means,vars,averaging_factor,running_means,running_invstds,src,gamma,beta);
+#endif
+    }
+
+    void batch_normalize_gradient (
+            const tensor& gradient_input,
+            const tensor& means,
+            const tensor& invstds,
+            const tensor& src,
+            const tensor& gamma,
+            tensor& src_grad,
+            tensor& gamma_grad, 
+            tensor& beta_grad 
+    )
+    {
+             
+#ifdef DLIB_USE_CUDA
+        cuda::batch_normalize_gradient(gradient_input, means, invstds, src, gamma, src_grad, gamma_grad, beta_grad);
 #else
-        cpu::batch_normalize(dest,means,vars,src,gamma,beta);
+        cpu::batch_normalize_gradient(gradient_input, means, invstds, src, gamma, src_grad, gamma_grad, beta_grad);
 #endif
     }
 
 // ----------------------------------------------------------------------------------------
 
+    void batch_normalize_conv_inference (
+        resizable_tensor& dest,
+        const tensor& src,
+        const tensor& gamma, 
+        const tensor& beta,
+        const tensor& running_means,
+        const tensor& running_invstds
+    )
+    {
+#ifdef DLIB_USE_CUDA
+        cuda::batch_normalize_conv_inference(dest,src,gamma,beta,running_means,running_invstds);
+#else
+        cpu::batch_normalize_conv_inference(dest,src,gamma,beta,running_means,running_invstds);
+#endif
+    }
+
     void batch_normalize_conv (
         resizable_tensor& dest,
         resizable_tensor& means,
         resizable_tensor& vars,
+        const double averaging_factor,
+        resizable_tensor& running_means,
+        resizable_tensor& running_invstds,
         const tensor& src,
         const tensor& gamma, 
         const tensor& beta 
     )
     {
 #ifdef DLIB_USE_CUDA
-        cuda::batch_normalize_conv(dest,means,vars,src,gamma,beta);
+        cuda::batch_normalize_conv(dest,means,vars,averaging_factor,running_means,running_invstds,src,gamma,beta);
+#else
+        cpu::batch_normalize_conv(dest,means,vars,averaging_factor,running_means,running_invstds,src,gamma,beta);
+#endif
+    }
+
+    void batch_normalize_conv_gradient (
+            const tensor& gradient_input,
+            const tensor& means,
+            const tensor& invstds,
+            const tensor& src,
+            const tensor& gamma,
+            tensor& src_grad,
+            tensor& gamma_grad, 
+            tensor& beta_grad 
+    )
+    {
+             
+#ifdef DLIB_USE_CUDA
+        cuda::batch_normalize_conv_gradient(gradient_input, means, invstds, src, gamma, src_grad, gamma_grad, beta_grad);
 #else
-        cpu::batch_normalize_conv(dest,means,vars,src,gamma,beta);
+        cpu::batch_normalize_conv_gradient(gradient_input, means, invstds, src, gamma, src_grad, gamma_grad, beta_grad);
 #endif
     }
 

dlib/dnn/tensor_tools.h

@@ -198,10 +198,38 @@ namespace dlib { namespace tt
 
 // ----------------------------------------------------------------------------------------
 
+    void batch_normalize_inference (
+        resizable_tensor& dest,
+        const tensor& src,
+        const tensor& gamma, 
+        const tensor& beta,
+        const tensor& running_means,
+        const tensor& running_invstds
+    );
+    /*!
+        requires
+            - gamma.num_samples() == 1 
+            - gamma.nr() == src.nr() 
+            - gamma.nc() == src.nc() 
+            - gamma.k()  == src.k()
+            - have_same_dimensions(gamma, beta) 
+            - have_same_dimensions(gamma, running_means) 
+            - have_same_dimensions(gamma, running_invstds)
+        ensures
+            - Just linearly transforms src as a call to batch_normalize() would if the resulting
+              means and invstds were running_means and running_invstds.  That is, this function 
+              performs: 
+                dest = gamma*(src-running_means)*running_invstds + beta
+              Note that it does it in a pointwise fashion over the samples in src.
+    !*/
+
     void batch_normalize (
         resizable_tensor& dest,
         resizable_tensor& means,
         resizable_tensor& invstds,
+        const double averaging_factor,
+        resizable_tensor& running_means,
+        resizable_tensor& running_invstds,
         const tensor& src,
         const tensor& gamma, 
         const tensor& beta 
@@ -214,6 +242,10 @@ namespace dlib { namespace tt
             - gamma.nr() == beta.nr() == src.nr()
             - gamma.nc() == beta.nc() == src.nc()
             - gamma.k()  == beta.k()  == src.k()
+            - 0 <= averaging_factor <= 1
+            - if (averaging_factor != 1)
+                - have_same_dimensions(running_means, means) == true
+                - have_same_dimensions(running_invstds, invstds) == true
         ensures
             - have_same_dimensions(#dest, src) == true
             - #means.num_samples() == 1
@@ -224,12 +256,11 @@ namespace dlib { namespace tt
             - #src == the batch normalized version of src.
             - #means == the mean values of the contents of src.
             - #invstds == 1/(the standard deviation values of the contents of src).
+            - #running_means = (1-averaging_factor)*mat(#running_means) + averaging_factor*mat(#means);
+            - #running_invstds = (1-averaging_factor)*mat(#running_invstds) + averaging_factor*mat(#invstds);
     !*/
 
-    class batch_normalize_gradient
-    {
-    public:
-        void operator() (
+    void batch_normalize_gradient (
         const tensor& gradient_input,
         const tensor& means,
         const tensor& invstds,
@@ -238,7 +269,7 @@ namespace dlib { namespace tt
         tensor& src_grad,
         tensor& gamma_grad, 
         tensor& beta_grad 
-        ){impl(gradient_input,means,invstds,src,gamma,src_grad,gamma_grad,beta_grad);}
+    );
     /*!
         requires
             - invstds and means should be the output of a call to
@@ -261,20 +292,42 @@ namespace dlib { namespace tt
             - Assigns the gradient of f() with respect to gamma to #gamma_grad.
             - Assigns the gradient of f() with respect to beta to #beta_grad.
     !*/
-    private:
-#ifdef DLIB_USE_CUDA
-        cuda::batch_normalize_gradient impl;
-#else
-        cpu::batch_normalize_gradient impl;
-#endif
-    };
 
 // ----------------------------------------------------------------------------------------
 
+    void batch_normalize_conv_inference (
+        resizable_tensor& dest,
+        const tensor& src,
+        const tensor& gamma, 
+        const tensor& beta,
+        const tensor& running_means,
+        const tensor& running_invstds
+    );
+    /*!
+        requires
+            - gamma.num_samples() == 1 
+            - gamma.nr() == 1 
+            - gamma.nc() == 1 
+            - gamma.k()  == src.k()
+            - have_same_dimensions(gamma, beta) 
+            - have_same_dimensions(gamma, running_means) 
+            - have_same_dimensions(gamma, running_invstds)
+        ensures
+            - Just linearly transforms src as a call to batch_normalize() would if the resulting
+              means and invstds were running_means and running_invstds.  That is, this function 
+              performs: 
+                dest = gamma*(src-running_means)*running_invstds + beta
+              Note that it does it in a pointwise fashion over the samples, rows, and
+              columns in src.
+    !*/
+
     void batch_normalize_conv (
         resizable_tensor& dest,
         resizable_tensor& means,
         resizable_tensor& invstds,
+        const double averaging_factor,
+        resizable_tensor& running_means,
+        resizable_tensor& running_invstds,
         const tensor& src,
         const tensor& gamma, 
         const tensor& beta 
@@ -285,6 +338,10 @@ namespace dlib { namespace tt
             - gamma.num_samples()==gamma.nr()==gamma.nc() == 1
             - beta.num_samples() ==beta.nr() ==gamma.nc() == 1
             - gamma.k()  == beta.k()  == src.k()
+            - 0 <= averaging_factor <= 1
+            - if (averaging_factor != 1)
+                - have_same_dimensions(running_means, means) == true
+                - have_same_dimensions(running_invstds, invstds) == true
         ensures
             - have_same_dimensions(#dest, src) == true
             - #means.num_samples()==means.nr()==means.nc() == 1
@@ -293,12 +350,11 @@ namespace dlib { namespace tt
             - #src == the batch normalized version of src.
             - #means == the mean values of the contents of src.
             - #invstds == 1/(the standard deviation values of the contents of src).
+            - #running_means = (1-averaging_factor)*mat(#running_means) + averaging_factor*mat(#means);
+            - #running_invstds = (1-averaging_factor)*mat(#running_invstds) + averaging_factor*mat(#invstds);
     !*/
 
-    class batch_normalize_conv_gradient
-    {
-    public:
-        void operator() (
+    void batch_normalize_conv_gradient (
             const tensor& gradient_input,
             const tensor& means,
             const tensor& invstds,
@@ -307,7 +363,7 @@ namespace dlib { namespace tt
             tensor& src_grad,
             tensor& gamma_grad, 
             tensor& beta_grad 
-        ){impl(gradient_input,means,invstds,src,gamma,src_grad,gamma_grad,beta_grad);}
+    );
     /*!
         requires
             - invstds and means should be the output of a call to
@@ -328,13 +384,6 @@ namespace dlib { namespace tt
             - Assigns the gradient of f() with respect to gamma to #gamma_grad.
             - Assigns the gradient of f() with respect to beta to #beta_grad.
     !*/
-    private:
-#ifdef DLIB_USE_CUDA
-        cuda::batch_normalize_conv_gradient impl;
-#else
-        cpu::batch_normalize_conv_gradient impl;
-#endif
-    };
 
 // -----------------------------------------------------------------------------------
 
@@ -540,14 +589,14 @@ namespace dlib { namespace tt
             ensures
                 - #dest.num_samples() == src.num_samples()
                 - #dest.k() == src.k()
-                - #dest.nr() == src.nr()/stride_y
-                - #dest.nc() == src.nc()/stride_x
+                - #dest.nr() == 1+(src.nr()-window_height%2)/stride_y
+                - #dest.nc() == 1+(src.nc()-window_width%2)/stride_x
                 - for all valid s, k, r, and c:
                     - image_plane(#dest,s,k)(r,c) == max(subm_clipped(image_plane(src,s,k),
+                                                                      centered_rect(c*stride_x,
                                                                                     r*stride_y,
-                                                                        c*stride_x,
-                                                                        window_height,
-                                                                        window_width))
+                                                                                    window_width,
+                                                                                    window_height)))
         !*/
 
         void get_gradient(

dlib/test/CMakeLists.txt

@@ -14,139 +14,6 @@ include(../cmake)
 # This variable contains a list of all the tests we are building
 # into the regression test suite.
 set (tests
-   example.cpp
-   active_learning.cpp
-   any.cpp
-   any_function.cpp
-   array2d.cpp
-   array.cpp
-   assignment_learning.cpp
-   base64.cpp
-   bayes_nets.cpp
-   bigint.cpp
-   binary_search_tree_kernel_1a.cpp
-   binary_search_tree_kernel_2a.cpp
-   binary_search_tree_mm1.cpp
-   binary_search_tree_mm2.cpp
-   bridge.cpp
-   bsp.cpp
-   byte_orderer.cpp
-   cca.cpp
-   clustering.cpp
-   cmd_line_parser.cpp
-   cmd_line_parser_wchar_t.cpp
-   compress_stream.cpp
-   conditioning_class_c.cpp
-   conditioning_class.cpp
-   config_reader.cpp
-   crc32.cpp
-   create_iris_datafile.cpp
-   data_io.cpp
-   directed_graph.cpp
-   discriminant_pca.cpp
-   disjoint_subsets.cpp
-   ekm_and_lisf.cpp
-   empirical_kernel_map.cpp
-   entropy_coder.cpp
-   entropy_encoder_model.cpp
-   example_args.cpp
-   face.cpp
-   fft.cpp
-   fhog.cpp
-   filtering.cpp
-   find_max_factor_graph_nmplp.cpp
-   find_max_factor_graph_viterbi.cpp
-   geometry.cpp
-   graph.cpp
-   graph_cuts.cpp
-   graph_labeler.cpp
-   hash.cpp
-   hash_map.cpp
-   hash_set.cpp
-   hash_table.cpp
-   hog_image.cpp
-   image.cpp
-   iosockstream.cpp
-   is_same_object.cpp
-   kcentroid.cpp
-   kernel_matrix.cpp
-   kmeans.cpp
-   learning_to_track.cpp
-   least_squares.cpp
-   linear_manifold_regularizer.cpp
-   lspi.cpp
-   lz77_buffer.cpp
-   map.cpp
-   matrix2.cpp
-   matrix3.cpp
-   matrix4.cpp
-   matrix_chol.cpp
-   matrix.cpp
-   matrix_eig.cpp
-   matrix_lu.cpp
-   matrix_qr.cpp
-   max_cost_assignment.cpp
-   max_sum_submatrix.cpp
-   md5.cpp
-   member_function_pointer.cpp
-   metaprogramming.cpp
-   mpc.cpp
-   multithreaded_object.cpp
-   numerical_integration.cpp
-   object_detector.cpp
-   oca.cpp
-   one_vs_all_trainer.cpp
-   one_vs_one_trainer.cpp
-   optimization.cpp
-   optimization_test_functions.cpp
-   opt_qp_solver.cpp
-   parallel_for.cpp
-   parse.cpp
-   pipe.cpp
-   pixel.cpp
-   probabilistic.cpp
-   pyramid_down.cpp
-   queue.cpp
-   rand.cpp
-   ranking.cpp
-   read_write_mutex.cpp
-   reference_counter.cpp
-   rls.cpp
-   sammon.cpp
-   scan_image.cpp
-   sequence.cpp
-   sequence_labeler.cpp
-   sequence_segmenter.cpp
-   serialize.cpp
-   set.cpp
-   sldf.cpp
-   sliding_buffer.cpp
-   smart_pointers.cpp
-   sockets2.cpp
-   sockets.cpp
-   sockstreambuf.cpp
-   sparse_vector.cpp
-   stack.cpp
-   static_map.cpp
-   static_set.cpp
-   statistics.cpp
-   std_vector_c.cpp
-   string.cpp
-   svm_c_linear.cpp
-   svm_c_linear_dcd.cpp
-   svm.cpp
-   svm_multiclass_linear.cpp
-   svm_struct.cpp
-   svr_linear_trainer.cpp
-   symmetric_matrix_cache.cpp
-   thread_pool.cpp
-   threads.cpp
-   timer.cpp
-   tokenizer.cpp
-   trust_region.cpp
-   tuple.cpp
-   type_safe_union.cpp
-   vectorstream.cpp
    )
 
 # Tests that require C++11 support
@@ -171,9 +38,3 @@ endif()
 TARGET_LINK_LIBRARIES(${target_name} dlib )
 
 
-if (NOT DLIB_NO_GUI_SUPPORT)
-   add_subdirectory(gui)
-   add_subdirectory(examples)
-endif()
-
-