Skip to content

D
dlib
Commit 3d1c22b9 authored by Davis King's avatar Davis King
Browse files
Options

Code cleanup

parent 51ea50b3
Hide whitespace changes
Inline Side-by-side
Showing with 159 additions and 277 deletions
dlib/dnn/cpu_dlib.cpp
View file @ 3d1c22b9
...@@ -164,7 +164,7 @@ namespace dlib ...@@ -164,7 +164,7 @@ namespace dlib
} }
} }
void batch_normalize_gradient ( void batch_normalize_gradient::operator() (
const tensor& gradient_input, const tensor& gradient_input,
const tensor& means, const tensor& means,
const tensor& invstds, const tensor& invstds,
...@@ -192,7 +192,6 @@ namespace dlib ...@@ -192,7 +192,6 @@ namespace dlib
const auto p_invstds = invstds.host(); const auto p_invstds = invstds.host();
const auto p_means = means.host(); const auto p_means = means.host();
resizable_tensor dvars, dmeans;
dvars.copy_size(invstds); dvars.copy_size(invstds);
dmeans.copy_size(means); dmeans.copy_size(means);
dvars = 0; dvars = 0;
...@@ -343,7 +342,7 @@ namespace dlib ...@@ -343,7 +342,7 @@ namespace dlib
} }
} }
void batch_normalize_conv_gradient ( void batch_normalize_conv_gradient::operator() (
const tensor& gradient_input, const tensor& gradient_input,
const tensor& means, const tensor& means,
const tensor& invstds, const tensor& invstds,
...@@ -371,7 +370,6 @@ namespace dlib ...@@ -371,7 +370,6 @@ namespace dlib
const auto p_invstds = invstds.host(); const auto p_invstds = invstds.host();
const auto p_means = means.host(); const auto p_means = means.host();
resizable_tensor dvars, dmeans;
dvars.copy_size(invstds); dvars.copy_size(invstds);
dmeans.copy_size(means); dmeans.copy_size(means);
dvars = 0; dvars = 0;
......
dlib/dnn/cpu_dlib.h
View file @ 3d1c22b9
...@@ -49,16 +49,22 @@ namespace dlib ...@@ -49,16 +49,22 @@ namespace dlib
const tensor& beta const tensor& beta
); );
void batch_normalize_gradient ( class batch_normalize_gradient
const tensor& gradient_input, {
const tensor& means, public:
const tensor& invstds, void operator() (
const tensor& src, const tensor& gradient_input,
const tensor& gamma, const tensor& means,
tensor& src_grad, const tensor& invstds,
tensor& gamma_grad, const tensor& src,
tensor& beta_grad const tensor& gamma,
); tensor& src_grad,
tensor& gamma_grad,
tensor& beta_grad
);
private:
resizable_tensor dvars, dmeans;
};
void batch_normalize_conv ( void batch_normalize_conv (
resizable_tensor& dest, resizable_tensor& dest,
...@@ -69,16 +75,22 @@ namespace dlib ...@@ -69,16 +75,22 @@ namespace dlib
const tensor& beta const tensor& beta
); );
void batch_normalize_conv_gradient ( class batch_normalize_conv_gradient
const tensor& gradient_input, {
const tensor& means, public:
const tensor& invstds, void operator() (
const tensor& src, const tensor& gradient_input,
const tensor& gamma, const tensor& means,
tensor& src_grad, const tensor& invstds,
tensor& gamma_grad, const tensor& src,
tensor& beta_grad const tensor& gamma,
); tensor& src_grad,
tensor& gamma_grad,
tensor& beta_grad
);
private:
resizable_tensor dvars, dmeans;
};
// ----------------------------------------------------------------------------------- // -----------------------------------------------------------------------------------
......
dlib/dnn/cuda_dlib.h
View file @ 3d1c22b9
...@@ -18,14 +18,6 @@ namespace dlib ...@@ -18,14 +18,6 @@ namespace dlib
tensor& dest, tensor& dest,
const tensor& src const tensor& src
); );
/*!
requires
- have_same_dimensions(dest,src) == true
ensures
- #dest == dest*src
That is, for all valid i:
#dest.host()[i] == dest.host()[i]*src.host()[i]
!*/
// ----------------------------------------------------------------------------------- // -----------------------------------------------------------------------------------
...@@ -35,11 +27,6 @@ namespace dlib ...@@ -35,11 +27,6 @@ namespace dlib
const float A, const float A,
const float B const float B
); );
/*!
ensures
- have_same_dimensions(#dest,src) == true
- #dest == A*src + B
!*/
// ----------------------------------------------------------------------------------- // -----------------------------------------------------------------------------------
...@@ -49,25 +36,6 @@ namespace dlib ...@@ -49,25 +36,6 @@ namespace dlib
const tensor& A, const tensor& A,
const tensor& B const tensor& B
); );
/*!
requires
- if (A.num_samples() == 1) then
- B.num_samples() == 1
- else
- A.num_samples() == src.num_samples()
- B.num_samples() == src.num_samples()
- A.nr() == B.nr() == src.nr()
- A.nc() == B.nc() == src.nc()
- A.k() == B.k() == src.k()
ensures
- have_same_dimensions(#dest,src) == true
- if (A.num_samples() == 1) then
- #dest == A*src + B
(done for each sample in src)
- else
- for all valid i:
- #dest.host()[i] == A.host()[i]*src.host()[i] + B.host()[i]
!*/
// ----------------------------------------------------------------------------------- // -----------------------------------------------------------------------------------
...@@ -79,58 +47,23 @@ namespace dlib ...@@ -79,58 +47,23 @@ namespace dlib
const tensor& gamma, const tensor& gamma,
const tensor& beta const tensor& beta
); );
/*!
requires class batch_normalize_gradient
- src.num_samples() > 1 {
- gamma.num_samples() == 1 public:
- beta.num_samples() == 1 void operator() (
- gamma.nr() == beta.nr() == src.nr() const tensor& gradient_input,
- gamma.nc() == beta.nc() == src.nc() const tensor& means,
- gamma.k() == beta.k() == src.k() const tensor& invstds,
ensures const tensor& src,
- have_same_dimensions(#dest, src) == true const tensor& gamma,
- #means.num_samples() == 1 tensor& src_grad,
- #invstds.num_samples() == 1 tensor& gamma_grad,
- means.nr() == invstds.nr() == src.nr() tensor& beta_grad
- means.nc() == invstds.nc() == src.nc() );
- means.k() == invstds.k() == src.k() private:
- #src == the batch normalized version of src. resizable_tensor dvars, dmeans;
- #means == the mean values of the contents of src. };
- #invstds == 1/(the standard deviation values of the contents of src).
!*/
void batch_normalize_gradient (
const tensor& gradient_input,
const tensor& means,
const tensor& invstds,
const tensor& src,
const tensor& gamma,
tensor& src_grad,
tensor& gamma_grad,
tensor& beta_grad
);
/*!
requires
- invstds and means should be the output of a call to
batch_normalize(dest,means,invstds,src,gamma,beta)
- have_same_dimensions(gradient_input, src) == true
- have_same_dimensions(src, src_grad) == true
- src.num_samples() > 1
- gamma.num_samples() == 1
- have_same_dimensions(gamma, gamma_grad) == true
- have_same_dimensions(gamma, beta_grad) == true
- gamma.nr() == src.nr()
- gamma.nc() == src.nc()
- gamma.k() == src.k()
- have_same_dimensions(means, gamma) == true
- have_same_dimensions(invstds, gamma) == true
ensures
- Let f(src,gamma,beta) == dot(gradient_input, dest output of
batch_normalize(dest,means,invstds,src,gamma,beta))
- Adds the gradient of f() with respect to src to #src_grad.
- Adds the gradient of f() with respect to gamma to #gamma_grad.
- Adds the gradient of f() with respect to beta to #beta_grad.
!*/
void batch_normalize_conv ( void batch_normalize_conv (
resizable_tensor& dest, resizable_tensor& dest,
...@@ -140,52 +73,23 @@ namespace dlib ...@@ -140,52 +73,23 @@ namespace dlib
const tensor& gamma, const tensor& gamma,
const tensor& beta const tensor& beta
); );
/*!
requires class batch_normalize_conv_gradient
- src.num_samples() > 1 {
- gamma.num_samples()==gamma.nr()==gamma.nc() == 1 public:
- beta.num_samples() ==beta.nr() ==gamma.nc() == 1 void operator() (
- gamma.k() == beta.k() == src.k() const tensor& gradient_input,
ensures const tensor& means,
- have_same_dimensions(#dest, src) == true const tensor& invstds,
- #means.num_samples()==means.nr()==means.nc() == 1 const tensor& src,
- #invstds.num_samples() ==invstds.nr() ==invstds.nc() == 1 const tensor& gamma,
- means.k() == invstds.k() == src.k() tensor& src_grad,
- #src == the batch normalized version of src. tensor& gamma_grad,
- #means == the mean values of the contents of src. tensor& beta_grad
- #invstds == 1/(the standard deviation values of the contents of src). );
!*/ private:
resizable_tensor dvars, dmeans;
void batch_normalize_conv_gradient ( };
const tensor& gradient_input,
const tensor& means,
const tensor& invstds,
const tensor& src,
const tensor& gamma,
tensor& src_grad,
tensor& gamma_grad,
tensor& beta_grad
);
/*!
requires
- invstds and means should be the output of a call to
batch_normalize_conv(dest,means,invstds,src,gamma,beta)
- have_same_dimensions(gradient_input, src) == true
- have_same_dimensions(src, src_grad) == true
- src.num_samples() > 1
- gamma.num_samples()==gamma.nr()==gamma.nc() == 1
- have_same_dimensions(gamma, gamma_grad) == true
- have_same_dimensions(gamma, beta_grad) == true
- gamma.k() == src.k()
- have_same_dimensions(means, gamma) == true
- have_same_dimensions(invstds, gamma) == true
ensures
- Let f(src,gamma,beta) == dot(gradient_input, dest output of
batch_normalize_conv(dest,means,invstds,src,gamma,beta))
- Adds the gradient of f() with respect to src to #src_grad.
- Adds the gradient of f() with respect to gamma to #gamma_grad.
- Adds the gradient of f() with respect to beta to #beta_grad.
!*/
// ----------------------------------------------------------------------------------- // -----------------------------------------------------------------------------------
...@@ -193,12 +97,6 @@ namespace dlib ...@@ -193,12 +97,6 @@ namespace dlib
tensor& data, tensor& data,
float thresh float thresh
); );
/*!
ensures
- Sets all elements of data to 1 or 0 depending on if they are above or
below the given threshold. Specifically, for all valid i:
- #data.host()[i] == data.host()[i]>thresh ? 1 : 0
!*/
// ------------------------------------------------------------------------------------ // ------------------------------------------------------------------------------------
......
dlib/dnn/tensor_tools.cpp
View file @ 3d1c22b9
...@@ -4,8 +4,6 @@ ...@@ -4,8 +4,6 @@
#define DLIB_TeNSOR_TOOLS_CPP_ #define DLIB_TeNSOR_TOOLS_CPP_
#include "tensor_tools.h" #include "tensor_tools.h"
#include "cpu_dlib.h"
#include "cuda_dlib.h"
namespace dlib { namespace tt namespace dlib { namespace tt
{ {
...@@ -143,30 +141,6 @@ namespace dlib { namespace tt ...@@ -143,30 +141,6 @@ namespace dlib { namespace tt
#endif #endif
} }
// ----------------------------------------------------------------------------------------
void batch_normalize_gradient (
const tensor& gradient_input,
const tensor& means,
const tensor& vars,
const tensor& src,
const tensor& gamma,
tensor& src_grad,
tensor& gamma_grad,
tensor& beta_grad
)
{
#ifdef DLIB_USE_CUDA
/*
cuda::batch_normalize_gradient(gradient_input,means,vars,src,gamma,
src_grad,gamma_grad,beta_grad);
*/
#else
cpu::batch_normalize_gradient(gradient_input,means,vars,src,gamma,
src_grad,gamma_grad,beta_grad);
#endif
}
// ---------------------------------------------------------------------------------------- // ----------------------------------------------------------------------------------------
void batch_normalize_conv ( void batch_normalize_conv (
...@@ -185,30 +159,6 @@ namespace dlib { namespace tt ...@@ -185,30 +159,6 @@ namespace dlib { namespace tt
#endif #endif
} }
// ----------------------------------------------------------------------------------------
void batch_normalize_conv_gradient (
const tensor& gradient_input,
const tensor& means,
const tensor& vars,
const tensor& src,
const tensor& gamma,
tensor& src_grad,
tensor& gamma_grad,
tensor& beta_grad
)
{
#ifdef DLIB_USE_CUDA
/*
cuda::batch_normalize_conv_gradient(gradient_input,means,vars,src,gamma,
src_grad,gamma_grad,beta_grad);
*/
#else
cpu::batch_normalize_conv_gradient(gradient_input,means,vars,src,gamma,
src_grad,gamma_grad,beta_grad);
#endif
}
// ---------------------------------------------------------------------------------------- // ----------------------------------------------------------------------------------------
void threshold ( void threshold (
......
dlib/dnn/tensor_tools.h
View file @ 3d1c22b9
...@@ -7,6 +7,8 @@ ...@@ -7,6 +7,8 @@
#include "cudnn_dlibapi.h" #include "cudnn_dlibapi.h"
#include "cublas_dlibapi.h" #include "cublas_dlibapi.h"
#include "curand_dlibapi.h" #include "curand_dlibapi.h"
#include "cpu_dlib.h"
#include "cuda_dlib.h"
#include "../rand.h" #include "../rand.h"
namespace dlib { namespace tt namespace dlib { namespace tt
...@@ -175,40 +177,50 @@ namespace dlib { namespace tt ...@@ -175,40 +177,50 @@ namespace dlib { namespace tt
- #invstds == 1/(the standard deviation values of the contents of src). - #invstds == 1/(the standard deviation values of the contents of src).
!*/ !*/
// ---------------------------------------------------------------------------------------- class batch_normalize_gradient
{
public:
void operator() (
const tensor& gradient_input,
const tensor& means,
const tensor& invstds,
const tensor& src,
const tensor& gamma,
tensor& src_grad,
tensor& gamma_grad,
tensor& beta_grad
){impl(gradient_input,means,invstds,src,gamma,src_grad,gamma_grad,beta_grad);}
/*!
requires
- invstds and means should be the output of a call to
batch_normalize(dest,means,invstds,src,gamma,beta)
- have_same_dimensions(gradient_input, src) == true
- have_same_dimensions(src, src_grad) == true
- src.num_samples() > 1
- gamma.num_samples() == 1
- have_same_dimensions(gamma, gamma_grad) == true
- have_same_dimensions(gamma, beta_grad) == true
- gamma.nr() == src.nr()
- gamma.nc() == src.nc()
- gamma.k() == src.k()
- have_same_dimensions(means, gamma) == true
- have_same_dimensions(invstds, gamma) == true
ensures
- Let f(src,gamma,beta) == dot(gradient_input, dest output of
batch_normalize(dest,means,invstds,src,gamma,beta))
- Adds the gradient of f() with respect to src to #src_grad.
- Adds the gradient of f() with respect to gamma to #gamma_grad.
- Adds the gradient of f() with respect to beta to #beta_grad.
!*/
private:
#ifdef DLIB_USE_CUDA
cuda::batch_normalize_conv_gradient impl;
#else
cpu::batch_normalize_conv_gradient impl;
#endif
};
void batch_normalize_gradient ( // ----------------------------------------------------------------------------------------
const tensor& gradient_input,
const tensor& means,
const tensor& invstds,
const tensor& src,
const tensor& gamma,
tensor& src_grad,
tensor& gamma_grad,
tensor& beta_grad
);
/*!
requires
- invstds and means should be the output of a call to
batch_normalize(dest,means,invstds,src,gamma,beta)
- have_same_dimensions(gradient_input, src) == true
- have_same_dimensions(src, src_grad) == true
- src.num_samples() > 1
- gamma.num_samples() == 1
- have_same_dimensions(gamma, gamma_grad) == true
- have_same_dimensions(gamma, beta_grad) == true
- gamma.nr() == src.nr()
- gamma.nc() == src.nc()
- gamma.k() == src.k()
- have_same_dimensions(means, gamma) == true
- have_same_dimensions(invstds, gamma) == true
ensures
- Let f(src,gamma,beta) == dot(gradient_input, dest output of
batch_normalize(dest,means,invstds,src,gamma,beta))
- Adds the gradient of f() with respect to src to #src_grad.
- Adds the gradient of f() with respect to gamma to #gamma_grad.
- Adds the gradient of f() with respect to beta to #beta_grad.
!*/
void batch_normalize_conv ( void batch_normalize_conv (
resizable_tensor& dest, resizable_tensor& dest,
...@@ -234,38 +246,48 @@ namespace dlib { namespace tt ...@@ -234,38 +246,48 @@ namespace dlib { namespace tt
- #invstds == 1/(the standard deviation values of the contents of src). - #invstds == 1/(the standard deviation values of the contents of src).
!*/ !*/
void batch_normalize_conv_gradient ( class batch_normalize_conv_gradient
const tensor& gradient_input, {
const tensor& means, public:
const tensor& invstds, void operator() (
const tensor& src, const tensor& gradient_input,
const tensor& gamma, const tensor& means,
tensor& src_grad, const tensor& invstds,
tensor& gamma_grad, const tensor& src,
tensor& beta_grad const tensor& gamma,
); tensor& src_grad,
/*! tensor& gamma_grad,
requires tensor& beta_grad
- invstds and means should be the output of a call to ){impl(gradient_input,means,invstds,src,gamma,src_grad,gamma_grad,beta_grad);}
batch_normalize_conv(dest,means,invstds,src,gamma,beta) /*!
- have_same_dimensions(gradient_input, src) == true requires
- have_same_dimensions(src, src_grad) == true - invstds and means should be the output of a call to
- src.num_samples() > 1 batch_normalize_conv(dest,means,invstds,src,gamma,beta)
- gamma.num_samples()==gamma.nr()==gamma.nc() == 1 - have_same_dimensions(gradient_input, src) == true
- have_same_dimensions(gamma, gamma_grad) == true - have_same_dimensions(src, src_grad) == true
- have_same_dimensions(gamma, beta_grad) == true - src.num_samples() > 1
- gamma.k() == src.k() - gamma.num_samples()==gamma.nr()==gamma.nc() == 1
- have_same_dimensions(means, gamma) == true - have_same_dimensions(gamma, gamma_grad) == true
- have_same_dimensions(invstds, gamma) == true - have_same_dimensions(gamma, beta_grad) == true
ensures - gamma.k() == src.k()
- Let f(src,gamma,beta) == dot(gradient_input, dest output of - have_same_dimensions(means, gamma) == true
batch_normalize_conv(dest,means,invstds,src,gamma,beta)) - have_same_dimensions(invstds, gamma) == true
- Adds the gradient of f() with respect to src to #src_grad. ensures
- Adds the gradient of f() with respect to gamma to #gamma_grad. - Let f(src,gamma,beta) == dot(gradient_input, dest output of
- Adds the gradient of f() with respect to beta to #beta_grad. batch_normalize_conv(dest,means,invstds,src,gamma,beta))
!*/ - Adds the gradient of f() with respect to src to #src_grad.
- Adds the gradient of f() with respect to gamma to #gamma_grad.
- Adds the gradient of f() with respect to beta to #beta_grad.
!*/
private:
#ifdef DLIB_USE_CUDA
cuda::batch_normalize_conv_gradient impl;
#else
cpu::batch_normalize_conv_gradient impl;
#endif
};
// ----------------------------------------------------------------------------------- // -----------------------------------------------------------------------------------
void threshold ( void threshold (
tensor& data, tensor& data,
......
dlib/test/dnn.cpp
View file @ 3d1c22b9
...@@ -17,7 +17,7 @@ namespace ...@@ -17,7 +17,7 @@ namespace
using namespace test; using namespace test;
using namespace dlib; using namespace dlib;
using namespace dlib::cpu; using namespace dlib::tt;
using namespace std; using namespace std;
logger dlog("test.dnn"); logger dlog("test.dnn");
...@@ -99,7 +99,8 @@ namespace ...@@ -99,7 +99,8 @@ namespace
gamma_grad = 0; gamma_grad = 0;
beta_grad = 0; beta_grad = 0;
batch_normalize_gradient(gradient_input, means, vars, src, gamma, src_grad, gamma_grad, beta_grad); batch_normalize_gradient bng;
bng(gradient_input, means, vars, src, gamma, src_grad, gamma_grad, beta_grad);
auto grad_error = compare_gradients(src_grad, grad_src); auto grad_error = compare_gradients(src_grad, grad_src);
dlog << LINFO << "src error: " << grad_error; dlog << LINFO << "src error: " << grad_error;
...@@ -175,7 +176,8 @@ namespace ...@@ -175,7 +176,8 @@ namespace
gamma_grad = 0; gamma_grad = 0;
beta_grad = 0; beta_grad = 0;
batch_normalize_conv_gradient(gradient_input, means, vars, src, gamma, src_grad, gamma_grad, beta_grad); batch_normalize_conv_gradient bng;
bng(gradient_input, means, vars, src, gamma, src_grad, gamma_grad, beta_grad);
auto grad_error = compare_gradients(src_grad, grad_src); auto grad_error = compare_gradients(src_grad, grad_src);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment