Skip to content

D
dlib
Commit 3d1c22b9 authored by Davis King's avatar Davis King
Browse files
Options

Code cleanup

parent 51ea50b3
Hide whitespace changes
Inline Side-by-side
Showing with 159 additions and 277 deletions
dlib/dnn/cpu_dlib.cpp
View file @ 3d1c22b9
......@@ -164,7 +164,7 @@ namespace dlib
}
}
void batch_normalize_gradient (
void batch_normalize_gradient::operator() (
const tensor& gradient_input,
const tensor& means,
const tensor& invstds,
......@@ -192,7 +192,6 @@ namespace dlib
const auto p_invstds = invstds.host();
const auto p_means = means.host();
resizable_tensor dvars, dmeans;
dvars.copy_size(invstds);
dmeans.copy_size(means);
dvars = 0;
......@@ -343,7 +342,7 @@ namespace dlib
}
}
void batch_normalize_conv_gradient (
void batch_normalize_conv_gradient::operator() (
const tensor& gradient_input,
const tensor& means,
const tensor& invstds,
......@@ -371,7 +370,6 @@ namespace dlib
const auto p_invstds = invstds.host();
const auto p_means = means.host();
resizable_tensor dvars, dmeans;
dvars.copy_size(invstds);
dmeans.copy_size(means);
dvars = 0;
......
dlib/dnn/cpu_dlib.h
View file @ 3d1c22b9
......@@ -49,16 +49,22 @@ namespace dlib
const tensor& beta
);
void batch_normalize_gradient (
const tensor& gradient_input,
const tensor& means,
const tensor& invstds,
const tensor& src,
const tensor& gamma,
tensor& src_grad,
tensor& gamma_grad,
tensor& beta_grad
);
class batch_normalize_gradient
{
public:
void operator() (
const tensor& gradient_input,
const tensor& means,
const tensor& invstds,
const tensor& src,
const tensor& gamma,
tensor& src_grad,
tensor& gamma_grad,
tensor& beta_grad
);
private:
resizable_tensor dvars, dmeans;
};
void batch_normalize_conv (
resizable_tensor& dest,
......@@ -69,16 +75,22 @@ namespace dlib
const tensor& beta
);
void batch_normalize_conv_gradient (
const tensor& gradient_input,
const tensor& means,
const tensor& invstds,
const tensor& src,
const tensor& gamma,
tensor& src_grad,
tensor& gamma_grad,
tensor& beta_grad
);
class batch_normalize_conv_gradient
{
public:
void operator() (
const tensor& gradient_input,
const tensor& means,
const tensor& invstds,
const tensor& src,
const tensor& gamma,
tensor& src_grad,
tensor& gamma_grad,
tensor& beta_grad
);
private:
resizable_tensor dvars, dmeans;
};
// -----------------------------------------------------------------------------------
......
dlib/dnn/cuda_dlib.h
View file @ 3d1c22b9
......@@ -18,14 +18,6 @@ namespace dlib
tensor& dest,
const tensor& src
);
/*!
requires
- have_same_dimensions(dest,src) == true
ensures
- #dest == dest*src
That is, for all valid i:
#dest.host()[i] == dest.host()[i]*src.host()[i]
!*/
// -----------------------------------------------------------------------------------
......@@ -35,11 +27,6 @@ namespace dlib
const float A,
const float B
);
/*!
ensures
- have_same_dimensions(#dest,src) == true
- #dest == A*src + B
!*/
// -----------------------------------------------------------------------------------
......@@ -49,25 +36,6 @@ namespace dlib
const tensor& A,
const tensor& B
);
/*!
requires
- if (A.num_samples() == 1) then
- B.num_samples() == 1
- else
- A.num_samples() == src.num_samples()
- B.num_samples() == src.num_samples()
- A.nr() == B.nr() == src.nr()
- A.nc() == B.nc() == src.nc()
- A.k() == B.k() == src.k()
ensures
- have_same_dimensions(#dest,src) == true
- if (A.num_samples() == 1) then
- #dest == A*src + B
(done for each sample in src)
- else
- for all valid i:
- #dest.host()[i] == A.host()[i]*src.host()[i] + B.host()[i]
!*/
// -----------------------------------------------------------------------------------
......@@ -79,58 +47,23 @@ namespace dlib
const tensor& gamma,
const tensor& beta
);
/*!
requires
- src.num_samples() > 1
- gamma.num_samples() == 1
- beta.num_samples() == 1
- gamma.nr() == beta.nr() == src.nr()
- gamma.nc() == beta.nc() == src.nc()
- gamma.k() == beta.k() == src.k()
ensures
- have_same_dimensions(#dest, src) == true
- #means.num_samples() == 1
- #invstds.num_samples() == 1
- means.nr() == invstds.nr() == src.nr()
- means.nc() == invstds.nc() == src.nc()
- means.k() == invstds.k() == src.k()
- #src == the batch normalized version of src.
- #means == the mean values of the contents of src.
- #invstds == 1/(the standard deviation values of the contents of src).
!*/
void batch_normalize_gradient (
const tensor& gradient_input,
const tensor& means,
const tensor& invstds,
const tensor& src,
const tensor& gamma,
tensor& src_grad,
tensor& gamma_grad,
tensor& beta_grad
);
/*!
requires
- invstds and means should be the output of a call to
batch_normalize(dest,means,invstds,src,gamma,beta)
- have_same_dimensions(gradient_input, src) == true
- have_same_dimensions(src, src_grad) == true
- src.num_samples() > 1
- gamma.num_samples() == 1
- have_same_dimensions(gamma, gamma_grad) == true
- have_same_dimensions(gamma, beta_grad) == true
- gamma.nr() == src.nr()
- gamma.nc() == src.nc()
- gamma.k() == src.k()
- have_same_dimensions(means, gamma) == true
- have_same_dimensions(invstds, gamma) == true
ensures
- Let f(src,gamma,beta) == dot(gradient_input, dest output of
batch_normalize(dest,means,invstds,src,gamma,beta))
- Adds the gradient of f() with respect to src to #src_grad.
- Adds the gradient of f() with respect to gamma to #gamma_grad.
- Adds the gradient of f() with respect to beta to #beta_grad.
!*/
class batch_normalize_gradient
{
public:
void operator() (
const tensor& gradient_input,
const tensor& means,
const tensor& invstds,
const tensor& src,
const tensor& gamma,
tensor& src_grad,
tensor& gamma_grad,
tensor& beta_grad
);
private:
resizable_tensor dvars, dmeans;
};
void batch_normalize_conv (
resizable_tensor& dest,
......@@ -140,52 +73,23 @@ namespace dlib
const tensor& gamma,
const tensor& beta
);
/*!
requires
- src.num_samples() > 1
- gamma.num_samples()==gamma.nr()==gamma.nc() == 1
- beta.num_samples() ==beta.nr() ==gamma.nc() == 1
- gamma.k() == beta.k() == src.k()
ensures
- have_same_dimensions(#dest, src) == true
- #means.num_samples()==means.nr()==means.nc() == 1
- #invstds.num_samples() ==invstds.nr() ==invstds.nc() == 1
- means.k() == invstds.k() == src.k()
- #src == the batch normalized version of src.
- #means == the mean values of the contents of src.
- #invstds == 1/(the standard deviation values of the contents of src).
!*/
void batch_normalize_conv_gradient (
const tensor& gradient_input,
const tensor& means,
const tensor& invstds,
const tensor& src,
const tensor& gamma,
tensor& src_grad,
tensor& gamma_grad,
tensor& beta_grad
);
/*!
requires
- invstds and means should be the output of a call to
batch_normalize_conv(dest,means,invstds,src,gamma,beta)
- have_same_dimensions(gradient_input, src) == true
- have_same_dimensions(src, src_grad) == true
- src.num_samples() > 1
- gamma.num_samples()==gamma.nr()==gamma.nc() == 1
- have_same_dimensions(gamma, gamma_grad) == true
- have_same_dimensions(gamma, beta_grad) == true
- gamma.k() == src.k()
- have_same_dimensions(means, gamma) == true
- have_same_dimensions(invstds, gamma) == true
ensures
- Let f(src,gamma,beta) == dot(gradient_input, dest output of
batch_normalize_conv(dest,means,invstds,src,gamma,beta))
- Adds the gradient of f() with respect to src to #src_grad.
- Adds the gradient of f() with respect to gamma to #gamma_grad.
- Adds the gradient of f() with respect to beta to #beta_grad.
!*/
class batch_normalize_conv_gradient
{
public:
void operator() (
const tensor& gradient_input,
const tensor& means,
const tensor& invstds,
const tensor& src,
const tensor& gamma,
tensor& src_grad,
tensor& gamma_grad,
tensor& beta_grad
);
private:
resizable_tensor dvars, dmeans;
};
// -----------------------------------------------------------------------------------
......@@ -193,12 +97,6 @@ namespace dlib
tensor& data,
float thresh
);
/*!
ensures
- Sets all elements of data to 1 or 0 depending on if they are above or
below the given threshold. Specifically, for all valid i:
- #data.host()[i] == data.host()[i]>thresh ? 1 : 0
!*/
// ------------------------------------------------------------------------------------
......
dlib/dnn/tensor_tools.cpp
View file @ 3d1c22b9
......@@ -4,8 +4,6 @@
#define DLIB_TeNSOR_TOOLS_CPP_
#include "tensor_tools.h"
#include "cpu_dlib.h"
#include "cuda_dlib.h"
namespace dlib { namespace tt
{
......@@ -143,30 +141,6 @@ namespace dlib { namespace tt
#endif
}
// ----------------------------------------------------------------------------------------
void batch_normalize_gradient (
const tensor& gradient_input,
const tensor& means,
const tensor& vars,
const tensor& src,
const tensor& gamma,
tensor& src_grad,
tensor& gamma_grad,
tensor& beta_grad
)
{
#ifdef DLIB_USE_CUDA
/*
cuda::batch_normalize_gradient(gradient_input,means,vars,src,gamma,
src_grad,gamma_grad,beta_grad);
*/
#else
cpu::batch_normalize_gradient(gradient_input,means,vars,src,gamma,
src_grad,gamma_grad,beta_grad);
#endif
}
// ----------------------------------------------------------------------------------------
void batch_normalize_conv (
......@@ -185,30 +159,6 @@ namespace dlib { namespace tt
#endif
}
// ----------------------------------------------------------------------------------------
void batch_normalize_conv_gradient (
const tensor& gradient_input,
const tensor& means,
const tensor& vars,
const tensor& src,
const tensor& gamma,
tensor& src_grad,
tensor& gamma_grad,
tensor& beta_grad
)
{
#ifdef DLIB_USE_CUDA
/*
cuda::batch_normalize_conv_gradient(gradient_input,means,vars,src,gamma,
src_grad,gamma_grad,beta_grad);
*/
#else
cpu::batch_normalize_conv_gradient(gradient_input,means,vars,src,gamma,
src_grad,gamma_grad,beta_grad);
#endif
}
// ----------------------------------------------------------------------------------------
void threshold (
......
dlib/dnn/tensor_tools.h
View file @ 3d1c22b9
......@@ -7,6 +7,8 @@
#include "cudnn_dlibapi.h"
#include "cublas_dlibapi.h"
#include "curand_dlibapi.h"
#include "cpu_dlib.h"
#include "cuda_dlib.h"
#include "../rand.h"
namespace dlib { namespace tt
......@@ -175,40 +177,50 @@ namespace dlib { namespace tt
- #invstds == 1/(the standard deviation values of the contents of src).
!*/
// ----------------------------------------------------------------------------------------
class batch_normalize_gradient
{
public:
void operator() (
const tensor& gradient_input,
const tensor& means,
const tensor& invstds,
const tensor& src,
const tensor& gamma,
tensor& src_grad,
tensor& gamma_grad,
tensor& beta_grad
){impl(gradient_input,means,invstds,src,gamma,src_grad,gamma_grad,beta_grad);}
/*!
requires
- invstds and means should be the output of a call to
batch_normalize(dest,means,invstds,src,gamma,beta)
- have_same_dimensions(gradient_input, src) == true
- have_same_dimensions(src, src_grad) == true
- src.num_samples() > 1
- gamma.num_samples() == 1
- have_same_dimensions(gamma, gamma_grad) == true
- have_same_dimensions(gamma, beta_grad) == true
- gamma.nr() == src.nr()
- gamma.nc() == src.nc()
- gamma.k() == src.k()
- have_same_dimensions(means, gamma) == true
- have_same_dimensions(invstds, gamma) == true
ensures
- Let f(src,gamma,beta) == dot(gradient_input, dest output of
batch_normalize(dest,means,invstds,src,gamma,beta))
- Adds the gradient of f() with respect to src to #src_grad.
- Adds the gradient of f() with respect to gamma to #gamma_grad.
- Adds the gradient of f() with respect to beta to #beta_grad.
!*/
private:
#ifdef DLIB_USE_CUDA
cuda::batch_normalize_conv_gradient impl;
#else
cpu::batch_normalize_conv_gradient impl;
#endif
};
void batch_normalize_gradient (
const tensor& gradient_input,
const tensor& means,
const tensor& invstds,
const tensor& src,
const tensor& gamma,
tensor& src_grad,
tensor& gamma_grad,
tensor& beta_grad
);
/*!
requires
- invstds and means should be the output of a call to
batch_normalize(dest,means,invstds,src,gamma,beta)
- have_same_dimensions(gradient_input, src) == true
- have_same_dimensions(src, src_grad) == true
- src.num_samples() > 1
- gamma.num_samples() == 1
- have_same_dimensions(gamma, gamma_grad) == true
- have_same_dimensions(gamma, beta_grad) == true
- gamma.nr() == src.nr()
- gamma.nc() == src.nc()
- gamma.k() == src.k()
- have_same_dimensions(means, gamma) == true
- have_same_dimensions(invstds, gamma) == true
ensures
- Let f(src,gamma,beta) == dot(gradient_input, dest output of
batch_normalize(dest,means,invstds,src,gamma,beta))
- Adds the gradient of f() with respect to src to #src_grad.
- Adds the gradient of f() with respect to gamma to #gamma_grad.
- Adds the gradient of f() with respect to beta to #beta_grad.
!*/
// ----------------------------------------------------------------------------------------
void batch_normalize_conv (
resizable_tensor& dest,
......@@ -234,38 +246,48 @@ namespace dlib { namespace tt
- #invstds == 1/(the standard deviation values of the contents of src).
!*/
void batch_normalize_conv_gradient (
const tensor& gradient_input,
const tensor& means,
const tensor& invstds,
const tensor& src,
const tensor& gamma,
tensor& src_grad,
tensor& gamma_grad,
tensor& beta_grad
);
/*!
requires
- invstds and means should be the output of a call to
batch_normalize_conv(dest,means,invstds,src,gamma,beta)
- have_same_dimensions(gradient_input, src) == true
- have_same_dimensions(src, src_grad) == true
- src.num_samples() > 1
- gamma.num_samples()==gamma.nr()==gamma.nc() == 1
- have_same_dimensions(gamma, gamma_grad) == true
- have_same_dimensions(gamma, beta_grad) == true
- gamma.k() == src.k()
- have_same_dimensions(means, gamma) == true
- have_same_dimensions(invstds, gamma) == true
ensures
- Let f(src,gamma,beta) == dot(gradient_input, dest output of
batch_normalize_conv(dest,means,invstds,src,gamma,beta))
- Adds the gradient of f() with respect to src to #src_grad.
- Adds the gradient of f() with respect to gamma to #gamma_grad.
- Adds the gradient of f() with respect to beta to #beta_grad.
!*/
class batch_normalize_conv_gradient
{
public:
void operator() (
const tensor& gradient_input,
const tensor& means,
const tensor& invstds,
const tensor& src,
const tensor& gamma,
tensor& src_grad,
tensor& gamma_grad,
tensor& beta_grad
){impl(gradient_input,means,invstds,src,gamma,src_grad,gamma_grad,beta_grad);}
/*!
requires
- invstds and means should be the output of a call to
batch_normalize_conv(dest,means,invstds,src,gamma,beta)
- have_same_dimensions(gradient_input, src) == true
- have_same_dimensions(src, src_grad) == true
- src.num_samples() > 1
- gamma.num_samples()==gamma.nr()==gamma.nc() == 1
- have_same_dimensions(gamma, gamma_grad) == true
- have_same_dimensions(gamma, beta_grad) == true
- gamma.k() == src.k()
- have_same_dimensions(means, gamma) == true
- have_same_dimensions(invstds, gamma) == true
ensures
- Let f(src,gamma,beta) == dot(gradient_input, dest output of
batch_normalize_conv(dest,means,invstds,src,gamma,beta))
- Adds the gradient of f() with respect to src to #src_grad.
- Adds the gradient of f() with respect to gamma to #gamma_grad.
- Adds the gradient of f() with respect to beta to #beta_grad.
!*/
private:
#ifdef DLIB_USE_CUDA
cuda::batch_normalize_conv_gradient impl;
#else
cpu::batch_normalize_conv_gradient impl;
#endif
};
// -----------------------------------------------------------------------------------
// -----------------------------------------------------------------------------------
void threshold (
tensor& data,
......
dlib/test/dnn.cpp
View file @ 3d1c22b9
......@@ -17,7 +17,7 @@ namespace
using namespace test;
using namespace dlib;
using namespace dlib::cpu;
using namespace dlib::tt;
using namespace std;
logger dlog("test.dnn");
......@@ -99,7 +99,8 @@ namespace
gamma_grad = 0;
beta_grad = 0;
batch_normalize_gradient(gradient_input, means, vars, src, gamma, src_grad, gamma_grad, beta_grad);
batch_normalize_gradient bng;
bng(gradient_input, means, vars, src, gamma, src_grad, gamma_grad, beta_grad);
auto grad_error = compare_gradients(src_grad, grad_src);
dlog << LINFO << "src error: " << grad_error;
......@@ -175,7 +176,8 @@ namespace
gamma_grad = 0;
beta_grad = 0;
batch_normalize_conv_gradient(gradient_input, means, vars, src, gamma, src_grad, gamma_grad, beta_grad);
batch_normalize_conv_gradient bng;
bng(gradient_input, means, vars, src, gamma, src_grad, gamma_grad, beta_grad);
auto grad_error = compare_gradients(src_grad, grad_src);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment