Skip to content

D
dlib
Commit 94979c79 authored by Davis King's avatar Davis King
Browse files
Options

Added outline for the CUDA portion of the DNN toolkit.

parent ca4f95ed
Hide whitespace changes
Inline Side-by-side
Showing with 604 additions and 0 deletions
dlib/dnn/cublas.h 0 → 100644
View file @ 94979c79
// Copyright (C) 2015 Davis E. King (davis@dlib.net)
// License: Boost Software License See LICENSE.txt for the full license.
#ifndef DLIB_DNN_CuBLAS_H_
#define DLIB_DNN_CuBLAS_H_
#ifdef DLIB_USE_CUDA
#include "tensor.h"
namespace dlib
{
namespace cuda
{
// -----------------------------------------------------------------------------------
struct cublas_error : public error
{
cublas_error(const std::string& message): error(message) {}
};
// -----------------------------------------------------------------------------------
class cublas_context
{
public:
cublas_context(const cublas_context&) = delete;
cublas_context& operator=(const cublas_context&) = delete;
cublas_context()
{
// TODO
}
~cublas_context()
{
// TODO
}
private:
void* handle;
};
// -----------------------------------------------------------------------------------
void gemm (
cublas_context& context,
float beta,
tensor& dest,
float alpha,
const tensor& lhs
bool trans_lhs,
const tensor& rhs,
bool trans_rhs
);
// ------------------------------------------------------------------------------------
}
}
#endif // DLIB_USE_CUDA
#endif // DLIB_DNN_CuBLAS_H_
dlib/dnn/cuda.h 0 → 100644
View file @ 94979c79
// Copyright (C) 2015 Davis E. King (davis@dlib.net)
// License: Boost Software License See LICENSE.txt for the full license.
#ifndef DLIB_DNN_CuDA_H_
#define DLIB_DNN_CuDA_H_
#ifdef DLIB_USE_CUDA
#include "tensor.h"
namespace dlib
{
namespace cuda
{
// -----------------------------------------------------------------------------------
void affine_transform(
resizable_tensor& dest,
const tensor& src,
const float A,
const float B
);
/*!
ensures
- have_same_dimensions(#dest,src) == true
- #dest == A*src + B
!*/
// -----------------------------------------------------------------------------------
void affine_transform(
resizable_tensor& dest,
const tensor& src,
const tensor& A,
const tensor& B
);
/*!
requires
- A.num_samples() == 1
- B.num_samples() == 1
- A.nr() == B.nr() == src.nr()
- A.nc() == B.nc() == src.nc()
- A.k() == B.k() == src.k()
ensures
- have_same_dimensions(#dest,src) == true
- #dest == A*src + B
(done for each sample in src)
!*/
// -----------------------------------------------------------------------------------
// TODO, add versions of batch_normalize() that output the gradients.
void batch_normalize (
resizable_tensor& dest,
resizable_tensor& means,
resizable_tensor& vars,
const tensor& src,
const tensor& gamma,
const tensor& beta
);
/*!
requires
- gamma.num_samples() == 1
- beta.num_samples() == 1
- gamma.nr() == beta.nr() == src.nr()
- gamma.nc() == beta.nc() == src.nc()
- gamma.k() == beta.k() == src.k()
ensures
- have_same_dimensions(#dest, src) == true
- #means.num_samples() == 1
- #vars.num_samples() == 1
- means.nr() == vars.nr() == src.nr()
- means.nc() == vars.nc() == src.nc()
- means.k() == vars.k() == src.k()
- #src == the batch normalized version of src.
- #means == the mean values of the contents of src.
- #vars == the variance values of the contents of src.
!*/
void batch_normalize_conv (
resizable_tensor& dest,
resizable_tensor& means,
resizable_tensor& vars,
const tensor& src,
const tensor& gamma,
const tensor& beta
);
/*!
requires
- gamma.num_samples()==gamma.nr()==gamma.nc() == 1
- beta.num_samples() ==beta.nr() ==gamma.nc() == 1
- gamma.k() == beta.k() == src.k()
ensures
- have_same_dimensions(#dest, src) == true
- #means.num_samples()==means.nr()==means.nc() == 1
- #vars.num_samples() ==vars.nr() ==vars.nc() == 1
- means.k() == vars.k() == src.k()
- #src == the batch normalized version of src.
- #means == the mean values of the contents of src.
- #vars == the variance values of the contents of src.
!*/
// -----------------------------------------------------------------------------------
class dropout
{
/*!
!*/
public:
dropout(const dropout&) = delete;
dropout& operator=(const dropout&) = delete;
dropout(float drop_rate = 0.5);
dropout(float drop_rate, int seed);
void operator() (
resizable_tensor& dest,
resizable_tensor& random_mask,
const tensor& src
);
/*!
ensures
- have_same_dimensions(src, #dest) == true
- have_same_dimensions(src, #random_mask) == true
!*/
void get_gradient(
const tensor& gradient_input,
const tensor& random_mask,
tensor& grad
);
/*!
requires
- have_same_dimensions(gradient_input, random_mask) == true
- have_same_dimensions(gradient_input, grad) == true
ensures
- let OUT and MASK be the output of (*this)(OUT,MASK,src)
- let f(src) == dot(gradient_input,OUT)
- Then this function computes the gradient of f() with respect to src
and adds it to grad.
!*/
};
// -----------------------------------------------------------------------------------
}
}
#endif // DLIB_USE_CUDA
#endif // DLIB_DNN_CuDA_H_
dlib/dnn/cudnn.h 0 → 100644
View file @ 94979c79
// Copyright (C) 2015 Davis E. King (davis@dlib.net)
// License: Boost Software License See LICENSE.txt for the full license.
#ifndef DLIB_DNN_CuDNN_H_
#define DLIB_DNN_CuDNN_H_
#ifdef DLIB_USE_CUDA
#include "tensor.h"
namespace dlib
{
namespace cuda
{
// -----------------------------------------------------------------------------------
struct cudnn_error : public error
{
cudnn_error(const std::string& message): error(message) {}
};
// ------------------------------------------------------------------------------------
class cudnn_context
{
public:
cudnn_context(const cudnn_context&) = delete;
cudnn_context& operator=(const cudnn_context&) = delete;
cudnn_context()
{
// TODO
// cudnnCreate()
}
~cudnn_context()
{
// TODO
// cudnnDestroy()
}
private:
void* handle;
};
// ------------------------------------------------------------------------------------
class tensor_descriptor
{
/*!
Each tensor object will carry a tensor_descriptor in it when compiled with
CUDA.
!*/
public:
tensor_descriptor(const tensor_descriptor&) = delete;
tensor_descriptor& operator=(const tensor_descriptor&) = delete;
tensor_descriptor()
{
//  cudnnCreateTensorDescriptor()
// Also call cudnnSetTensor4dDescriptor() somewhere. Probably in a set()
// method.
}
~tensor_descriptor()
{
// cudnnDestroyTensorDescriptor()
}
private:
void* handle;
};
// ------------------------------------------------------------------------------------
void add(
cudnn_context& context,
float beta,
tensor& dest,
float alpha,
const tensor& src
);
/*!
requires
- dest.num_samples()==src.num_samples() || src.num_samples()==1
- dest.nr()==src.nr() || src.nr()==1
- dest.nc()==src.nc() || src.nc()==1
- dest.k()==src.k() || src.k()==1
ensures
- performs: dest = beta*dest + alpha*src
TODO, clarify:
calls cudnnAddTensor_v3()
This function adds the scaled values of one bias tensor to another
tensor. Each dimension of the bias tensor must match the coresponding
dimension of the srcDest tensor or must be equal to 1. In the latter
case, the same value from the bias tensor for thoses dimensions will be
used to blend into the srcDest tensor.
!*/
void set_tensor (
cudnn_context& context,
tensor& t,
float value
);
/*!
ensures
- sets all elements in t equal to value.
Uses cudnnSetTensor().
!*/
void scale_tensor (
cudnn_context& context,
tensor& t,
float value
);
/*!
ensures
- scales all elements of t by the given value. I.e. for all elements E in
t, this function performs:
- E = E*value
uses cudnnScaleTensor()
!*/
// ------------------------------------------------------------------------------------
class conv
{
public:
conv(const conv&) = delete;
conv& operator=(const conv&) = delete;
conv(
cudnn_context& context,
const tensor& data,
const tensor& filters,
int stride_y,
int stride_x
);
void operator() (
resizable_tensor& output,
const tensor& data,
const tensor& filters
);
/*!
requires
- the dimensions of data and filters are the same as the ones given
to the constructor.
ensures
- convolves filters over data.
- filters contains filters.num_samples() filters.
- #output.num_samples() == data.num_samples()
- #output.k() == filters.num_samples()
- #output.nr() == 1+(data.nr()-1)/stride_y
- #output.nc() == 1+(data.nc()-1)/stride_x
!*/
// get gradient of data: 4.49. cudnnConvolutionBackwardData_v3
void get_gradient_for_data (
const tensor& gradient_input,
const tensor& filters,
tensor& data_gradient
);
/*!
requires
- filters has the same dimensions as the filters object give to the
constructor.
- data_gradient has the same dimensions as the data object give to the
constructor.
- gradient_input has the same dimensions as the output of operator().
ensures
- let OUT be the output of (*this)(OUT,data,filters).
- let f(data,filters) == dot(OUT, gradient_input)
- This function finds the gradient of f() with respect to data
and adds this gradient to data_gradient.
!*/
// get gradient of filters: 4.44. cudnnConvolutionBackwardFilter_v3
void get_gradient_for_filters (
const tensor& gradient_input,
const tensor& data,
tensor& filters_gradient
);
/*!
requires
- filters_gradient has the same dimensions as the filters object give
to the constructor.
- data has the same dimensions as the data object give to the constructor.
- gradient_input has the same dimensions as the output of operator().
ensures
- let OUT be the output of (*this)(OUT,data,filters).
- let f(data,filters) == dot(OUT, gradient_input)
- This function finds the gradient of f() with respect to filters
and adds this gradient to filters_gradient.
!*/
};
// ------------------------------------------------------------------------------------
void soft_max (
cudnn_context& context,
resizable_tensor& dest,
const tensor& src
);
/*!
probably uses CUDNN_SOFTMAX_MODE_CHANNEL
!*/
void soft_max_gradient (
cudnn_context& context,
tensor& grad,
const tensor& src,
const tensor& gradient_input
);
/*!
- let OUT be the output of soft_max(context,OUT,src)
- let f(src) == dot(gradient_input,OUT)
- Then this function computes the gradient of f() with respect to src
and adds it to grad.
!*/
// ------------------------------------------------------------------------------------
class max_pool
{
/*!
CUDNN_POOLING_MAX
!*/
public:
max_pool(const max_pool&) = delete;
max_pool& operator=(const max_pool&) = delete;
// cudnnCreatePoolingDescriptor(), cudnnSetPooling2dDescriptor()
max_pool (
cudnn_context& context,
int window_height,
int window_width,
int stride_y,
int stride_x
);
// cudnnDestroyPoolingDescriptor ()
~max_pool(
);
// cudnnGetPooling2dForwardOutputDim(), cudnnPoolingForward()
void operator() (
resizable_tensor& dest,
const tensor& src
);
/*!
!*/
// cudnnPoolingBackward()
void get_gradient(
const tensor& gradient_input,
const tensor& src,
tensor& grad
);
/*!
- let OUT be the output of (*this)(OUT,src)
- let f(src) == dot(gradient_input,OUT)
- Then this function computes the gradient of f() with respect to src and
adds it to grad.
!*/
};
// TODO, make the order of parameters of all these functions consistent.
// ------------------------------------------------------------------------------------
// cudnnActivationForward(), CUDNN_ACTIVATION_SIGMOID
void sigmoid (
cudnn_context& context,
resizable_tensor& dest,
const tensor& src
);
/*!
ensures
- have_same_dimensions(#dest, src) == true
- for all valid i:
- #dest.host()[i] == 1/(1+std::exp(-src.host()[i]))
!*/
// cudnnActivationBackward()
void sigmoid_gradient (
cudnn_context& context,
tensor& grad,
const tensor& src,
const tensor& gradient_input
);
/*!
requires
- have_same_dimensions(src,gradient_input) == true
- have_same_dimensions(src,grad) == true
ensures
- let OUT be the output of sigmoid(context,OUT,src)
- let f(src) == dot(gradient_input,OUT)
- Then this function computes the gradient of f() with respect to src and
adds it to grad.
!*/
// ------------------------------------------------------------------------------------
// cudnnActivationForward(), CUDNN_ACTIVATION_RELU
void relu (
cudnn_context& context,
resizable_tensor& dest,
const tensor& src
);
/*!
ensures
- have_same_dimensions(#dest, src) == true
- for all valid i:
- #dest.host()[i] == std::max(0,src.host()[i])
!*/
// cudnnActivationBackward()
void relu_gradient (
cudnn_context& context,
tensor& grad,
const tensor& src,
const tensor& gradient_input
);
/*!
requires
- have_same_dimensions(src,gradient_input) == true
- have_same_dimensions(src,grad) == true
ensures
- let OUT be the output of relu(context,OUT,src)
- let f(src) == dot(gradient_input,OUT)
- Then this function computes the gradient of f() with respect to src and
adds it to grad.
!*/
// ------------------------------------------------------------------------------------
// cudnnActivationForward(), CUDNN_ACTIVATION_TANH
void tanh (
cudnn_context& context,
resizable_tensor& dest,
const tensor& src
);
/*!
ensures
- have_same_dimensions(#dest, src) == true
- for all valid i:
- #dest.host()[i] == std::tanh(src.host()[i])
!*/
// cudnnActivationBackward()
void tanh_gradient (
cudnn_context& context,
tensor& grad,
const tensor& src,
const tensor& gradient_input
);
/*!
requires
- have_same_dimensions(src,gradient_input) == true
- have_same_dimensions(src,grad) == true
ensures
- let OUT be the output of tanh(context,OUT,src)
- let f(src) == dot(gradient_input,OUT)
- Then this function computes the gradient of f() with respect to src and
adds it to grad.
!*/
// ------------------------------------------------------------------------------------
}
}
#endif // DLIB_USE_CUDA
#endif // DLIB_DNN_CuDNN_H_
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment