Skip to content

D
dlib
Commit 8c550d4c authored by Davis E. King's avatar Davis E. King
Browse files
Options

Merge pull request #114 from e-fominov/dnn_group_layer 

Concat layer
parents cbd37d56 01b3b08b
Hide whitespace changes
Inline Side-by-side
Showing with 757 additions and 5 deletions
dlib/dnn/core.h
View file @ 8c550d4c
...@@ -536,7 +536,7 @@ namespace dlib ...@@ -536,7 +536,7 @@ namespace dlib
subnet_wrapper(const subnet_wrapper&) = delete; subnet_wrapper(const subnet_wrapper&) = delete;
subnet_wrapper& operator=(const subnet_wrapper&) = delete; subnet_wrapper& operator=(const subnet_wrapper&) = delete;
subnet_wrapper(T& l_) {} subnet_wrapper(T& /*l_*/) {}
// Nothing here because in this case T is one of the input layer types // Nothing here because in this case T is one of the input layer types
// that doesn't have anything in it. // that doesn't have anything in it.
}; };
...@@ -600,7 +600,7 @@ namespace dlib ...@@ -600,7 +600,7 @@ namespace dlib
struct is_nonloss_layer_type<add_layer<T,U>> : std::true_type {}; struct is_nonloss_layer_type<add_layer<T,U>> : std::true_type {};
template <typename LAYER_DETAILS, typename SUBNET> template <typename LAYER_DETAILS, typename SUBNET>
class add_layer<LAYER_DETAILS,SUBNET, class add_layer<LAYER_DETAILS,SUBNET,
typename std::enable_if<is_nonloss_layer_type<SUBNET>::value>::type> typename std::enable_if<is_nonloss_layer_type<SUBNET>::value>::type>
{ {
public: public:
......
dlib/dnn/cpu_dlib.cpp
View file @ 8c550d4c
...@@ -1783,6 +1783,36 @@ namespace dlib ...@@ -1783,6 +1783,36 @@ namespace dlib
filters_gradient += gi*temp; filters_gradient += gi*temp;
} }
} }
// ------------------------------------------------------------------------------------
void copy_tensor(
tensor& dest,
size_t dest_k_offset,
const tensor& src,
size_t src_k_offset,
size_t count_k
)
{
const size_t dest_sample_size = static_cast<size_t>(dest.nc() * dest.nr() * dest.k());
const size_t src_sample_size = static_cast<size_t>(src.nc() * src.nr() * src.k());
const size_t block_size = count_k * dest.nc() * dest.nr();
DLIB_CASSERT(dest.num_samples() == src.num_samples() &&
dest.nc() == src.nc() && dest.nr() == src.nr(), "All sources should fit into dest tensor size");
DLIB_CASSERT(dest.k() - dest_k_offset >= count_k, "Not enough space in dest tensor");
DLIB_CASSERT(src.k() - src_k_offset >= count_k, "Not enough space in src tensor");
float* dest_p = dest.host() + dest_k_offset * dest.nc() * dest.nr();
const float* src_p = src.host() + src_k_offset * src.nc() * src.nr();
for (unsigned long i = 0; i < src.num_samples(); ++i)
{
::memcpy(dest_p, src_p, block_size * sizeof(float));
dest_p += dest_sample_size;
src_p += src_sample_size;
}
}
// ------------------------------------------------------------------------------------ // ------------------------------------------------------------------------------------
// ------------------------------------------------------------------------------------ // ------------------------------------------------------------------------------------
......
dlib/dnn/cpu_dlib.h
View file @ 8c550d4c
...@@ -385,6 +385,14 @@ namespace dlib ...@@ -385,6 +385,14 @@ namespace dlib
}; };
// ----------------------------------------------------------------------------------- // -----------------------------------------------------------------------------------
void copy_tensor(
tensor& dest,
size_t dest_k_offset,
const tensor& src,
size_t src_k_offset,
size_t count_k
);
// -----------------------------------------------------------------------------------
} }
} }
......
dlib/dnn/cuda_dlib.cu
View file @ 8c550d4c
...@@ -796,7 +796,38 @@ namespace dlib ...@@ -796,7 +796,38 @@ namespace dlib
grad.device(), src.device(), gradient_input.device(), grad.size(), grad.device(), src.device(), gradient_input.device(), grad.size(),
param.device(), params_grad.device()); param.device(), params_grad.device());
} }
// ----------------------------------------------------------------------------------------
void copy_tensor(
tensor& dest,
size_t dest_k_offset,
const tensor& src,
size_t src_k_offset,
size_t count_k
)
{
const size_t dest_sample_size = static_cast<size_t>(dest.nc() * dest.nr() * dest.k());
const size_t src_sample_size = static_cast<size_t>(src.nc() * src.nr() * src.k());
const size_t block_size = count_k * dest.nc() * dest.nr();
DLIB_CASSERT(dest.num_samples() == src.num_samples() &&
dest.nc() == src.nc() && dest.nr() == src.nr(), "All sources should fit into dest tensor size");
DLIB_CASSERT(dest.k() - dest_k_offset >= count_k, "Not enough space in dest tensor");
DLIB_CASSERT(src.k() - src_k_offset >= count_k, "Not enough space in src tensor");
float* dest_p = dest.device() + dest_k_offset * dest.nc() * dest.nr();
const float* src_p = src.device() + src_k_offset * src.nc() * src.nr();;
for (unsigned long i = 0; i < src.num_samples(); ++i)
{
CHECK_CUDA(cudaMemcpy(dest_p, src_p, block_size * sizeof(float), cudaMemcpyDeviceToDevice));
dest_p += dest_sample_size;
src_p += src_sample_size;
}
}
// ---------------------------------------------------------------------------------------- // ----------------------------------------------------------------------------------------
} }
......
dlib/dnn/cuda_dlib.h
View file @ 8c550d4c
...@@ -258,6 +258,13 @@ namespace dlib ...@@ -258,6 +258,13 @@ namespace dlib
tensor& params_grad tensor& params_grad
); );
void copy_tensor(
tensor& dest,
size_t dest_k_offset,
const tensor& src,
size_t src_k_offset,
size_t count_k
);
// ------------------------------------------------------------------------------------ // ------------------------------------------------------------------------------------
// ------------------------------------------------------------------------------------ // ------------------------------------------------------------------------------------
// ------------------------------------------------------------------------------------ // ------------------------------------------------------------------------------------
......
dlib/dnn/layers.h
View file @ 8c550d4c
...@@ -1836,6 +1836,199 @@ namespace dlib ...@@ -1836,6 +1836,199 @@ namespace dlib
template <typename SUBNET> template <typename SUBNET>
using softmax = add_layer<softmax_, SUBNET>; using softmax = add_layer<softmax_, SUBNET>;
// ----------------------------------------------------------------------------------------
namespace impl{
// helper classes for layer concat processing
template <template<typename> class... TAG_TYPES>
struct concat_helper_impl {
};
template <template<typename> class TAG_TYPE>
struct concat_helper_impl<TAG_TYPE>{
constexpr static size_t tag_count() {return 1;}
template<typename SUBNET>
static void resize_out(resizable_tensor& out, const SUBNET& sub, long sum_k)
{
auto& t = layer<TAG_TYPE>(sub).get_output();
out.set_size(t.num_samples(), t.k() + sum_k, t.nr(), t.nc());
}
template<typename SUBNET>
static void concat(tensor& out, const SUBNET& sub, size_t k_offset)
{
auto& t = layer<TAG_TYPE>(sub).get_output();
tt::copy_tensor(out, k_offset, t, 0, t.k());
}
template<typename SUBNET>
static void split(const tensor& input, SUBNET& sub, size_t k_offset)
{
auto& t = layer<TAG_TYPE>(sub).get_gradient_input();
tt::copy_tensor(t, 0, input, k_offset, t.k());
}
};
template <template<typename> class TAG_TYPE, template<typename> class... TAG_TYPES>
struct concat_helper_impl<TAG_TYPE, TAG_TYPES...>{
constexpr static size_t tag_count() {return 1 + concat_helper_impl<TAG_TYPES...>::tag_count();}
template<typename SUBNET>
static void resize_out(resizable_tensor& out, const SUBNET& sub, long sum_k)
{
auto& t = layer<TAG_TYPE>(sub).get_output();
concat_helper_impl<TAG_TYPES...>::resize_out(out, sub, sum_k + t.k());
}
template<typename SUBNET>
static void concat(tensor& out, const SUBNET& sub, size_t k_offset)
{
auto& t = layer<TAG_TYPE>(sub).get_output();
tt::copy_tensor(out, k_offset, t, 0, t.k());
k_offset += t.k();
concat_helper_impl<TAG_TYPES...>::concat(out, sub, k_offset);
}
template<typename SUBNET>
static void split(const tensor& input, SUBNET& sub, size_t k_offset)
{
auto& t = layer<TAG_TYPE>(sub).get_gradient_input();
tt::copy_tensor(t, 0, input, k_offset, t.k());
k_offset += t.k();
concat_helper_impl<TAG_TYPES...>::split(input, sub, k_offset);
}
};
}
// concat layer
template<
template<typename> class... TAG_TYPES
>
class concat_
{
public:
constexpr static size_t tag_count() {return impl::concat_helper_impl<TAG_TYPES...>::tag_count();};
template <typename SUBNET>
void setup (const SUBNET&)
{
// do nothing
}
template <typename SUBNET>
void forward(const SUBNET& sub, resizable_tensor& output)
{
// the total depth of result is the sum of depths from all tags
impl::concat_helper_impl<TAG_TYPES...>::resize_out(output, sub, 0);
// copy output from each tag into different part result
impl::concat_helper_impl<TAG_TYPES...>::concat(output, sub, 0);
}
template <typename SUBNET>
void backward(const tensor& gradient_input, SUBNET& sub, tensor&)
{
// Gradient is splitted into parts for each tag layer
impl::concat_helper_impl<TAG_TYPES...>::split(gradient_input, sub, 0);
}
const tensor& get_layer_params() const { return params; }
tensor& get_layer_params() { return params; }
friend void serialize(const concat_& item, std::ostream& out)
{
serialize("concat_", out);
size_t count = tag_count();
serialize(count, out);
}
friend void deserialize(concat_& item, std::istream& in)
{
std::string version;
deserialize(version, in);
if (version != "concat_")
throw serialization_error("Unexpected version '"+version+"' found while deserializing dlib::concat_.");
size_t count_tags;
deserialize(count_tags, in);
if (count_tags != tag_count())
throw serialization_error("Invalid count of tags "+ std::to_string(count_tags) +", expecting " +
std::to_string(tag_count()) +
" found while deserializing dlib::concat_.");
}
friend std::ostream& operator<<(std::ostream& out, const concat_& item)
{
out << "concat\t ("
<< tag_count()
<< ")";
return out;
}
private:
resizable_tensor params; // unused
};
// concat layer definitions
template <template<typename> class TAG1, typename SUBNET>
using concat1 = add_layer<concat_<TAG1>, SUBNET>;
template <template<typename> class TAG1,
template<typename> class TAG2,
typename SUBNET>
using concat2 = add_layer<concat_<TAG1, TAG2>, SUBNET>;
template <template<typename> class TAG1,
template<typename> class TAG2,
template<typename> class TAG3,
typename SUBNET>
using concat3 = add_layer<concat_<TAG1, TAG2, TAG3>, SUBNET>;
template <template<typename> class TAG1,
template<typename> class TAG2,
template<typename> class TAG3,
template<typename> class TAG4,
typename SUBNET>
using concat4 = add_layer<concat_<TAG1, TAG2, TAG3, TAG4>, SUBNET>;
template <template<typename> class TAG1,
template<typename> class TAG2,
template<typename> class TAG3,
template<typename> class TAG4,
template<typename> class TAG5,
typename SUBNET>
using concat5 = add_layer<concat_<TAG1, TAG2, TAG3, TAG4, TAG5>, SUBNET>;
// inception layer will use tags internally. If user will use tags too,
// some conflicts possible
// to exclude them, here are new tags specially for inceptions
template <typename SUBNET> using itag0 = add_tag_layer< 1000 + 0, SUBNET>;
template <typename SUBNET> using itag1 = add_tag_layer< 1000 + 1, SUBNET>;
template <typename SUBNET> using itag2 = add_tag_layer< 1000 + 2, SUBNET>;
template <typename SUBNET> using itag3 = add_tag_layer< 1000 + 3, SUBNET>;
template <typename SUBNET> using itag4 = add_tag_layer< 1000 + 4, SUBNET>;
template <typename SUBNET> using itag5 = add_tag_layer< 1000 + 5, SUBNET>;
// skip to inception input
template <typename SUBNET> using iskip = add_skip_layer< itag0, SUBNET>;
// here are some templates to be used for creating inception layer groups
template <template<typename>class B1,
typename SUBNET>
using inception1 = concat1<itag1, itag1<B1<iskip< itag0<SUBNET>>>>>;
template <template<typename>class B1,
template<typename>class B2,
typename SUBNET>
using inception2 = concat2<itag1, itag2, itag1<B1<iskip< itag2<B2< itag0<SUBNET>>>>>>>;
template <template<typename>class B1,
template<typename>class B2,
template<typename>class B3,
typename SUBNET>
using inception3 = concat3<itag1, itag2, itag3, itag1<B1<iskip< itag2<B2<iskip< itag3<B3< itag0<SUBNET>>>>>>>>>>;
template <template<typename>class B1,
template<typename>class B2,
template<typename>class B3,
template<typename>class B4,
typename SUBNET>
using inception4 = concat4<itag1, itag2, itag3, itag4,
itag1<B1<iskip< itag2<B2<iskip< itag3<B3<iskip< itag4<B4< itag0<SUBNET>>>>>>>>>>>>
>;
template <template<typename>class B1,
template<typename>class B2,
template<typename>class B3,
template<typename>class B4,
template<typename>class B5,
typename SUBNET>
using inception5 = concat5<itag1, itag2, itag3, itag4, itag5,
itag1<B1<iskip< itag2<B2<iskip< itag3<B3<iskip< itag4<B4<iskip< itag5<B5< itag0<SUBNET>>>>>>>>>>>>>>>
>;
// ---------------------------------------------------------------------------------------- // ----------------------------------------------------------------------------------------
} }
......
dlib/dnn/layers_abstract.h
View file @ 8c550d4c
...@@ -1652,6 +1652,116 @@ namespace dlib ...@@ -1652,6 +1652,116 @@ namespace dlib
using add_prev9_ = add_prev_<tag9>; using add_prev9_ = add_prev_<tag9>;
using add_prev10_ = add_prev_<tag10>; using add_prev10_ = add_prev_<tag10>;
// ----------------------------------------------------------------------------------------
template<
template<typename> class... TAG_TYPES
>
class concat_
{
/*!
WHAT THIS OBJECT REPRESENTS
This is an implementation of the EXAMPLE_COMPUTATIONAL_LAYER_ interface
defined above. This layer simply concatenates the output of requiered layers
In particular, it copies each layer's output from TAG_TYPES into the corresponding
place of the result tensor, those producing combined output
The output of each tag layer is stored in a separate part of final output.
FORWARD:
for each (tag in TAG_TYPES)
outout[i, k + tag.k(), r, c] = layer<tag>(subnet).get_output[i, k, r, c]
BACKWARD:
for each (tag in TAG_TYPES)
layer<tag>(subnet).get_gradient_input[i, k, r, c] = input[i, k + tag.k(), r, c]
This layer can be only used with tags inside.
Each tagged layer should have identical num_samples, R and C size
The output will have K = sum(k) of tags, and the, and the output's num_samples,
R and C will be the same as tagged layers
!*/
public:
template <typename SUBNET> void setup (const SUBNET& sub);
template <typename SUBNET> void forward(const SUBNET& sub, resizable_tensor& output);
template <typename SUBNET> void backward(const tensor& gradient_input, SUBNET& sub, tensor& params_grad);
const tensor& get_layer_params() const;
tensor& get_layer_params();
/*!
These functions are implemented as described in the EXAMPLE_COMPUTATIONAL_LAYER_ interface.
!*/
};
// concat layer definitions
template <template<typename> class TAG1, typename SUBNET>
using concat1 = add_layer<concat_<TAG1>, SUBNET>;
template <template<typename> class TAG1,
template<typename> class TAG2,
typename SUBNET>
using concat2 = add_layer<concat_<TAG1, TAG2>, SUBNET>;
template <template<typename> class TAG1,
template<typename> class TAG2,
template<typename> class TAG3,
typename SUBNET>
using concat3 = add_layer<concat_<TAG1, TAG2, TAG3>, SUBNET>;
template <template<typename> class TAG1,
template<typename> class TAG2,
template<typename> class TAG3,
template<typename> class TAG4,
typename SUBNET>
using concat4 = add_layer<concat_<TAG1, TAG2, TAG3, TAG4>, SUBNET>;
template <template<typename> class TAG1,
template<typename> class TAG2,
template<typename> class TAG3,
template<typename> class TAG4,
template<typename> class TAG5,
typename SUBNET>
using concat5 = add_layer<concat_<TAG1, TAG2, TAG3, TAG4, TAG5>, SUBNET>;
// inception layer will use tags internally. If user will use tags too,
// some conflicts possible
// to exclude them, here are new tags specially for inceptions
template <typename SUBNET> using itag0 = add_tag_layer< 1000 + 0, SUBNET>;
template <typename SUBNET> using itag1 = add_tag_layer< 1000 + 1, SUBNET>;
template <typename SUBNET> using itag2 = add_tag_layer< 1000 + 2, SUBNET>;
template <typename SUBNET> using itag3 = add_tag_layer< 1000 + 3, SUBNET>;
template <typename SUBNET> using itag4 = add_tag_layer< 1000 + 4, SUBNET>;
template <typename SUBNET> using itag5 = add_tag_layer< 1000 + 5, SUBNET>;
// skip to inception input
template <typename SUBNET> using iskip = add_skip_layer< itag0, SUBNET>;
// here are some templates to be used for creating inception layer groups
template <template<typename>class B1,
typename SUBNET>
using inception1 = concat1<itag1, itag1<B1<iskip< itag0<SUBNET>>>>>;
template <template<typename>class B1,
template<typename>class B2,
typename SUBNET>
using inception2 = concat2<itag1, itag2, itag1<B1<iskip< itag2<B2< itag0<SUBNET>>>>>>>;
template <template<typename>class B1,
template<typename>class B2,
template<typename>class B3,
typename SUBNET>
using inception3 = concat3<itag1, itag2, itag3, itag1<B1<iskip< itag2<B2<iskip< itag3<B3< itag0<SUBNET>>>>>>>>>>;
template <template<typename>class B1,
template<typename>class B2,
template<typename>class B3,
template<typename>class B4,
typename SUBNET>
using inception4 = concat4<itag1, itag2, itag3, itag4,
itag1<B1<iskip< itag2<B2<iskip< itag3<B3<iskip< itag4<B4< itag0<SUBNET>>>>>>>>>>>>
>;
template <template<typename>class B1,
template<typename>class B2,
template<typename>class B3,
template<typename>class B4,
template<typename>class B5,
typename SUBNET>
using inception5 = concat5<itag1, itag2, itag3, itag4, itag5,
itag1<B1<iskip< itag2<B2<iskip< itag3<B3<iskip< itag4<B4<iskip< itag5<B5< itag0<SUBNET>>>>>>>>>>>>>>>
>;
// ---------------------------------------------------------------------------------------- // ----------------------------------------------------------------------------------------
} }
......
dlib/dnn/tensor_tools.cpp
View file @ 8c550d4c
...@@ -678,6 +678,23 @@ namespace dlib { namespace tt ...@@ -678,6 +678,23 @@ namespace dlib { namespace tt
#endif #endif
} }
// ------------------------------------------------------------------------------------
void copy_tensor(
tensor& dest,
size_t dest_k_offset,
const tensor& src,
size_t src_k_offset,
size_t count_k
)
{
#ifdef DLIB_USE_CUDA
cuda::copy_tensor(dest, dest_k_offset, src, src_k_offset, count_k);
#else
cpu::copy_tensor(dest, dest_k_offset, src, src_k_offset, count_k);
#endif
}
// ---------------------------------------------------------------------------------------- // ----------------------------------------------------------------------------------------
}} }}
......
dlib/dnn/tensor_tools.h
View file @ 8c550d4c
...@@ -1232,6 +1232,27 @@ namespace dlib { namespace tt ...@@ -1232,6 +1232,27 @@ namespace dlib { namespace tt
resizable_tensor accum_buffer; resizable_tensor accum_buffer;
}; };
// ----------------------------------------------------------------------------------------
void copy_tensor(
tensor& dest,
size_t dest_k_offset,
const tensor& src,
size_t src_k_offset,
size_t count_k
);
/*!
requires
- dest.nc() == src.nc()
- dest.nr() == src.nr()
- dest.num_samples() == src.num_samples()
- dest.k() - dest_k_offset >= count_k
- src.k() - src_k_offset >= count_k
- is_same_object(dest,src) == false
ensures
- performs: dest[i, k + dest_k_offset, r, c] = src[i, k + src_k_offset, r, c], where k in [0..count_k]
Copies content of each sample from src in to corresponding place of sample at dst
!*/
// ---------------------------------------------------------------------------------------- // ----------------------------------------------------------------------------------------
......
dlib/test/dnn.cpp
View file @ 8c550d4c
...@@ -11,8 +11,7 @@ ...@@ -11,8 +11,7 @@
#include "tester.h" #include "tester.h"
namespace
namespace
{ {
using namespace test; using namespace test;
...@@ -1405,6 +1404,174 @@ namespace ...@@ -1405,6 +1404,174 @@ namespace
DLIB_TEST(count == pnet.num_computational_layers); DLIB_TEST(count == pnet.num_computational_layers);
} }
float tensor_read_cpu(const tensor& t, long i, long k, long r, long c)
{
const float* p = t.host() + t.k() * t.nr() * t.nc() * i +
t.nr() * t.nc() * k + t.nc() * r + c;
return *p;
}
void test_copy_tensor_cpu()
{
using namespace dlib::tt;
print_spinner();
resizable_tensor dest(10, 9, 7, 15);
resizable_tensor src1(10, 3, 7, 15);
resizable_tensor src2(10, 3, 7, 15);
resizable_tensor src3(10, 9, 7, 15);
dest = matrix_cast<float>(gaussian_randm(dest.num_samples(), dest.k() * dest.nr() * dest.nc(), 1));
src1 = matrix_cast<float>(gaussian_randm(src1.num_samples(), src1.k() * src1.nr() * src1.nc(), 0));
src2 = matrix_cast<float>(gaussian_randm(src1.num_samples(), src2.k() * src2.nr() * src2.nc(), 0));
src3 = matrix_cast<float>(gaussian_randm(src1.num_samples(), src3.k() * src3.nr() * src3.nc(), 0));
cpu::copy_tensor(dest, 0, src1, 0, src1.k()); //full copy src1->dest
cpu::copy_tensor(dest, src1.k(), src2, 0, src2.k()); //full copy src2->dest with offset of src1
cpu::copy_tensor(dest, src1.k() + src2.k(), src3, 3, 3); //partial copy src3 into the rest place of dest
for (long i = 0; i < dest.num_samples(); ++i)
{
for (long k = 0; k < dest.k(); ++k)
{
for (long r = 0; r < dest.nr(); ++r)
{
for (long c = 0; c < dest.nc(); ++c)
{
float dest_value = tensor_read_cpu(dest, i, k, r, c);
// first part is from src1
if (k < src1.k())
{
float src_value = tensor_read_cpu(src1, i, k, r, c);
DLIB_TEST(src_value == dest_value);
}
// second part is from src2
else if (k < src1.k() + src2.k())
{
float src_value = tensor_read_cpu(src2, i, k - src1.k(), r, c);
DLIB_TEST(src_value == dest_value);
}
// third part is from src3
else
{
float src_value = tensor_read_cpu(src3, i, k - src1.k() - src2.k() + 3, r, c);
DLIB_TEST(src_value == dest_value);
}
}
}
}
}
}
#ifdef DLIB_USE_CUDA
void test_copy_tensor_gpu()
{
using namespace dlib::tt;
print_spinner();
resizable_tensor dest(10, 9, 7, 15);
resizable_tensor src1(10, 3, 7, 15);
resizable_tensor src2(10, 3, 7, 15);
resizable_tensor src3(10, 9, 7, 15);
dest = matrix_cast<float>(gaussian_randm(dest.num_samples(), dest.k() * dest.nr() * dest.nc(), 1));
src1 = matrix_cast<float>(gaussian_randm(src1.num_samples(), src1.k() * src1.nr() * src1.nc(), 0));
src2 = matrix_cast<float>(gaussian_randm(src1.num_samples(), src2.k() * src2.nr() * src2.nc(), 0));
src3 = matrix_cast<float>(gaussian_randm(src1.num_samples(), src3.k() * src3.nr() * src3.nc(), 0));
cuda::copy_tensor(dest, 0, src1, 0, src1.k()); //full copy src1->dest
cuda::copy_tensor(dest, src1.k(), src2, 0, src2.k()); //full copy src2->dest with offset of src1
cuda::copy_tensor(dest, src1.k() + src2.k(), src3, 3, 3); //partial copy src3 into the rest place of dest
for (long i = 0; i < dest.num_samples(); ++i)
{
for (long k = 0; k < dest.k(); ++k)
{
for (long r = 0; r < dest.nr(); ++r)
{
for (long c = 0; c < dest.nc(); ++c)
{
float dest_value = tensor_read_cpu(dest, i, k, r, c);
// first part is from src1
if (k < src1.k())
{
float src_value = tensor_read_cpu(src1, i, k, r, c);
DLIB_TEST(src_value == dest_value);
}
// second part is from src2
else if (k < src1.k() + src2.k())
{
float src_value = tensor_read_cpu(src2, i, k - src1.k(), r, c);
DLIB_TEST(src_value == dest_value);
}
// third part is from src3
else
{
float src_value = tensor_read_cpu(src3, i, k - src1.k() - src2.k() + 3, r, c);
DLIB_TEST(src_value == dest_value);
}
}
}
}
}
}
#endif//DLIB_USE_CUDA
template <typename SUBNET> using concat_block1 = con<5,1,1,1,1,SUBNET>;
template <typename SUBNET> using concat_block2 = con<8,3,3,1,1,SUBNET>;
template <typename SUBNET> using concat_block3 = max_pool<3,3,1,1,SUBNET>;
template <typename SUBNET> using concat_incept = inception3<concat_block1,concat_block2,concat_block3,SUBNET>;
void test_concat()
{
using namespace dlib::tt;
print_spinner();
using net_type = concat_incept<input<matrix<float>>>;
resizable_tensor data(10, 1, 111, 222);
data = matrix_cast<float>(gaussian_randm(data.num_samples(), data.k() * data.nr() * data.nc(), 1));
net_type net;
auto& out = net.forward(data);
auto& b1o = layer<itag1>(net).get_output();
auto& b2o = layer<itag2>(net).get_output();
auto& b3o = layer<itag3>(net).get_output();
resizable_tensor dest(10, 14, 111, 222);
copy_tensor(dest, 0, b1o, 0, b1o.k());
copy_tensor(dest, b1o.k(), b2o, 0, b2o.k());
copy_tensor(dest, b1o.k() + b2o.k(), b3o, 0, b3o.k());
DLIB_TEST(dest.size() == out.size());
int error = memcmp(dest.host(), out.host(), dest.size());
DLIB_TEST(error == 0);
resizable_tensor gr(10, 14, 111, 222);
gr = matrix_cast<float>(gaussian_randm(gr.num_samples(), gr.k() * gr.nr() * gr.nc(), 1));
resizable_tensor params;
net.layer_details().backward(gr, net, params);
auto& b1g = layer<itag1>(net).subnet().get_gradient_input();
auto& b2g = layer<itag2>(net).subnet().get_gradient_input();
auto& b3g = layer<itag3>(net).subnet().get_gradient_input();
resizable_tensor g1(10, 5, 111, 222);
resizable_tensor g2(10, 8, 111, 222);
resizable_tensor g3(10, 1, 111, 222);
copy_tensor(g1, 0, gr, 0, g1.k());
copy_tensor(g2, 0, gr, g1.k(), g2.k());
copy_tensor(g3, 0, gr, g1.k() + g2.k(), g3.k());
DLIB_TEST(g1.size() == b1g.size());
error = memcmp(g1.host(), b1g.host(), b1g.size());
DLIB_TEST(error == 0);
DLIB_TEST(g2.size() == b2g.size());
error = memcmp(g2.host(), b2g.host(), b2g.size());
DLIB_TEST(error == 0);
DLIB_TEST(g3.size() == b3g.size());
error = memcmp(g3.host(), b3g.host(), b3g.size());
DLIB_TEST(error == 0);
}
// ---------------------------------------------------------------------------------------- // ----------------------------------------------------------------------------------------
class dnn_tester : public tester class dnn_tester : public tester
...@@ -1433,6 +1600,7 @@ namespace ...@@ -1433,6 +1600,7 @@ namespace
compare_bn_conv_gpu_and_cpu(); compare_bn_conv_gpu_and_cpu();
test_add(); test_add();
compare_adam(); compare_adam();
test_copy_tensor_gpu();
#endif #endif
test_max_pool(1,1,2,3,0,0); test_max_pool(1,1,2,3,0,0);
test_max_pool(3,3,1,1,0,0); test_max_pool(3,3,1,1,0,0);
...@@ -1466,9 +1634,10 @@ namespace ...@@ -1466,9 +1634,10 @@ namespace
test_basic_tensor_ops(); test_basic_tensor_ops();
test_layers(); test_layers();
test_visit_funcions(); test_visit_funcions();
test_copy_tensor_cpu();
test_concat();
} }
} a; } a;
} }
examples/CMakeLists.txt
View file @ 8c550d4c
...@@ -33,6 +33,7 @@ ENDMACRO() ...@@ -33,6 +33,7 @@ ENDMACRO()
if (COMPILER_CAN_DO_CPP_11) if (COMPILER_CAN_DO_CPP_11)
add_example(dnn_mnist_ex) add_example(dnn_mnist_ex)
add_example(dnn_mnist_advanced_ex) add_example(dnn_mnist_advanced_ex)
add_example(dnn_inception_ex)
endif() endif()
#here we apply our macros #here we apply our macros
......
examples/dnn_inception_ex.cpp 0 → 100644
View file @ 8c550d4c
// The contents of this file are in the public domain. See LICENSE_FOR_EXAMPLE_PROGRAMS.txt
/*
This is an example illustrating the use of the deep learning tools from the
dlib C++ Library. I'm assuming you have already read the dnn_mnist_ex.cpp
example. So in this example program I'm going to go over a number of more
advanced parts of the API, including:
- Using grp layer for constructing inception layer
Inception layer is a kind of NN architecture for running sevelar convolution types
on the same input area and joining all convolution results into one output.
For further reading refer http://www.cs.unc.edu/~wliu/papers/GoogLeNet.pdf
*/
#include <dlib/dnn.h>
#include <iostream>
#include <dlib/data_io.h>
using namespace std;
using namespace dlib;
// Inception layer has some different convolutions inside
// Here we define blocks as convolutions with different kernel size that we will use in
// inception layer block.
template <typename SUBNET> using block_a1 = relu<con<10,1,1,1,1,SUBNET>>;
template <typename SUBNET> using block_a2 = relu<con<10,3,3,1,1,relu<con<16,1,1,1,1,SUBNET>>>>;
template <typename SUBNET> using block_a3 = relu<con<10,5,5,1,1,relu<con<16,1,1,1,1,SUBNET>>>>;
template <typename SUBNET> using block_a4 = relu<con<10,1,1,1,1,max_pool<3,3,1,1,SUBNET>>>;
// Here is inception layer definition. It uses different blocks to process input and returns combined output
template <typename SUBNET> using incept_a = inception4<block_a1,block_a2,block_a3,block_a4, SUBNET>;
// Network can have inception layers of different structure.
// Here are blocks with different convolutions
template <typename SUBNET> using block_b1 = relu<con<4,1,1,1,1,SUBNET>>;
template <typename SUBNET> using block_b2 = relu<con<4,3,3,1,1,SUBNET>>;
template <typename SUBNET> using block_b3 = relu<con<4,1,1,1,1,max_pool<3,3,1,1,SUBNET>>>;
// Here is inception layer definition. It uses different blocks to process input and returns combined output
template <typename SUBNET> using incept_b = inception3<block_b1,block_b2,block_b3,SUBNET>;
// and then the network type is
using net_type = loss_multiclass_log<
fc<10,
relu<fc<32,
max_pool<2,2,2,2,incept_b<
max_pool<2,2,2,2,tag1<incept_a<
input<matrix<unsigned char>>
>>>>>>>>>;
int main(int argc, char** argv) try
{
// This example is going to run on the MNIST dataset.
if (argc != 2)
{
cout << "This example needs the MNIST dataset to run!" << endl;
cout << "You can get MNIST from http://yann.lecun.com/exdb/mnist/" << endl;
cout << "Download the 4 files that comprise the dataset, decompress them, and" << endl;
cout << "put them in a folder. Then give that folder as input to this program." << endl;
return 1;
}
std::vector<matrix<unsigned char>> training_images;
std::vector<unsigned long> training_labels;
std::vector<matrix<unsigned char>> testing_images;
std::vector<unsigned long> testing_labels;
load_mnist_dataset(argv[1], training_images, training_labels, testing_images, testing_labels);
// Create network of predefined type.
net_type net;
// Now let's print the details of the pnet to the screen and inspect it.
cout << "The net has " << net.num_layers << " layers in it." << endl;
cout << net << endl;
// we can access inner layers with layer<> function:
// with tags
auto& in_b = layer<tag1>(net);
cout << "Found inception B layer: " << endl << in_b << endl;
// and we can access layers inside inceptions with itags
auto& in_b_1 = layer<itag1>(in_b);
cout << "Found inception B/1 layer: " << endl << in_b_1 << endl;
// or this is identical to
auto& in_b_1_a = layer<tag1,2>(net);
cout << "Found inception B/1 layer alternative way: " << endl << in_b_1_a << endl;
cout << "Traning NN..." << endl;
// The rest of the sample is identical to dnn_minst_ex
// And then train it using the MNIST data. The code below uses mini-batch stochastic
// gradient descent with an initial learning rate of 0.01 to accomplish this.
dnn_trainer<net_type> trainer(net);
trainer.set_learning_rate(0.01);
trainer.set_min_learning_rate(0.00001);
trainer.set_mini_batch_size(128);
trainer.be_verbose();
// Since DNN training can take a long time, we can ask the trainer to save its state to
// a file named "mnist_sync" every 20 seconds. This way, if we kill this program and
// start it again it will begin where it left off rather than restarting the training
// from scratch. This is because, when the program restarts, this call to
// set_synchronization_file() will automatically reload the settings from mnist_sync if
// the file exists.
trainer.set_synchronization_file("inception_sync", std::chrono::seconds(20));
// Finally, this line begins training. By default, it runs SGD with our specified
// learning rate until the loss stops decreasing. Then it reduces the learning rate by
// a factor of 10 and continues running until the loss stops decreasing again. It will
// keep doing this until the learning rate has dropped below the min learning rate
// defined above or the maximum number of epochs as been executed (defaulted to 10000).
trainer.train(training_images, training_labels);
// At this point our net object should have learned how to classify MNIST images. But
// before we try it out let's save it to disk. Note that, since the trainer has been
// running images through the network, net will have a bunch of state in it related to
// the last batch of images it processed (e.g. outputs from each layer). Since we
// don't care about saving that kind of stuff to disk we can tell the network to forget
// about that kind of transient data so that our file will be smaller. We do this by
// "cleaning" the network before saving it.
net.clean();
serialize("mnist_network_inception.dat") << net;
// Now if we later wanted to recall the network from disk we can simply say:
// deserialize("mnist_network.dat") >> net;
// Now let's run the training images through the network. This statement runs all the
// images through it and asks the loss layer to convert the network's raw output into
// labels. In our case, these labels are the numbers between 0 and 9.
std::vector<unsigned long> predicted_labels = net(training_images);
int num_right = 0;
int num_wrong = 0;
// And then let's see if it classified them correctly.
for (size_t i = 0; i < training_images.size(); ++i)
{
if (predicted_labels[i] == training_labels[i])
++num_right;
else
++num_wrong;
}
cout << "training num_right: " << num_right << endl;
cout << "training num_wrong: " << num_wrong << endl;
cout << "training accuracy: " << num_right/(double)(num_right+num_wrong) << endl;
// Let's also see if the network can correctly classify the testing images. Since
// MNIST is an easy dataset, we should see at least 99% accuracy.
predicted_labels = net(testing_images);
num_right = 0;
num_wrong = 0;
for (size_t i = 0; i < testing_images.size(); ++i)
{
if (predicted_labels[i] == testing_labels[i])
++num_right;
else
++num_wrong;
}
cout << "testing num_right: " << num_right << endl;
cout << "testing num_wrong: " << num_wrong << endl;
cout << "testing accuracy: " << num_right/(double)(num_right+num_wrong) << endl;
}
catch(std::exception& e)
{
cout << e.what() << endl;
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment