Skip to content
Projects
Groups
Snippets
Help
Loading...
Sign in
Toggle navigation
D
dlib
Project
Project
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
钟尚武
dlib
Commits
6acddf99
Commit
6acddf99
authored
Apr 10, 2016
by
Davis King
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Just renamed variables to reflect the new meaning of the batch normalization
running variance output.
parent
538de238
Hide whitespace changes
Inline
Side-by-side
Showing
8 changed files
with
123 additions
and
121 deletions
+123
-121
cpu_dlib.cpp
dlib/dnn/cpu_dlib.cpp
+26
-24
cpu_dlib.h
dlib/dnn/cpu_dlib.h
+4
-4
cudnn_dlibapi.cpp
dlib/dnn/cudnn_dlibapi.cpp
+22
-22
cudnn_dlibapi.h
dlib/dnn/cudnn_dlibapi.h
+4
-4
layers.h
dlib/dnn/layers.h
+12
-12
tensor_tools.cpp
dlib/dnn/tensor_tools.cpp
+12
-12
tensor_tools.h
dlib/dnn/tensor_tools.h
+19
-19
dnn.cpp
dlib/test/dnn.cpp
+24
-24
No files found.
dlib/dnn/cpu_dlib.cpp
View file @
6acddf99
...
@@ -466,7 +466,7 @@ namespace dlib
...
@@ -466,7 +466,7 @@ namespace dlib
const
tensor
&
gamma
,
const
tensor
&
gamma
,
const
tensor
&
beta
,
const
tensor
&
beta
,
const
tensor
&
running_means
,
const
tensor
&
running_means
,
const
tensor
&
running_
invstd
s
const
tensor
&
running_
variance
s
)
)
{
{
DLIB_CASSERT
(
DLIB_CASSERT
(
...
@@ -476,7 +476,7 @@ namespace dlib
...
@@ -476,7 +476,7 @@ namespace dlib
gamma
.
k
()
==
src
.
k
()
&&
gamma
.
k
()
==
src
.
k
()
&&
have_same_dimensions
(
gamma
,
beta
)
&&
have_same_dimensions
(
gamma
,
beta
)
&&
have_same_dimensions
(
gamma
,
running_means
)
&&
have_same_dimensions
(
gamma
,
running_means
)
&&
have_same_dimensions
(
gamma
,
running_
invstd
s
),
have_same_dimensions
(
gamma
,
running_
variance
s
),
"
\n
gamma.num_samples(): "
<<
gamma
.
num_samples
()
<<
"
\n
gamma.num_samples(): "
<<
gamma
.
num_samples
()
<<
"
\n
gamma.k(): "
<<
gamma
.
k
()
<<
"
\n
gamma.k(): "
<<
gamma
.
k
()
<<
"
\n
gamma.nr(): "
<<
gamma
.
nr
()
<<
"
\n
gamma.nr(): "
<<
gamma
.
nr
()
<<
...
@@ -489,10 +489,10 @@ namespace dlib
...
@@ -489,10 +489,10 @@ namespace dlib
"
\n
running_means.k(): "
<<
running_means
.
k
()
<<
"
\n
running_means.k(): "
<<
running_means
.
k
()
<<
"
\n
running_means.nr(): "
<<
running_means
.
nr
()
<<
"
\n
running_means.nr(): "
<<
running_means
.
nr
()
<<
"
\n
running_means.nc(): "
<<
running_means
.
nc
()
<<
"
\n
running_means.nc(): "
<<
running_means
.
nc
()
<<
"
\n
running_
invstds.num_samples(): "
<<
running_invstd
s
.
num_samples
()
<<
"
\n
running_
variances.num_samples(): "
<<
running_variance
s
.
num_samples
()
<<
"
\n
running_
invstds.k(): "
<<
running_invstd
s
.
k
()
<<
"
\n
running_
variances.k(): "
<<
running_variance
s
.
k
()
<<
"
\n
running_
invstds.nr(): "
<<
running_invstd
s
.
nr
()
<<
"
\n
running_
variances.nr(): "
<<
running_variance
s
.
nr
()
<<
"
\n
running_
invstds.nc(): "
<<
running_invstd
s
.
nc
()
<<
"
\n
running_
variances.nc(): "
<<
running_variance
s
.
nc
()
<<
"
\n
src.k(): "
<<
src
.
k
()
<<
"
\n
src.k(): "
<<
src
.
k
()
<<
"
\n
src.nr(): "
<<
src
.
nr
()
<<
"
\n
src.nr(): "
<<
src
.
nr
()
<<
"
\n
src.nc(): "
<<
src
.
nc
()
"
\n
src.nc(): "
<<
src
.
nc
()
...
@@ -504,14 +504,14 @@ namespace dlib
...
@@ -504,14 +504,14 @@ namespace dlib
auto
g
=
gamma
.
host
();
auto
g
=
gamma
.
host
();
auto
b
=
beta
.
host
();
auto
b
=
beta
.
host
();
auto
m
=
running_means
.
host
();
auto
m
=
running_means
.
host
();
auto
i
=
running_invstd
s
.
host
();
auto
v
=
running_variance
s
.
host
();
const
long
num
=
src
.
k
()
*
src
.
nr
()
*
src
.
nc
();
const
long
num
=
src
.
k
()
*
src
.
nr
()
*
src
.
nc
();
for
(
long
n
=
0
;
n
<
src
.
num_samples
();
++
n
)
for
(
long
n
=
0
;
n
<
src
.
num_samples
();
++
n
)
{
{
for
(
long
k
=
0
;
k
<
num
;
++
k
)
for
(
long
k
=
0
;
k
<
num
;
++
k
)
{
{
*
d
=
g
[
k
]
*
(
*
s
-
m
[
k
])
/
std
::
sqrt
(
i
[
k
]
+
dlib
::
tt
::
BATCH_NORM_EPS
)
+
b
[
k
];
*
d
=
g
[
k
]
*
(
*
s
-
m
[
k
])
/
std
::
sqrt
(
v
[
k
]
+
dlib
::
tt
::
BATCH_NORM_EPS
)
+
b
[
k
];
++
d
;
++
d
;
++
s
;
++
s
;
}
}
...
@@ -524,7 +524,7 @@ namespace dlib
...
@@ -524,7 +524,7 @@ namespace dlib
resizable_tensor
&
invstds
,
resizable_tensor
&
invstds
,
const
double
averaging_factor
,
const
double
averaging_factor
,
resizable_tensor
&
running_means
,
resizable_tensor
&
running_means
,
resizable_tensor
&
running_
invstd
s
,
resizable_tensor
&
running_
variance
s
,
const
tensor
&
src
,
const
tensor
&
src
,
const
tensor
&
gamma
,
const
tensor
&
gamma
,
const
tensor
&
beta
const
tensor
&
beta
...
@@ -532,7 +532,7 @@ namespace dlib
...
@@ -532,7 +532,7 @@ namespace dlib
{
{
DLIB_CASSERT
(
0
<=
averaging_factor
&&
averaging_factor
<=
1
,
"averaging_factor: "
<<
averaging_factor
);
DLIB_CASSERT
(
0
<=
averaging_factor
&&
averaging_factor
<=
1
,
"averaging_factor: "
<<
averaging_factor
);
DLIB_CASSERT
(
averaging_factor
==
1
||
have_same_dimensions
(
running_means
,
means
),
""
);
DLIB_CASSERT
(
averaging_factor
==
1
||
have_same_dimensions
(
running_means
,
means
),
""
);
DLIB_CASSERT
(
averaging_factor
==
1
||
have_same_dimensions
(
running_
invstd
s
,
invstds
),
""
);
DLIB_CASSERT
(
averaging_factor
==
1
||
have_same_dimensions
(
running_
variance
s
,
invstds
),
""
);
DLIB_CASSERT
(
DLIB_CASSERT
(
src
.
num_samples
()
>
1
&&
src
.
num_samples
()
>
1
&&
gamma
.
num_samples
()
==
1
&&
gamma
.
num_samples
()
==
1
&&
...
@@ -580,8 +580,9 @@ namespace dlib
...
@@ -580,8 +580,9 @@ namespace dlib
invstds
.
host
();
means
.
host
();
invstds
.
host
();
means
.
host
();
// compute variances
// compute variances
running_invstds
.
copy_size
(
invstds
);
running_variances
.
copy_size
(
invstds
);
auto
rvar
=
running_invstds
.
host
();
auto
rvar
=
running_variances
.
host
();
// This scale makes the running variances unbiased.
const
double
scale
=
(
src
.
num_samples
())
/
(
src
.
num_samples
()
-
1.0
);
const
double
scale
=
(
src
.
num_samples
())
/
(
src
.
num_samples
()
-
1.0
);
for
(
long
i
=
0
;
i
<
num
;
++
i
)
for
(
long
i
=
0
;
i
<
num
;
++
i
)
{
{
...
@@ -718,7 +719,7 @@ namespace dlib
...
@@ -718,7 +719,7 @@ namespace dlib
const
tensor
&
gamma
,
const
tensor
&
gamma
,
const
tensor
&
beta
,
const
tensor
&
beta
,
const
tensor
&
running_means
,
const
tensor
&
running_means
,
const
tensor
&
running_
invstd
s
const
tensor
&
running_
variance
s
)
)
{
{
DLIB_CASSERT
(
DLIB_CASSERT
(
...
@@ -728,7 +729,7 @@ namespace dlib
...
@@ -728,7 +729,7 @@ namespace dlib
gamma
.
k
()
==
src
.
k
()
&&
gamma
.
k
()
==
src
.
k
()
&&
have_same_dimensions
(
gamma
,
beta
)
&&
have_same_dimensions
(
gamma
,
beta
)
&&
have_same_dimensions
(
gamma
,
running_means
)
&&
have_same_dimensions
(
gamma
,
running_means
)
&&
have_same_dimensions
(
gamma
,
running_
invstd
s
),
have_same_dimensions
(
gamma
,
running_
variance
s
),
"
\n
gamma.num_samples(): "
<<
gamma
.
num_samples
()
<<
"
\n
gamma.num_samples(): "
<<
gamma
.
num_samples
()
<<
"
\n
gamma.k(): "
<<
gamma
.
k
()
<<
"
\n
gamma.k(): "
<<
gamma
.
k
()
<<
"
\n
gamma.nr(): "
<<
gamma
.
nr
()
<<
"
\n
gamma.nr(): "
<<
gamma
.
nr
()
<<
...
@@ -741,10 +742,10 @@ namespace dlib
...
@@ -741,10 +742,10 @@ namespace dlib
"
\n
running_means.k(): "
<<
running_means
.
k
()
<<
"
\n
running_means.k(): "
<<
running_means
.
k
()
<<
"
\n
running_means.nr(): "
<<
running_means
.
nr
()
<<
"
\n
running_means.nr(): "
<<
running_means
.
nr
()
<<
"
\n
running_means.nc(): "
<<
running_means
.
nc
()
<<
"
\n
running_means.nc(): "
<<
running_means
.
nc
()
<<
"
\n
running_
invstds.num_samples(): "
<<
running_invstd
s
.
num_samples
()
<<
"
\n
running_
variances.num_samples(): "
<<
running_variance
s
.
num_samples
()
<<
"
\n
running_
invstds.k(): "
<<
running_invstd
s
.
k
()
<<
"
\n
running_
variances.k(): "
<<
running_variance
s
.
k
()
<<
"
\n
running_
invstds.nr(): "
<<
running_invstd
s
.
nr
()
<<
"
\n
running_
variances.nr(): "
<<
running_variance
s
.
nr
()
<<
"
\n
running_
invstds.nc(): "
<<
running_invstd
s
.
nc
()
<<
"
\n
running_
variances.nc(): "
<<
running_variance
s
.
nc
()
<<
"
\n
src.k(): "
<<
src
.
k
()
<<
"
\n
src.k(): "
<<
src
.
k
()
<<
"
\n
src.nr(): "
<<
src
.
nr
()
<<
"
\n
src.nr(): "
<<
src
.
nr
()
<<
"
\n
src.nc(): "
<<
src
.
nc
()
"
\n
src.nc(): "
<<
src
.
nc
()
...
@@ -756,14 +757,14 @@ namespace dlib
...
@@ -756,14 +757,14 @@ namespace dlib
auto
g
=
gamma
.
host
();
auto
g
=
gamma
.
host
();
auto
b
=
beta
.
host
();
auto
b
=
beta
.
host
();
auto
m
=
running_means
.
host
();
auto
m
=
running_means
.
host
();
auto
i
=
running_invstd
s
.
host
();
auto
v
=
running_variance
s
.
host
();
const
long
num
=
src
.
nr
()
*
src
.
nc
();
const
long
num
=
src
.
nr
()
*
src
.
nc
();
for
(
long
n
=
0
;
n
<
src
.
num_samples
();
++
n
)
for
(
long
n
=
0
;
n
<
src
.
num_samples
();
++
n
)
{
{
for
(
long
k
=
0
;
k
<
src
.
k
();
++
k
)
for
(
long
k
=
0
;
k
<
src
.
k
();
++
k
)
{
{
const
float
invstd
=
1.0
f
/
std
::
sqrt
(
i
[
k
]
+
dlib
::
tt
::
BATCH_NORM_EPS
);
const
float
invstd
=
1.0
f
/
std
::
sqrt
(
v
[
k
]
+
dlib
::
tt
::
BATCH_NORM_EPS
);
for
(
long
j
=
0
;
j
<
num
;
++
j
)
for
(
long
j
=
0
;
j
<
num
;
++
j
)
{
{
*
d
=
g
[
k
]
*
(
*
s
-
m
[
k
])
*
invstd
+
b
[
k
];
*
d
=
g
[
k
]
*
(
*
s
-
m
[
k
])
*
invstd
+
b
[
k
];
...
@@ -780,7 +781,7 @@ namespace dlib
...
@@ -780,7 +781,7 @@ namespace dlib
resizable_tensor
&
invstds
,
resizable_tensor
&
invstds
,
const
double
averaging_factor
,
const
double
averaging_factor
,
resizable_tensor
&
running_means
,
resizable_tensor
&
running_means
,
resizable_tensor
&
running_
invstd
s
,
resizable_tensor
&
running_
variance
s
,
const
tensor
&
src
,
const
tensor
&
src
,
const
tensor
&
gamma
,
const
tensor
&
gamma
,
const
tensor
&
beta
const
tensor
&
beta
...
@@ -788,7 +789,7 @@ namespace dlib
...
@@ -788,7 +789,7 @@ namespace dlib
{
{
DLIB_CASSERT
(
0
<=
averaging_factor
&&
averaging_factor
<=
1
,
"averaging_factor: "
<<
averaging_factor
);
DLIB_CASSERT
(
0
<=
averaging_factor
&&
averaging_factor
<=
1
,
"averaging_factor: "
<<
averaging_factor
);
DLIB_CASSERT
(
averaging_factor
==
1
||
have_same_dimensions
(
running_means
,
means
),
""
);
DLIB_CASSERT
(
averaging_factor
==
1
||
have_same_dimensions
(
running_means
,
means
),
""
);
DLIB_CASSERT
(
averaging_factor
==
1
||
have_same_dimensions
(
running_
invstd
s
,
invstds
),
""
);
DLIB_CASSERT
(
averaging_factor
==
1
||
have_same_dimensions
(
running_
variance
s
,
invstds
),
""
);
DLIB_CASSERT
(
DLIB_CASSERT
(
src
.
num_samples
()
>
1
&&
src
.
num_samples
()
>
1
&&
gamma
.
num_samples
()
==
1
&&
gamma
.
num_samples
()
==
1
&&
...
@@ -844,8 +845,9 @@ namespace dlib
...
@@ -844,8 +845,9 @@ namespace dlib
p_src
=
src
.
host
();
p_src
=
src
.
host
();
// compute variances
// compute variances
running_invstds
.
copy_size
(
invstds
);
running_variances
.
copy_size
(
invstds
);
auto
rvar
=
running_invstds
.
host
();
auto
rvar
=
running_variances
.
host
();
// This scale makes the running variances unbiased.
const
double
scale
=
(
src
.
num_samples
()
*
num
)
/
(
src
.
num_samples
()
*
num
-
1.0
);
const
double
scale
=
(
src
.
num_samples
()
*
num
)
/
(
src
.
num_samples
()
*
num
-
1.0
);
for
(
long
k
=
0
;
k
<
src
.
k
();
++
k
)
for
(
long
k
=
0
;
k
<
src
.
k
();
++
k
)
{
{
...
...
dlib/dnn/cpu_dlib.h
View file @
6acddf99
...
@@ -120,7 +120,7 @@ namespace dlib
...
@@ -120,7 +120,7 @@ namespace dlib
const
tensor
&
gamma
,
const
tensor
&
gamma
,
const
tensor
&
beta
,
const
tensor
&
beta
,
const
tensor
&
running_means
,
const
tensor
&
running_means
,
const
tensor
&
running_
invstd
s
const
tensor
&
running_
variance
s
);
);
void
batch_normalize
(
void
batch_normalize
(
...
@@ -129,7 +129,7 @@ namespace dlib
...
@@ -129,7 +129,7 @@ namespace dlib
resizable_tensor
&
invstds
,
resizable_tensor
&
invstds
,
const
double
averaging_factor
,
const
double
averaging_factor
,
resizable_tensor
&
running_means
,
resizable_tensor
&
running_means
,
resizable_tensor
&
running_
invstd
s
,
resizable_tensor
&
running_
variance
s
,
const
tensor
&
src
,
const
tensor
&
src
,
const
tensor
&
gamma
,
const
tensor
&
gamma
,
const
tensor
&
beta
const
tensor
&
beta
...
@@ -152,7 +152,7 @@ namespace dlib
...
@@ -152,7 +152,7 @@ namespace dlib
const
tensor
&
gamma
,
const
tensor
&
gamma
,
const
tensor
&
beta
,
const
tensor
&
beta
,
const
tensor
&
running_means
,
const
tensor
&
running_means
,
const
tensor
&
running_
invstd
s
const
tensor
&
running_
variance
s
);
);
void
batch_normalize_conv
(
void
batch_normalize_conv
(
...
@@ -161,7 +161,7 @@ namespace dlib
...
@@ -161,7 +161,7 @@ namespace dlib
resizable_tensor
&
invstds
,
resizable_tensor
&
invstds
,
const
double
averaging_factor
,
const
double
averaging_factor
,
resizable_tensor
&
running_means
,
resizable_tensor
&
running_means
,
resizable_tensor
&
running_
invstd
s
,
resizable_tensor
&
running_
variance
s
,
const
tensor
&
src
,
const
tensor
&
src
,
const
tensor
&
gamma
,
const
tensor
&
gamma
,
const
tensor
&
beta
const
tensor
&
beta
...
...
dlib/dnn/cudnn_dlibapi.cpp
View file @
6acddf99
...
@@ -343,7 +343,7 @@ namespace dlib
...
@@ -343,7 +343,7 @@ namespace dlib
const
tensor
&
gamma
,
const
tensor
&
gamma
,
const
tensor
&
beta
,
const
tensor
&
beta
,
const
tensor
&
running_means
,
const
tensor
&
running_means
,
const
tensor
&
running_
invstd
s
const
tensor
&
running_
variance
s
)
)
{
{
DLIB_CASSERT
(
DLIB_CASSERT
(
...
@@ -353,7 +353,7 @@ namespace dlib
...
@@ -353,7 +353,7 @@ namespace dlib
gamma
.
k
()
==
src
.
k
()
&&
gamma
.
k
()
==
src
.
k
()
&&
have_same_dimensions
(
gamma
,
beta
)
&&
have_same_dimensions
(
gamma
,
beta
)
&&
have_same_dimensions
(
gamma
,
running_means
)
&&
have_same_dimensions
(
gamma
,
running_means
)
&&
have_same_dimensions
(
gamma
,
running_
invstd
s
),
have_same_dimensions
(
gamma
,
running_
variance
s
),
"
\n
gamma.num_samples(): "
<<
gamma
.
num_samples
()
<<
"
\n
gamma.num_samples(): "
<<
gamma
.
num_samples
()
<<
"
\n
gamma.k(): "
<<
gamma
.
k
()
<<
"
\n
gamma.k(): "
<<
gamma
.
k
()
<<
"
\n
gamma.nr(): "
<<
gamma
.
nr
()
<<
"
\n
gamma.nr(): "
<<
gamma
.
nr
()
<<
...
@@ -366,10 +366,10 @@ namespace dlib
...
@@ -366,10 +366,10 @@ namespace dlib
"
\n
running_means.k(): "
<<
running_means
.
k
()
<<
"
\n
running_means.k(): "
<<
running_means
.
k
()
<<
"
\n
running_means.nr(): "
<<
running_means
.
nr
()
<<
"
\n
running_means.nr(): "
<<
running_means
.
nr
()
<<
"
\n
running_means.nc(): "
<<
running_means
.
nc
()
<<
"
\n
running_means.nc(): "
<<
running_means
.
nc
()
<<
"
\n
running_
invstds.num_samples(): "
<<
running_invstd
s
.
num_samples
()
<<
"
\n
running_
variances.num_samples(): "
<<
running_variance
s
.
num_samples
()
<<
"
\n
running_
invstds.k(): "
<<
running_invstd
s
.
k
()
<<
"
\n
running_
variances.k(): "
<<
running_variance
s
.
k
()
<<
"
\n
running_
invstds.nr(): "
<<
running_invstd
s
.
nr
()
<<
"
\n
running_
variances.nr(): "
<<
running_variance
s
.
nr
()
<<
"
\n
running_
invstds.nc(): "
<<
running_invstd
s
.
nc
()
<<
"
\n
running_
variances.nc(): "
<<
running_variance
s
.
nc
()
<<
"
\n
src.k(): "
<<
src
.
k
()
<<
"
\n
src.k(): "
<<
src
.
k
()
<<
"
\n
src.nr(): "
<<
src
.
nr
()
<<
"
\n
src.nr(): "
<<
src
.
nr
()
<<
"
\n
src.nc(): "
<<
src
.
nc
()
"
\n
src.nc(): "
<<
src
.
nc
()
...
@@ -392,7 +392,7 @@ namespace dlib
...
@@ -392,7 +392,7 @@ namespace dlib
gamma
.
device
(),
gamma
.
device
(),
beta
.
device
(),
beta
.
device
(),
running_means
.
device
(),
running_means
.
device
(),
running_
invstd
s
.
device
(),
running_
variance
s
.
device
(),
dlib
::
tt
::
BATCH_NORM_EPS
));
dlib
::
tt
::
BATCH_NORM_EPS
));
}
}
...
@@ -402,7 +402,7 @@ namespace dlib
...
@@ -402,7 +402,7 @@ namespace dlib
resizable_tensor
&
invstds
,
resizable_tensor
&
invstds
,
const
double
averaging_factor
,
const
double
averaging_factor
,
resizable_tensor
&
running_means
,
resizable_tensor
&
running_means
,
resizable_tensor
&
running_
invstd
s
,
resizable_tensor
&
running_
variance
s
,
const
tensor
&
src
,
const
tensor
&
src
,
const
tensor
&
gamma
,
const
tensor
&
gamma
,
const
tensor
&
beta
const
tensor
&
beta
...
@@ -410,7 +410,7 @@ namespace dlib
...
@@ -410,7 +410,7 @@ namespace dlib
{
{
DLIB_CASSERT
(
0
<=
averaging_factor
&&
averaging_factor
<=
1
,
"averaging_factor: "
<<
averaging_factor
);
DLIB_CASSERT
(
0
<=
averaging_factor
&&
averaging_factor
<=
1
,
"averaging_factor: "
<<
averaging_factor
);
DLIB_CASSERT
(
averaging_factor
==
1
||
have_same_dimensions
(
running_means
,
means
),
""
);
DLIB_CASSERT
(
averaging_factor
==
1
||
have_same_dimensions
(
running_means
,
means
),
""
);
DLIB_CASSERT
(
averaging_factor
==
1
||
have_same_dimensions
(
running_
invstd
s
,
invstds
),
""
);
DLIB_CASSERT
(
averaging_factor
==
1
||
have_same_dimensions
(
running_
variance
s
,
invstds
),
""
);
DLIB_CASSERT
(
DLIB_CASSERT
(
src
.
num_samples
()
>
1
&&
src
.
num_samples
()
>
1
&&
gamma
.
num_samples
()
==
1
&&
gamma
.
num_samples
()
==
1
&&
...
@@ -438,7 +438,7 @@ namespace dlib
...
@@ -438,7 +438,7 @@ namespace dlib
means
.
set_size
(
1
,
src
.
k
(),
src
.
nr
(),
src
.
nc
());
means
.
set_size
(
1
,
src
.
k
(),
src
.
nr
(),
src
.
nc
());
invstds
.
copy_size
(
means
);
invstds
.
copy_size
(
means
);
running_means
.
copy_size
(
means
);
running_means
.
copy_size
(
means
);
running_
invstd
s
.
copy_size
(
means
);
running_
variance
s
.
copy_size
(
means
);
CHECK_CUDNN
(
cudnnBatchNormalizationForwardTraining
(
CHECK_CUDNN
(
cudnnBatchNormalizationForwardTraining
(
context
(),
context
(),
...
@@ -454,7 +454,7 @@ namespace dlib
...
@@ -454,7 +454,7 @@ namespace dlib
beta
.
device
(),
beta
.
device
(),
averaging_factor
,
averaging_factor
,
running_means
.
device
(),
running_means
.
device
(),
running_
invstd
s
.
device
(),
running_
variance
s
.
device
(),
dlib
::
tt
::
BATCH_NORM_EPS
,
dlib
::
tt
::
BATCH_NORM_EPS
,
means
.
device
(),
means
.
device
(),
invstds
.
device
()));
invstds
.
device
()));
...
@@ -516,7 +516,7 @@ namespace dlib
...
@@ -516,7 +516,7 @@ namespace dlib
const
tensor
&
gamma
,
const
tensor
&
gamma
,
const
tensor
&
beta
,
const
tensor
&
beta
,
const
tensor
&
running_means
,
const
tensor
&
running_means
,
const
tensor
&
running_
invstd
s
const
tensor
&
running_
variance
s
)
)
{
{
DLIB_CASSERT
(
DLIB_CASSERT
(
...
@@ -526,7 +526,7 @@ namespace dlib
...
@@ -526,7 +526,7 @@ namespace dlib
gamma
.
k
()
==
src
.
k
()
&&
gamma
.
k
()
==
src
.
k
()
&&
have_same_dimensions
(
gamma
,
beta
)
&&
have_same_dimensions
(
gamma
,
beta
)
&&
have_same_dimensions
(
gamma
,
running_means
)
&&
have_same_dimensions
(
gamma
,
running_means
)
&&
have_same_dimensions
(
gamma
,
running_
invstd
s
),
have_same_dimensions
(
gamma
,
running_
variance
s
),
"
\n
gamma.num_samples(): "
<<
gamma
.
num_samples
()
<<
"
\n
gamma.num_samples(): "
<<
gamma
.
num_samples
()
<<
"
\n
gamma.k(): "
<<
gamma
.
k
()
<<
"
\n
gamma.k(): "
<<
gamma
.
k
()
<<
"
\n
gamma.nr(): "
<<
gamma
.
nr
()
<<
"
\n
gamma.nr(): "
<<
gamma
.
nr
()
<<
...
@@ -539,10 +539,10 @@ namespace dlib
...
@@ -539,10 +539,10 @@ namespace dlib
"
\n
running_means.k(): "
<<
running_means
.
k
()
<<
"
\n
running_means.k(): "
<<
running_means
.
k
()
<<
"
\n
running_means.nr(): "
<<
running_means
.
nr
()
<<
"
\n
running_means.nr(): "
<<
running_means
.
nr
()
<<
"
\n
running_means.nc(): "
<<
running_means
.
nc
()
<<
"
\n
running_means.nc(): "
<<
running_means
.
nc
()
<<
"
\n
running_
invstds.num_samples(): "
<<
running_invstd
s
.
num_samples
()
<<
"
\n
running_
variances.num_samples(): "
<<
running_variance
s
.
num_samples
()
<<
"
\n
running_
invstds.k(): "
<<
running_invstd
s
.
k
()
<<
"
\n
running_
variances.k(): "
<<
running_variance
s
.
k
()
<<
"
\n
running_
invstds.nr(): "
<<
running_invstd
s
.
nr
()
<<
"
\n
running_
variances.nr(): "
<<
running_variance
s
.
nr
()
<<
"
\n
running_
invstds.nc(): "
<<
running_invstd
s
.
nc
()
<<
"
\n
running_
variances.nc(): "
<<
running_variance
s
.
nc
()
<<
"
\n
src.k(): "
<<
src
.
k
()
<<
"
\n
src.k(): "
<<
src
.
k
()
<<
"
\n
src.nr(): "
<<
src
.
nr
()
<<
"
\n
src.nr(): "
<<
src
.
nr
()
<<
"
\n
src.nc(): "
<<
src
.
nc
()
"
\n
src.nc(): "
<<
src
.
nc
()
...
@@ -565,7 +565,7 @@ namespace dlib
...
@@ -565,7 +565,7 @@ namespace dlib
gamma
.
device
(),
gamma
.
device
(),
beta
.
device
(),
beta
.
device
(),
running_means
.
device
(),
running_means
.
device
(),
running_
invstd
s
.
device
(),
running_
variance
s
.
device
(),
dlib
::
tt
::
BATCH_NORM_EPS
));
dlib
::
tt
::
BATCH_NORM_EPS
));
}
}
...
@@ -575,7 +575,7 @@ namespace dlib
...
@@ -575,7 +575,7 @@ namespace dlib
resizable_tensor
&
invstds
,
resizable_tensor
&
invstds
,
const
double
averaging_factor
,
const
double
averaging_factor
,
resizable_tensor
&
running_means
,
resizable_tensor
&
running_means
,
resizable_tensor
&
running_
invstd
s
,
resizable_tensor
&
running_
variance
s
,
const
tensor
&
src
,
const
tensor
&
src
,
const
tensor
&
gamma
,
const
tensor
&
gamma
,
const
tensor
&
beta
const
tensor
&
beta
...
@@ -583,7 +583,7 @@ namespace dlib
...
@@ -583,7 +583,7 @@ namespace dlib
{
{
DLIB_CASSERT
(
0
<=
averaging_factor
&&
averaging_factor
<=
1
,
"averaging_factor: "
<<
averaging_factor
);
DLIB_CASSERT
(
0
<=
averaging_factor
&&
averaging_factor
<=
1
,
"averaging_factor: "
<<
averaging_factor
);
DLIB_CASSERT
(
averaging_factor
==
1
||
have_same_dimensions
(
running_means
,
means
),
""
);
DLIB_CASSERT
(
averaging_factor
==
1
||
have_same_dimensions
(
running_means
,
means
),
""
);
DLIB_CASSERT
(
averaging_factor
==
1
||
have_same_dimensions
(
running_
invstd
s
,
invstds
),
""
);
DLIB_CASSERT
(
averaging_factor
==
1
||
have_same_dimensions
(
running_
variance
s
,
invstds
),
""
);
DLIB_CASSERT
(
DLIB_CASSERT
(
src
.
num_samples
()
>
1
&&
src
.
num_samples
()
>
1
&&
gamma
.
num_samples
()
==
1
&&
gamma
.
num_samples
()
==
1
&&
...
@@ -612,7 +612,7 @@ namespace dlib
...
@@ -612,7 +612,7 @@ namespace dlib
means
.
set_size
(
1
,
src
.
k
());
means
.
set_size
(
1
,
src
.
k
());
invstds
.
copy_size
(
means
);
invstds
.
copy_size
(
means
);
running_means
.
copy_size
(
means
);
running_means
.
copy_size
(
means
);
running_
invstd
s
.
copy_size
(
means
);
running_
variance
s
.
copy_size
(
means
);
CHECK_CUDNN
(
cudnnBatchNormalizationForwardTraining
(
CHECK_CUDNN
(
cudnnBatchNormalizationForwardTraining
(
context
(),
context
(),
...
@@ -628,7 +628,7 @@ namespace dlib
...
@@ -628,7 +628,7 @@ namespace dlib
beta
.
device
(),
beta
.
device
(),
averaging_factor
,
averaging_factor
,
running_means
.
device
(),
running_means
.
device
(),
running_
invstd
s
.
device
(),
running_
variance
s
.
device
(),
dlib
::
tt
::
BATCH_NORM_EPS
,
dlib
::
tt
::
BATCH_NORM_EPS
,
means
.
device
(),
means
.
device
(),
invstds
.
device
()));
invstds
.
device
()));
...
...
dlib/dnn/cudnn_dlibapi.h
View file @
6acddf99
...
@@ -140,7 +140,7 @@ namespace dlib
...
@@ -140,7 +140,7 @@ namespace dlib
const
tensor
&
gamma
,
const
tensor
&
gamma
,
const
tensor
&
beta
,
const
tensor
&
beta
,
const
tensor
&
running_means
,
const
tensor
&
running_means
,
const
tensor
&
running_
invstd
s
const
tensor
&
running_
variance
s
);
);
void
batch_normalize
(
void
batch_normalize
(
...
@@ -149,7 +149,7 @@ namespace dlib
...
@@ -149,7 +149,7 @@ namespace dlib
resizable_tensor
&
invstds
,
resizable_tensor
&
invstds
,
const
double
averaging_factor
,
const
double
averaging_factor
,
resizable_tensor
&
running_means
,
resizable_tensor
&
running_means
,
resizable_tensor
&
running_
invstd
s
,
resizable_tensor
&
running_
variance
s
,
const
tensor
&
src
,
const
tensor
&
src
,
const
tensor
&
gamma
,
const
tensor
&
gamma
,
const
tensor
&
beta
const
tensor
&
beta
...
@@ -174,7 +174,7 @@ namespace dlib
...
@@ -174,7 +174,7 @@ namespace dlib
const
tensor
&
gamma
,
const
tensor
&
gamma
,
const
tensor
&
beta
,
const
tensor
&
beta
,
const
tensor
&
running_means
,
const
tensor
&
running_means
,
const
tensor
&
running_
invstd
s
const
tensor
&
running_
variance
s
);
);
void
batch_normalize_conv
(
void
batch_normalize_conv
(
...
@@ -183,7 +183,7 @@ namespace dlib
...
@@ -183,7 +183,7 @@ namespace dlib
resizable_tensor
&
invstds
,
resizable_tensor
&
invstds
,
const
double
averaging_factor
,
const
double
averaging_factor
,
resizable_tensor
&
running_means
,
resizable_tensor
&
running_means
,
resizable_tensor
&
running_
invstd
s
,
resizable_tensor
&
running_
variance
s
,
const
tensor
&
src
,
const
tensor
&
src
,
const
tensor
&
gamma
,
const
tensor
&
gamma
,
const
tensor
&
beta
const
tensor
&
beta
...
...
dlib/dnn/layers.h
View file @
6acddf99
...
@@ -453,9 +453,9 @@ namespace dlib
...
@@ -453,9 +453,9 @@ namespace dlib
beta
(
params
,
gamma
.
size
())
=
0
;
beta
(
params
,
gamma
.
size
())
=
0
;
running_means
.
copy_size
(
gamma
(
params
,
0
));
running_means
.
copy_size
(
gamma
(
params
,
0
));
running_
invstd
s
.
copy_size
(
gamma
(
params
,
0
));
running_
variance
s
.
copy_size
(
gamma
(
params
,
0
));
running_means
=
0
;
running_means
=
0
;
running_
invstd
s
=
1
;
running_
variance
s
=
1
;
num_updates
=
0
;
num_updates
=
0
;
}
}
...
@@ -470,16 +470,16 @@ namespace dlib
...
@@ -470,16 +470,16 @@ namespace dlib
if
(
num_updates
<
running_stats_window_size
)
if
(
num_updates
<
running_stats_window_size
)
++
num_updates
;
++
num_updates
;
if
(
mode
==
FC_MODE
)
if
(
mode
==
FC_MODE
)
tt
::
batch_normalize
(
output
,
means
,
invstds
,
decay
,
running_means
,
running_
invstd
s
,
sub
.
get_output
(),
g
,
b
);
tt
::
batch_normalize
(
output
,
means
,
invstds
,
decay
,
running_means
,
running_
variance
s
,
sub
.
get_output
(),
g
,
b
);
else
else
tt
::
batch_normalize_conv
(
output
,
means
,
invstds
,
decay
,
running_means
,
running_
invstd
s
,
sub
.
get_output
(),
g
,
b
);
tt
::
batch_normalize_conv
(
output
,
means
,
invstds
,
decay
,
running_means
,
running_
variance
s
,
sub
.
get_output
(),
g
,
b
);
}
}
else
// we are running in testing mode so we just linearly scale the input tensor.
else
// we are running in testing mode so we just linearly scale the input tensor.
{
{
if
(
mode
==
FC_MODE
)
if
(
mode
==
FC_MODE
)
tt
::
batch_normalize_inference
(
output
,
sub
.
get_output
(),
g
,
b
,
running_means
,
running_
invstd
s
);
tt
::
batch_normalize_inference
(
output
,
sub
.
get_output
(),
g
,
b
,
running_means
,
running_
variance
s
);
else
else
tt
::
batch_normalize_conv_inference
(
output
,
sub
.
get_output
(),
g
,
b
,
running_means
,
running_
invstd
s
);
tt
::
batch_normalize_conv_inference
(
output
,
sub
.
get_output
(),
g
,
b
,
running_means
,
running_
variance
s
);
}
}
}
}
...
@@ -510,7 +510,7 @@ namespace dlib
...
@@ -510,7 +510,7 @@ namespace dlib
serialize
(
item
.
means
,
out
);
serialize
(
item
.
means
,
out
);
serialize
(
item
.
invstds
,
out
);
serialize
(
item
.
invstds
,
out
);
serialize
(
item
.
running_means
,
out
);
serialize
(
item
.
running_means
,
out
);
serialize
(
item
.
running_
invstd
s
,
out
);
serialize
(
item
.
running_
variance
s
,
out
);
serialize
(
item
.
num_updates
,
out
);
serialize
(
item
.
num_updates
,
out
);
serialize
(
item
.
running_stats_window_size
,
out
);
serialize
(
item
.
running_stats_window_size
,
out
);
}
}
...
@@ -539,7 +539,7 @@ namespace dlib
...
@@ -539,7 +539,7 @@ namespace dlib
deserialize
(
item
.
means
,
in
);
deserialize
(
item
.
means
,
in
);
deserialize
(
item
.
invstds
,
in
);
deserialize
(
item
.
invstds
,
in
);
deserialize
(
item
.
running_means
,
in
);
deserialize
(
item
.
running_means
,
in
);
deserialize
(
item
.
running_
invstd
s
,
in
);
deserialize
(
item
.
running_
variance
s
,
in
);
deserialize
(
item
.
num_updates
,
in
);
deserialize
(
item
.
num_updates
,
in
);
deserialize
(
item
.
running_stats_window_size
,
in
);
deserialize
(
item
.
running_stats_window_size
,
in
);
...
@@ -551,9 +551,9 @@ namespace dlib
...
@@ -551,9 +551,9 @@ namespace dlib
deserialize
(
_mode
,
in
);
deserialize
(
_mode
,
in
);
if
(
mode
!=
(
layer_mode
)
_mode
)
throw
serialization_error
(
"Wrong mode found while deserializing dlib::bn_"
);
if
(
mode
!=
(
layer_mode
)
_mode
)
throw
serialization_error
(
"Wrong mode found while deserializing dlib::bn_"
);
// We also need to flip the running_
invstd
s around since the previous
// We also need to flip the running_
variance
s around since the previous
// format saved the inverse standard deviations instead of variances.
// format saved the inverse standard deviations instead of variances.
item
.
running_
invstds
=
1
.
0
f
/
squared
(
mat
(
item
.
running_invstd
s
))
-
tt
::
BATCH_NORM_EPS
;
item
.
running_
variances
=
1
.
0
f
/
squared
(
mat
(
item
.
running_variance
s
))
-
tt
::
BATCH_NORM_EPS
;
}
}
}
}
...
@@ -564,7 +564,7 @@ namespace dlib
...
@@ -564,7 +564,7 @@ namespace dlib
resizable_tensor
params
;
resizable_tensor
params
;
alias_tensor
gamma
,
beta
;
alias_tensor
gamma
,
beta
;
resizable_tensor
means
,
running_means
;
resizable_tensor
means
,
running_means
;
resizable_tensor
invstds
,
running_
invstd
s
;
resizable_tensor
invstds
,
running_
variance
s
;
unsigned
long
num_updates
;
unsigned
long
num_updates
;
unsigned
long
running_stats_window_size
;
unsigned
long
running_stats_window_size
;
};
};
...
@@ -911,7 +911,7 @@ namespace dlib
...
@@ -911,7 +911,7 @@ namespace dlib
auto
sg
=
gamma
(
temp
,
0
);
auto
sg
=
gamma
(
temp
,
0
);
auto
sb
=
beta
(
temp
,
gamma
.
size
());
auto
sb
=
beta
(
temp
,
gamma
.
size
());
g
=
pointwise_multiply
(
mat
(
sg
),
1
.
0
f
/
sqrt
(
mat
(
item
.
running_
invstd
s
)
+
tt
::
BATCH_NORM_EPS
));
g
=
pointwise_multiply
(
mat
(
sg
),
1
.
0
f
/
sqrt
(
mat
(
item
.
running_
variance
s
)
+
tt
::
BATCH_NORM_EPS
));
b
=
mat
(
sb
)
-
pointwise_multiply
(
mat
(
g
),
mat
(
item
.
running_means
));
b
=
mat
(
sb
)
-
pointwise_multiply
(
mat
(
g
),
mat
(
item
.
running_means
));
}
}
...
...
dlib/dnn/tensor_tools.cpp
View file @
6acddf99
...
@@ -274,13 +274,13 @@ namespace dlib { namespace tt
...
@@ -274,13 +274,13 @@ namespace dlib { namespace tt
const
tensor
&
gamma
,
const
tensor
&
gamma
,
const
tensor
&
beta
,
const
tensor
&
beta
,
const
tensor
&
running_means
,
const
tensor
&
running_means
,
const
tensor
&
running_
invstd
s
const
tensor
&
running_
variance
s
)
)
{
{
#ifdef DLIB_USE_CUDA
#ifdef DLIB_USE_CUDA
cuda
::
batch_normalize_inference
(
dest
,
src
,
gamma
,
beta
,
running_means
,
running_
invstd
s
);
cuda
::
batch_normalize_inference
(
dest
,
src
,
gamma
,
beta
,
running_means
,
running_
variance
s
);
#else
#else
cpu
::
batch_normalize_inference
(
dest
,
src
,
gamma
,
beta
,
running_means
,
running_
invstd
s
);
cpu
::
batch_normalize_inference
(
dest
,
src
,
gamma
,
beta
,
running_means
,
running_
variance
s
);
#endif
#endif
}
}
...
@@ -290,16 +290,16 @@ namespace dlib { namespace tt
...
@@ -290,16 +290,16 @@ namespace dlib { namespace tt
resizable_tensor
&
vars
,
resizable_tensor
&
vars
,
const
double
averaging_factor
,
const
double
averaging_factor
,
resizable_tensor
&
running_means
,
resizable_tensor
&
running_means
,
resizable_tensor
&
running_
invstd
s
,
resizable_tensor
&
running_
variance
s
,
const
tensor
&
src
,
const
tensor
&
src
,
const
tensor
&
gamma
,
const
tensor
&
gamma
,
const
tensor
&
beta
const
tensor
&
beta
)
)
{
{
#ifdef DLIB_USE_CUDA
#ifdef DLIB_USE_CUDA
cuda
::
batch_normalize
(
dest
,
means
,
vars
,
averaging_factor
,
running_means
,
running_
invstd
s
,
src
,
gamma
,
beta
);
cuda
::
batch_normalize
(
dest
,
means
,
vars
,
averaging_factor
,
running_means
,
running_
variance
s
,
src
,
gamma
,
beta
);
#else
#else
cpu
::
batch_normalize
(
dest
,
means
,
vars
,
averaging_factor
,
running_means
,
running_
invstd
s
,
src
,
gamma
,
beta
);
cpu
::
batch_normalize
(
dest
,
means
,
vars
,
averaging_factor
,
running_means
,
running_
variance
s
,
src
,
gamma
,
beta
);
#endif
#endif
}
}
...
@@ -330,13 +330,13 @@ namespace dlib { namespace tt
...
@@ -330,13 +330,13 @@ namespace dlib { namespace tt
const
tensor
&
gamma
,
const
tensor
&
gamma
,
const
tensor
&
beta
,
const
tensor
&
beta
,
const
tensor
&
running_means
,
const
tensor
&
running_means
,
const
tensor
&
running_
invstd
s
const
tensor
&
running_
variance
s
)
)
{
{
#ifdef DLIB_USE_CUDA
#ifdef DLIB_USE_CUDA
cuda
::
batch_normalize_conv_inference
(
dest
,
src
,
gamma
,
beta
,
running_means
,
running_
invstd
s
);
cuda
::
batch_normalize_conv_inference
(
dest
,
src
,
gamma
,
beta
,
running_means
,
running_
variance
s
);
#else
#else
cpu
::
batch_normalize_conv_inference
(
dest
,
src
,
gamma
,
beta
,
running_means
,
running_
invstd
s
);
cpu
::
batch_normalize_conv_inference
(
dest
,
src
,
gamma
,
beta
,
running_means
,
running_
variance
s
);
#endif
#endif
}
}
...
@@ -346,16 +346,16 @@ namespace dlib { namespace tt
...
@@ -346,16 +346,16 @@ namespace dlib { namespace tt
resizable_tensor
&
vars
,
resizable_tensor
&
vars
,
const
double
averaging_factor
,
const
double
averaging_factor
,
resizable_tensor
&
running_means
,
resizable_tensor
&
running_means
,
resizable_tensor
&
running_
invstd
s
,
resizable_tensor
&
running_
variance
s
,
const
tensor
&
src
,
const
tensor
&
src
,
const
tensor
&
gamma
,
const
tensor
&
gamma
,
const
tensor
&
beta
const
tensor
&
beta
)
)
{
{
#ifdef DLIB_USE_CUDA
#ifdef DLIB_USE_CUDA
cuda
::
batch_normalize_conv
(
dest
,
means
,
vars
,
averaging_factor
,
running_means
,
running_
invstd
s
,
src
,
gamma
,
beta
);
cuda
::
batch_normalize_conv
(
dest
,
means
,
vars
,
averaging_factor
,
running_means
,
running_
variance
s
,
src
,
gamma
,
beta
);
#else
#else
cpu
::
batch_normalize_conv
(
dest
,
means
,
vars
,
averaging_factor
,
running_means
,
running_
invstd
s
,
src
,
gamma
,
beta
);
cpu
::
batch_normalize_conv
(
dest
,
means
,
vars
,
averaging_factor
,
running_means
,
running_
variance
s
,
src
,
gamma
,
beta
);
#endif
#endif
}
}
...
...
dlib/dnn/tensor_tools.h
View file @
6acddf99
...
@@ -294,7 +294,7 @@ namespace dlib { namespace tt
...
@@ -294,7 +294,7 @@ namespace dlib { namespace tt
const
tensor
&
gamma
,
const
tensor
&
gamma
,
const
tensor
&
beta
,
const
tensor
&
beta
,
const
tensor
&
running_means
,
const
tensor
&
running_means
,
const
tensor
&
running_
invstd
s
const
tensor
&
running_
variance
s
);
);
/*!
/*!
requires
requires
...
@@ -304,12 +304,12 @@ namespace dlib { namespace tt
...
@@ -304,12 +304,12 @@ namespace dlib { namespace tt
- gamma.k() == src.k()
- gamma.k() == src.k()
- have_same_dimensions(gamma, beta)
- have_same_dimensions(gamma, beta)
- have_same_dimensions(gamma, running_means)
- have_same_dimensions(gamma, running_means)
- have_same_dimensions(gamma, running_
invstd
s)
- have_same_dimensions(gamma, running_
variance
s)
ensures
ensures
-
Just linearly transforms src as a call to batch_normalize() would if the resulting
-
Linearly transforms src as a call to batch_normalize() would if src had means
means and invstds were running_means and running_invstds. That is, this function
and variances as given by running_means and running_variances. That is, this
performs:
function
performs:
dest = gamma*(src-running_means)
*running_invstds
+ beta
dest = gamma*(src-running_means)
/sqrt(running_variances+BATCH_NORM_EPS)
+ beta
Note that it does it in a pointwise fashion over the samples in src.
Note that it does it in a pointwise fashion over the samples in src.
!*/
!*/
...
@@ -319,7 +319,7 @@ namespace dlib { namespace tt
...
@@ -319,7 +319,7 @@ namespace dlib { namespace tt
resizable_tensor
&
invstds
,
resizable_tensor
&
invstds
,
const
double
averaging_factor
,
const
double
averaging_factor
,
resizable_tensor
&
running_means
,
resizable_tensor
&
running_means
,
resizable_tensor
&
running_
invstd
s
,
resizable_tensor
&
running_
variance
s
,
const
tensor
&
src
,
const
tensor
&
src
,
const
tensor
&
gamma
,
const
tensor
&
gamma
,
const
tensor
&
beta
const
tensor
&
beta
...
@@ -335,7 +335,7 @@ namespace dlib { namespace tt
...
@@ -335,7 +335,7 @@ namespace dlib { namespace tt
- 0 <= averaging_factor <= 1
- 0 <= averaging_factor <= 1
- if (averaging_factor != 1)
- if (averaging_factor != 1)
- have_same_dimensions(running_means, means) == true
- have_same_dimensions(running_means, means) == true
- have_same_dimensions(running_
invstd
s, invstds) == true
- have_same_dimensions(running_
variance
s, invstds) == true
ensures
ensures
- have_same_dimensions(#dest, src) == true
- have_same_dimensions(#dest, src) == true
- #means.num_samples() == 1
- #means.num_samples() == 1
...
@@ -347,7 +347,7 @@ namespace dlib { namespace tt
...
@@ -347,7 +347,7 @@ namespace dlib { namespace tt
- #means == the mean values of the contents of src.
- #means == the mean values of the contents of src.
- #invstds == 1/(the standard deviation values of the contents of src).
- #invstds == 1/(the standard deviation values of the contents of src).
- #running_means = (1-averaging_factor)*mat(#running_means) + averaging_factor*mat(#means);
- #running_means = (1-averaging_factor)*mat(#running_means) + averaging_factor*mat(#means);
- #running_
invstds = (1-averaging_factor)*mat(#running_invstds) + averaging_factor*mat(#invstds
);
- #running_
variances = (1-averaging_factor)*mat(#running_variances) + averaging_factor*(variance of contents of src
);
!*/
!*/
void
batch_normalize_gradient
(
void
batch_normalize_gradient
(
...
@@ -391,7 +391,7 @@ namespace dlib { namespace tt
...
@@ -391,7 +391,7 @@ namespace dlib { namespace tt
const
tensor
&
gamma
,
const
tensor
&
gamma
,
const
tensor
&
beta
,
const
tensor
&
beta
,
const
tensor
&
running_means
,
const
tensor
&
running_means
,
const
tensor
&
running_
invstd
s
const
tensor
&
running_
variance
s
);
);
/*!
/*!
requires
requires
...
@@ -401,13 +401,13 @@ namespace dlib { namespace tt
...
@@ -401,13 +401,13 @@ namespace dlib { namespace tt
- gamma.k() == src.k()
- gamma.k() == src.k()
- have_same_dimensions(gamma, beta)
- have_same_dimensions(gamma, beta)
- have_same_dimensions(gamma, running_means)
- have_same_dimensions(gamma, running_means)
- have_same_dimensions(gamma, running_
invstd
s)
- have_same_dimensions(gamma, running_
variance
s)
ensures
ensures
-
Just linearly transforms src as a call to batch_normalize_conv() would if the resulting
-
Linearly transforms src as a call to batch_normalize_conv() would if src had
means and
invstds were running_means and running_invstds. That is, this function
means and
variances as given by running_means and running_variances. That
performs:
is, this function
performs:
dest = gamma*(src-running_means)
*running_invstds
+ beta
dest = gamma*(src-running_means)
/sqrt(running_variances+BATCH_NORM_EPS)
+ beta
Note that it does
it
in a pointwise fashion over the samples, rows, and
Note that it does
this
in a pointwise fashion over the samples, rows, and
columns in src.
columns in src.
!*/
!*/
...
@@ -417,7 +417,7 @@ namespace dlib { namespace tt
...
@@ -417,7 +417,7 @@ namespace dlib { namespace tt
resizable_tensor
&
invstds
,
resizable_tensor
&
invstds
,
const
double
averaging_factor
,
const
double
averaging_factor
,
resizable_tensor
&
running_means
,
resizable_tensor
&
running_means
,
resizable_tensor
&
running_
invstd
s
,
resizable_tensor
&
running_
variance
s
,
const
tensor
&
src
,
const
tensor
&
src
,
const
tensor
&
gamma
,
const
tensor
&
gamma
,
const
tensor
&
beta
const
tensor
&
beta
...
@@ -431,7 +431,7 @@ namespace dlib { namespace tt
...
@@ -431,7 +431,7 @@ namespace dlib { namespace tt
- 0 <= averaging_factor <= 1
- 0 <= averaging_factor <= 1
- if (averaging_factor != 1)
- if (averaging_factor != 1)
- have_same_dimensions(running_means, means) == true
- have_same_dimensions(running_means, means) == true
- have_same_dimensions(running_
invstd
s, invstds) == true
- have_same_dimensions(running_
variance
s, invstds) == true
ensures
ensures
- have_same_dimensions(#dest, src) == true
- have_same_dimensions(#dest, src) == true
- #means.num_samples()==means.nr()==means.nc() == 1
- #means.num_samples()==means.nr()==means.nc() == 1
...
@@ -441,7 +441,7 @@ namespace dlib { namespace tt
...
@@ -441,7 +441,7 @@ namespace dlib { namespace tt
- #means == the mean values of the contents of src.
- #means == the mean values of the contents of src.
- #invstds == 1/(the standard deviation values of the contents of src).
- #invstds == 1/(the standard deviation values of the contents of src).
- #running_means = (1-averaging_factor)*mat(#running_means) + averaging_factor*mat(#means);
- #running_means = (1-averaging_factor)*mat(#running_means) + averaging_factor*mat(#means);
- #running_
invstds = (1-averaging_factor)*mat(#running_invstds) + averaging_factor*mat(#invstds
);
- #running_
variances = (1-averaging_factor)*mat(#running_variances) + averaging_factor*(variance of contents of src
);
!*/
!*/
void
batch_normalize_conv_gradient
(
void
batch_normalize_conv_gradient
(
...
...
dlib/test/dnn.cpp
View file @
6acddf99
...
@@ -164,12 +164,12 @@ namespace
...
@@ -164,12 +164,12 @@ namespace
beta
=
0
;
beta
=
0
;
resizable_tensor
running_means
;
resizable_tensor
running_means
;
resizable_tensor
running_
invstd
s
;
resizable_tensor
running_
variance
s
;
batch_normalize
(
dest
,
means
,
vars
,
1
,
running_means
,
running_
invstd
s
,
src
,
gamma
,
beta
);
batch_normalize
(
dest
,
means
,
vars
,
1
,
running_means
,
running_
variance
s
,
src
,
gamma
,
beta
);
const
double
scale
=
(
src
.
num_samples
())
/
(
src
.
num_samples
()
-
1.0
);
const
double
scale
=
(
src
.
num_samples
())
/
(
src
.
num_samples
()
-
1.0
);
// Turn back into biased variance estimate because that's how batch_normalize() works, so if we want to match it this is necessary.
// Turn back into biased variance estimate because that's how batch_normalize() works, so if we want to match it this is necessary.
running_
invstds
=
mat
(
running_invstd
s
)
/
scale
;
running_
variances
=
mat
(
running_variance
s
)
/
scale
;
batch_normalize_inference
(
dest2
,
src
,
gamma
,
beta
,
running_means
,
running_
invstd
s
);
batch_normalize_inference
(
dest2
,
src
,
gamma
,
beta
,
running_means
,
running_
variance
s
);
DLIB_TEST_MSG
(
max
(
abs
(
mat
(
dest2
)
-
mat
(
dest
)))
<
1e-5
,
max
(
abs
(
mat
(
dest2
)
-
mat
(
dest
))));
DLIB_TEST_MSG
(
max
(
abs
(
mat
(
dest2
)
-
mat
(
dest
)))
<
1e-5
,
max
(
abs
(
mat
(
dest2
)
-
mat
(
dest
))));
...
@@ -177,7 +177,7 @@ namespace
...
@@ -177,7 +177,7 @@ namespace
auto
f
=
[
&
](
float
eps
)
{
auto
f
=
[
&
](
float
eps
)
{
const
float
old
=
src
.
host
()[
idx
];
const
float
old
=
src
.
host
()[
idx
];
src
.
host
()[
idx
]
+=
eps
;
src
.
host
()[
idx
]
+=
eps
;
batch_normalize
(
dest
,
means
,
vars
,
1
,
running_means
,
running_
invstd
s
,
src
,
gamma
,
beta
);
batch_normalize
(
dest
,
means
,
vars
,
1
,
running_means
,
running_
variance
s
,
src
,
gamma
,
beta
);
float
result
=
dot
(
gradient_input
,
dest
);
float
result
=
dot
(
gradient_input
,
dest
);
src
.
host
()[
idx
]
=
old
;
src
.
host
()[
idx
]
=
old
;
return
result
;
return
result
;
...
@@ -189,7 +189,7 @@ namespace
...
@@ -189,7 +189,7 @@ namespace
auto
f
=
[
&
](
float
eps
)
{
auto
f
=
[
&
](
float
eps
)
{
const
float
old
=
gamma
.
host
()[
idx
];
const
float
old
=
gamma
.
host
()[
idx
];
gamma
.
host
()[
idx
]
+=
eps
;
gamma
.
host
()[
idx
]
+=
eps
;
batch_normalize
(
dest
,
means
,
vars
,
1
,
running_means
,
running_
invstd
s
,
src
,
gamma
,
beta
);
batch_normalize
(
dest
,
means
,
vars
,
1
,
running_means
,
running_
variance
s
,
src
,
gamma
,
beta
);
float
result
=
dot
(
gradient_input
,
dest
);
float
result
=
dot
(
gradient_input
,
dest
);
gamma
.
host
()[
idx
]
=
old
;
gamma
.
host
()[
idx
]
=
old
;
return
result
;
return
result
;
...
@@ -201,7 +201,7 @@ namespace
...
@@ -201,7 +201,7 @@ namespace
auto
f
=
[
&
](
float
eps
)
{
auto
f
=
[
&
](
float
eps
)
{
const
float
old
=
beta
.
host
()[
idx
];
const
float
old
=
beta
.
host
()[
idx
];
beta
.
host
()[
idx
]
+=
eps
;
beta
.
host
()[
idx
]
+=
eps
;
batch_normalize
(
dest
,
means
,
vars
,
1
,
running_means
,
running_
invstd
s
,
src
,
gamma
,
beta
);
batch_normalize
(
dest
,
means
,
vars
,
1
,
running_means
,
running_
variance
s
,
src
,
gamma
,
beta
);
float
result
=
dot
(
gradient_input
,
dest
);
float
result
=
dot
(
gradient_input
,
dest
);
beta
.
host
()[
idx
]
=
old
;
beta
.
host
()[
idx
]
=
old
;
return
result
;
return
result
;
...
@@ -247,13 +247,13 @@ namespace
...
@@ -247,13 +247,13 @@ namespace
beta
=
0
;
beta
=
0
;
resizable_tensor
running_means
;
resizable_tensor
running_means
;
resizable_tensor
running_
invstd
s
;
resizable_tensor
running_
variance
s
;
batch_normalize_conv
(
dest
,
means
,
vars
,
1
,
running_means
,
running_
invstd
s
,
src
,
gamma
,
beta
);
batch_normalize_conv
(
dest
,
means
,
vars
,
1
,
running_means
,
running_
variance
s
,
src
,
gamma
,
beta
);
const
double
scale
=
(
src
.
num_samples
()
*
src
.
nr
()
*
src
.
nc
())
/
(
src
.
num_samples
()
*
src
.
nr
()
*
src
.
nc
()
-
1.0
);
const
double
scale
=
(
src
.
num_samples
()
*
src
.
nr
()
*
src
.
nc
())
/
(
src
.
num_samples
()
*
src
.
nr
()
*
src
.
nc
()
-
1.0
);
// Turn back into biased variance estimate because that's how
// Turn back into biased variance estimate because that's how
// batch_normalize_conv() works, so if we want to match it this is necessary.
// batch_normalize_conv() works, so if we want to match it this is necessary.
running_
invstds
=
mat
(
running_invstd
s
)
/
scale
;
running_
variances
=
mat
(
running_variance
s
)
/
scale
;
batch_normalize_conv_inference
(
dest2
,
src
,
gamma
,
beta
,
running_means
,
running_
invstd
s
);
batch_normalize_conv_inference
(
dest2
,
src
,
gamma
,
beta
,
running_means
,
running_
variance
s
);
DLIB_TEST
(
max
(
abs
(
mat
(
dest2
)
-
mat
(
dest
)))
<
1e-5
);
DLIB_TEST
(
max
(
abs
(
mat
(
dest2
)
-
mat
(
dest
)))
<
1e-5
);
...
@@ -261,7 +261,7 @@ namespace
...
@@ -261,7 +261,7 @@ namespace
auto
f
=
[
&
](
float
eps
)
{
auto
f
=
[
&
](
float
eps
)
{
const
float
old
=
src
.
host
()[
idx
];
const
float
old
=
src
.
host
()[
idx
];
src
.
host
()[
idx
]
+=
eps
;
src
.
host
()[
idx
]
+=
eps
;
batch_normalize_conv
(
dest
,
means
,
vars
,
1
,
running_means
,
running_
invstd
s
,
src
,
gamma
,
beta
);
batch_normalize_conv
(
dest
,
means
,
vars
,
1
,
running_means
,
running_
variance
s
,
src
,
gamma
,
beta
);
float
result
=
dot
(
gradient_input
,
dest
);
float
result
=
dot
(
gradient_input
,
dest
);
src
.
host
()[
idx
]
=
old
;
src
.
host
()[
idx
]
=
old
;
return
result
;
return
result
;
...
@@ -273,7 +273,7 @@ namespace
...
@@ -273,7 +273,7 @@ namespace
auto
f
=
[
&
](
float
eps
)
{
auto
f
=
[
&
](
float
eps
)
{
const
float
old
=
gamma
.
host
()[
idx
];
const
float
old
=
gamma
.
host
()[
idx
];
gamma
.
host
()[
idx
]
+=
eps
;
gamma
.
host
()[
idx
]
+=
eps
;
batch_normalize_conv
(
dest
,
means
,
vars
,
1
,
running_means
,
running_
invstd
s
,
src
,
gamma
,
beta
);
batch_normalize_conv
(
dest
,
means
,
vars
,
1
,
running_means
,
running_
variance
s
,
src
,
gamma
,
beta
);
float
result
=
dot
(
gradient_input
,
dest
);
float
result
=
dot
(
gradient_input
,
dest
);
gamma
.
host
()[
idx
]
=
old
;
gamma
.
host
()[
idx
]
=
old
;
return
result
;
return
result
;
...
@@ -285,7 +285,7 @@ namespace
...
@@ -285,7 +285,7 @@ namespace
auto
f
=
[
&
](
float
eps
)
{
auto
f
=
[
&
](
float
eps
)
{
const
float
old
=
beta
.
host
()[
idx
];
const
float
old
=
beta
.
host
()[
idx
];
beta
.
host
()[
idx
]
+=
eps
;
beta
.
host
()[
idx
]
+=
eps
;
batch_normalize_conv
(
dest
,
means
,
vars
,
1
,
running_means
,
running_
invstd
s
,
src
,
gamma
,
beta
);
batch_normalize_conv
(
dest
,
means
,
vars
,
1
,
running_means
,
running_
variance
s
,
src
,
gamma
,
beta
);
float
result
=
dot
(
gradient_input
,
dest
);
float
result
=
dot
(
gradient_input
,
dest
);
beta
.
host
()[
idx
]
=
old
;
beta
.
host
()[
idx
]
=
old
;
return
result
;
return
result
;
...
@@ -775,7 +775,7 @@ namespace
...
@@ -775,7 +775,7 @@ namespace
resizable_tensor
means
,
means2
;
resizable_tensor
means
,
means2
;
resizable_tensor
invstds
,
invstds2
;
resizable_tensor
invstds
,
invstds2
;
resizable_tensor
running_means
,
running_means2
;
resizable_tensor
running_means
,
running_means2
;
resizable_tensor
running_
invstds
,
running_invstd
s2
;
resizable_tensor
running_
variances
,
running_variance
s2
;
resizable_tensor
src
(
64
,
20
,
100
,
100
);
resizable_tensor
src
(
64
,
20
,
100
,
100
);
resizable_tensor
gamma
(
1
,
20
,
100
,
100
);
resizable_tensor
gamma
(
1
,
20
,
100
,
100
);
resizable_tensor
beta
(
1
,
20
,
100
,
100
);
resizable_tensor
beta
(
1
,
20
,
100
,
100
);
...
@@ -785,20 +785,20 @@ namespace
...
@@ -785,20 +785,20 @@ namespace
rnd
.
fill_uniform
(
src
);
rnd
.
fill_uniform
(
src
);
cpu
::
batch_normalize
(
dest
,
means
,
invstds
,
1
,
running_means
,
running_
invstd
s
,
src
,
gamma
,
beta
);
cpu
::
batch_normalize
(
dest
,
means
,
invstds
,
1
,
running_means
,
running_
variance
s
,
src
,
gamma
,
beta
);
cuda
::
batch_normalize
(
dest2
,
means2
,
invstds2
,
1
,
running_means2
,
running_
invstd
s2
,
src
,
gamma
,
beta
);
cuda
::
batch_normalize
(
dest2
,
means2
,
invstds2
,
1
,
running_means2
,
running_
variance
s2
,
src
,
gamma
,
beta
);
dlog
<<
LINFO
<<
"dest error: "
<<
max
(
abs
(
mat
(
dest
)
-
mat
(
dest2
)));
dlog
<<
LINFO
<<
"dest error: "
<<
max
(
abs
(
mat
(
dest
)
-
mat
(
dest2
)));
dlog
<<
LINFO
<<
"means error: "
<<
max
(
abs
(
mat
(
means
)
-
mat
(
means2
)));
dlog
<<
LINFO
<<
"means error: "
<<
max
(
abs
(
mat
(
means
)
-
mat
(
means2
)));
dlog
<<
LINFO
<<
"invstds error: "
<<
max
(
abs
(
mat
(
invstds
)
-
mat
(
invstds2
)));
dlog
<<
LINFO
<<
"invstds error: "
<<
max
(
abs
(
mat
(
invstds
)
-
mat
(
invstds2
)));
dlog
<<
LINFO
<<
"running_means error: "
<<
max
(
abs
(
mat
(
running_means
)
-
mat
(
running_means2
)));
dlog
<<
LINFO
<<
"running_means error: "
<<
max
(
abs
(
mat
(
running_means
)
-
mat
(
running_means2
)));
dlog
<<
LINFO
<<
"running_
invstds error: "
<<
max
(
abs
(
mat
(
running_invstds
)
-
mat
(
running_invstd
s2
)));
dlog
<<
LINFO
<<
"running_
variances error: "
<<
max
(
abs
(
mat
(
running_variances
)
-
mat
(
running_variance
s2
)));
DLIB_TEST
(
max
(
abs
(
mat
(
dest
)
-
mat
(
dest2
)))
<
1e-4
);
DLIB_TEST
(
max
(
abs
(
mat
(
dest
)
-
mat
(
dest2
)))
<
1e-4
);
DLIB_TEST
(
max
(
abs
(
mat
(
means
)
-
mat
(
means2
)))
<
1e-4
);
DLIB_TEST
(
max
(
abs
(
mat
(
means
)
-
mat
(
means2
)))
<
1e-4
);
DLIB_TEST
(
max
(
abs
(
mat
(
invstds
)
-
mat
(
invstds2
)))
<
1e-4
);
DLIB_TEST
(
max
(
abs
(
mat
(
invstds
)
-
mat
(
invstds2
)))
<
1e-4
);
DLIB_TEST
(
max
(
abs
(
mat
(
running_means
)
-
mat
(
running_means2
)))
<
1e-4
);
DLIB_TEST
(
max
(
abs
(
mat
(
running_means
)
-
mat
(
running_means2
)))
<
1e-4
);
DLIB_TEST
(
max
(
abs
(
mat
(
running_
invstds
)
-
mat
(
running_invstd
s2
)))
<
1e-4
);
DLIB_TEST
(
max
(
abs
(
mat
(
running_
variances
)
-
mat
(
running_variance
s2
)))
<
1e-4
);
// now check that the gradients match as well
// now check that the gradients match as well
...
@@ -830,7 +830,7 @@ namespace
...
@@ -830,7 +830,7 @@ namespace
resizable_tensor
means
,
means2
;
resizable_tensor
means
,
means2
;
resizable_tensor
invstds
,
invstds2
;
resizable_tensor
invstds
,
invstds2
;
resizable_tensor
running_means
,
running_means2
;
resizable_tensor
running_means
,
running_means2
;
resizable_tensor
running_
invstds
,
running_invstd
s2
;
resizable_tensor
running_
variances
,
running_variance
s2
;
resizable_tensor
src
(
2
,
8
,
10
,
9
);
resizable_tensor
src
(
2
,
8
,
10
,
9
);
resizable_tensor
gamma
(
1
,
8
);
resizable_tensor
gamma
(
1
,
8
);
resizable_tensor
beta
(
1
,
8
);
resizable_tensor
beta
(
1
,
8
);
...
@@ -839,20 +839,20 @@ namespace
...
@@ -839,20 +839,20 @@ namespace
tt
::
tensor_rand
rnd
;
tt
::
tensor_rand
rnd
;
rnd
.
fill_uniform
(
src
);
rnd
.
fill_uniform
(
src
);
cpu
::
batch_normalize_conv
(
dest
,
means
,
invstds
,
1
,
running_means
,
running_
invstd
s
,
src
,
gamma
,
beta
);
cpu
::
batch_normalize_conv
(
dest
,
means
,
invstds
,
1
,
running_means
,
running_
variance
s
,
src
,
gamma
,
beta
);
cuda
::
batch_normalize_conv
(
dest2
,
means2
,
invstds2
,
1
,
running_means2
,
running_
invstd
s2
,
src
,
gamma
,
beta
);
cuda
::
batch_normalize_conv
(
dest2
,
means2
,
invstds2
,
1
,
running_means2
,
running_
variance
s2
,
src
,
gamma
,
beta
);
dlog
<<
LINFO
<<
"dest error: "
<<
max
(
abs
(
mat
(
dest
)
-
mat
(
dest2
)));
dlog
<<
LINFO
<<
"dest error: "
<<
max
(
abs
(
mat
(
dest
)
-
mat
(
dest2
)));
dlog
<<
LINFO
<<
"means error: "
<<
max
(
abs
(
mat
(
means
)
-
mat
(
means2
)));
dlog
<<
LINFO
<<
"means error: "
<<
max
(
abs
(
mat
(
means
)
-
mat
(
means2
)));
dlog
<<
LINFO
<<
"invstds error: "
<<
max
(
abs
(
mat
(
invstds
)
-
mat
(
invstds2
)));
dlog
<<
LINFO
<<
"invstds error: "
<<
max
(
abs
(
mat
(
invstds
)
-
mat
(
invstds2
)));
dlog
<<
LINFO
<<
"running_means error: "
<<
max
(
abs
(
mat
(
running_means
)
-
mat
(
running_means2
)));
dlog
<<
LINFO
<<
"running_means error: "
<<
max
(
abs
(
mat
(
running_means
)
-
mat
(
running_means2
)));
dlog
<<
LINFO
<<
"running_
invstds error: "
<<
max
(
abs
(
mat
(
running_invstds
)
-
mat
(
running_invstd
s2
)));
dlog
<<
LINFO
<<
"running_
variances error: "
<<
max
(
abs
(
mat
(
running_variances
)
-
mat
(
running_variance
s2
)));
DLIB_TEST
(
max
(
abs
(
mat
(
dest
)
-
mat
(
dest2
)))
<
1e-4
);
DLIB_TEST
(
max
(
abs
(
mat
(
dest
)
-
mat
(
dest2
)))
<
1e-4
);
DLIB_TEST
(
max
(
abs
(
mat
(
means
)
-
mat
(
means2
)))
<
1e-4
);
DLIB_TEST
(
max
(
abs
(
mat
(
means
)
-
mat
(
means2
)))
<
1e-4
);
DLIB_TEST
(
max
(
abs
(
mat
(
invstds
)
-
mat
(
invstds2
)))
<
1e-4
);
DLIB_TEST
(
max
(
abs
(
mat
(
invstds
)
-
mat
(
invstds2
)))
<
1e-4
);
DLIB_TEST
(
max
(
abs
(
mat
(
running_means
)
-
mat
(
running_means2
)))
<
1e-4
);
DLIB_TEST
(
max
(
abs
(
mat
(
running_means
)
-
mat
(
running_means2
)))
<
1e-4
);
DLIB_TEST
(
max
(
abs
(
mat
(
running_
invstds
)
-
mat
(
running_invstd
s2
)))
<
1e-4
);
DLIB_TEST
(
max
(
abs
(
mat
(
running_
variances
)
-
mat
(
running_variance
s2
)))
<
1e-4
);
resizable_tensor
gradient_input
;
resizable_tensor
gradient_input
;
resizable_tensor
src_grad
,
gamma_grad
,
beta_grad
;
resizable_tensor
src_grad
,
gamma_grad
,
beta_grad
;
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment