Commit bbeabe45 authored by Davis King's avatar Davis King

Removed the old find_min* functions, updated the regression tests, and

modified any code that used the old functions.

--HG--
extra : convert_revision : svn%3Afdd8eb12-d10e-0410-9acb-85c331704f74/trunk%403187
parent 5cf9b7e1
......@@ -1033,163 +1033,6 @@ namespace dlib
);
}
// ----------------------------------------------------------------------------------------
// ----------------------------------------------------------------------------------------
// ----------------------------------------------------------------------------------------
template <
typename funct,
typename funct_der,
typename T
>
void find_min_quasi_newton (
const funct& f,
const funct_der& der,
T& x,
double min_f,
double min_delta = 1e-7
)
{
// You get an error on this line when you pass in a global function to this function.
// You have to either use a function object or pass a pointer to your global function
// by taking its address using the & operator. (This check is here because gcc 4.0
// has a bug that causes it to silently corrupt return values from functions that
// invoked through a reference)
COMPILE_TIME_ASSERT(is_function<funct>::value == false);
COMPILE_TIME_ASSERT(is_function<funct_der>::value == false);
COMPILE_TIME_ASSERT(is_matrix<T>::value);
DLIB_ASSERT (
min_delta >= 0 && x.nc() == 1,
"\tdouble find_min_quasi_newton()"
<< "\n\tYou have to supply column vectors to this function"
<< "\n\tmin_delta: " << min_delta
<< "\n\tx.nc(): " << x.nc()
);
find_min(
bfgs_search_strategy(),
objective_delta_stop_strategy(min_delta),
f, der, x, min_f
);
}
// ----------------------------------------------------------------------------------------
template <
typename funct,
typename funct_der,
typename T
>
void find_min_conjugate_gradient (
const funct& f,
const funct_der& der,
T& x,
double min_f,
double min_delta = 1e-7
)
{
// You get an error on this line when you pass in a global function to this function.
// You have to either use a function object or pass a pointer to your global function
// by taking its address using the & operator. (This check is here because gcc 4.0
// has a bug that causes it to silently corrupt return values from functions that
// invoked through a reference)
COMPILE_TIME_ASSERT(is_function<funct>::value == false);
COMPILE_TIME_ASSERT(is_function<funct_der>::value == false);
COMPILE_TIME_ASSERT(is_matrix<T>::value);
DLIB_ASSERT (
min_delta >= 0 && x.nc() == 1,
"\tdouble find_min_conjugate_gradient()"
<< "\n\tYou have to supply column vectors to this function"
<< "\n\tmin_delta: " << min_delta
<< "\n\tx.nc(): " << x.nc()
);
find_min(
cg_search_strategy(),
objective_delta_stop_strategy(min_delta),
f, der, x, min_f
);
}
// ----------------------------------------------------------------------------------------
template <
typename funct,
typename T
>
void find_min_quasi_newton2 (
const funct& f,
T& x,
double min_f,
double min_delta = 1e-7,
double derivative_eps = 1e-7
)
{
// You get an error on this line when you pass in a global function to this function.
// You have to either use a function object or pass a pointer to your global function
// by taking its address using the & operator. (This check is here because gcc 4.0
// has a bug that causes it to silently corrupt return values from functions that
// invoked through a reference)
COMPILE_TIME_ASSERT(is_function<funct>::value == false);
COMPILE_TIME_ASSERT(is_matrix<T>::value);
DLIB_ASSERT (
min_delta >= 0 && x.nc() == 1,
"\tdouble find_min_quasi_newton2()"
<< "\n\tYou have to supply column vectors to this function"
<< "\n\tmin_delta: " << min_delta
<< "\n\tx.nc(): " << x.nc()
<< "\n\tderivative_eps: " << derivative_eps
);
find_min_using_approximate_derivatives(
bfgs_search_strategy(),
objective_delta_stop_strategy(min_delta),
f, x, min_f, derivative_eps
);
}
// ----------------------------------------------------------------------------------------
template <
typename funct,
typename T
>
void find_min_conjugate_gradient2 (
const funct& f,
T& x,
double min_f,
double min_delta = 1e-7,
double derivative_eps = 1e-7
)
{
// You get an error on this line when you pass in a global function to this function.
// You have to either use a function object or pass a pointer to your global function
// by taking its address using the & operator. (This check is here because gcc 4.0
// has a bug that causes it to silently corrupt return values from functions that
// invoked through a reference)
COMPILE_TIME_ASSERT(is_function<funct>::value == false);
COMPILE_TIME_ASSERT(is_matrix<T>::value);
DLIB_ASSERT (
min_delta >= 0 && x.nc() == 1 && derivative_eps > 0,
"\tdouble find_min_conjugate_gradient2()"
<< "\n\tYou have to supply column vectors to this function"
<< "\n\tmin_delta: " << min_delta
<< "\n\tx.nc(): " << x.nc()
<< "\n\tderivative_eps: " << derivative_eps
);
find_min_using_approximate_derivatives(
cg_search_strategy(),
objective_delta_stop_strategy(min_delta),
f, x, min_f, derivative_eps
);
}
// ----------------------------------------------------------------------------------------
}
......
......@@ -546,7 +546,9 @@ namespace dlib
// perform the actual optimization
find_min_conjugate_gradient(obj, obj_der, opt_starting_point, 0, eps);
find_min(lbfgs_search_strategy(20),
objective_delta_stop_strategy(eps),
obj, obj_der, opt_starting_point, 0);
// now make sure that the final optimized value is loaded into the beta and
// out_vectors matrices
......
......@@ -10,6 +10,7 @@
#include <vector>
#include "../stl_checked.h"
#include "../array.h"
#include "../rand.h"
#include "tester.h"
......@@ -81,6 +82,24 @@ namespace
return res;
}
// ----------------------------------------------------------------------------------------
// negative of Rosenbrock's function. minimum at (1,1)
double neg_rosen ( const matrix<double,2,1>& x)
{
++total_count;
return -(100*pow(x(1) - x(0)*x(0),2) + pow(1 - x(0),2));
}
matrix<double,2,1> der_neg_rosen ( const matrix<double,2,1>& x)
{
++total_count;
matrix<double,2,1> res;
res(0) = -400*x(0)*(x(1)-x(0)*x(0)) - 2*(1-x(0));
res(1) = 200*(x(1)-x(0)*x(0));
return -res;
}
// ----------------------------------------------------------------------------------------
double simple ( const matrix<double,2,1>& x)
......@@ -124,43 +143,98 @@ namespace
matrix<double,0,1> x(p.nr()), opt(p.nr());
set_all_elements(opt, 0);
if (p.size() < 20)
dlog << LINFO << "testing with apq and the start point: " << trans(p);
else
dlog << LINFO << "testing with apq and a big vector with " << p.size() << " components.";
// don't use bfgs on really large vectors
if (p.size() < 20)
{
total_count = 0;
x = p;
find_min(bfgs_search_strategy(),
objective_delta_stop_strategy(eps),
wrap_function(apq<T>), wrap_function(der_apq<T>), x, minf);
DLIB_TEST_MSG(dlib::equal(x,opt, 1e-5),opt-x);
dlog << LINFO << "find_min() bgfs: got apq in " << total_count;
}
total_count = 0;
x = p;
find_min(lbfgs_search_strategy(10),
objective_delta_stop_strategy(eps),
wrap_function(apq<T>), wrap_function(der_apq<T>), x, minf);
DLIB_TEST_MSG(dlib::equal(x,opt, 1e-5),opt-x);
dlog << LINFO << "find_min() lbgfs-10: got apq in " << total_count;
total_count = 0;
x = p;
find_min_quasi_newton(wrap_function(apq<T>), wrap_function(der_apq<T>), x, minf, eps);
find_min(lbfgs_search_strategy(1),
objective_delta_stop_strategy(eps),
wrap_function(apq<T>), wrap_function(der_apq<T>), x, minf);
DLIB_TEST_MSG(dlib::equal(x,opt, 1e-5),opt-x);
dlog << LINFO << "find_min_quasi_newton got apq in " << total_count;
dlog << LINFO << "find_min() lbgfs-1: got apq in " << total_count;
total_count = 0;
x = p;
find_min_conjugate_gradient(wrap_function(apq<T>), wrap_function(der_apq<T>), x, minf, eps);
find_min(cg_search_strategy(),
objective_delta_stop_strategy(eps),
wrap_function(apq<T>), wrap_function(der_apq<T>), x, minf);
DLIB_TEST_MSG(dlib::equal(x,opt, 1e-5),opt-x);
dlog << LINFO << "find_min_conjugate_gradient got apq in " << total_count;
dlog << LINFO << "find_min() cg: got apq in " << total_count;
// don't do approximate derivative tests if the input point is really long
if (p.size() < 20)
{
total_count = 0;
x = p;
find_min_quasi_newton(wrap_function(apq<T>), derivative(wrap_function(apq<T>)), x, minf, eps);
find_min(bfgs_search_strategy(),
objective_delta_stop_strategy(eps),
wrap_function(apq<T>), derivative(wrap_function(apq<T>)), x, minf);
DLIB_TEST_MSG(dlib::equal(x,opt, 1e-5),opt-x);
dlog << LINFO << "find_min_quasi_newton got apq/noder in " << total_count;
dlog << LINFO << "find_min() bfgs: got apq/noder in " << total_count;
total_count = 0;
x = p;
find_min_conjugate_gradient(wrap_function(apq<T>), derivative(wrap_function(apq<T>)), x, minf, eps);
find_min(cg_search_strategy(),
objective_delta_stop_strategy(eps),
wrap_function(apq<T>), derivative(wrap_function(apq<T>)), x, minf);
DLIB_TEST_MSG(dlib::equal(x,opt, 1e-5),opt-x);
dlog << LINFO << "find_min_conjugate_gradient got apq/noder in " << total_count;
dlog << LINFO << "find_min() cg: got apq/noder in " << total_count;
total_count = 0;
x = p;
find_min_quasi_newton2(wrap_function(apq<T>), x, minf, eps);
find_min_using_approximate_derivatives(bfgs_search_strategy(),
objective_delta_stop_strategy(eps),
wrap_function(apq<T>), x, minf);
DLIB_TEST_MSG(dlib::equal(x,opt, 1e-5),opt-x);
dlog << LINFO << "find_min_quasi_newton got apq/noder2 in " << total_count;
dlog << LINFO << "find_min() bfgs: got apq/noder2 in " << total_count;
total_count = 0;
x = p;
find_min_conjugate_gradient2(wrap_function(apq<T>), x, minf, eps);
find_min_using_approximate_derivatives(lbfgs_search_strategy(10),
objective_delta_stop_strategy(eps),
wrap_function(apq<T>), x, minf);
DLIB_TEST_MSG(dlib::equal(x,opt, 1e-5),opt-x);
dlog << LINFO << "find_min_conjugate_gradient got apq/noder2 in " << total_count;
dlog << LINFO << "find_min() lbfgs-10: got apq/noder2 in " << total_count;
total_count = 0;
x = p;
find_min_using_approximate_derivatives(cg_search_strategy(),
objective_delta_stop_strategy(eps),
wrap_function(apq<T>), x, minf);
DLIB_TEST_MSG(dlib::equal(x,opt, 1e-5),opt-x);
dlog << LINFO << "find_min() cg: got apq/noder2 in " << total_count;
}
}
void test_powell (
......@@ -180,28 +254,47 @@ namespace
/*
total_count = 0;
x = p;
find_min_quasi_newton(&powell, derivative(&powell,1e-8), x, minf, eps);
find_min(bfgs_search_strategy(),
objective_delta_stop_strategy(eps),
&powell, derivative(&powell,1e-8), x, minf);
DLIB_TEST_MSG(dlib::equal(x,opt, 1e-2),opt-x);
dlog << LINFO << "find_min_quasi_newton got powell/noder in " << total_count;
dlog << LINFO << "find_min() bfgs: got powell/noder in " << total_count;
total_count = 0;
x = p;
find_min_conjugate_gradient(&powell, derivative(&powell,1e-9), x, minf, eps);
find_min(cg_search_strategy(),
objective_delta_stop_strategy(eps),
&powell, derivative(&powell,1e-9), x, minf);
DLIB_TEST_MSG(dlib::equal(x,opt, 1e-2),opt-x);
dlog << LINFO << "find_min_conjugate_gradient got powell/noder in " << total_count;
dlog << LINFO << "find_min() cg: got powell/noder in " << total_count;
*/
total_count = 0;
x = p;
find_min_quasi_newton2(&powell, x, minf, eps, 1e-10);
find_min_using_approximate_derivatives(bfgs_search_strategy(),
objective_delta_stop_strategy(eps),
&powell, x, minf, 1e-10);
DLIB_TEST_MSG(dlib::equal(x,opt, 1e-1),opt-x);
dlog << LINFO << "find_min() bfgs: got powell/noder2 in " << total_count;
total_count = 0;
x = p;
find_min_using_approximate_derivatives(lbfgs_search_strategy(4),
objective_delta_stop_strategy(eps),
&powell, x, minf, 1e-10);
DLIB_TEST_MSG(dlib::equal(x,opt, 1e-1),opt-x);
dlog << LINFO << "find_min_quasi_newton got powell/noder2 in " << total_count;
dlog << LINFO << "find_min() lbfgs-4: got powell/noder2 in " << total_count;
total_count = 0;
x = p;
find_min_conjugate_gradient2(&powell, x, minf, eps, 1e-10);
find_min_using_approximate_derivatives(cg_search_strategy(),
objective_delta_stop_strategy(eps),
&powell, x, minf, 1e-10);
DLIB_TEST_MSG(dlib::equal(x,opt, 1e-1),opt-x);
dlog << LINFO << "find_min_conjugate_gradient got powell/noder2 in " << total_count;
dlog << LINFO << "find_min() cg: got powell/noder2 in " << total_count;
}
......@@ -220,39 +313,86 @@ namespace
total_count = 0;
x = p;
find_min_quasi_newton(&simple, &der_simple, x, minf, eps);
find_min(bfgs_search_strategy(),
objective_delta_stop_strategy(eps),
&simple, &der_simple, x, minf);
DLIB_TEST_MSG(dlib::equal(x,opt, 1e-5),opt-x);
dlog << LINFO << "find_min_quasi_newton got simple in " << total_count;
dlog << LINFO << "find_min() bfgs: got simple in " << total_count;
total_count = 0;
x = p;
find_min_conjugate_gradient(&simple, &der_simple, x, minf, eps);
find_min(lbfgs_search_strategy(3),
objective_delta_stop_strategy(eps),
&simple, &der_simple, x, minf);
DLIB_TEST_MSG(dlib::equal(x,opt, 1e-5),opt-x);
dlog << LINFO << "find_min_conjugate_gradient got simple in " << total_count;
dlog << LINFO << "find_min() lbfgs-3: got simple in " << total_count;
total_count = 0;
x = p;
find_min_quasi_newton(&simple, derivative(&simple), x, minf, eps);
find_min(cg_search_strategy(),
objective_delta_stop_strategy(eps),
&simple, &der_simple, x, minf);
DLIB_TEST_MSG(dlib::equal(x,opt, 1e-5),opt-x);
dlog << LINFO << "find_min_quasi_newton got simple/noder in " << total_count;
dlog << LINFO << "find_min() cg: got simple in " << total_count;
total_count = 0;
x = p;
find_min_conjugate_gradient(&simple, derivative(&simple), x, minf, eps);
find_min(bfgs_search_strategy(),
objective_delta_stop_strategy(eps),
&simple, derivative(&simple), x, minf);
DLIB_TEST_MSG(dlib::equal(x,opt, 1e-5),opt-x);
dlog << LINFO << "find_min_conjugate_gradient got simple/noder in " << total_count;
dlog << LINFO << "find_min() bfgs: got simple/noder in " << total_count;
total_count = 0;
x = p;
find_min(lbfgs_search_strategy(8),
objective_delta_stop_strategy(eps),
&simple, derivative(&simple), x, minf);
DLIB_TEST_MSG(dlib::equal(x,opt, 1e-5),opt-x);
dlog << LINFO << "find_min() lbfgs-8: got simple/noder in " << total_count;
total_count = 0;
x = p;
find_min_quasi_newton2(&simple, x, minf, eps);
find_min(cg_search_strategy(),
objective_delta_stop_strategy(eps),
&simple, derivative(&simple), x, minf);
DLIB_TEST_MSG(dlib::equal(x,opt, 1e-5),opt-x);
dlog << LINFO << "find_min_quasi_newton got simple/noder2 in " << total_count;
dlog << LINFO << "find_min() cg: got simple/noder in " << total_count;
total_count = 0;
x = p;
find_min_conjugate_gradient2(&simple, x, minf, eps);
find_min_using_approximate_derivatives(bfgs_search_strategy(),
objective_delta_stop_strategy(eps),
&simple, x, minf);
DLIB_TEST_MSG(dlib::equal(x,opt, 1e-5),opt-x);
dlog << LINFO << "find_min_conjugate_gradient got simple/noder2 in " << total_count;
dlog << LINFO << "find_min() bfgs: got simple/noder2 in " << total_count;
total_count = 0;
x = p;
find_min_using_approximate_derivatives(lbfgs_search_strategy(6),
objective_delta_stop_strategy(eps),
&simple, x, minf);
DLIB_TEST_MSG(dlib::equal(x,opt, 1e-5),opt-x);
dlog << LINFO << "find_min() lbfgs-6: got simple/noder2 in " << total_count;
total_count = 0;
x = p;
find_min_using_approximate_derivatives(cg_search_strategy(),
objective_delta_stop_strategy(eps),
&simple, x, minf);
DLIB_TEST_MSG(dlib::equal(x,opt, 1e-5),opt-x);
dlog << LINFO << "find_min() cg: got simple/noder2 in " << total_count;
}
......@@ -270,41 +410,122 @@ namespace
total_count = 0;
x = p;
find_min_quasi_newton(&rosen, &der_rosen, x, minf, eps);
find_min(bfgs_search_strategy(),
objective_delta_stop_strategy(eps),
&rosen, &der_rosen, x, minf);
DLIB_TEST_MSG(dlib::equal(x,opt, 1e-7),opt-x);
dlog << LINFO << "find_min_quasi_newton got rosen in " << total_count;
dlog << LINFO << "find_min() bfgs: got rosen in " << total_count;
total_count = 0;
x = p;
find_min_conjugate_gradient(&rosen, &der_rosen, x, minf, eps);
DLIB_TEST_MSG(dlib::equal(x,opt, 1e-5),opt-x);
dlog << LINFO << "find_min_conjugate_gradient got rosen in " << total_count;
find_min(lbfgs_search_strategy(20),
objective_delta_stop_strategy(eps),
&rosen, &der_rosen, x, minf);
DLIB_TEST_MSG(dlib::equal(x,opt, 1e-7),opt-x);
dlog << LINFO << "find_min() lbfgs-20: got rosen in " << total_count;
total_count = 0;
x = p;
find_min_quasi_newton(&rosen, derivative(&rosen), x, minf, eps);
find_min(cg_search_strategy(),
objective_delta_stop_strategy(eps),
&rosen, &der_rosen, x, minf);
DLIB_TEST_MSG(dlib::equal(x,opt, 1e-7),opt-x);
dlog << LINFO << "find_min() cg: got rosen in " << total_count;
total_count = 0;
x = p;
find_min(bfgs_search_strategy(),
objective_delta_stop_strategy(eps),
&rosen, derivative(&rosen,1e-5), x, minf);
DLIB_TEST_MSG(dlib::equal(x,opt, 1e-4),opt-x);
dlog << LINFO << "find_min_quasi_newton got rosen/noder in " << total_count;
dlog << LINFO << "find_min() bfgs: got rosen/noder in " << total_count;
total_count = 0;
x = p;
find_min_conjugate_gradient(&rosen, derivative(&rosen), x, minf, eps);
find_min(lbfgs_search_strategy(5),
objective_delta_stop_strategy(eps),
&rosen, derivative(&rosen,1e-5), x, minf);
DLIB_TEST_MSG(dlib::equal(x,opt, 1e-4),opt-x);
dlog << LINFO << "find_min_conjugate_gradient got rosen/noder in " << total_count;
dlog << LINFO << "find_min() lbfgs-5: got rosen/noder in " << total_count;
/* This test fails
total_count = 0;
x = p;
find_min_quasi_newton2(&rosen, x, minf, eps, 1e-13);
find_min(cg_search_strategy(),
objective_delta_stop_strategy(eps),
&rosen, derivative(&rosen,1e-5), x, minf);
DLIB_TEST_MSG(dlib::equal(x,opt, 1e-4),opt-x);
dlog << LINFO << "find_min() cg: got rosen/noder in " << total_count;
// TODO uncomment
/*
total_count = 0;
x = p;
find_min_quasi_newton2(&rosen, x, minf, eps);
DLIB_TEST_MSG(dlib::equal(x,opt, 1e-2),opt-x);
dlog << LINFO << "find_min_quasi_newton got rosen/noder2 in " << total_count;
*/
total_count = 0;
x = p;
find_min_conjugate_gradient2(&rosen, x, minf, eps, 1e-11);
find_min_using_approximate_derivatives(cg_search_strategy(),
objective_delta_stop_strategy(eps),
&rosen, x, minf);
DLIB_TEST_MSG(dlib::equal(x,opt, 1e-4),opt-x);
dlog << LINFO << "find_min_conjugate_gradient got rosen/noder2 in " << total_count;
dlog << LINFO << "find_min() cg: got rosen/noder2 in " << total_count;
}
void test_neg_rosen (
const matrix<double,2,1> p
)
{
const double eps = 1e-15;
const double maxf = 10;
matrix<double,2,1> x, opt;
opt(0) = 1;
opt(1) = 1;
dlog << LINFO << "testing with neg_rosen and the start point: " << trans(p);
total_count = 0;
x = p;
find_max(
bfgs_search_strategy(),
objective_delta_stop_strategy(eps), &neg_rosen, &der_neg_rosen, x, maxf);
DLIB_TEST_MSG(dlib::equal(x,opt, 1e-7),opt-x);
dlog << LINFO << "find_max() bfgs: got neg_rosen in " << total_count;
total_count = 0;
x = p;
find_max(
lbfgs_search_strategy(5),
objective_delta_stop_strategy(eps), &neg_rosen, &der_neg_rosen, x, maxf);
DLIB_TEST_MSG(dlib::equal(x,opt, 1e-7),opt-x);
dlog << LINFO << "find_max() lbfgs-5: got neg_rosen in " << total_count;
total_count = 0;
x = p;
find_max(
lbfgs_search_strategy(5),
objective_delta_stop_strategy(eps), &neg_rosen, derivative(&neg_rosen), x, maxf);
DLIB_TEST_MSG(dlib::equal(x,opt, 1e-7),opt-x);
dlog << LINFO << "find_max() lbfgs-5: got neg_rosen/noder in " << total_count;
total_count = 0;
x = p;
find_max_using_approximate_derivatives(
cg_search_strategy(),
objective_delta_stop_strategy(eps), &neg_rosen, x, maxf);
DLIB_TEST_MSG(dlib::equal(x,opt, 1e-7),opt-x);
dlog << LINFO << "find_max() cg: got neg_rosen/noder2 in " << total_count;
}
// ----------------------------------------------------------------------------------------
......@@ -318,6 +539,8 @@ namespace
{
matrix<double,0,1> p;
print_spinner();
p.set_size(2);
......@@ -325,6 +548,7 @@ namespace
p(0) = 9;
p(1) = -4.9;
test_rosen(p);
test_neg_rosen(p);
p(0) = 0;
p(1) = 0;
......@@ -347,6 +571,8 @@ namespace
p(1) = 839485;
test_simple(p);
print_spinner();
// test with the apq function
p.set_size(5);
......@@ -371,6 +597,8 @@ namespace
p(4) = 5;
test_apq(p);
print_spinner();
p(0) = 1;
p(1) = 2324;
p(2) = -3;
......@@ -391,6 +619,17 @@ namespace
p(9) = 5;
test_apq(p);
// test apq with a big vector
p.set_size(500);
dlib::rand::float_1a rnd;
for (long i = 0; i < p.size(); ++i)
{
p(i) = rnd.get_random_double()*20 - 10;
}
test_apq(p);
print_spinner();
// test with the powell function
p.set_size(4);
......@@ -433,6 +672,12 @@ namespace
make_line_search_function(&der_rosen,m,m)(1)) < 1e-5,"");
}
{
matrix<double,2,1> m;
m = 1,2;
DLIB_TEST(std::abs(neg_rosen(m) - negate_function(&rosen)(m) ) < 1e-16);
}
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment