Commit e273f515 authored by Davis King's avatar Davis King

Added find_min_global() overloads.

parent 3284f575
......@@ -112,75 +112,109 @@ template <typename T> static auto go(T&& f, const matrix<double, 0, 1>& a) -> de
// ----------------------------------------------------------------------------------------
template <
typename funct
>
std::pair<size_t,function_evaluation> find_max_global (
std::vector<funct>& functions,
std::vector<function_spec> specs,
const max_function_calls num,
const std::chrono::nanoseconds max_runtime = FOREVER,
double solver_epsilon = 0
)
namespace impl
{
// Decide which parameters should be searched on a log scale. Basically, it's
// common for machine learning models to have parameters that should be searched on
// a log scale (e.g. SVM C). These parameters are usually identifiable because
// they have bounds like [1e-5 1e10], that is, they span a very large range of
// magnitudes from really small to really big. So there we are going to check for
// that and if we find parameters with that kind of bound constraints we will
// transform them to a log scale automatically.
std::vector<std::vector<bool>> log_scale(specs.size());
for (size_t i = 0; i < specs.size(); ++i)
template <
typename funct
>
std::pair<size_t,function_evaluation> find_max_global (
std::vector<funct>& functions,
std::vector<function_spec> specs,
const max_function_calls num,
const std::chrono::nanoseconds max_runtime,
double solver_epsilon,
double ymult
)
{
for (long j = 0; j < specs[i].lower.size(); ++j)
// Decide which parameters should be searched on a log scale. Basically, it's
// common for machine learning models to have parameters that should be searched on
// a log scale (e.g. SVM C). These parameters are usually identifiable because
// they have bounds like [1e-5 1e10], that is, they span a very large range of
// magnitudes from really small to really big. So there we are going to check for
// that and if we find parameters with that kind of bound constraints we will
// transform them to a log scale automatically.
std::vector<std::vector<bool>> log_scale(specs.size());
for (size_t i = 0; i < specs.size(); ++i)
{
if (!specs[i].is_integer_variable[j] && specs[i].lower(j) > 0 && specs[i].upper(j)/specs[i].lower(j) > 1000)
for (long j = 0; j < specs[i].lower.size(); ++j)
{
log_scale[i].push_back(true);
specs[i].lower(j) = std::log(specs[i].lower(j));
specs[i].upper(j) = std::log(specs[i].upper(j));
if (!specs[i].is_integer_variable[j] && specs[i].lower(j) > 0 && specs[i].upper(j)/specs[i].lower(j) > 1000)
{
log_scale[i].push_back(true);
specs[i].lower(j) = std::log(specs[i].lower(j));
specs[i].upper(j) = std::log(specs[i].upper(j));
}
else
{
log_scale[i].push_back(false);
}
}
else
}
global_function_search opt(specs);
opt.set_solver_epsilon(solver_epsilon);
const auto time_to_stop = std::chrono::steady_clock::now() + max_runtime;
// Now run the main solver loop.
for (size_t i = 0; i < num.max_calls && std::chrono::steady_clock::now() < time_to_stop; ++i)
{
auto next = opt.get_next_x();
matrix<double,0,1> x = next.x();
// Undo any log-scaling that was applied to the variables before we pass them
// to the functions being optimized.
for (long j = 0; j < x.size(); ++j)
{
log_scale[i].push_back(false);
if (log_scale[next.function_idx()][j])
x(j) = std::exp(x(j));
}
double y = ymult*call_function_and_expand_args(functions[next.function_idx()], x);
next.set(y);
}
}
global_function_search opt(specs);
opt.set_solver_epsilon(solver_epsilon);
const auto time_to_stop = std::chrono::steady_clock::now() + max_runtime;
// Now run the main solver loop.
for (size_t i = 0; i < num.max_calls && std::chrono::steady_clock::now() < time_to_stop; ++i)
{
auto next = opt.get_next_x();
matrix<double,0,1> x = next.x();
// Undo any log-scaling that was applied to the variables before we pass them
// to the functions being optimized.
matrix<double,0,1> x;
double y;
size_t function_idx;
opt.get_best_function_eval(x,y,function_idx);
// Undo any log-scaling that was applied to the variables before we output them.
for (long j = 0; j < x.size(); ++j)
{
if (log_scale[next.function_idx()][j])
if (log_scale[function_idx][j])
x(j) = std::exp(x(j));
}
double y = call_function_and_expand_args(functions[next.function_idx()], x);
next.set(y);
return std::make_pair(function_idx, function_evaluation(x,y/ymult));
}
}
// ----------------------------------------------------------------------------------------
matrix<double,0,1> x;
double y;
size_t function_idx;
opt.get_best_function_eval(x,y,function_idx);
// Undo any log-scaling that was applied to the variables before we output them.
for (long j = 0; j < x.size(); ++j)
{
if (log_scale[function_idx][j])
x(j) = std::exp(x(j));
}
return std::make_pair(function_idx, function_evaluation(x,std::move(y)));
template <
typename funct
>
std::pair<size_t,function_evaluation> find_max_global (
std::vector<funct>& functions,
std::vector<function_spec> specs,
const max_function_calls num,
const std::chrono::nanoseconds max_runtime = FOREVER,
double solver_epsilon = 0
)
{
return impl::find_max_global(functions, std::move(specs), num, max_runtime, solver_epsilon, +1);
}
template <
typename funct
>
std::pair<size_t,function_evaluation> find_min_global (
std::vector<funct>& functions,
std::vector<function_spec> specs,
const max_function_calls num,
const std::chrono::nanoseconds max_runtime = FOREVER,
double solver_epsilon = 0
)
{
return impl::find_max_global(functions, std::move(specs), num, max_runtime, solver_epsilon, -1);
}
// ----------------------------------------------------------------------------------------
......@@ -203,6 +237,24 @@ template <typename T> static auto go(T&& f, const matrix<double, 0, 1>& a) -> de
return find_max_global(functions, std::move(specs), num, max_runtime, solver_epsilon).second;
}
template <
typename funct
>
function_evaluation find_min_global (
funct f,
const matrix<double,0,1>& bound1,
const matrix<double,0,1>& bound2,
const std::vector<bool>& is_integer_variable,
const max_function_calls num,
const std::chrono::nanoseconds max_runtime = FOREVER,
double solver_epsilon = 0
)
{
std::vector<funct> functions(1,std::move(f));
std::vector<function_spec> specs(1, function_spec(bound1, bound2, is_integer_variable));
return find_min_global(functions, std::move(specs), num, max_runtime, solver_epsilon).second;
}
// ----------------------------------------------------------------------------------------
template <
......@@ -220,6 +272,21 @@ template <typename T> static auto go(T&& f, const matrix<double, 0, 1>& a) -> de
return find_max_global(std::move(f), bound1, bound2, is_integer_variable, num, FOREVER, solver_epsilon);
}
template <
typename funct
>
function_evaluation find_min_global (
funct f,
const matrix<double,0,1>& bound1,
const matrix<double,0,1>& bound2,
const std::vector<bool>& is_integer_variable,
const max_function_calls num,
double solver_epsilon
)
{
return find_min_global(std::move(f), bound1, bound2, is_integer_variable, num, FOREVER, solver_epsilon);
}
// ----------------------------------------------------------------------------------------
template <
......@@ -237,6 +304,21 @@ template <typename T> static auto go(T&& f, const matrix<double, 0, 1>& a) -> de
return find_max_global(std::move(f), bound1, bound2, std::vector<bool>(bound1.size(),false), num, max_runtime, solver_epsilon);
}
template <
typename funct
>
function_evaluation find_min_global (
funct f,
const matrix<double,0,1>& bound1,
const matrix<double,0,1>& bound2,
const max_function_calls num,
const std::chrono::nanoseconds max_runtime = FOREVER,
double solver_epsilon = 0
)
{
return find_min_global(std::move(f), bound1, bound2, std::vector<bool>(bound1.size(),false), num, max_runtime, solver_epsilon);
}
// ----------------------------------------------------------------------------------------
template <
......@@ -253,6 +335,20 @@ template <typename T> static auto go(T&& f, const matrix<double, 0, 1>& a) -> de
return find_max_global(std::move(f), bound1, bound2, std::vector<bool>(bound1.size(),false), num, FOREVER, solver_epsilon);
}
template <
typename funct
>
function_evaluation find_min_global (
funct f,
const matrix<double,0,1>& bound1,
const matrix<double,0,1>& bound2,
const max_function_calls num,
double solver_epsilon
)
{
return find_min_global(std::move(f), bound1, bound2, std::vector<bool>(bound1.size(),false), num, FOREVER, solver_epsilon);
}
// ----------------------------------------------------------------------------------------
template <
......@@ -270,6 +366,21 @@ template <typename T> static auto go(T&& f, const matrix<double, 0, 1>& a) -> de
return find_max_global(std::move(f), matrix<double,0,1>({bound1}), matrix<double,0,1>({bound2}), num, max_runtime, solver_epsilon);
}
template <
typename funct
>
function_evaluation find_min_global (
funct f,
const double bound1,
const double bound2,
const max_function_calls num,
const std::chrono::nanoseconds max_runtime = FOREVER,
double solver_epsilon = 0
)
{
return find_min_global(std::move(f), matrix<double,0,1>({bound1}), matrix<double,0,1>({bound2}), num, max_runtime, solver_epsilon);
}
// ----------------------------------------------------------------------------------------
template <
......@@ -286,6 +397,20 @@ template <typename T> static auto go(T&& f, const matrix<double, 0, 1>& a) -> de
return find_max_global(std::move(f), matrix<double,0,1>({bound1}), matrix<double,0,1>({bound2}), num, FOREVER, solver_epsilon);
}
template <
typename funct
>
function_evaluation find_min_global (
funct f,
const double bound1,
const double bound2,
const max_function_calls num,
double solver_epsilon
)
{
return find_min_global(std::move(f), matrix<double,0,1>({bound1}), matrix<double,0,1>({bound2}), num, FOREVER, solver_epsilon);
}
// ----------------------------------------------------------------------------------------
template <
......@@ -302,6 +427,20 @@ template <typename T> static auto go(T&& f, const matrix<double, 0, 1>& a) -> de
return find_max_global(std::move(f), bound1, bound2, max_function_calls(), max_runtime, solver_epsilon);
}
template <
typename funct
>
function_evaluation find_min_global (
funct f,
const matrix<double,0,1>& bound1,
const matrix<double,0,1>& bound2,
const std::chrono::nanoseconds max_runtime,
double solver_epsilon = 0
)
{
return find_min_global(std::move(f), bound1, bound2, max_function_calls(), max_runtime, solver_epsilon);
}
// ----------------------------------------------------------------------------------------
template <
......@@ -318,6 +457,20 @@ template <typename T> static auto go(T&& f, const matrix<double, 0, 1>& a) -> de
return find_max_global(std::move(f), bound1, bound2, max_function_calls(), max_runtime, solver_epsilon);
}
template <
typename funct
>
function_evaluation find_min_global (
funct f,
const double bound1,
const double bound2,
const std::chrono::nanoseconds max_runtime,
double solver_epsilon = 0
)
{
return find_min_global(std::move(f), bound1, bound2, max_function_calls(), max_runtime, solver_epsilon);
}
// ----------------------------------------------------------------------------------------
template <
......@@ -335,6 +488,21 @@ template <typename T> static auto go(T&& f, const matrix<double, 0, 1>& a) -> de
return find_max_global(std::move(f), bound1, bound2, is_integer_variable, max_function_calls(), max_runtime, solver_epsilon);
}
template <
typename funct
>
function_evaluation find_min_global (
funct f,
const matrix<double,0,1>& bound1,
const matrix<double,0,1>& bound2,
const std::vector<bool>& is_integer_variable,
const std::chrono::nanoseconds max_runtime,
double solver_epsilon = 0
)
{
return find_min_global(std::move(f), bound1, bound2, is_integer_variable, max_function_calls(), max_runtime, solver_epsilon);
}
// ----------------------------------------------------------------------------------------
}
......
......@@ -53,7 +53,7 @@ namespace dlib
/*!
WHAT THIS OBJECT REPRESENTS
This is a simple typed integer class used to strongly type the "max number
of function calls" argument to find_max_global().
of function calls" argument to find_max_global() and find_min_global().
!*/
......@@ -136,6 +136,21 @@ namespace dlib
optimizer.
!*/
template <
typename funct
>
std::pair<size_t,function_evaluation> find_min_global (
std::vector<funct>& functions,
const std::vector<function_spec>& specs,
const max_function_calls num,
const std::chrono::nanoseconds max_runtime = FOREVER,
double solver_epsilon = 0
);
/*!
This function is identical to the find_max_global() defined immediately above,
except that we perform minimization rather than maximization.
!*/
// ----------------------------------------------------------------------------------------
template <
......@@ -204,9 +219,26 @@ namespace dlib
optimizer.
!*/
template <
typename funct
>
function_evaluation find_min_global (
funct f,
const matrix<double,0,1>& bound1,
const matrix<double,0,1>& bound2,
const std::vector<bool>& is_integer_variable,
const max_function_calls num,
const std::chrono::nanoseconds max_runtime = FOREVER,
double solver_epsilon = 0
);
/*!
This function is identical to the find_max_global() defined immediately above,
except that we perform minimization rather than maximization.
!*/
// ----------------------------------------------------------------------------------------
// The following functions are just convenient overloads for calling the above defined
// find_max_global() routines.
// find_max_global() and find_min_global() routines.
// ----------------------------------------------------------------------------------------
template <
......@@ -224,6 +256,21 @@ namespace dlib
return find_max_global(std::move(f), bound1, bound2, std::vector<bool>(bound1.size(),false), num, FOREVER, solver_epsilon);
}
template <
typename funct
>
function_evaluation find_min_global (
funct f,
const matrix<double,0,1>& bound1,
const matrix<double,0,1>& bound2,
const std::vector<bool>& is_integer_variable,
const max_function_calls num,
double solver_epsilon
)
{
return find_min_global(std::move(f), bound1, bound2, std::vector<bool>(bound1.size(),false), num, FOREVER, solver_epsilon);
}
// ----------------------------------------------------------------------------------------
template <
......@@ -241,6 +288,21 @@ namespace dlib
return find_max_global(std::move(f), bound1, bound2, std::vector<bool>(bound1.size(),false), num, max_runtime, solver_epsilon);
}
template <
typename funct
>
function_evaluation find_min_global (
funct f,
const matrix<double,0,1>& bound1,
const matrix<double,0,1>& bound2,
const max_function_calls num,
const std::chrono::nanoseconds max_runtime = FOREVER,
double solver_epsilon = 0
)
{
return find_min_global(std::move(f), bound1, bound2, std::vector<bool>(bound1.size(),false), num, max_runtime, solver_epsilon);
}
// ----------------------------------------------------------------------------------------
template <
......@@ -257,6 +319,20 @@ namespace dlib
return find_max_global(std::move(f), bound1, bound2, std::vector<bool>(bound1.size(),false), num, FOREVER, solver_epsilon);
}
template <
typename funct
>
function_evaluation find_min_global (
funct f,
const matrix<double,0,1>& bound1,
const matrix<double,0,1>& bound2,
const max_function_calls num,
double solver_epsilon
)
{
return find_min_global(std::move(f), bound1, bound2, std::vector<bool>(bound1.size(),false), num, FOREVER, solver_epsilon);
}
// ----------------------------------------------------------------------------------------
template <
......@@ -274,6 +350,21 @@ namespace dlib
return find_max_global(std::move(f), matrix<double,0,1>({bound1}), matrix<double,0,1>({bound2}), num, max_runtime, solver_epsilon);
}
template <
typename funct
>
function_evaluation find_min_global (
funct f,
const double bound1,
const double bound2,
const max_function_calls num,
const std::chrono::nanoseconds max_runtime = FOREVER,
double solver_epsilon = 0
)
{
return find_min_global(std::move(f), matrix<double,0,1>({bound1}), matrix<double,0,1>({bound2}), num, max_runtime, solver_epsilon);
}
// ----------------------------------------------------------------------------------------
template <
......@@ -290,6 +381,20 @@ namespace dlib
return find_max_global(std::move(f), matrix<double,0,1>({bound1}), matrix<double,0,1>({bound2}), num, FOREVER, solver_epsilon);
}
template <
typename funct
>
function_evaluation find_min_global (
funct f,
const double bound1,
const double bound2,
const max_function_calls num,
double solver_epsilon
)
{
return find_min_global(std::move(f), matrix<double,0,1>({bound1}), matrix<double,0,1>({bound2}), num, FOREVER, solver_epsilon);
}
// ----------------------------------------------------------------------------------------
template <
......@@ -306,6 +411,20 @@ namespace dlib
return find_max_global(std::move(f), bound1, bound2, max_function_calls(), max_runtime, solver_epsilon);
}
template <
typename funct
>
function_evaluation find_min_global (
funct f,
const matrix<double,0,1>& bound1,
const matrix<double,0,1>& bound2,
const std::chrono::nanoseconds max_runtime,
double solver_epsilon = 0
)
{
return find_min_global(std::move(f), bound1, bound2, max_function_calls(), max_runtime, solver_epsilon);
}
// ----------------------------------------------------------------------------------------
template <
......@@ -322,6 +441,20 @@ namespace dlib
return find_max_global(std::move(f), bound1, bound2, max_function_calls(), max_runtime, solver_epsilon);
}
template <
typename funct
>
function_evaluation find_min_global (
funct f,
const double bound1,
const double bound2,
const std::chrono::nanoseconds max_runtime,
double solver_epsilon = 0
)
{
return find_min_global(std::move(f), bound1, bound2, max_function_calls(), max_runtime, solver_epsilon);
}
// ----------------------------------------------------------------------------------------
template <
......@@ -339,6 +472,21 @@ namespace dlib
return find_max_global(std::move(f), bound1, bound2, is_integer_variable, max_function_calls(), max_runtime, solver_epsilon);
}
template <
typename funct
>
function_evaluation find_min_global (
funct f,
const matrix<double,0,1>& bound1,
const matrix<double,0,1>& bound2,
const std::vector<bool>& is_integer_variable,
const std::chrono::nanoseconds max_runtime,
double solver_epsilon = 0
)
{
return find_min_global(std::move(f), bound1, bound2, is_integer_variable, max_function_calls(), max_runtime, solver_epsilon);
}
// ----------------------------------------------------------------------------------------
}
......
......@@ -213,6 +213,66 @@ namespace
DLIB_TEST_MSG(std::abs(result.y - 21.9210397) < 0.0001, std::abs(result.y - 21.9210397));
}
// ----------------------------------------------------------------------------------------
void test_find_min_global(
)
{
print_spinner();
auto rosen = [](const matrix<double,0,1>& x) { return +1*( 100*std::pow(x(1) - x(0)*x(0),2.0) + std::pow(1 - x(0),2)); };
auto result = find_min_global(rosen, {0, 0}, {2, 2}, max_function_calls(100), 0);
matrix<double,0,1> true_x = {1,1};
dlog << LINFO << "rosen: " << trans(result.x);
DLIB_TEST_MSG(min(abs(true_x-result.x)) < 1e-5, min(abs(true_x-result.x)));
print_spinner();
result = find_min_global(rosen, {0, 0}, {2, 2}, max_function_calls(100));
dlog << LINFO << "rosen: " << trans(result.x);
DLIB_TEST_MSG(min(abs(true_x-result.x)) < 1e-5, min(abs(true_x-result.x)));
print_spinner();
result = find_min_global(rosen, {0, 0}, {2, 2}, std::chrono::seconds(5));
dlog << LINFO << "rosen: " << trans(result.x);
DLIB_TEST_MSG(min(abs(true_x-result.x)) < 1e-5, min(abs(true_x-result.x)));
print_spinner();
result = find_min_global(rosen, {0, 0}, {2, 2}, {false,false}, max_function_calls(100));
dlog << LINFO << "rosen: " << trans(result.x);
DLIB_TEST_MSG(min(abs(true_x-result.x)) < 1e-5, min(abs(true_x-result.x)));
print_spinner();
result = find_min_global(rosen, {0, 0}, {0.9, 0.9}, {false,false}, max_function_calls(100));
true_x = {0.9, 0.81};
dlog << LINFO << "rosen, bounded at 0.9: " << trans(result.x);
DLIB_TEST_MSG(min(abs(true_x-result.x)) < 1e-5, min(abs(true_x-result.x)));
print_spinner();
result = find_min_global([](double x){ return std::pow(x-2,2.0); }, -10, 10, max_function_calls(10), 0);
dlog << LINFO << "(x-2)^2: " << trans(result.x);
DLIB_TEST(result.x.size()==1);
DLIB_TEST(std::abs(result.x - 2) < 1e-9);
print_spinner();
result = find_min_global([](double x){ return std::pow(x-2,2.0); }, -10, 1, max_function_calls(10));
dlog << LINFO << "(x-2)^2, bound at 1: " << trans(result.x);
DLIB_TEST(result.x.size()==1);
DLIB_TEST(std::abs(result.x - 1) < 1e-9);
print_spinner();
result = find_min_global([](double x){ return std::pow(x-2,2.0); }, -10, 1, std::chrono::seconds(2));
dlog << LINFO << "(x-2)^2, bound at 1: " << trans(result.x);
DLIB_TEST(result.x.size()==1);
DLIB_TEST(std::abs(result.x - 1) < 1e-9);
print_spinner();
result = find_min_global([](double a, double b){ return complex_holder_table(a,b);},
{-10, -10}, {10, 10}, max_function_calls(300), 0);
dlog << LINFO << "complex_holder_table y: "<< result.y;
DLIB_TEST_MSG(std::abs(result.y + 21.9210397) < 0.0001, std::abs(result.y + 21.9210397));
}
// ----------------------------------------------------------------------------------------
......@@ -233,6 +293,7 @@ namespace
test_upper_bound_function(0.0, 1e-1);
test_global_function_search();
test_find_max_global();
test_find_min_global();
}
} a;
......
......@@ -263,14 +263,14 @@ int main() try
// Finally, let's try the find_max_global() routine. Like find_min_bobyqa(),
// this technique is specially designed to optimize a function in the absence
// Finally, let's try the find_min_global() routine. Like find_min_bobyqa(),
// this technique is specially designed to minimize a function in the absence
// of derivative information. However, it is also designed to handle
// functions with many local optima. Where BOBYQA would get stuck at the
// nearest local optima, find_max_global() won't. find_max_global() uses a
// nearest local optima, find_min_global() won't. find_min_global() uses a
// global optimization method based on a combination of non-parametric global
// function modeling and BOBYQA style quadratic trust region modeling to
// efficiently find a global maximizer. It usually does a good job with a
// efficiently find a global minimizer. It usually does a good job with a
// relatively small number of calls to the function being optimized.
//
// You also don't have to give it a starting point or set any parameters,
......@@ -296,20 +296,20 @@ int main() try
}
// Holder table function tilted towards 10,10 and with additional
// high frequency terms to add more local optima.
return std::abs(sin(x0)*cos(x1)*exp(std::abs(1-std::sqrt(x0*x0+x1*x1)/pi))) -(x0+x1)/10 - sin(x0*10)*cos(x1*10);
return -( std::abs(sin(x0)*cos(x1)*exp(std::abs(1-std::sqrt(x0*x0+x1*x1)/pi))) -(x0+x1)/10 - sin(x0*10)*cos(x1*10));
};
// To optimize this difficult function all we need to do is call
// find_max_global()
auto result = find_max_global(complex_holder_table,
// find_min_global()
auto result = find_min_global(complex_holder_table,
{-10,-10}, // lower bounds
{10,10}, // upper bounds
max_function_calls(300));
cout.precision(9);
// These cout statements will show that find_max_global() found the
// These cout statements will show that find_min_global() found the
// globally optimal solution to 9 digits of precision:
cout << "complex holder table function solution y (should be 21.9210397): " << result.y << endl;
cout << "complex holder table function solution y (should be -21.9210397): " << result.y << endl;
cout << "complex holder table function solution x:\n" << result.x << endl;
}
catch (std::exception& e)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment