Commit 7e460a05 authored by Davis King's avatar Davis King

Worked on the spec for the manifold regularization stuff more.

--HG--
extra : convert_revision : svn%3Afdd8eb12-d10e-0410-9acb-85c331704f74/trunk%403594
parent e9595abe
...@@ -180,13 +180,13 @@ namespace dlib ...@@ -180,13 +180,13 @@ namespace dlib
// ---------------------------------------------------------------------------------------- // ----------------------------------------------------------------------------------------
template < template <
typename alloc typename vector_type
> >
bool contains_duplicate_pairs ( bool contains_duplicate_pairs (
const std::vector<sample_pair, alloc>& pairs const vector_type& pairs
) )
{ {
std::vector<sample_pair, alloc> temp(pairs); vector_type temp(pairs);
std::sort(temp.begin(), temp.end(), &order_by_index); std::sort(temp.begin(), temp.end(), &order_by_index);
for (unsigned long i = 1; i < temp.size(); ++i) for (unsigned long i = 1; i < temp.size(); ++i)
...@@ -199,6 +199,34 @@ namespace dlib ...@@ -199,6 +199,34 @@ namespace dlib
return false; return false;
} }
// ----------------------------------------------------------------------------------------
template <
typename vector_type
>
unsigned long max_index_value_plus_one (
const vector_type& pairs
)
{
if (pairs.size() == 0)
{
return 0;
}
else
{
unsigned long max_idx = 0;
for (unsigned long i = 0; i < pairs.size(); ++i)
{
if (pairs[i].index1() > max_idx)
max_idx = pairs[i].index1();
if (pairs[i].index2() > max_idx)
max_idx = pairs[i].index2();
}
return max_idx + 1;
}
}
// ---------------------------------------------------------------------------------------- // ----------------------------------------------------------------------------------------
} }
......
...@@ -80,12 +80,15 @@ namespace dlib ...@@ -80,12 +80,15 @@ namespace dlib
// ---------------------------------------------------------------------------------------- // ----------------------------------------------------------------------------------------
template < template <
typename alloc typename vector_type
> >
bool contains_duplicate_pairs ( bool contains_duplicate_pairs (
const std::vector<sample_pair, alloc>& pairs const vector_type& pairs
); );
/*! /*!
requires
- vector_type == a type with an interface compatible with std::vector and
it must in turn contain objects with an interface compatible with dlib::sample_pair
ensures ensures
- if (pairs contains any elements that are equal according to operator==) then - if (pairs contains any elements that are equal according to operator==) then
- returns true - returns true
...@@ -93,6 +96,27 @@ namespace dlib ...@@ -93,6 +96,27 @@ namespace dlib
- returns false - returns false
!*/ !*/
// ----------------------------------------------------------------------------------------
template <
typename vector_type
>
unsigned long max_index_value_plus_one (
const vector_type& pairs
);
/*!
requires
- vector_type == a type with an interface compatible with std::vector and
it must in turn contain objects with an interface compatible with dlib::sample_pair
ensures
- if (pairs.size() == 0) then
- returns 0
- else
- returns a number N such that:
- for all i: pairs[i].index1() < N && pairs[i].index2() < N
- for some j: pairs[j].index1()+1 == N || pairs[j].index2()+1 == N
!*/
// ---------------------------------------------------------------------------------------- // ----------------------------------------------------------------------------------------
} }
......
...@@ -3,7 +3,7 @@ ...@@ -3,7 +3,7 @@
#ifndef DLIB_LINEAR_MANIFOLD_ReGULARIZER_H__ #ifndef DLIB_LINEAR_MANIFOLD_ReGULARIZER_H__
#define DLIB_LINEAR_MANIFOLD_ReGULARIZER_H__ #define DLIB_LINEAR_MANIFOLD_ReGULARIZER_H__
#include "linear_manifold_regularizer.h" #include "linear_manifold_regularizer_abstract.h"
#include <limits> #include <limits>
#include <vector> #include <vector>
#include "../serialize.h" #include "../serialize.h"
...@@ -41,6 +41,10 @@ namespace dlib ...@@ -41,6 +41,10 @@ namespace dlib
unsigned long size ( unsigned long size (
) const ) const
/*!
ensures
- returns the number of vertices in this graph
!*/
{ {
return _size; return _size;
} }
...@@ -51,6 +55,9 @@ namespace dlib ...@@ -51,6 +55,9 @@ namespace dlib
/*! /*!
requires requires
- idx < size() - idx < size()
ensures
- returns an iterator that points to the first neighbor of
the idx'th vertex.
!*/ !*/
{ {
return blocks[idx]; return blocks[idx];
...@@ -62,6 +69,9 @@ namespace dlib ...@@ -62,6 +69,9 @@ namespace dlib
/*! /*!
requires requires
- idx < size() - idx < size()
ensures
- returns an iterator that points one past the last neighbor
of the idx'th vertex.
!*/ !*/
{ {
return blocks[idx+1]; return blocks[idx+1];
...@@ -78,11 +88,14 @@ namespace dlib ...@@ -78,11 +88,14 @@ namespace dlib
- vector_type == a type with an interface compatible with std::vector and - vector_type == a type with an interface compatible with std::vector and
it must in turn contain objects with an interface compatible with dlib::sample_pair it must in turn contain objects with an interface compatible with dlib::sample_pair
- edges.size() > 0 - edges.size() > 0
- all the elements of edges are unique. That is: - contains_duplicate_pairs(edges) == false
- for all valid i and j where i != j:
it must be true that edges[i] != edges[j]
- weight_funct(edges[i]) must be a valid expression that evaluates to a - weight_funct(edges[i]) must be a valid expression that evaluates to a
floating point number floating point number >= 0
ensures
- #size() == one greater than the max index in edges.
- builds the adjacency list so that it contains all the given edges.
- The weight in each neighbor is set to the output of the weight_funct()
for the associated edge.
!*/ !*/
{ {
...@@ -160,15 +173,6 @@ namespace dlib ...@@ -160,15 +173,6 @@ namespace dlib
> >
class linear_manifold_regularizer class linear_manifold_regularizer
{ {
/*!
REQUIREMENTS ON matrix_type
Must be some type of dlib::matrix.
WHAT THIS OBJECT REPRESENTS
This object computes the inv(T) matrix described in the following paper:
Linear Manifold Regularization for Large Scale Semi-supervised Learning
by Vikas Sindhwani, Partha Niyogi, and Mikhail Belkin
!*/
public: public:
typedef typename matrix_type::mem_manager_type mem_manager_type; typedef typename matrix_type::mem_manager_type mem_manager_type;
...@@ -194,14 +198,18 @@ namespace dlib ...@@ -194,14 +198,18 @@ namespace dlib
make_mr_matrix(samples, graph); make_mr_matrix(samples, graph);
} }
long dimensionality (
) const { return reg_mat.nr(); }
general_matrix get_transformation_matrix ( general_matrix get_transformation_matrix (
scalar_type intrinsic_regularization_strength scalar_type intrinsic_regularization_strength
) const ) const
/*!
requires
- intrinsic_regularization_strength >= 0
!*/
{ {
if (dimensionality() == 0)
return general_matrix();
// TODO: should we divide intrinsic_regularization_strength by the number of edges? That maybe a more
// reasonable interface.
return inv_lower_triangular(chol(identity_matrix<scalar_type>(reg_mat.nr()) + intrinsic_regularization_strength*reg_mat)); return inv_lower_triangular(chol(identity_matrix<scalar_type>(reg_mat.nr()) + intrinsic_regularization_strength*reg_mat));
} }
...@@ -215,15 +223,17 @@ namespace dlib ...@@ -215,15 +223,17 @@ namespace dlib
/*! /*!
requires requires
- samples.size() == graph.size() - samples.size() == graph.size()
ensures
- computes trans(X)*lap(graph)*X where X is the data matrix
(i.e. the matrix that contains all the samples in its rows)
and lap(graph) is the laplacian matrix of the graph. The
resulting matrix is stored in reg_mat.
!*/ !*/
{ {
const unsigned long dims = samples[0].size(); const unsigned long dims = samples[0].size();
reg_mat.set_size(dims,dims); reg_mat.set_size(dims,dims);
reg_mat = 0; reg_mat = 0;
// Compute trans(X)*lap(graph)*X where X is the data matrix
// (i.e. the matrix that contains all the samples in its rows)
// and lap(graph) is the laplacian matrix of the graph.
typename impl::undirected_adjacency_list::const_iterator beg, end; typename impl::undirected_adjacency_list::const_iterator beg, end;
......
// Copyright (C) 2010 Davis E. King (davis@dlib.net)
// License: Boost Software License See LICENSE.txt for the full license.
#undef DLIB_LINEAR_MANIFOLD_ReGULARIZER_ABSTRACT_H__
#ifdef DLIB_LINEAR_MANIFOLD_ReGULARIZER_ABSTRACT_H__
#include <limits>
#include <vector>
#include "../serialize.h"
#include "../matrix.h"
namespace dlib
{
// ----------------------------------------------------------------------------------------
template <
typename matrix_type
>
class linear_manifold_regularizer
{
/*!
REQUIREMENTS ON matrix_type
Must be some type of dlib::matrix.
INITIAL VALUE
- dimensionality() == 0
WHAT THIS OBJECT REPRESENTS
Many learning algorithms attempt to minimize a loss function that,
at a high level, looks like this:
loss(w) == complexity + training_set_error
The idea is to find the set of parameters, w, that gives low error on
your training data but also is not "complex" according to some particular
measure of complexity. This strategy of penalizing complexity is
usually called regularization.
In the above setting all the training data consists of labeled samples.
However, it would be nice to be able to benefit from unlabeled data
(see the example program for this object for an example where unlabeled
data is useful). The idea of manifold regularization is to extract useful
information from unlabeled data by defining which data samples are "close"
to each other (perhaps by using their 3 nearest neighbors) and then adding
a term to the loss function that penalizes any decision rule which produces
different output on data samples that we have designated as being close.
It turns out that it is possible to turn these manifold regularized loss
functions into the normal form shown above by applying a certain kind
of processing to all of our data samples. Once this is done we can use
a normal learning algorithm, such as the svm_c_linear_trainer, on just the
labeled data samples and obtain the same output as the manifold regularized
learner would have produced. Therefore, the linear_manifold_regularizer is
a tool for creating this preprocessing transformation. In particular, the
transformation is linear. That is, it is just a matrix you multiply with
all your samples.
For a more detailed discussion of this topic you should consult the following
paper. In particular, see section 4.2. This object computes the inverse T
matrix described in that section.
Linear Manifold Regularization for Large Scale Semi-supervised Learning
by Vikas Sindhwani, Partha Niyogi, and Mikhail Belkin
!*/
public:
typedef typename matrix_type::mem_manager_type mem_manager_type;
typedef typename matrix_type::type scalar_type;
typedef typename matrix_type::layout_type layout_type;
typedef matrix<scalar_type,0,0,mem_manager_type,layout_type> general_matrix;
template <
typename vector_type1,
typename vector_type2,
typename weight_function_type
>
void build (
const vector_type1& samples,
const vector_type2& edges,
const weight_function_type& weight_funct
);
/*!
requires
- vector_type1 == a type with an interface compatible with std::vector and it must
in turn contain dlib::matrix objects.
- vector_type2 == a type with an interface compatible with std::vector and
it must in turn contain objects with an interface compatible with dlib::sample_pair
- edges.size() > 0
- contains_duplicate_pairs(edges) == false
- max_index_value_plus_one(edges) <= samples.size()
- weight_funct(edges[i]) must be a valid expression that evaluates to a
floating point number >= 0
ensures
- #dimensionality() == samples[0].size()
- This function sets up the transformation matrix describe above. The manifold
regularization is done assuming that the following samples are meant to
be "close" according to the graph defined by the given edges. I.e:
- for all valid i: samples[edges[i].index1()] is close to samples[edges[i].index2()].
How much we care about these two samples having similar outputs according
to the learned rule is given by weight_funct(edges[i]). Bigger weights mean
we care more.
!*/
long dimensionality (
) const;
/*!
ensures
- returns the number of rows and columns in the transformation matrix
produced by this object
!*/
general_matrix get_transformation_matrix (
scalar_type intrinsic_regularization_strength
) const;
/*!
requires
- intrinsic_regularization_strength >= 0
ensures
- returns a matrix that represents the preprocessing transformation described above.
- You must choose how important the manifold regularizer is relative to the basic
"don't be complex" regularizer describe above. The intrinsic_regularization_strength
is the parameter that controls this trade-off. A large value of
intrinsic_regularization_strength means that more emphasis should be placed on
finding decision rules which produce the same output on similar samples. On
the other hand, a small value would mean that we don't care much about the
manifold regularizer. For example, using 0 will cause this function to return the
identity matrix.
- The returned matrix will have dimensionality() rows and columns
!*/
};
}
// ----------------------------------------------------------------------------------------
#endif // DLIB_LINEAR_MANIFOLD_ReGULARIZER_ABSTRACT_H__
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment