Skip to content

D
dlib
Commit 3e1edfaa authored by Davis King's avatar Davis King
Browse files
Options

- Added a max dictionary setting to the kcentroid object 

   - Removed the "discount" thing and replaced it with a
     much less confusing pair of scale arguments to the
     training functions.
   - Changed the kcentroid's serialization format.

--HG--
extra : convert_revision : svn%3Afdd8eb12-d10e-0410-9acb-85c331704f74/trunk%402313
parent 94ebc5c3
Hide whitespace changes
Inline Side-by-side
Showing with 156 additions and 92 deletions
dlib/svm/kcentroid.h
View file @ 3e1edfaa
......@@ -36,52 +36,24 @@ namespace dlib
explicit kcentroid (
const kernel_type& kernel_,
scalar_type tolerance_ = 0.001
scalar_type tolerance_ = 0.001,
unsigned long max_dictionary_size_ = 1000000
) :
kernel(kernel_),
tolerance(tolerance_),
max_dis(1e6)
max_dictionary_size(max_dictionary_size_)
{
clear_dictionary();
}
void set_tolerance (scalar_type tolerance_)
{
// make sure requires clause is not broken
DLIB_ASSERT(tolerance_ >= 0,
"\tvoid kcentroid::set_tolerance"
<< "\n\tinvalid tolerance value"
<< "\n\ttolerance: " << tolerance_
<< "\n\tthis: " << this
);
tolerance = tolerance_;
}
scalar_type get_tolerance() const
{
return tolerance;
}
void set_max_discount (
scalar_type value
)
{
// make sure requires clause is not broken
DLIB_ASSERT(value >= 0,
"\tvoid kcentroid::set_max_discount"
<< "\n\tinvalid discount value"
<< "\n\tvalue: " << value
<< "\n\tthis: " << this
);
max_dis = value;
if (samples_seen > value)
samples_seen = value;
}
scalar_type get_max_discount(
) const
unsigned long get_max_dictionary_size() const
{
return max_dis;
return max_dictionary_size;
}
void clear_dictionary ()
......@@ -90,6 +62,7 @@ namespace dlib
alpha.clear();
K_inv.set_size(0,0);
K.set_size(0,0);
samples_seen = 0;
bias = 0;
}
......@@ -105,18 +78,50 @@ namespace dlib
return std::sqrt(kernel(x,x) + bias - 2*temp);
}
double samples_trained (
) const
{
return samples_seen;
}
scalar_type test_and_train (
const sample_type& x
)
{
return train_and_maybe_test(x,true);
++samples_seen;
const double xscale = 1.0/samples_seen;
const double cscale = 1-xscale;
return train_and_maybe_test(x,cscale,xscale,true);
}
void train (
const sample_type& x
)
{
train_and_maybe_test(x,false);
++samples_seen;
const double xscale = 1.0/samples_seen;
const double cscale = 1-xscale;
train_and_maybe_test(x,cscale,xscale,false);
}
scalar_type test_and_train (
const sample_type& x,
double cscale,
double xscale
)
{
++samples_seen;
return train_and_maybe_test(x,cscale,xscale,true);
}
void train (
const sample_type& x,
double cscale,
double xscale
)
{
++samples_seen;
train_and_maybe_test(x,cscale,xscale,false);
}
void swap (
......@@ -131,7 +136,6 @@ namespace dlib
exchange(tolerance, item.tolerance);
exchange(samples_seen, item.samples_seen);
exchange(bias, item.bias);
exchange(max_dis, item.max_dis);
a.swap(item.a);
k.swap(item.k);
}
......@@ -149,7 +153,6 @@ namespace dlib
serialize(item.tolerance, out);
serialize(item.samples_seen, out);
serialize(item.bias, out);
serialize(item.max_dis, out);
}
friend void deserialize(kcentroid& item, std::istream& in)
......@@ -162,13 +165,14 @@ namespace dlib
deserialize(item.tolerance, in);
deserialize(item.samples_seen, in);
deserialize(item.bias, in);
deserialize(item.max_dis, in);
}
private:
scalar_type train_and_maybe_test (
const sample_type& x,
double cscale,
double xscale,
bool do_test
)
{
......@@ -183,7 +187,7 @@ namespace dlib
K.set_size(1,1);
K(0,0) = kx;
alpha.push_back(1.0);
alpha.push_back(xscale);
dictionary.push_back(x);
}
else
......@@ -201,12 +205,25 @@ namespace dlib
// compute the error we would have if we approximated the new x sample
// with the dictionary. That is, do the ALD test from the KRLS paper.
a = K_inv*k;
const scalar_type delta = kx - trans(k)*a;
scalar_type delta = kx - trans(k)*a;
// if this new vector isn't approximately linearly dependent on the vectors
// in our dictionary.
if (std::abs(delta) > tolerance)
{
if (dictionary.size() >= max_dictionary_size)
{
// We need to remove one of the old members of the dictionary before
// we proceed with adding a new one. So remove the oldest one.
remove_dictionary_vector(0);
// recompute these guys since they were computed with the old
// kernel matrix
k = remove_row(k,0);
a = K_inv*k;
delta = kx - trans(k)*a;
}
// add x to the dictionary
dictionary.push_back(x);
......@@ -239,37 +256,68 @@ namespace dlib
// now update the alpha vector
const double alpha_scale = samples_seen/(samples_seen+1);
for (unsigned long i = 0; i < alpha.size(); ++i)
{
alpha[i] *= alpha_scale;
alpha[i] *= cscale;
}
alpha.push_back(1.0-alpha_scale);
alpha.push_back(xscale);
}
else
{
// update the alpha vector so that this new sample has been added into
// the mean vector we are accumulating
const double alpha_scale = samples_seen/(samples_seen+1);
const double a_scale = 1.0-alpha_scale;
for (unsigned long i = 0; i < alpha.size(); ++i)
{
alpha[i] = alpha_scale*alpha[i] + a_scale*a(i);
alpha[i] = cscale*alpha[i] + xscale*a(i);
}
}
}
++samples_seen;
// recompute the bias term
bias = sum(pointwise_multiply(K, vector_to_matrix(alpha)*trans(vector_to_matrix(alpha))));
if (samples_seen > max_dis)
samples_seen = max_dis;
return test_result;
}
void remove_dictionary_vector (
long i
)
/*!
requires
- 0 <= i < dictionary.size()
ensures
- #dictionary.size() == dictionary.size() - 1
- #alpha.size() == alpha.size() - 1
- updates the K_inv matrix so that it is still a proper inverse of the
kernel matrix
- also removes the necessary row and column from the K matrix
- uses the this->a variable so after this function runs that variable
will contain a different value.
!*/
{
// remove the dictionary vector
dictionary.erase(dictionary.begin()+i);
// remove the i'th vector from the inverse kernel matrix. This formula is basically
// just the reverse of the way K_inv is updated by equation 3.14 during normal training.
K_inv = removerc(K_inv,i,i) - remove_row(colm(K_inv,i)/K_inv(i,i),i)*remove_col(rowm(K_inv,i),i);
// now compute the updated alpha values to take account that we just removed one of
// our dictionary vectors
a = (K_inv*remove_row(K,i)*vector_to_matrix(alpha));
// now copy over the new alpha values
alpha.resize(alpha.size()-1);
for (unsigned long k = 0; k < alpha.size(); ++k)
{
alpha[k] = a(k);
}
// update the K matrix as well
K = removerc(K,i,i);
}
typedef std_allocator<sample_type, mem_manager_type> alloc_sample_type;
typedef std_allocator<scalar_type, mem_manager_type> alloc_scalar_type;
......@@ -285,9 +333,9 @@ namespace dlib
matrix<scalar_type,0,0,mem_manager_type> K;
scalar_type tolerance;
unsigned long max_dictionary_size;
scalar_type samples_seen;
scalar_type bias;
scalar_type max_dis;
// temp variables here just so we don't have to reconstruct them over and over. Thus,
......
dlib/svm/kcentroid_abstract.h
View file @ 3e1edfaa
......@@ -23,7 +23,7 @@ namespace dlib
INITIAL VALUE
- dictionary_size() == 0
- max_discount() == 1e6
- samples_trained() == 0
WHAT THIS OBJECT REPRESENTS
This is an implementation of an online algorithm for recursively estimating the
......@@ -34,6 +34,13 @@ namespace dlib
and any test points. So you can use this object to predict how similar a test
point is to the data this object has been trained on (larger distances from the
centroid indicate dissimilarity/anomalous points).
Also note that the algorithm internally keeps a set of "dictionary vectors"
that are used to represent the regression function. You can force the
algorithm to use no more than a set number of vectors by setting
the 3rd constructor argument to whatever you want. However, note that
doing this causes the algorithm to bias it's results towards more
recent training examples.
!*/
public:
......@@ -44,7 +51,8 @@ namespace dlib
explicit kcentroid (
const kernel_type& kernel_,
scalar_type tolerance_ = 0.001
scalar_type tolerance_ = 0.001,
unsigned long max_dictionary_size_ = 1000000
);
/*!
ensures
......@@ -52,16 +60,23 @@ namespace dlib
- #get_tolerance() == tolerance_
- #get_decision_function().kernel_function == kernel_
(i.e. this object will use the given kernel function)
- #get_max_dictionary_size() == max_dictionary_size_
!*/
void set_tolerance (
scalar_type tolerance_
);
unsigned long get_max_dictionary_size(
) const;
/*!
requires
- tolerance_ >= 0
ensures
- #get_tolerance() == tolerance_
- returns the maximum number of dictionary vectors this object
will use at a time. That is, dictionary_size() will never be
greater than get_max_dictionary_size().
!*/
scalar_type samples_trained (
) const;
/*!
ensures
- returns the number of samples this object has been trained on so far
!*/
scalar_type get_tolerance(
......@@ -77,52 +92,47 @@ namespace dlib
less accurate estimate but also in less support vectors.
!*/
void set_max_discount (
scalar_type value
void clear_dictionary (
);
/*!
requires
- value > 0
ensures
- #get_max_discount() == value
- clears out all learned data (e.g. #dictionary_size() == 0)
- #samples_seen() == 0
!*/
scalar_type get_max_discount(
scalar_type operator() (
const sample_type& x
) const;
/*!
ensures
- If you have shown this object N samples so far then it has found
the centroid of those N samples. That is, it has found the average
of all of them in some high dimensional feature space.
- if (N <= get_max_discount()) then
- The next sample you show this object will be added to the centroid
with a weight of 1/(N+1).
- else
- The next sample you show this object will be added to the centroid
with a weight of 1/(get_max_discount()+1).
- If you think your samples are from a stationary source then you
should set the max discount to some really big number. However,
if you think the source isn't stationary then use a smaller number.
This will cause the centroid in this object to be closer to the
centroid of the more recent points.
- returns the distance in feature space between the sample x and the
current estimate of the centroid of the training samples given
to this object so far.
!*/
void clear_dictionary (
void train (
const sample_type& x
);
/*!
ensures
- clears out all learned data (e.g. #dictionary_size() == 0)
- adds the sample x into the current estimate of the centroid
- also note that calling this function is equivalent to calling
train(x, samples_trained()/(samples_trained()+1.0, 1.0/(samples_trained()+1.0).
That is, this function finds the normal unweighted centroid of all training points.
!*/
scalar_type operator() (
const sample_type& x
) const;
void train (
const sample_type& x,
double cscale,
double xscale
);
/*!
ensures
- returns the distance in feature space between the sample x and the
current estimate of the centroid of the training samples given
to this object so far.
- adds the sample x into the current estimate of the centroid but
uses a user given scale. That is, this function performs:
- new_centroid = cscale*old_centroid + xscale*x
- This function allows you to weight different samples however
you want.
!*/
scalar_type test_and_train (
......@@ -137,12 +147,18 @@ namespace dlib
than calling both individually.
!*/
void train (
const sample_type& x
scalar_type test_and_train (
const sample_type& x,
double cscale,
double xscale
);
/*!
ensures
- adds the sample x into the current estimate of the centroid
- calls train(x,cscale,xscale)
- returns (*this)(x)
- The reason this function exists is because train() and operator()
both compute some of the same things. So this function is more efficient
than calling both individually.
!*/
void swap (
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment