Faiss

- Main Page
- Namespaces
- Classes
- Files

- File List

All Classes Namespaces Functions Variables Typedefs Enumerations Enumerator Friends
/data/users/matthijs/github_faiss/faiss/VectorTransform.cpp
1
2 /**
3 * Copyright (c) 2015-present, Facebook, Inc.
4 * All rights reserved.
5 *
6 * This source code is licensed under the CC-by-NC license found in the
7 * LICENSE file in the root directory of this source tree.
8 */
9
10 // Copyright 2004-present Facebook. All Rights Reserved
11 // -*- c++ -*-
12
13 #include "VectorTransform.h"
14
15 #include <cstdio>
16 #include <cmath>
17 #include <cstring>
18
19 #include "utils.h"
20 #include "FaissAssert.h"
21 #include "IndexPQ.h"
22
23 using namespace faiss;
24
25
26 extern "C" {
27
28 // this is to keep the clang syntax checker happy
29 #ifndef FINTEGER
30 #define FINTEGER int
31 #endif
32
33
34 /* declare BLAS functions, see http://www.netlib.org/clapack/cblas/ */
35
36 int sgemm_ (
37 const char *transa, const char *transb, FINTEGER *m, FINTEGER *
38 n, FINTEGER *k, const float *alpha, const float *a,
39 FINTEGER *lda, const float *b,
40 FINTEGER *ldb, float *beta,
41 float *c, FINTEGER *ldc);
42
43 int ssyrk_ (
44 const char *uplo, const char *trans, FINTEGER *n, FINTEGER *k,
45 float *alpha, float *a, FINTEGER *lda,
46 float *beta, float *c, FINTEGER *ldc);
47
48 /* Lapack functions from http://www.netlib.org/clapack/old/single/ */
49
50 int ssyev_ (
51 const char *jobz, const char *uplo, FINTEGER *n, float *a,
52 FINTEGER *lda, float *w, float *work, FINTEGER *lwork,
53 FINTEGER *info);
54
55 int sgesvd_(
56 const char *jobu, const char *jobvt, FINTEGER *m, FINTEGER *n,
57 float *a, FINTEGER *lda, float *s, float *u, FINTEGER *ldu, float *vt,
58 FINTEGER *ldvt, float *work, FINTEGER *lwork, FINTEGER *info);
59
60 }
61
62 /*********************************************
63 * VectorTransform
64 *********************************************/
65
66
67
68 float * VectorTransform::apply (Index::idx_t n, const float * x) const
69 {
70 float * xt = new float[n * d_out];
71 apply_noalloc (n, x, xt);
72 return xt;
73 }
74
75
76 void VectorTransform::train (idx_t, const float *) {
77 // does nothing by default
78 }
79
80
81 void VectorTransform::reverse_transform (
82 idx_t , const float *,
83 float *) const
84 {
85 FAISS_ASSERT (!"reverse transform not implemented");
86 }
87
88
89
90
91 /*********************************************
92 * LinearTransform
93 *********************************************/
94 /// both d_in > d_out and d_out < d_in are supported
95 LinearTransform::LinearTransform (int d_in, int d_out,
96 bool have_bias):
97 VectorTransform (d_in, d_out), have_bias (have_bias),
98 verbose (false)
99 {}
100
101 void LinearTransform::apply_noalloc (Index::idx_t n, const float * x,
102 float * xt) const
103 {
104 FAISS_ASSERT(is_trained || !"Transformation not trained yet");
105
106 float c_factor;
107 if (have_bias) {
108 FAISS_ASSERT (b.size() == d_out || !"Bias not initialized");
109 float * xi = xt;
110 for (int i = 0; i < n; i++)
111 for(int j = 0; j < d_out; j++)
112 *xi++ = b[j];
113 c_factor = 1.0;
114 } else {
115 c_factor = 0.0;
116 }
117
118 FAISS_ASSERT (A.size() == d_out * d_in ||
119 !"Transformation matrix not initialized");
120
121 float one = 1;
122 FINTEGER nbiti = d_out, ni = n, di = d_in;
123 sgemm_ ("Transposed", "Not transposed",
124 &nbiti, &ni, &di,
125 &one, A.data(), &di, x, &di, &c_factor, xt, &nbiti);
126
127 }
128
129
130 void LinearTransform::transform_transpose (idx_t n, const float * y,
131 float *x) const
132 {
133 if (have_bias) { // allocate buffer to store bias-corrected data
134 float *y_new = new float [n * d_out];
135 const float *yr = y;
136 float *yw = y_new;
137 for (idx_t i = 0; i < n; i++) {
138 for (int j = 0; j < d_out; j++) {
139 *yw++ = *yr++ - b [j];
140 }
141 }
142 y = y_new;
143 }
144
145 {
146 FINTEGER dii = d_in, doi = d_out, ni = n;
147 float one = 1.0, zero = 0.0;
148 sgemm_ ("Not", "Not", &dii, &ni, &doi,
149 &one, A.data (), &dii, y, &doi, &zero, x, &dii);
150 }
151
152 if (have_bias) delete [] y;
153 }
154
155
156 /*********************************************
157 * RandomRotationMatrix
158 *********************************************/
159
160 void RandomRotationMatrix::init (int seed)
161 {
162
163 if(d_out <= d_in) {
164 A.resize (d_out * d_in);
165 float *q = A.data();
166 float_randn(q, d_out * d_in, seed);
167 matrix_qr(d_in, d_out, q);
168 } else {
169 A.resize (d_out * d_out);
170 float *q = A.data();
171 float_randn(q, d_out * d_out, seed);
172 matrix_qr(d_out, d_out, q);
173 // remove columns
174 int i, j;
175 for (i = 0; i < d_out; i++) {
176 for(j = 0; j < d_in; j++) {
177 q[i * d_in + j] = q[i * d_out + j];
178 }
179 }
180 A.resize(d_in * d_out);
181 }
182
183 }
184
185 void RandomRotationMatrix::reverse_transform (idx_t n, const float * xt,
186 float *x) const
187 {
188 transform_transpose (n, xt, x);
189 }
190
191 /*********************************************
192 * PCAMatrix
193 *********************************************/
194
195 PCAMatrix::PCAMatrix (int d_in, int d_out,
196 float eigen_power, bool random_rotation):
197 LinearTransform(d_in, d_out, true),
198 eigen_power(eigen_power), random_rotation(random_rotation)
199 {
200 is_trained = false;
201 max_points_per_d = 1000;
202 balanced_bins = 0;
203 }
204
205
206 void PCAMatrix::train (Index::idx_t n, const float *x)
207 {
208 const float * x_in = x;
209
210 x = fvecs_maybe_subsample (d_in, (size_t*)&n,
211 max_points_per_d * d_in, x, verbose);
212
213 // compute mean
214 mean.clear(); mean.resize(d_in, 0.0);
215 if (have_bias) { // we may want to skip the bias
216 const float *xi = x;
217 for (int i = 0; i < n; i++) {
218 for(int j = 0; j < d_in; j++)
219 mean[j] += *xi++;
220 }
221 for(int j = 0; j < d_in; j++)
222 mean[j] /= n;
223 }
224 if(verbose) {
225 printf("mean=[");
226 for(int j = 0; j < d_in; j++) printf("%g ", mean[j]);
227 printf("]\n");
228 }
229
230 if(n >= d_in) {
231 // compute covariance matrix, store it in PCA matrix
232 PCAMat.resize(d_in * d_in);
233 float * cov = PCAMat.data();
234 { // initialize with mean * mean^T term
235 float *ci = cov;
236 for(int i = 0; i < d_in; i++) {
237 for(int j = 0; j < d_in; j++)
238 *ci++ = - n * mean[i] * mean[j];
239 }
240 }
241 {
242 FINTEGER di = d_in, ni = n;
243 float one = 1.0;
244 ssyrk_ ("Up", "Non transposed",
245 &di, &ni, &one, (float*)x, &di, &one, cov, &di);
246
247 }
248 if(verbose && d_in <= 10) {
249 float *ci = cov;
250 printf("cov=\n");
251 for(int i = 0; i < d_in; i++) {
252 for(int j = 0; j < d_in; j++)
253 printf("%10g ", *ci++);
254 printf("\n");
255 }
256 }
257
258 { // compute eigenvalues and vectors
259 eigenvalues.resize(d_in);
260 FINTEGER info = 0, lwork = -1, di = d_in;
261 float workq;
262
263 ssyev_ ("Vectors as well", "Upper",
264 &di, cov, &di, eigenvalues.data(), &workq, &lwork, &info);
265 lwork = FINTEGER(workq);
266 float *work = new float[lwork];
267
268 ssyev_ ("Vectors as well", "Upper",
269 &di, cov, &di, eigenvalues.data(), work, &lwork, &info);
270
271 if (info != 0) {
272 fprintf (stderr, "WARN ssyev info returns %d, "
273 "a very bad PCA matrix is learnt\n",
274 int(info));
275
276 }
277
278 delete [] work;
279
280 if(verbose && d_in <= 10) {
281 printf("info=%ld new eigvals=[", long(info));
282 for(int j = 0; j < d_in; j++) printf("%g ", eigenvalues[j]);
283 printf("]\n");
284
285 float *ci = cov;
286 printf("eigenvecs=\n");
287 for(int i = 0; i < d_in; i++) {
288 for(int j = 0; j < d_in; j++)
289 printf("%10.4g ", *ci++);
290 printf("\n");
291 }
292 }
293
294 }
295
296 // revert order of eigenvectors & values
297
298 for(int i = 0; i < d_in / 2; i++) {
299
300 std::swap(eigenvalues[i], eigenvalues[d_in - 1 - i]);
301 float *v1 = cov + i * d_in;
302 float *v2 = cov + (d_in - 1 - i) * d_in;
303 for(int j = 0; j < d_in; j++)
304 std::swap(v1[j], v2[j]);
305 }
306
307 } else {
308 FAISS_ASSERT(!"Gramm matrix version not implemented "
309 "-- provide more training examples than dimensions");
310 }
311
312
313 if (x != x_in) delete [] x;
314
315 prepare_Ab();
316 is_trained = true;
317 }
318
319 void PCAMatrix::copy_from (const PCAMatrix & other)
320 {
321 FAISS_ASSERT (other.is_trained);
322 mean = other.mean;
323 eigenvalues = other.eigenvalues;
324 PCAMat = other.PCAMat;
325 prepare_Ab ();
326 is_trained = true;
327 }
328
329 void PCAMatrix::prepare_Ab ()
330 {
331
332 if (!random_rotation) {
333 A = PCAMat;
334 A.resize(d_out * d_in); // strip off useless dimensions
335
336 // first scale the components
337 if (eigen_power != 0) {
338 float *ai = A.data();
339 for (int i = 0; i < d_out; i++) {
340 float factor = pow(eigenvalues[i], eigen_power);
341 for(int j = 0; j < d_in; j++)
342 *ai++ *= factor;
343 }
344 }
345
346 if (balanced_bins != 0) {
347 FAISS_ASSERT (d_out % balanced_bins == 0);
348 int dsub = d_out / balanced_bins;
349 std::vector <float> Ain;
350 std::swap(A, Ain);
351 A.resize(d_out * d_in);
352
353 std::vector <float> accu(balanced_bins);
354 std::vector <int> counter(balanced_bins);
355
356 // greedy assignment
357 for (int i = 0; i < d_out; i++) {
358 // find best bin
359 int best_j = -1;
360 float min_w = 1e30;
361 for (int j = 0; j < balanced_bins; j++) {
362 if (counter[j] < dsub && accu[j] < min_w) {
363 min_w = accu[j];
364 best_j = j;
365 }
366 }
367 int row_dst = best_j * dsub + counter[best_j];
368 accu[best_j] += eigenvalues[i];
369 counter[best_j] ++;
370 memcpy (&A[row_dst * d_in], &Ain[i * d_in],
371 d_in * sizeof (A[0]));
372 }
373
374 if (verbose) {
375 printf(" bin accu=[");
376 for (int i = 0; i < balanced_bins; i++)
377 printf("%g ", accu[i]);
378 printf("]\n");
379 }
380 }
381
382
383 } else {
384 FAISS_ASSERT (balanced_bins == 0 ||
385 !"both balancing bins and applying a random rotation "
386 "does not make sense");
387 RandomRotationMatrix rr(d_out, d_out);
388
389 rr.init(5);
390
391 // apply scaling on the rotation matrix (right multiplication)
392 if (eigen_power != 0) {
393 for (int i = 0; i < d_out; i++) {
394 float factor = pow(eigenvalues[i], eigen_power);
395 for(int j = 0; j < d_out; j++)
396 rr.A[j * d_out + i] *= factor;
397 }
398 }
399
400 A.resize(d_in * d_out);
401 {
402 FINTEGER dii = d_in, doo = d_out;
403 float one = 1.0, zero = 0.0;
404
405 sgemm_ ("Not", "Not", &dii, &doo, &doo,
406 &one, PCAMat.data(), &dii, rr.A.data(), &doo, &zero,
407 A.data(), &dii);
408
409 }
410
411 }
412
413 b.clear(); b.resize(d_out);
414
415 for (int i = 0; i < d_out; i++) {
416 float accu = 0;
417 for (int j = 0; j < d_in; j++)
418 accu -= mean[j] * A[j + i * d_in];
419 b[i] = accu;
420 }
421
422 }
423
424 void PCAMatrix::reverse_transform (idx_t n, const float * xt,
425 float *x) const
426 {
427 FAISS_ASSERT (eigen_power == 0 ||
428 !"reverse only implemented for orthogonal transforms");
429 transform_transpose (n, xt, x);
430 }
431
432 /*********************************************
433 * OPQMatrix
434 *********************************************/
435
436
437 OPQMatrix::OPQMatrix (int d, int M, int d2):
438 LinearTransform (d, d2 == -1 ? d : d2, false), M(M),
439 niter (50),
440 niter_pq (4), niter_pq_0 (40),
441 verbose(false)
442 {
443 is_trained = false;
444 // OPQ is quite expensive to train, so set this right.
445 max_train_points = 256 * 256;
446 }
447
448
449
450 void OPQMatrix::train (Index::idx_t n, const float *x)
451 {
452
453 const float * x_in = x;
454
455 x = fvecs_maybe_subsample (d_in, (size_t*)&n,
456 max_train_points, x, verbose);
457
458 // To support d_out > d_in, we pad input vectors with 0s to d_out
459 size_t d = d_out <= d_in ? d_in : d_out;
460 size_t d2 = d_out;
461
462 #if 0
463 // what this test shows: the only way of getting bit-exact
464 // reproducible results with sgeqrf and sgesvd seems to be forcing
465 // single-threading.
466 { // test repro
467 std::vector<float> r (d * d);
468 float * rotation = r.data();
469 float_randn (rotation, d * d, 1234);
470 printf("CS0: %016lx\n",
471 ivec_checksum (128*128, (int*)rotation));
472 matrix_qr (d, d, rotation);
473 printf("CS1: %016lx\n",
474 ivec_checksum (128*128, (int*)rotation));
475 return;
476 }
477 #endif
478
479 if (verbose) {
480 printf ("OPQMatrix::train: training an OPQ rotation matrix "
481 "for M=%d from %ld vectors in %dD -> %dD\n",
482 M, n, d_in, d_out);
483 }
484
485 std::vector<float> xtrain (n * d);
486 // center x
487 {
488 std::vector<float> sum (d);
489 const float *xi = x;
490 for (size_t i = 0; i < n; i++) {
491 for (int j = 0; j < d_in; j++)
492 sum [j] += *xi++;
493 }
494 for (int i = 0; i < d; i++) sum[i] /= n;
495 float *yi = xtrain.data();
496 xi = x;
497 for (size_t i = 0; i < n; i++) {
498 for (int j = 0; j < d_in; j++)
499 *yi++ = *xi++ - sum[j];
500 yi += d - d_in;
501 }
502 }
503 float *rotation;
504
505 if (A.size () == 0) {
506 A.resize (d * d);
507 rotation = A.data();
508 if (verbose)
509 printf(" OPQMatrix::train: making random %ld*%ld rotation\n",
510 d, d);
511 float_randn (rotation, d * d, 1234);
512 matrix_qr (d, d, rotation);
513 // we use only the d * d2 upper part of the matrix
514 A.resize (d * d2);
515 } else {
516 FAISS_ASSERT (A.size() == d * d2);
517 rotation = A.data();
518 }
519
520
521 std::vector<float>
522 xproj (d2 * n), pq_recons (d2 * n), xxr (d * n),
523 tmp(d * d * 4);
524
525 std::vector<uint8_t> codes (M * n);
526 ProductQuantizer pq_regular (d2, M, 8);
527 double t0 = getmillisecs();
528 for (int iter = 0; iter < niter; iter++) {
529
530 { // torch.mm(xtrain, rotation:t())
531 FINTEGER di = d, d2i = d2, ni = n;
532 float zero = 0, one = 1;
533 sgemm_ ("Transposed", "Not transposed",
534 &d2i, &ni, &di,
535 &one, rotation, &di,
536 xtrain.data(), &di,
537 &zero, xproj.data(), &d2i);
538 }
539
540 pq_regular.cp.max_points_per_centroid = 1000;
541 pq_regular.cp.niter = iter == 0 ? niter_pq_0 : niter_pq;
542 pq_regular.cp.verbose = verbose;
543 pq_regular.train (n, xproj.data());
544
545 pq_regular.compute_codes (xproj.data(), codes.data(), n);
546 pq_regular.decode (codes.data(), pq_recons.data(), n);
547
548 float pq_err = fvec_L2sqr (pq_recons.data(), xproj.data(), n * d2) / n;
549
550 if (verbose)
551 printf (" Iteration %d (%d PQ iterations):"
552 "%.3f s, obj=%g\n", iter, pq_regular.cp.niter,
553 (getmillisecs () - t0) / 1000.0, pq_err);
554
555 {
556 float *u = tmp.data(), *vt = &tmp [d * d];
557 float *sing_val = &tmp [2 * d * d];
558 FINTEGER di = d, d2i = d2, ni = n;
559 float one = 1, zero = 0;
560
561 // torch.mm(xtrain:t(), pq_recons)
562 sgemm_ ("Not", "Transposed",
563 &d2i, &di, &ni,
564 &one, pq_recons.data(), &d2i,
565 xtrain.data(), &di,
566 &zero, xxr.data(), &d2i);
567
568
569 FINTEGER lwork = -1, info = -1;
570 float worksz;
571 // workspace query
572 sgesvd_ ("All", "All",
573 &d2i, &di, xxr.data(), &d2i,
574 sing_val,
575 vt, &d2i, u, &di,
576 &worksz, &lwork, &info);
577
578 lwork = int(worksz);
579 std::vector<float> work (lwork);
580 // u and vt swapped
581 sgesvd_ ("All", "All",
582 &d2i, &di, xxr.data(), &d2i,
583 sing_val,
584 vt, &d2i, u, &di,
585 work.data(), &lwork, &info);
586
587 sgemm_ ("Transposed", "Transposed",
588 &di, &d2i, &d2i,
589 &one, u, &di, vt, &d2i,
590 &zero, rotation, &di);
591
592 }
593 pq_regular.train_type = ProductQuantizer::Train_hot_start;
594 }
595
596 // revert A matrix
597 if (d > d_in) {
598 for (long i = 0; i < d_out; i++)
599 memmove (&A[i * d_in], &A[i * d], sizeof(A[0]) * d_in);
600 A.resize (d_in * d_out);
601 }
602
603 if (x != x_in)
604 delete [] x;
605
606 is_trained = true;
607 }
608
609
610
611
612 void OPQMatrix::reverse_transform (idx_t n, const float * xt,
613 float *x) const
614 {
615 transform_transpose (n, xt, x);
616 }
617
618 /*********************************************
619 * IndexPreTransform
620 *********************************************/
621
622 IndexPreTransform::IndexPreTransform ():
623 index(nullptr), own_fields (false)
624 {
625 }
626
627
628 IndexPreTransform::IndexPreTransform (
629 Index * index):
630 Index (index->d, index->metric_type),
631 index (index), own_fields (false)
632 {
633 is_trained = index->is_trained;
634 set_typename();
635 }
636
637
638
639
640 IndexPreTransform::IndexPreTransform (
641 VectorTransform * ltrans,
642 Index * index):
643 Index (index->d, index->metric_type),
644 index (index), own_fields (false)
645 {
646 is_trained = index->is_trained;
647 prepend_transform (ltrans);
648 set_typename();
649 }
650
651 void IndexPreTransform::prepend_transform (VectorTransform *ltrans)
652 {
653 FAISS_ASSERT (ltrans->d_out == d);
654 is_trained = is_trained && ltrans->is_trained;
655 chain.insert (chain.begin(), ltrans);
656 d = ltrans->d_in;
657 set_typename ();
658 }
659
660
661 void IndexPreTransform::set_typename ()
662 {
663 // TODO correct this according to actual type
664 index_typename = "PreLT[" + index->index_typename + "]";
665 }
666
667
668 IndexPreTransform::~IndexPreTransform ()
669 {
670 if (own_fields) {
671 for (int i = 0; i < chain.size(); i++)
672 delete chain[i];
673 delete index;
674 }
675 }
676
677
678
679
680 void IndexPreTransform::train (idx_t n, const float *x)
681 {
682 int last_untrained = 0;
683 for (int i = 0; i < chain.size(); i++)
684 if (!chain[i]->is_trained) last_untrained = i;
685 if (!index->is_trained) last_untrained = chain.size();
686 const float *prev_x = x;
687
688 for (int i = 0; i <= last_untrained; i++) {
689 if (i < chain.size()) {
690 VectorTransform *ltrans = chain [i];
691 if (!ltrans->is_trained)
692 ltrans->train(n, prev_x);
693 } else {
694 index->train (n, prev_x);
695 }
696 if (i == last_untrained) break;
697
698 float * xt = chain[i]->apply (n, prev_x);
699 if (prev_x != x) delete [] prev_x;
700 prev_x = xt;
701 }
702
703 if (prev_x != x) delete [] prev_x;
704 is_trained = true;
705 }
706
707
708 const float *IndexPreTransform::apply_chain (idx_t n, const float *x) const
709 {
710 const float *prev_x = x;
711 for (int i = 0; i < chain.size(); i++) {
712 float * xt = chain[i]->apply (n, prev_x);
713 if (prev_x != x) delete [] prev_x;
714 prev_x = xt;
715 }
716 return prev_x;
717 }
718
719 void IndexPreTransform::add (idx_t n, const float *x)
720 {
721 FAISS_ASSERT (is_trained);
722 const float *xt = apply_chain (n, x);
723 index->add (n, xt);
724 if (xt != x) delete [] xt;
725 ntotal = index->ntotal;
726 }
727
728 void IndexPreTransform::add_with_ids (idx_t n, const float * x,
729 const long *xids)
730 {
731 FAISS_ASSERT (is_trained);
732 const float *xt = apply_chain (n, x);
733 index->add_with_ids (n, xt, xids);
734 if (xt != x) delete [] xt;
735 ntotal = index->ntotal;
736 }
737
738
739
740
741 void IndexPreTransform::search (idx_t n, const float *x, idx_t k,
742 float *distances, idx_t *labels) const
743 {
744 FAISS_ASSERT (is_trained);
745 const float *xt = apply_chain (n, x);
746 index->search (n, xt, k, distances, labels);
747 if (xt != x) delete [] xt;
748 }
749
750
751 void IndexPreTransform::reset () {
752 index->reset();
753 ntotal = 0;
754 }
755
756 long IndexPreTransform::remove_ids (const IDSelector & sel) {
757 long nremove = index->remove_ids (sel);
758 ntotal = index->ntotal;
759 return nremove;
760 }
761
762
763 void IndexPreTransform::reconstruct_n (idx_t i0, idx_t ni, float *recons) const
764 {
765 float *x = chain.empty() ? recons : new float [ni * index->d];
766 // initial reconstruction
767 index->reconstruct_n (i0, ni, x);
768
769 // revert transformations from last to first
770 for (int i = chain.size() - 1; i >= 0; i--) {
771 float *x_pre = i == 0 ? recons : new float [chain[i]->d_in * ni];
772 chain [i]->reverse_transform (ni, x, x_pre);
773 delete [] x;
774 x = x_pre;
775 }
776 }
777
778
779
780 /*********************************************
781 * RemapDimensionsTransform
782 *********************************************/
783
784
785 RemapDimensionsTransform::RemapDimensionsTransform (
786 int d_in, int d_out, const int *map_in):
787 VectorTransform (d_in, d_out)
788 {
789 map.resize (d_out);
790 for (int i = 0; i < d_out; i++) {
791 map[i] = map_in[i];
792 FAISS_ASSERT (map[i] == -1 || (map[i] >= 0 && map[i] < d_in));
793 }
794 }
795
796 RemapDimensionsTransform::RemapDimensionsTransform (
797 int d_in, int d_out, bool uniform): VectorTransform (d_in, d_out)
798 {
799 map.resize (d_out, -1);
800
801 if (uniform) {
802 if (d_in < d_out) {
803 for (int i = 0; i < d_in; i++) {
804 map [i * d_out / d_in] = i;
805 }
806 } else {
807 for (int i = 0; i < d_out; i++) {
808 map [i] = i * d_in / d_out;
809 }
810 }
811 } else {
812 for (int i = 0; i < d_in && i < d_out; i++)
813 map [i] = i;
814 }
815 }
816
817
818 void RemapDimensionsTransform::apply_noalloc (idx_t n, const float * x,
819 float *xt) const
820 {
821 for (idx_t i = 0; i < n; i++) {
822 for (int j = 0; j < d_out; j++) {
823 xt[j] = map[j] < 0 ? 0 : x[map[j]];
824 }
825 x += d_in;
826 xt += d_out;
827 }
828 }
829
830 void RemapDimensionsTransform::reverse_transform (idx_t n, const float * xt,
831 float *x) const
832 {
833 memset (x, 0, sizeof (*x) * n * d_in);
834 for (idx_t i = 0; i < n; i++) {
835 for (int j = 0; j < d_out; j++) {
836 if (map[j] >= 0) x[map[j]] = xt[j];
837 }
838 x += d_in;
839 xt += d_out;
840 }
841 }
faiss::LinearTransform::transform_transpose
void transform_transpose(idx_t n, const float *y, float *x) const
Definition: VectorTransform.cpp:130
faiss::IndexPreTransform::index
Index * index
! chain of tranforms
Definition: VectorTransform.h:240
faiss::RandomRotationMatrix
Randomly rotate a set of vectors.
Definition: VectorTransform.h:113
faiss::ClusteringParameters::niter
int niter
clustering iterations
Definition: Clustering.h:26
faiss::OPQMatrix::niter
int niter
Number of outer training iterations.
Definition: VectorTransform.h:189
faiss::ProductQuantizer::decode
void decode(const uint8_t *code, float *x) const
decode a vector from a given code (or n vectors if third argument)
Definition: ProductQuantizer.cpp:338
faiss::fvec_L2sqr
float fvec_L2sqr(const float *x, const float *y, size_t d)
Squared L2 distance between two vectors.
Definition: utils.cpp:432
faiss::RandomRotationMatrix::init
void init(int seed)
must be called before the transform is used
Definition: VectorTransform.cpp:160
faiss::IndexPreTransform::reset
virtual void reset() override
removes all elements from the database.
Definition: VectorTransform.cpp:751
faiss::Index::reset
virtual void reset()=0
removes all elements from the database.
faiss::OPQMatrix::niter_pq
int niter_pq
Number of training iterations for the PQ.
Definition: VectorTransform.h:190
faiss::LinearTransform::A
std::vector< float > A
! whether to use the bias term
Definition: VectorTransform.h:84
faiss::fvecs_maybe_subsample
const float * fvecs_maybe_subsample(size_t d, size_t *n, size_t nmax, const float *x, bool verbose, long seed)
Definition: utils.cpp:1793
faiss::PCAMatrix::eigen_power
float eigen_power
Definition: VectorTransform.h:139
faiss::IDSelector
Definition: AuxIndexStructures.h:58
faiss::LinearTransform::LinearTransform
LinearTransform(int d_in=0, int d_out=0, bool have_bias=false)
both d_in &gt; d_out and d_out &lt; d_in are supported
Definition: VectorTransform.cpp:95
faiss::PCAMatrix
Definition: VectorTransform.h:131
faiss::Index::add_with_ids
virtual void add_with_ids(idx_t n, const float *x, const long *xids)
Definition: Index.cpp:32
faiss::PCAMatrix::train
virtual void train(Index::idx_t n, const float *x) override
Definition: VectorTransform.cpp:206
faiss::PCAMatrix::mean
std::vector< float > mean
Mean, size d_in.
Definition: VectorTransform.h:151
faiss::RemapDimensionsTransform::map
std::vector< int > map
Definition: VectorTransform.h:215
faiss::IndexPreTransform::apply_chain
const float * apply_chain(idx_t n, const float *x) const
Definition: VectorTransform.cpp:708
faiss::PCAMatrix::PCAMat
std::vector< float > PCAMat
PCA matrix, size d_in * d_in.
Definition: VectorTransform.h:157
faiss::IndexPreTransform::train
virtual void train(idx_t n, const float *x) override
Definition: VectorTransform.cpp:680
faiss::ProductQuantizer::compute_codes
void compute_codes(const float *x, uint8_t *codes, size_t n) const
same as compute_code for several vectors
Definition: ProductQuantizer.cpp:385
faiss::Index::d
int d
vector dimension
Definition: Index.h:66
faiss::LinearTransform::b
std::vector< float > b
bias vector, size d_out
Definition: VectorTransform.h:87
faiss::Index::reconstruct_n
virtual void reconstruct_n(idx_t i0, idx_t ni, float *recons) const
Definition: Index.cpp:50
faiss::Index::add
virtual void add(idx_t n, const float *x)=0
faiss::LinearTransform
Definition: VectorTransform.h:78
faiss::OPQMatrix::train
virtual void train(Index::idx_t n, const float *x) override
Definition: VectorTransform.cpp:450
faiss::PCAMatrix::balanced_bins
int balanced_bins
try to distribute output eigenvectors in this many bins
Definition: VectorTransform.h:148
faiss::VectorTransform::is_trained
bool is_trained
Definition: VectorTransform.h:43
faiss::Index::idx_t
long idx_t
all indices are this type
Definition: Index.h:64
faiss::IndexPreTransform::reconstruct_n
void reconstruct_n(idx_t i0, idx_t ni, float *recons) const override
Definition: VectorTransform.cpp:763
faiss::Index::ntotal
idx_t ntotal
total nb of indexed vectors
Definition: Index.h:67
faiss::ProductQuantizer::Train_hot_start
the centroids are already initialized
Definition: ProductQuantizer.h:43
faiss::getmillisecs
double getmillisecs()
ms elapsed since some arbitrary epoch
Definition: utils.cpp:71
faiss::Index::remove_ids
virtual long remove_ids(const IDSelector &sel)
Definition: Index.cpp:38
faiss::Index::search
virtual void search(idx_t n, const float *x, idx_t k, float *distances, idx_t *labels) const =0
faiss::LinearTransform::apply_noalloc
virtual void apply_noalloc(idx_t n, const float *x, float *xt) const
same as apply, but result is pre-allocated
Definition: VectorTransform.cpp:101
faiss::VectorTransform
Definition: VectorTransform.h:29
faiss::matrix_qr
void matrix_qr(int m, int n, float *a)
Definition: utils.cpp:1206
faiss::IndexPreTransform::own_fields
bool own_fields
! the sub-index
Definition: VectorTransform.h:242
faiss::OPQMatrix::niter_pq_0
int niter_pq_0
same, for the first outer iteration
Definition: VectorTransform.h:191
faiss::OPQMatrix::reverse_transform
virtual void reverse_transform(idx_t n, const float *xt, float *x) const override
Definition: VectorTransform.cpp:612
faiss::ProductQuantizer::cp
ClusteringParameters cp
parameters used during clustering
Definition: ProductQuantizer.h:50
faiss::ivec_checksum
size_t ivec_checksum(size_t n, const int *a)
compute a checksum on a table.
Definition: utils.cpp:1490
faiss::VectorTransform::train
virtual void train(idx_t n, const float *x)
Definition: VectorTransform.cpp:76
faiss::RandomRotationMatrix::reverse_transform
virtual void reverse_transform(idx_t n, const float *xt, float *x) const override
Definition: VectorTransform.cpp:185
faiss::VectorTransform::reverse_transform
virtual void reverse_transform(idx_t n, const float *xt, float *x) const
Definition: VectorTransform.cpp:81
faiss::RemapDimensionsTransform::reverse_transform
virtual void reverse_transform(idx_t n, const float *xt, float *x) const override
reverse transform correct only when the mapping is a permuation
Definition: VectorTransform.cpp:830
faiss::PCAMatrix::reverse_transform
virtual void reverse_transform(idx_t n, const float *xt, float *x) const override
Definition: VectorTransform.cpp:424
faiss::OPQMatrix::max_train_points
size_t max_train_points
if there are too many training points, resample
Definition: VectorTransform.h:194
faiss::PCAMatrix::copy_from
void copy_from(const PCAMatrix &other)
copy pre-trained PCA matrix
Definition: VectorTransform.cpp:319
faiss::VectorTransform::d_out
int d_out
! input dimension
Definition: VectorTransform.h:34
faiss::OPQMatrix::OPQMatrix
OPQMatrix(int d=0, int M=1, int d2=-1)
if d2 != -1, output vectors of this dimension
Definition: VectorTransform.cpp:437
faiss::Index
Definition: Index.h:61
faiss::PCAMatrix::prepare_Ab
void prepare_Ab()
called after mean, PCAMat and eigenvalues are computed
Definition: VectorTransform.cpp:329
faiss::IndexPreTransform::add
virtual void add(idx_t n, const float *x) override
Definition: VectorTransform.cpp:719
faiss::RemapDimensionsTransform::apply_noalloc
virtual void apply_noalloc(idx_t n, const float *x, float *xt) const override
same as apply, but result is pre-allocated
Definition: VectorTransform.cpp:818
faiss::ProductQuantizer
Definition: ProductQuantizer.h:26
faiss::Index::is_trained
bool is_trained
set if the Index does not require training, or if training is done already
Definition: Index.h:71
faiss::PCAMatrix::eigenvalues
std::vector< float > eigenvalues
eigenvalues of covariance matrix (= squared singular values)
Definition: VectorTransform.h:154
faiss::IndexPreTransform::search
virtual void search(idx_t n, const float *x, idx_t k, float *distances, idx_t *labels) const override
Definition: VectorTransform.cpp:741
faiss::Index::train
virtual void train(idx_t n, const float *x)
Definition: Index.h:92
faiss::IndexPreTransform::add_with_ids
virtual void add_with_ids(idx_t n, const float *x, const long *xids) override
Definition: VectorTransform.cpp:728
faiss::PCAMatrix::random_rotation
bool random_rotation
random rotation after PCA
Definition: VectorTransform.h:142
faiss::PCAMatrix::max_points_per_d
size_t max_points_per_d
ratio between # training vectors and dimension
Definition: VectorTransform.h:145
faiss::ClusteringParameters::max_points_per_centroid
int max_points_per_centroid
to limit size of dataset
Definition: Clustering.h:34
faiss::VectorTransform::apply
float * apply(idx_t n, const float *x) const
Definition: VectorTransform.cpp:68
faiss::IndexPreTransform::remove_ids
virtual long remove_ids(const IDSelector &sel) override
Definition: VectorTransform.cpp:756
faiss::VectorTransform::apply_noalloc
virtual void apply_noalloc(idx_t n, const float *x, float *xt) const =0
same as apply, but result is pre-allocated
faiss::OPQMatrix::M
int M
nb of subquantizers
Definition: VectorTransform.h:188