Faiss

- Main Page
- Namespaces
- Classes
- Files

- File List

All Classes Namespaces Functions Variables Typedefs Enumerations Enumerator Friends
/data/users/matthijs/github_faiss/faiss/IndexIVFPQ.cpp
1 /**
2 * Copyright (c) 2015-present, Facebook, Inc.
3 * All rights reserved.
4 *
5 * This source code is licensed under the BSD+Patents license found in the
6 * LICENSE file in the root directory of this source tree.
7 */
8
9 /* Copyright 2004-present Facebook. All Rights Reserved.
10 Inverted list structure.
11 */
12
13 #include "IndexIVFPQ.h"
14
15 #include <cmath>
16 #include <cstdio>
17 #include <cassert>
18
19 #include <algorithm>
20
21 #include "Heap.h"
22 #include "utils.h"
23
24 #include "Clustering.h"
25 #include "IndexFlat.h"
26
27 #include "hamming.h"
28
29 #include "FaissAssert.h"
30
31 #include "AuxIndexStructures.h"
32
33 namespace faiss {
34
35
36
37
38
39 /*****************************************
40 * IndexIVFPQ implementation
41 ******************************************/
42
43 IndexIVFPQ::IndexIVFPQ (Index * quantizer, size_t d, size_t nlist,
44 size_t M, size_t nbits_per_idx):
45 IndexIVF (quantizer, d, nlist, 0, METRIC_L2),
46 pq (d, M, nbits_per_idx)
47 {
48 FAISS_THROW_IF_NOT (nbits_per_idx <= 8);
49 code_size = pq.code_size;
50 invlists->code_size = code_size;
51 is_trained = false;
52 by_residual = true;
53 use_precomputed_table = 0;
54 scan_table_threshold = 0;
55
56 polysemous_training = nullptr;
57 do_polysemous_training = false;
58 polysemous_ht = 0;
59
60 }
61
62
63 /****************************************************************
64 * training */
65
66 void IndexIVFPQ::train_residual (idx_t n, const float *x)
67 {
68 train_residual_o (n, x, nullptr);
69 }
70
71
72 void IndexIVFPQ::train_residual_o (idx_t n, const float *x, float *residuals_2)
73 {
74 const float * x_in = x;
75
76 x = fvecs_maybe_subsample (
77 d, (size_t*)&n, pq.cp.max_points_per_centroid * pq.ksub,
78 x, verbose, pq.cp.seed);
79
80 ScopeDeleter<float> del_x (x_in == x ? nullptr : x);
81
82 const float *trainset;
83 ScopeDeleter<float> del_residuals;
84 if (by_residual) {
85 if(verbose) printf("computing residuals\n");
86 idx_t * assign = new idx_t [n]; // assignement to coarse centroids
87 ScopeDeleter<idx_t> del (assign);
88 quantizer->assign (n, x, assign);
89 float *residuals = new float [n * d];
90 del_residuals.set (residuals);
91 for (idx_t i = 0; i < n; i++)
92 quantizer->compute_residual (x + i * d, residuals+i*d, assign[i]);
93
94 trainset = residuals;
95 } else {
96 trainset = x;
97 }
98 if (verbose)
99 printf ("training %zdx%zd product quantizer on %ld vectors in %dD\n",
100 pq.M, pq.ksub, n, d);
101 pq.verbose = verbose;
102 pq.train (n, trainset);
103
104 if (do_polysemous_training) {
105 if (verbose)
106 printf("doing polysemous training for PQ\n");
107 PolysemousTraining default_pt;
108 PolysemousTraining *pt = polysemous_training;
109 if (!pt) pt = &default_pt;
110 pt->optimize_pq_for_hamming (pq, n, trainset);
111 }
112
113 // prepare second-level residuals for refine PQ
114 if (residuals_2) {
115 uint8_t *train_codes = new uint8_t [pq.code_size * n];
116 ScopeDeleter<uint8_t> del (train_codes);
117 pq.compute_codes (trainset, train_codes, n);
118
119 for (idx_t i = 0; i < n; i++) {
120 const float *xx = trainset + i * d;
121 float * res = residuals_2 + i * d;
122 pq.decode (train_codes + i * pq.code_size, res);
123 for (int j = 0; j < d; j++)
124 res[j] = xx[j] - res[j];
125 }
126
127 }
128
129 if (by_residual) {
130 precompute_table ();
131 }
132
133 }
134
135
136
137
138
139
140 /****************************************************************
141 * IVFPQ as codec */
142
143
144 /* produce a binary signature based on the residual vector */
145 void IndexIVFPQ::encode (long key, const float * x, uint8_t * code) const
146 {
147 if (by_residual) {
148 float residual_vec[d];
149 quantizer->compute_residual (x, residual_vec, key);
150 pq.compute_code (residual_vec, code);
151 }
152 else pq.compute_code (x, code);
153 }
154
155 void IndexIVFPQ::encode_multiple (size_t n, long *keys,
156 const float * x, uint8_t * xcodes,
157 bool compute_keys) const
158 {
159 if (compute_keys)
160 quantizer->assign (n, x, keys);
161
162 if (by_residual) {
163 float *residuals = new float [n * d];
164 ScopeDeleter<float> del (residuals);
165 // TODO: parallelize?
166 for (size_t i = 0; i < n; i++)
167 quantizer->compute_residual (x + i * d, residuals + i * d, keys[i]);
168 pq.compute_codes (residuals, xcodes, n);
169 } else {
170 pq.compute_codes (x, xcodes, n);
171 }
172 }
173
174 void IndexIVFPQ::decode_multiple (size_t n, const long *keys,
175 const uint8_t * xcodes, float * x) const
176 {
177 pq.decode (xcodes, x, n);
178 if (by_residual) {
179 std::vector<float> centroid (d);
180 for (size_t i = 0; i < n; i++) {
181 quantizer->reconstruct (keys[i], centroid.data());
182 float *xi = x + i * d;
183 for (size_t j = 0; j < d; j++) {
184 xi [j] += centroid [j];
185 }
186 }
187 }
188 }
189
190
191
192
193 /****************************************************************
194 * add */
195
196
197 void IndexIVFPQ::add_with_ids (idx_t n, const float * x, const long *xids)
198 {
199 add_core_o (n, x, xids, nullptr);
200 }
201
202
203 void IndexIVFPQ::add_core_o (idx_t n, const float * x, const long *xids,
204 float *residuals_2, const long *precomputed_idx)
205 {
206
207 idx_t bs = 32768;
208 if (n > bs) {
209 for (idx_t i0 = 0; i0 < n; i0 += bs) {
210 idx_t i1 = std::min(i0 + bs, n);
211 if (verbose) {
212 printf("IndexIVFPQ::add_core_o: adding %ld:%ld / %ld\n",
213 i0, i1, n);
214 }
215 add_core_o (i1 - i0, x + i0 * d,
216 xids ? xids + i0 : nullptr,
217 residuals_2 ? residuals_2 + i0 * d : nullptr,
218 precomputed_idx ? precomputed_idx + i0 : nullptr);
219 }
220 return;
221 }
222
223 FAISS_THROW_IF_NOT (is_trained);
224 double t0 = getmillisecs ();
225 const long * idx;
226 ScopeDeleter<long> del_idx;
227
228 if (precomputed_idx) {
229 idx = precomputed_idx;
230 } else {
231 long * idx0 = new long [n];
232 del_idx.set (idx0);
233 quantizer->assign (n, x, idx0);
234 idx = idx0;
235 }
236
237 double t1 = getmillisecs ();
238 uint8_t * xcodes = new uint8_t [n * code_size];
239 ScopeDeleter<uint8_t> del_xcodes (xcodes);
240
241 const float *to_encode = nullptr;
242 ScopeDeleter<float> del_to_encode;
243
244 if (by_residual) {
245 float *residuals = new float [n * d];
246 // TODO: parallelize?
247 for (size_t i = 0; i < n; i++) {
248 if (idx[i] < 0)
249 memset (residuals + i * d, 0, sizeof(*residuals) * d);
250 else
251 quantizer->compute_residual (
252 x + i * d, residuals + i * d, idx[i]);
253 }
254 to_encode = residuals;
255 del_to_encode.set (to_encode);
256 } else {
257 to_encode = x;
258 }
259 pq.compute_codes (to_encode, xcodes, n);
260
261 double t2 = getmillisecs ();
262 // TODO: parallelize?
263 size_t n_ignore = 0;
264 for (size_t i = 0; i < n; i++) {
265 idx_t key = idx[i];
266 if (key < 0) {
267 n_ignore ++;
268 if (residuals_2)
269 memset (residuals_2, 0, sizeof(*residuals_2) * d);
270 continue;
271 }
272 idx_t id = xids ? xids[i] : ntotal + i;
273
274 uint8_t *code = xcodes + i * code_size;
275 size_t offset = invlists->add_entry (key, id, code);
276
277 if (residuals_2) {
278 float *res2 = residuals_2 + i * d;
279 const float *xi = to_encode + i * d;
280 pq.decode (code, res2);
281 for (int j = 0; j < d; j++)
282 res2[j] = xi[j] - res2[j];
283 }
284
285 if (maintain_direct_map)
286 direct_map.push_back (key << 32 | offset);
287 }
288
289
290 double t3 = getmillisecs ();
291 if(verbose) {
292 char comment[100] = {0};
293 if (n_ignore > 0)
294 snprintf (comment, 100, "(%ld vectors ignored)", n_ignore);
295 printf(" add_core times: %.3f %.3f %.3f %s\n",
296 t1 - t0, t2 - t1, t3 - t2, comment);
297 }
298 ntotal += n;
299 }
300
301
302 void IndexIVFPQ::reconstruct_from_offset (long list_no, long offset,
303 float* recons) const
304 {
305 const uint8_t* code = invlists->get_single_code (list_no, offset);
306
307 if (by_residual) {
308 std::vector<float> centroid(d);
309 quantizer->reconstruct (list_no, centroid.data());
310
311 pq.decode (code, recons);
312 for (int i = 0; i < d; ++i) {
313 recons[i] += centroid[i];
314 }
315 } else {
316 pq.decode (code, recons);
317 }
318 }
319
320
321
322
323
324
325 /** Precomputed tables for residuals
326 *
327 * During IVFPQ search with by_residual, we compute
328 *
329 * d = || x - y_C - y_R ||^2
330 *
331 * where x is the query vector, y_C the coarse centroid, y_R the
332 * refined PQ centroid. The expression can be decomposed as:
333 *
334 * d = || x - y_C ||^2 + || y_R ||^2 + 2 * (y_C|y_R) - 2 * (x|y_R)
335 * --------------- --------------------------- -------
336 * term 1 term 2 term 3
337 *
338 * When using multiprobe, we use the following decomposition:
339 * - term 1 is the distance to the coarse centroid, that is computed
340 * during the 1st stage search.
341 * - term 2 can be precomputed, as it does not involve x. However,
342 * because of the PQ, it needs nlist * M * ksub storage. This is why
343 * use_precomputed_table is off by default
344 * - term 3 is the classical non-residual distance table.
345 *
346 * Since y_R defined by a product quantizer, it is split across
347 * subvectors and stored separately for each subvector. If the coarse
348 * quantizer is a MultiIndexQuantizer then the table can be stored
349 * more compactly.
350 *
351 * At search time, the tables for term 2 and term 3 are added up. This
352 * is faster when the length of the lists is > ksub * M.
353 */
354
355 void IndexIVFPQ::precompute_table ()
356 {
357 if (use_precomputed_table == 0) { // then choose the type of table
358 if (quantizer->metric_type == METRIC_INNER_PRODUCT) {
359 fprintf(stderr, "IndexIVFPQ::precompute_table: WARN precomputed "
360 "tables not needed for inner product quantizers\n");
361 return;
362 }
363 const MultiIndexQuantizer *miq =
364 dynamic_cast<const MultiIndexQuantizer *> (quantizer);
365 if (miq && pq.M % miq->pq.M == 0)
366 use_precomputed_table = 2;
367 else
368 use_precomputed_table = 1;
369 } // otherwise assume user has set appropriate flag on input
370
371 if (verbose) {
372 printf ("precomputing IVFPQ tables type %d\n",
373 use_precomputed_table);
374 }
375
376 // squared norms of the PQ centroids
377 std::vector<float> r_norms (pq.M * pq.ksub, NAN);
378 for (int m = 0; m < pq.M; m++)
379 for (int j = 0; j < pq.ksub; j++)
380 r_norms [m * pq.ksub + j] =
381 fvec_norm_L2sqr (pq.get_centroids (m, j), pq.dsub);
382
383 if (use_precomputed_table == 1) {
384
385 precomputed_table.resize (nlist * pq.M * pq.ksub);
386 std::vector<float> centroid (d);
387
388 for (size_t i = 0; i < nlist; i++) {
389 quantizer->reconstruct (i, centroid.data());
390
391 float *tab = &precomputed_table[i * pq.M * pq.ksub];
392 pq.compute_inner_prod_table (centroid.data(), tab);
393 fvec_madd (pq.M * pq.ksub, r_norms.data(), 2.0, tab, tab);
394 }
395 } else if (use_precomputed_table == 2) {
396 const MultiIndexQuantizer *miq =
397 dynamic_cast<const MultiIndexQuantizer *> (quantizer);
398 FAISS_THROW_IF_NOT (miq);
399 const ProductQuantizer &cpq = miq->pq;
400 FAISS_THROW_IF_NOT (pq.M % cpq.M == 0);
401
402 precomputed_table.resize(cpq.ksub * pq.M * pq.ksub);
403
404 // reorder PQ centroid table
405 std::vector<float> centroids (d * cpq.ksub, NAN);
406
407 for (int m = 0; m < cpq.M; m++) {
408 for (size_t i = 0; i < cpq.ksub; i++) {
409 memcpy (centroids.data() + i * d + m * cpq.dsub,
410 cpq.get_centroids (m, i),
411 sizeof (*centroids.data()) * cpq.dsub);
412 }
413 }
414
415 pq.compute_inner_prod_tables (cpq.ksub, centroids.data (),
416 precomputed_table.data ());
417
418 for (size_t i = 0; i < cpq.ksub; i++) {
419 float *tab = &precomputed_table[i * pq.M * pq.ksub];
420 fvec_madd (pq.M * pq.ksub, r_norms.data(), 2.0, tab, tab);
421 }
422
423 }
424 }
425
426 namespace {
427
428 static uint64_t get_cycles () {
429 uint32_t high, low;
430 asm volatile("rdtsc \n\t"
431 : "=a" (low),
432 "=d" (high));
433 return ((uint64_t)high << 32) | (low);
434 }
435
436 #define TIC t0 = get_cycles()
437 #define TOC get_cycles () - t0
438
439
440
441 /** QueryTables manages the various ways of searching an
442 * IndexIVFPQ. The code contains a lot of branches, depending on:
443 * - metric_type: are we computing L2 or Inner product similarity?
444 * - by_residual: do we encode raw vectors or residuals?
445 * - use_precomputed_table: are x_R|x_C tables precomputed?
446 * - polysemous_ht: are we filtering with polysemous codes?
447 */
448 struct QueryTables {
449
450 /*****************************************************
451 * General data from the IVFPQ
452 *****************************************************/
453
454 const IndexIVFPQ & ivfpq;
455
456 // copied from IndexIVFPQ for easier access
457 int d;
458 const ProductQuantizer & pq;
459 MetricType metric_type;
460 bool by_residual;
461 int use_precomputed_table;
462
463 // pre-allocated data buffers
464 float * sim_table, * sim_table_2;
465 float * residual_vec, *decoded_vec;
466
467 // single data buffer
468 std::vector<float> mem;
469
470 // for table pointers
471 std::vector<const float *> sim_table_ptrs;
472
473 explicit QueryTables (const IndexIVFPQ & ivfpq):
474 ivfpq(ivfpq),
475 d(ivfpq.d),
476 pq (ivfpq.pq),
477 metric_type (ivfpq.metric_type),
478 by_residual (ivfpq.by_residual),
479 use_precomputed_table (ivfpq.use_precomputed_table)
480 {
481 mem.resize (pq.ksub * pq.M * 2 + d * 2);
482 sim_table = mem.data ();
483 sim_table_2 = sim_table + pq.ksub * pq.M;
484 residual_vec = sim_table_2 + pq.ksub * pq.M;
485 decoded_vec = residual_vec + d;
486
487 // for polysemous
488 if (ivfpq.polysemous_ht != 0) {
489 q_code.resize (pq.code_size);
490 }
491 init_list_cycles = 0;
492 sim_table_ptrs.resize (pq.M);
493 }
494
495 /*****************************************************
496 * What we do when query is known
497 *****************************************************/
498
499 // field specific to query
500 const float * qi;
501
502 // query-specific intialization
503 void init_query (const float * qi) {
504 this->qi = qi;
505 if (metric_type == METRIC_INNER_PRODUCT)
506 init_query_IP ();
507 else
508 init_query_L2 ();
509 if (!by_residual && ivfpq.polysemous_ht != 0)
510 pq.compute_code (qi, q_code.data());
511 }
512
513 void init_query_IP () {
514 // precompute some tables specific to the query qi
515 pq.compute_inner_prod_table (qi, sim_table);
516 // we compute negated inner products for use with the maxheap
517 for (int i = 0; i < pq.ksub * pq.M; i++) {
518 sim_table[i] = - sim_table[i];
519 }
520 }
521
522 void init_query_L2 () {
523 if (!by_residual) {
524 pq.compute_distance_table (qi, sim_table);
525 } else if (use_precomputed_table) {
526 pq.compute_inner_prod_table (qi, sim_table_2);
527 }
528 }
529
530 /*****************************************************
531 * When inverted list is known: prepare computations
532 *****************************************************/
533
534 // fields specific to list
535 Index::idx_t key;
536 float coarse_dis;
537 std::vector<uint8_t> q_code;
538
539 uint64_t init_list_cycles;
540
541 /// once we know the query and the centroid, we can prepare the
542 /// sim_table that will be used for accumulation
543 /// and dis0, the initial value
544 float precompute_list_tables () {
545 float dis0 = 0;
546 uint64_t t0; TIC;
547 if (by_residual) {
548 if (metric_type == METRIC_INNER_PRODUCT)
549 dis0 = precompute_list_tables_IP ();
550 else
551 dis0 = precompute_list_tables_L2 ();
552 }
553 init_list_cycles += TOC;
554 return dis0;
555 }
556
557 float precompute_list_table_pointers () {
558 float dis0 = 0;
559 uint64_t t0; TIC;
560 if (by_residual) {
561 if (metric_type == METRIC_INNER_PRODUCT)
562 FAISS_THROW_MSG ("not implemented");
563 else
564 dis0 = precompute_list_table_pointers_L2 ();
565 }
566 init_list_cycles += TOC;
567 return dis0;
568 }
569
570 /*****************************************************
571 * compute tables for inner prod
572 *****************************************************/
573
574 float precompute_list_tables_IP ()
575 {
576 // prepare the sim_table that will be used for accumulation
577 // and dis0, the initial value
578 ivfpq.quantizer->reconstruct (key, decoded_vec);
579 // decoded_vec = centroid
580 float dis0 = -fvec_inner_product (qi, decoded_vec, d);
581
582 if (ivfpq.polysemous_ht) {
583 for (int i = 0; i < d; i++) {
584 residual_vec [i] = qi[i] - decoded_vec[i];
585 }
586 pq.compute_code (residual_vec, q_code.data());
587 }
588 return dis0;
589 }
590
591
592 /*****************************************************
593 * compute tables for L2 distance
594 *****************************************************/
595
596 float precompute_list_tables_L2 ()
597 {
598 float dis0 = 0;
599
600 if (use_precomputed_table == 0) {
601 ivfpq.quantizer->compute_residual (qi, residual_vec, key);
602 pq.compute_distance_table (residual_vec, sim_table);
603 } else if (use_precomputed_table == 1) {
604 dis0 = coarse_dis;
605
606 fvec_madd (pq.M * pq.ksub,
607 &ivfpq.precomputed_table [key * pq.ksub * pq.M],
608 -2.0, sim_table_2,
609 sim_table);
610 } else if (use_precomputed_table == 2) {
611 dis0 = coarse_dis;
612
613 const MultiIndexQuantizer *miq =
614 dynamic_cast<const MultiIndexQuantizer *> (ivfpq.quantizer);
615 FAISS_THROW_IF_NOT (miq);
616 const ProductQuantizer &cpq = miq->pq;
617 int Mf = pq.M / cpq.M;
618
619 const float *qtab = sim_table_2; // query-specific table
620 float *ltab = sim_table; // (output) list-specific table
621
622 long k = key;
623 for (int cm = 0; cm < cpq.M; cm++) {
624 // compute PQ index
625 int ki = k & ((uint64_t(1) << cpq.nbits) - 1);
626 k >>= cpq.nbits;
627
628 // get corresponding table
629 const float *pc = &ivfpq.precomputed_table
630 [(ki * pq.M + cm * Mf) * pq.ksub];
631
632 if (ivfpq.polysemous_ht == 0) {
633
634 // sum up with query-specific table
635 fvec_madd (Mf * pq.ksub,
636 pc,
637 -2.0, qtab,
638 ltab);
639 ltab += Mf * pq.ksub;
640 qtab += Mf * pq.ksub;
641 } else {
642 for (int m = cm * Mf; m < (cm + 1) * Mf; m++) {
643 q_code[m] = fvec_madd_and_argmin
644 (pq.ksub, pc, -2, qtab, ltab);
645 pc += pq.ksub;
646 ltab += pq.ksub;
647 qtab += pq.ksub;
648 }
649 }
650
651 }
652 }
653
654 return dis0;
655 }
656
657 float precompute_list_table_pointers_L2 ()
658 {
659 float dis0 = 0;
660
661 if (use_precomputed_table == 1) {
662 dis0 = coarse_dis;
663
664 const float * s = &ivfpq.precomputed_table [key * pq.ksub * pq.M];
665 for (int m = 0; m < pq.M; m++) {
666 sim_table_ptrs [m] = s;
667 s += pq.ksub;
668 }
669 } else if (use_precomputed_table == 2) {
670 dis0 = coarse_dis;
671
672 const MultiIndexQuantizer *miq =
673 dynamic_cast<const MultiIndexQuantizer *> (ivfpq.quantizer);
674 FAISS_THROW_IF_NOT (miq);
675 const ProductQuantizer &cpq = miq->pq;
676 int Mf = pq.M / cpq.M;
677
678 long k = key;
679 int m0 = 0;
680 for (int cm = 0; cm < cpq.M; cm++) {
681 int ki = k & ((uint64_t(1) << cpq.nbits) - 1);
682 k >>= cpq.nbits;
683
684 const float *pc = &ivfpq.precomputed_table
685 [(ki * pq.M + cm * Mf) * pq.ksub];
686
687 for (int m = m0; m < m0 + Mf; m++) {
688 sim_table_ptrs [m] = pc;
689 pc += pq.ksub;
690 }
691 m0 += Mf;
692 }
693 } else {
694 FAISS_THROW_MSG ("need precomputed tables");
695 }
696
697 if (ivfpq.polysemous_ht) {
698 FAISS_THROW_MSG ("not implemented");
699 // Not clear that it makes sense to implemente this,
700 // because it costs M * ksub, which is what we wanted to
701 // avoid with the tables pointers.
702 }
703
704 return dis0;
705 }
706
707
708 };
709
710
711 /*****************************************************
712 * Scaning the codes.
713 * The scanning functions call their favorite precompute_*
714 * function to precompute the tables they need.
715 *****************************************************/
716 template <typename IDType>
717 struct InvertedListScanner: QueryTables {
718
719 const uint8_t * __restrict list_codes;
720 const IDType * list_ids;
721 size_t list_size;
722
723 explicit InvertedListScanner (const IndexIVFPQ & ivfpq):
724 QueryTables (ivfpq)
725 {
726 FAISS_THROW_IF_NOT (pq.byte_per_idx == 1);
727 n_hamming_pass = 0;
728 }
729
730 /// list_specific intialization
731 void init_list (Index::idx_t key, float coarse_dis,
732 size_t list_size_in, const IDType *list_ids_in,
733 const uint8_t *list_codes_in) {
734 this->key = key;
735 this->coarse_dis = coarse_dis;
736 list_size = list_size_in;
737 list_codes = list_codes_in;
738 list_ids = list_ids_in;
739 }
740
741 /*****************************************************
742 * Scaning the codes: simple PQ scan.
743 *****************************************************/
744
745 /// version of the scan where we use precomputed tables
746 void scan_list_with_table (
747 size_t k, float * heap_sim, long * heap_ids, bool store_pairs)
748 {
749 float dis0 = precompute_list_tables ();
750
751 for (size_t j = 0; j < list_size; j++) {
752
753 float dis = dis0;
754 const float *tab = sim_table;
755
756 for (size_t m = 0; m < pq.M; m++) {
757 dis += tab[*list_codes++];
758 tab += pq.ksub;
759 }
760
761 if (dis < heap_sim[0]) {
762 maxheap_pop (k, heap_sim, heap_ids);
763 long id = store_pairs ? (key << 32 | j) : list_ids[j];
764 maxheap_push (k, heap_sim, heap_ids, dis, id);
765 }
766 }
767 }
768
769
770 /// tables are not precomputed, but pointers are provided to the
771 /// relevant X_c|x_r tables
772 void scan_list_with_pointer (
773 size_t k, float * heap_sim, long * heap_ids, bool store_pairs)
774 {
775
776 float dis0 = precompute_list_table_pointers ();
777
778 for (size_t j = 0; j < list_size; j++) {
779
780 float dis = dis0;
781 const float *tab = sim_table_2;
782
783 for (size_t m = 0; m < pq.M; m++) {
784 int ci = *list_codes++;
785 dis += sim_table_ptrs [m][ci] - 2 * tab [ci];
786 tab += pq.ksub;
787 }
788
789 if (dis < heap_sim[0]) {
790 maxheap_pop (k, heap_sim, heap_ids);
791 long id = store_pairs ? (key << 32 | j) : list_ids[j];
792 maxheap_push (k, heap_sim, heap_ids, dis, id);
793 }
794 }
795
796 }
797
798 /// nothing is precomputed: access residuals on-the-fly
799 void scan_on_the_fly_dist (
800 size_t k, float * heap_sim, long * heap_ids, bool store_pairs)
801 {
802
803 if (by_residual && use_precomputed_table) {
804 scan_list_with_pointer (k, heap_sim, heap_ids, store_pairs);
805 return;
806 }
807
808 const float *dvec;
809 float dis0 = 0;
810
811 if (by_residual) {
812 if (metric_type == METRIC_INNER_PRODUCT) {
813 ivfpq.quantizer->reconstruct (key, residual_vec);
814 dis0 = fvec_inner_product (residual_vec, qi, d);
815 } else {
816 ivfpq.quantizer->compute_residual (qi, residual_vec, key);
817 }
818 dvec = residual_vec;
819 } else {
820 dvec = qi;
821 dis0 = 0;
822 }
823
824 for (size_t j = 0; j < list_size; j++) {
825
826 pq.decode (list_codes, decoded_vec);
827 list_codes += pq.code_size;
828
829 float dis;
830 if (metric_type == METRIC_INNER_PRODUCT) {
831 dis = -dis0 - fvec_inner_product (decoded_vec, qi, d);
832 } else {
833 dis = fvec_L2sqr (decoded_vec, dvec, d);
834 }
835
836 if (dis < heap_sim[0]) {
837 maxheap_pop (k, heap_sim, heap_ids);
838 long id = store_pairs ? (key << 32 | j) : list_ids[j];
839 maxheap_push (k, heap_sim, heap_ids, dis, id);
840 }
841 }
842 }
843
844 /*****************************************************
845 * Scanning codes with polysemous filtering
846 *****************************************************/
847
848 // code for the query
849 size_t n_hamming_pass;
850
851
852 template <class HammingComputer>
853 void scan_list_polysemous_hc (
854 size_t k, float * heap_sim, long * heap_ids, bool store_pairs)
855 {
856 float dis0 = precompute_list_tables ();
857 int ht = ivfpq.polysemous_ht;
858
859 int code_size = pq.code_size;
860
861 HammingComputer hc (q_code.data(), code_size);
862
863 for (size_t j = 0; j < list_size; j++) {
864 const uint8_t *b_code = list_codes;
865 int hd = hc.hamming (b_code);
866 if (hd < ht) {
867 n_hamming_pass ++;
868
869 float dis = dis0;
870 const float *tab = sim_table;
871
872 for (size_t m = 0; m < pq.M; m++) {
873 dis += tab[*b_code++];
874 tab += pq.ksub;
875 }
876
877 if (dis < heap_sim[0]) {
878 maxheap_pop (k, heap_sim, heap_ids);
879 long id = store_pairs ? (key << 32 | j) : list_ids[j];
880 maxheap_push (k, heap_sim, heap_ids, dis, id);
881 }
882 }
883 list_codes += code_size;
884 }
885 }
886
887 void scan_list_polysemous (
888 size_t k, float * heap_sim, long * heap_ids, bool store_pairs)
889 {
890 switch (pq.code_size) {
891 #define HANDLE_CODE_SIZE(cs) \
892 case cs: \
893 scan_list_polysemous_hc <HammingComputer ## cs> \
894 (k, heap_sim, heap_ids, store_pairs); \
895 break
896 HANDLE_CODE_SIZE(4);
897 HANDLE_CODE_SIZE(8);
898 HANDLE_CODE_SIZE(16);
899 HANDLE_CODE_SIZE(20);
900 HANDLE_CODE_SIZE(32);
901 HANDLE_CODE_SIZE(64);
902 #undef HANDLE_CODE_SIZE
903 default:
904 if (pq.code_size % 8 == 0)
905 scan_list_polysemous_hc <HammingComputerM8>
906 (k, heap_sim, heap_ids, store_pairs);
907 else
908 scan_list_polysemous_hc <HammingComputerM4>
909 (k, heap_sim, heap_ids, store_pairs);
910 break;
911 }
912 }
913
914 };
915
916
917
918
919 } // anonymous namespace
920
921
922 IndexIVFPQStats indexIVFPQ_stats;
923
924 void IndexIVFPQStats::reset () {
925 memset (this, 0, sizeof (*this));
926 }
927
928
929
930 void IndexIVFPQ::search_preassigned (idx_t nx, const float *qx, idx_t k,
931 const idx_t *keys,
932 const float *coarse_dis,
933 float *distances, idx_t *labels,
934 bool store_pairs) const
935 {
936 float_maxheap_array_t res = {
937 size_t(nx), size_t(k),
938 labels, distances
939 };
940
941 #pragma omp parallel
942 {
943 InvertedListScanner<long> qt (*this);
944 size_t stats_nlist = 0;
945 size_t stats_ncode = 0;
946 uint64_t init_query_cycles = 0;
947 uint64_t scan_cycles = 0;
948 uint64_t heap_cycles = 0;
949
950 #pragma omp for
951 for (size_t i = 0; i < nx; i++) {
952 const float *qi = qx + i * d;
953 const long * keysi = keys + i * nprobe;
954 const float *coarse_dis_i = coarse_dis + i * nprobe;
955 float * heap_sim = res.get_val (i);
956 long * heap_ids = res.get_ids (i);
957
958 uint64_t t0;
959 TIC;
960 maxheap_heapify (k, heap_sim, heap_ids);
961 heap_cycles += TOC;
962
963 TIC;
964 qt.init_query (qi);
965 init_query_cycles += TOC;
966
967 size_t nscan = 0;
968
969 for (size_t ik = 0; ik < nprobe; ik++) {
970 long key = keysi[ik]; /* select the list */
971 if (key < 0) {
972 // not enough centroids for multiprobe
973 continue;
974 }
975
976 size_t list_size = invlists->list_size (key);
977 stats_nlist ++;
978 nscan += list_size;
979
980 if (list_size == 0) continue;
981
982 qt.init_list (key, coarse_dis_i[ik],
983 list_size, invlists->get_ids (key),
984 invlists->get_codes (key));
985
986 TIC;
987 if (polysemous_ht > 0) {
988 qt.scan_list_polysemous
989 (k, heap_sim, heap_ids, store_pairs);
990 } else if (list_size > scan_table_threshold) {
991 qt.scan_list_with_table (k, heap_sim, heap_ids, store_pairs);
992 } else {
993 qt.scan_on_the_fly_dist (k, heap_sim, heap_ids, store_pairs);
994 }
995 scan_cycles += TOC;
996
997 if (max_codes && nscan >= max_codes) break;
998 }
999 stats_ncode += nscan;
1000 TIC;
1001 maxheap_reorder (k, heap_sim, heap_ids);
1002
1003 if (metric_type == METRIC_INNER_PRODUCT) {
1004 for (size_t j = 0; j < k; j++)
1005 heap_sim[j] = -heap_sim[j];
1006 }
1007 heap_cycles += TOC;
1008 }
1009
1010 #pragma omp critical
1011 {
1012 indexIVFPQ_stats.n_hamming_pass += qt.n_hamming_pass;
1013 indexIVFPQ_stats.nlist += stats_nlist;
1014 indexIVFPQ_stats.ncode += stats_ncode;
1015
1016 indexIVFPQ_stats.init_query_cycles += init_query_cycles;
1017 indexIVFPQ_stats.init_list_cycles += qt.init_list_cycles;
1018 indexIVFPQ_stats.scan_cycles += scan_cycles - qt.init_list_cycles;
1019 indexIVFPQ_stats.heap_cycles += heap_cycles;
1020 }
1021
1022 }
1023 indexIVFPQ_stats.nq += nx;
1024 }
1025
1026
1027 IndexIVFPQ::IndexIVFPQ ()
1028 {
1029 // initialize some runtime values
1030 use_precomputed_table = 0;
1031 scan_table_threshold = 0;
1032 do_polysemous_training = false;
1033 polysemous_ht = 0;
1034 polysemous_training = nullptr;
1035 }
1036
1037
1038 struct CodeCmp {
1039 const uint8_t *tab;
1040 size_t code_size;
1041 bool operator () (int a, int b) const {
1042 return cmp (a, b) > 0;
1043 }
1044 int cmp (int a, int b) const {
1045 return memcmp (tab + a * code_size, tab + b * code_size,
1046 code_size);
1047 }
1048 };
1049
1050
1051 size_t IndexIVFPQ::find_duplicates (idx_t *dup_ids, size_t *lims) const
1052 {
1053 size_t ngroup = 0;
1054 lims[0] = 0;
1055 for (size_t list_no = 0; list_no < nlist; list_no++) {
1056 size_t n = invlists->list_size (list_no);
1057 std::vector<int> ord (n);
1058 for (int i = 0; i < n; i++) ord[i] = i;
1059 CodeCmp cs = { invlists->get_codes (list_no), code_size };
1060 std::sort (ord.begin(), ord.end(), cs);
1061
1062 const idx_t *list_ids = invlists->get_ids (list_no);
1063 int prev = -1; // all elements from prev to i-1 are equal
1064 for (int i = 0; i < n; i++) {
1065 if (prev >= 0 && cs.cmp (ord [prev], ord [i]) == 0) {
1066 // same as previous => remember
1067 if (prev + 1 == i) { // start new group
1068 ngroup++;
1069 lims[ngroup] = lims[ngroup - 1];
1070 dup_ids [lims [ngroup]++] = list_ids [ord [prev]];
1071 }
1072 dup_ids [lims [ngroup]++] = list_ids [ord [i]];
1073 } else { // not same as previous.
1074 prev = i;
1075 }
1076 }
1077 }
1078 return ngroup;
1079 }
1080
1081
1082
1083
1084 /*****************************************
1085 * IndexIVFPQR implementation
1086 ******************************************/
1087
1088 IndexIVFPQR::IndexIVFPQR (
1089 Index * quantizer, size_t d, size_t nlist,
1090 size_t M, size_t nbits_per_idx,
1091 size_t M_refine, size_t nbits_per_idx_refine):
1092 IndexIVFPQ (quantizer, d, nlist, M, nbits_per_idx),
1093 refine_pq (d, M_refine, nbits_per_idx_refine),
1094 k_factor (4)
1095 {
1096 by_residual = true;
1097 }
1098
1099 IndexIVFPQR::IndexIVFPQR ():
1100 k_factor (1)
1101 {
1102 by_residual = true;
1103 }
1104
1105
1106
1107 void IndexIVFPQR::reset()
1108 {
1109 IndexIVFPQ::reset();
1110 refine_codes.clear();
1111 }
1112
1113
1114
1115
1116 void IndexIVFPQR::train_residual (idx_t n, const float *x)
1117 {
1118
1119 float * residual_2 = new float [n * d];
1120 ScopeDeleter <float> del(residual_2);
1121
1122 train_residual_o (n, x, residual_2);
1123
1124 if (verbose)
1125 printf ("training %zdx%zd 2nd level PQ quantizer on %ld %dD-vectors\n",
1126 refine_pq.M, refine_pq.ksub, n, d);
1127
1128 refine_pq.cp.max_points_per_centroid = 1000;
1129 refine_pq.cp.verbose = verbose;
1130
1131 refine_pq.train (n, residual_2);
1132
1133 }
1134
1135
1136 void IndexIVFPQR::add_with_ids (idx_t n, const float *x, const long *xids) {
1137 add_core (n, x, xids, nullptr);
1138 }
1139
1140 void IndexIVFPQR::add_core (idx_t n, const float *x, const long *xids,
1141 const long *precomputed_idx) {
1142
1143 float * residual_2 = new float [n * d];
1144 ScopeDeleter <float> del(residual_2);
1145
1146 idx_t n0 = ntotal;
1147
1148 add_core_o (n, x, xids, residual_2, precomputed_idx);
1149
1150 refine_codes.resize (ntotal * refine_pq.code_size);
1151
1152 refine_pq.compute_codes (
1153 residual_2, &refine_codes[n0 * refine_pq.code_size], n);
1154
1155
1156 }
1157
1158
1159 void IndexIVFPQR::search_preassigned (idx_t n, const float *x, idx_t k,
1160 const idx_t *idx,
1161 const float *L1_dis,
1162 float *distances, idx_t *labels,
1163 bool store_pairs) const
1164 {
1165 uint64_t t0;
1166 TIC;
1167 size_t k_coarse = long(k * k_factor);
1168 idx_t *coarse_labels = new idx_t [k_coarse * n];
1169 ScopeDeleter<idx_t> del1 (coarse_labels);
1170 { // query with quantizer levels 1 and 2.
1171 float *coarse_distances = new float [k_coarse * n];
1172 ScopeDeleter<float> del(coarse_distances);
1173
1174 IndexIVFPQ::search_preassigned (n, x, k_coarse,
1175 idx, L1_dis, coarse_distances, coarse_labels,
1176 true);
1177 }
1178
1179
1180 indexIVFPQ_stats.search_cycles += TOC;
1181
1182 TIC;
1183
1184 // 3rd level refinement
1185 size_t n_refine = 0;
1186 #pragma omp parallel reduction(+ : n_refine)
1187 {
1188 // tmp buffers
1189 float *residual_1 = new float [2 * d];
1190 ScopeDeleter<float> del (residual_1);
1191 float *residual_2 = residual_1 + d;
1192 #pragma omp for
1193 for (idx_t i = 0; i < n; i++) {
1194 const float *xq = x + i * d;
1195 const long * shortlist = coarse_labels + k_coarse * i;
1196 float * heap_sim = distances + k * i;
1197 long * heap_ids = labels + k * i;
1198 maxheap_heapify (k, heap_sim, heap_ids);
1199
1200 for (int j = 0; j < k_coarse; j++) {
1201 long sl = shortlist[j];
1202
1203 if (sl == -1) continue;
1204
1205 int list_no = sl >> 32;
1206 int ofs = sl & 0xffffffff;
1207
1208 assert (list_no >= 0 && list_no < nlist);
1209 assert (ofs >= 0 && ofs < invlists->list_size (list_no));
1210
1211 // 1st level residual
1212 quantizer->compute_residual (xq, residual_1, list_no);
1213
1214 // 2nd level residual
1215 const uint8_t * l2code =
1216 invlists->get_single_code (list_no, ofs);
1217
1218 pq.decode (l2code, residual_2);
1219 for (int l = 0; l < d; l++)
1220 residual_2[l] = residual_1[l] - residual_2[l];
1221
1222 // 3rd level residual's approximation
1223 idx_t id = invlists->get_single_id (list_no, ofs);
1224 assert (0 <= id && id < ntotal);
1225 refine_pq.decode (&refine_codes [id * refine_pq.code_size],
1226 residual_1);
1227
1228 float dis = fvec_L2sqr (residual_1, residual_2, d);
1229
1230 if (dis < heap_sim[0]) {
1231 maxheap_pop (k, heap_sim, heap_ids);
1232 long id_or_pair = store_pairs ? sl : id;
1233 maxheap_push (k, heap_sim, heap_ids, dis, id_or_pair);
1234 }
1235 n_refine ++;
1236 }
1237 maxheap_reorder (k, heap_sim, heap_ids);
1238 }
1239 }
1240 indexIVFPQ_stats.nrefine += n_refine;
1241 indexIVFPQ_stats.refine_cycles += TOC;
1242 }
1243
1244 void IndexIVFPQR::reconstruct_from_offset (long list_no, long offset,
1245 float* recons) const
1246 {
1247 IndexIVFPQ::reconstruct_from_offset (list_no, offset, recons);
1248
1249 idx_t id = invlists->get_single_id (list_no, offset);
1250 assert (0 <= id && id < ntotal);
1251
1252 std::vector<float> r3(d);
1253 refine_pq.decode (&refine_codes [id * refine_pq.code_size], r3.data());
1254 for (int i = 0; i < d; ++i) {
1255 recons[i] += r3[i];
1256 }
1257 }
1258
1259 void IndexIVFPQR::merge_from (IndexIVF &other_in, idx_t add_id)
1260 {
1261 IndexIVFPQR *other = dynamic_cast<IndexIVFPQR *> (&other_in);
1262 FAISS_THROW_IF_NOT(other);
1263
1264 IndexIVF::merge_from (other_in, add_id);
1265
1266 refine_codes.insert (refine_codes.end(),
1267 other->refine_codes.begin(),
1268 other->refine_codes.end());
1269 other->refine_codes.clear();
1270 }
1271
1272 long IndexIVFPQR::remove_ids(const IDSelector& /*sel*/) {
1273 FAISS_THROW_MSG("not implemented");
1274 return 0;
1275 }
1276
1277 /*************************************
1278 * Index2Layer implementation
1279 *************************************/
1280
1281
1282 Index2Layer::Index2Layer (Index * quantizer, size_t nlist,
1283 int M,
1284 MetricType metric):
1285 Index (quantizer->d, metric),
1286 q1 (quantizer, nlist),
1287 pq (quantizer->d, M, 8)
1288 {
1289 is_trained = false;
1290 for (int nbyte = 0; nbyte < 7; nbyte++) {
1291 if ((1L << (8 * nbyte)) >= nlist) {
1292 code_size_1 = nbyte;
1293 break;
1294 }
1295 }
1296 code_size_2 = pq.code_size;
1297 code_size = code_size_1 + code_size_2;
1298 }
1299
1300 Index2Layer::Index2Layer ()
1301 {
1302 code_size = code_size_1 = code_size_2 = 0;
1303 }
1304
1305 Index2Layer::~Index2Layer ()
1306 {}
1307
1308 void Index2Layer::train(idx_t n, const float* x)
1309 {
1310 if (verbose) {
1311 printf ("training level-1 quantizer %ld vectors in %dD\n",
1312 n, d);
1313 }
1314
1315 q1.train_q1 (n, x, verbose, metric_type);
1316
1317 if (verbose) {
1318 printf("computing residuals\n");
1319 }
1320
1321 const float * x_in = x;
1322
1323 x = fvecs_maybe_subsample (
1324 d, (size_t*)&n, pq.cp.max_points_per_centroid * pq.ksub,
1325 x, verbose, pq.cp.seed);
1326
1327 ScopeDeleter<float> del_x (x_in == x ? nullptr : x);
1328
1329 std::vector<idx_t> assign(n); // assignement to coarse centroids
1330 q1.quantizer->assign (n, x, assign.data());
1331 std::vector<float> residuals(n * d);
1332 for (idx_t i = 0; i < n; i++) {
1333 q1.quantizer->compute_residual (
1334 x + i * d, residuals.data() + i * d, assign[i]);
1335 }
1336
1337 if (verbose)
1338 printf ("training %zdx%zd product quantizer on %ld vectors in %dD\n",
1339 pq.M, pq.ksub, n, d);
1340 pq.verbose = verbose;
1341 pq.train (n, residuals.data());
1342
1343 is_trained = true;
1344 }
1345
1346 void Index2Layer::add(idx_t n, const float* x)
1347 {
1348 idx_t bs = 32768;
1349 if (n > bs) {
1350 for (idx_t i0 = 0; i0 < n; i0 += bs) {
1351 idx_t i1 = std::min(i0 + bs, n);
1352 if (verbose) {
1353 printf("Index2Layer::add: adding %ld:%ld / %ld\n",
1354 i0, i1, n);
1355 }
1356 add (i1 - i0, x + i0 * d);
1357 }
1358 return;
1359 }
1360
1361 std::vector<idx_t> codes1 (n);
1362 q1.quantizer->assign (n, x, codes1.data());
1363 std::vector<float> residuals(n * d);
1364 for (idx_t i = 0; i < n; i++) {
1365 q1.quantizer->compute_residual (
1366 x + i * d, residuals.data() + i * d, codes1[i]);
1367 }
1368 std::vector<uint8_t> codes2 (n * code_size_2);
1369
1370 pq.compute_codes (residuals.data(), codes2.data(), n);
1371
1372 codes.resize ((ntotal + n) * code_size);
1373 uint8_t *wp = &codes[ntotal * code_size];
1374
1375 {
1376 int i = 0x11223344;
1377 const char *ip = (char*)&i;
1378 FAISS_THROW_IF_NOT_MSG (ip[0] == 0x44,
1379 "works only on a little-endian CPU");
1380 }
1381
1382 // copy to output table
1383 for (idx_t i = 0; i < n; i++) {
1384 memcpy (wp, &codes1[i], code_size_1);
1385 wp += code_size_1;
1386 memcpy (wp, &codes2[i * code_size_2], code_size_2);
1387 wp += code_size_2;
1388 }
1389
1390 ntotal += n;
1391
1392 }
1393
1394 void Index2Layer::search(
1395 idx_t n,
1396 const float* x,
1397 idx_t k,
1398 float* distances,
1399 idx_t* labels) const
1400 {
1401 FAISS_THROW_MSG ("not implemented");
1402 }
1403
1404
1405 void Index2Layer::reconstruct_n(idx_t i0, idx_t ni, float* recons) const
1406 {
1407 float recons1[d];
1408 FAISS_THROW_IF_NOT (i0 >= 0 && i0 + ni <= ntotal);
1409 const uint8_t *rp = &codes[i0 * code_size];
1410
1411 for (idx_t i = 0; i < ni; i++) {
1412 idx_t key = 0;
1413 memcpy (&key, rp, code_size_1);
1414 q1.quantizer->reconstruct (key, recons1);
1415 rp += code_size_1;
1416 pq.decode (rp, recons);
1417 for (idx_t j = 0; j < d; j++) {
1418 recons[j] += recons1[j];
1419 }
1420 rp += code_size_2;
1421 recons += d;
1422 }
1423 }
1424
1425 void Index2Layer::transfer_to_IVFPQ (IndexIVFPQ & other) const
1426 {
1427 FAISS_THROW_IF_NOT (other.nlist == q1.nlist);
1428 FAISS_THROW_IF_NOT (other.code_size == code_size_2);
1429 FAISS_THROW_IF_NOT (other.ntotal == 0);
1430
1431 const uint8_t *rp = codes.data();
1432
1433 for (idx_t i = 0; i < ntotal; i++) {
1434 idx_t key = 0;
1435 memcpy (&key, rp, code_size_1);
1436 rp += code_size_1;
1437 other.invlists->add_entry (key, i, rp);
1438 rp += code_size_2;
1439 }
1440
1441 other.ntotal = ntotal;
1442
1443 }
1444
1445
1446
1447 void Index2Layer::reconstruct(idx_t key, float* recons) const
1448 {
1449 reconstruct_n (key, 1, recons);
1450 }
1451
1452 void Index2Layer::reset()
1453 {
1454 ntotal = 0;
1455 codes.clear ();
1456 }
1457
1458
1459
1460
1461
1462
1463
1464
1465 } // namespace faiss
faiss::IndexIVF
Definition: IndexIVF.h:166
faiss::IndexIVFPQ::precompute_table
void precompute_table()
build precomputed table
Definition: IndexIVFPQ.cpp:355
faiss::IndexIVFPQR::merge_from
void merge_from(IndexIVF &other, idx_t add_id) override
Definition: IndexIVFPQ.cpp:1259
faiss::Index2Layer::transfer_to_IVFPQ
void transfer_to_IVFPQ(IndexIVFPQ &other) const
transfer the flat codes to an IVFPQ index
Definition: IndexIVFPQ.cpp:1425
faiss::Index2Layer::code_size_2
size_t code_size_2
size of the code for the second level
Definition: IndexIVFPQ.h:210
faiss::IndexIVFPQR::reconstruct_from_offset
void reconstruct_from_offset(long list_no, long offset, float *recons) const override
Definition: IndexIVFPQ.cpp:1244
faiss::IndexIVFPQR::search_preassigned
void search_preassigned(idx_t n, const float *x, idx_t k, const idx_t *assign, const float *centroid_dis, float *distances, idx_t *labels, bool store_pairs) const override
Definition: IndexIVFPQ.cpp:1159
faiss::IndexIVFPQR
Definition: IndexIVFPQ.h:149
faiss::ProductQuantizer::decode
void decode(const uint8_t *code, float *x) const
decode a vector from a given code (or n vectors if third argument)
Definition: ProductQuantizer.cpp:339
faiss::IndexIVFPQR::refine_pq
ProductQuantizer refine_pq
3rd level quantizer
Definition: IndexIVFPQ.h:150
faiss::fvec_L2sqr
float fvec_L2sqr(const float *x, const float *y, size_t d)
Squared L2 distance between two vectors.
Definition: utils.cpp:574
faiss::IndexIVFPQ::polysemous_training
PolysemousTraining * polysemous_training
if NULL, use default
Definition: IndexIVFPQ.h:35
faiss::InvertedLists::get_ids
virtual const idx_t * get_ids(size_t list_no) const =0
faiss::HeapArray::get_val
T * get_val(size_t key)
Return the list of values for a heap.
Definition: Heap.h:360
faiss::Index2Layer::reconstruct_n
void reconstruct_n(idx_t i0, idx_t ni, float *recons) const override
Definition: IndexIVFPQ.cpp:1405
faiss::Index2Layer::code_size
size_t code_size
code_size_1 + code_size_2
Definition: IndexIVFPQ.h:213
faiss::fvecs_maybe_subsample
const float * fvecs_maybe_subsample(size_t d, size_t *n, size_t nmax, const float *x, bool verbose, long seed)
Definition: utils.cpp:1975
faiss::IndexIVF::nprobe
size_t nprobe
number of probes at query time
Definition: IndexIVF.h:173
faiss::Index2Layer::reset
void reset() override
removes all elements from the database.
Definition: IndexIVFPQ.cpp:1452
faiss::Index::assign
void assign(idx_t n, const float *x, idx_t *labels, idx_t k=1)
Definition: Index.cpp:34
faiss::IDSelector
Definition: AuxIndexStructures.h:52
faiss::InvertedLists::list_size
virtual size_t list_size(size_t list_no) const =0
get the size of a list
faiss::IndexIVFPQ::decode_multiple
void decode_multiple(size_t n, const long *keys, const uint8_t *xcodes, float *x) const
inverse of encode_multiple
Definition: IndexIVFPQ.cpp:174
faiss::IndexIVFPQ::train_residual_o
void train_residual_o(idx_t n, const float *x, float *residuals_2)
same as train_residual, also output 2nd level residuals
Definition: IndexIVFPQ.cpp:72
faiss::IndexIVFPQ::do_polysemous_training
bool do_polysemous_training
reorder PQ centroids after training?
Definition: IndexIVFPQ.h:34
faiss::IndexIVFPQ::scan_table_threshold
size_t scan_table_threshold
use table computation or on-the-fly?
Definition: IndexIVFPQ.h:38
faiss::IndexIVFPQR::train_residual
void train_residual(idx_t n, const float *x) override
trains the two product quantizers
Definition: IndexIVFPQ.cpp:1116
faiss::IndexIVFPQR::add_core
void add_core(idx_t n, const float *x, const long *xids, const long *precomputed_idx=nullptr)
same as add_with_ids, but optionally use the precomputed list ids
Definition: IndexIVFPQ.cpp:1140
faiss::ScopeDeleter
Definition: FaissException.h:39
faiss::ProductQuantizer::dsub
size_t dsub
dimensionality of each subvector
Definition: ProductQuantizer.h:32
faiss::ClusteringParameters::seed
int seed
seed for the random number generator
Definition: Clustering.h:36
faiss::IndexIVFPQ::precomputed_table
std::vector< float > precomputed_table
Definition: IndexIVFPQ.h:44
faiss::Index2Layer::q1
Level1Quantizer q1
first level quantizer
Definition: IndexIVFPQ.h:198
faiss::fvec_madd
void fvec_madd(size_t n, const float *a, float bf, const float *b, float *c)
Definition: utils.cpp:1889
faiss::IndexIVFPQ::polysemous_ht
int polysemous_ht
Hamming thresh for polysemous filtering.
Definition: IndexIVFPQ.h:39
faiss::IndexIVFPQR::reset
void reset() override
removes all elements from the database.
Definition: IndexIVFPQ.cpp:1107
faiss::Index2Layer::codes
std::vector< uint8_t > codes
Codes. Size ntotal * code_size.
Definition: IndexIVFPQ.h:204
faiss::IndexIVFPQ::add_with_ids
void add_with_ids(idx_t n, const float *x, const long *xids=nullptr) override
Definition: IndexIVFPQ.cpp:197
faiss::ProductQuantizer::compute_codes
void compute_codes(const float *x, uint8_t *codes, size_t n) const
same as compute_code for several vectors
Definition: ProductQuantizer.cpp:386
faiss::InvertedLists::get_single_id
virtual idx_t get_single_id(size_t list_no, size_t offset) const
Definition: IndexIVF.cpp:118
faiss::Index::d
int d
vector dimension
Definition: Index.h:64
faiss::InvertedLists::code_size
size_t code_size
code size per vector in bytes
Definition: IndexIVF.h:71
faiss::CodeCmp
Definition: IndexIVFPQ.cpp:1038
faiss::InvertedLists::get_single_code
virtual const uint8_t * get_single_code(size_t list_no, size_t offset) const
Definition: IndexIVF.cpp:129
faiss::IndexIVFPQR::refine_codes
std::vector< uint8_t > refine_codes
corresponding codes
Definition: IndexIVFPQ.h:151
faiss::ProductQuantizer::code_size
size_t code_size
byte per indexed vector
Definition: ProductQuantizer.h:34
faiss::IndexIVFPQ::train_residual
void train_residual(idx_t n, const float *x) override
trains the product quantizer
Definition: IndexIVFPQ.cpp:66
faiss::IndexIVFPQ::reconstruct_from_offset
void reconstruct_from_offset(long list_no, long offset, float *recons) const override
Definition: IndexIVFPQ.cpp:302
faiss::IndexIVFPQ
Definition: IndexIVFPQ.h:29
faiss::IndexIVFPQ::encode_multiple
void encode_multiple(size_t n, long *keys, const float *x, uint8_t *codes, bool compute_keys=false) const
Definition: IndexIVFPQ.cpp:155
faiss::InvertedLists::add_entry
virtual size_t add_entry(size_t list_no, idx_t theid, const uint8_t *code)
add one entry to an inverted list
Definition: IndexIVF.cpp:136
faiss::ProductQuantizer::ksub
size_t ksub
number of centroids for each subquantizer
Definition: ProductQuantizer.h:35
faiss::HeapArray
Definition: Heap.h:350
faiss::Index::idx_t
long idx_t
all indices are this type
Definition: Index.h:62
faiss::ProductQuantizer::compute_code
void compute_code(const float *x, uint8_t *code) const
Quantize one vector with the product quantizer.
Definition: ProductQuantizer.cpp:312
faiss::Index::ntotal
idx_t ntotal
total nb of indexed vectors
Definition: Index.h:65
faiss::Index2Layer::train
void train(idx_t n, const float *x) override
Definition: IndexIVFPQ.cpp:1308
faiss::Index::verbose
bool verbose
verbosity level
Definition: Index.h:66
faiss::IndexIVF::reset
void reset() override
removes all elements from the database.
Definition: IndexIVF.cpp:376
faiss::getmillisecs
double getmillisecs()
ms elapsed since some arbitrary epoch
Definition: utils.cpp:74
faiss::PolysemousTraining
optimizes the order of indices in a ProductQuantizer
Definition: PolysemousTraining.h:125
faiss::Index2Layer::add
void add(idx_t n, const float *x) override
Definition: IndexIVFPQ.cpp:1346
faiss::MultiIndexQuantizer
Definition: IndexPQ.h:135
faiss::fvec_norm_L2sqr
float fvec_norm_L2sqr(const float *x, size_t d)
Definition: utils.cpp:632
faiss::Level1Quantizer::train_q1
void train_q1(size_t n, const float *x, bool verbose, MetricType metric_type)
Trains the quantizer and calls train_residual to train sub-quantizers.
Definition: IndexIVF.cpp:56
faiss::ProductQuantizer::cp
ClusteringParameters cp
parameters used during clustering
Definition: ProductQuantizer.h:49
faiss::Index2Layer::search
void search(idx_t n, const float *x, idx_t k, float *distances, idx_t *labels) const override
not implemented
Definition: IndexIVFPQ.cpp:1394
faiss::IndexIVFPQ::by_residual
bool by_residual
Encode residual or plain vector?
Definition: IndexIVFPQ.h:30
faiss::HeapArray::get_ids
TI * get_ids(size_t key)
Correspponding identifiers.
Definition: Heap.h:363
faiss::Index::metric_type
MetricType metric_type
type of metric this index uses for search
Definition: Index.h:72
faiss::IndexIVFPQ::pq
ProductQuantizer pq
produces the codes
Definition: IndexIVFPQ.h:32
faiss::IndexIVF::invlists
InvertedLists * invlists
Acess to the actual data.
Definition: IndexIVF.h:168
faiss::ProductQuantizer::M
size_t M
number of subquantizers
Definition: ProductQuantizer.h:28
faiss::Index2Layer::code_size_1
size_t code_size_1
size of the code for the first level (ceil(log8(q1.nlist)))
Definition: IndexIVFPQ.h:207
faiss::Index
Definition: Index.h:60
faiss::IndexIVFPQ::add_core_o
void add_core_o(idx_t n, const float *x, const long *xids, float *residuals_2, const long *precomputed_idx=nullptr)
Definition: IndexIVFPQ.cpp:203
faiss::fvec_madd_and_argmin
int fvec_madd_and_argmin(size_t n, const float *a, float bf, const float *b, float *c)
Definition: utils.cpp:1963
faiss::InvertedLists::get_codes
virtual const uint8_t * get_codes(size_t list_no) const =0
faiss::IndexIVFPQR::remove_ids
long remove_ids(const IDSelector &sel) override
Dataset manipulation functions.
Definition: IndexIVFPQ.cpp:1272
faiss::IndexIVFPQ::search_preassigned
void search_preassigned(idx_t n, const float *x, idx_t k, const idx_t *assign, const float *centroid_dis, float *distances, idx_t *labels, bool store_pairs) const override
Definition: IndexIVFPQ.cpp:930
faiss::Level1Quantizer::quantizer
Index * quantizer
quantizer that maps vectors to inverted lists
Definition: IndexIVF.h:33
faiss::ProductQuantizer
Definition: ProductQuantizer.h:25
faiss::Index::is_trained
bool is_trained
set if the Index does not require training, or if training is done already
Definition: Index.h:69
faiss::Index::compute_residual
void compute_residual(const float *x, float *residual, idx_t key) const
Definition: Index.cpp:86
faiss::ProductQuantizer::get_centroids
float * get_centroids(size_t m, size_t i)
return the centroids associated with subvector m
Definition: ProductQuantizer.h:59
faiss::IndexIVF::max_codes
size_t max_codes
max nb of codes to visit to do a query
Definition: IndexIVF.h:174
faiss::Index::reconstruct
virtual void reconstruct(idx_t key, float *recons) const
Definition: Index.cpp:54
faiss::Index2Layer::pq
ProductQuantizer pq
second level quantizer is always a PQ
Definition: IndexIVFPQ.h:201
faiss::IndexIVF::maintain_direct_map
bool maintain_direct_map
map for direct access to the elements. Enables reconstruct().
Definition: IndexIVF.h:177
faiss::PolysemousTraining::optimize_pq_for_hamming
void optimize_pq_for_hamming(ProductQuantizer &pq, size_t n, const float *x) const
Definition: PolysemousTraining.cpp:933
faiss::ClusteringParameters::max_points_per_centroid
int max_points_per_centroid
to limit size of dataset
Definition: Clustering.h:34
faiss::ProductQuantizer::verbose
bool verbose
verbose during training?
Definition: ProductQuantizer.h:36
faiss::IndexIVFPQR::add_with_ids
void add_with_ids(idx_t n, const float *x, const long *xids) override
Definition: IndexIVFPQ.cpp:1136
faiss::Index2Layer::reconstruct
void reconstruct(idx_t key, float *recons) const override
Definition: IndexIVFPQ.cpp:1447
faiss::IndexIVF::merge_from
virtual void merge_from(IndexIVF &other, idx_t add_id)
Definition: IndexIVF.cpp:476
faiss::Level1Quantizer::nlist
size_t nlist
number of possible key values
Definition: IndexIVF.h:34
faiss::IndexIVF::code_size
size_t code_size
code size per vector in bytes
Definition: IndexIVF.h:171
faiss::IndexIVFPQ::find_duplicates
size_t find_duplicates(idx_t *ids, size_t *lims) const
Definition: IndexIVFPQ.cpp:1051
faiss::MetricType
MetricType
Some algorithms support both an inner product version and a L2 search version.
Definition: Index.h:43
faiss::IndexIVFPQR::k_factor
float k_factor
factor between k requested in search and the k requested from the IVFPQ
Definition: IndexIVFPQ.h:154
faiss::IndexIVFPQ::use_precomputed_table
int use_precomputed_table
if by_residual, build precompute tables
Definition: IndexIVFPQ.h:31