Faiss

- Main Page
- Namespaces
- Classes
- Files

- File List

All Classes Namespaces Functions Variables Typedefs Enumerations Enumerator Friends
/data/users/matthijs/github_faiss/faiss/IndexIVF.cpp
1 /**
2 * Copyright (c) 2015-present, Facebook, Inc.
3 * All rights reserved.
4 *
5 * This source code is licensed under the BSD+Patents license found in the
6 * LICENSE file in the root directory of this source tree.
7 */
8
9 /* Copyright 2004-present Facebook. All Rights Reserved.
10 Inverted list structure.
11 */
12
13 #include "IndexIVF.h"
14
15 #include <cstdio>
16
17 #include "utils.h"
18 #include "hamming.h"
19
20 #include "FaissAssert.h"
21 #include "IndexFlat.h"
22 #include "AuxIndexStructures.h"
23
24 namespace faiss {
25
26 /*****************************************
27 * Level1Quantizer implementation
28 ******************************************/
29
30
31 Level1Quantizer::Level1Quantizer (Index * quantizer, size_t nlist):
32 quantizer (quantizer),
33 nlist (nlist),
34 quantizer_trains_alone (0),
35 own_fields (false),
36 clustering_index (nullptr)
37 {
38 // here we set a low # iterations because this is typically used
39 // for large clusterings (nb this is not used for the MultiIndex,
40 // for which quantizer_trains_alone = true)
41 cp.niter = 10;
42 }
43
44 Level1Quantizer::Level1Quantizer ():
45 quantizer (nullptr),
46 nlist (0),
47 quantizer_trains_alone (0), own_fields (false),
48 clustering_index (nullptr)
49 {}
50
51 Level1Quantizer::~Level1Quantizer ()
52 {
53 if (own_fields) delete quantizer;
54 }
55
56 void Level1Quantizer::train_q1 (size_t n, const float *x, bool verbose, MetricType metric_type)
57 {
58 size_t d = quantizer->d;
59 if (quantizer->is_trained && (quantizer->ntotal == nlist)) {
60 if (verbose)
61 printf ("IVF quantizer does not need training.\n");
62 } else if (quantizer_trains_alone == 1) {
63 if (verbose)
64 printf ("IVF quantizer trains alone...\n");
65 quantizer->train (n, x);
66 quantizer->verbose = verbose;
67 FAISS_THROW_IF_NOT_MSG (quantizer->ntotal == nlist,
68 "nlist not consistent with quantizer size");
69 } else if (quantizer_trains_alone == 0) {
70 if (verbose)
71 printf ("Training level-1 quantizer on %ld vectors in %ldD\n",
72 n, d);
73
74 Clustering clus (d, nlist, cp);
75 quantizer->reset();
76 if (clustering_index) {
77 clus.train (n, x, *clustering_index);
78 quantizer->add (nlist, clus.centroids.data());
79 } else {
80 clus.train (n, x, *quantizer);
81 }
82 quantizer->is_trained = true;
83 } else if (quantizer_trains_alone == 2) {
84 if (verbose)
85 printf (
86 "Training L2 quantizer on %ld vectors in %ldD%s\n",
87 n, d,
88 clustering_index ? "(user provided index)" : "");
89 FAISS_THROW_IF_NOT (metric_type == METRIC_L2);
90 Clustering clus (d, nlist, cp);
91 if (!clustering_index) {
92 IndexFlatL2 assigner (d);
93 clus.train(n, x, assigner);
94 } else {
95 clus.train(n, x, *clustering_index);
96 }
97 if (verbose)
98 printf ("Adding centroids to quantizer\n");
99 quantizer->add (nlist, clus.centroids.data());
100 }
101 }
102
103 /*****************************************
104 * InvertedLists implementation
105 ******************************************/
106
107
108 InvertedLists::InvertedLists (size_t nlist, size_t code_size):
109 nlist (nlist), code_size (code_size)
110 {
111 }
112
113
114 InvertedLists::~InvertedLists ()
115 {}
116
117
118 InvertedLists::idx_t InvertedLists::get_single_id (
119 size_t list_no, size_t offset) const
120 {
121 assert (offset < list_size (list_no));
122 return get_ids(list_no)[offset];
123 }
124
125
126 void InvertedLists::prefetch_lists (const long *, int) const
127 {}
128
129 const uint8_t * InvertedLists::get_single_code (
130 size_t list_no, size_t offset) const
131 {
132 assert (offset < list_size (list_no));
133 return get_codes(list_no) + offset * code_size;
134 }
135
136 size_t InvertedLists::add_entry (size_t list_no, idx_t theid,
137 const uint8_t *code)
138 {
139 return add_entries (list_no, 1, &theid, code);
140 }
141
142 void InvertedLists::update_entry (size_t list_no, size_t offset,
143 idx_t id, const uint8_t *code)
144 {
145 update_entries (list_no, offset, 1, &id, code);
146 }
147
148 void InvertedLists::reset () {
149 for (size_t i = 0; i < nlist; i++) {
150 resize (i, 0);
151 }
152 }
153
154 /*****************************************
155 * ArrayInvertedLists implementation
156 ******************************************/
157
158 ArrayInvertedLists::ArrayInvertedLists (size_t nlist, size_t code_size):
159 InvertedLists (nlist, code_size)
160 {
161 ids.resize (nlist);
162 codes.resize (nlist);
163 }
164
165 size_t ArrayInvertedLists::add_entries (
166 size_t list_no, size_t n_entry,
167 const idx_t* ids_in, const uint8_t *code)
168 {
169 if (n_entry == 0) return 0;
170 assert (list_no < nlist);
171 size_t o = ids [list_no].size();
172 ids [list_no].resize (o + n_entry);
173 memcpy (&ids[list_no][o], ids_in, sizeof (ids_in[0]) * n_entry);
174 codes [list_no].resize ((o + n_entry) * code_size);
175 memcpy (&codes[list_no][o * code_size], code, code_size * n_entry);
176 return o;
177 }
178
179 size_t ArrayInvertedLists::list_size(size_t list_no) const
180 {
181 assert (list_no < nlist);
182 return ids[list_no].size();
183 }
184
185 const uint8_t * ArrayInvertedLists::get_codes (size_t list_no) const
186 {
187 assert (list_no < nlist);
188 return codes[list_no].data();
189 }
190
191 const InvertedLists::idx_t * ArrayInvertedLists::get_ids (size_t list_no) const
192 {
193 assert (list_no < nlist);
194 return ids[list_no].data();
195 }
196
197 void ArrayInvertedLists::resize (size_t list_no, size_t new_size)
198 {
199 ids[list_no].resize (new_size);
200 codes[list_no].resize (new_size * code_size);
201 }
202
203 void ArrayInvertedLists::update_entries (
204 size_t list_no, size_t offset, size_t n_entry,
205 const idx_t *ids_in, const uint8_t *codes_in)
206 {
207 assert (list_no < nlist);
208 assert (n_entry + offset <= ids[list_no].size());
209 memcpy (&ids[list_no][offset], ids_in, sizeof(ids_in[0]) * n_entry);
210 memcpy (&codes[list_no][offset * code_size], codes_in, code_size * n_entry);
211 }
212
213
214 ArrayInvertedLists::~ArrayInvertedLists ()
215 {}
216
217
218
219 /*****************************************
220 * IndexIVF implementation
221 ******************************************/
222
223
224 IndexIVF::IndexIVF (Index * quantizer, size_t d,
225 size_t nlist, size_t code_size,
226 MetricType metric):
227 Index (d, metric),
228 Level1Quantizer (quantizer, nlist),
229 invlists (new ArrayInvertedLists (nlist, code_size)),
230 own_invlists (true),
231 code_size (code_size),
232 nprobe (1),
233 max_codes (0),
234 maintain_direct_map (false)
235 {
236 FAISS_THROW_IF_NOT (d == quantizer->d);
237 is_trained = quantizer->is_trained && (quantizer->ntotal == nlist);
238 // Spherical by default if the metric is inner_product
239 if (metric_type == METRIC_INNER_PRODUCT) {
240 cp.spherical = true;
241 }
242
243 }
244
245 IndexIVF::IndexIVF ():
246 invlists (nullptr), own_invlists (false),
247 code_size (0),
248 nprobe (1), max_codes (0),
249 maintain_direct_map (false)
250 {}
251
252 void IndexIVF::add (idx_t n, const float * x)
253 {
254 add_with_ids (n, x, nullptr);
255 }
256
257 void IndexIVF::make_direct_map (bool new_maintain_direct_map)
258 {
259 // nothing to do
260 if (new_maintain_direct_map == maintain_direct_map)
261 return;
262
263 if (new_maintain_direct_map) {
264 direct_map.resize (ntotal, -1);
265 for (size_t key = 0; key < nlist; key++) {
266 size_t list_size = invlists->list_size (key);
267 const idx_t *idlist = invlists->get_ids (key);
268
269 for (long ofs = 0; ofs < list_size; ofs++) {
270 FAISS_THROW_IF_NOT_MSG (
271 0 <= idlist [ofs] && idlist[ofs] < ntotal,
272 "direct map supported only for seuquential ids");
273 direct_map [idlist [ofs]] = key << 32 | ofs;
274 }
275 }
276 } else {
277 direct_map.clear ();
278 }
279 maintain_direct_map = new_maintain_direct_map;
280 }
281
282
283 void IndexIVF::search (idx_t n, const float *x, idx_t k,
284 float *distances, idx_t *labels) const
285 {
286 long * idx = new long [n * nprobe];
287 ScopeDeleter<long> del (idx);
288 float * coarse_dis = new float [n * nprobe];
289 ScopeDeleter<float> del2 (coarse_dis);
290
291 quantizer->search (n, x, nprobe, coarse_dis, idx);
292
293 invlists->prefetch_lists (idx, n * nprobe);
294
295 search_preassigned (n, x, k, idx, coarse_dis,
296 distances, labels, false);
297
298 }
299
300
301 void IndexIVF::reconstruct (idx_t key, float* recons) const
302 {
303 FAISS_THROW_IF_NOT_MSG (direct_map.size() == ntotal,
304 "direct map is not initialized");
305 long list_no = direct_map[key] >> 32;
306 long offset = direct_map[key] & 0xffffffff;
307 reconstruct_from_offset (list_no, offset, recons);
308 }
309
310
311 void IndexIVF::reconstruct_n (idx_t i0, idx_t ni, float* recons) const
312 {
313 FAISS_THROW_IF_NOT (ni == 0 || (i0 >= 0 && i0 + ni <= ntotal));
314
315 for (long list_no = 0; list_no < nlist; list_no++) {
316 size_t list_size = invlists->list_size (list_no);
317 const Index::idx_t * idlist = invlists->get_ids (list_no);
318
319 for (long offset = 0; offset < list_size; offset++) {
320 long id = idlist[offset];
321 if (!(id >= i0 && id < i0 + ni)) {
322 continue;
323 }
324
325 float* reconstructed = recons + (id - i0) * d;
326 reconstruct_from_offset (list_no, offset, reconstructed);
327 }
328 }
329 }
330
331
332 void IndexIVF::search_and_reconstruct (idx_t n, const float *x, idx_t k,
333 float *distances, idx_t *labels,
334 float *recons) const
335 {
336 long * idx = new long [n * nprobe];
337 ScopeDeleter<long> del (idx);
338 float * coarse_dis = new float [n * nprobe];
339 ScopeDeleter<float> del2 (coarse_dis);
340
341 quantizer->search (n, x, nprobe, coarse_dis, idx);
342
343 invlists->prefetch_lists (idx, n * nprobe);
344
345 // search_preassigned() with `store_pairs` enabled to obtain the list_no
346 // and offset into `codes` for reconstruction
347 search_preassigned (n, x, k, idx, coarse_dis,
348 distances, labels, true /* store_pairs */);
349 for (idx_t i = 0; i < n; ++i) {
350 for (idx_t j = 0; j < k; ++j) {
351 idx_t ij = i * k + j;
352 idx_t key = labels[ij];
353 float* reconstructed = recons + ij * d;
354 if (key < 0) {
355 // Fill with NaNs
356 memset(reconstructed, -1, sizeof(*reconstructed) * d);
357 } else {
358 int list_no = key >> 32;
359 int offset = key & 0xffffffff;
360
361 // Update label to the actual id
362 labels[ij] = invlists->get_single_id (list_no, offset);
363
364 reconstruct_from_offset (list_no, offset, reconstructed);
365 }
366 }
367 }
368 }
369
370 void IndexIVF::reconstruct_from_offset (long list_no, long offset,
371 float* recons) const
372 {
373 FAISS_THROW_MSG ("reconstruct_from_offset not implemented");
374 }
375
376 void IndexIVF::reset ()
377 {
378 direct_map.clear ();
379 invlists->reset ();
380 ntotal = 0;
381 }
382
383
384 long IndexIVF::remove_ids (const IDSelector & sel)
385 {
386 FAISS_THROW_IF_NOT_MSG (!maintain_direct_map,
387 "direct map remove not implemented");
388
389 std::vector<long> toremove(nlist);
390
391 #pragma omp parallel for
392 for (long i = 0; i < nlist; i++) {
393 long l0 = invlists->list_size (i), l = l0, j = 0;
394 const idx_t *idsi = invlists->get_ids (i);
395 while (j < l) {
396 if (sel.is_member (idsi[j])) {
397 l--;
398 invlists->update_entry (
399 i, j,
400 invlists->get_single_id (i, l),
401 invlists->get_single_code (i, l));
402 } else {
403 j++;
404 }
405 }
406 toremove[i] = l0 - l;
407 }
408 // this will not run well in parallel on ondisk because of possible shrinks
409 long nremove = 0;
410 for (long i = 0; i < nlist; i++) {
411 if (toremove[i] > 0) {
412 nremove += toremove[i];
413 invlists->resize(
414 i, invlists->list_size(i) - toremove[i]);
415 }
416 }
417 ntotal -= nremove;
418 return nremove;
419 }
420
421
422
423
424 void IndexIVF::train (idx_t n, const float *x)
425 {
426 if (verbose)
427 printf ("Training level-1 quantizer\n");
428
429 train_q1 (n, x, verbose, metric_type);
430
431 if (verbose)
432 printf ("Training IVF residual\n");
433
434 train_residual (n, x);
435 is_trained = true;
436
437 }
438
439 void IndexIVF::train_residual(idx_t /*n*/, const float* /*x*/) {
440 if (verbose)
441 printf("IndexIVF: no residual training\n");
442 // does nothing by default
443 }
444
445
446
447 double IndexIVF::imbalance_factor () const
448 {
449 std::vector<int> hist (nlist);
450 for (int i = 0; i < nlist; i++) {
451 hist[i] = invlists->list_size(i);
452 }
453 return faiss::imbalance_factor (nlist, hist.data());
454 }
455
456 void IndexIVF::print_stats () const
457 {
458 std::vector<int> sizes(40);
459 for (int i = 0; i < nlist; i++) {
460 for (int j = 0; j < sizes.size(); j++) {
461 if ((invlists->list_size(i) >> j) == 0) {
462 sizes[j]++;
463 break;
464 }
465 }
466 }
467 for (int i = 0; i < sizes.size(); i++) {
468 if (sizes[i]) {
469 printf ("list size in < %d: %d instances\n",
470 1 << i, sizes[i]);
471 }
472 }
473
474 }
475
476 void IndexIVF::merge_from (IndexIVF &other, idx_t add_id)
477 {
478 // minimal sanity checks
479 FAISS_THROW_IF_NOT (other.d == d);
480 FAISS_THROW_IF_NOT (other.nlist == nlist);
481 FAISS_THROW_IF_NOT (other.code_size == code_size);
482 FAISS_THROW_IF_NOT_MSG ((!maintain_direct_map &&
483 !other.maintain_direct_map),
484 "direct map copy not implemented");
485 FAISS_THROW_IF_NOT_MSG (typeid (*this) == typeid (other),
486 "can only merge indexes of the same type");
487
488 InvertedLists *oivf = other.invlists;
489 #pragma omp parallel for
490 for (long i = 0; i < nlist; i++) {
491 size_t list_size = oivf->list_size (i);
492 const idx_t * ids = oivf->get_ids (i);
493 if (add_id == 0) {
494 invlists->add_entries (i, list_size, ids,
495 oivf->get_codes (i));
496 } else {
497 std::vector <idx_t> new_ids (list_size);
498
499 for (size_t j = 0; j < list_size; j++) {
500 new_ids [j] = ids[j] + add_id;
501 }
502
503 invlists->add_entries (i, list_size, new_ids.data(),
504 oivf->get_codes (i));
505 }
506 oivf->resize (i, 0);
507 }
508
509 ntotal += other.ntotal;
510 other.ntotal = 0;
511 }
512
513
514 void IndexIVF::replace_invlists (InvertedLists *il, bool own)
515 {
516 //FAISS_THROW_IF_NOT (ntotal == 0);
517 FAISS_THROW_IF_NOT (il->nlist == nlist &&
518 il->code_size == code_size);
519 if (own_invlists) {
520 delete invlists;
521 }
522 invlists = il;
523 own_invlists = own;
524 }
525
526
527 void IndexIVF::copy_subset_to (IndexIVF & other, int subset_type,
528 long a1, long a2) const
529 {
530
531 FAISS_THROW_IF_NOT (nlist == other.nlist);
532 FAISS_THROW_IF_NOT (code_size == other.code_size);
533 FAISS_THROW_IF_NOT (!other.maintain_direct_map);
534 FAISS_THROW_IF_NOT_FMT (
535 subset_type == 0 || subset_type == 1 || subset_type == 2,
536 "subset type %d not implemented", subset_type);
537
538 size_t accu_n = 0;
539 size_t accu_a1 = 0;
540 size_t accu_a2 = 0;
541
542 InvertedLists *oivf = other.invlists;
543
544 for (long list_no = 0; list_no < nlist; list_no++) {
545 size_t n = invlists->list_size (list_no);
546 const idx_t *ids_in = invlists->get_ids (list_no);
547
548 if (subset_type == 0) {
549 for (long i = 0; i < n; i++) {
550 idx_t id = ids_in[i];
551 if (a1 <= id && id < a2) {
552 oivf->add_entry (list_no,
553 invlists->get_single_id (list_no, i),
554 invlists->get_single_code (list_no, i));
555 other.ntotal++;
556 }
557 }
558 } else if (subset_type == 1) {
559 for (long i = 0; i < n; i++) {
560 idx_t id = ids_in[i];
561 if (id % a1 == a2) {
562 oivf->add_entry (list_no,
563 invlists->get_single_id (list_no, i),
564 invlists->get_single_code (list_no, i));
565 other.ntotal++;
566 }
567 }
568 } else if (subset_type == 2) {
569 // see what is allocated to a1 and to a2
570 size_t next_accu_n = accu_n + n;
571 size_t next_accu_a1 = next_accu_n * a1 / ntotal;
572 size_t i1 = next_accu_a1 - accu_a1;
573 size_t next_accu_a2 = next_accu_n * a2 / ntotal;
574 size_t i2 = next_accu_a2 - accu_a2;
575
576 for (long i = i1; i < i2; i++) {
577 oivf->add_entry (list_no,
578 invlists->get_single_id (list_no, i),
579 invlists->get_single_code (list_no, i));
580 }
581
582 other.ntotal += i2 - i1;
583 accu_a1 = next_accu_a1;
584 accu_a2 = next_accu_a2;
585 }
586 accu_n += n;
587 }
588 FAISS_ASSERT(accu_n == ntotal);
589
590 }
591
592
593
594 IndexIVF::~IndexIVF()
595 {
596 if (own_invlists) {
597 delete invlists;
598 }
599 }
600
601
602 void IndexIVFStats::reset()
603 {
604 memset ((void*)this, 0, sizeof (*this));
605 }
606
607
608 IndexIVFStats indexIVF_stats;
609
610
611
612 } // namespace faiss
faiss::IndexIVF::search_preassigned
virtual void search_preassigned(idx_t n, const float *x, idx_t k, const idx_t *assign, const float *centroid_dis, float *distances, idx_t *labels, bool store_pairs) const =0
faiss::IndexIVF
Definition: IndexIVF.h:166
faiss::ArrayInvertedLists::get_codes
const uint8_t * get_codes(size_t list_no) const override
Definition: IndexIVF.cpp:185
faiss::Level1Quantizer
Definition: IndexIVF.h:32
faiss::ArrayInvertedLists
Definition: IndexIVF.h:123
faiss::ArrayInvertedLists::get_ids
const idx_t * get_ids(size_t list_no) const override
Definition: IndexIVF.cpp:191
faiss::InvertedLists::get_ids
virtual const idx_t * get_ids(size_t list_no) const =0
faiss::IndexIVF::imbalance_factor
double imbalance_factor() const
1= perfectly balanced, &gt;1: imbalanced
Definition: IndexIVF.cpp:447
faiss::IndexIVF::search_and_reconstruct
void search_and_reconstruct(idx_t n, const float *x, idx_t k, float *distances, idx_t *labels, float *recons) const override
Definition: IndexIVF.cpp:332
faiss::Index::reset
virtual void reset()=0
removes all elements from the database.
faiss::IndexIVF::copy_subset_to
virtual void copy_subset_to(IndexIVF &other, int subset_type, long a1, long a2) const
Definition: IndexIVF.cpp:527
faiss::IndexIVF::reconstruct_from_offset
virtual void reconstruct_from_offset(long list_no, long offset, float *recons) const
Definition: IndexIVF.cpp:370
faiss::IndexIVF::nprobe
size_t nprobe
number of probes at query time
Definition: IndexIVF.h:173
faiss::IDSelector
Definition: AuxIndexStructures.h:52
faiss::IndexFlatL2
Definition: IndexFlat.h:79
faiss::InvertedLists::list_size
virtual size_t list_size(size_t list_no) const =0
get the size of a list
faiss::IndexIVF::reconstruct
void reconstruct(idx_t key, float *recons) const override
Definition: IndexIVF.cpp:301
faiss::Index::train
virtual void train(idx_t n, const float *x)
Definition: Index.cpp:23
faiss::Index::add_with_ids
virtual void add_with_ids(idx_t n, const float *x, const long *xids)
Definition: Index.cpp:41
faiss::IndexIVF::train_residual
virtual void train_residual(idx_t n, const float *x)
Definition: IndexIVF.cpp:439
faiss::ScopeDeleter
Definition: FaissException.h:39
faiss::imbalance_factor
double imbalance_factor(int n, int k, const long *assign)
a balanced assignment has a IF of 1
Definition: utils.cpp:1627
faiss::InvertedLists
Definition: IndexIVF.h:67
faiss::InvertedLists::get_single_id
virtual idx_t get_single_id(size_t list_no, size_t offset) const
Definition: IndexIVF.cpp:118
faiss::Index::d
int d
vector dimension
Definition: Index.h:64
faiss::InvertedLists::code_size
size_t code_size
code size per vector in bytes
Definition: IndexIVF.h:71
faiss::IndexIVF::train
void train(idx_t n, const float *x) override
Trains the quantizer and calls train_residual to train sub-quantizers.
Definition: IndexIVF.cpp:424
faiss::InvertedLists::get_single_code
virtual const uint8_t * get_single_code(size_t list_no, size_t offset) const
Definition: IndexIVF.cpp:129
faiss::IndexIVF::reconstruct_n
void reconstruct_n(idx_t i0, idx_t ni, float *recons) const override
Definition: IndexIVF.cpp:311
faiss::Level1Quantizer::quantizer_trains_alone
char quantizer_trains_alone
Definition: IndexIVF.h:41
faiss::Clustering
Definition: Clustering.h:57
faiss::Index::add
virtual void add(idx_t n, const float *x)=0
faiss::InvertedLists::add_entry
virtual size_t add_entry(size_t list_no, idx_t theid, const uint8_t *code)
add one entry to an inverted list
Definition: IndexIVF.cpp:136
faiss::Index::idx_t
long idx_t
all indices are this type
Definition: Index.h:62
faiss::Level1Quantizer::cp
ClusteringParameters cp
to override default clustering params
Definition: IndexIVF.h:44
faiss::Index::ntotal
idx_t ntotal
total nb of indexed vectors
Definition: Index.h:65
faiss::Index::verbose
bool verbose
verbosity level
Definition: Index.h:66
faiss::IndexIVF::reset
void reset() override
removes all elements from the database.
Definition: IndexIVF.cpp:376
faiss::Clustering::centroids
std::vector< float > centroids
centroids (k * d)
Definition: Clustering.h:63
faiss::InvertedLists::prefetch_lists
virtual void prefetch_lists(const long *list_nos, int nlist) const
Definition: IndexIVF.cpp:126
faiss::Index::search
virtual void search(idx_t n, const float *x, idx_t k, float *distances, idx_t *labels) const =0
faiss::Level1Quantizer::clustering_index
Index * clustering_index
to override index used during clustering
Definition: IndexIVF.h:45
faiss::Level1Quantizer::train_q1
void train_q1(size_t n, const float *x, bool verbose, MetricType metric_type)
Trains the quantizer and calls train_residual to train sub-quantizers.
Definition: IndexIVF.cpp:56
faiss::InvertedLists::nlist
size_t nlist
number of possible key values
Definition: IndexIVF.h:70
faiss::IndexIVF::make_direct_map
void make_direct_map(bool new_maintain_direct_map=true)
Definition: IndexIVF.cpp:257
faiss::Index::metric_type
MetricType metric_type
type of metric this index uses for search
Definition: Index.h:72
faiss::IndexIVF::print_stats
void print_stats() const
display some stats about the inverted lists
Definition: IndexIVF.cpp:456
faiss::IndexIVF::invlists
InvertedLists * invlists
Acess to the actual data.
Definition: IndexIVF.h:168
faiss::ArrayInvertedLists::ids
std::vector< std::vector< idx_t > > ids
Inverted lists for indexes.
Definition: IndexIVF.h:125
faiss::Index
Definition: Index.h:60
faiss::IndexIVF::add
void add(idx_t n, const float *x) override
Quantizes x and calls add_with_key.
Definition: IndexIVF.cpp:252
faiss::Clustering::train
virtual void train(idx_t n, const float *x, faiss::Index &index)
Index is used during the assignment stage.
Definition: Clustering.cpp:67
faiss::InvertedLists::get_codes
virtual const uint8_t * get_codes(size_t list_no) const =0
faiss::Level1Quantizer::quantizer
Index * quantizer
quantizer that maps vectors to inverted lists
Definition: IndexIVF.h:33
faiss::Index::is_trained
bool is_trained
set if the Index does not require training, or if training is done already
Definition: Index.h:69
faiss::IndexIVF::remove_ids
long remove_ids(const IDSelector &sel) override
Dataset manipulation functions.
Definition: IndexIVF.cpp:384
faiss::IndexIVF::maintain_direct_map
bool maintain_direct_map
map for direct access to the elements. Enables reconstruct().
Definition: IndexIVF.h:177
faiss::ClusteringParameters::spherical
bool spherical
do we want normalized centroids?
Definition: Clustering.h:29
faiss::Level1Quantizer::own_fields
bool own_fields
whether object owns the quantizer
Definition: IndexIVF.h:42
faiss::ArrayInvertedLists::list_size
size_t list_size(size_t list_no) const override
get the size of a list
Definition: IndexIVF.cpp:179
faiss::IndexIVF::search
virtual void search(idx_t n, const float *x, idx_t k, float *distances, idx_t *labels) const override
Definition: IndexIVF.cpp:283
faiss::IndexIVF::merge_from
virtual void merge_from(IndexIVF &other, idx_t add_id)
Definition: IndexIVF.cpp:476
faiss::Level1Quantizer::nlist
size_t nlist
number of possible key values
Definition: IndexIVF.h:34
faiss::IndexIVF::code_size
size_t code_size
code size per vector in bytes
Definition: IndexIVF.h:171
faiss::MetricType
MetricType
Some algorithms support both an inner product version and a L2 search version.
Definition: Index.h:43