Faiss

- Main Page
- Namespaces
- Classes
- Files

- File List

All Classes Namespaces Functions Variables Typedefs Enumerations Enumerator Friends
/data/users/matthijs/github_faiss/faiss/IndexIVF.cpp
1 /**
2 * Copyright (c) 2015-present, Facebook, Inc.
3 * All rights reserved.
4 *
5 * This source code is licensed under the CC-by-NC license found in the
6 * LICENSE file in the root directory of this source tree.
7 */
8
9 /* Copyright 2004-present Facebook. All Rights Reserved.
10 Inverted list structure.
11 */
12
13 #include "IndexIVF.h"
14
15 #include <cstdio>
16
17 #include "utils.h"
18 #include "hamming.h"
19
20 #include "FaissAssert.h"
21 #include "IndexFlat.h"
22 #include "AuxIndexStructures.h"
23
24 namespace faiss {
25
26 /*****************************************
27 * IndexIVF implementation
28 ******************************************/
29
30
31 IndexIVF::IndexIVF (Index * quantizer, size_t d, size_t nlist,
32 MetricType metric):
33 Index (d, metric),
34 nlist (nlist),
35 nprobe (1),
36 quantizer (quantizer),
37 quantizer_trains_alone (false),
38 own_fields (false),
39 ids (nlist),
40 maintain_direct_map (false)
41 {
42 FAISS_THROW_IF_NOT (d == quantizer->d);
43 is_trained = quantizer->is_trained && (quantizer->ntotal == nlist);
44 // Spherical by default if the metric is inner_product
45 if (metric_type == METRIC_INNER_PRODUCT) {
46 cp.spherical = true;
47 }
48 // here we set a low # iterations because this is typically used
49 // for large clusterings (nb this is not used for the MultiIndex,
50 // for which quantizer_trains_alone = true)
51 cp.niter = 10;
52 cp.verbose = verbose;
53
54 }
55
56 IndexIVF::IndexIVF ():
57 nlist (0), nprobe (1), quantizer (nullptr),
58 quantizer_trains_alone (false), own_fields (false),
59 maintain_direct_map (false)
60 {}
61
62
63 void IndexIVF::add (idx_t n, const float * x)
64 {
65 add_with_ids (n, x, nullptr);
66 }
67
68 void IndexIVF::make_direct_map ()
69 {
70 if (maintain_direct_map) return;
71
72 direct_map.resize (ntotal, -1);
73 for (size_t key = 0; key < nlist; key++) {
74 const std::vector<long> & idlist = ids[key];
75
76 for (long ofs = 0; ofs < idlist.size(); ofs++) {
77 direct_map [idlist [ofs]] =
78 key << 32 | ofs;
79 }
80 }
81
82 maintain_direct_map = true;
83 }
84
85
86 void IndexIVF::reset ()
87 {
88 ntotal = 0;
89 direct_map.clear();
90 for (size_t i = 0; i < ids.size(); i++)
91 ids[i].clear();
92 }
93
94
95 void IndexIVF::train (idx_t n, const float *x)
96 {
97 if (quantizer->is_trained && (quantizer->ntotal == nlist)) {
98 if (verbose)
99 printf ("IVF quantizer does not need training.\n");
100 } else if (quantizer_trains_alone) {
101 if (verbose)
102 printf ("IVF quantizer trains alone...\n");
103 quantizer->train (n, x);
104 FAISS_THROW_IF_NOT_MSG (quantizer->ntotal == nlist,
105 "nlist not consistent with quantizer size");
106 } else {
107 if (verbose)
108 printf ("Training IVF quantizer on %ld vectors in %dD\n",
109 n, d);
110
111 Clustering clus (d, nlist, cp);
112
113 quantizer->reset();
114 clus.train (n, x, *quantizer);
115 quantizer->is_trained = true;
116 }
117 if (verbose)
118 printf ("Training IVF residual\n");
119
120 train_residual (n, x);
121 is_trained = true;
122 }
123
124 void IndexIVF::train_residual (idx_t n, const float *x)
125 {
126 if (verbose)
127 printf ("IndexIVF: no residual training\n");
128 // does nothing by default
129 }
130
131
132
133 double IndexIVF::imbalance_factor () const
134 {
135 std::vector<int> hist (nlist);
136 for (int i = 0; i < nlist; i++) {
137 hist[i] = ids[i].size();
138 }
139 return faiss::imbalance_factor (nlist, hist.data());
140 }
141
142 void IndexIVF::print_stats () const
143 {
144 std::vector<int> sizes(40);
145 for (int i = 0; i < nlist; i++) {
146 for (int j = 0; j < sizes.size(); j++) {
147 if ((ids[i].size() >> j) == 0) {
148 sizes[j]++;
149 break;
150 }
151 }
152 }
153 for (int i = 0; i < sizes.size(); i++) {
154 if (sizes[i]) {
155 printf ("list size in < %d: %d instances\n",
156 1 << i, sizes[i]);
157 }
158 }
159
160 }
161
162 void IndexIVF::merge_from (IndexIVF &other, idx_t add_id)
163 {
164 // minimal sanity checks
165 FAISS_THROW_IF_NOT (other.d == d);
166 FAISS_THROW_IF_NOT (other.nlist == nlist);
167 FAISS_THROW_IF_NOT_MSG ((!maintain_direct_map &&
168 !other.maintain_direct_map),
169 "direct map copy not implemented");
170 FAISS_THROW_IF_NOT_MSG (typeid (*this) == typeid (other),
171 "can only merge indexes of the same type");
172 for (long i = 0; i < nlist; i++) {
173 std::vector<idx_t> & src = other.ids[i];
174 std::vector<idx_t> & dest = ids[i];
175 for (long j = 0; j < src.size(); j++)
176 dest.push_back (src[j] + add_id);
177 src.clear();
178 }
179 merge_from_residuals (other);
180 ntotal += other.ntotal;
181 other.ntotal = 0;
182 }
183
184
185
186
187 IndexIVF::~IndexIVF()
188 {
189 if (own_fields) delete quantizer;
190 }
191
192
193
194 /*****************************************
195 * IndexIVFFlat implementation
196 ******************************************/
197
198 IndexIVFFlat::IndexIVFFlat (Index * quantizer,
199 size_t d, size_t nlist, MetricType metric):
200 IndexIVF (quantizer, d, nlist, metric)
201 {
202 vecs.resize (nlist);
203 }
204
205
206
207
208
209
210 void IndexIVFFlat::add_with_ids (idx_t n, const float * x, const long *xids)
211 {
212 add_core (n, x, xids, nullptr);
213 }
214
215 void IndexIVFFlat::add_core (idx_t n, const float * x, const long *xids,
216 const long *precomputed_idx)
217
218 {
219 FAISS_THROW_IF_NOT (is_trained);
220 const long * idx;
221 ScopeDeleter<long> del;
222
223 if (precomputed_idx) {
224 idx = precomputed_idx;
225 } else {
226 long * idx0 = new long [n];
227 quantizer->assign (n, x, idx0);
228 idx = idx0;
229 del.set (idx);
230 }
231 long n_add = 0;
232 for (size_t i = 0; i < n; i++) {
233 long id = xids ? xids[i] : ntotal + i;
234 long list_no = idx [i];
235 if (list_no < 0)
236 continue;
237 assert (list_no < nlist);
238
239 ids[list_no].push_back (id);
240 const float *xi = x + i * d;
241 /* store the vectors */
242 for (size_t j = 0 ; j < d ; j++)
243 vecs[list_no].push_back (xi [j]);
244
245 if (maintain_direct_map)
246 direct_map.push_back (list_no << 32 | (ids[list_no].size() - 1));
247 n_add++;
248 }
249 if (verbose) {
250 printf("IndexIVFFlat::add_core: added %ld / %ld vectors\n",
251 n_add, n);
252 }
253 ntotal += n_add;
254 }
255
256 void IndexIVFFlatStats::reset()
257 {
258 memset ((void*)this, 0, sizeof (*this));
259 }
260
261
262 IndexIVFFlatStats indexIVFFlat_stats;
263
264 void IndexIVFFlat::search_knn_inner_product (
265 size_t nx,
266 const float * x,
267 const long * __restrict keys,
268 float_minheap_array_t * res) const
269 {
270
271 const size_t k = res->k;
272 size_t nlistv = 0, ndis = 0;
273
274 #pragma omp parallel for reduction(+: nlistv, ndis)
275 for (size_t i = 0; i < nx; i++) {
276 const float * xi = x + i * d;
277 const long * keysi = keys + i * nprobe;
278 float * __restrict simi = res->get_val (i);
279 long * __restrict idxi = res->get_ids (i);
280 minheap_heapify (k, simi, idxi);
281
282 for (size_t ik = 0; ik < nprobe; ik++) {
283 long key = keysi[ik]; /* select the list */
284 if (key < 0) {
285 // not enough centroids for multiprobe
286 continue;
287 }
288 if (key >= (long) nlist) {
289 fprintf (stderr, "Invalid key=%ld at ik=%ld nlist=%ld\n",
290 key, ik, nlist);
291 throw;
292 }
293 nlistv++;
294 const size_t list_size = ids[key].size();
295 const float * list_vecs = vecs[key].data();
296
297 for (size_t j = 0; j < list_size; j++) {
298 const float * yj = list_vecs + d * j;
299 float ip = fvec_inner_product (xi, yj, d);
300 if (ip > simi[0]) {
301 minheap_pop (k, simi, idxi);
302 minheap_push (k, simi, idxi, ip, ids[key][j]);
303 }
304 }
305 ndis += list_size;
306 }
307 minheap_reorder (k, simi, idxi);
308 }
309 indexIVFFlat_stats.nq += nx;
310 indexIVFFlat_stats.nlist += nlistv;
311 indexIVFFlat_stats.ndis += ndis;
312 }
313
314
315 void IndexIVFFlat::search_knn_L2sqr (
316 size_t nx,
317 const float * x,
318 const long * __restrict keys,
319 float_maxheap_array_t * res) const
320 {
321 const size_t k = res->k;
322 size_t nlistv = 0, ndis = 0;
323
324 #pragma omp parallel for reduction(+: nlistv, ndis)
325 for (size_t i = 0; i < nx; i++) {
326 const float * xi = x + i * d;
327 const long * keysi = keys + i * nprobe;
328 float * __restrict disi = res->get_val (i);
329 long * __restrict idxi = res->get_ids (i);
330 maxheap_heapify (k, disi, idxi);
331
332 for (size_t ik = 0; ik < nprobe; ik++) {
333 long key = keysi[ik]; /* select the list */
334 if (key < 0) {
335 // not enough centroids for multiprobe
336 continue;
337 }
338 if (key >= (long) nlist) {
339 fprintf (stderr, "Invalid key=%ld at ik=%ld nlist=%ld\n",
340 key, ik, nlist);
341 throw;
342 }
343 nlistv++;
344 const size_t list_size = ids[key].size();
345 const float * list_vecs = vecs[key].data();
346
347 for (size_t j = 0; j < list_size; j++) {
348 const float * yj = list_vecs + d * j;
349 float disij = fvec_L2sqr (xi, yj, d);
350 if (disij < disi[0]) {
351 maxheap_pop (k, disi, idxi);
352 maxheap_push (k, disi, idxi, disij, ids[key][j]);
353 }
354 }
355 ndis += list_size;
356 }
357 maxheap_reorder (k, disi, idxi);
358 }
359 indexIVFFlat_stats.nq += nx;
360 indexIVFFlat_stats.nlist += nlistv;
361 indexIVFFlat_stats.ndis += ndis;
362 }
363
364
365 void IndexIVFFlat::search (idx_t n, const float *x, idx_t k,
366 float *distances, idx_t *labels) const
367 {
368 idx_t * idx = new idx_t [n * nprobe];
369 ScopeDeleter <idx_t> del (idx);
370 quantizer->assign (n, x, idx, nprobe);
371 search_preassigned (n, x, k, idx, distances, labels);
372 }
373
374
375 void IndexIVFFlat::search_preassigned (idx_t n, const float *x, idx_t k,
376 const idx_t *idx,
377 float *distances, idx_t *labels) const
378 {
379 if (metric_type == METRIC_INNER_PRODUCT) {
380 float_minheap_array_t res = {
381 size_t(n), size_t(k), labels, distances};
382 search_knn_inner_product (n, x, idx, &res);
383
384 } else if (metric_type == METRIC_L2) {
385 float_maxheap_array_t res = {
386 size_t(n), size_t(k), labels, distances};
387 search_knn_L2sqr (n, x, idx, &res);
388 }
389
390 }
391
392
393 void IndexIVFFlat::range_search (idx_t nx, const float *x, float radius,
394 RangeSearchResult *result) const
395 {
396 idx_t * keys = new idx_t [nx * nprobe];
397 ScopeDeleter<idx_t> del (keys);
398 quantizer->assign (nx, x, keys, nprobe);
399
400 #pragma omp parallel
401 {
402 RangeSearchPartialResult pres(result);
403
404 for (size_t i = 0; i < nx; i++) {
405 const float * xi = x + i * d;
406 const long * keysi = keys + i * nprobe;
407
408 RangeSearchPartialResult::QueryResult & qres =
409 pres.new_result (i);
410
411 for (size_t ik = 0; ik < nprobe; ik++) {
412 long key = keysi[ik]; /* select the list */
413 if (key < 0 || key >= (long) nlist) {
414 fprintf (stderr, "Invalid key=%ld at ik=%ld nlist=%ld\n",
415 key, ik, nlist);
416 throw;
417 }
418
419 const size_t list_size = ids[key].size();
420 const float * list_vecs = vecs[key].data();
421
422 for (size_t j = 0; j < list_size; j++) {
423 const float * yj = list_vecs + d * j;
424 if (metric_type == METRIC_L2) {
425 float disij = fvec_L2sqr (xi, yj, d);
426 if (disij < radius) {
427 qres.add (disij, ids[key][j]);
428 }
429 } else if (metric_type == METRIC_INNER_PRODUCT) {
430 float disij = fvec_inner_product(xi, yj, d);
431 if (disij > radius) {
432 qres.add (disij, ids[key][j]);
433 }
434 }
435 }
436 }
437 }
438
439 pres.finalize ();
440 }
441 }
442
443 void IndexIVFFlat::merge_from_residuals (IndexIVF &other_in)
444 {
445 IndexIVFFlat &other = dynamic_cast<IndexIVFFlat &> (other_in);
446 for (int i = 0; i < nlist; i++) {
447 std::vector<float> & src = other.vecs[i];
448 std::vector<float> & dest = vecs[i];
449 for (int j = 0; j < src.size(); j++)
450 dest.push_back (src[j]);
451 src.clear();
452 }
453 }
454
455 void IndexIVFFlat::copy_subset_to (IndexIVFFlat & other, int subset_type,
456 long a1, long a2) const
457 {
458 FAISS_THROW_IF_NOT (nlist == other.nlist);
459 FAISS_THROW_IF_NOT (!other.maintain_direct_map);
460
461 for (long list_no = 0; list_no < nlist; list_no++) {
462 const std::vector<idx_t> & ids_in = ids[list_no];
463 std::vector<idx_t> & ids_out = other.ids[list_no];
464 const std::vector<float> & vecs_in = vecs[list_no];
465 std::vector<float> & vecs_out = other.vecs[list_no];
466
467 for (long i = 0; i < ids_in.size(); i++) {
468 idx_t id = ids_in[i];
469 if (subset_type == 0 && a1 <= id && id < a2) {
470 ids_out.push_back (id);
471 vecs_out.insert (vecs_out.end(),
472 vecs_in.begin() + i * d,
473 vecs_in.begin() + (i + 1) * d);
474 other.ntotal++;
475 }
476 }
477 }
478 }
479
480
481
482 void IndexIVFFlat::reset()
483 {
484 IndexIVF::reset();
485 for (size_t key = 0; key < nlist; key++) {
486 vecs[key].clear();
487 }
488 }
489
490 long IndexIVFFlat::remove_ids (const IDSelector & sel)
491 {
492 FAISS_THROW_IF_NOT_MSG (!maintain_direct_map,
493 "direct map remove not implemented");
494 long nremove = 0;
495 #pragma omp parallel for reduction(+: nremove)
496 for (long i = 0; i < nlist; i++) {
497 std::vector<idx_t> & idsi = ids[i];
498 float *vecsi = vecs[i].data();
499
500 long l = idsi.size(), j = 0;
501 while (j < l) {
502 if (sel.is_member (idsi[j])) {
503 l--;
504 idsi [j] = idsi [l];
505 memmove (vecsi + j * d,
506 vecsi + l * d, d * sizeof (float));
507 } else {
508 j++;
509 }
510 }
511 if (l < idsi.size()) {
512 nremove += idsi.size() - l;
513 idsi.resize (l);
514 vecs[i].resize (l * d);
515 }
516 }
517 ntotal -= nremove;
518 return nremove;
519 }
520
521
522 void IndexIVFFlat::reconstruct (idx_t key, float * recons) const
523 {
524 FAISS_THROW_IF_NOT_MSG (direct_map.size() == ntotal,
525 "direct map is not initialized");
526 int list_no = direct_map[key] >> 32;
527 int ofs = direct_map[key] & 0xffffffff;
528 memcpy (recons, &vecs[list_no][ofs * d], d * sizeof(recons[0]));
529 }
530
531
532
533
534 /*****************************************
535 * IndexIVFFlatIPBounds implementation
536 ******************************************/
537
538 IndexIVFFlatIPBounds::IndexIVFFlatIPBounds (
539 Index * quantizer, size_t d, size_t nlist,
540 size_t fsize):
541 IndexIVFFlat(quantizer, d, nlist, METRIC_INNER_PRODUCT), fsize(fsize)
542 {
543 part_norms.resize(nlist);
544 }
545
546
547
548 void IndexIVFFlatIPBounds::add_core (idx_t n, const float * x, const long *xids,
549 const long *precomputed_idx) {
550
551 FAISS_THROW_IF_NOT (is_trained);
552 const long * idx;
553 ScopeDeleter<long> del;
554
555 if (precomputed_idx) {
556 idx = precomputed_idx;
557 } else {
558 long * idx0 = new long [n];
559 quantizer->assign (n, x, idx0);
560 idx = idx0;
561 del.set (idx);
562 }
563 IndexIVFFlat::add_core(n, x, xids, idx);
564
565 // compute
566 const float * xi = x + fsize;
567 for (size_t i = 0; i < n; i++) {
568 float norm = std::sqrt (fvec_norm_L2sqr (xi, d - fsize));
569 part_norms[idx[i]].push_back(norm);
570 xi += d;
571 }
572
573
574 }
575
576 namespace {
577
578 void search_bounds_knn_inner_product (
579 const IndexIVFFlatIPBounds & ivf,
580 const float *x,
581 const long *keys,
582 float_minheap_array_t *res,
583 const float *qnorms)
584 {
585
586 size_t k = res->k, nx = res->nh, nprobe = ivf.nprobe;
587 size_t d = ivf.d;
588 int fsize = ivf.fsize;
589
590 size_t nlistv = 0, ndis = 0, npartial = 0;
591
592 #pragma omp parallel for reduction(+: nlistv, ndis, npartial)
593 for (size_t i = 0; i < nx; i++) {
594 const float * xi = x + i * d;
595 const long * keysi = keys + i * nprobe;
596 float qnorm = qnorms[i];
597 float * __restrict simi = res->get_val (i);
598 long * __restrict idxi = res->get_ids (i);
599 minheap_heapify (k, simi, idxi);
600
601 for (size_t ik = 0; ik < nprobe; ik++) {
602 long key = keysi[ik]; /* select the list */
603 if (key < 0) {
604 // not enough centroids for multiprobe
605 continue;
606 }
607 assert (key < (long) ivf.nlist);
608 nlistv++;
609
610 const size_t list_size = ivf.ids[key].size();
611 const float * yj = ivf.vecs[key].data();
612 const float * bnorms = ivf.part_norms[key].data();
613
614 for (size_t j = 0; j < list_size; j++) {
615 float ip_part = fvec_inner_product (xi, yj, fsize);
616 float bound = ip_part + bnorms[j] * qnorm;
617
618 if (bound > simi[0]) {
619 float ip = ip_part + fvec_inner_product (
620 xi + fsize, yj + fsize, d - fsize);
621 if (ip > simi[0]) {
622 minheap_pop (k, simi, idxi);
623 minheap_push (k, simi, idxi, ip, ivf.ids[key][j]);
624 }
625 ndis ++;
626 }
627 yj += d;
628 }
629 npartial += list_size;
630 }
631 minheap_reorder (k, simi, idxi);
632 }
633 indexIVFFlat_stats.nq += nx;
634 indexIVFFlat_stats.nlist += nlistv;
635 indexIVFFlat_stats.ndis += ndis;
636 indexIVFFlat_stats.npartial += npartial;
637 }
638
639
640 }
641
642
643 void IndexIVFFlatIPBounds::search (
644 idx_t n, const float *x, idx_t k,
645 float *distances, idx_t *labels) const
646 {
647 // compute query remainder norms and distances
648 idx_t * idx = new idx_t [n * nprobe];
649 ScopeDeleter<idx_t> del (idx);
650 quantizer->assign (n, x, idx, nprobe);
651
652 float * qnorms = new float [n];
653 ScopeDeleter <float> del2 (qnorms);
654
655 #pragma omp parallel for
656 for (size_t i = 0; i < n; i++) {
657 qnorms[i] = std::sqrt (fvec_norm_L2sqr (
658 x + i * d + fsize, d - fsize));
659 }
660
661 float_minheap_array_t res = {
662 size_t(n), size_t(k), labels, distances};
663
664 search_bounds_knn_inner_product (*this, x, idx, &res, qnorms);
665
666 }
667
668 } // namespace faiss
faiss::IndexIVF
Definition: IndexIVF.h:45
faiss::IndexIVFFlat::search_preassigned
void search_preassigned(idx_t n, const float *x, idx_t k, const idx_t *assign, float *distances, idx_t *labels) const
perform search, without computing the assignment to the quantizer
Definition: IndexIVF.cpp:375
faiss::ClusteringParameters::niter
int niter
clustering iterations
Definition: Clustering.h:25
faiss::RangeSearchPartialResult::QueryResult
result structure for a single query
Definition: AuxIndexStructures.h:154
faiss::fvec_L2sqr
float fvec_L2sqr(const float *x, const float *y, size_t d)
Squared L2 distance between two vectors.
Definition: utils.cpp:481
faiss::IndexIVFFlat::search_knn_L2sqr
void search_knn_L2sqr(size_t nx, const float *x, const long *keys, float_maxheap_array_t *res) const
Implementation of the search for the L2 metric.
Definition: IndexIVF.cpp:315
faiss::HeapArray::get_val
T * get_val(size_t key)
Return the list of values for a heap.
Definition: Heap.h:360
faiss::IndexIVF::imbalance_factor
double imbalance_factor() const
1= perfectly balanced, &gt;1: imbalanced
Definition: IndexIVF.cpp:133
faiss::Index::reset
virtual void reset()=0
removes all elements from the database.
faiss::IndexIVF::nprobe
size_t nprobe
number of probes at query time
Definition: IndexIVF.h:47
faiss::IndexIVFFlat::reconstruct
void reconstruct(idx_t key, float *recons) const override
Definition: IndexIVF.cpp:522
faiss::Index::assign
void assign(idx_t n, const float *x, idx_t *labels, idx_t k=1)
Definition: Index.cpp:23
faiss::IDSelector
Definition: AuxIndexStructures.h:57
faiss::IndexIVF::quantizer_trains_alone
bool quantizer_trains_alone
just pass over the trainset to quantizer
Definition: IndexIVF.h:50
faiss::IndexIVFFlat::range_search
void range_search(idx_t n, const float *x, float radius, RangeSearchResult *result) const override
Definition: IndexIVF.cpp:393
faiss::IndexIVFFlat::copy_subset_to
void copy_subset_to(IndexIVFFlat &other, int subset_type, long a1, long a2) const
Definition: IndexIVF.cpp:455
faiss::IndexIVFFlat::merge_from_residuals
void merge_from_residuals(IndexIVF &other) override
Definition: IndexIVF.cpp:443
faiss::Index::add_with_ids
virtual void add_with_ids(idx_t n, const float *x, const long *xids)
Definition: Index.cpp:31
faiss::IndexIVF::train_residual
virtual void train_residual(idx_t n, const float *x)
Definition: IndexIVF.cpp:124
faiss::HeapArray::k
size_t k
allocated size per heap
Definition: Heap.h:355
faiss::ScopeDeleter
Definition: FaissException.h:39
faiss::imbalance_factor
double imbalance_factor(int n, int k, const long *assign)
a balanced assignment has a IF of 1
Definition: utils.cpp:1593
faiss::IndexIVFFlat::remove_ids
long remove_ids(const IDSelector &sel) override
Definition: IndexIVF.cpp:490
faiss::IndexIVFFlat
Definition: IndexIVF.h:129
faiss::IndexIVF::ids
std::vector< std::vector< long > > ids
Inverted lists for indexes.
Definition: IndexIVF.h:55
faiss::Index::d
int d
vector dimension
Definition: Index.h:64
faiss::IndexIVF::quantizer
Index * quantizer
quantizer that maps vectors to inverted lists
Definition: IndexIVF.h:49
faiss::IndexIVF::train
void train(idx_t n, const float *x) override
Trains the quantizer and calls train_residual to train sub-quantizers.
Definition: IndexIVF.cpp:95
faiss::IndexIVF::cp
ClusteringParameters cp
to override default clustering params
Definition: IndexIVF.h:53
faiss::IndexIVFFlat::add_with_ids
void add_with_ids(idx_t n, const float *x, const long *xids) override
implemented for all IndexIVF* classes
Definition: IndexIVF.cpp:210
faiss::Clustering
Definition: Clustering.h:56
faiss::IndexIVF::own_fields
bool own_fields
whether object owns the quantizer
Definition: IndexIVF.h:51
faiss::HeapArray
Definition: Heap.h:350
faiss::Index::idx_t
long idx_t
all indices are this type
Definition: Index.h:62
faiss::IndexIVFFlat::reset
void reset() override
removes all elements from the database.
Definition: IndexIVF.cpp:482
faiss::IndexIVF::make_direct_map
void make_direct_map()
intialize a direct map
Definition: IndexIVF.cpp:68
faiss::Index::ntotal
idx_t ntotal
total nb of indexed vectors
Definition: Index.h:65
faiss::Index::verbose
bool verbose
verbosity level
Definition: Index.h:66
faiss::IndexIVF::reset
void reset() override
removes all elements from the database.
Definition: IndexIVF.cpp:86
faiss::RangeSearchPartialResult::new_result
QueryResult & new_result(idx_t qno)
begin a new result
Definition: AuxIndexStructures.cpp:113
faiss::IndexIVFFlatIPBounds::search
void search(idx_t n, const float *x, idx_t k, float *distances, idx_t *labels) const override
Definition: IndexIVF.cpp:643
faiss::IndexIVFFlatIPBounds::part_norms
std::vector< std::vector< float > > part_norms
norm of remainder (dimensions fsize:d)
Definition: IndexIVF.h:200
faiss::fvec_norm_L2sqr
float fvec_norm_L2sqr(const float *x, size_t d)
Definition: utils.cpp:538
faiss::IndexIVFFlatIPBounds
Definition: IndexIVF.h:194
faiss::IndexIVFFlatIPBounds::fsize
size_t fsize
nb of dimensions of pre-filter
Definition: IndexIVF.h:197
faiss::RangeSearchPartialResult
the entries in the buffers are split per query
Definition: AuxIndexStructures.h:148
faiss::RangeSearchResult
Definition: AuxIndexStructures.h:34
faiss::IndexIVF::merge_from_residuals
virtual void merge_from_residuals(IndexIVF &other)=0
faiss::HeapArray::get_ids
TI * get_ids(size_t key)
Correspponding identifiers.
Definition: Heap.h:363
faiss::Index::metric_type
MetricType metric_type
type of metric this index uses for search
Definition: Index.h:72
faiss::IndexIVF::print_stats
void print_stats() const
display some stats about the inverted lists
Definition: IndexIVF.cpp:142
faiss::IndexIVFFlatIPBounds::add_core
void add_core(idx_t n, const float *x, const long *xids, const long *precomputed_idx) override
same as add_with_ids, with precomputed coarse quantizer
Definition: IndexIVF.cpp:548
faiss::HeapArray::nh
size_t nh
number of heaps
Definition: Heap.h:354
faiss::IndexIVF::nlist
size_t nlist
number of possible key values
Definition: IndexIVF.h:46
faiss::Index
Definition: Index.h:60
faiss::IndexIVF::add
void add(idx_t n, const float *x) override
Quantizes x and calls add_with_key.
Definition: IndexIVF.cpp:63
faiss::Clustering::train
virtual void train(idx_t n, const float *x, faiss::Index &index)
Index is used during the assignment stage.
Definition: Clustering.cpp:66
faiss::Index::is_trained
bool is_trained
set if the Index does not require training, or if training is done already
Definition: Index.h:69
faiss::IndexIVFFlat::search_knn_inner_product
void search_knn_inner_product(size_t nx, const float *x, const long *keys, float_minheap_array_t *res) const
Implementation of the search for the inner product metric.
Definition: IndexIVF.cpp:264
faiss::Index::train
virtual void train(idx_t n, const float *x)
Definition: Index.h:89
faiss::IndexIVF::maintain_direct_map
bool maintain_direct_map
map for direct access to the elements. Enables reconstruct().
Definition: IndexIVF.h:58
faiss::ClusteringParameters::spherical
bool spherical
do we want normalized centroids?
Definition: Clustering.h:29
faiss::IndexIVF::merge_from
virtual void merge_from(IndexIVF &other, idx_t add_id)
Definition: IndexIVF.cpp:162
faiss::MetricType
MetricType
Some algorithms support both an inner product vetsion and a L2 search version.
Definition: Index.h:43
faiss::IndexIVFFlat::vecs
std::vector< std::vector< float > > vecs
Definition: IndexIVF.h:132
faiss::IndexIVFFlat::add_core
virtual void add_core(idx_t n, const float *x, const long *xids, const long *precomputed_idx)
same as add_with_ids, with precomputed coarse quantizer
Definition: IndexIVF.cpp:215
faiss::IndexIVFFlat::search
void search(idx_t n, const float *x, idx_t k, float *distances, idx_t *labels) const override
Definition: IndexIVF.cpp:365