All Classes Namespaces Functions Variables Typedefs Enumerations Enumerator Friends
/data/users/matthijs/github_faiss/faiss/MetaIndexes.h
/**
* Copyright (c) 2015-present, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under the CC-by-NC license found in the
* LICENSE file in the root directory of this source tree.
*/
// Copyright 2004-present Facebook. All Rights Reserved.
// -*- c++ -*-
#ifndef META_INDEXES_H
#define META_INDEXES_H
#include <vector>
#include <unordered_map>
#include "Index.h"
namespace faiss {
/** Index that translates search results to ids */
struct IndexIDMap : Index {
Index * index; ///! the sub-index
bool own_fields; ///! whether pointers are deleted in destructo
std::vector<long> id_map;
explicit IndexIDMap (Index *index);
/// Same as add_core, but stores xids instead of sequential ids
/// @param xids if non-null, ids to store for the vectors (size n)
void add_with_ids(idx_t n, const float* x, const long* xids) override;
/// this will fail. Use add_with_ids
void add(idx_t n, const float* x) override;
void search(
idx_t n,
const float* x,
idx_t k,
float* distances,
idx_t* labels) const override;
void train(idx_t n, const float* x) override;
void reset() override;
/// remove ids adapted to IndexFlat
long remove_ids(const IDSelector& sel) override;
~IndexIDMap() override;
IndexIDMap () {own_fields=false; index=nullptr; }
};
/** same as IndexIDMap but also provides an efficient reconstruction
implementation via a 2-way index */
struct IndexIDMap2 : IndexIDMap {
std::unordered_map<idx_t, idx_t> rev_map;
explicit IndexIDMap2 (Index *index);
/// make the rev_map from scratch
void construct_rev_map ();
void add_with_ids(idx_t n, const float* x, const long* xids) override;
long remove_ids(const IDSelector& sel) override;
void reconstruct (idx_t key, float * recons) const override;
~IndexIDMap2() override {}
IndexIDMap2 () {}
};
/** Index that concatenates the results from several sub-indexes
*
*/
struct IndexShards : Index {
std::vector<Index*> shard_indexes;
bool own_fields; /// should the sub-indexes be deleted along with this?
bool threaded;
bool successive_ids;
/**
* @param threaded do we use one thread per sub_index or do
* queries sequentially?
* @param successive_ids should we shift the returned ids by
* the size of each sub-index or return them
* as they are?
*/
explicit IndexShards (idx_t d, bool threaded = false,
bool successive_ids = true);
void add_shard (Index *);
// update metric_type and ntotal. Call if you changes something in
// the shard indexes.
void sync_with_shard_indexes ();
Index *at(int i) {return shard_indexes[i]; }
/// supported only for sub-indices that implement add_with_ids
void add(idx_t n, const float* x) override;
void add_with_ids(idx_t n, const float* x, const long* xids) override;
void search(
idx_t n,
const float* x,
idx_t k,
float* distances,
idx_t* labels) const override;
void train(idx_t n, const float* x) override;
void reset() override;
~IndexShards() override;
};
/** splits input vectors in segments and assigns each segment to a sub-index
* used to distribute a MultiIndexQuantizer
*/
struct IndexSplitVectors: Index {
bool own_fields;
bool threaded;
std::vector<Index*> sub_indexes;
idx_t sum_d; /// sum of dimensions seen so far
explicit IndexSplitVectors (idx_t d, bool threaded = false);
void add_sub_index (Index *);
void sync_with_sub_indexes ();
void add(idx_t n, const float* x) override;
void search(
idx_t n,
const float* x,
idx_t k,
float* distances,
idx_t* labels) const override;
void train(idx_t n, const float* x) override;
void reset() override;
~IndexSplitVectors() override;
};
}
#endif
faiss::IndexShards::train
void train(idx_t n, const float *x) override
Definition: MetaIndexes.cpp:385
faiss::IndexIDMap2
Definition: MetaIndexes.h:60
faiss::IndexShards::IndexShards
IndexShards(idx_t d, bool threaded=false, bool successive_ids=true)
Definition: MetaIndexes.cpp:355
faiss::IndexIDMap::search
void search(idx_t n, const float *x, idx_t k, float *distances, idx_t *labels) const override
Definition: MetaIndexes.cpp:69
faiss::IndexShards::add_with_ids
void add_with_ids(idx_t n, const float *x, const long *xids) override
Definition: MetaIndexes.cpp:425
faiss::IDSelector
Definition: AuxIndexStructures.h:52
faiss::IndexShards::search
void search(idx_t n, const float *x, idx_t k, float *distances, idx_t *labels) const override
Definition: MetaIndexes.cpp:486
faiss::IndexIDMap2::add_with_ids
void add_with_ids(idx_t n, const float *x, const long *xids) override
Definition: MetaIndexes.cpp:130
faiss::IndexIDMap
Definition: MetaIndexes.h:26
faiss::IndexSplitVectors::reset
void reset() override
removes all elements from the database.
Definition: MetaIndexes.cpp:718
faiss::IndexIDMap::add
void add(idx_t n, const float *x) override
this will fail. Use add_with_ids
Definition: MetaIndexes.cpp:40
faiss::IndexSplitVectors
Definition: MetaIndexes.h:131
faiss::Index::d
int d
vector dimension
Definition: Index.h:64
faiss::IndexIDMap::id_map
std::vector< long > id_map
! whether pointers are deleted in destructo
Definition: MetaIndexes.h:29
faiss::IndexSplitVectors::train
void train(idx_t n, const float *x) override
Definition: MetaIndexes.cpp:713
faiss::IndexShards::add
void add(idx_t n, const float *x) override
supported only for sub-indices that implement add_with_ids
Definition: MetaIndexes.cpp:408
faiss::Index::idx_t
long idx_t
all indices are this type
Definition: Index.h:62
faiss::IndexIDMap2::construct_rev_map
void construct_rev_map()
make the rev_map from scratch
Definition: MetaIndexes.cpp:139
faiss::IndexSplitVectors::add
void add(idx_t n, const float *x) override
Definition: MetaIndexes.cpp:610
faiss::IndexIDMap2::remove_ids
long remove_ids(const IDSelector &sel) override
remove ids adapted to IndexFlat
Definition: MetaIndexes.cpp:148
faiss::IndexShards::threaded
bool threaded
should the sub-indexes be deleted along with this?
Definition: MetaIndexes.h:87
faiss::IndexShards::reset
void reset() override
removes all elements from the database.
Definition: MetaIndexes.cpp:478
faiss::IndexIDMap::reset
void reset() override
removes all elements from the database.
Definition: MetaIndexes.cpp:53
faiss::IndexSplitVectors::search
void search(idx_t n, const float *x, idx_t k, float *distances, idx_t *labels) const override
Definition: MetaIndexes.cpp:655
faiss::IndexIDMap::remove_ids
long remove_ids(const IDSelector &sel) override
remove ids adapted to IndexFlat
Definition: MetaIndexes.cpp:95
faiss::IndexIDMap2::reconstruct
void reconstruct(idx_t key, float *recons) const override
Definition: MetaIndexes.cpp:156
faiss::IndexIDMap::train
void train(idx_t n, const float *x) override
Definition: MetaIndexes.cpp:47
faiss::Index
Definition: Index.h:60
faiss::IndexShards
Definition: MetaIndexes.h:83
faiss::IndexIDMap::add_with_ids
void add_with_ids(idx_t n, const float *x, const long *xids) override
Definition: MetaIndexes.cpp:60
faiss::IndexSplitVectors::IndexSplitVectors
IndexSplitVectors(idx_t d, bool threaded=false)
sum of dimensions seen so far
Definition: MetaIndexes.cpp:580
faiss::IndexIDMap::own_fields
bool own_fields
! the sub-index
Definition: MetaIndexes.h:28