Faiss
 All Classes Namespaces Functions Variables Typedefs Enumerations Enumerator Friends
test_dealloc_invlists.cpp
1 /**
2  * Copyright (c) 2015-present, Facebook, Inc.
3  * All rights reserved.
4  *
5  * This source code is licensed under the BSD+Patents license found in the
6  * LICENSE file in the root directory of this source tree.
7  */
8 
9 #include <cstdio>
10 #include <cstdlib>
11 
12 #include <memory>
13 #include <vector>
14 
15 #include <gtest/gtest.h>
16 
17 #include <faiss/IndexIVF.h>
18 #include <faiss/AutoTune.h>
19 #include <faiss/index_io.h>
20 #include <faiss/IVFlib.h>
21 
22 using namespace faiss;
23 
24 namespace {
25 
26 typedef Index::idx_t idx_t;
27 
28 
29 // dimension of the vectors to index
30 int d = 32;
31 
32 // nb of training vectors
33 size_t nt = 5000;
34 
35 // size of the database points per window step
36 size_t nb = 1000;
37 
38 // nb of queries
39 size_t nq = 200;
40 
41 
42 int total_size = 40;
43 int window_size = 10;
44 
45 
46 
47 
48 
49 std::vector<float> make_data(size_t n)
50 {
51  std::vector <float> database (n * d);
52  for (size_t i = 0; i < n * d; i++) {
53  database[i] = drand48();
54  }
55  return database;
56 }
57 
58 std::unique_ptr<Index> make_trained_index(const char *index_type)
59 {
60  auto index = std::unique_ptr<Index>(index_factory(d, index_type));
61  auto xt = make_data(nt * d);
62  index->train(nt, xt.data());
63  ParameterSpace().set_index_parameter (index.get(), "nprobe", 4);
64  return index;
65 }
66 
67 std::vector<idx_t> search_index(Index *index, const float *xq) {
68  int k = 10;
69  std::vector<idx_t> I(k * nq);
70  std::vector<float> D(k * nq);
71  index->search (nq, xq, k, D.data(), I.data());
72  return I;
73 }
74 
75 
76 
77 
78 
79 /*************************************************************
80  * Test functions for a given index type
81  *************************************************************/
82 
83 struct EncapsulateInvertedLists: InvertedLists {
84 
85  const InvertedLists *il;
86 
87  EncapsulateInvertedLists(const InvertedLists *il):
88  InvertedLists(il->nlist, il->code_size),
89  il(il)
90  {}
91 
92  static void * memdup (const void *m, size_t size) {
93  if (size == 0) return nullptr;
94  return memcpy (malloc(size), m, size);
95  }
96 
97  size_t list_size(size_t list_no) const override {
98  return il->list_size (list_no);
99  }
100 
101  const uint8_t * get_codes (size_t list_no) const override {
102  return (uint8_t*)memdup (il->get_codes(list_no),
103  list_size(list_no) * code_size);
104  }
105 
106  const idx_t * get_ids (size_t list_no) const override {
107  return (idx_t*)memdup (il->get_ids(list_no),
108  list_size(list_no) * sizeof(idx_t));
109  }
110 
111  void release_codes (const uint8_t *codes) const override {
112  free ((void*)codes);
113  }
114 
115  void release_ids (const idx_t *ids) const override {
116  free ((void*)ids);
117  }
118 
119  const uint8_t * get_single_code (size_t list_no, size_t offset)
120  const override {
121  return (uint8_t*)memdup (il->get_single_code(list_no, offset),
122  code_size);
123  }
124 
125  virtual size_t add_entries (
126  size_t, size_t,
127  const idx_t*, const uint8_t *) override
128  {
129  assert(!"not implemented");
130  return 0;
131  }
132 
133 
134  virtual void update_entries (size_t , size_t , size_t ,
135  const idx_t *, const uint8_t *)
136  override {
137  assert(!"not implemented");
138  }
139 
140  virtual void resize (size_t , size_t ) override {
141  assert(!"not implemented");
142  }
143 
144  ~EncapsulateInvertedLists () {}
145 };
146 
147 
148 
149 int test_dealloc_invlists (const char *index_key) {
150 
151  std::unique_ptr<Index> index = make_trained_index(index_key);
152  IndexIVF * index_ivf = ivflib::extract_index_ivf (index.get());
153 
154  auto xb = make_data (nb * d);
155  index->add(nb, xb.data());
156 
157  auto xq = make_data (nq * d);
158 
159  auto ref_res = search_index (index.get(), xq.data());
160 
161  EncapsulateInvertedLists eil(index_ivf->invlists);
162 
163  index_ivf->own_invlists = false;
164  index_ivf->replace_invlists (&eil, false);
165 
166  // TEST: this could crash or leak mem
167  auto new_res = search_index (index.get(), xq.data());
168 
169  // delete explicitly
170  delete eil.il;
171 
172  // just to make sure
173  EXPECT_EQ (ref_res, new_res);
174  return 0;
175 }
176 
177 } // anonymous namespace
178 
179 
180 
181 /*************************************************************
182  * Test entry points
183  *************************************************************/
184 
185 TEST(TestIvlistDealloc, IVFFlat) {
186  test_dealloc_invlists ("IVF32,Flat");
187 }
188 
189 TEST(TestIvlistDealloc, IVFSQ) {
190  test_dealloc_invlists ("IVF32,SQ8");
191 }
192 
193 TEST(TestIvlistDealloc, IVFPQ) {
194  test_dealloc_invlists ("IVF32,PQ4np");
195 }
void train(idx_t n, const float *x) override
long idx_t
all indices are this type
Definition: Index.h:64
void replace_invlists(InvertedLists *il, bool own=false)
replace the inverted lists, old one is deallocated if own_invlists
Definition: IndexIVF.cpp:486
virtual void search(idx_t n, const float *x, idx_t k, float *distances, idx_t *labels) const =0
virtual void set_index_parameter(Index *index, const std::string &name, double val) const
set one of the parameters
Definition: AutoTune.cpp:452
InvertedLists * invlists
Acess to the actual data.
Definition: IndexIVF.h:93
Index * index_factory(int d, const char *description_in, MetricType metric)
Definition: AutoTune.cpp:722