Faiss

- Main Page
- Namespaces
- Classes
- Files

- File List

All Classes Namespaces Functions Variables Typedefs Enumerations Enumerator Friends
test_dealloc_invlists.cpp
1 /**
2 * Copyright (c) 2015-present, Facebook, Inc.
3 * All rights reserved.
4 *
5 * This source code is licensed under the BSD+Patents license found in the
6 * LICENSE file in the root directory of this source tree.
7 */
8
9 #include <cstdio>
10 #include <cstdlib>
11
12 #include <memory>
13 #include <vector>
14
15 #include <gtest/gtest.h>
16
17 #include <faiss/IndexIVF.h>
18 #include <faiss/AutoTune.h>
19 #include <faiss/index_io.h>
20 #include <faiss/IVFlib.h>
21
22 using namespace faiss;
23
24 namespace {
25
26 typedef Index::idx_t idx_t;
27
28
29 // dimension of the vectors to index
30 int d = 32;
31
32 // nb of training vectors
33 size_t nt = 5000;
34
35 // size of the database points per window step
36 size_t nb = 1000;
37
38 // nb of queries
39 size_t nq = 200;
40
41
42 int total_size = 40;
43 int window_size = 10;
44
45
46
47
48
49 std::vector<float> make_data(size_t n)
50 {
51 std::vector <float> database (n * d);
52 for (size_t i = 0; i < n * d; i++) {
53 database[i] = drand48();
54 }
55 return database;
56 }
57
58 std::unique_ptr<Index> make_trained_index(const char *index_type)
59 {
60 auto index = std::unique_ptr<Index>(index_factory(d, index_type));
61 auto xt = make_data(nt * d);
62 index->train(nt, xt.data());
63 ParameterSpace().set_index_parameter (index.get(), "nprobe", 4);
64 return index;
65 }
66
67 std::vector<idx_t> search_index(Index *index, const float *xq) {
68 int k = 10;
69 std::vector<idx_t> I(k * nq);
70 std::vector<float> D(k * nq);
71 index->search (nq, xq, k, D.data(), I.data());
72 return I;
73 }
74
75
76
77
78
79 /*************************************************************
80 * Test functions for a given index type
81 *************************************************************/
82
83 struct EncapsulateInvertedLists: InvertedLists {
84
85 const InvertedLists *il;
86
87 EncapsulateInvertedLists(const InvertedLists *il):
88 InvertedLists(il->nlist, il->code_size),
89 il(il)
90 {}
91
92 static void * memdup (const void *m, size_t size) {
93 if (size == 0) return nullptr;
94 return memcpy (malloc(size), m, size);
95 }
96
97 size_t list_size(size_t list_no) const override {
98 return il->list_size (list_no);
99 }
100
101 const uint8_t * get_codes (size_t list_no) const override {
102 return (uint8_t*)memdup (il->get_codes(list_no),
103 list_size(list_no) * code_size);
104 }
105
106 const idx_t * get_ids (size_t list_no) const override {
107 return (idx_t*)memdup (il->get_ids(list_no),
108 list_size(list_no) * sizeof(idx_t));
109 }
110
111 void release_codes (const uint8_t *codes) const override {
112 free ((void*)codes);
113 }
114
115 void release_ids (const idx_t *ids) const override {
116 free ((void*)ids);
117 }
118
119 const uint8_t * get_single_code (size_t list_no, size_t offset)
120 const override {
121 return (uint8_t*)memdup (il->get_single_code(list_no, offset),
122 code_size);
123 }
124
125 virtual size_t add_entries (
126 size_t, size_t,
127 const idx_t*, const uint8_t *) override
128 {
129 assert(!"not implemented");
130 return 0;
131 }
132
133
134 virtual void update_entries (size_t , size_t , size_t ,
135 const idx_t *, const uint8_t *)
136 override {
137 assert(!"not implemented");
138 }
139
140 virtual void resize (size_t , size_t ) override {
141 assert(!"not implemented");
142 }
143
144 ~EncapsulateInvertedLists () {}
145 };
146
147
148
149 int test_dealloc_invlists (const char *index_key) {
150
151 std::unique_ptr<Index> index = make_trained_index(index_key);
152 IndexIVF * index_ivf = ivflib::extract_index_ivf (index.get());
153
154 auto xb = make_data (nb * d);
155 index->add(nb, xb.data());
156
157 auto xq = make_data (nq * d);
158
159 auto ref_res = search_index (index.get(), xq.data());
160
161 EncapsulateInvertedLists eil(index_ivf->invlists);
162
163 index_ivf->own_invlists = false;
164 index_ivf->replace_invlists (&eil, false);
165
166 // TEST: this could crash or leak mem
167 auto new_res = search_index (index.get(), xq.data());
168
169 // delete explicitly
170 delete eil.il;
171
172 // just to make sure
173 EXPECT_EQ (ref_res, new_res);
174 return 0;
175 }
176
177 } // anonymous namespace
178
179
180
181 /*************************************************************
182 * Test entry points
183 *************************************************************/
184
185 TEST(TestIvlistDealloc, IVFFlat) {
186 test_dealloc_invlists ("IVF32,Flat");
187 }
188
189 TEST(TestIvlistDealloc, IVFSQ) {
190 test_dealloc_invlists ("IVF32,SQ8");
191 }
192
193 TEST(TestIvlistDealloc, IVFPQ) {
194 test_dealloc_invlists ("IVF32,PQ4np");
195 }
faiss::IndexShards::train
void train(idx_t n, const float *x) override
Definition: MetaIndexes.cpp:399
faiss::IndexIVF
Definition: IndexIVF.h:91
faiss::InvertedLists
Definition: InvertedLists.h:32
faiss::Index::idx_t
long idx_t
all indices are this type
Definition: Index.h:64
faiss::IndexIVF::replace_invlists
void replace_invlists(InvertedLists *il, bool own=false)
replace the inverted lists, old one is deallocated if own_invlists
Definition: IndexIVF.cpp:486
faiss::Index::search
virtual void search(idx_t n, const float *x, idx_t k, float *distances, idx_t *labels) const =0
faiss::ParameterSpace::set_index_parameter
virtual void set_index_parameter(Index *index, const std::string &name, double val) const
set one of the parameters
Definition: AutoTune.cpp:452
faiss::IndexIVF::invlists
InvertedLists * invlists
Acess to the actual data.
Definition: IndexIVF.h:93
faiss::ParameterSpace
Definition: AutoTune.h:132
faiss::Index
Definition: Index.h:62
faiss::index_factory
Index * index_factory(int d, const char *description_in, MetricType metric)
Definition: AutoTune.cpp:722