Faiss

- Main Page
- Namespaces
- Classes
- Files

- File List

All Classes Namespaces Functions Variables Typedefs Enumerations Enumerator Friends
/tmp/faiss/InvertedLists.cpp
1 /**
2 * Copyright (c) 2015-present, Facebook, Inc.
3 * All rights reserved.
4 *
5 * This source code is licensed under the BSD+Patents license found in the
6 * LICENSE file in the root directory of this source tree.
7 */
8
9 // -*- c++ -*-
10
11 #include "InvertedLists.h"
12
13 #include <cstdio>
14
15 #include "utils.h"
16 #include "FaissAssert.h"
17
18 namespace faiss {
19
20 using ScopedIds = InvertedLists::ScopedIds;
21 using ScopedCodes = InvertedLists::ScopedCodes;
22
23
24 /*****************************************
25 * InvertedLists implementation
26 ******************************************/
27
28 InvertedLists::InvertedLists (size_t nlist, size_t code_size):
29 nlist (nlist), code_size (code_size)
30 {
31 }
32
33 InvertedLists::~InvertedLists ()
34 {}
35
36 InvertedLists::idx_t InvertedLists::get_single_id (
37 size_t list_no, size_t offset) const
38 {
39 assert (offset < list_size (list_no));
40 return get_ids(list_no)[offset];
41 }
42
43
44 void InvertedLists::release_codes (const uint8_t *) const
45 {}
46
47 void InvertedLists::release_ids (const idx_t *) const
48 {}
49
50 void InvertedLists::prefetch_lists (const long *, int) const
51 {}
52
53 const uint8_t * InvertedLists::get_single_code (
54 size_t list_no, size_t offset) const
55 {
56 assert (offset < list_size (list_no));
57 return get_codes(list_no) + offset * code_size;
58 }
59
60 size_t InvertedLists::add_entry (size_t list_no, idx_t theid,
61 const uint8_t *code)
62 {
63 return add_entries (list_no, 1, &theid, code);
64 }
65
66 void InvertedLists::update_entry (size_t list_no, size_t offset,
67 idx_t id, const uint8_t *code)
68 {
69 update_entries (list_no, offset, 1, &id, code);
70 }
71
72 void InvertedLists::reset () {
73 for (size_t i = 0; i < nlist; i++) {
74 resize (i, 0);
75 }
76 }
77
78 void InvertedLists::merge_from (InvertedLists *oivf, size_t add_id) {
79
80 #pragma omp parallel for
81 for (long i = 0; i < nlist; i++) {
82 size_t list_size = oivf->list_size (i);
83 ScopedIds ids (oivf, i);
84 if (add_id == 0) {
85 add_entries (i, list_size, ids.get (),
86 ScopedCodes (oivf, i).get());
87 } else {
88 std::vector <idx_t> new_ids (list_size);
89
90 for (size_t j = 0; j < list_size; j++) {
91 new_ids [j] = ids[j] + add_id;
92 }
93 add_entries (i, list_size, new_ids.data(),
94 ScopedCodes (oivf, i).get());
95 }
96 oivf->resize (i, 0);
97 }
98 }
99
100 /*****************************************
101 * ArrayInvertedLists implementation
102 ******************************************/
103
104 ArrayInvertedLists::ArrayInvertedLists (size_t nlist, size_t code_size):
105 InvertedLists (nlist, code_size)
106 {
107 ids.resize (nlist);
108 codes.resize (nlist);
109 }
110
111 size_t ArrayInvertedLists::add_entries (
112 size_t list_no, size_t n_entry,
113 const idx_t* ids_in, const uint8_t *code)
114 {
115 if (n_entry == 0) return 0;
116 assert (list_no < nlist);
117 size_t o = ids [list_no].size();
118 ids [list_no].resize (o + n_entry);
119 memcpy (&ids[list_no][o], ids_in, sizeof (ids_in[0]) * n_entry);
120 codes [list_no].resize ((o + n_entry) * code_size);
121 memcpy (&codes[list_no][o * code_size], code, code_size * n_entry);
122 return o;
123 }
124
125 size_t ArrayInvertedLists::list_size(size_t list_no) const
126 {
127 assert (list_no < nlist);
128 return ids[list_no].size();
129 }
130
131 const uint8_t * ArrayInvertedLists::get_codes (size_t list_no) const
132 {
133 assert (list_no < nlist);
134 return codes[list_no].data();
135 }
136
137
138 const InvertedLists::idx_t * ArrayInvertedLists::get_ids (size_t list_no) const
139 {
140 assert (list_no < nlist);
141 return ids[list_no].data();
142 }
143
144 void ArrayInvertedLists::resize (size_t list_no, size_t new_size)
145 {
146 ids[list_no].resize (new_size);
147 codes[list_no].resize (new_size * code_size);
148 }
149
150 void ArrayInvertedLists::update_entries (
151 size_t list_no, size_t offset, size_t n_entry,
152 const idx_t *ids_in, const uint8_t *codes_in)
153 {
154 assert (list_no < nlist);
155 assert (n_entry + offset <= ids[list_no].size());
156 memcpy (&ids[list_no][offset], ids_in, sizeof(ids_in[0]) * n_entry);
157 memcpy (&codes[list_no][offset * code_size], codes_in, code_size * n_entry);
158 }
159
160
161 ArrayInvertedLists::~ArrayInvertedLists ()
162 {}
163
164
165 /*****************************************
166 * ConcatenatedInvertedLists implementation
167 ******************************************/
168
169 ConcatenatedInvertedLists::ConcatenatedInvertedLists (
170 int nil, const InvertedLists **ils_in):
171 InvertedLists (nil > 0 ? ils_in[0]->nlist : 0,
172 nil > 0 ? ils_in[0]->code_size : 0)
173 {
174 FAISS_THROW_IF_NOT (nil > 0);
175 for (int i = 0; i < nil; i++) {
176 ils.push_back (ils_in[i]);
177 FAISS_THROW_IF_NOT (ils_in[i]->code_size == code_size &&
178 ils_in[i]->nlist == nlist);
179 }
180 }
181
182 size_t ConcatenatedInvertedLists::list_size(size_t list_no) const
183 {
184 size_t sz = 0;
185 for (int i = 0; i < ils.size(); i++) {
186 const InvertedLists *il = ils[i];
187 sz += il->list_size (list_no);
188 }
189 return sz;
190 }
191
192 const uint8_t * ConcatenatedInvertedLists::get_codes (size_t list_no) const
193 {
194 uint8_t *codes = new uint8_t [code_size * list_size(list_no)], *c = codes;
195
196 for (int i = 0; i < ils.size(); i++) {
197 const InvertedLists *il = ils[i];
198 size_t sz = il->list_size(list_no) * code_size;
199 if (sz > 0) {
200 memcpy (c, ScopedCodes (il, list_no).get(), sz);
201 c += sz;
202 }
203 }
204 return codes;
205 }
206
207 const uint8_t * ConcatenatedInvertedLists::get_single_code (
208 size_t list_no, size_t offset) const
209 {
210 for (int i = 0; i < ils.size(); i++) {
211 const InvertedLists *il = ils[i];
212 size_t sz = il->list_size (list_no);
213 if (offset < sz) {
214 // here we have to copy the code, otherwise it will crash at dealloc
215 uint8_t * code = new uint8_t [code_size];
216 memcpy (code, ScopedCodes (il, list_no, offset).get(), code_size);
217 return code;
218 }
219 offset -= sz;
220 }
221 FAISS_THROW_FMT ("offset %ld unknown", offset);
222 }
223
224
225 void ConcatenatedInvertedLists::release_codes (const uint8_t *codes) const {
226 delete [] codes;
227 }
228
229 const Index::idx_t * ConcatenatedInvertedLists::get_ids (size_t list_no) const
230 {
231 idx_t *ids = new idx_t [list_size(list_no)], *c = ids;
232
233 for (int i = 0; i < ils.size(); i++) {
234 const InvertedLists *il = ils[i];
235 size_t sz = il->list_size(list_no);
236 if (sz > 0) {
237 memcpy (c, ScopedIds (il, list_no).get(), sz * sizeof(idx_t));
238 c += sz;
239 }
240 }
241 return ids;
242 }
243
244 Index::idx_t ConcatenatedInvertedLists::get_single_id (
245 size_t list_no, size_t offset) const
246 {
247
248 for (int i = 0; i < ils.size(); i++) {
249 const InvertedLists *il = ils[i];
250 size_t sz = il->list_size (list_no);
251 if (offset < sz) {
252 return il->get_single_id (list_no, offset);
253 }
254 offset -= sz;
255 }
256 FAISS_THROW_FMT ("offset %ld unknown", offset);
257 }
258
259
260 void ConcatenatedInvertedLists::release_ids (const idx_t *ids) const {
261 delete [] ids;
262 }
263
264 size_t ConcatenatedInvertedLists::add_entries (
265 size_t , size_t ,
266 const idx_t* , const uint8_t *)
267 {
268 FAISS_THROW_MSG ("not implemented");
269 }
270
271 void ConcatenatedInvertedLists::update_entries (size_t, size_t , size_t ,
272 const idx_t *, const uint8_t *)
273 {
274 FAISS_THROW_MSG ("not implemented");
275 }
276
277 void ConcatenatedInvertedLists::resize (size_t , size_t )
278 {
279 FAISS_THROW_MSG ("not implemented");
280 }
281
282
283
284
285 } // namespace faiss
faiss::ArrayInvertedLists::get_codes
const uint8_t * get_codes(size_t list_no) const override
Definition: InvertedLists.cpp:131
faiss::ArrayInvertedLists::get_ids
const idx_t * get_ids(size_t list_no) const override
Definition: InvertedLists.cpp:138
faiss::InvertedLists::get_ids
virtual const idx_t * get_ids(size_t list_no) const =0
faiss::InvertedLists::list_size
virtual size_t list_size(size_t list_no) const =0
get the size of a list
faiss::ConcatenatedInvertedLists::release_codes
void release_codes(const uint8_t *codes) const override
release codes returned by get_codes (default implementation is nop
Definition: InvertedLists.cpp:225
faiss::ConcatenatedInvertedLists::release_ids
void release_ids(const idx_t *ids) const override
release ids returned by get_ids
Definition: InvertedLists.cpp:260
faiss::ConcatenatedInvertedLists::ConcatenatedInvertedLists
ConcatenatedInvertedLists(int nil, const InvertedLists **ils)
build InvertedLists by concatenating nil of them
Definition: InvertedLists.cpp:169
faiss::InvertedLists
Definition: InvertedLists.h:32
faiss::InvertedLists::merge_from
void merge_from(InvertedLists *oivf, size_t add_id)
move all entries from oivf (empty on output)
Definition: InvertedLists.cpp:78
faiss::InvertedLists::get_single_id
virtual idx_t get_single_id(size_t list_no, size_t offset) const
Definition: InvertedLists.cpp:36
faiss::InvertedLists::code_size
size_t code_size
code size per vector in bytes
Definition: InvertedLists.h:36
faiss::InvertedLists::get_single_code
virtual const uint8_t * get_single_code(size_t list_no, size_t offset) const
Definition: InvertedLists.cpp:53
faiss::ConcatenatedInvertedLists::get_ids
const idx_t * get_ids(size_t list_no) const override
Definition: InvertedLists.cpp:229
faiss::InvertedLists::add_entry
virtual size_t add_entry(size_t list_no, idx_t theid, const uint8_t *code)
add one entry to an inverted list
Definition: InvertedLists.cpp:60
faiss::Index::idx_t
long idx_t
all indices are this type
Definition: Index.h:64
faiss::ConcatenatedInvertedLists::get_single_code
const uint8_t * get_single_code(size_t list_no, size_t offset) const override
Definition: InvertedLists.cpp:207
faiss::ConcatenatedInvertedLists::get_single_id
idx_t get_single_id(size_t list_no, size_t offset) const override
Definition: InvertedLists.cpp:244
faiss::InvertedLists::prefetch_lists
virtual void prefetch_lists(const long *list_nos, int nlist) const
Definition: InvertedLists.cpp:50
faiss::InvertedLists::nlist
size_t nlist
number of possible key values
Definition: InvertedLists.h:35
faiss::ConcatenatedInvertedLists::list_size
size_t list_size(size_t list_no) const override
get the size of a list
Definition: InvertedLists.cpp:182
faiss::ArrayInvertedLists::ids
std::vector< std::vector< idx_t > > ids
Inverted lists for indexes.
Definition: InvertedLists.h:170
faiss::ConcatenatedInvertedLists::get_codes
const uint8_t * get_codes(size_t list_no) const override
Definition: InvertedLists.cpp:192
faiss::InvertedLists::release_codes
virtual void release_codes(const uint8_t *codes) const
release codes returned by get_codes (default implementation is nop
Definition: InvertedLists.cpp:44
faiss::InvertedLists::get_codes
virtual const uint8_t * get_codes(size_t list_no) const =0
faiss::InvertedLists::ScopedIds
Definition: InvertedLists.h:125
faiss::ArrayInvertedLists::list_size
size_t list_size(size_t list_no) const override
get the size of a list
Definition: InvertedLists.cpp:125
faiss::InvertedLists::release_ids
virtual void release_ids(const idx_t *ids) const
release ids returned by get_ids
Definition: InvertedLists.cpp:47
faiss::InvertedLists::ScopedCodes
Definition: InvertedLists.h:144