Faiss

- Main Page
- Namespaces
- Classes
- Files

- File List

All Classes Namespaces Functions Variables Typedefs Enumerations Enumerator Friends
/tmp/faiss/InvertedLists.h
1 /**
2 * Copyright (c) 2015-present, Facebook, Inc.
3 * All rights reserved.
4 *
5 * This source code is licensed under the BSD+Patents license found in the
6 * LICENSE file in the root directory of this source tree.
7 */
8
9 // -*- c++ -*-
10
11 #ifndef FAISS_INVERTEDLISTS_IVF_H
12 #define FAISS_INVERTEDLISTS_IVF_H
13
14 /**
15 * Definition of inverted lists + a few common classes that implement
16 * the interface.
17 */
18
19 #include <vector>
20 #include "Index.h"
21
22
23 namespace faiss {
24
25 /** Table of inverted lists
26 * multithreading rules:
27 * - concurrent read accesses are allowed
28 * - concurrent update accesses are allowed
29 * - for resize and add_entries, only concurrent access to different lists
30 * are allowed
31 */
32 struct InvertedLists {
33 typedef Index::idx_t idx_t;
34
35 size_t nlist; ///< number of possible key values
36 size_t code_size; ///< code size per vector in bytes
37
38 InvertedLists (size_t nlist, size_t code_size);
39
40 /*************************
41 * Read only functions */
42
43 /// get the size of a list
44 virtual size_t list_size(size_t list_no) const = 0;
45
46 /** get the codes for an inverted list
47 * must be released by release_codes
48 *
49 * @return codes size list_size * code_size
50 */
51 virtual const uint8_t * get_codes (size_t list_no) const = 0;
52
53 /** get the ids for an inverted list
54 * must be released by release_ids
55 *
56 * @return ids size list_size
57 */
58 virtual const idx_t * get_ids (size_t list_no) const = 0;
59
60 /// release codes returned by get_codes (default implementation is nop
61 virtual void release_codes (const uint8_t *codes) const;
62
63 /// release ids returned by get_ids
64 virtual void release_ids (const idx_t *ids) const;
65
66 /// @return a single id in an inverted list
67 virtual idx_t get_single_id (size_t list_no, size_t offset) const;
68
69 /// @return a single code in an inverted list
70 /// (should be deallocated with release_codes)
71 virtual const uint8_t * get_single_code (
72 size_t list_no, size_t offset) const;
73
74 /// prepare the following lists (default does nothing)
75 /// a list can be -1 hence the signed long
76 virtual void prefetch_lists (const long *list_nos, int nlist) const;
77
78 /*************************
79 * writing functions */
80
81 /// add one entry to an inverted list
82 virtual size_t add_entry (size_t list_no, idx_t theid,
83 const uint8_t *code);
84
85 virtual size_t add_entries (
86 size_t list_no, size_t n_entry,
87 const idx_t* ids, const uint8_t *code) = 0;
88
89 virtual void update_entry (size_t list_no, size_t offset,
90 idx_t id, const uint8_t *code);
91
92 virtual void update_entries (size_t list_no, size_t offset, size_t n_entry,
93 const idx_t *ids, const uint8_t *code) = 0;
94
95 virtual void resize (size_t list_no, size_t new_size) = 0;
96
97 virtual void reset ();
98
99 /// move all entries from oivf (empty on output)
100 void merge_from (InvertedLists *oivf, size_t add_id);
101
102 virtual ~InvertedLists ();
103
104 /**************************************
105 * Scoped inverted lists (for automatic deallocation)
106 *
107 * instead of writing:
108 *
109 * uint8_t * codes = invlists->get_codes (10);
110 * ... use codes
111 * invlists->release_codes(codes)
112 *
113 * write:
114 *
115 * ScopedCodes codes (invlists, 10);
116 * ... use codes.get()
117 * // release called automatically when codes goes out of scope
118 *
119 * the following function call also works:
120 *
121 * foo (123, ScopedCodes (invlists, 10).get(), 456);
122 *
123 */
124
125 struct ScopedIds {
126 const InvertedLists *il;
127 const idx_t *ids;
128
129 ScopedIds (const InvertedLists *il, size_t list_no):
130 il (il), ids (il->get_ids (list_no))
131 {}
132
133 const idx_t *get() {return ids; }
134
135 idx_t operator [] (size_t i) const {
136 return ids[i];
137 }
138
139 ~ScopedIds () {
140 il->release_ids (ids);
141 }
142 };
143
144 struct ScopedCodes {
145 const InvertedLists *il;
146 const uint8_t *codes;
147
148 ScopedCodes (const InvertedLists *il, size_t list_no):
149 il (il), codes (il->get_codes (list_no))
150 {}
151
152 ScopedCodes (const InvertedLists *il, size_t list_no, size_t offset):
153 il (il), codes (il->get_single_code (list_no, offset))
154 {}
155
156 const uint8_t *get() {return codes; }
157
158 ~ScopedCodes () {
159 il->release_codes (codes);
160 }
161 };
162
163
164 };
165
166
167 /// simple (default) implementation as an array of inverted lists
168 struct ArrayInvertedLists: InvertedLists {
169 std::vector < std::vector<uint8_t> > codes; // binary codes, size nlist
170 std::vector < std::vector<idx_t> > ids; ///< Inverted lists for indexes
171
172 ArrayInvertedLists (size_t nlist, size_t code_size);
173
174 size_t list_size(size_t list_no) const override;
175 const uint8_t * get_codes (size_t list_no) const override;
176 const idx_t * get_ids (size_t list_no) const override;
177
178 size_t add_entries (
179 size_t list_no, size_t n_entry,
180 const idx_t* ids, const uint8_t *code) override;
181
182 void update_entries (size_t list_no, size_t offset, size_t n_entry,
183 const idx_t *ids, const uint8_t *code) override;
184
185 void resize (size_t list_no, size_t new_size) override;
186
187 virtual ~ArrayInvertedLists ();
188 };
189
190
191 /// inverted lists built as the concatenation of a set of invlists
192 /// (read-only)
193 struct ConcatenatedInvertedLists: InvertedLists {
194
195 std::vector<const InvertedLists *>ils;
196
197 /// build InvertedLists by concatenating nil of them
198 ConcatenatedInvertedLists (int nil, const InvertedLists **ils);
199
200 size_t list_size(size_t list_no) const override;
201 const uint8_t * get_codes (size_t list_no) const override;
202 const idx_t * get_ids (size_t list_no) const override;
203
204 void release_codes (const uint8_t *codes) const override;
205 void release_ids (const idx_t *ids) const override;
206
207 idx_t get_single_id (size_t list_no, size_t offset) const override;
208
209 const uint8_t * get_single_code (
210 size_t list_no, size_t offset) const override;
211
212 size_t add_entries (
213 size_t list_no, size_t n_entry,
214 const idx_t* ids, const uint8_t *code) override;
215
216 void update_entries (size_t list_no, size_t offset, size_t n_entry,
217 const idx_t *ids, const uint8_t *code) override;
218
219 void resize (size_t list_no, size_t new_size) override;
220
221 };
222
223 } // namespace faiss
224
225
226 #endif
faiss::ArrayInvertedLists::get_codes
const uint8_t * get_codes(size_t list_no) const override
Definition: InvertedLists.cpp:131
faiss::ConcatenatedInvertedLists
Definition: InvertedLists.h:193
faiss::ArrayInvertedLists
simple (default) implementation as an array of inverted lists
Definition: InvertedLists.h:168
faiss::ArrayInvertedLists::get_ids
const idx_t * get_ids(size_t list_no) const override
Definition: InvertedLists.cpp:138
faiss::InvertedLists::get_ids
virtual const idx_t * get_ids(size_t list_no) const =0
faiss::InvertedLists::list_size
virtual size_t list_size(size_t list_no) const =0
get the size of a list
faiss::ConcatenatedInvertedLists::release_codes
void release_codes(const uint8_t *codes) const override
release codes returned by get_codes (default implementation is nop
Definition: InvertedLists.cpp:225
faiss::ConcatenatedInvertedLists::release_ids
void release_ids(const idx_t *ids) const override
release ids returned by get_ids
Definition: InvertedLists.cpp:260
faiss::ConcatenatedInvertedLists::ConcatenatedInvertedLists
ConcatenatedInvertedLists(int nil, const InvertedLists **ils)
build InvertedLists by concatenating nil of them
Definition: InvertedLists.cpp:169
faiss::InvertedLists
Definition: InvertedLists.h:32
faiss::InvertedLists::merge_from
void merge_from(InvertedLists *oivf, size_t add_id)
move all entries from oivf (empty on output)
Definition: InvertedLists.cpp:78
faiss::InvertedLists::get_single_id
virtual idx_t get_single_id(size_t list_no, size_t offset) const
Definition: InvertedLists.cpp:36
faiss::InvertedLists::code_size
size_t code_size
code size per vector in bytes
Definition: InvertedLists.h:36
faiss::InvertedLists::get_single_code
virtual const uint8_t * get_single_code(size_t list_no, size_t offset) const
Definition: InvertedLists.cpp:53
faiss::ConcatenatedInvertedLists::get_ids
const idx_t * get_ids(size_t list_no) const override
Definition: InvertedLists.cpp:229
faiss::InvertedLists::add_entry
virtual size_t add_entry(size_t list_no, idx_t theid, const uint8_t *code)
add one entry to an inverted list
Definition: InvertedLists.cpp:60
faiss::Index::idx_t
long idx_t
all indices are this type
Definition: Index.h:64
faiss::ConcatenatedInvertedLists::get_single_code
const uint8_t * get_single_code(size_t list_no, size_t offset) const override
Definition: InvertedLists.cpp:207
faiss::ConcatenatedInvertedLists::get_single_id
idx_t get_single_id(size_t list_no, size_t offset) const override
Definition: InvertedLists.cpp:244
faiss::InvertedLists::prefetch_lists
virtual void prefetch_lists(const long *list_nos, int nlist) const
Definition: InvertedLists.cpp:50
faiss::InvertedLists::nlist
size_t nlist
number of possible key values
Definition: InvertedLists.h:35
faiss::ConcatenatedInvertedLists::list_size
size_t list_size(size_t list_no) const override
get the size of a list
Definition: InvertedLists.cpp:182
faiss::ArrayInvertedLists::ids
std::vector< std::vector< idx_t > > ids
Inverted lists for indexes.
Definition: InvertedLists.h:170
faiss::ConcatenatedInvertedLists::get_codes
const uint8_t * get_codes(size_t list_no) const override
Definition: InvertedLists.cpp:192
faiss::InvertedLists::release_codes
virtual void release_codes(const uint8_t *codes) const
release codes returned by get_codes (default implementation is nop
Definition: InvertedLists.cpp:44
faiss::InvertedLists::get_codes
virtual const uint8_t * get_codes(size_t list_no) const =0
faiss::InvertedLists::ScopedIds
Definition: InvertedLists.h:125
faiss::ArrayInvertedLists::list_size
size_t list_size(size_t list_no) const override
get the size of a list
Definition: InvertedLists.cpp:125
faiss::InvertedLists::release_ids
virtual void release_ids(const idx_t *ids) const
release ids returned by get_ids
Definition: InvertedLists.cpp:47
faiss::InvertedLists::ScopedCodes
Definition: InvertedLists.h:144