Faiss

- Main Page
- Namespaces
- Classes
- Files

- File List

All Classes Namespaces Functions Variables Typedefs Enumerations Enumerator Friends
/data/users/hoss/faiss/AutoTune.cpp
1 /**
2 * Copyright (c) Facebook, Inc. and its affiliates.
3 *
4 * This source code is licensed under the MIT license found in the
5 * LICENSE file in the root directory of this source tree.
6 */
7
8 // -*- c++ -*-
9
10 /*
11 * implementation of Hyper-parameter auto-tuning
12 */
13
14 #include "AutoTune.h"
15
16 #include <cmath>
17 #include <stdarg.h> /* va_list, va_start, va_arg, va_end */
18
19
20 #include "FaissAssert.h"
21 #include "utils.h"
22
23 #include "IndexFlat.h"
24 #include "VectorTransform.h"
25 #include "IndexLSH.h"
26 #include "IndexPQ.h"
27 #include "IndexIVF.h"
28 #include "IndexIVFPQ.h"
29 #include "IndexIVFFlat.h"
30 #include "MetaIndexes.h"
31 #include "IndexScalarQuantizer.h"
32 #include "IndexHNSW.h"
33 #include "IndexBinaryFlat.h"
34 #include "IndexBinaryHNSW.h"
35 #include "IndexBinaryIVF.h"
36
37 namespace faiss {
38
39
40 AutoTuneCriterion::AutoTuneCriterion (idx_t nq, idx_t nnn):
41 nq (nq), nnn (nnn), gt_nnn (0)
42 {}
43
44
45 void AutoTuneCriterion::set_groundtruth (
46 int gt_nnn, const float *gt_D_in, const idx_t *gt_I_in)
47 {
48 this->gt_nnn = gt_nnn;
49 if (gt_D_in) { // allow null for this, as it is often not used
50 gt_D.resize (nq * gt_nnn);
51 memcpy (gt_D.data(), gt_D_in, sizeof (gt_D[0]) * nq * gt_nnn);
52 }
53 gt_I.resize (nq * gt_nnn);
54 memcpy (gt_I.data(), gt_I_in, sizeof (gt_I[0]) * nq * gt_nnn);
55 }
56
57
58
59 OneRecallAtRCriterion::OneRecallAtRCriterion (idx_t nq, idx_t R):
60 AutoTuneCriterion(nq, R), R(R)
61 {}
62
63 double OneRecallAtRCriterion::evaluate(const float* /*D*/, const idx_t* I)
64 const {
65 FAISS_THROW_IF_NOT_MSG(
66 (gt_I.size() == gt_nnn * nq && gt_nnn >= 1 && nnn >= R),
67 "ground truth not initialized");
68 idx_t n_ok = 0;
69 for (idx_t q = 0; q < nq; q++) {
70 idx_t gt_nn = gt_I[q * gt_nnn];
71 const idx_t* I_line = I + q * nnn;
72 for (int i = 0; i < R; i++) {
73 if (I_line[i] == gt_nn) {
74 n_ok++;
75 break;
76 }
77 }
78 }
79 return n_ok / double(nq);
80 }
81
82
83 IntersectionCriterion::IntersectionCriterion (idx_t nq, idx_t R):
84 AutoTuneCriterion(nq, R), R(R)
85 {}
86
87 double IntersectionCriterion::evaluate(const float* /*D*/, const idx_t* I)
88 const {
89 FAISS_THROW_IF_NOT_MSG(
90 (gt_I.size() == gt_nnn * nq && gt_nnn >= R && nnn >= R),
91 "ground truth not initialized");
92 long n_ok = 0;
93 #pragma omp parallel for reduction(+: n_ok)
94 for (idx_t q = 0; q < nq; q++) {
95 n_ok += ranklist_intersection_size (
96 R, &gt_I [q * gt_nnn],
97 R, I + q * nnn);
98 }
99 return n_ok / double (nq * R);
100 }
101
102 /***************************************************************
103 * OperatingPoints
104 ***************************************************************/
105
106 OperatingPoints::OperatingPoints ()
107 {
108 clear();
109 }
110
111 void OperatingPoints::clear ()
112 {
113 all_pts.clear();
114 optimal_pts.clear();
115 /// default point: doing nothing gives 0 performance and takes 0 time
116 OperatingPoint op = {0, 0, "", -1};
117 optimal_pts.push_back(op);
118 }
119
120 /// add a performance measure
121 bool OperatingPoints::add (double perf, double t, const std::string & key,
122 size_t cno)
123 {
124 OperatingPoint op = {perf, t, key, long(cno)};
125 all_pts.push_back (op);
126 if (perf == 0) {
127 return false; // no method for 0 accuracy is faster than doing nothing
128 }
129 std::vector<OperatingPoint> & a = optimal_pts;
130 if (perf > a.back().perf) {
131 // keep unconditionally
132 a.push_back (op);
133 } else if (perf == a.back().perf) {
134 if (t < a.back ().t) {
135 a.back() = op;
136 } else {
137 return false;
138 }
139 } else {
140 int i;
141 // stricto sensu this should be a bissection
142 for (i = 0; i < a.size(); i++) {
143 if (a[i].perf >= perf) break;
144 }
145 assert (i < a.size());
146 if (t < a[i].t) {
147 if (a[i].perf == perf) {
148 a[i] = op;
149 } else {
150 a.insert (a.begin() + i, op);
151 }
152 } else {
153 return false;
154 }
155 }
156 { // remove non-optimal points from array
157 int i = a.size() - 1;
158 while (i > 0) {
159 if (a[i].t < a[i - 1].t)
160 a.erase (a.begin() + (i - 1));
161 i--;
162 }
163 }
164 return true;
165 }
166
167
168 int OperatingPoints::merge_with (const OperatingPoints &other,
169 const std::string & prefix)
170 {
171 int n_add = 0;
172 for (int i = 0; i < other.all_pts.size(); i++) {
173 const OperatingPoint & op = other.all_pts[i];
174 if (add (op.perf, op.t, prefix + op.key, op.cno))
175 n_add++;
176 }
177 return n_add;
178 }
179
180
181
182 /// get time required to obtain a given performance measure
183 double OperatingPoints::t_for_perf (double perf) const
184 {
185 const std::vector<OperatingPoint> & a = optimal_pts;
186 if (perf > a.back().perf) return 1e50;
187 int i0 = -1, i1 = a.size() - 1;
188 while (i0 + 1 < i1) {
189 int imed = (i0 + i1 + 1) / 2;
190 if (a[imed].perf < perf) i0 = imed;
191 else i1 = imed;
192 }
193 return a[i1].t;
194 }
195
196
197 void OperatingPoints::all_to_gnuplot (const char *fname) const
198 {
199 FILE *f = fopen(fname, "w");
200 if (!f) {
201 fprintf (stderr, "cannot open %s", fname);
202 perror("");
203 abort();
204 }
205 for (int i = 0; i < all_pts.size(); i++) {
206 const OperatingPoint & op = all_pts[i];
207 fprintf (f, "%g %g %s\n", op.perf, op.t, op.key.c_str());
208 }
209 fclose(f);
210 }
211
212 void OperatingPoints::optimal_to_gnuplot (const char *fname) const
213 {
214 FILE *f = fopen(fname, "w");
215 if (!f) {
216 fprintf (stderr, "cannot open %s", fname);
217 perror("");
218 abort();
219 }
220 double prev_perf = 0.0;
221 for (int i = 0; i < optimal_pts.size(); i++) {
222 const OperatingPoint & op = optimal_pts[i];
223 fprintf (f, "%g %g\n", prev_perf, op.t);
224 fprintf (f, "%g %g %s\n", op.perf, op.t, op.key.c_str());
225 prev_perf = op.perf;
226 }
227 fclose(f);
228 }
229
230 void OperatingPoints::display (bool only_optimal) const
231 {
232 const std::vector<OperatingPoint> &pts =
233 only_optimal ? optimal_pts : all_pts;
234 printf("Tested %ld operating points, %ld ones are optimal:\n",
235 all_pts.size(), optimal_pts.size());
236
237 for (int i = 0; i < pts.size(); i++) {
238 const OperatingPoint & op = pts[i];
239 const char *star = "";
240 if (!only_optimal) {
241 for (int j = 0; j < optimal_pts.size(); j++) {
242 if (op.cno == optimal_pts[j].cno) {
243 star = "*";
244 break;
245 }
246 }
247 }
248 printf ("cno=%ld key=%s perf=%.4f t=%.3f %s\n",
249 op.cno, op.key.c_str(), op.perf, op.t, star);
250 }
251
252 }
253
254 /***************************************************************
255 * ParameterSpace
256 ***************************************************************/
257
258 ParameterSpace::ParameterSpace ():
259 verbose (1), n_experiments (500),
260 batchsize (1<<30), thread_over_batches (false),
261 min_test_duration (0)
262 {
263 }
264
265 /* not keeping this constructor as inheritors will call the parent
266 initialize()
267 */
268
269 #if 0
270 ParameterSpace::ParameterSpace (Index *index):
271 verbose (1), n_experiments (500),
272 batchsize (1<<30), thread_over_batches (false)
273
274 {
275 initialize(index);
276 }
277 #endif
278
279 size_t ParameterSpace::n_combinations () const
280 {
281 size_t n = 1;
282 for (int i = 0; i < parameter_ranges.size(); i++)
283 n *= parameter_ranges[i].values.size();
284 return n;
285 }
286
287 /// get string representation of the combination
288 std::string ParameterSpace::combination_name (size_t cno) const {
289 char buf[1000], *wp = buf;
290 *wp = 0;
291 for (int i = 0; i < parameter_ranges.size(); i++) {
292 const ParameterRange & pr = parameter_ranges[i];
293 size_t j = cno % pr.values.size();
294 cno /= pr.values.size();
295 wp += snprintf (
296 wp, buf + 1000 - wp, "%s%s=%g", i == 0 ? "" : ",",
297 pr.name.c_str(), pr.values[j]);
298 }
299 return std::string (buf);
300 }
301
302
303 bool ParameterSpace::combination_ge (size_t c1, size_t c2) const
304 {
305 for (int i = 0; i < parameter_ranges.size(); i++) {
306 int nval = parameter_ranges[i].values.size();
307 size_t j1 = c1 % nval;
308 size_t j2 = c2 % nval;
309 if (!(j1 >= j2)) return false;
310 c1 /= nval;
311 c2 /= nval;
312 }
313 return true;
314 }
315
316
317
318 #define DC(classname) \
319 const classname *ix = dynamic_cast<const classname *>(index)
320
321 static void init_pq_ParameterRange (const ProductQuantizer & pq,
322 ParameterRange & pr)
323 {
324 if (pq.code_size % 4 == 0) {
325 // Polysemous not supported for code sizes that are not a
326 // multiple of 4
327 for (int i = 2; i <= pq.code_size * 8 / 2; i+= 2)
328 pr.values.push_back(i);
329 }
330 pr.values.push_back (pq.code_size * 8);
331 }
332
333 ParameterRange &ParameterSpace::add_range(const char * name)
334 {
335 for (auto & pr : parameter_ranges) {
336 if (pr.name == name) {
337 return pr;
338 }
339 }
340 parameter_ranges.push_back (ParameterRange ());
341 parameter_ranges.back ().name = name;
342 return parameter_ranges.back ();
343 }
344
345
346 /// initialize with reasonable parameters for the index
347 void ParameterSpace::initialize (const Index * index)
348 {
349 if (DC (IndexPreTransform)) {
350 index = ix->index;
351 }
352 if (DC (IndexRefineFlat)) {
353 ParameterRange & pr = add_range("k_factor_rf");
354 for (int i = 0; i <= 6; i++) {
355 pr.values.push_back (1 << i);
356 }
357 index = ix->base_index;
358 }
359 if (DC (IndexPreTransform)) {
360 index = ix->index;
361 }
362
363 if (DC (IndexIVF)) {
364 {
365 ParameterRange & pr = add_range("nprobe");
366 for (int i = 0; i < 13; i++) {
367 size_t nprobe = 1 << i;
368 if (nprobe >= ix->nlist) break;
369 pr.values.push_back (nprobe);
370 }
371 }
372 if (dynamic_cast<const IndexHNSW*>(ix->quantizer)) {
373 ParameterRange & pr = add_range("efSearch");
374 for (int i = 2; i <= 9; i++) {
375 pr.values.push_back (1 << i);
376 }
377 }
378 }
379 if (DC (IndexPQ)) {
380 ParameterRange & pr = add_range("ht");
381 init_pq_ParameterRange (ix->pq, pr);
382 }
383 if (DC (IndexIVFPQ)) {
384 ParameterRange & pr = add_range("ht");
385 init_pq_ParameterRange (ix->pq, pr);
386 }
387
388 if (DC (IndexIVF)) {
389 const MultiIndexQuantizer *miq =
390 dynamic_cast<const MultiIndexQuantizer *> (ix->quantizer);
391 if (miq) {
392 ParameterRange & pr_max_codes = add_range("max_codes");
393 for (int i = 8; i < 20; i++) {
394 pr_max_codes.values.push_back (1 << i);
395 }
396 pr_max_codes.values.push_back (
397 std::numeric_limits<double>::infinity()
398 );
399 }
400 }
401 if (DC (IndexIVFPQR)) {
402 ParameterRange & pr = add_range("k_factor");
403 for (int i = 0; i <= 6; i++) {
404 pr.values.push_back (1 << i);
405 }
406 }
407 if (dynamic_cast<const IndexHNSW*>(index)) {
408 ParameterRange & pr = add_range("efSearch");
409 for (int i = 2; i <= 9; i++) {
410 pr.values.push_back (1 << i);
411 }
412 }
413 }
414
415 #undef DC
416
417 // non-const version
418 #define DC(classname) classname *ix = dynamic_cast<classname *>(index)
419
420
421 /// set a combination of parameters on an index
422 void ParameterSpace::set_index_parameters (Index *index, size_t cno) const
423 {
424
425 for (int i = 0; i < parameter_ranges.size(); i++) {
426 const ParameterRange & pr = parameter_ranges[i];
427 size_t j = cno % pr.values.size();
428 cno /= pr.values.size();
429 double val = pr.values [j];
430 set_index_parameter (index, pr.name, val);
431 }
432 }
433
434 /// set a combination of parameters on an index
435 void ParameterSpace::set_index_parameters (
436 Index *index, const char *description_in) const
437 {
438 char description[strlen(description_in) + 1];
439 char *ptr;
440 memcpy (description, description_in, strlen(description_in) + 1);
441
442 for (char *tok = strtok_r (description, " ,", &ptr);
443 tok;
444 tok = strtok_r (nullptr, " ,", &ptr)) {
445 char name[100];
446 double val;
447 int ret = sscanf (tok, "%100[^=]=%lf", name, &val);
448 FAISS_THROW_IF_NOT_FMT (
449 ret == 2, "could not interpret parameters %s", tok);
450 set_index_parameter (index, name, val);
451 }
452
453 }
454
455 void ParameterSpace::set_index_parameter (
456 Index * index, const std::string & name, double val) const
457 {
458 if (verbose > 1)
459 printf(" set %s=%g\n", name.c_str(), val);
460
461 if (name == "verbose") {
462 index->verbose = int(val);
463 // and fall through to also enable it on sub-indexes
464 }
465 if (DC (IndexPreTransform)) {
466 set_index_parameter (ix->index, name, val);
467 return;
468 }
469 if (DC (IndexShards)) {
470 // call on all sub-indexes
471 auto fn =
472 [this, name, val](int, Index* subIndex) {
473 set_index_parameter(subIndex, name, val);
474 };
475
476 ix->runOnIndex(fn);
477 return;
478 }
479 if (DC (IndexReplicas)) {
480 // call on all sub-indexes
481 auto fn =
482 [this, name, val](int, Index* subIndex) {
483 set_index_parameter(subIndex, name, val);
484 };
485
486 ix->runOnIndex(fn);
487 return;
488 }
489 if (DC (IndexRefineFlat)) {
490 if (name == "k_factor_rf") {
491 ix->k_factor = int(val);
492 return;
493 }
494 // otherwise it is for the sub-index
495 set_index_parameter (&ix->refine_index, name, val);
496 return;
497 }
498
499 if (name == "verbose") {
500 index->verbose = int(val);
501 return; // last verbose that we could find
502 }
503
504 if (name == "nprobe") {
505 if (DC (IndexIDMap)) {
506 set_index_parameter (ix->index, name, val);
507 return;
508 } else if (DC (IndexIVF)) {
509 ix->nprobe = int(val);
510 return;
511 }
512 }
513
514 if (name == "ht") {
515 if (DC (IndexPQ)) {
516 if (val >= ix->pq.code_size * 8) {
517 ix->search_type = IndexPQ::ST_PQ;
518 } else {
519 ix->search_type = IndexPQ::ST_polysemous;
520 ix->polysemous_ht = int(val);
521 }
522 return;
523 } else if (DC (IndexIVFPQ)) {
524 if (val >= ix->pq.code_size * 8) {
525 ix->polysemous_ht = 0;
526 } else {
527 ix->polysemous_ht = int(val);
528 }
529 return;
530 }
531 }
532
533 if (name == "k_factor") {
534 if (DC (IndexIVFPQR)) {
535 ix->k_factor = val;
536 return;
537 }
538 }
539 if (name == "max_codes") {
540 if (DC (IndexIVF)) {
541 ix->max_codes = std::isfinite(val) ? size_t(val) : 0;
542 return;
543 }
544 }
545
546 if (name == "efSearch") {
547 if (DC (IndexHNSW)) {
548 ix->hnsw.efSearch = int(val);
549 return;
550 }
551 if (DC (IndexIVF)) {
552 if (IndexHNSW *cq =
553 dynamic_cast<IndexHNSW *>(ix->quantizer)) {
554 cq->hnsw.efSearch = int(val);
555 return;
556 }
557 }
558 }
559
560 FAISS_THROW_FMT ("ParameterSpace::set_index_parameter:"
561 "could not set parameter %s",
562 name.c_str());
563 }
564
565 void ParameterSpace::display () const
566 {
567 printf ("ParameterSpace, %ld parameters, %ld combinations:\n",
568 parameter_ranges.size (), n_combinations ());
569 for (int i = 0; i < parameter_ranges.size(); i++) {
570 const ParameterRange & pr = parameter_ranges[i];
571 printf (" %s: ", pr.name.c_str ());
572 char sep = '[';
573 for (int j = 0; j < pr.values.size(); j++) {
574 printf ("%c %g", sep, pr.values [j]);
575 sep = ',';
576 }
577 printf ("]\n");
578 }
579 }
580
581
582
583 void ParameterSpace::update_bounds (size_t cno, const OperatingPoint & op,
584 double *upper_bound_perf,
585 double *lower_bound_t) const
586 {
587 if (combination_ge (cno, op.cno)) {
588 if (op.t > *lower_bound_t) *lower_bound_t = op.t;
589 }
590 if (combination_ge (op.cno, cno)) {
591 if (op.perf < *upper_bound_perf) *upper_bound_perf = op.perf;
592 }
593 }
594
595
596
597 void ParameterSpace::explore (Index *index,
598 size_t nq, const float *xq,
599 const AutoTuneCriterion & crit,
600 OperatingPoints * ops) const
601 {
602 FAISS_THROW_IF_NOT_MSG (nq == crit.nq,
603 "criterion does not have the same nb of queries");
604
605 size_t n_comb = n_combinations ();
606
607 if (n_experiments == 0) {
608
609 for (size_t cno = 0; cno < n_comb; cno++) {
610 set_index_parameters (index, cno);
611 std::vector<Index::idx_t> I(nq * crit.nnn);
612 std::vector<float> D(nq * crit.nnn);
613
614 double t0 = getmillisecs ();
615 index->search (nq, xq, crit.nnn, D.data(), I.data());
616 double t_search = (getmillisecs() - t0) / 1e3;
617
618 double perf = crit.evaluate (D.data(), I.data());
619
620 bool keep = ops->add (perf, t_search, combination_name (cno), cno);
621
622 if (verbose)
623 printf(" %ld/%ld: %s perf=%.3f t=%.3f s %s\n", cno, n_comb,
624 combination_name (cno).c_str(), perf, t_search,
625 keep ? "*" : "");
626 }
627 return;
628 }
629
630 int n_exp = n_experiments;
631
632 if (n_exp > n_comb) n_exp = n_comb;
633 FAISS_THROW_IF_NOT (n_comb == 1 || n_exp > 2);
634 std::vector<int> perm (n_comb);
635 // make sure the slowest and fastest experiment are run
636 perm[0] = 0;
637 if (n_comb > 1) {
638 perm[1] = n_comb - 1;
639 rand_perm (&perm[2], n_comb - 2, 1234);
640 for (int i = 2; i < perm.size(); i++) perm[i] ++;
641 }
642
643 for (size_t xp = 0; xp < n_exp; xp++) {
644 size_t cno = perm[xp];
645
646 if (verbose)
647 printf(" %ld/%d: cno=%ld %s ", xp, n_exp, cno,
648 combination_name (cno).c_str());
649
650 {
651 double lower_bound_t = 0.0;
652 double upper_bound_perf = 1.0;
653 for (int i = 0; i < ops->all_pts.size(); i++) {
654 update_bounds (cno, ops->all_pts[i],
655 &upper_bound_perf, &lower_bound_t);
656 }
657 double best_t = ops->t_for_perf (upper_bound_perf);
658 if (verbose)
659 printf ("bounds [perf<=%.3f t>=%.3f] %s",
660 upper_bound_perf, lower_bound_t,
661 best_t <= lower_bound_t ? "skip\n" : "");
662 if (best_t <= lower_bound_t) continue;
663 }
664
665 set_index_parameters (index, cno);
666 std::vector<Index::idx_t> I(nq * crit.nnn);
667 std::vector<float> D(nq * crit.nnn);
668
669 double t0 = getmillisecs ();
670
671 int nrun = 0;
672 double t_search;
673
674 do {
675
676 if (thread_over_batches) {
677 #pragma omp parallel for
678 for (size_t q0 = 0; q0 < nq; q0 += batchsize) {
679 size_t q1 = q0 + batchsize;
680 if (q1 > nq) q1 = nq;
681 index->search (q1 - q0, xq + q0 * index->d,
682 crit.nnn,
683 D.data() + q0 * crit.nnn,
684 I.data() + q0 * crit.nnn);
685 }
686 } else {
687 for (size_t q0 = 0; q0 < nq; q0 += batchsize) {
688 size_t q1 = q0 + batchsize;
689 if (q1 > nq) q1 = nq;
690 index->search (q1 - q0, xq + q0 * index->d,
691 crit.nnn,
692 D.data() + q0 * crit.nnn,
693 I.data() + q0 * crit.nnn);
694 }
695 }
696 nrun ++;
697 t_search = (getmillisecs() - t0) / 1e3;
698
699 } while (t_search < min_test_duration);
700
701 t_search /= nrun;
702
703 double perf = crit.evaluate (D.data(), I.data());
704
705 bool keep = ops->add (perf, t_search, combination_name (cno), cno);
706
707 if (verbose)
708 printf(" perf %.3f t %.3f (%d runs) %s\n",
709 perf, t_search, nrun,
710 keep ? "*" : "");
711 }
712 }
713
714 /***************************************************************
715 * index_factory
716 ***************************************************************/
717
718 namespace {
719
720 struct VTChain {
721 std::vector<VectorTransform *> chain;
722 ~VTChain () {
723 for (int i = 0; i < chain.size(); i++) {
724 delete chain[i];
725 }
726 }
727 };
728
729
730 /// what kind of training does this coarse quantizer require?
731 char get_trains_alone(const Index *coarse_quantizer) {
732 return
733 dynamic_cast<const MultiIndexQuantizer*>(coarse_quantizer) ? 1 :
734 dynamic_cast<const IndexHNSWFlat*>(coarse_quantizer) ? 2 :
735 0;
736 }
737
738
739 }
740
741 Index *index_factory (int d, const char *description_in, MetricType metric)
742 {
743 VTChain vts;
744 Index *coarse_quantizer = nullptr;
745 Index *index = nullptr;
746 bool add_idmap = false;
747 bool make_IndexRefineFlat = false;
748
749 ScopeDeleter1<Index> del_coarse_quantizer, del_index;
750
751 char description[strlen(description_in) + 1];
752 char *ptr;
753 memcpy (description, description_in, strlen(description_in) + 1);
754
755 int ncentroids = -1;
756
757 for (char *tok = strtok_r (description, " ,", &ptr);
758 tok;
759 tok = strtok_r (nullptr, " ,", &ptr)) {
760 int d_out, opq_M, nbit, M, M2, pq_m, ncent;
761 std::string stok(tok);
762
763 // to avoid mem leaks with exceptions:
764 // do all tests before any instanciation
765
766 VectorTransform *vt_1 = nullptr;
767 Index *coarse_quantizer_1 = nullptr;
768 Index *index_1 = nullptr;
769
770 // VectorTransforms
771 if (sscanf (tok, "PCA%d", &d_out) == 1) {
772 vt_1 = new PCAMatrix (d, d_out);
773 d = d_out;
774 } else if (sscanf (tok, "PCAR%d", &d_out) == 1) {
775 vt_1 = new PCAMatrix (d, d_out, 0, true);
776 d = d_out;
777 } else if (sscanf (tok, "RR%d", &d_out) == 1) {
778 vt_1 = new RandomRotationMatrix (d, d_out);
779 d = d_out;
780 } else if (sscanf (tok, "PCAW%d", &d_out) == 1) {
781 vt_1 = new PCAMatrix (d, d_out, -0.5, false);
782 d = d_out;
783 } else if (sscanf (tok, "PCAWR%d", &d_out) == 1) {
784 vt_1 = new PCAMatrix (d, d_out, -0.5, true);
785 d = d_out;
786 } else if (sscanf (tok, "OPQ%d_%d", &opq_M, &d_out) == 2) {
787 vt_1 = new OPQMatrix (d, opq_M, d_out);
788 d = d_out;
789 } else if (sscanf (tok, "OPQ%d", &opq_M) == 1) {
790 vt_1 = new OPQMatrix (d, opq_M);
791 } else if (stok == "L2norm") {
792 vt_1 = new NormalizationTransform (d, 2.0);
793
794 // coarse quantizers
795 } else if (!coarse_quantizer &&
796 sscanf (tok, "IVF%d_HNSW%d", &ncentroids, &M) == 2) {
797 FAISS_THROW_IF_NOT (metric == METRIC_L2);
798 coarse_quantizer_1 = new IndexHNSWFlat (d, M);
799
800 } else if (!coarse_quantizer &&
801 sscanf (tok, "IVF%d", &ncentroids) == 1) {
802 if (metric == METRIC_L2) {
803 coarse_quantizer_1 = new IndexFlatL2 (d);
804 } else { // if (metric == METRIC_IP)
805 coarse_quantizer_1 = new IndexFlatIP (d);
806 }
807 } else if (!coarse_quantizer && sscanf (tok, "IMI2x%d", &nbit) == 1) {
808 FAISS_THROW_IF_NOT_MSG (metric == METRIC_L2,
809 "MultiIndex not implemented for inner prod search");
810 coarse_quantizer_1 = new MultiIndexQuantizer (d, 2, nbit);
811 ncentroids = 1 << (2 * nbit);
812 } else if (stok == "IDMap") {
813 add_idmap = true;
814
815 // IVFs
816 } else if (!index && (stok == "Flat" || stok == "FlatDedup")) {
817 if (coarse_quantizer) {
818 // if there was an IVF in front, then it is an IVFFlat
819 IndexIVF *index_ivf = stok == "Flat" ?
820 new IndexIVFFlat (
821 coarse_quantizer, d, ncentroids, metric) :
822 new IndexIVFFlatDedup (
823 coarse_quantizer, d, ncentroids, metric);
824 index_ivf->quantizer_trains_alone =
825 get_trains_alone (coarse_quantizer);
826 index_ivf->cp.spherical = metric == METRIC_INNER_PRODUCT;
827 del_coarse_quantizer.release ();
828 index_ivf->own_fields = true;
829 index_1 = index_ivf;
830 } else {
831 FAISS_THROW_IF_NOT_MSG (stok != "FlatDedup",
832 "dedup supported only for IVFFlat");
833 index_1 = new IndexFlat (d, metric);
834 }
835 } else if (!index && (stok == "SQ8" || stok == "SQ4" ||
836 stok == "SQfp16")) {
837 ScalarQuantizer::QuantizerType qt =
838 stok == "SQ8" ? ScalarQuantizer::QT_8bit :
839 stok == "SQ4" ? ScalarQuantizer::QT_4bit :
840 stok == "SQfp16" ? ScalarQuantizer::QT_fp16 :
841 ScalarQuantizer::QT_4bit;
842 if (coarse_quantizer) {
843 IndexIVFScalarQuantizer *index_ivf =
844 new IndexIVFScalarQuantizer (
845 coarse_quantizer, d, ncentroids, qt, metric);
846 index_ivf->quantizer_trains_alone =
847 get_trains_alone (coarse_quantizer);
848 del_coarse_quantizer.release ();
849 index_ivf->own_fields = true;
850 index_1 = index_ivf;
851 } else {
852 index_1 = new IndexScalarQuantizer (d, qt, metric);
853 }
854 } else if (!index && sscanf (tok, "PQ%d+%d", &M, &M2) == 2) {
855 FAISS_THROW_IF_NOT_MSG(coarse_quantizer,
856 "PQ with + works only with an IVF");
857 FAISS_THROW_IF_NOT_MSG(metric == METRIC_L2,
858 "IVFPQR not implemented for inner product search");
859 IndexIVFPQR *index_ivf = new IndexIVFPQR (
860 coarse_quantizer, d, ncentroids, M, 8, M2, 8);
861 index_ivf->quantizer_trains_alone =
862 get_trains_alone (coarse_quantizer);
863 del_coarse_quantizer.release ();
864 index_ivf->own_fields = true;
865 index_1 = index_ivf;
866 } else if (!index && (sscanf (tok, "PQ%d", &M) == 1 ||
867 sscanf (tok, "PQ%dnp", &M) == 1)) {
868 bool do_polysemous_training = stok.find("np") == std::string::npos;
869 if (coarse_quantizer) {
870 IndexIVFPQ *index_ivf = new IndexIVFPQ (
871 coarse_quantizer, d, ncentroids, M, 8);
872 index_ivf->quantizer_trains_alone =
873 get_trains_alone (coarse_quantizer);
874 index_ivf->metric_type = metric;
875 index_ivf->cp.spherical = metric == METRIC_INNER_PRODUCT;
876 del_coarse_quantizer.release ();
877 index_ivf->own_fields = true;
878 index_ivf->do_polysemous_training = do_polysemous_training;
879 index_1 = index_ivf;
880 } else {
881 IndexPQ *index_pq = new IndexPQ (d, M, 8, metric);
882 index_pq->do_polysemous_training = do_polysemous_training;
883 index_1 = index_pq;
884 }
885 } else if (!index &&
886 sscanf (tok, "HNSW%d_%d+PQ%d", &M, &ncent, &pq_m) == 3) {
887 Index * quant = new IndexFlatL2 (d);
888 IndexHNSW2Level * hidx2l = new IndexHNSW2Level (quant, ncent, pq_m, M);
889 Index2Layer * idx2l = dynamic_cast<Index2Layer*>(hidx2l->storage);
890 idx2l->q1.own_fields = true;
891 index_1 = hidx2l;
892 } else if (!index &&
893 sscanf (tok, "HNSW%d_2x%d+PQ%d", &M, &nbit, &pq_m) == 3) {
894 Index * quant = new MultiIndexQuantizer (d, 2, nbit);
895 IndexHNSW2Level * hidx2l =
896 new IndexHNSW2Level (quant, 1 << (2 * nbit), pq_m, M);
897 Index2Layer * idx2l = dynamic_cast<Index2Layer*>(hidx2l->storage);
898 idx2l->q1.own_fields = true;
899 idx2l->q1.quantizer_trains_alone = 1;
900 index_1 = hidx2l;
901 } else if (!index &&
902 sscanf (tok, "HNSW%d_PQ%d", &M, &pq_m) == 2) {
903 index_1 = new IndexHNSWPQ (d, pq_m, M);
904 } else if (!index &&
905 sscanf (tok, "HNSW%d", &M) == 1) {
906 index_1 = new IndexHNSWFlat (d, M);
907 } else if (!index &&
908 sscanf (tok, "HNSW%d_SQ%d", &M, &pq_m) == 2 &&
909 pq_m == 8) {
910 index_1 = new IndexHNSWSQ (d, ScalarQuantizer::QT_8bit, M);
911 } else if (stok == "RFlat") {
912 make_IndexRefineFlat = true;
913 } else {
914 FAISS_THROW_FMT( "could not parse token \"%s\" in %s\n",
915 tok, description_in);
916 }
917
918 if (index_1 && add_idmap) {
919 IndexIDMap *idmap = new IndexIDMap(index_1);
920 del_index.set (idmap);
921 idmap->own_fields = true;
922 index_1 = idmap;
923 add_idmap = false;
924 }
925
926 if (vt_1) {
927 vts.chain.push_back (vt_1);
928 }
929
930 if (coarse_quantizer_1) {
931 coarse_quantizer = coarse_quantizer_1;
932 del_coarse_quantizer.set (coarse_quantizer);
933 }
934
935 if (index_1) {
936 index = index_1;
937 del_index.set (index);
938 }
939 }
940
941 FAISS_THROW_IF_NOT_FMT(index, "descrption %s did not generate an index",
942 description_in);
943
944 // nothing can go wrong now
945 del_index.release ();
946 del_coarse_quantizer.release ();
947
948 if (add_idmap) {
949 fprintf(stderr, "index_factory: WARNING: "
950 "IDMap option not used\n");
951 }
952
953 if (vts.chain.size() > 0) {
954 IndexPreTransform *index_pt = new IndexPreTransform (index);
955 index_pt->own_fields = true;
956 // add from back
957 while (vts.chain.size() > 0) {
958 index_pt->prepend_transform (vts.chain.back ());
959 vts.chain.pop_back ();
960 }
961 index = index_pt;
962 }
963
964 if (make_IndexRefineFlat) {
965 IndexRefineFlat *index_rf = new IndexRefineFlat (index);
966 index_rf->own_fields = true;
967 index = index_rf;
968 }
969
970 return index;
971 }
972
973 IndexBinary *index_binary_factory(int d, const char *description)
974 {
975 IndexBinary *index = nullptr;
976
977 int ncentroids = -1;
978 int M;
979
980 if (sscanf(description, "BIVF%d_HNSW%d", &ncentroids, &M) == 2) {
981 IndexBinaryIVF *index_ivf = new IndexBinaryIVF(
982 new IndexBinaryHNSW(d, M), d, ncentroids
983 );
984 index_ivf->own_fields = true;
985 index = index_ivf;
986
987 } else if (sscanf(description, "BIVF%d", &ncentroids) == 1) {
988 IndexBinaryIVF *index_ivf = new IndexBinaryIVF(
989 new IndexBinaryFlat(d), d, ncentroids
990 );
991 index_ivf->own_fields = true;
992 index = index_ivf;
993
994 } else if (sscanf(description, "BHNSW%d", &M) == 1) {
995 IndexBinaryHNSW *index_hnsw = new IndexBinaryHNSW(d, M);
996 index = index_hnsw;
997
998 } else if (std::string(description) == "BFlat") {
999 index = new IndexBinaryFlat(d);
1000
1001 } else {
1002 FAISS_THROW_IF_NOT_FMT(index, "descrption %s did not generate an index",
1003 description);
1004 }
1005
1006 return index;
1007 }
1008
1009 /*********************************************************************
1010 * MatrixStats
1011 *********************************************************************/
1012
1013 MatrixStats::PerDimStats::PerDimStats():
1014 n(0), n_nan(0), n_inf(0), n0(0),
1015 min(HUGE_VALF), max(-HUGE_VALF),
1016 sum(0), sum2(0),
1017 mean(NAN), stddev(NAN)
1018 {}
1019
1020
1021 void MatrixStats::PerDimStats::add (float x)
1022 {
1023 n++;
1024 if (std::isnan(x)) {
1025 n_nan++;
1026 return;
1027 }
1028 if (!std::isfinite(x)) {
1029 n_inf++;
1030 return;
1031 }
1032 if (x == 0) n0++;
1033 if (x < min) min = x;
1034 if (x > max) max = x;
1035 sum += x;
1036 sum2 += (double)x * (double)x;
1037 }
1038
1039 void MatrixStats::PerDimStats::compute_mean_std ()
1040 {
1041 n_valid = n - n_nan - n_inf;
1042 mean = sum / n_valid;
1043 double var = sum2 / n_valid - mean * mean;
1044 if (var < 0) var = 0;
1045 stddev = sqrt(var);
1046 }
1047
1048
1049 void MatrixStats::do_comment (const char *fmt, ...)
1050 {
1051 va_list ap;
1052
1053 /* Determine required size */
1054 va_start(ap, fmt);
1055 size_t size = vsnprintf(buf, nbuf, fmt, ap);
1056 va_end(ap);
1057
1058 nbuf -= size;
1059 buf += size;
1060 }
1061
1062
1063
1064 MatrixStats::MatrixStats (size_t n, size_t d, const float *x):
1065 n(n), d(d),
1066 n_collision(0), n_valid(0), n0(0),
1067 min_norm2(HUGE_VAL), max_norm2(0)
1068 {
1069 std::vector<char> comment_buf (10000);
1070 buf = comment_buf.data ();
1071 nbuf = comment_buf.size();
1072
1073 do_comment ("analyzing %ld vectors of size %ld\n", n, d);
1074
1075 if (d > 1024) {
1076 do_comment (
1077 "indexing this many dimensions is hard, "
1078 "please consider dimensionality reducution (with PCAMatrix)\n");
1079 }
1080
1081 size_t nbytes = sizeof (x[0]) * d;
1082 per_dim_stats.resize (d);
1083
1084 for (size_t i = 0; i < n; i++) {
1085 const float *xi = x + d * i;
1086 double sum2 = 0;
1087 for (size_t j = 0; j < d; j++) {
1088 per_dim_stats[j].add (xi[j]);
1089 sum2 += xi[j] * (double)xi[j];
1090 }
1091
1092 if (std::isfinite (sum2)) {
1093 n_valid++;
1094 if (sum2 == 0) {
1095 n0 ++;
1096 } else {
1097 if (sum2 < min_norm2) min_norm2 = sum2;
1098 if (sum2 > max_norm2) max_norm2 = sum2;
1099 }
1100 }
1101
1102 { // check hash
1103 uint64_t hash = hash_bytes((const uint8_t*)xi, nbytes);
1104 auto elt = occurrences.find (hash);
1105 if (elt == occurrences.end()) {
1106 Occurrence occ = {i, 1};
1107 occurrences[hash] = occ;
1108 } else {
1109 if (!memcmp (xi, x + elt->second.first * d, nbytes)) {
1110 elt->second.count ++;
1111 } else {
1112 n_collision ++;
1113 // we should use a list of collisions but overkill
1114 }
1115 }
1116 }
1117 }
1118
1119 // invalid vecor stats
1120 if (n_valid == n) {
1121 do_comment ("no NaN or Infs in data\n");
1122 } else {
1123 do_comment ("%ld vectors contain NaN or Inf "
1124 "(or have too large components), "
1125 "expect bad results with indexing!\n", n - n_valid);
1126 }
1127
1128 // copies in dataset
1129 if (occurrences.size() == n) {
1130 do_comment ("all vectors are distinct\n");
1131 } else {
1132 do_comment ("%ld vectors are distinct (%.2f%%)\n",
1133 occurrences.size(),
1134 occurrences.size() * 100.0 / n);
1135
1136 if (n_collision > 0) {
1137 do_comment ("%ld collisions in hash table, "
1138 "counts may be invalid\n", n_collision);
1139 }
1140
1141 Occurrence max = {0, 0};
1142 for (auto it = occurrences.begin();
1143 it != occurrences.end(); ++it) {
1144 if (it->second.count > max.count) {
1145 max = it->second;
1146 }
1147 }
1148 do_comment ("vector %ld has %ld copies\n", max.first, max.count);
1149 }
1150
1151 { // norm stats
1152 min_norm2 = sqrt (min_norm2);
1153 max_norm2 = sqrt (max_norm2);
1154 do_comment ("range of L2 norms=[%g, %g] (%ld null vectors)\n",
1155 min_norm2, max_norm2, n0);
1156
1157 if (max_norm2 < min_norm2 * 1.0001) {
1158 do_comment ("vectors are normalized, inner product and "
1159 "L2 search are equivalent\n");
1160 }
1161
1162 if (max_norm2 > min_norm2 * 100) {
1163 do_comment ("vectors have very large differences in norms, "
1164 "is this normal?\n");
1165 }
1166 }
1167
1168 { // per dimension stats
1169
1170 double max_std = 0, min_std = HUGE_VAL;
1171
1172 size_t n_dangerous_range = 0, n_0_range = 0, n0 = 0;
1173
1174 for (size_t j = 0; j < d; j++) {
1175 PerDimStats &st = per_dim_stats[j];
1176 st.compute_mean_std ();
1177 n0 += st.n0;
1178
1179 if (st.max == st.min) {
1180 n_0_range ++;
1181 } else if (st.max < 1.001 * st.min) {
1182 n_dangerous_range ++;
1183 }
1184
1185 if (st.stddev > max_std) max_std = st.stddev;
1186 if (st.stddev < min_std) min_std = st.stddev;
1187 }
1188
1189
1190
1191 if (n0 == 0) {
1192 do_comment ("matrix contains no 0s\n");
1193 } else {
1194 do_comment ("matrix contains %.2f %% 0 entries\n",
1195 n0 * 100.0 / (n * d));
1196 }
1197
1198 if (n_0_range == 0) {
1199 do_comment ("no constant dimensions\n");
1200 } else {
1201 do_comment ("%ld dimensions are constant: they can be removed\n",
1202 n_0_range);
1203 }
1204
1205 if (n_dangerous_range == 0) {
1206 do_comment ("no dimension has a too large mean\n");
1207 } else {
1208 do_comment ("%ld dimensions are too large "
1209 "wrt. their variance, may loose precision "
1210 "in IndexFlatL2 (use CenteringTransform)\n",
1211 n_dangerous_range);
1212 }
1213
1214 do_comment ("stddevs per dimension are in [%g %g]\n", min_std, max_std);
1215
1216 size_t n_small_var = 0;
1217
1218 for (size_t j = 0; j < d; j++) {
1219 const PerDimStats &st = per_dim_stats[j];
1220 if (st.stddev < max_std * 1e-4) {
1221 n_small_var++;
1222 }
1223 }
1224
1225 if (n_small_var > 0) {
1226 do_comment ("%ld dimensions have negligible stddev wrt. "
1227 "the largest dimension, they could be ignored",
1228 n_small_var);
1229 }
1230
1231 }
1232 comments = comment_buf.data ();
1233 buf = nullptr;
1234 nbuf = 0;
1235 }
1236
1237
1238
1239
1240 } // namespace faiss
faiss::ParameterSpace::explore
void explore(Index *index, size_t nq, const float *xq, const AutoTuneCriterion &crit, OperatingPoints *ops) const
Definition: AutoTune.cpp:597
faiss::IndexIVF
Definition: IndexIVF.h:90
faiss::ParameterSpace::parameter_ranges
std::vector< ParameterRange > parameter_ranges
all tunable parameters
Definition: AutoTune.h:134
faiss::OperatingPoint::key
std::string key
key that identifies this op pt
Definition: AutoTune.h:89
faiss::IndexHNSWSQ
Definition: IndexHNSW.h:158
faiss::ScalarQuantizer::QuantizerType
QuantizerType
Definition: IndexScalarQuantizer.h:34
faiss::IndexFlat
Definition: IndexFlat.h:21
faiss::RandomRotationMatrix
Randomly rotate a set of vectors.
Definition: VectorTransform.h:115
faiss::OperatingPoint::cno
long cno
integer identifer
Definition: AutoTune.h:90
faiss::IndexIVFPQR
Definition: IndexIVFPQ.h:155
faiss::IndexPQ::do_polysemous_training
bool do_polysemous_training
false = standard PQ
Definition: IndexPQ.h:69
faiss::OperatingPoints::clear
void clear()
Definition: AutoTune.cpp:111
faiss::ParameterSpace::min_test_duration
double min_test_duration
Definition: AutoTune.h:153
faiss::OneRecallAtRCriterion::evaluate
double evaluate(const float *D, const idx_t *I) const override
Definition: AutoTune.cpp:63
faiss::OperatingPoint
Definition: AutoTune.h:86
faiss::IndexScalarQuantizer
Definition: IndexScalarQuantizer.h:92
faiss::OperatingPoints::display
void display(bool only_optimal=true) const
easy-to-read output
Definition: AutoTune.cpp:230
faiss::IndexPQ
Definition: IndexPQ.h:26
faiss::OperatingPoint::perf
double perf
performance measure (output of a Criterion)
Definition: AutoTune.h:87
faiss::ScalarQuantizer::QT_4bit
4 bits per component
Definition: IndexScalarQuantizer.h:36
faiss::OperatingPoints
Definition: AutoTune.h:93
faiss::OperatingPoints::t_for_perf
double t_for_perf(double perf) const
get time required to obtain a given performance measure
Definition: AutoTune.cpp:183
faiss::AutoTuneCriterion::nnn
idx_t nnn
nb of NNs that the query should request
Definition: AutoTune.h:29
faiss::PCAMatrix
Definition: VectorTransform.h:133
faiss::IndexFlatL2
Definition: IndexFlat.h:77
faiss::IntersectionCriterion::evaluate
double evaluate(const float *D, const idx_t *I) const override
Definition: AutoTune.cpp:87
faiss::OperatingPoints::add
bool add(double perf, double t, const std::string &key, size_t cno=0)
add a performance measure. Return whether it is an optimal point
Definition: AutoTune.cpp:121
faiss::OPQMatrix
Definition: VectorTransform.h:188
faiss::ScopeDeleter1
Definition: FaissException.h:58
faiss::IndexIVFPQ::do_polysemous_training
bool do_polysemous_training
reorder PQ centroids after training?
Definition: IndexIVFPQ.h:39
faiss::ParameterSpace::batchsize
size_t batchsize
maximum number of queries to submit at a time.
Definition: AutoTune.h:145
faiss::Index2Layer::q1
Level1Quantizer q1
first level quantizer
Definition: IndexIVFPQ.h:206
faiss::AutoTuneCriterion::evaluate
virtual double evaluate(const float *D, const idx_t *I) const =0
faiss::IndexPreTransform
Definition: VectorTransform.h:271
faiss::IndexIDMap
Definition: MetaIndexes.h:22
faiss::AutoTuneCriterion::nq
idx_t nq
nb of queries this criterion is evaluated on
Definition: AutoTune.h:28
faiss::IndexIVFFlat
Definition: IndexIVFFlat.h:24
faiss::OperatingPoints::optimal_pts
std::vector< OperatingPoint > optimal_pts
optimal operating points, sorted by perf
Definition: AutoTune.h:98
faiss::IndexFlatIP
Definition: IndexFlat.h:71
faiss::AutoTuneCriterion::set_groundtruth
void set_groundtruth(int gt_nnn, const float *gt_D_in, const idx_t *gt_I_in)
Definition: AutoTune.cpp:45
faiss::ParameterSpace::add_range
ParameterRange & add_range(const char *name)
add a new parameter (or return it if it exists)
Definition: AutoTune.cpp:333
faiss::IndexRefineFlat
Definition: IndexFlat.h:101
faiss::Index2Layer
Definition: IndexIVFPQ.h:204
faiss::AutoTuneCriterion::gt_nnn
idx_t gt_nnn
nb of GT NNs required to evaluate crterion
Definition: AutoTune.h:30
faiss::OperatingPoints::all_to_gnuplot
void all_to_gnuplot(const char *fname) const
output to a format easy to digest by gnuplot
Definition: AutoTune.cpp:197
faiss::IndexRefineFlat::own_fields
bool own_fields
should the base index be deallocated?
Definition: IndexFlat.h:108
faiss::IndexHNSW2Level
Definition: IndexHNSW.h:167
faiss::Index::d
int d
vector dimension
Definition: Index.h:66
faiss::ProductQuantizer::code_size
size_t code_size
byte per indexed vector
Definition: ProductQuantizer.h:33
faiss::IndexIVFPQ
Definition: IndexIVFPQ.h:34
faiss::Level1Quantizer::quantizer_trains_alone
char quantizer_trains_alone
Definition: IndexIVF.h:40
faiss::ParameterSpace::thread_over_batches
bool thread_over_batches
Definition: AutoTune.h:149
faiss::hash_bytes
uint64_t hash_bytes(const uint8_t *bytes, long n)
Definition: utils.cpp:1584
faiss::OperatingPoints::all_pts
std::vector< OperatingPoint > all_pts
all operating points
Definition: AutoTune.h:95
faiss::ranklist_intersection_size
size_t ranklist_intersection_size(size_t k1, const long *v1, size_t k2, const long *v2_in)
Definition: utils.cpp:1253
faiss::Level1Quantizer::cp
ClusteringParameters cp
to override default clustering params
Definition: IndexIVF.h:43
faiss::AutoTuneCriterion
Definition: AutoTune.h:26
faiss::Index::verbose
bool verbose
verbosity level
Definition: Index.h:68
faiss::getmillisecs
double getmillisecs()
ms elapsed since some arbitrary epoch
Definition: utils.cpp:69
faiss::IndexShardsTemplate
Definition: IndexShards.h:20
faiss::AutoTuneCriterion::gt_D
std::vector< float > gt_D
Ground-truth distances (size nq * gt_nnn)
Definition: AutoTune.h:32
faiss::IndexIVFScalarQuantizer
Definition: IndexScalarQuantizer.h:141
faiss::ParameterSpace::combination_name
std::string combination_name(size_t cno) const
get string representation of the combination
Definition: AutoTune.cpp:288
faiss::Index::search
virtual void search(idx_t n, const float *x, idx_t k, float *distances, idx_t *labels) const =0
faiss::MultiIndexQuantizer
Definition: IndexPQ.h:135
faiss::ParameterSpace::update_bounds
void update_bounds(size_t cno, const OperatingPoint &op, double *upper_bound_perf, double *lower_bound_t) const
Definition: AutoTune.cpp:583
faiss::VectorTransform
Definition: VectorTransform.h:26
faiss::IndexPreTransform::own_fields
bool own_fields
! the sub-index
Definition: VectorTransform.h:276
faiss::ParameterSpace::initialize
virtual void initialize(const Index *index)
initialize with reasonable parameters for the index
Definition: AutoTune.cpp:347
faiss::ParameterSpace::verbose
int verbose
verbosity during exploration
Definition: AutoTune.h:139
faiss::IndexHNSWPQ
Definition: IndexHNSW.h:147
faiss::OperatingPoints::merge_with
int merge_with(const OperatingPoints &other, const std::string &prefix="")
add operating points from other to this, with a prefix to the keys
Definition: AutoTune.cpp:168
faiss::IndexHNSW
Definition: IndexHNSW.h:69
faiss::ParameterSpace::set_index_parameter
virtual void set_index_parameter(Index *index, const std::string &name, double val) const
set one of the parameters
Definition: AutoTune.cpp:455
faiss::ParameterSpace::n_combinations
size_t n_combinations() const
nb of combinations, = product of values sizes
Definition: AutoTune.cpp:279
faiss::Index::metric_type
MetricType metric_type
type of metric this index uses for search
Definition: Index.h:74
faiss::ParameterSpace::set_index_parameters
void set_index_parameters(Index *index, size_t cno) const
set a combination of parameters on an index
Definition: AutoTune.cpp:422
faiss::IndexHNSWFlat
Definition: IndexHNSW.h:137
faiss::ScalarQuantizer::QT_8bit
8 bits per component
Definition: IndexScalarQuantizer.h:35
faiss::Index
Definition: Index.h:61
faiss::IndexPQ::ST_PQ
asymmetric product quantizer (default)
Definition: IndexPQ.h:76
faiss::ParameterSpace::display
void display() const
print a description on stdout
Definition: AutoTune.cpp:565
faiss::IndexPQ::ST_polysemous
HE filter (using ht) + PQ combination.
Definition: IndexPQ.h:80
faiss::ProductQuantizer
Definition: ProductQuantizer.h:23
faiss::NormalizationTransform
Definition: VectorTransform.h:236
faiss::ParameterSpace::combination_ge
bool combination_ge(size_t c1, size_t c2) const
returns whether combinations c1 &gt;= c2 in the tuple sense
Definition: AutoTune.cpp:303
faiss::IndexReplicasTemplate
Definition: IndexReplicas.h:21
faiss::ClusteringParameters::spherical
bool spherical
do we want normalized centroids?
Definition: Clustering.h:27
faiss::Level1Quantizer::own_fields
bool own_fields
whether object owns the quantizer
Definition: IndexIVF.h:41
faiss::ParameterRange
possible values of a parameter, sorted from least to most expensive/accurate
Definition: AutoTune.h:125
faiss::index_factory
Index * index_factory(int d, const char *description_in, MetricType metric)
Definition: AutoTune.cpp:741
faiss::ParameterSpace::n_experiments
int n_experiments
nb of experiments during optimization (0 = try all combinations)
Definition: AutoTune.h:142
faiss::AutoTuneCriterion::gt_I
std::vector< idx_t > gt_I
Ground-truth indexes (size nq * gt_nnn)
Definition: AutoTune.h:33
faiss::OperatingPoint::t
double t
corresponding execution time (ms)
Definition: AutoTune.h:88
faiss::MetricType
MetricType
Some algorithms support both an inner product version and a L2 search version.
Definition: Index.h:44
faiss::IndexIVFFlatDedup
Definition: IndexIVFFlat.h:62
faiss::IndexIDMap::own_fields
bool own_fields
! the sub-index
Definition: MetaIndexes.h:24