Faiss

- Main Page
- Namespaces
- Classes
- Files

- File List

All Classes Namespaces Functions Variables Typedefs Enumerations Enumerator Friends
/tmp/faiss/PolysemousTraining.cpp
1 /**
2 * Copyright (c) 2015-present, Facebook, Inc.
3 * All rights reserved.
4 *
5 * This source code is licensed under the BSD+Patents license found in the
6 * LICENSE file in the root directory of this source tree.
7 */
8
9 // -*- c++ -*-
10
11 #include "PolysemousTraining.h"
12
13 #include <cstdlib>
14 #include <cmath>
15 #include <cstring>
16
17 #include <algorithm>
18
19 #include "utils.h"
20 #include "hamming.h"
21
22 #include "FaissAssert.h"
23
24 /*****************************************
25 * Mixed PQ / Hamming
26 ******************************************/
27
28 namespace faiss {
29
30
31 /****************************************************
32 * Optimization code
33 ****************************************************/
34
35 SimulatedAnnealingParameters::SimulatedAnnealingParameters ()
36 {
37 // set some reasonable defaults for the optimization
38 init_temperature = 0.7;
39 temperature_decay = pow (0.9, 1/500.);
40 // reduce by a factor 0.9 every 500 it
41 n_iter = 500000;
42 n_redo = 2;
43 seed = 123;
44 verbose = 0;
45 only_bit_flips = false;
46 init_random = false;
47 }
48
49 // what would the cost update be if iw and jw were swapped?
50 // default implementation just computes both and computes the difference
51 double PermutationObjective::cost_update (
52 const int *perm, int iw, int jw) const
53 {
54 double orig_cost = compute_cost (perm);
55
56 std::vector<int> perm2 (n);
57 for (int i = 0; i < n; i++)
58 perm2[i] = perm[i];
59 perm2[iw] = perm[jw];
60 perm2[jw] = perm[iw];
61
62 double new_cost = compute_cost (perm2.data());
63 return new_cost - orig_cost;
64 }
65
66
67
68
69 SimulatedAnnealingOptimizer::SimulatedAnnealingOptimizer (
70 PermutationObjective *obj,
71 const SimulatedAnnealingParameters &p):
72 SimulatedAnnealingParameters (p),
73 obj (obj),
74 n(obj->n),
75 logfile (nullptr)
76 {
77 rnd = new RandomGenerator (p.seed);
78 FAISS_THROW_IF_NOT (n < 100000 && n >=0 );
79 }
80
81 SimulatedAnnealingOptimizer::~SimulatedAnnealingOptimizer ()
82 {
83 delete rnd;
84 }
85
86 // run the optimization and return the best result in best_perm
87 double SimulatedAnnealingOptimizer::run_optimization (int * best_perm)
88 {
89 double min_cost = 1e30;
90
91 // just do a few runs of the annealing and keep the lowest output cost
92 for (int it = 0; it < n_redo; it++) {
93 std::vector<int> perm(n);
94 for (int i = 0; i < n; i++)
95 perm[i] = i;
96 if (init_random) {
97 for (int i = 0; i < n; i++) {
98 int j = i + rnd->rand_int (n - i);
99 std::swap (perm[i], perm[j]);
100 }
101 }
102 float cost = optimize (perm.data());
103 if (logfile) fprintf (logfile, "\n");
104 if(verbose > 1) {
105 printf (" optimization run %d: cost=%g %s\n",
106 it, cost, cost < min_cost ? "keep" : "");
107 }
108 if (cost < min_cost) {
109 memcpy (best_perm, perm.data(), sizeof(perm[0]) * n);
110 min_cost = cost;
111 }
112 }
113 return min_cost;
114 }
115
116 // perform the optimization loop, starting from and modifying
117 // permutation in-place
118 double SimulatedAnnealingOptimizer::optimize (int *perm)
119 {
120 double cost = init_cost = obj->compute_cost (perm);
121 int log2n = 0;
122 while (!(n <= (1 << log2n))) log2n++;
123 double temperature = init_temperature;
124 int n_swap = 0, n_hot = 0;
125 for (int it = 0; it < n_iter; it++) {
126 temperature = temperature * temperature_decay;
127 int iw, jw;
128 if (only_bit_flips) {
129 iw = rnd->rand_int (n);
130 jw = iw ^ (1 << rnd->rand_int (log2n));
131 } else {
132 iw = rnd->rand_int (n);
133 jw = rnd->rand_int (n - 1);
134 if (jw == iw) jw++;
135 }
136 double delta_cost = obj->cost_update (perm, iw, jw);
137 if (delta_cost < 0 || rnd->rand_float () < temperature) {
138 std::swap (perm[iw], perm[jw]);
139 cost += delta_cost;
140 n_swap++;
141 if (delta_cost >= 0) n_hot++;
142 }
143 if (verbose > 2 || (verbose > 1 && it % 10000 == 0)) {
144 printf (" iteration %d cost %g temp %g n_swap %d "
145 "(%d hot) \r",
146 it, cost, temperature, n_swap, n_hot);
147 fflush(stdout);
148 }
149 if (logfile) {
150 fprintf (logfile, "%d %g %g %d %d\n",
151 it, cost, temperature, n_swap, n_hot);
152 }
153 }
154 if (verbose > 1) printf("\n");
155 return cost;
156 }
157
158
159
160
161
162 /****************************************************
163 * Cost functions: ReproduceDistanceTable
164 ****************************************************/
165
166
167
168
169
170
171 static inline int hamming_dis (uint64_t a, uint64_t b)
172 {
173 return __builtin_popcountl (a ^ b);
174 }
175
176 namespace {
177
178 /// optimize permutation to reproduce a distance table with Hamming distances
179 struct ReproduceWithHammingObjective : PermutationObjective {
180 int nbits;
181 double dis_weight_factor;
182
183 static double sqr (double x) { return x * x; }
184
185
186 // weihgting of distances: it is more important to reproduce small
187 // distances well
188 double dis_weight (double x) const
189 {
190 return exp (-dis_weight_factor * x);
191 }
192
193 std::vector<double> target_dis; // wanted distances (size n^2)
194 std::vector<double> weights; // weights for each distance (size n^2)
195
196 // cost = quadratic difference between actual distance and Hamming distance
197 double compute_cost(const int* perm) const override {
198 double cost = 0;
199 for (int i = 0; i < n; i++) {
200 for (int j = 0; j < n; j++) {
201 double wanted = target_dis[i * n + j];
202 double w = weights[i * n + j];
203 double actual = hamming_dis(perm[i], perm[j]);
204 cost += w * sqr(wanted - actual);
205 }
206 }
207 return cost;
208 }
209
210
211 // what would the cost update be if iw and jw were swapped?
212 // computed in O(n) instead of O(n^2) for the full re-computation
213 double cost_update(const int* perm, int iw, int jw) const override {
214 double delta_cost = 0;
215
216 for (int i = 0; i < n; i++) {
217 if (i == iw) {
218 for (int j = 0; j < n; j++) {
219 double wanted = target_dis[i * n + j], w = weights[i * n + j];
220 double actual = hamming_dis(perm[i], perm[j]);
221 delta_cost -= w * sqr(wanted - actual);
222 double new_actual =
223 hamming_dis(perm[jw], perm[j == iw ? jw : j == jw ? iw : j]);
224 delta_cost += w * sqr(wanted - new_actual);
225 }
226 } else if (i == jw) {
227 for (int j = 0; j < n; j++) {
228 double wanted = target_dis[i * n + j], w = weights[i * n + j];
229 double actual = hamming_dis(perm[i], perm[j]);
230 delta_cost -= w * sqr(wanted - actual);
231 double new_actual =
232 hamming_dis(perm[iw], perm[j == iw ? jw : j == jw ? iw : j]);
233 delta_cost += w * sqr(wanted - new_actual);
234 }
235 } else {
236 int j = iw;
237 {
238 double wanted = target_dis[i * n + j], w = weights[i * n + j];
239 double actual = hamming_dis(perm[i], perm[j]);
240 delta_cost -= w * sqr(wanted - actual);
241 double new_actual = hamming_dis(perm[i], perm[jw]);
242 delta_cost += w * sqr(wanted - new_actual);
243 }
244 j = jw;
245 {
246 double wanted = target_dis[i * n + j], w = weights[i * n + j];
247 double actual = hamming_dis(perm[i], perm[j]);
248 delta_cost -= w * sqr(wanted - actual);
249 double new_actual = hamming_dis(perm[i], perm[iw]);
250 delta_cost += w * sqr(wanted - new_actual);
251 }
252 }
253 }
254
255 return delta_cost;
256 }
257
258
259
260 ReproduceWithHammingObjective (
261 int nbits,
262 const std::vector<double> & dis_table,
263 double dis_weight_factor):
264 nbits (nbits), dis_weight_factor (dis_weight_factor)
265 {
266 n = 1 << nbits;
267 FAISS_THROW_IF_NOT (dis_table.size() == n * n);
268 set_affine_target_dis (dis_table);
269 }
270
271 void set_affine_target_dis (const std::vector<double> & dis_table)
272 {
273 double sum = 0, sum2 = 0;
274 int n2 = n * n;
275 for (int i = 0; i < n2; i++) {
276 sum += dis_table [i];
277 sum2 += dis_table [i] * dis_table [i];
278 }
279 double mean = sum / n2;
280 double stddev = sqrt(sum2 / n2 - (sum / n2) * (sum / n2));
281
282 target_dis.resize (n2);
283
284 for (int i = 0; i < n2; i++) {
285 // the mapping function
286 double td = (dis_table [i] - mean) / stddev * sqrt(nbits / 4) +
287 nbits / 2;
288 target_dis[i] = td;
289 // compute a weight
290 weights.push_back (dis_weight (td));
291 }
292
293 }
294
295 ~ReproduceWithHammingObjective() override {}
296 };
297
298 } // anonymous namespace
299
300 // weihgting of distances: it is more important to reproduce small
301 // distances well
302 double ReproduceDistancesObjective::dis_weight (double x) const
303 {
304 return exp (-dis_weight_factor * x);
305 }
306
307
308 double ReproduceDistancesObjective::get_source_dis (int i, int j) const
309 {
310 return source_dis [i * n + j];
311 }
312
313 // cost = quadratic difference between actual distance and Hamming distance
314 double ReproduceDistancesObjective::compute_cost (const int *perm) const
315 {
316 double cost = 0;
317 for (int i = 0; i < n; i++) {
318 for (int j = 0; j < n; j++) {
319 double wanted = target_dis [i * n + j];
320 double w = weights [i * n + j];
321 double actual = get_source_dis (perm[i], perm[j]);
322 cost += w * sqr (wanted - actual);
323 }
324 }
325 return cost;
326 }
327
328 // what would the cost update be if iw and jw were swapped?
329 // computed in O(n) instead of O(n^2) for the full re-computation
330 double ReproduceDistancesObjective::cost_update(
331 const int *perm, int iw, int jw) const
332 {
333 double delta_cost = 0;
334 for (int i = 0; i < n; i++) {
335 if (i == iw) {
336 for (int j = 0; j < n; j++) {
337 double wanted = target_dis [i * n + j],
338 w = weights [i * n + j];
339 double actual = get_source_dis (perm[i], perm[j]);
340 delta_cost -= w * sqr (wanted - actual);
341 double new_actual = get_source_dis (
342 perm[jw],
343 perm[j == iw ? jw : j == jw ? iw : j]);
344 delta_cost += w * sqr (wanted - new_actual);
345 }
346 } else if (i == jw) {
347 for (int j = 0; j < n; j++) {
348 double wanted = target_dis [i * n + j],
349 w = weights [i * n + j];
350 double actual = get_source_dis (perm[i], perm[j]);
351 delta_cost -= w * sqr (wanted - actual);
352 double new_actual = get_source_dis (
353 perm[iw],
354 perm[j == iw ? jw : j == jw ? iw : j]);
355 delta_cost += w * sqr (wanted - new_actual);
356 }
357 } else {
358 int j = iw;
359 {
360 double wanted = target_dis [i * n + j],
361 w = weights [i * n + j];
362 double actual = get_source_dis (perm[i], perm[j]);
363 delta_cost -= w * sqr (wanted - actual);
364 double new_actual = get_source_dis (perm[i], perm[jw]);
365 delta_cost += w * sqr (wanted - new_actual);
366 }
367 j = jw;
368 {
369 double wanted = target_dis [i * n + j],
370 w = weights [i * n + j];
371 double actual = get_source_dis (perm[i], perm[j]);
372 delta_cost -= w * sqr (wanted - actual);
373 double new_actual = get_source_dis (perm[i], perm[iw]);
374 delta_cost += w * sqr (wanted - new_actual);
375 }
376 }
377 }
378 return delta_cost;
379 }
380
381
382
383 ReproduceDistancesObjective::ReproduceDistancesObjective (
384 int n,
385 const double *source_dis_in,
386 const double *target_dis_in,
387 double dis_weight_factor):
388 dis_weight_factor (dis_weight_factor),
389 target_dis (target_dis_in)
390 {
391 this->n = n;
392 set_affine_target_dis (source_dis_in);
393 }
394
395 void ReproduceDistancesObjective::compute_mean_stdev (
396 const double *tab, size_t n2,
397 double *mean_out, double *stddev_out)
398 {
399 double sum = 0, sum2 = 0;
400 for (int i = 0; i < n2; i++) {
401 sum += tab [i];
402 sum2 += tab [i] * tab [i];
403 }
404 double mean = sum / n2;
405 double stddev = sqrt(sum2 / n2 - (sum / n2) * (sum / n2));
406 *mean_out = mean;
407 *stddev_out = stddev;
408 }
409
410 void ReproduceDistancesObjective::set_affine_target_dis (
411 const double *source_dis_in)
412 {
413 int n2 = n * n;
414
415 double mean_src, stddev_src;
416 compute_mean_stdev (source_dis_in, n2, &mean_src, &stddev_src);
417
418 double mean_target, stddev_target;
419 compute_mean_stdev (target_dis, n2, &mean_target, &stddev_target);
420
421 printf ("map mean %g std %g -> mean %g std %g\n",
422 mean_src, stddev_src, mean_target, stddev_target);
423
424 source_dis.resize (n2);
425 weights.resize (n2);
426
427 for (int i = 0; i < n2; i++) {
428 // the mapping function
429 source_dis[i] = (source_dis_in[i] - mean_src) / stddev_src
430 * stddev_target + mean_target;
431
432 // compute a weight
433 weights [i] = dis_weight (target_dis[i]);
434 }
435
436 }
437
438 /****************************************************
439 * Cost functions: RankingScore
440 ****************************************************/
441
442 /// Maintains a 3D table of elementary costs.
443 /// Accumulates elements based on Hamming distance comparisons
444 template <typename Ttab, typename Taccu>
445 struct Score3Computer: PermutationObjective {
446
447 int nc;
448
449 // cost matrix of size nc * nc *nc
450 // n_gt (i,j,k) = count of d_gt(x, y-) < d_gt(x, y+)
451 // where x has PQ code i, y- PQ code j and y+ PQ code k
452 std::vector<Ttab> n_gt;
453
454
455 /// the cost is a triple loop on the nc * nc * nc matrix of entries.
456 ///
457 Taccu compute (const int * perm) const
458 {
459 Taccu accu = 0;
460 const Ttab *p = n_gt.data();
461 for (int i = 0; i < nc; i++) {
462 int ip = perm [i];
463 for (int j = 0; j < nc; j++) {
464 int jp = perm [j];
465 for (int k = 0; k < nc; k++) {
466 int kp = perm [k];
467 if (hamming_dis (ip, jp) <
468 hamming_dis (ip, kp)) {
469 accu += *p; // n_gt [ ( i * nc + j) * nc + k];
470 }
471 p++;
472 }
473 }
474 }
475 return accu;
476 }
477
478
479 /** cost update if entries iw and jw of the permutation would be
480 * swapped.
481 *
482 * The computation is optimized by avoiding elements in the
483 * nc*nc*nc cube that are known not to change. For nc=256, this
484 * reduces the nb of cells to visit to about 6/256 th of the
485 * cells. Practical speedup is about 8x, and the code is quite
486 * complex :-/
487 */
488 Taccu compute_update (const int *perm, int iw, int jw) const
489 {
490 assert (iw != jw);
491 if (iw > jw) std::swap (iw, jw);
492
493 Taccu accu = 0;
494 const Ttab * n_gt_i = n_gt.data();
495 for (int i = 0; i < nc; i++) {
496 int ip0 = perm [i];
497 int ip = perm [i == iw ? jw : i == jw ? iw : i];
498
499 //accu += update_i (perm, iw, jw, ip0, ip, n_gt_i);
500
501 accu += update_i_cross (perm, iw, jw,
502 ip0, ip, n_gt_i);
503
504 if (ip != ip0)
505 accu += update_i_plane (perm, iw, jw,
506 ip0, ip, n_gt_i);
507
508 n_gt_i += nc * nc;
509 }
510
511 return accu;
512 }
513
514
515 Taccu update_i (const int *perm, int iw, int jw,
516 int ip0, int ip, const Ttab * n_gt_i) const
517 {
518 Taccu accu = 0;
519 const Ttab *n_gt_ij = n_gt_i;
520 for (int j = 0; j < nc; j++) {
521 int jp0 = perm[j];
522 int jp = perm [j == iw ? jw : j == jw ? iw : j];
523 for (int k = 0; k < nc; k++) {
524 int kp0 = perm [k];
525 int kp = perm [k == iw ? jw : k == jw ? iw : k];
526 int ng = n_gt_ij [k];
527 if (hamming_dis (ip, jp) < hamming_dis (ip, kp)) {
528 accu += ng;
529 }
530 if (hamming_dis (ip0, jp0) < hamming_dis (ip0, kp0)) {
531 accu -= ng;
532 }
533 }
534 n_gt_ij += nc;
535 }
536 return accu;
537 }
538
539 // 2 inner loops for the case ip0 != ip
540 Taccu update_i_plane (const int *perm, int iw, int jw,
541 int ip0, int ip, const Ttab * n_gt_i) const
542 {
543 Taccu accu = 0;
544 const Ttab *n_gt_ij = n_gt_i;
545
546 for (int j = 0; j < nc; j++) {
547 if (j != iw && j != jw) {
548 int jp = perm[j];
549 for (int k = 0; k < nc; k++) {
550 if (k != iw && k != jw) {
551 int kp = perm [k];
552 Ttab ng = n_gt_ij [k];
553 if (hamming_dis (ip, jp) < hamming_dis (ip, kp)) {
554 accu += ng;
555 }
556 if (hamming_dis (ip0, jp) < hamming_dis (ip0, kp)) {
557 accu -= ng;
558 }
559 }
560 }
561 }
562 n_gt_ij += nc;
563 }
564 return accu;
565 }
566
567 /// used for the 8 cells were the 3 indices are swapped
568 inline Taccu update_k (const int *perm, int iw, int jw,
569 int ip0, int ip, int jp0, int jp,
570 int k,
571 const Ttab * n_gt_ij) const
572 {
573 Taccu accu = 0;
574 int kp0 = perm [k];
575 int kp = perm [k == iw ? jw : k == jw ? iw : k];
576 Ttab ng = n_gt_ij [k];
577 if (hamming_dis (ip, jp) < hamming_dis (ip, kp)) {
578 accu += ng;
579 }
580 if (hamming_dis (ip0, jp0) < hamming_dis (ip0, kp0)) {
581 accu -= ng;
582 }
583 return accu;
584 }
585
586 /// compute update on a line of k's, where i and j are swapped
587 Taccu update_j_line (const int *perm, int iw, int jw,
588 int ip0, int ip, int jp0, int jp,
589 const Ttab * n_gt_ij) const
590 {
591 Taccu accu = 0;
592 for (int k = 0; k < nc; k++) {
593 if (k == iw || k == jw) continue;
594 int kp = perm [k];
595 Ttab ng = n_gt_ij [k];
596 if (hamming_dis (ip, jp) < hamming_dis (ip, kp)) {
597 accu += ng;
598 }
599 if (hamming_dis (ip0, jp0) < hamming_dis (ip0, kp)) {
600 accu -= ng;
601 }
602 }
603 return accu;
604 }
605
606
607 /// considers the 2 pairs of crossing lines j=iw or jw and k = iw or kw
608 Taccu update_i_cross (const int *perm, int iw, int jw,
609 int ip0, int ip, const Ttab * n_gt_i) const
610 {
611 Taccu accu = 0;
612 const Ttab *n_gt_ij = n_gt_i;
613
614 for (int j = 0; j < nc; j++) {
615 int jp0 = perm[j];
616 int jp = perm [j == iw ? jw : j == jw ? iw : j];
617
618 accu += update_k (perm, iw, jw, ip0, ip, jp0, jp, iw, n_gt_ij);
619 accu += update_k (perm, iw, jw, ip0, ip, jp0, jp, jw, n_gt_ij);
620
621 if (jp != jp0)
622 accu += update_j_line (perm, iw, jw, ip0, ip, jp0, jp, n_gt_ij);
623
624 n_gt_ij += nc;
625 }
626 return accu;
627 }
628
629
630 /// PermutationObjective implementeation (just negates the scores
631 /// for minimization)
632
633 double compute_cost(const int* perm) const override {
634 return -compute(perm);
635 }
636
637 double cost_update(const int* perm, int iw, int jw) const override {
638 double ret = -compute_update(perm, iw, jw);
639 return ret;
640 }
641
642 ~Score3Computer() override {}
643 };
644
645
646
647
648
649 struct IndirectSort {
650 const float *tab;
651 bool operator () (int a, int b) {return tab[a] < tab[b]; }
652 };
653
654
655
656 struct RankingScore2: Score3Computer<float, double> {
657 int nbits;
658 int nq, nb;
659 const uint32_t *qcodes, *bcodes;
660 const float *gt_distances;
661
662 RankingScore2 (int nbits, int nq, int nb,
663 const uint32_t *qcodes, const uint32_t *bcodes,
664 const float *gt_distances):
665 nbits(nbits), nq(nq), nb(nb), qcodes(qcodes),
666 bcodes(bcodes), gt_distances(gt_distances)
667 {
668 n = nc = 1 << nbits;
669 n_gt.resize (nc * nc * nc);
670 init_n_gt ();
671 }
672
673
674 double rank_weight (int r)
675 {
676 return 1.0 / (r + 1);
677 }
678
679 /// count nb of i, j in a x b st. i < j
680 /// a and b should be sorted on input
681 /// they are the ranks of j and k respectively.
682 /// specific version for diff-of-rank weighting, cannot optimized
683 /// with a cumulative table
684 double accum_gt_weight_diff (const std::vector<int> & a,
685 const std::vector<int> & b)
686 {
687 int nb = b.size(), na = a.size();
688
689 double accu = 0;
690 int j = 0;
691 for (int i = 0; i < na; i++) {
692 int ai = a[i];
693 while (j < nb && ai >= b[j]) j++;
694
695 double accu_i = 0;
696 for (int k = j; k < b.size(); k++)
697 accu_i += rank_weight (b[k] - ai);
698
699 accu += rank_weight (ai) * accu_i;
700
701 }
702 return accu;
703 }
704
705 void init_n_gt ()
706 {
707 for (int q = 0; q < nq; q++) {
708 const float *gtd = gt_distances + q * nb;
709 const uint32_t *cb = bcodes;// all same codes
710 float * n_gt_q = & n_gt [qcodes[q] * nc * nc];
711
712 printf("init gt for q=%d/%d \r", q, nq); fflush(stdout);
713
714 std::vector<int> rankv (nb);
715 int * ranks = rankv.data();
716
717 // elements in each code bin, ordered by rank within each bin
718 std::vector<std::vector<int> > tab (nc);
719
720 { // build rank table
721 IndirectSort s = {gtd};
722 for (int j = 0; j < nb; j++) ranks[j] = j;
723 std::sort (ranks, ranks + nb, s);
724 }
725
726 for (int rank = 0; rank < nb; rank++) {
727 int i = ranks [rank];
728 tab [cb[i]].push_back (rank);
729 }
730
731
732 // this is very expensive. Any suggestion for improvement
733 // welcome.
734 for (int i = 0; i < nc; i++) {
735 std::vector<int> & di = tab[i];
736 for (int j = 0; j < nc; j++) {
737 std::vector<int> & dj = tab[j];
738 n_gt_q [i * nc + j] += accum_gt_weight_diff (di, dj);
739
740 }
741 }
742
743 }
744
745 }
746
747 };
748
749
750 /*****************************************
751 * PolysemousTraining
752 ******************************************/
753
754
755
756 PolysemousTraining::PolysemousTraining ()
757 {
758 optimization_type = OT_ReproduceDistances_affine;
759 ntrain_permutation = 0;
760 dis_weight_factor = log(2);
761 }
762
763
764
765 void PolysemousTraining::optimize_reproduce_distances (
766 ProductQuantizer &pq) const
767 {
768
769 int dsub = pq.dsub;
770
771 int n = pq.ksub;
772 int nbits = pq.nbits;
773
774 #pragma omp parallel for
775 for (int m = 0; m < pq.M; m++) {
776 std::vector<double> dis_table;
777
778 // printf ("Optimizing quantizer %d\n", m);
779
780 float * centroids = pq.get_centroids (m, 0);
781
782 for (int i = 0; i < n; i++) {
783 for (int j = 0; j < n; j++) {
784 dis_table.push_back (fvec_L2sqr (centroids + i * dsub,
785 centroids + j * dsub,
786 dsub));
787 }
788 }
789
790 std::vector<int> perm (n);
791 ReproduceWithHammingObjective obj (
792 nbits, dis_table,
793 dis_weight_factor);
794
795
796 SimulatedAnnealingOptimizer optim (&obj, *this);
797
798 if (log_pattern.size()) {
799 char fname[256];
800 snprintf (fname, 256, log_pattern.c_str(), m);
801 printf ("opening log file %s\n", fname);
802 optim.logfile = fopen (fname, "w");
803 FAISS_THROW_IF_NOT_MSG (optim.logfile, "could not open logfile");
804 }
805 double final_cost = optim.run_optimization (perm.data());
806
807 if (verbose > 0) {
808 printf ("SimulatedAnnealingOptimizer for m=%d: %g -> %g\n",
809 m, optim.init_cost, final_cost);
810 }
811
812 if (log_pattern.size()) fclose (optim.logfile);
813
814 std::vector<float> centroids_copy;
815 for (int i = 0; i < dsub * n; i++)
816 centroids_copy.push_back (centroids[i]);
817
818 for (int i = 0; i < n; i++)
819 memcpy (centroids + perm[i] * dsub,
820 centroids_copy.data() + i * dsub,
821 dsub * sizeof(centroids[0]));
822
823 }
824
825 }
826
827
828 void PolysemousTraining::optimize_ranking (
829 ProductQuantizer &pq, size_t n, const float *x) const
830 {
831
832 int dsub = pq.dsub;
833
834 int nbits = pq.nbits;
835
836 std::vector<uint8_t> all_codes (pq.code_size * n);
837
838 pq.compute_codes (x, all_codes.data(), n);
839
840 FAISS_THROW_IF_NOT (pq.byte_per_idx == 1);
841
842 if (n == 0)
843 pq.compute_sdc_table ();
844
845 #pragma omp parallel for
846 for (int m = 0; m < pq.M; m++) {
847 size_t nq, nb;
848 std::vector <uint32_t> codes; // query codes, then db codes
849 std::vector <float> gt_distances; // nq * nb matrix of distances
850
851 if (n > 0) {
852 std::vector<float> xtrain (n * dsub);
853 for (int i = 0; i < n; i++)
854 memcpy (xtrain.data() + i * dsub,
855 x + i * pq.d + m * dsub,
856 sizeof(float) * dsub);
857
858 codes.resize (n);
859 for (int i = 0; i < n; i++)
860 codes [i] = all_codes [i * pq.code_size + m];
861
862 nq = n / 4; nb = n - nq;
863 const float *xq = xtrain.data();
864 const float *xb = xq + nq * dsub;
865
866 gt_distances.resize (nq * nb);
867
868 pairwise_L2sqr (dsub,
869 nq, xq,
870 nb, xb,
871 gt_distances.data());
872 } else {
873 nq = nb = pq.ksub;
874 codes.resize (2 * nq);
875 for (int i = 0; i < nq; i++)
876 codes[i] = codes [i + nq] = i;
877
878 gt_distances.resize (nq * nb);
879
880 memcpy (gt_distances.data (),
881 pq.sdc_table.data () + m * nq * nb,
882 sizeof (float) * nq * nb);
883 }
884
885 double t0 = getmillisecs ();
886
887 PermutationObjective *obj = new RankingScore2 (
888 nbits, nq, nb,
889 codes.data(), codes.data() + nq,
890 gt_distances.data ());
891 ScopeDeleter1<PermutationObjective> del (obj);
892
893 if (verbose > 0) {
894 printf(" m=%d, nq=%ld, nb=%ld, intialize RankingScore "
895 "in %.3f ms\n",
896 m, nq, nb, getmillisecs () - t0);
897 }
898
899 SimulatedAnnealingOptimizer optim (obj, *this);
900
901 if (log_pattern.size()) {
902 char fname[256];
903 snprintf (fname, 256, log_pattern.c_str(), m);
904 printf ("opening log file %s\n", fname);
905 optim.logfile = fopen (fname, "w");
906 FAISS_THROW_IF_NOT_FMT (optim.logfile,
907 "could not open logfile %s", fname);
908 }
909
910 std::vector<int> perm (pq.ksub);
911
912 double final_cost = optim.run_optimization (perm.data());
913 printf ("SimulatedAnnealingOptimizer for m=%d: %g -> %g\n",
914 m, optim.init_cost, final_cost);
915
916 if (log_pattern.size()) fclose (optim.logfile);
917
918 float * centroids = pq.get_centroids (m, 0);
919
920 std::vector<float> centroids_copy;
921 for (int i = 0; i < dsub * pq.ksub; i++)
922 centroids_copy.push_back (centroids[i]);
923
924 for (int i = 0; i < pq.ksub; i++)
925 memcpy (centroids + perm[i] * dsub,
926 centroids_copy.data() + i * dsub,
927 dsub * sizeof(centroids[0]));
928
929 }
930
931 }
932
933
934
935 void PolysemousTraining::optimize_pq_for_hamming (ProductQuantizer &pq,
936 size_t n, const float *x) const
937 {
938 if (optimization_type == OT_None) {
939
940 } else if (optimization_type == OT_ReproduceDistances_affine) {
941 optimize_reproduce_distances (pq);
942 } else {
943 optimize_ranking (pq, n, x);
944 }
945
946 pq.compute_sdc_table ();
947
948 }
949
950
951 } // namespace faiss
faiss::RandomGenerator
random generator that can be used in multithreaded contexts
Definition: utils.h:48
faiss::ProductQuantizer::nbits
size_t nbits
number of bits per quantization index
Definition: ProductQuantizer.h:28
faiss::fvec_L2sqr
float fvec_L2sqr(const float *x, const float *y, size_t d)
Squared L2 distance between two vectors.
Definition: utils_simd.cpp:502
faiss::ProductQuantizer::byte_per_idx
size_t byte_per_idx
nb bytes per code component (1 or 2)
Definition: ProductQuantizer.h:32
faiss::Score3Computer::compute_update
Taccu compute_update(const int *perm, int iw, int jw) const
Definition: PolysemousTraining.cpp:488
faiss::ProductQuantizer::sdc_table
std::vector< float > sdc_table
Symmetric Distance Table.
Definition: ProductQuantizer.h:158
faiss::SimulatedAnnealingOptimizer::SimulatedAnnealingOptimizer
SimulatedAnnealingOptimizer(PermutationObjective *obj, const SimulatedAnnealingParameters &p)
logs values of the cost function
Definition: PolysemousTraining.cpp:69
faiss::ScopeDeleter1
Definition: FaissException.h:52
faiss::SimulatedAnnealingOptimizer::n
int n
size of the permutation
Definition: PolysemousTraining.h:98
faiss::Score3Computer::compute
Taccu compute(const int *perm) const
Definition: PolysemousTraining.cpp:457
faiss::ProductQuantizer::dsub
size_t dsub
dimensionality of each subvector
Definition: ProductQuantizer.h:31
faiss::ProductQuantizer::compute_codes
void compute_codes(const float *x, uint8_t *codes, size_t n) const
same as compute_code for several vectors
Definition: ProductQuantizer.cpp:382
faiss::Score3Computer::update_j_line
Taccu update_j_line(const int *perm, int iw, int jw, int ip0, int ip, int jp0, int jp, const Ttab *n_gt_ij) const
compute update on a line of k&#39;s, where i and j are swapped
Definition: PolysemousTraining.cpp:587
faiss::ReproduceDistancesObjective::target_dis
const double * target_dis
wanted distances (size n^2)
Definition: PolysemousTraining.h:66
faiss::ProductQuantizer::code_size
size_t code_size
byte per indexed vector
Definition: ProductQuantizer.h:33
faiss::SimulatedAnnealingOptimizer::init_cost
double init_cost
remember intial cost of optimization
Definition: PolysemousTraining.h:106
faiss::RandomGenerator::rand_int
int rand_int()
random positive integer
Definition: utils.cpp:115
faiss::ProductQuantizer::ksub
size_t ksub
number of centroids for each subquantizer
Definition: ProductQuantizer.h:34
faiss::PolysemousTraining::optimize_ranking
void optimize_ranking(ProductQuantizer &pq, size_t n, const float *x) const
called by optimize_pq_for_hamming
Definition: PolysemousTraining.cpp:828
faiss::pairwise_L2sqr
void pairwise_L2sqr(long d, long nq, const float *xq, long nb, const float *xb, float *dis, long ldq, long ldb, long ldd)
Definition: utils.cpp:1009
faiss::Score3Computer::compute_cost
double compute_cost(const int *perm) const override
Definition: PolysemousTraining.cpp:633
faiss::Score3Computer
Definition: PolysemousTraining.cpp:445
faiss::getmillisecs
double getmillisecs()
ms elapsed since some arbitrary epoch
Definition: utils.cpp:70
faiss::ReproduceDistancesObjective::weights
std::vector< double > weights
weights for each distance (size n^2)
Definition: PolysemousTraining.h:67
faiss::RankingScore2::accum_gt_weight_diff
double accum_gt_weight_diff(const std::vector< int > &a, const std::vector< int > &b)
Definition: PolysemousTraining.cpp:684
faiss::SimulatedAnnealingParameters
parameters used for the simulated annealing method
Definition: PolysemousTraining.h:22
faiss::IndirectSort
Definition: PolysemousTraining.cpp:649
faiss::Score3Computer::update_i_cross
Taccu update_i_cross(const int *perm, int iw, int jw, int ip0, int ip, const Ttab *n_gt_i) const
considers the 2 pairs of crossing lines j=iw or jw and k = iw or kw
Definition: PolysemousTraining.cpp:608
faiss::ProductQuantizer::M
size_t M
number of subquantizers
Definition: ProductQuantizer.h:27
faiss::PolysemousTraining::OT_ReproduceDistances_affine
default
Definition: PolysemousTraining.h:126
faiss::PermutationObjective
abstract class for the loss function
Definition: PolysemousTraining.h:41
faiss::ProductQuantizer
Definition: ProductQuantizer.h:24
faiss::Score3Computer::update_k
Taccu update_k(const int *perm, int iw, int jw, int ip0, int ip, int jp0, int jp, int k, const Ttab *n_gt_ij) const
used for the 8 cells were the 3 indices are swapped
Definition: PolysemousTraining.cpp:568
faiss::ReproduceDistancesObjective::source_dis
std::vector< double > source_dis
&quot;real&quot; corrected distances (size n^2)
Definition: PolysemousTraining.h:65
faiss::ProductQuantizer::get_centroids
float * get_centroids(size_t m, size_t i)
return the centroids associated with subvector m
Definition: ProductQuantizer.h:58
faiss::PolysemousTraining::optimize_reproduce_distances
void optimize_reproduce_distances(ProductQuantizer &pq) const
called by optimize_pq_for_hamming
Definition: PolysemousTraining.cpp:765
faiss::PolysemousTraining::optimize_pq_for_hamming
void optimize_pq_for_hamming(ProductQuantizer &pq, size_t n, const float *x) const
Definition: PolysemousTraining.cpp:935
faiss::ProductQuantizer::d
size_t d
size of the input vectors
Definition: ProductQuantizer.h:26
faiss::RankingScore2
Definition: PolysemousTraining.cpp:656
faiss::SimulatedAnnealingOptimizer
Simulated annealing optimization algorithm for permutations.
Definition: PolysemousTraining.h:95