Faiss

- Main Page
- Namespaces
- Classes
- Files

- File List

All Classes Namespaces Functions Variables Typedefs Enumerations Enumerator Friends
/data/users/hoss/faiss/PolysemousTraining.cpp
1 /**
2 * Copyright (c) Facebook, Inc. and its affiliates.
3 *
4 * This source code is licensed under the MIT license found in the
5 * LICENSE file in the root directory of this source tree.
6 */
7
8 // -*- c++ -*-
9
10 #include "PolysemousTraining.h"
11
12 #include <cstdlib>
13 #include <cmath>
14 #include <cstring>
15
16 #include <algorithm>
17
18 #include "utils.h"
19 #include "hamming.h"
20
21 #include "FaissAssert.h"
22
23 /*****************************************
24 * Mixed PQ / Hamming
25 ******************************************/
26
27 namespace faiss {
28
29
30 /****************************************************
31 * Optimization code
32 ****************************************************/
33
34 SimulatedAnnealingParameters::SimulatedAnnealingParameters ()
35 {
36 // set some reasonable defaults for the optimization
37 init_temperature = 0.7;
38 temperature_decay = pow (0.9, 1/500.);
39 // reduce by a factor 0.9 every 500 it
40 n_iter = 500000;
41 n_redo = 2;
42 seed = 123;
43 verbose = 0;
44 only_bit_flips = false;
45 init_random = false;
46 }
47
48 // what would the cost update be if iw and jw were swapped?
49 // default implementation just computes both and computes the difference
50 double PermutationObjective::cost_update (
51 const int *perm, int iw, int jw) const
52 {
53 double orig_cost = compute_cost (perm);
54
55 std::vector<int> perm2 (n);
56 for (int i = 0; i < n; i++)
57 perm2[i] = perm[i];
58 perm2[iw] = perm[jw];
59 perm2[jw] = perm[iw];
60
61 double new_cost = compute_cost (perm2.data());
62 return new_cost - orig_cost;
63 }
64
65
66
67
68 SimulatedAnnealingOptimizer::SimulatedAnnealingOptimizer (
69 PermutationObjective *obj,
70 const SimulatedAnnealingParameters &p):
71 SimulatedAnnealingParameters (p),
72 obj (obj),
73 n(obj->n),
74 logfile (nullptr)
75 {
76 rnd = new RandomGenerator (p.seed);
77 FAISS_THROW_IF_NOT (n < 100000 && n >=0 );
78 }
79
80 SimulatedAnnealingOptimizer::~SimulatedAnnealingOptimizer ()
81 {
82 delete rnd;
83 }
84
85 // run the optimization and return the best result in best_perm
86 double SimulatedAnnealingOptimizer::run_optimization (int * best_perm)
87 {
88 double min_cost = 1e30;
89
90 // just do a few runs of the annealing and keep the lowest output cost
91 for (int it = 0; it < n_redo; it++) {
92 std::vector<int> perm(n);
93 for (int i = 0; i < n; i++)
94 perm[i] = i;
95 if (init_random) {
96 for (int i = 0; i < n; i++) {
97 int j = i + rnd->rand_int (n - i);
98 std::swap (perm[i], perm[j]);
99 }
100 }
101 float cost = optimize (perm.data());
102 if (logfile) fprintf (logfile, "\n");
103 if(verbose > 1) {
104 printf (" optimization run %d: cost=%g %s\n",
105 it, cost, cost < min_cost ? "keep" : "");
106 }
107 if (cost < min_cost) {
108 memcpy (best_perm, perm.data(), sizeof(perm[0]) * n);
109 min_cost = cost;
110 }
111 }
112 return min_cost;
113 }
114
115 // perform the optimization loop, starting from and modifying
116 // permutation in-place
117 double SimulatedAnnealingOptimizer::optimize (int *perm)
118 {
119 double cost = init_cost = obj->compute_cost (perm);
120 int log2n = 0;
121 while (!(n <= (1 << log2n))) log2n++;
122 double temperature = init_temperature;
123 int n_swap = 0, n_hot = 0;
124 for (int it = 0; it < n_iter; it++) {
125 temperature = temperature * temperature_decay;
126 int iw, jw;
127 if (only_bit_flips) {
128 iw = rnd->rand_int (n);
129 jw = iw ^ (1 << rnd->rand_int (log2n));
130 } else {
131 iw = rnd->rand_int (n);
132 jw = rnd->rand_int (n - 1);
133 if (jw == iw) jw++;
134 }
135 double delta_cost = obj->cost_update (perm, iw, jw);
136 if (delta_cost < 0 || rnd->rand_float () < temperature) {
137 std::swap (perm[iw], perm[jw]);
138 cost += delta_cost;
139 n_swap++;
140 if (delta_cost >= 0) n_hot++;
141 }
142 if (verbose > 2 || (verbose > 1 && it % 10000 == 0)) {
143 printf (" iteration %d cost %g temp %g n_swap %d "
144 "(%d hot) \r",
145 it, cost, temperature, n_swap, n_hot);
146 fflush(stdout);
147 }
148 if (logfile) {
149 fprintf (logfile, "%d %g %g %d %d\n",
150 it, cost, temperature, n_swap, n_hot);
151 }
152 }
153 if (verbose > 1) printf("\n");
154 return cost;
155 }
156
157
158
159
160
161 /****************************************************
162 * Cost functions: ReproduceDistanceTable
163 ****************************************************/
164
165
166
167
168
169
170 static inline int hamming_dis (uint64_t a, uint64_t b)
171 {
172 return __builtin_popcountl (a ^ b);
173 }
174
175 namespace {
176
177 /// optimize permutation to reproduce a distance table with Hamming distances
178 struct ReproduceWithHammingObjective : PermutationObjective {
179 int nbits;
180 double dis_weight_factor;
181
182 static double sqr (double x) { return x * x; }
183
184
185 // weihgting of distances: it is more important to reproduce small
186 // distances well
187 double dis_weight (double x) const
188 {
189 return exp (-dis_weight_factor * x);
190 }
191
192 std::vector<double> target_dis; // wanted distances (size n^2)
193 std::vector<double> weights; // weights for each distance (size n^2)
194
195 // cost = quadratic difference between actual distance and Hamming distance
196 double compute_cost(const int* perm) const override {
197 double cost = 0;
198 for (int i = 0; i < n; i++) {
199 for (int j = 0; j < n; j++) {
200 double wanted = target_dis[i * n + j];
201 double w = weights[i * n + j];
202 double actual = hamming_dis(perm[i], perm[j]);
203 cost += w * sqr(wanted - actual);
204 }
205 }
206 return cost;
207 }
208
209
210 // what would the cost update be if iw and jw were swapped?
211 // computed in O(n) instead of O(n^2) for the full re-computation
212 double cost_update(const int* perm, int iw, int jw) const override {
213 double delta_cost = 0;
214
215 for (int i = 0; i < n; i++) {
216 if (i == iw) {
217 for (int j = 0; j < n; j++) {
218 double wanted = target_dis[i * n + j], w = weights[i * n + j];
219 double actual = hamming_dis(perm[i], perm[j]);
220 delta_cost -= w * sqr(wanted - actual);
221 double new_actual =
222 hamming_dis(perm[jw], perm[j == iw ? jw : j == jw ? iw : j]);
223 delta_cost += w * sqr(wanted - new_actual);
224 }
225 } else if (i == jw) {
226 for (int j = 0; j < n; j++) {
227 double wanted = target_dis[i * n + j], w = weights[i * n + j];
228 double actual = hamming_dis(perm[i], perm[j]);
229 delta_cost -= w * sqr(wanted - actual);
230 double new_actual =
231 hamming_dis(perm[iw], perm[j == iw ? jw : j == jw ? iw : j]);
232 delta_cost += w * sqr(wanted - new_actual);
233 }
234 } else {
235 int j = iw;
236 {
237 double wanted = target_dis[i * n + j], w = weights[i * n + j];
238 double actual = hamming_dis(perm[i], perm[j]);
239 delta_cost -= w * sqr(wanted - actual);
240 double new_actual = hamming_dis(perm[i], perm[jw]);
241 delta_cost += w * sqr(wanted - new_actual);
242 }
243 j = jw;
244 {
245 double wanted = target_dis[i * n + j], w = weights[i * n + j];
246 double actual = hamming_dis(perm[i], perm[j]);
247 delta_cost -= w * sqr(wanted - actual);
248 double new_actual = hamming_dis(perm[i], perm[iw]);
249 delta_cost += w * sqr(wanted - new_actual);
250 }
251 }
252 }
253
254 return delta_cost;
255 }
256
257
258
259 ReproduceWithHammingObjective (
260 int nbits,
261 const std::vector<double> & dis_table,
262 double dis_weight_factor):
263 nbits (nbits), dis_weight_factor (dis_weight_factor)
264 {
265 n = 1 << nbits;
266 FAISS_THROW_IF_NOT (dis_table.size() == n * n);
267 set_affine_target_dis (dis_table);
268 }
269
270 void set_affine_target_dis (const std::vector<double> & dis_table)
271 {
272 double sum = 0, sum2 = 0;
273 int n2 = n * n;
274 for (int i = 0; i < n2; i++) {
275 sum += dis_table [i];
276 sum2 += dis_table [i] * dis_table [i];
277 }
278 double mean = sum / n2;
279 double stddev = sqrt(sum2 / n2 - (sum / n2) * (sum / n2));
280
281 target_dis.resize (n2);
282
283 for (int i = 0; i < n2; i++) {
284 // the mapping function
285 double td = (dis_table [i] - mean) / stddev * sqrt(nbits / 4) +
286 nbits / 2;
287 target_dis[i] = td;
288 // compute a weight
289 weights.push_back (dis_weight (td));
290 }
291
292 }
293
294 ~ReproduceWithHammingObjective() override {}
295 };
296
297 } // anonymous namespace
298
299 // weihgting of distances: it is more important to reproduce small
300 // distances well
301 double ReproduceDistancesObjective::dis_weight (double x) const
302 {
303 return exp (-dis_weight_factor * x);
304 }
305
306
307 double ReproduceDistancesObjective::get_source_dis (int i, int j) const
308 {
309 return source_dis [i * n + j];
310 }
311
312 // cost = quadratic difference between actual distance and Hamming distance
313 double ReproduceDistancesObjective::compute_cost (const int *perm) const
314 {
315 double cost = 0;
316 for (int i = 0; i < n; i++) {
317 for (int j = 0; j < n; j++) {
318 double wanted = target_dis [i * n + j];
319 double w = weights [i * n + j];
320 double actual = get_source_dis (perm[i], perm[j]);
321 cost += w * sqr (wanted - actual);
322 }
323 }
324 return cost;
325 }
326
327 // what would the cost update be if iw and jw were swapped?
328 // computed in O(n) instead of O(n^2) for the full re-computation
329 double ReproduceDistancesObjective::cost_update(
330 const int *perm, int iw, int jw) const
331 {
332 double delta_cost = 0;
333 for (int i = 0; i < n; i++) {
334 if (i == iw) {
335 for (int j = 0; j < n; j++) {
336 double wanted = target_dis [i * n + j],
337 w = weights [i * n + j];
338 double actual = get_source_dis (perm[i], perm[j]);
339 delta_cost -= w * sqr (wanted - actual);
340 double new_actual = get_source_dis (
341 perm[jw],
342 perm[j == iw ? jw : j == jw ? iw : j]);
343 delta_cost += w * sqr (wanted - new_actual);
344 }
345 } else if (i == jw) {
346 for (int j = 0; j < n; j++) {
347 double wanted = target_dis [i * n + j],
348 w = weights [i * n + j];
349 double actual = get_source_dis (perm[i], perm[j]);
350 delta_cost -= w * sqr (wanted - actual);
351 double new_actual = get_source_dis (
352 perm[iw],
353 perm[j == iw ? jw : j == jw ? iw : j]);
354 delta_cost += w * sqr (wanted - new_actual);
355 }
356 } else {
357 int j = iw;
358 {
359 double wanted = target_dis [i * n + j],
360 w = weights [i * n + j];
361 double actual = get_source_dis (perm[i], perm[j]);
362 delta_cost -= w * sqr (wanted - actual);
363 double new_actual = get_source_dis (perm[i], perm[jw]);
364 delta_cost += w * sqr (wanted - new_actual);
365 }
366 j = jw;
367 {
368 double wanted = target_dis [i * n + j],
369 w = weights [i * n + j];
370 double actual = get_source_dis (perm[i], perm[j]);
371 delta_cost -= w * sqr (wanted - actual);
372 double new_actual = get_source_dis (perm[i], perm[iw]);
373 delta_cost += w * sqr (wanted - new_actual);
374 }
375 }
376 }
377 return delta_cost;
378 }
379
380
381
382 ReproduceDistancesObjective::ReproduceDistancesObjective (
383 int n,
384 const double *source_dis_in,
385 const double *target_dis_in,
386 double dis_weight_factor):
387 dis_weight_factor (dis_weight_factor),
388 target_dis (target_dis_in)
389 {
390 this->n = n;
391 set_affine_target_dis (source_dis_in);
392 }
393
394 void ReproduceDistancesObjective::compute_mean_stdev (
395 const double *tab, size_t n2,
396 double *mean_out, double *stddev_out)
397 {
398 double sum = 0, sum2 = 0;
399 for (int i = 0; i < n2; i++) {
400 sum += tab [i];
401 sum2 += tab [i] * tab [i];
402 }
403 double mean = sum / n2;
404 double stddev = sqrt(sum2 / n2 - (sum / n2) * (sum / n2));
405 *mean_out = mean;
406 *stddev_out = stddev;
407 }
408
409 void ReproduceDistancesObjective::set_affine_target_dis (
410 const double *source_dis_in)
411 {
412 int n2 = n * n;
413
414 double mean_src, stddev_src;
415 compute_mean_stdev (source_dis_in, n2, &mean_src, &stddev_src);
416
417 double mean_target, stddev_target;
418 compute_mean_stdev (target_dis, n2, &mean_target, &stddev_target);
419
420 printf ("map mean %g std %g -> mean %g std %g\n",
421 mean_src, stddev_src, mean_target, stddev_target);
422
423 source_dis.resize (n2);
424 weights.resize (n2);
425
426 for (int i = 0; i < n2; i++) {
427 // the mapping function
428 source_dis[i] = (source_dis_in[i] - mean_src) / stddev_src
429 * stddev_target + mean_target;
430
431 // compute a weight
432 weights [i] = dis_weight (target_dis[i]);
433 }
434
435 }
436
437 /****************************************************
438 * Cost functions: RankingScore
439 ****************************************************/
440
441 /// Maintains a 3D table of elementary costs.
442 /// Accumulates elements based on Hamming distance comparisons
443 template <typename Ttab, typename Taccu>
444 struct Score3Computer: PermutationObjective {
445
446 int nc;
447
448 // cost matrix of size nc * nc *nc
449 // n_gt (i,j,k) = count of d_gt(x, y-) < d_gt(x, y+)
450 // where x has PQ code i, y- PQ code j and y+ PQ code k
451 std::vector<Ttab> n_gt;
452
453
454 /// the cost is a triple loop on the nc * nc * nc matrix of entries.
455 ///
456 Taccu compute (const int * perm) const
457 {
458 Taccu accu = 0;
459 const Ttab *p = n_gt.data();
460 for (int i = 0; i < nc; i++) {
461 int ip = perm [i];
462 for (int j = 0; j < nc; j++) {
463 int jp = perm [j];
464 for (int k = 0; k < nc; k++) {
465 int kp = perm [k];
466 if (hamming_dis (ip, jp) <
467 hamming_dis (ip, kp)) {
468 accu += *p; // n_gt [ ( i * nc + j) * nc + k];
469 }
470 p++;
471 }
472 }
473 }
474 return accu;
475 }
476
477
478 /** cost update if entries iw and jw of the permutation would be
479 * swapped.
480 *
481 * The computation is optimized by avoiding elements in the
482 * nc*nc*nc cube that are known not to change. For nc=256, this
483 * reduces the nb of cells to visit to about 6/256 th of the
484 * cells. Practical speedup is about 8x, and the code is quite
485 * complex :-/
486 */
487 Taccu compute_update (const int *perm, int iw, int jw) const
488 {
489 assert (iw != jw);
490 if (iw > jw) std::swap (iw, jw);
491
492 Taccu accu = 0;
493 const Ttab * n_gt_i = n_gt.data();
494 for (int i = 0; i < nc; i++) {
495 int ip0 = perm [i];
496 int ip = perm [i == iw ? jw : i == jw ? iw : i];
497
498 //accu += update_i (perm, iw, jw, ip0, ip, n_gt_i);
499
500 accu += update_i_cross (perm, iw, jw,
501 ip0, ip, n_gt_i);
502
503 if (ip != ip0)
504 accu += update_i_plane (perm, iw, jw,
505 ip0, ip, n_gt_i);
506
507 n_gt_i += nc * nc;
508 }
509
510 return accu;
511 }
512
513
514 Taccu update_i (const int *perm, int iw, int jw,
515 int ip0, int ip, const Ttab * n_gt_i) const
516 {
517 Taccu accu = 0;
518 const Ttab *n_gt_ij = n_gt_i;
519 for (int j = 0; j < nc; j++) {
520 int jp0 = perm[j];
521 int jp = perm [j == iw ? jw : j == jw ? iw : j];
522 for (int k = 0; k < nc; k++) {
523 int kp0 = perm [k];
524 int kp = perm [k == iw ? jw : k == jw ? iw : k];
525 int ng = n_gt_ij [k];
526 if (hamming_dis (ip, jp) < hamming_dis (ip, kp)) {
527 accu += ng;
528 }
529 if (hamming_dis (ip0, jp0) < hamming_dis (ip0, kp0)) {
530 accu -= ng;
531 }
532 }
533 n_gt_ij += nc;
534 }
535 return accu;
536 }
537
538 // 2 inner loops for the case ip0 != ip
539 Taccu update_i_plane (const int *perm, int iw, int jw,
540 int ip0, int ip, const Ttab * n_gt_i) const
541 {
542 Taccu accu = 0;
543 const Ttab *n_gt_ij = n_gt_i;
544
545 for (int j = 0; j < nc; j++) {
546 if (j != iw && j != jw) {
547 int jp = perm[j];
548 for (int k = 0; k < nc; k++) {
549 if (k != iw && k != jw) {
550 int kp = perm [k];
551 Ttab ng = n_gt_ij [k];
552 if (hamming_dis (ip, jp) < hamming_dis (ip, kp)) {
553 accu += ng;
554 }
555 if (hamming_dis (ip0, jp) < hamming_dis (ip0, kp)) {
556 accu -= ng;
557 }
558 }
559 }
560 }
561 n_gt_ij += nc;
562 }
563 return accu;
564 }
565
566 /// used for the 8 cells were the 3 indices are swapped
567 inline Taccu update_k (const int *perm, int iw, int jw,
568 int ip0, int ip, int jp0, int jp,
569 int k,
570 const Ttab * n_gt_ij) const
571 {
572 Taccu accu = 0;
573 int kp0 = perm [k];
574 int kp = perm [k == iw ? jw : k == jw ? iw : k];
575 Ttab ng = n_gt_ij [k];
576 if (hamming_dis (ip, jp) < hamming_dis (ip, kp)) {
577 accu += ng;
578 }
579 if (hamming_dis (ip0, jp0) < hamming_dis (ip0, kp0)) {
580 accu -= ng;
581 }
582 return accu;
583 }
584
585 /// compute update on a line of k's, where i and j are swapped
586 Taccu update_j_line (const int *perm, int iw, int jw,
587 int ip0, int ip, int jp0, int jp,
588 const Ttab * n_gt_ij) const
589 {
590 Taccu accu = 0;
591 for (int k = 0; k < nc; k++) {
592 if (k == iw || k == jw) continue;
593 int kp = perm [k];
594 Ttab ng = n_gt_ij [k];
595 if (hamming_dis (ip, jp) < hamming_dis (ip, kp)) {
596 accu += ng;
597 }
598 if (hamming_dis (ip0, jp0) < hamming_dis (ip0, kp)) {
599 accu -= ng;
600 }
601 }
602 return accu;
603 }
604
605
606 /// considers the 2 pairs of crossing lines j=iw or jw and k = iw or kw
607 Taccu update_i_cross (const int *perm, int iw, int jw,
608 int ip0, int ip, const Ttab * n_gt_i) const
609 {
610 Taccu accu = 0;
611 const Ttab *n_gt_ij = n_gt_i;
612
613 for (int j = 0; j < nc; j++) {
614 int jp0 = perm[j];
615 int jp = perm [j == iw ? jw : j == jw ? iw : j];
616
617 accu += update_k (perm, iw, jw, ip0, ip, jp0, jp, iw, n_gt_ij);
618 accu += update_k (perm, iw, jw, ip0, ip, jp0, jp, jw, n_gt_ij);
619
620 if (jp != jp0)
621 accu += update_j_line (perm, iw, jw, ip0, ip, jp0, jp, n_gt_ij);
622
623 n_gt_ij += nc;
624 }
625 return accu;
626 }
627
628
629 /// PermutationObjective implementeation (just negates the scores
630 /// for minimization)
631
632 double compute_cost(const int* perm) const override {
633 return -compute(perm);
634 }
635
636 double cost_update(const int* perm, int iw, int jw) const override {
637 double ret = -compute_update(perm, iw, jw);
638 return ret;
639 }
640
641 ~Score3Computer() override {}
642 };
643
644
645
646
647
648 struct IndirectSort {
649 const float *tab;
650 bool operator () (int a, int b) {return tab[a] < tab[b]; }
651 };
652
653
654
655 struct RankingScore2: Score3Computer<float, double> {
656 int nbits;
657 int nq, nb;
658 const uint32_t *qcodes, *bcodes;
659 const float *gt_distances;
660
661 RankingScore2 (int nbits, int nq, int nb,
662 const uint32_t *qcodes, const uint32_t *bcodes,
663 const float *gt_distances):
664 nbits(nbits), nq(nq), nb(nb), qcodes(qcodes),
665 bcodes(bcodes), gt_distances(gt_distances)
666 {
667 n = nc = 1 << nbits;
668 n_gt.resize (nc * nc * nc);
669 init_n_gt ();
670 }
671
672
673 double rank_weight (int r)
674 {
675 return 1.0 / (r + 1);
676 }
677
678 /// count nb of i, j in a x b st. i < j
679 /// a and b should be sorted on input
680 /// they are the ranks of j and k respectively.
681 /// specific version for diff-of-rank weighting, cannot optimized
682 /// with a cumulative table
683 double accum_gt_weight_diff (const std::vector<int> & a,
684 const std::vector<int> & b)
685 {
686 int nb = b.size(), na = a.size();
687
688 double accu = 0;
689 int j = 0;
690 for (int i = 0; i < na; i++) {
691 int ai = a[i];
692 while (j < nb && ai >= b[j]) j++;
693
694 double accu_i = 0;
695 for (int k = j; k < b.size(); k++)
696 accu_i += rank_weight (b[k] - ai);
697
698 accu += rank_weight (ai) * accu_i;
699
700 }
701 return accu;
702 }
703
704 void init_n_gt ()
705 {
706 for (int q = 0; q < nq; q++) {
707 const float *gtd = gt_distances + q * nb;
708 const uint32_t *cb = bcodes;// all same codes
709 float * n_gt_q = & n_gt [qcodes[q] * nc * nc];
710
711 printf("init gt for q=%d/%d \r", q, nq); fflush(stdout);
712
713 std::vector<int> rankv (nb);
714 int * ranks = rankv.data();
715
716 // elements in each code bin, ordered by rank within each bin
717 std::vector<std::vector<int> > tab (nc);
718
719 { // build rank table
720 IndirectSort s = {gtd};
721 for (int j = 0; j < nb; j++) ranks[j] = j;
722 std::sort (ranks, ranks + nb, s);
723 }
724
725 for (int rank = 0; rank < nb; rank++) {
726 int i = ranks [rank];
727 tab [cb[i]].push_back (rank);
728 }
729
730
731 // this is very expensive. Any suggestion for improvement
732 // welcome.
733 for (int i = 0; i < nc; i++) {
734 std::vector<int> & di = tab[i];
735 for (int j = 0; j < nc; j++) {
736 std::vector<int> & dj = tab[j];
737 n_gt_q [i * nc + j] += accum_gt_weight_diff (di, dj);
738
739 }
740 }
741
742 }
743
744 }
745
746 };
747
748
749 /*****************************************
750 * PolysemousTraining
751 ******************************************/
752
753
754
755 PolysemousTraining::PolysemousTraining ()
756 {
757 optimization_type = OT_ReproduceDistances_affine;
758 ntrain_permutation = 0;
759 dis_weight_factor = log(2);
760 }
761
762
763
764 void PolysemousTraining::optimize_reproduce_distances (
765 ProductQuantizer &pq) const
766 {
767
768 int dsub = pq.dsub;
769
770 int n = pq.ksub;
771 int nbits = pq.nbits;
772
773 #pragma omp parallel for
774 for (int m = 0; m < pq.M; m++) {
775 std::vector<double> dis_table;
776
777 // printf ("Optimizing quantizer %d\n", m);
778
779 float * centroids = pq.get_centroids (m, 0);
780
781 for (int i = 0; i < n; i++) {
782 for (int j = 0; j < n; j++) {
783 dis_table.push_back (fvec_L2sqr (centroids + i * dsub,
784 centroids + j * dsub,
785 dsub));
786 }
787 }
788
789 std::vector<int> perm (n);
790 ReproduceWithHammingObjective obj (
791 nbits, dis_table,
792 dis_weight_factor);
793
794
795 SimulatedAnnealingOptimizer optim (&obj, *this);
796
797 if (log_pattern.size()) {
798 char fname[256];
799 snprintf (fname, 256, log_pattern.c_str(), m);
800 printf ("opening log file %s\n", fname);
801 optim.logfile = fopen (fname, "w");
802 FAISS_THROW_IF_NOT_MSG (optim.logfile, "could not open logfile");
803 }
804 double final_cost = optim.run_optimization (perm.data());
805
806 if (verbose > 0) {
807 printf ("SimulatedAnnealingOptimizer for m=%d: %g -> %g\n",
808 m, optim.init_cost, final_cost);
809 }
810
811 if (log_pattern.size()) fclose (optim.logfile);
812
813 std::vector<float> centroids_copy;
814 for (int i = 0; i < dsub * n; i++)
815 centroids_copy.push_back (centroids[i]);
816
817 for (int i = 0; i < n; i++)
818 memcpy (centroids + perm[i] * dsub,
819 centroids_copy.data() + i * dsub,
820 dsub * sizeof(centroids[0]));
821
822 }
823
824 }
825
826
827 void PolysemousTraining::optimize_ranking (
828 ProductQuantizer &pq, size_t n, const float *x) const
829 {
830
831 int dsub = pq.dsub;
832
833 int nbits = pq.nbits;
834
835 std::vector<uint8_t> all_codes (pq.code_size * n);
836
837 pq.compute_codes (x, all_codes.data(), n);
838
839 FAISS_THROW_IF_NOT (pq.nbits == 8);
840
841 if (n == 0)
842 pq.compute_sdc_table ();
843
844 #pragma omp parallel for
845 for (int m = 0; m < pq.M; m++) {
846 size_t nq, nb;
847 std::vector <uint32_t> codes; // query codes, then db codes
848 std::vector <float> gt_distances; // nq * nb matrix of distances
849
850 if (n > 0) {
851 std::vector<float> xtrain (n * dsub);
852 for (int i = 0; i < n; i++)
853 memcpy (xtrain.data() + i * dsub,
854 x + i * pq.d + m * dsub,
855 sizeof(float) * dsub);
856
857 codes.resize (n);
858 for (int i = 0; i < n; i++)
859 codes [i] = all_codes [i * pq.code_size + m];
860
861 nq = n / 4; nb = n - nq;
862 const float *xq = xtrain.data();
863 const float *xb = xq + nq * dsub;
864
865 gt_distances.resize (nq * nb);
866
867 pairwise_L2sqr (dsub,
868 nq, xq,
869 nb, xb,
870 gt_distances.data());
871 } else {
872 nq = nb = pq.ksub;
873 codes.resize (2 * nq);
874 for (int i = 0; i < nq; i++)
875 codes[i] = codes [i + nq] = i;
876
877 gt_distances.resize (nq * nb);
878
879 memcpy (gt_distances.data (),
880 pq.sdc_table.data () + m * nq * nb,
881 sizeof (float) * nq * nb);
882 }
883
884 double t0 = getmillisecs ();
885
886 PermutationObjective *obj = new RankingScore2 (
887 nbits, nq, nb,
888 codes.data(), codes.data() + nq,
889 gt_distances.data ());
890 ScopeDeleter1<PermutationObjective> del (obj);
891
892 if (verbose > 0) {
893 printf(" m=%d, nq=%ld, nb=%ld, intialize RankingScore "
894 "in %.3f ms\n",
895 m, nq, nb, getmillisecs () - t0);
896 }
897
898 SimulatedAnnealingOptimizer optim (obj, *this);
899
900 if (log_pattern.size()) {
901 char fname[256];
902 snprintf (fname, 256, log_pattern.c_str(), m);
903 printf ("opening log file %s\n", fname);
904 optim.logfile = fopen (fname, "w");
905 FAISS_THROW_IF_NOT_FMT (optim.logfile,
906 "could not open logfile %s", fname);
907 }
908
909 std::vector<int> perm (pq.ksub);
910
911 double final_cost = optim.run_optimization (perm.data());
912 printf ("SimulatedAnnealingOptimizer for m=%d: %g -> %g\n",
913 m, optim.init_cost, final_cost);
914
915 if (log_pattern.size()) fclose (optim.logfile);
916
917 float * centroids = pq.get_centroids (m, 0);
918
919 std::vector<float> centroids_copy;
920 for (int i = 0; i < dsub * pq.ksub; i++)
921 centroids_copy.push_back (centroids[i]);
922
923 for (int i = 0; i < pq.ksub; i++)
924 memcpy (centroids + perm[i] * dsub,
925 centroids_copy.data() + i * dsub,
926 dsub * sizeof(centroids[0]));
927
928 }
929
930 }
931
932
933
934 void PolysemousTraining::optimize_pq_for_hamming (ProductQuantizer &pq,
935 size_t n, const float *x) const
936 {
937 if (optimization_type == OT_None) {
938
939 } else if (optimization_type == OT_ReproduceDistances_affine) {
940 optimize_reproduce_distances (pq);
941 } else {
942 optimize_ranking (pq, n, x);
943 }
944
945 pq.compute_sdc_table ();
946
947 }
948
949
950 } // namespace faiss
faiss::RandomGenerator
random generator that can be used in multithreaded contexts
Definition: utils.h:47
faiss::ProductQuantizer::nbits
size_t nbits
number of bits per quantization index
Definition: ProductQuantizer.h:29
faiss::fvec_L2sqr
float fvec_L2sqr(const float *x, const float *y, size_t d)
Squared L2 distance between two vectors.
Definition: utils_simd.cpp:501
faiss::Score3Computer::compute_update
Taccu compute_update(const int *perm, int iw, int jw) const
Definition: PolysemousTraining.cpp:487
faiss::ProductQuantizer::sdc_table
std::vector< float > sdc_table
Symmetric Distance Table.
Definition: ProductQuantizer.h:165
faiss::SimulatedAnnealingOptimizer::SimulatedAnnealingOptimizer
SimulatedAnnealingOptimizer(PermutationObjective *obj, const SimulatedAnnealingParameters &p)
logs values of the cost function
Definition: PolysemousTraining.cpp:68
faiss::ScopeDeleter1
Definition: FaissException.h:58
faiss::SimulatedAnnealingOptimizer::n
int n
size of the permutation
Definition: PolysemousTraining.h:97
faiss::Score3Computer::compute
Taccu compute(const int *perm) const
Definition: PolysemousTraining.cpp:456
faiss::ProductQuantizer::dsub
size_t dsub
dimensionality of each subvector
Definition: ProductQuantizer.h:32
faiss::ProductQuantizer::compute_codes
void compute_codes(const float *x, uint8_t *codes, size_t n) const
same as compute_code for several vectors
Definition: ProductQuantizer.cpp:482
faiss::Score3Computer::update_j_line
Taccu update_j_line(const int *perm, int iw, int jw, int ip0, int ip, int jp0, int jp, const Ttab *n_gt_ij) const
compute update on a line of k&#39;s, where i and j are swapped
Definition: PolysemousTraining.cpp:586
faiss::ReproduceDistancesObjective::target_dis
const double * target_dis
wanted distances (size n^2)
Definition: PolysemousTraining.h:65
faiss::ProductQuantizer::code_size
size_t code_size
byte per indexed vector
Definition: ProductQuantizer.h:33
faiss::SimulatedAnnealingOptimizer::init_cost
double init_cost
remember intial cost of optimization
Definition: PolysemousTraining.h:105
faiss::RandomGenerator::rand_int
int rand_int()
random positive integer
Definition: utils.cpp:114
faiss::ProductQuantizer::ksub
size_t ksub
number of centroids for each subquantizer
Definition: ProductQuantizer.h:34
faiss::PolysemousTraining::optimize_ranking
void optimize_ranking(ProductQuantizer &pq, size_t n, const float *x) const
called by optimize_pq_for_hamming
Definition: PolysemousTraining.cpp:827
faiss::pairwise_L2sqr
void pairwise_L2sqr(long d, long nq, const float *xq, long nb, const float *xb, float *dis, long ldq, long ldb, long ldd)
Definition: utils.cpp:1021
faiss::Score3Computer::compute_cost
double compute_cost(const int *perm) const override
Definition: PolysemousTraining.cpp:632
faiss::Score3Computer
Definition: PolysemousTraining.cpp:444
faiss::getmillisecs
double getmillisecs()
ms elapsed since some arbitrary epoch
Definition: utils.cpp:69
faiss::ReproduceDistancesObjective::weights
std::vector< double > weights
weights for each distance (size n^2)
Definition: PolysemousTraining.h:66
faiss::RankingScore2::accum_gt_weight_diff
double accum_gt_weight_diff(const std::vector< int > &a, const std::vector< int > &b)
Definition: PolysemousTraining.cpp:683
faiss::SimulatedAnnealingParameters
parameters used for the simulated annealing method
Definition: PolysemousTraining.h:21
faiss::IndirectSort
Definition: PolysemousTraining.cpp:648
faiss::Score3Computer::update_i_cross
Taccu update_i_cross(const int *perm, int iw, int jw, int ip0, int ip, const Ttab *n_gt_i) const
considers the 2 pairs of crossing lines j=iw or jw and k = iw or kw
Definition: PolysemousTraining.cpp:607
faiss::ProductQuantizer::M
size_t M
number of subquantizers
Definition: ProductQuantizer.h:28
faiss::PolysemousTraining::OT_ReproduceDistances_affine
default
Definition: PolysemousTraining.h:125
faiss::PermutationObjective
abstract class for the loss function
Definition: PolysemousTraining.h:40
faiss::ProductQuantizer
Definition: ProductQuantizer.h:23
faiss::Score3Computer::update_k
Taccu update_k(const int *perm, int iw, int jw, int ip0, int ip, int jp0, int jp, int k, const Ttab *n_gt_ij) const
used for the 8 cells were the 3 indices are swapped
Definition: PolysemousTraining.cpp:567
faiss::ReproduceDistancesObjective::source_dis
std::vector< double > source_dis
&quot;real&quot; corrected distances (size n^2)
Definition: PolysemousTraining.h:64
faiss::ProductQuantizer::get_centroids
float * get_centroids(size_t m, size_t i)
return the centroids associated with subvector m
Definition: ProductQuantizer.h:58
faiss::PolysemousTraining::optimize_reproduce_distances
void optimize_reproduce_distances(ProductQuantizer &pq) const
called by optimize_pq_for_hamming
Definition: PolysemousTraining.cpp:764
faiss::PolysemousTraining::optimize_pq_for_hamming
void optimize_pq_for_hamming(ProductQuantizer &pq, size_t n, const float *x) const
Definition: PolysemousTraining.cpp:934
faiss::ProductQuantizer::d
size_t d
size of the input vectors
Definition: ProductQuantizer.h:27
faiss::RankingScore2
Definition: PolysemousTraining.cpp:655
faiss::SimulatedAnnealingOptimizer
Simulated annealing optimization algorithm for permutations.
Definition: PolysemousTraining.h:94