source: trunk/lib/carryQ.h @ 890

Last change on this file since 890 was 890, checked in by cameron, 8 years ago

Force Bitblock_... functions inline

File size: 20.7 KB
Line 
1//
2// carryQ.h
3// Robert D. Cameron
4// Dec. 5, 2010 - first queuing implementation
5// November 29, 2010 - first version without actual queueing.
6//
7#ifdef SIMD_CARRY_Q
8#define CARRY_Q
9#endif
10#ifdef ADC64_CARRY_Q
11#define CARRY_Q
12#endif
13
14
15#ifndef CARRY_Q
16#include "block_carry.h"
17
18
19#define CarryQtype CarryType *
20
21#define CarryDeclare(name, count)\
22CarryType name[count];\
23
24#define CarryInit(name, count)\
25for (int j=0; j < count; j++) name[j] = Carry0
26
27static inline BitBlock BitBlock_advance_ci_co(BitBlock strm, CarryQtype cq, const int carryno) __attribute__ ((always_inline));
28
29static inline BitBlock BitBlock_advance_co(BitBlock strm, CarryQtype cq, const int carryno) __attribute__ ((always_inline));
30
31static inline BitBlock BitBlock_add_ci_co(BitBlock strm1, BitBlock strm2, CarryQtype cq, const int carryno) __attribute__ ((always_inline));
32
33static inline BitBlock BitBlock_add_co(BitBlock strm1, BitBlock strm2, CarryQtype cq, const int carryno) __attribute__ ((always_inline));
34
35static inline BitBlock BitBlock_sub_ci_co(BitBlock strm1, BitBlock strm2, CarryQtype cq, const int carryno) __attribute__ ((always_inline));
36
37static inline BitBlock BitBlock_sub_co(BitBlock strm1, BitBlock strm2, CarryQtype cq, const int carryno) __attribute__ ((always_inline));
38
39static inline BitBlock BitBlock_scanthru_ci_co(BitBlock markers0, BitBlock charclass, CarryQtype cq, const int carryno) __attribute__ ((always_inline));
40
41static inline BitBlock BitBlock_scanthru_co(BitBlock markers0, BitBlock charclass, CarryQtype cq, const int carryno) __attribute__ ((always_inline));
42
43static inline bool CarryTest(CarryQtype cq, const int carryno, const int carry_count) __attribute__ ((always_inline));
44
45static inline void CarryDequeueEnqueue(CarryQtype cq, const int carryno, const int carry_count) __attribute__ ((always_inline));
46
47static inline void CarryQ_Adjust(CarryQtype cq, const int carry_count) __attribute__ ((always_inline));
48
49static inline void CarryCombine(CarryQtype cq, CarryQtype local_cq, const int carryno, const int carry_count) __attribute__ ((always_inline));
50
51
52static inline BitBlock BitBlock_advance_ci_co(BitBlock strm, CarryQtype cq, const int carryno) {
53 BitBlock rslt;
54 advance_with_carry(strm, cq[carryno], rslt);
55 return rslt;
56}
57
58static inline BitBlock BitBlock_advance_co(BitBlock strm, CarryQtype cq, const int carryno) {
59 BitBlock rslt;
60 cq[carryno] = Carry0;
61 advance_with_carry(strm, cq[carryno], rslt);
62 return rslt;
63}
64
65static inline BitBlock BitBlock_add_ci_co(BitBlock strm1, BitBlock strm2, CarryQtype cq, const int carryno) {
66 BitBlock sum;
67 adc128(strm1, strm2, cq[carryno], sum);
68 return sum;
69}
70
71static inline BitBlock BitBlock_add_co(BitBlock strm1, BitBlock strm2, CarryQtype cq, const int carryno) {
72 BitBlock sum;
73 cq[carryno] = Carry0;
74 adc128(strm1, strm2, cq[carryno], sum);
75 return sum;
76}
77
78static inline BitBlock BitBlock_sub_ci_co(BitBlock strm1, BitBlock strm2, CarryQtype cq, const int carryno) {
79 BitBlock diff;
80 sbb128(strm1, strm2, cq[carryno], diff);
81 return diff;
82}
83
84static inline BitBlock BitBlock_sub_co(BitBlock strm1, BitBlock strm2, CarryQtype cq, const int carryno) {
85 BitBlock diff;
86 cq[carryno] = Carry0;
87 sbb128(strm1, strm2, cq[carryno], diff);
88 return diff;
89}
90
91static inline BitBlock BitBlock_scanthru_ci_co(BitBlock markers0, BitBlock charclass, CarryQtype cq, const int carryno) {
92 BitBlock markers1;
93 adc128(markers0, charclass, cq[carryno], markers1);
94 return simd_andc(markers1, charclass);
95}
96
97static inline BitBlock BitBlock_scanthru_co(BitBlock markers0, BitBlock charclass, CarryQtype cq, const int carryno) {
98 BitBlock markers1;
99 cq[carryno] = Carry0;
100 adc128(markers0, charclass, cq[carryno], markers1);
101 return simd_andc(markers1, charclass);
102}
103
104static inline bool CarryTest(CarryQtype cq, const int carryno, const int carry_count) {
105  CarryType c1 = cq[carryno];
106  int i;
107  for (i = carryno + 1; i < carryno + carry_count; i++) {
108    c1 = carry_or(c1, cq[i]);
109  }
110  return test_carry(c1);
111}
112
113static inline void CarryDequeueEnqueue(CarryQtype cq, const int carryno, const int carry_count) {
114  // Given carryin queue with carry_count carries starting from carryno are 0,
115  // ensure that the carryout queue has carry_count carries starting from carryno set to 0.
116  // Nothing to do when the queues are the same!
117  return;
118}
119
120static inline void CarryQ_Adjust(CarryQtype cq, const int carry_count) {
121  // Adjust the carryQ so that carries enqueued are readied for dequeiing.
122  // Nothing to do with indexed queues.
123  return;
124}
125
126static inline void CarryCombine(CarryQtype cq, CarryQtype local_cq, const int carryno, const int carry_count) {
127  int i;
128  for (i = 0; i < carry_count; i++) {
129    cq[carryno+i] = carry_or(cq[carryno+i], local_cq[i]);
130  }
131}
132#endif
133#ifdef SIMD_CARRY_Q
134
135#define CarryQtype SIMD_type
136
137#define CarryDeclare(name, count)\
138  CarryQtype name
139
140#define CarryInit(name, count)\
141  name = simd_const_1(0)
142
143SIMD_type carryQ_ci_mask = sisd_from_int(1);
144SIMD_type carryQ_co_mask = sisd_slli(carryQ_ci_mask, 127);
145
146#define carryQ_adc128_ci_co(x, y, carryQ,  sum) \
147do{ \
148  SIMD_type gen = simd_and(x, y); \
149  SIMD_type prop = simd_or(x, y); \
150  SIMD_type partial = simd_add_64(simd_add_64(x, y), simd_and(carryQ, carryQ_ci_mask)); \
151  carryQ = simd_srli_64(carryQ, 1); \
152  SIMD_type c1 = sisd_slli(simd_srli_64(simd_or(gen, simd_andc(prop, partial)), 63), 64); \
153  sum = simd_add_64(c1, partial); \
154  carryQ = simd_or(carryQ, simd_and(simd_or(gen, simd_andc(prop, sum)), carryQ_co_mask)); \
155} while(0)
156
157#define carryQ_adc128_co(x, y, carryQ,  sum) \
158do{ \
159  SIMD_type gen = simd_and(x, y); \
160  SIMD_type prop = simd_or(x, y); \
161  SIMD_type partial = simd_add_64(x, y); \
162  carryQ = simd_srli_64(carryQ, 1); \
163  SIMD_type c1 = sisd_slli(simd_srli_64(simd_or(gen, simd_andc(prop, partial)), 63), 64); \
164  sum = simd_add_64(c1, partial); \
165  carryQ = simd_or(carryQ, simd_and(simd_or(gen, simd_andc(prop, sum)), carryQ_co_mask)); \
166} while(0)
167
168#define carryQ_sbb128_ci_co(x, y, carryQ, difference) \
169do {\
170  SIMD_type gen = simd_andc(y, x); \
171  SIMD_type prop = simd_not(simd_xor(x, y)); \
172  SIMD_type partial = simd_sub_64(simd_sub_64(x, y), simd_and(carryQ, carryQ_ci_mask)); \
173  carryQ = simd_srli_64(carryQ, 1); \
174  SIMD_type b1 = sisd_slli(simd_srli_64(simd_or(gen, simd_and(prop, partial)), 63), 64); \
175  difference = simd_sub_64(partial, b1); \
176  carryQ = simd_or(carryQ, simd_and(simd_or(gen, simd_and(prop, difference)), carryQ_co_mask)); \
177}while(0)
178
179#define carryQ_sbb128_co(x, y, carryQ, difference) \
180do {\
181  SIMD_type gen = simd_andc(y, x); \
182  SIMD_type prop = simd_not(simd_xor(x, y)); \
183  SIMD_type partial = simd_sub_64(x, y); \
184  carryQ = simd_srli_64(carryQ, 1); \
185  SIMD_type b1 = sisd_slli(simd_srli_64(simd_or(gen, simd_and(prop, partial)), 63), 64); \
186  difference = simd_sub_64(partial, b1); \
187  carryQ = simd_or(carryQ, simd_and(simd_or(gen, simd_and(prop, difference)), carryQ_co_mask)); \
188}while(0)
189
190#define carryQ_advance_with_carry_ci_co(cursor, carryQ, rslt)\
191do {\
192  SIMD_type carry_out = simd_and(cursor, carryQ_co_mask);\
193  SIMD_type carry_in = simd_and(carryQ, carryQ_ci_mask);\
194  carryQ = simd_or(simd_srli_64(carryQ, 1), carry_out); \
195  SIMD_type shift_out = simd_srli_64(cursor, 63);\
196  SIMD_type low_bits = simd_mergel_64(shift_out, carry_in);\
197  rslt = simd_or(simd_add_64(cursor, cursor), low_bits);\
198} while(0)
199
200#define carryQ_advance_with_carry_co(cursor, carryQ, rslt)\
201do {\
202  SIMD_type carry_out = simd_and(cursor, carryQ_co_mask);\
203  carryQ = simd_or(simd_srli_64(carryQ, 1), carry_out); \
204  SIMD_type shift_out = simd_srli_64(cursor, 63);\
205  SIMD_type low_bits = simd_mergel_64(shift_out, simd_const_1(0));\
206  rslt = simd_or(simd_add_64(cursor, cursor), low_bits);\
207} while(0)
208
209
210static inline BitBlock BitBlock_advance_ci_co(BitBlock strm, CarryQtype cq, const int carryno) __attribute__ ((always_inline));
211
212static inline BitBlock BitBlock_advance_co(BitBlock strm, CarryQtype cq, const int carryno) __attribute__ ((always_inline));
213
214static inline BitBlock BitBlock_add_ci_co(BitBlock strm1, BitBlock strm2, CarryQtype cq, const int carryno) __attribute__ ((always_inline));
215
216static inline BitBlock BitBlock_add_co(BitBlock strm1, BitBlock strm2, CarryQtype cq, const int carryno) __attribute__ ((always_inline));
217
218static inline BitBlock BitBlock_sub_ci_co(BitBlock strm1, BitBlock strm2, CarryQtype cq, const int carryno) __attribute__ ((always_inline));
219
220static inline BitBlock BitBlock_sub_co(BitBlock strm1, BitBlock strm2, CarryQtype cq, const int carryno) __attribute__ ((always_inline));
221
222static inline BitBlock BitBlock_scanthru_ci_co(BitBlock markers0, BitBlock charclass, CarryQtype cq, const int carryno) __attribute__ ((always_inline));
223
224static inline BitBlock BitBlock_scanthru_co(BitBlock markers0, BitBlock charclass, CarryQtype cq, const int carryno) __attribute__ ((always_inline));
225
226static inline bool CarryTest(CarryQtype cq, const int carryno, const int carry_count) __attribute__ ((always_inline));
227
228static inline void CarryDequeueEnqueue(CarryQtype cq, const int carryno, const int carry_count) __attribute__ ((always_inline));
229
230static inline void CarryQ_Adjust(CarryQtype cq, const int carry_count) __attribute__ ((always_inline));
231
232static inline void CarryCombine(CarryQtype cq, CarryQtype local_cq, const int carryno, const int carry_count) __attribute__ ((always_inline));
233
234
235static inline BitBlock BitBlock_advance_ci_co(BitBlock strm, CarryQtype & cq, const int carryno) {
236 BitBlock rslt;
237 carryQ_advance_with_carry_ci_co(strm, cq, rslt);
238 return rslt;
239}
240
241static inline BitBlock BitBlock_advance_co(BitBlock strm, CarryQtype & cq, const int carryno) {
242 BitBlock rslt;
243 carryQ_advance_with_carry_co(strm, cq, rslt);
244 return rslt;
245}
246
247static inline BitBlock BitBlock_add_ci_co(BitBlock strm1, BitBlock strm2, CarryQtype & cq, const int carryno) {
248 BitBlock sum;
249 carryQ_adc128_ci_co(strm1, strm2, cq, sum);
250 return sum;
251}
252
253static inline BitBlock BitBlock_add_co(BitBlock strm1, BitBlock strm2, CarryQtype & cq, const int carryno) {
254 BitBlock sum;
255 carryQ_adc128_co(strm1, strm2, cq, sum);
256 return sum;
257}
258
259static inline BitBlock BitBlock_sub_ci_co(BitBlock strm1, BitBlock strm2, CarryQtype & cq, const int carryno) {
260 BitBlock diff;
261 carryQ_sbb128_ci_co(strm1, strm2, cq, diff);
262 return diff;
263}
264
265static inline BitBlock BitBlock_sub_co(BitBlock strm1, BitBlock strm2, CarryQtype & cq, const int carryno) {
266 BitBlock diff;
267 carryQ_sbb128_co(strm1, strm2, cq, diff);
268 return diff;
269}
270
271static inline BitBlock BitBlock_scanthru_ci_co(BitBlock markers0, BitBlock charclass, CarryQtype & cq, const int carryno) {
272 BitBlock markers1;
273 carryQ_adc128_ci_co(markers0, charclass, cq, markers1);
274 return simd_andc(markers1, charclass);
275}
276
277static inline BitBlock BitBlock_scanthru_co(BitBlock markers0, BitBlock charclass, CarryQtype & cq, const int carryno) {
278 BitBlock markers1;
279 carryQ_adc128_co(markers0, charclass, cq, markers1);
280 return simd_andc(markers1, charclass);
281}
282
283typedef union {SIMD_type bitblock; uint64_t int64[2];} BitBlock_int64;
284
285static inline bool CarryTest(CarryQtype cq, const int carryno, const int carry_count) {
286  BitBlock_int64 t;
287  t.bitblock = cq;
288  uint64_t carryQ_top_N_mask = ((1 << carry_count) -1);
289  return t.int64[0] & carryQ_top_N_mask;
290}
291
292static inline void CarryDequeueEnqueue(CarryQtype & cq, const int carryno, const int carry_count) {
293  // Given carryin queue with carry_count carries starting from carryno are 0,
294  // ensure that the carryout queue has carry_count carries starting from carryno set to 0.
295  cq = sisd_srli(cq, carry_count);
296}
297
298static inline void CarryCombine(CarryQtype & cq, CarryQtype local_cq, const int carryno, const int carry_count) {
299  cq = simd_or(cq, local_cq);
300}
301
302static inline void CarryQ_Adjust(CarryQtype & cq, const int carry_count) {
303  // Adjust the carryQ so that carries enqueued are readied for dequeiing.
304  cq = sisd_srli(cq, (128-carry_count));
305}
306
307
308#endif
309
310#ifdef ADC64_CARRY_Q
311
312//
313// CarryQueue implementation using 64-bit integer queues.
314// A single 64-bit integer holds both the input and output
315// carries, with bits moving right-to-left.   Thus the
316// high bit in the queue is always the next carry to be
317// dequeued; a newly enqueued carry is always inserted as
318// the low bit.
319//
320// The two typical operations for dequeueing and enqueueing
321// carryies from/to a CarryQueue cq are the following.
322// 1.  Dequeueing:  add(cq, cq)
323//     The high carry bit is dequeued and sets the processor
324//     carry flag to be used as a carry-in variable in the
325//     following bitblock operation.   This also shifts cq
326//     right one position, making room for enqueuing a new carry.
327// 2.  Enqueueing:  adc($0, cq)
328//     The carry out value of an operation as recorded in the
329//     processor carry flag is enqueued by adding it in to the
330//     low bit position of cq (this bit will have been cleared
331//     by the dequeue operation.
332
333#define CarryQtype uint64_t
334
335#define CarryDeclare(name, count)\
336CarryQtype name
337
338#define CarryInit(name, count)\
339name = 0
340
341typedef union {SIMD_type bitblock; uint64_t int64[2];} BitBlock_int64;
342
343#define double_int64_adc_ci_co(x1, x2, y1, y2, rslt1, rslt2, carryQ) \
344   __asm__ __volatile__ ("add %[cq], %[cq]\n\t" \
345         "adc %[e1], %[z1]\n\t" \
346         "adc %[e2], %[z2]\n\t" \
347         "adc $0, %[cq]\n\t" \
348     : [z1] "=r" (rslt1), [z2] "=r" (rslt2), [cq] "=r" (carryQ) \
349         : "[z1]" (x1), "[z2]" (x2), \
350           [e1] "r" (y1), [e2] "r" (y2), \
351           "[cq]" (carryQ) \
352         : "cc")
353
354
355#define carryQ_adc128_ci_co(first, second, carryQ, sum) \
356do {\
357  BitBlock_int64 rslt, x, y;\
358  x.bitblock = first;\
359  y.bitblock = second;\
360  double_int64_adc_ci_co(x.int64[0], x.int64[1], y.int64[0], y.int64[1], rslt.int64[0], rslt.int64[1], carryQ);\
361  sum = rslt.bitblock;\
362} while(0)
363
364#define double_int64_adc_co(x1, x2, y1, y2, rslt1, rslt2, carryQ) \
365   __asm__ __volatile__ ("add %[cq], %[cq]\n\t" \
366         "add %[e1], %[z1]\n\t" \
367         "adc %[e2], %[z2]\n\t" \
368         "adc $0, %[cq]\n\t" \
369     : [z1] "=r" (rslt1), [z2] "=r" (rslt2), [cq] "=r" (carryQ) \
370         : "[z1]" (x1), "[z2]" (x2), \
371           [e1] "r" (y1), [e2] "r" (y2), \
372           "[cq]" (carryQ) \
373         : "cc")
374
375
376#define carryQ_adc128_co(first, second, carryQ, sum) \
377do {\
378  BitBlock_int64 rslt, x, y;\
379  x.bitblock = first;\
380  y.bitblock = second;\
381  double_int64_adc_co(x.int64[0], x.int64[1], y.int64[0], y.int64[1], rslt.int64[0], rslt.int64[1], carryQ);\
382  sum = rslt.bitblock;\
383} while(0)
384
385
386#define double_int64_sbb_ci_co(x1, x2, y1, y2, rslt1, rslt2, brwQ) \
387  __asm__  __volatile__ ("add %[cq], %[cq]\n\t" \
388        "sbb %[e1], %[z1]\n\t" \
389        "sbb %[e2], %[z2]\n\t" \
390        "adc $0, %[cq]\n\t" \
391     : [z1] "=r" (rslt1), [z2] "=r" (rslt2), [cq] "=r" (brwQ) \
392         : "[z1]" (x1), "[z2]" (x2), \
393           [e1] "r" (y1), [e2] "r" (y2), \
394           "[cq]" (brwQ) \
395         : "cc")
396
397#define carryQ_sbb128_ci_co(first, second, borrowQ, diff) \
398do {\
399  BitBlock_int64 rslt, x, y;\
400  x.bitblock = first;\
401  y.bitblock = second;\
402  double_int64_sbb_ci_co(x.int64[0], x.int64[1], y.int64[0], y.int64[1], \
403                   rslt.int64[0], rslt.int64[1], borrowQ);\
404  diff = rslt.bitblock;\
405} while(0)
406
407#define double_int64_sbb_co(x1, x2, y1, y2, rslt1, rslt2, brwQ) \
408  __asm__  __volatile__ ("add %[cq], %[cq]\n\t" \
409        "sub %[e1], %[z1]\n\t" \
410        "sbb %[e2], %[z2]\n\t" \
411        "adc $0, %[cq]\n\t" \
412     : [z1] "=r" (rslt1), [z2] "=r" (rslt2), [cq] "=r" (brwQ) \
413         : "[z1]" (x1), "[z2]" (x2), \
414           [e1] "r" (y1), [e2] "r" (y2), \
415           "[cq]" (brwQ) \
416         : "cc")
417
418#define carryQ_sbb128_co(first, second, borrowQ, diff) \
419do {\
420  BitBlock_int64 rslt, x, y;\
421  x.bitblock = first;\
422  y.bitblock = second;\
423  double_int64_sbb_co(x.int64[0], x.int64[1], y.int64[0], y.int64[1], \
424                   rslt.int64[0], rslt.int64[1], borrowQ);\
425  diff = rslt.bitblock;\
426} while(0)
427
428#define double_int64_advance_ci_co(x1, x2, rslt1, rslt2, carryQ) \
429  __asm__  __volatile__ ("add %[cq], %[cq]\n\t" \
430        "adc %[z1], %[z1]\n\t" \
431        "adc %[z2], %[z2]\n\t" \
432        "adc $0, %[cq]\n\t" \
433         : [z1] "=r" (rslt1), [z2] "=r" (rslt2), [cq] "=r" (carryQ) \
434         : "[z1]" (x1), "[z2]" (x2), \
435           "[cq]" (carryQ) \
436         : "cc")
437
438#define carryQ_advance_with_carry_ci_co(cursor, carryQ, rslt)\
439do {\
440  BitBlock_int64 x, z;\
441  x.bitblock = cursor;\
442  double_int64_advance_ci_co(x.int64[0], x.int64[1], z.int64[0], z.int64[1], carryQ);\
443  rslt = z.bitblock;\
444} while(0)
445
446
447#define double_int64_advance_co(x1, x2, rslt1, rslt2, carryQ) \
448  __asm__  __volatile__ ("add %[cq], %[cq]\n\t" \
449        "add %[z1], %[z1]\n\t" \
450        "adc %[z2], %[z2]\n\t" \
451        "adc $0, %[cq]\n\t" \
452         : [z1] "=r" (rslt1), [z2] "=r" (rslt2), [cq] "=r" (carryQ) \
453         : "[z1]" (x1), "[z2]" (x2), \
454           "[cq]" (carryQ) \
455         : "cc")
456
457#define carryQ_advance_with_carry_co(cursor, carryQ, rslt)\
458do {\
459  BitBlock_int64 x, z;\
460  x.bitblock = cursor;\
461  double_int64_advance_co(x.int64[0], x.int64[1], z.int64[0], z.int64[1], carryQ);\
462  rslt = z.bitblock;\
463} while(0)
464
465static inline BitBlock BitBlock_advance_ci_co(BitBlock strm, CarryQtype cq, const int carryno) __attribute__ ((always_inline));
466
467static inline BitBlock BitBlock_advance_co(BitBlock strm, CarryQtype cq, const int carryno) __attribute__ ((always_inline));
468
469static inline BitBlock BitBlock_add_ci_co(BitBlock strm1, BitBlock strm2, CarryQtype cq, const int carryno) __attribute__ ((always_inline));
470
471static inline BitBlock BitBlock_add_co(BitBlock strm1, BitBlock strm2, CarryQtype cq, const int carryno) __attribute__ ((always_inline));
472
473static inline BitBlock BitBlock_sub_ci_co(BitBlock strm1, BitBlock strm2, CarryQtype cq, const int carryno) __attribute__ ((always_inline));
474
475static inline BitBlock BitBlock_sub_co(BitBlock strm1, BitBlock strm2, CarryQtype cq, const int carryno) __attribute__ ((always_inline));
476
477static inline BitBlock BitBlock_scanthru_ci_co(BitBlock markers0, BitBlock charclass, CarryQtype cq, const int carryno) __attribute__ ((always_inline));
478
479static inline BitBlock BitBlock_scanthru_co(BitBlock markers0, BitBlock charclass, CarryQtype cq, const int carryno) __attribute__ ((always_inline));
480
481static inline bool CarryTest(CarryQtype cq, const int carryno, const int carry_count) __attribute__ ((always_inline));
482
483static inline void CarryDequeueEnqueue(CarryQtype cq, const int carryno, const int carry_count) __attribute__ ((always_inline));
484
485static inline void CarryQ_Adjust(CarryQtype cq, const int carry_count) __attribute__ ((always_inline));
486
487static inline void CarryCombine(CarryQtype cq, CarryQtype local_cq, const int carryno, const int carry_count) __attribute__ ((always_inline));
488
489
490static inline BitBlock BitBlock_advance_ci_co(BitBlock strm, CarryQtype & cq, const int carryno) {
491        BitBlock rslt;
492        carryQ_advance_with_carry_ci_co(strm, cq, rslt);
493        return rslt;
494}
495
496static inline BitBlock BitBlock_advance_co(BitBlock strm, CarryQtype & cq, const int carryno) {
497        BitBlock rslt;
498        carryQ_advance_with_carry_co(strm, cq, rslt);
499        return rslt;
500}
501
502static inline BitBlock BitBlock_add_ci_co(BitBlock strm1, BitBlock strm2, CarryQtype & cq, const int carryno) {
503        BitBlock sum;
504        carryQ_adc128_ci_co(strm1, strm2, cq, sum);
505        return sum;
506}
507
508static inline BitBlock BitBlock_add_co(BitBlock strm1, BitBlock strm2, CarryQtype & cq, const int carryno) {
509        BitBlock sum;
510        carryQ_adc128_co(strm1, strm2, cq, sum);
511        return sum;
512}
513
514static inline BitBlock BitBlock_sub_ci_co(BitBlock strm1, BitBlock strm2, CarryQtype & cq, const int carryno) {
515        BitBlock diff;
516        carryQ_sbb128_ci_co(strm1, strm2, cq, diff);
517        return diff;
518}
519
520static inline BitBlock BitBlock_sub_co(BitBlock strm1, BitBlock strm2, CarryQtype & cq, const int carryno) {
521        BitBlock diff;
522        carryQ_sbb128_co(strm1, strm2, cq, diff);
523        return diff;
524}
525
526static inline BitBlock BitBlock_scanthru_ci_co(BitBlock markers0, BitBlock charclass, CarryQtype & cq, const int carryno) {
527        BitBlock markers1;
528        carryQ_adc128_ci_co(markers0, charclass, cq, markers1);
529        return simd_andc(markers1, charclass);
530}
531
532static inline BitBlock BitBlock_scanthru_co(BitBlock markers0, BitBlock charclass, CarryQtype & cq, const int carryno) {
533        BitBlock markers1;
534        carryQ_adc128_co(markers0, charclass, cq, markers1);
535        return simd_andc(markers1, charclass);
536}
537
538static inline bool CarryTest(CarryQtype cq, const int carryno, const int carry_count) {
539//         print_general_register_64("cq", cq);
540        uint64_t carryQ_top_N_mask = ~(0xFFFFFFFFFFFFFFFFULL >> carry_count);
541        return (cq & carryQ_top_N_mask) != 0;
542}
543
544static inline void CarryDequeueEnqueue(CarryQtype & cq, const int carryno, const int carry_count) {
545        // Given carryin queue with carry_count carries starting from carryno are 0,
546        // ensure that the carryout queue has carry_count carries starting from carryno set to 0.
547        cq <<= carry_count;
548}
549
550static inline void CarryCombine(CarryQtype & cq, CarryQtype local_cq, const int carryno, const int carry_count) {
551        cq |= local_cq;
552}
553
554static inline void CarryQ_Adjust(CarryQtype & cq, int total_carries) {
555        // Adjust the carryQ so that carries enqueued are readied for dequeiing.
556        cq <<= (64-total_carries);
557}
558
559
560#endif
Note: See TracBrowser for help on using the repository browser.