Barretenberg
The ZK-SNARK library at the core of Aztec
Loading...
Searching...
No Matches
field_impl_x64.hpp
Go to the documentation of this file.
1// === AUDIT STATUS ===
2// internal: { status: Completed, auditors: [Raju], commit: }
3// external_1: { status: not started, auditors: [], commit: }
4// external_2: { status: not started, auditors: [], commit: }
5// =====================
6
7#pragma once
8
9#if (BBERG_NO_ASM == 0)
10#include "./field_impl.hpp"
11#include "asm_macros.hpp"
12namespace bb {
13
14template <class T> field<T> field<T>::asm_mul_with_coarse_reduction(const field& a, const field& b) noexcept
15{
16 field r;
17 constexpr uint64_t r_inv = T::r_inv;
18 constexpr uint64_t modulus_0 = modulus.data[0];
19 constexpr uint64_t modulus_1 = modulus.data[1];
20 constexpr uint64_t modulus_2 = modulus.data[2];
21 constexpr uint64_t modulus_3 = modulus.data[3];
22 constexpr uint64_t zero_ref = 0;
23
33 __asm__(MUL("0(%0)", "8(%0)", "16(%0)", "24(%0)", "%1")
34 STORE_FIELD_ELEMENT("%2", "%%r12", "%%r13", "%%r14", "%%r15")
35 :
36 : "%r"(&a),
37 "%r"(&b),
38 "r"(&r),
39 [modulus_0] "m"(modulus_0),
40 [modulus_1] "m"(modulus_1),
41 [modulus_2] "m"(modulus_2),
42 [modulus_3] "m"(modulus_3),
43 [r_inv] "m"(r_inv),
44 [zero_reference] "m"(zero_ref)
45 : "%rdx", "%rdi", "%r8", "%r9", "%r10", "%r11", "%r12", "%r13", "%r14", "%r15", "cc", "memory");
46 return r;
47}
48
49template <class T> void field<T>::asm_self_mul_with_coarse_reduction(field& a, const field& b) noexcept
50{
51 constexpr uint64_t r_inv = T::r_inv;
52 constexpr uint64_t modulus_0 = modulus.data[0];
53 constexpr uint64_t modulus_1 = modulus.data[1];
54 constexpr uint64_t modulus_2 = modulus.data[2];
55 constexpr uint64_t modulus_3 = modulus.data[3];
56 constexpr uint64_t zero_ref = 0;
66 __asm__(MUL("0(%0)", "8(%0)", "16(%0)", "24(%0)", "%1")
67 STORE_FIELD_ELEMENT("%0", "%%r12", "%%r13", "%%r14", "%%r15")
68 :
69 : "r"(&a),
70 "r"(&b),
71 [modulus_0] "m"(modulus_0),
72 [modulus_1] "m"(modulus_1),
73 [modulus_2] "m"(modulus_2),
74 [modulus_3] "m"(modulus_3),
75 [r_inv] "m"(r_inv),
76 [zero_reference] "m"(zero_ref)
77 : "%rdx", "%rdi", "%r8", "%r9", "%r10", "%r11", "%r12", "%r13", "%r14", "%r15", "cc", "memory");
78}
79
80template <class T> field<T> field<T>::asm_sqr_with_coarse_reduction(const field& a) noexcept
81{
82 field r;
83 constexpr uint64_t r_inv = T::r_inv;
84 constexpr uint64_t modulus_0 = modulus.data[0];
85 constexpr uint64_t modulus_1 = modulus.data[1];
86 constexpr uint64_t modulus_2 = modulus.data[2];
87 constexpr uint64_t modulus_3 = modulus.data[3];
88 constexpr uint64_t zero_ref = 0;
89
90// Our SQR implementation with BMI2 but without ADX has a bug.
91// The case is extremely rare so fixing it is a bit of a waste of time.
92// We'll use MUL instead.
93#if !defined(__ADX__) || defined(DISABLE_ADX)
103 __asm__(MUL("0(%0)", "8(%0)", "16(%0)", "24(%0)", "%1")
104 STORE_FIELD_ELEMENT("%2", "%%r12", "%%r13", "%%r14", "%%r15")
105 :
106 : "%r"(&a),
107 "%r"(&a),
108 "r"(&r),
109 [modulus_0] "m"(modulus_0),
110 [modulus_1] "m"(modulus_1),
111 [modulus_2] "m"(modulus_2),
112 [modulus_3] "m"(modulus_3),
113 [r_inv] "m"(r_inv),
114 [zero_reference] "m"(zero_ref)
115 : "%rdx", "%rdi", "%r8", "%r9", "%r10", "%r11", "%r12", "%r13", "%r14", "%r15", "cc", "memory");
116
117#else
118
127 __asm__(SQR("%0")
128 // "movq %[r_ptr], %%rsi \n\t"
129 STORE_FIELD_ELEMENT("%1", "%%r12", "%%r13", "%%r14", "%%r15")
130 :
131 : "r"(&a),
132 "r"(&r),
133 [zero_reference] "m"(zero_ref),
134 [modulus_0] "m"(modulus_0),
135 [modulus_1] "m"(modulus_1),
136 [modulus_2] "m"(modulus_2),
137 [modulus_3] "m"(modulus_3),
138 [r_inv] "m"(r_inv)
139 : "%rcx", "%rdx", "%rdi", "%r8", "%r9", "%r10", "%r11", "%r12", "%r13", "%r14", "%r15", "cc", "memory");
140#endif
141 return r;
142}
143
144template <class T> void field<T>::asm_self_sqr_with_coarse_reduction(field& a) noexcept
145{
146 constexpr uint64_t r_inv = T::r_inv;
147 constexpr uint64_t modulus_0 = modulus.data[0];
148 constexpr uint64_t modulus_1 = modulus.data[1];
149 constexpr uint64_t modulus_2 = modulus.data[2];
150 constexpr uint64_t modulus_3 = modulus.data[3];
151 constexpr uint64_t zero_ref = 0;
152
153// Our SQR implementation with BMI2 but without ADX has a bug.
154// The case is extremely rare so fixing it is a bit of a waste of time.
155// We'll use MUL instead.
156#if !defined(__ADX__) || defined(DISABLE_ADX)
166 __asm__(MUL("0(%0)", "8(%0)", "16(%0)", "24(%0)", "%1")
167 STORE_FIELD_ELEMENT("%0", "%%r12", "%%r13", "%%r14", "%%r15")
168 :
169 : "r"(&a),
170 "r"(&a),
171 [modulus_0] "m"(modulus_0),
172 [modulus_1] "m"(modulus_1),
173 [modulus_2] "m"(modulus_2),
174 [modulus_3] "m"(modulus_3),
175 [r_inv] "m"(r_inv),
176 [zero_reference] "m"(zero_ref)
177 : "%rdx", "%rdi", "%r8", "%r9", "%r10", "%r11", "%r12", "%r13", "%r14", "%r15", "cc", "memory");
178
179#else
188 __asm__(SQR("%0")
189 // "movq %[r_ptr], %%rsi \n\t"
190 STORE_FIELD_ELEMENT("%0", "%%r12", "%%r13", "%%r14", "%%r15")
191 :
192 : "r"(&a),
193 [zero_reference] "m"(zero_ref),
194 [modulus_0] "m"(modulus_0),
195 [modulus_1] "m"(modulus_1),
196 [modulus_2] "m"(modulus_2),
197 [modulus_3] "m"(modulus_3),
198 [r_inv] "m"(r_inv)
199 : "%rcx", "%rdx", "%rdi", "%r8", "%r9", "%r10", "%r11", "%r12", "%r13", "%r14", "%r15", "cc", "memory");
200#endif
201}
202
203template <class T> field<T> field<T>::asm_add_with_coarse_reduction(const field& a, const field& b) noexcept
204{
205 field r;
206
207 constexpr uint64_t twice_not_modulus_0 = twice_not_modulus.data[0];
208 constexpr uint64_t twice_not_modulus_1 = twice_not_modulus.data[1];
209 constexpr uint64_t twice_not_modulus_2 = twice_not_modulus.data[2];
210 constexpr uint64_t twice_not_modulus_3 = twice_not_modulus.data[3];
211
212 __asm__(CLEAR_FLAGS("%%r12") LOAD_FIELD_ELEMENT("%0", "%%r12", "%%r13", "%%r14", "%%r15")
213 ADD_REDUCE("%1",
214 "%[twice_not_modulus_0]",
215 "%[twice_not_modulus_1]",
216 "%[twice_not_modulus_2]",
217 "%[twice_not_modulus_3]") STORE_FIELD_ELEMENT("%2", "%%r12", "%%r13", "%%r14", "%%r15")
218 :
219 : "%r"(&a),
220 "%r"(&b),
221 "r"(&r),
222 [twice_not_modulus_0] "m"(twice_not_modulus_0),
223 [twice_not_modulus_1] "m"(twice_not_modulus_1),
224 [twice_not_modulus_2] "m"(twice_not_modulus_2),
225 [twice_not_modulus_3] "m"(twice_not_modulus_3)
226 : "%r8", "%r9", "%r10", "%r11", "%r12", "%r13", "%r14", "%r15", "cc", "memory");
227 return r;
228}
229
230template <class T> void field<T>::asm_self_add_with_coarse_reduction(field& a, const field& b) noexcept
231{
232 constexpr uint64_t twice_not_modulus_0 = twice_not_modulus.data[0];
233 constexpr uint64_t twice_not_modulus_1 = twice_not_modulus.data[1];
234 constexpr uint64_t twice_not_modulus_2 = twice_not_modulus.data[2];
235 constexpr uint64_t twice_not_modulus_3 = twice_not_modulus.data[3];
236
237 __asm__(CLEAR_FLAGS("%%r12") LOAD_FIELD_ELEMENT("%0", "%%r12", "%%r13", "%%r14", "%%r15")
238 ADD_REDUCE("%1",
239 "%[twice_not_modulus_0]",
240 "%[twice_not_modulus_1]",
241 "%[twice_not_modulus_2]",
242 "%[twice_not_modulus_3]") STORE_FIELD_ELEMENT("%0", "%%r12", "%%r13", "%%r14", "%%r15")
243 :
244 : "r"(&a),
245 "r"(&b),
246 [twice_not_modulus_0] "m"(twice_not_modulus_0),
247 [twice_not_modulus_1] "m"(twice_not_modulus_1),
248 [twice_not_modulus_2] "m"(twice_not_modulus_2),
249 [twice_not_modulus_3] "m"(twice_not_modulus_3)
250 : "%r8", "%r9", "%r10", "%r11", "%r12", "%r13", "%r14", "%r15", "cc", "memory");
251}
252
253template <class T> field<T> field<T>::asm_sub_with_coarse_reduction(const field& a, const field& b) noexcept
254{
255 field r;
256
257 constexpr uint64_t twice_modulus_0 = twice_modulus.data[0];
258 constexpr uint64_t twice_modulus_1 = twice_modulus.data[1];
259 constexpr uint64_t twice_modulus_2 = twice_modulus.data[2];
260 constexpr uint64_t twice_modulus_3 = twice_modulus.data[3];
261
262 /* Note: the ADX CONDITIONAL_ADD reads OF via adoxq, but OF is guaranteed to be 0 here. */
263 /* SUB's sbbq sets OF on signed overflow, which cannot occur for field elements in [0,2p) */
264 /* because the top limb is always < 2^63 (positive in signed interpretation). */
265 __asm__(CLEAR_FLAGS("%%r12") LOAD_FIELD_ELEMENT("%0", "%%r12", "%%r13", "%%r14", "%%r15") SUB("%1")
266 CONDITIONAL_ADD("%[twice_modulus_0]", "%[twice_modulus_1]", "%[twice_modulus_2]", "%[twice_modulus_3]")
267 STORE_FIELD_ELEMENT("%2", "%%r12", "%%r13", "%%r14", "%%r15")
268 :
269 : "r"(&a),
270 "r"(&b),
271 "r"(&r),
272 [twice_modulus_0] "m"(twice_modulus_0),
273 [twice_modulus_1] "m"(twice_modulus_1),
274 [twice_modulus_2] "m"(twice_modulus_2),
275 [twice_modulus_3] "m"(twice_modulus_3)
276 : "%r8", "%r9", "%r10", "%r11", "%r12", "%r13", "%r14", "%r15", "cc", "memory");
277 return r;
278}
279
280template <class T> void field<T>::asm_self_sub_with_coarse_reduction(field& a, const field& b) noexcept
281{
282 constexpr uint64_t twice_modulus_0 = twice_modulus.data[0];
283 constexpr uint64_t twice_modulus_1 = twice_modulus.data[1];
284 constexpr uint64_t twice_modulus_2 = twice_modulus.data[2];
285 constexpr uint64_t twice_modulus_3 = twice_modulus.data[3];
286
287 /* See comment in asm_sub_with_coarse_reduction: OF is guaranteed 0 after SUB. */
288 __asm__(CLEAR_FLAGS("%%r12") LOAD_FIELD_ELEMENT("%0", "%%r12", "%%r13", "%%r14", "%%r15") SUB("%1")
289 CONDITIONAL_ADD("%[twice_modulus_0]", "%[twice_modulus_1]", "%[twice_modulus_2]", "%[twice_modulus_3]")
290 STORE_FIELD_ELEMENT("%0", "%%r12", "%%r13", "%%r14", "%%r15")
291 :
292 : "r"(&a),
293 "r"(&b),
294 [twice_modulus_0] "m"(twice_modulus_0),
295 [twice_modulus_1] "m"(twice_modulus_1),
296 [twice_modulus_2] "m"(twice_modulus_2),
297 [twice_modulus_3] "m"(twice_modulus_3)
298 : "%r8", "%r9", "%r10", "%r11", "%r12", "%r13", "%r14", "%r15", "cc", "memory");
299}
300
301template <class T> void field<T>::asm_conditional_negate(field& r, const uint64_t predicate) noexcept
302{
303 constexpr uint64_t twice_modulus_0 = twice_modulus.data[0];
304 constexpr uint64_t twice_modulus_1 = twice_modulus.data[1];
305 constexpr uint64_t twice_modulus_2 = twice_modulus.data[2];
306 constexpr uint64_t twice_modulus_3 = twice_modulus.data[3];
307
308 __asm__(CLEAR_FLAGS("%%r8") LOAD_FIELD_ELEMENT(
309 "%1", "%%r8", "%%r9", "%%r10", "%%r11") "movq %[twice_modulus_0], %%r12 \n\t"
310 "movq %[twice_modulus_1], %%r13 \n\t"
311 "movq %[twice_modulus_2], %%r14 \n\t"
312 "movq %[twice_modulus_3], %%r15 \n\t"
313 "subq %%r8, %%r12 \n\t"
314 "sbbq %%r9, %%r13 \n\t"
315 "sbbq %%r10, %%r14 \n\t"
316 "sbbq %%r11, %%r15 \n\t"
317 "testq %0, %0 \n\t"
318 "cmovnzq %%r12, %%r8 \n\t"
319 "cmovnzq %%r13, %%r9 \n\t"
320 "cmovnzq %%r14, %%r10 \n\t"
321 "cmovnzq %%r15, %%r11 \n\t" STORE_FIELD_ELEMENT(
322 "%1", "%%r8", "%%r9", "%%r10", "%%r11")
323 :
324 : "r"(predicate),
325 "r"(&r),
326 [twice_modulus_0] "i"(twice_modulus_0),
327 [twice_modulus_1] "i"(twice_modulus_1),
328 [twice_modulus_2] "i"(twice_modulus_2),
329 [twice_modulus_3] "i"(twice_modulus_3)
330 : "%r8", "%r9", "%r10", "%r11", "%r12", "%r13", "%r14", "%r15", "cc", "memory");
331}
332
333template <class T> field<T> field<T>::asm_reduce_once(const field& a) noexcept
334{
335 field r;
336
337 constexpr uint64_t not_modulus_0 = not_modulus.data[0];
338 constexpr uint64_t not_modulus_1 = not_modulus.data[1];
339 constexpr uint64_t not_modulus_2 = not_modulus.data[2];
340 constexpr uint64_t not_modulus_3 = not_modulus.data[3];
341
342 __asm__(CLEAR_FLAGS("%%r12") LOAD_FIELD_ELEMENT("%0", "%%r12", "%%r13", "%%r14", "%%r15")
343 CONDITIONAL_ADD("%[not_modulus_0]", "%[not_modulus_1]", "%[not_modulus_2]", "%[not_modulus_3]")
344 STORE_FIELD_ELEMENT("%1", "%%r12", "%%r13", "%%r14", "%%r15")
345 :
346 : "r"(&a),
347 "r"(&r),
348 [not_modulus_0] "m"(not_modulus_0),
349 [not_modulus_1] "m"(not_modulus_1),
350 [not_modulus_2] "m"(not_modulus_2),
351 [not_modulus_3] "m"(not_modulus_3)
352 : "%r8", "%r9", "%r10", "%r11", "%r12", "%r13", "%r14", "%r15", "cc", "memory");
353 return r;
354}
355
356template <class T> void field<T>::asm_self_reduce_once(field& a) noexcept
357{
358 constexpr uint64_t not_modulus_0 = not_modulus.data[0];
359 constexpr uint64_t not_modulus_1 = not_modulus.data[1];
360 constexpr uint64_t not_modulus_2 = not_modulus.data[2];
361 constexpr uint64_t not_modulus_3 = not_modulus.data[3];
362
363 __asm__(CLEAR_FLAGS("%%r12") LOAD_FIELD_ELEMENT("%0", "%%r12", "%%r13", "%%r14", "%%r15")
364 CONDITIONAL_ADD("%[not_modulus_0]", "%[not_modulus_1]", "%[not_modulus_2]", "%[not_modulus_3]")
365 STORE_FIELD_ELEMENT("%0", "%%r12", "%%r13", "%%r14", "%%r15")
366 :
367 : "r"(&a),
368 [not_modulus_0] "m"(not_modulus_0),
369 [not_modulus_1] "m"(not_modulus_1),
370 [not_modulus_2] "m"(not_modulus_2),
371 [not_modulus_3] "m"(not_modulus_3)
372 : "%r8", "%r9", "%r10", "%r11", "%r12", "%r13", "%r14", "%r15", "cc", "memory");
373}
374} // namespace bb
375#endif
asm_macros.hpp
ADD_REDUCE
#define ADD_REDUCE(b, twice_not_modulus_0, twice_not_modulus_1, twice_not_modulus_2, twice_not_modulus_3)
Definition asm_macros.hpp:66
STORE_FIELD_ELEMENT
#define STORE_FIELD_ELEMENT(r, lolo, lohi, hilo, hihi)
Definition asm_macros.hpp:31
LOAD_FIELD_ELEMENT
#define LOAD_FIELD_ELEMENT(a, lolo, lohi, hilo, hihi)
Definition asm_macros.hpp:20
CLEAR_FLAGS
#define CLEAR_FLAGS(empty_reg)
Definition asm_macros.hpp:13
CONDITIONAL_ADD
#define CONDITIONAL_ADD(b_0, b_1, b_2, b_3)
Definition asm_macros.hpp:95
a
FF a
Definition field_gt.test.cpp:52
b
FF b
Definition field_gt.test.cpp:53
field_impl.hpp
bb
Entry point for Barretenberg command-line interface.
Definition api.hpp:5
bb::Instruction::SQR
@ SQR
Square a field element.
bb::Instruction::SUB
@ SUB
Subtract two field elements.
bb::Instruction::MUL
@ MUL
Multiply two field elements.