graviola 0.4.0

graviola is a modern, fast cryptography library
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729

// generated source. do not edit.
#![allow(non_upper_case_globals, unused_macros, unused_imports)]
use crate::low::macros::*;

// Copyright (c) 2024 The mlkem-native project authors
// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
// SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT

// ----------------------------------------------------------------------------
// Inverse number-theoretic transform from ML-KEM
// Input a[256], z_01234[80], z_56[384] (all signed 16-bit words); output a[256] (signed 16-bit words).
//
// The transform is in-place with input and output a[256], with the input in
// bitreversed order and the output mapped into the Montgomery domain via
// x |-> (2^16 * x) mod 3329. The second parameter is expected to point to
// a table of constants whose definitions can be found in the mlkem-native
// repo or our "tests/test.c".
//
// extern void mlkem_intt_x86(int16_t a[static 256], const int16_t qdata[static 624]);
//
// Standard x86-64 ABI: RDI = a, RSI = qdata
// Microsoft x64 ABI:   RCX = a, RDX = qdata
// ----------------------------------------------------------------------------

/// Inverse number-theoretic transform from ML-KEM
///
/// Input a[256], z_01234[80], z_56[384] (all signed 16-bit words); output a[256] (signed 16-bit words).
///
/// The transform is in-place with input and output a[256], with the input in
/// bitreversed order and the output mapped into the Montgomery domain via
/// x |-> (2^16 * x) mod 3329. The second parameter is expected to point to
/// a table of constants whose definitions can be found in the mlkem-native
/// repo or our "tests/test.c".
pub(crate) fn mlkem_intt(a: &mut [i16; 256], qdata: &[i16; 624]) {
    debug_assert!((a.as_mut_ptr() as usize).is_multiple_of(32));
    debug_assert!((qdata.as_ptr() as usize).is_multiple_of(32));
    // SAFETY: inline assembly. see [crate::low::inline_assembly_safety] for safety info.
    unsafe {
        core::arch::asm!(

        Q!("    endbr64         " ),

        Q!("    mov             " "eax, 0xd010d01"),
        Q!("    vmovd           " "xmm0, eax"),
        Q!("    vpbroadcastd    " "ymm0, xmm0"),
        Q!("    mov             " "eax, 0xd8a1d8a1"),
        Q!("    vmovd           " "xmm2, eax"),
        Q!("    vpbroadcastd    " "ymm2, xmm2"),
        Q!("    mov             " "eax, 0x5a105a1"),
        Q!("    vmovd           " "xmm3, eax"),
        Q!("    vpbroadcastd    " "ymm3, xmm3"),
        Q!("    vmovdqa         " "ymm4, [rdi]"),
        Q!("    vmovdqa         " "ymm6, [64 + rdi]"),
        Q!("    vmovdqa         " "ymm5, [32 + rdi]"),
        Q!("    vmovdqa         " "ymm7, [96 + rdi]"),
        Q!("    vpmullw         " "ymm12, ymm4, ymm2"),
        Q!("    vpmulhw         " "ymm4, ymm4, ymm3"),
        Q!("    vpmulhw         " "ymm12, ymm12, ymm0"),
        Q!("    vpsubw          " "ymm4, ymm4, ymm12"),
        Q!("    vpmullw         " "ymm12, ymm6, ymm2"),
        Q!("    vpmulhw         " "ymm6, ymm6, ymm3"),
        Q!("    vpmulhw         " "ymm12, ymm12, ymm0"),
        Q!("    vpsubw          " "ymm6, ymm6, ymm12"),
        Q!("    vpmullw         " "ymm12, ymm5, ymm2"),
        Q!("    vpmulhw         " "ymm5, ymm5, ymm3"),
        Q!("    vpmulhw         " "ymm12, ymm12, ymm0"),
        Q!("    vpsubw          " "ymm5, ymm5, ymm12"),
        Q!("    vpmullw         " "ymm12, ymm7, ymm2"),
        Q!("    vpmulhw         " "ymm7, ymm7, ymm3"),
        Q!("    vpmulhw         " "ymm12, ymm12, ymm0"),
        Q!("    vpsubw          " "ymm7, ymm7, ymm12"),
        Q!("    vmovdqa         " "ymm8, [128 + rdi]"),
        Q!("    vmovdqa         " "ymm10, [192 + rdi]"),
        Q!("    vmovdqa         " "ymm9, [160 + rdi]"),
        Q!("    vmovdqa         " "ymm11, [224 + rdi]"),
        Q!("    vpmullw         " "ymm12, ymm8, ymm2"),
        Q!("    vpmulhw         " "ymm8, ymm8, ymm3"),
        Q!("    vpmulhw         " "ymm12, ymm12, ymm0"),
        Q!("    vpsubw          " "ymm8, ymm8, ymm12"),
        Q!("    vpmullw         " "ymm12, ymm10, ymm2"),
        Q!("    vpmulhw         " "ymm10, ymm10, ymm3"),
        Q!("    vpmulhw         " "ymm12, ymm12, ymm0"),
        Q!("    vpsubw          " "ymm10, ymm10, ymm12"),
        Q!("    vpmullw         " "ymm12, ymm9, ymm2"),
        Q!("    vpmulhw         " "ymm9, ymm9, ymm3"),
        Q!("    vpmulhw         " "ymm12, ymm12, ymm0"),
        Q!("    vpsubw          " "ymm9, ymm9, ymm12"),
        Q!("    vpmullw         " "ymm12, ymm11, ymm2"),
        Q!("    vpmulhw         " "ymm11, ymm11, ymm3"),
        Q!("    vpmulhw         " "ymm12, ymm12, ymm0"),
        Q!("    vpsubw          " "ymm11, ymm11, ymm12"),
        Q!("    vpermq          " "ymm15, [928 + rsi], 0x4e"),
        Q!("    vpermq          " "ymm1, [864 + rsi], 0x4e"),
        Q!("    vpermq          " "ymm2, [960 + rsi], 0x4e"),
        Q!("    vpermq          " "ymm3, [896 + rsi], 0x4e"),
        Q!("    vmovdqa         " "ymm12, [rsi]"),
        Q!("    vpshufb         " "ymm15, ymm15, ymm12"),
        Q!("    vpshufb         " "ymm1, ymm1, ymm12"),
        Q!("    vpshufb         " "ymm2, ymm2, ymm12"),
        Q!("    vpshufb         " "ymm3, ymm3, ymm12"),
        Q!("    vpsubw          " "ymm12, ymm6, ymm4"),
        Q!("    vpaddw          " "ymm4, ymm4, ymm6"),
        Q!("    vpsubw          " "ymm13, ymm7, ymm5"),
        Q!("    vpmullw         " "ymm6, ymm12, ymm15"),
        Q!("    vpaddw          " "ymm5, ymm5, ymm7"),
        Q!("    vpsubw          " "ymm14, ymm10, ymm8"),
        Q!("    vpmullw         " "ymm7, ymm13, ymm15"),
        Q!("    vpaddw          " "ymm8, ymm8, ymm10"),
        Q!("    vpsubw          " "ymm15, ymm11, ymm9"),
        Q!("    vpmullw         " "ymm10, ymm14, ymm1"),
        Q!("    vpaddw          " "ymm9, ymm9, ymm11"),
        Q!("    vpmullw         " "ymm11, ymm15, ymm1"),
        Q!("    vpmulhw         " "ymm12, ymm12, ymm2"),
        Q!("    vpmulhw         " "ymm13, ymm13, ymm2"),
        Q!("    vpmulhw         " "ymm14, ymm14, ymm3"),
        Q!("    vpmulhw         " "ymm15, ymm15, ymm3"),
        Q!("    vpmulhw         " "ymm6, ymm6, ymm0"),
        Q!("    vpmulhw         " "ymm7, ymm7, ymm0"),
        Q!("    vpmulhw         " "ymm10, ymm10, ymm0"),
        Q!("    vpmulhw         " "ymm11, ymm11, ymm0"),
        Q!("    vpsubw          " "ymm6, ymm12, ymm6"),
        Q!("    vpsubw          " "ymm7, ymm13, ymm7"),
        Q!("    vpsubw          " "ymm10, ymm14, ymm10"),
        Q!("    vpsubw          " "ymm11, ymm15, ymm11"),
        Q!("    vpermq          " "ymm2, [800 + rsi], 0x4e"),
        Q!("    vpermq          " "ymm3, [832 + rsi], 0x4e"),
        Q!("    vmovdqa         " "ymm1, [rsi]"),
        Q!("    vpshufb         " "ymm2, ymm2, ymm1"),
        Q!("    vpshufb         " "ymm3, ymm3, ymm1"),
        Q!("    vpsubw          " "ymm12, ymm8, ymm4"),
        Q!("    vpaddw          " "ymm4, ymm4, ymm8"),
        Q!("    vpsubw          " "ymm13, ymm9, ymm5"),
        Q!("    vpmullw         " "ymm8, ymm12, ymm2"),
        Q!("    vpaddw          " "ymm5, ymm5, ymm9"),
        Q!("    vpsubw          " "ymm14, ymm10, ymm6"),
        Q!("    vpmullw         " "ymm9, ymm13, ymm2"),
        Q!("    vpaddw          " "ymm6, ymm6, ymm10"),
        Q!("    vpsubw          " "ymm15, ymm11, ymm7"),
        Q!("    vpmullw         " "ymm10, ymm14, ymm2"),
        Q!("    vpaddw          " "ymm7, ymm7, ymm11"),
        Q!("    vpmullw         " "ymm11, ymm15, ymm2"),
        Q!("    vpmulhw         " "ymm12, ymm12, ymm3"),
        Q!("    vpmulhw         " "ymm13, ymm13, ymm3"),
        Q!("    vpmulhw         " "ymm14, ymm14, ymm3"),
        Q!("    vpmulhw         " "ymm15, ymm15, ymm3"),
        Q!("    vpmulhw         " "ymm8, ymm8, ymm0"),
        Q!("    vpmulhw         " "ymm9, ymm9, ymm0"),
        Q!("    vpmulhw         " "ymm10, ymm10, ymm0"),
        Q!("    vpmulhw         " "ymm11, ymm11, ymm0"),
        Q!("    vpsubw          " "ymm8, ymm12, ymm8"),
        Q!("    vpsubw          " "ymm9, ymm13, ymm9"),
        Q!("    vpsubw          " "ymm10, ymm14, ymm10"),
        Q!("    vpsubw          " "ymm11, ymm15, ymm11"),
        Q!("    vpslld          " "ymm3, ymm5, 0x10"),
        Q!("    vpblendw        " "ymm3, ymm4, ymm3, 0xaa"),
        Q!("    vpsrld          " "ymm4, ymm4, 0x10"),
        Q!("    vpblendw        " "ymm5, ymm4, ymm5, 0xaa"),
        Q!("    vpslld          " "ymm4, ymm7, 0x10"),
        Q!("    vpblendw        " "ymm4, ymm6, ymm4, 0xaa"),
        Q!("    vpsrld          " "ymm6, ymm6, 0x10"),
        Q!("    vpblendw        " "ymm7, ymm6, ymm7, 0xaa"),
        Q!("    vpslld          " "ymm6, ymm9, 0x10"),
        Q!("    vpblendw        " "ymm6, ymm8, ymm6, 0xaa"),
        Q!("    vpsrld          " "ymm8, ymm8, 0x10"),
        Q!("    vpblendw        " "ymm9, ymm8, ymm9, 0xaa"),
        Q!("    vpslld          " "ymm8, ymm11, 0x10"),
        Q!("    vpblendw        " "ymm8, ymm10, ymm8, 0xaa"),
        Q!("    vpsrld          " "ymm10, ymm10, 0x10"),
        Q!("    vpblendw        " "ymm11, ymm10, ymm11, 0xaa"),
        Q!("    vmovdqa         " "ymm12, [32 + rsi]"),
        Q!("    vpermd          " "ymm2, ymm12, [736 + rsi]"),
        Q!("    vpermd          " "ymm10, ymm12, [768 + rsi]"),
        Q!("    vpsubw          " "ymm12, ymm5, ymm3"),
        Q!("    vpaddw          " "ymm3, ymm3, ymm5"),
        Q!("    vpsubw          " "ymm13, ymm7, ymm4"),
        Q!("    vpmullw         " "ymm5, ymm12, ymm2"),
        Q!("    vpaddw          " "ymm4, ymm4, ymm7"),
        Q!("    vpsubw          " "ymm14, ymm9, ymm6"),
        Q!("    vpmullw         " "ymm7, ymm13, ymm2"),
        Q!("    vpaddw          " "ymm6, ymm6, ymm9"),
        Q!("    vpsubw          " "ymm15, ymm11, ymm8"),
        Q!("    vpmullw         " "ymm9, ymm14, ymm2"),
        Q!("    vpaddw          " "ymm8, ymm8, ymm11"),
        Q!("    vpmullw         " "ymm11, ymm15, ymm2"),
        Q!("    vpmulhw         " "ymm12, ymm12, ymm10"),
        Q!("    vpmulhw         " "ymm13, ymm13, ymm10"),
        Q!("    vpmulhw         " "ymm14, ymm14, ymm10"),
        Q!("    vpmulhw         " "ymm15, ymm15, ymm10"),
        Q!("    vpmulhw         " "ymm5, ymm5, ymm0"),
        Q!("    vpmulhw         " "ymm7, ymm7, ymm0"),
        Q!("    vpmulhw         " "ymm9, ymm9, ymm0"),
        Q!("    vpmulhw         " "ymm11, ymm11, ymm0"),
        Q!("    vpsubw          " "ymm5, ymm12, ymm5"),
        Q!("    vpsubw          " "ymm7, ymm13, ymm7"),
        Q!("    vpsubw          " "ymm9, ymm14, ymm9"),
        Q!("    vpsubw          " "ymm11, ymm15, ymm11"),
        Q!("    mov             " "eax, 0x4ebf4ebf"),
        Q!("    vmovd           " "xmm1, eax"),
        Q!("    vpbroadcastd    " "ymm1, xmm1"),
        Q!("    vpmulhw         " "ymm12, ymm3, ymm1"),
        Q!("    vpsraw          " "ymm12, ymm12, 0xa"),
        Q!("    vpmullw         " "ymm12, ymm12, ymm0"),
        Q!("    vpsubw          " "ymm3, ymm3, ymm12"),
        Q!("    vmovsldup       " "ymm10, ymm4"),
        Q!("    vpblendd        " "ymm10, ymm3, ymm10, 0xaa"),
        Q!("    vpsrlq          " "ymm3, ymm3, 0x20"),
        Q!("    vpblendd        " "ymm4, ymm3, ymm4, 0xaa"),
        Q!("    vmovsldup       " "ymm3, ymm8"),
        Q!("    vpblendd        " "ymm3, ymm6, ymm3, 0xaa"),
        Q!("    vpsrlq          " "ymm6, ymm6, 0x20"),
        Q!("    vpblendd        " "ymm8, ymm6, ymm8, 0xaa"),
        Q!("    vmovsldup       " "ymm6, ymm7"),
        Q!("    vpblendd        " "ymm6, ymm5, ymm6, 0xaa"),
        Q!("    vpsrlq          " "ymm5, ymm5, 0x20"),
        Q!("    vpblendd        " "ymm7, ymm5, ymm7, 0xaa"),
        Q!("    vmovsldup       " "ymm5, ymm11"),
        Q!("    vpblendd        " "ymm5, ymm9, ymm5, 0xaa"),
        Q!("    vpsrlq          " "ymm9, ymm9, 0x20"),
        Q!("    vpblendd        " "ymm11, ymm9, ymm11, 0xaa"),
        Q!("    vpermq          " "ymm2, [672 + rsi], 0x1b"),
        Q!("    vpermq          " "ymm9, [704 + rsi], 0x1b"),
        Q!("    vpsubw          " "ymm12, ymm4, ymm10"),
        Q!("    vpaddw          " "ymm10, ymm10, ymm4"),
        Q!("    vpsubw          " "ymm13, ymm8, ymm3"),
        Q!("    vpmullw         " "ymm4, ymm12, ymm2"),
        Q!("    vpaddw          " "ymm3, ymm3, ymm8"),
        Q!("    vpsubw          " "ymm14, ymm7, ymm6"),
        Q!("    vpmullw         " "ymm8, ymm13, ymm2"),
        Q!("    vpaddw          " "ymm6, ymm6, ymm7"),
        Q!("    vpsubw          " "ymm15, ymm11, ymm5"),
        Q!("    vpmullw         " "ymm7, ymm14, ymm2"),
        Q!("    vpaddw          " "ymm5, ymm5, ymm11"),
        Q!("    vpmullw         " "ymm11, ymm15, ymm2"),
        Q!("    vpmulhw         " "ymm12, ymm12, ymm9"),
        Q!("    vpmulhw         " "ymm13, ymm13, ymm9"),
        Q!("    vpmulhw         " "ymm14, ymm14, ymm9"),
        Q!("    vpmulhw         " "ymm15, ymm15, ymm9"),
        Q!("    vpmulhw         " "ymm4, ymm4, ymm0"),
        Q!("    vpmulhw         " "ymm8, ymm8, ymm0"),
        Q!("    vpmulhw         " "ymm7, ymm7, ymm0"),
        Q!("    vpmulhw         " "ymm11, ymm11, ymm0"),
        Q!("    vpsubw          " "ymm4, ymm12, ymm4"),
        Q!("    vpsubw          " "ymm8, ymm13, ymm8"),
        Q!("    vpsubw          " "ymm7, ymm14, ymm7"),
        Q!("    vpsubw          " "ymm11, ymm15, ymm11"),
        Q!("    vpmulhw         " "ymm12, ymm10, ymm1"),
        Q!("    vpsraw          " "ymm12, ymm12, 0xa"),
        Q!("    vpmullw         " "ymm12, ymm12, ymm0"),
        Q!("    vpsubw          " "ymm10, ymm10, ymm12"),
        Q!("    vpunpcklqdq     " "ymm9, ymm10, ymm3"),
        Q!("    vpunpckhqdq     " "ymm3, ymm10, ymm3"),
        Q!("    vpunpcklqdq     " "ymm10, ymm6, ymm5"),
        Q!("    vpunpckhqdq     " "ymm5, ymm6, ymm5"),
        Q!("    vpunpcklqdq     " "ymm6, ymm4, ymm8"),
        Q!("    vpunpckhqdq     " "ymm8, ymm4, ymm8"),
        Q!("    vpunpcklqdq     " "ymm4, ymm7, ymm11"),
        Q!("    vpunpckhqdq     " "ymm11, ymm7, ymm11"),
        Q!("    vpermq          " "ymm2, [608 + rsi], 0x4e"),
        Q!("    vpermq          " "ymm7, [640 + rsi], 0x4e"),
        Q!("    vpsubw          " "ymm12, ymm3, ymm9"),
        Q!("    vpaddw          " "ymm9, ymm9, ymm3"),
        Q!("    vpsubw          " "ymm13, ymm5, ymm10"),
        Q!("    vpmullw         " "ymm3, ymm12, ymm2"),
        Q!("    vpaddw          " "ymm10, ymm10, ymm5"),
        Q!("    vpsubw          " "ymm14, ymm8, ymm6"),
        Q!("    vpmullw         " "ymm5, ymm13, ymm2"),
        Q!("    vpaddw          " "ymm6, ymm6, ymm8"),
        Q!("    vpsubw          " "ymm15, ymm11, ymm4"),
        Q!("    vpmullw         " "ymm8, ymm14, ymm2"),
        Q!("    vpaddw          " "ymm4, ymm4, ymm11"),
        Q!("    vpmullw         " "ymm11, ymm15, ymm2"),
        Q!("    vpmulhw         " "ymm12, ymm12, ymm7"),
        Q!("    vpmulhw         " "ymm13, ymm13, ymm7"),
        Q!("    vpmulhw         " "ymm14, ymm14, ymm7"),
        Q!("    vpmulhw         " "ymm15, ymm15, ymm7"),
        Q!("    vpmulhw         " "ymm3, ymm3, ymm0"),
        Q!("    vpmulhw         " "ymm5, ymm5, ymm0"),
        Q!("    vpmulhw         " "ymm8, ymm8, ymm0"),
        Q!("    vpmulhw         " "ymm11, ymm11, ymm0"),
        Q!("    vpsubw          " "ymm3, ymm12, ymm3"),
        Q!("    vpsubw          " "ymm5, ymm13, ymm5"),
        Q!("    vpsubw          " "ymm8, ymm14, ymm8"),
        Q!("    vpsubw          " "ymm11, ymm15, ymm11"),
        Q!("    vpmulhw         " "ymm12, ymm9, ymm1"),
        Q!("    vpsraw          " "ymm12, ymm12, 0xa"),
        Q!("    vpmullw         " "ymm12, ymm12, ymm0"),
        Q!("    vpsubw          " "ymm9, ymm9, ymm12"),
        Q!("    vperm2i128      " "ymm7, ymm9, ymm10, 0x20"),
        Q!("    vperm2i128      " "ymm10, ymm9, ymm10, 0x31"),
        Q!("    vperm2i128      " "ymm9, ymm6, ymm4, 0x20"),
        Q!("    vperm2i128      " "ymm4, ymm6, ymm4, 0x31"),
        Q!("    vperm2i128      " "ymm6, ymm3, ymm5, 0x20"),
        Q!("    vperm2i128      " "ymm5, ymm3, ymm5, 0x31"),
        Q!("    vperm2i128      " "ymm3, ymm8, ymm11, 0x20"),
        Q!("    vperm2i128      " "ymm11, ymm8, ymm11, 0x31"),
        Q!("    vmovdqa         " "ymm2, [544 + rsi]"),
        Q!("    vmovdqa         " "ymm8, [576 + rsi]"),
        Q!("    vpsubw          " "ymm12, ymm10, ymm7"),
        Q!("    vpaddw          " "ymm7, ymm7, ymm10"),
        Q!("    vpsubw          " "ymm13, ymm4, ymm9"),
        Q!("    vpmullw         " "ymm10, ymm12, ymm2"),
        Q!("    vpaddw          " "ymm9, ymm9, ymm4"),
        Q!("    vpsubw          " "ymm14, ymm5, ymm6"),
        Q!("    vpmullw         " "ymm4, ymm13, ymm2"),
        Q!("    vpaddw          " "ymm6, ymm6, ymm5"),
        Q!("    vpsubw          " "ymm15, ymm11, ymm3"),
        Q!("    vpmullw         " "ymm5, ymm14, ymm2"),
        Q!("    vpaddw          " "ymm3, ymm3, ymm11"),
        Q!("    vpmullw         " "ymm11, ymm15, ymm2"),
        Q!("    vpmulhw         " "ymm12, ymm12, ymm8"),
        Q!("    vpmulhw         " "ymm13, ymm13, ymm8"),
        Q!("    vpmulhw         " "ymm14, ymm14, ymm8"),
        Q!("    vpmulhw         " "ymm15, ymm15, ymm8"),
        Q!("    vpmulhw         " "ymm10, ymm10, ymm0"),
        Q!("    vpmulhw         " "ymm4, ymm4, ymm0"),
        Q!("    vpmulhw         " "ymm5, ymm5, ymm0"),
        Q!("    vpmulhw         " "ymm11, ymm11, ymm0"),
        Q!("    vpsubw          " "ymm10, ymm12, ymm10"),
        Q!("    vpsubw          " "ymm4, ymm13, ymm4"),
        Q!("    vpsubw          " "ymm5, ymm14, ymm5"),
        Q!("    vpsubw          " "ymm11, ymm15, ymm11"),
        Q!("    vpmulhw         " "ymm12, ymm7, ymm1"),
        Q!("    vpsraw          " "ymm12, ymm12, 0xa"),
        Q!("    vpmullw         " "ymm12, ymm12, ymm0"),
        Q!("    vpsubw          " "ymm7, ymm7, ymm12"),
        Q!("    vmovdqa         " "[rdi], ymm7"),
        Q!("    vmovdqa         " "[32 + rdi], ymm9"),
        Q!("    vmovdqa         " "[64 + rdi], ymm6"),
        Q!("    vmovdqa         " "[96 + rdi], ymm3"),
        Q!("    vmovdqa         " "[128 + rdi], ymm10"),
        Q!("    vmovdqa         " "[160 + rdi], ymm4"),
        Q!("    vmovdqa         " "[192 + rdi], ymm5"),
        Q!("    vmovdqa         " "[224 + rdi], ymm11"),
        Q!("    mov             " "eax, 0xd8a1d8a1"),
        Q!("    vmovd           " "xmm2, eax"),
        Q!("    vpbroadcastd    " "ymm2, xmm2"),
        Q!("    mov             " "eax, 0x5a105a1"),
        Q!("    vmovd           " "xmm3, eax"),
        Q!("    vpbroadcastd    " "ymm3, xmm3"),
        Q!("    vmovdqa         " "ymm4, [256 + rdi]"),
        Q!("    vmovdqa         " "ymm6, [320 + rdi]"),
        Q!("    vmovdqa         " "ymm5, [288 + rdi]"),
        Q!("    vmovdqa         " "ymm7, [352 + rdi]"),
        Q!("    vpmullw         " "ymm12, ymm4, ymm2"),
        Q!("    vpmulhw         " "ymm4, ymm4, ymm3"),
        Q!("    vpmulhw         " "ymm12, ymm12, ymm0"),
        Q!("    vpsubw          " "ymm4, ymm4, ymm12"),
        Q!("    vpmullw         " "ymm12, ymm6, ymm2"),
        Q!("    vpmulhw         " "ymm6, ymm6, ymm3"),
        Q!("    vpmulhw         " "ymm12, ymm12, ymm0"),
        Q!("    vpsubw          " "ymm6, ymm6, ymm12"),
        Q!("    vpmullw         " "ymm12, ymm5, ymm2"),
        Q!("    vpmulhw         " "ymm5, ymm5, ymm3"),
        Q!("    vpmulhw         " "ymm12, ymm12, ymm0"),
        Q!("    vpsubw          " "ymm5, ymm5, ymm12"),
        Q!("    vpmullw         " "ymm12, ymm7, ymm2"),
        Q!("    vpmulhw         " "ymm7, ymm7, ymm3"),
        Q!("    vpmulhw         " "ymm12, ymm12, ymm0"),
        Q!("    vpsubw          " "ymm7, ymm7, ymm12"),
        Q!("    vmovdqa         " "ymm8, [384 + rdi]"),
        Q!("    vmovdqa         " "ymm10, [448 + rdi]"),
        Q!("    vmovdqa         " "ymm9, [416 + rdi]"),
        Q!("    vmovdqa         " "ymm11, [480 + rdi]"),
        Q!("    vpmullw         " "ymm12, ymm8, ymm2"),
        Q!("    vpmulhw         " "ymm8, ymm8, ymm3"),
        Q!("    vpmulhw         " "ymm12, ymm12, ymm0"),
        Q!("    vpsubw          " "ymm8, ymm8, ymm12"),
        Q!("    vpmullw         " "ymm12, ymm10, ymm2"),
        Q!("    vpmulhw         " "ymm10, ymm10, ymm3"),
        Q!("    vpmulhw         " "ymm12, ymm12, ymm0"),
        Q!("    vpsubw          " "ymm10, ymm10, ymm12"),
        Q!("    vpmullw         " "ymm12, ymm9, ymm2"),
        Q!("    vpmulhw         " "ymm9, ymm9, ymm3"),
        Q!("    vpmulhw         " "ymm12, ymm12, ymm0"),
        Q!("    vpsubw          " "ymm9, ymm9, ymm12"),
        Q!("    vpmullw         " "ymm12, ymm11, ymm2"),
        Q!("    vpmulhw         " "ymm11, ymm11, ymm3"),
        Q!("    vpmulhw         " "ymm12, ymm12, ymm0"),
        Q!("    vpsubw          " "ymm11, ymm11, ymm12"),
        Q!("    vpermq          " "ymm15, [480 + rsi], 0x4e"),
        Q!("    vpermq          " "ymm1, [416 + rsi], 0x4e"),
        Q!("    vpermq          " "ymm2, [512 + rsi], 0x4e"),
        Q!("    vpermq          " "ymm3, [448 + rsi], 0x4e"),
        Q!("    vmovdqa         " "ymm12, [rsi]"),
        Q!("    vpshufb         " "ymm15, ymm15, ymm12"),
        Q!("    vpshufb         " "ymm1, ymm1, ymm12"),
        Q!("    vpshufb         " "ymm2, ymm2, ymm12"),
        Q!("    vpshufb         " "ymm3, ymm3, ymm12"),
        Q!("    vpsubw          " "ymm12, ymm6, ymm4"),
        Q!("    vpaddw          " "ymm4, ymm4, ymm6"),
        Q!("    vpsubw          " "ymm13, ymm7, ymm5"),
        Q!("    vpmullw         " "ymm6, ymm12, ymm15"),
        Q!("    vpaddw          " "ymm5, ymm5, ymm7"),
        Q!("    vpsubw          " "ymm14, ymm10, ymm8"),
        Q!("    vpmullw         " "ymm7, ymm13, ymm15"),
        Q!("    vpaddw          " "ymm8, ymm8, ymm10"),
        Q!("    vpsubw          " "ymm15, ymm11, ymm9"),
        Q!("    vpmullw         " "ymm10, ymm14, ymm1"),
        Q!("    vpaddw          " "ymm9, ymm9, ymm11"),
        Q!("    vpmullw         " "ymm11, ymm15, ymm1"),
        Q!("    vpmulhw         " "ymm12, ymm12, ymm2"),
        Q!("    vpmulhw         " "ymm13, ymm13, ymm2"),
        Q!("    vpmulhw         " "ymm14, ymm14, ymm3"),
        Q!("    vpmulhw         " "ymm15, ymm15, ymm3"),
        Q!("    vpmulhw         " "ymm6, ymm6, ymm0"),
        Q!("    vpmulhw         " "ymm7, ymm7, ymm0"),
        Q!("    vpmulhw         " "ymm10, ymm10, ymm0"),
        Q!("    vpmulhw         " "ymm11, ymm11, ymm0"),
        Q!("    vpsubw          " "ymm6, ymm12, ymm6"),
        Q!("    vpsubw          " "ymm7, ymm13, ymm7"),
        Q!("    vpsubw          " "ymm10, ymm14, ymm10"),
        Q!("    vpsubw          " "ymm11, ymm15, ymm11"),
        Q!("    vpermq          " "ymm2, [352 + rsi], 0x4e"),
        Q!("    vpermq          " "ymm3, [384 + rsi], 0x4e"),
        Q!("    vmovdqa         " "ymm1, [rsi]"),
        Q!("    vpshufb         " "ymm2, ymm2, ymm1"),
        Q!("    vpshufb         " "ymm3, ymm3, ymm1"),
        Q!("    vpsubw          " "ymm12, ymm8, ymm4"),
        Q!("    vpaddw          " "ymm4, ymm4, ymm8"),
        Q!("    vpsubw          " "ymm13, ymm9, ymm5"),
        Q!("    vpmullw         " "ymm8, ymm12, ymm2"),
        Q!("    vpaddw          " "ymm5, ymm5, ymm9"),
        Q!("    vpsubw          " "ymm14, ymm10, ymm6"),
        Q!("    vpmullw         " "ymm9, ymm13, ymm2"),
        Q!("    vpaddw          " "ymm6, ymm6, ymm10"),
        Q!("    vpsubw          " "ymm15, ymm11, ymm7"),
        Q!("    vpmullw         " "ymm10, ymm14, ymm2"),
        Q!("    vpaddw          " "ymm7, ymm7, ymm11"),
        Q!("    vpmullw         " "ymm11, ymm15, ymm2"),
        Q!("    vpmulhw         " "ymm12, ymm12, ymm3"),
        Q!("    vpmulhw         " "ymm13, ymm13, ymm3"),
        Q!("    vpmulhw         " "ymm14, ymm14, ymm3"),
        Q!("    vpmulhw         " "ymm15, ymm15, ymm3"),
        Q!("    vpmulhw         " "ymm8, ymm8, ymm0"),
        Q!("    vpmulhw         " "ymm9, ymm9, ymm0"),
        Q!("    vpmulhw         " "ymm10, ymm10, ymm0"),
        Q!("    vpmulhw         " "ymm11, ymm11, ymm0"),
        Q!("    vpsubw          " "ymm8, ymm12, ymm8"),
        Q!("    vpsubw          " "ymm9, ymm13, ymm9"),
        Q!("    vpsubw          " "ymm10, ymm14, ymm10"),
        Q!("    vpsubw          " "ymm11, ymm15, ymm11"),
        Q!("    vpslld          " "ymm3, ymm5, 0x10"),
        Q!("    vpblendw        " "ymm3, ymm4, ymm3, 0xaa"),
        Q!("    vpsrld          " "ymm4, ymm4, 0x10"),
        Q!("    vpblendw        " "ymm5, ymm4, ymm5, 0xaa"),
        Q!("    vpslld          " "ymm4, ymm7, 0x10"),
        Q!("    vpblendw        " "ymm4, ymm6, ymm4, 0xaa"),
        Q!("    vpsrld          " "ymm6, ymm6, 0x10"),
        Q!("    vpblendw        " "ymm7, ymm6, ymm7, 0xaa"),
        Q!("    vpslld          " "ymm6, ymm9, 0x10"),
        Q!("    vpblendw        " "ymm6, ymm8, ymm6, 0xaa"),
        Q!("    vpsrld          " "ymm8, ymm8, 0x10"),
        Q!("    vpblendw        " "ymm9, ymm8, ymm9, 0xaa"),
        Q!("    vpslld          " "ymm8, ymm11, 0x10"),
        Q!("    vpblendw        " "ymm8, ymm10, ymm8, 0xaa"),
        Q!("    vpsrld          " "ymm10, ymm10, 0x10"),
        Q!("    vpblendw        " "ymm11, ymm10, ymm11, 0xaa"),
        Q!("    vmovdqa         " "ymm12, [32 + rsi]"),
        Q!("    vpermd          " "ymm2, ymm12, [288 + rsi]"),
        Q!("    vpermd          " "ymm10, ymm12, [320 + rsi]"),
        Q!("    vpsubw          " "ymm12, ymm5, ymm3"),
        Q!("    vpaddw          " "ymm3, ymm3, ymm5"),
        Q!("    vpsubw          " "ymm13, ymm7, ymm4"),
        Q!("    vpmullw         " "ymm5, ymm12, ymm2"),
        Q!("    vpaddw          " "ymm4, ymm4, ymm7"),
        Q!("    vpsubw          " "ymm14, ymm9, ymm6"),
        Q!("    vpmullw         " "ymm7, ymm13, ymm2"),
        Q!("    vpaddw          " "ymm6, ymm6, ymm9"),
        Q!("    vpsubw          " "ymm15, ymm11, ymm8"),
        Q!("    vpmullw         " "ymm9, ymm14, ymm2"),
        Q!("    vpaddw          " "ymm8, ymm8, ymm11"),
        Q!("    vpmullw         " "ymm11, ymm15, ymm2"),
        Q!("    vpmulhw         " "ymm12, ymm12, ymm10"),
        Q!("    vpmulhw         " "ymm13, ymm13, ymm10"),
        Q!("    vpmulhw         " "ymm14, ymm14, ymm10"),
        Q!("    vpmulhw         " "ymm15, ymm15, ymm10"),
        Q!("    vpmulhw         " "ymm5, ymm5, ymm0"),
        Q!("    vpmulhw         " "ymm7, ymm7, ymm0"),
        Q!("    vpmulhw         " "ymm9, ymm9, ymm0"),
        Q!("    vpmulhw         " "ymm11, ymm11, ymm0"),
        Q!("    vpsubw          " "ymm5, ymm12, ymm5"),
        Q!("    vpsubw          " "ymm7, ymm13, ymm7"),
        Q!("    vpsubw          " "ymm9, ymm14, ymm9"),
        Q!("    vpsubw          " "ymm11, ymm15, ymm11"),
        Q!("    mov             " "eax, 0x4ebf4ebf"),
        Q!("    vmovd           " "xmm1, eax"),
        Q!("    vpbroadcastd    " "ymm1, xmm1"),
        Q!("    vpmulhw         " "ymm12, ymm3, ymm1"),
        Q!("    vpsraw          " "ymm12, ymm12, 0xa"),
        Q!("    vpmullw         " "ymm12, ymm12, ymm0"),
        Q!("    vpsubw          " "ymm3, ymm3, ymm12"),
        Q!("    vmovsldup       " "ymm10, ymm4"),
        Q!("    vpblendd        " "ymm10, ymm3, ymm10, 0xaa"),
        Q!("    vpsrlq          " "ymm3, ymm3, 0x20"),
        Q!("    vpblendd        " "ymm4, ymm3, ymm4, 0xaa"),
        Q!("    vmovsldup       " "ymm3, ymm8"),
        Q!("    vpblendd        " "ymm3, ymm6, ymm3, 0xaa"),
        Q!("    vpsrlq          " "ymm6, ymm6, 0x20"),
        Q!("    vpblendd        " "ymm8, ymm6, ymm8, 0xaa"),
        Q!("    vmovsldup       " "ymm6, ymm7"),
        Q!("    vpblendd        " "ymm6, ymm5, ymm6, 0xaa"),
        Q!("    vpsrlq          " "ymm5, ymm5, 0x20"),
        Q!("    vpblendd        " "ymm7, ymm5, ymm7, 0xaa"),
        Q!("    vmovsldup       " "ymm5, ymm11"),
        Q!("    vpblendd        " "ymm5, ymm9, ymm5, 0xaa"),
        Q!("    vpsrlq          " "ymm9, ymm9, 0x20"),
        Q!("    vpblendd        " "ymm11, ymm9, ymm11, 0xaa"),
        Q!("    vpermq          " "ymm2, [224 + rsi], 0x1b"),
        Q!("    vpermq          " "ymm9, [256 + rsi], 0x1b"),
        Q!("    vpsubw          " "ymm12, ymm4, ymm10"),
        Q!("    vpaddw          " "ymm10, ymm10, ymm4"),
        Q!("    vpsubw          " "ymm13, ymm8, ymm3"),
        Q!("    vpmullw         " "ymm4, ymm12, ymm2"),
        Q!("    vpaddw          " "ymm3, ymm3, ymm8"),
        Q!("    vpsubw          " "ymm14, ymm7, ymm6"),
        Q!("    vpmullw         " "ymm8, ymm13, ymm2"),
        Q!("    vpaddw          " "ymm6, ymm6, ymm7"),
        Q!("    vpsubw          " "ymm15, ymm11, ymm5"),
        Q!("    vpmullw         " "ymm7, ymm14, ymm2"),
        Q!("    vpaddw          " "ymm5, ymm5, ymm11"),
        Q!("    vpmullw         " "ymm11, ymm15, ymm2"),
        Q!("    vpmulhw         " "ymm12, ymm12, ymm9"),
        Q!("    vpmulhw         " "ymm13, ymm13, ymm9"),
        Q!("    vpmulhw         " "ymm14, ymm14, ymm9"),
        Q!("    vpmulhw         " "ymm15, ymm15, ymm9"),
        Q!("    vpmulhw         " "ymm4, ymm4, ymm0"),
        Q!("    vpmulhw         " "ymm8, ymm8, ymm0"),
        Q!("    vpmulhw         " "ymm7, ymm7, ymm0"),
        Q!("    vpmulhw         " "ymm11, ymm11, ymm0"),
        Q!("    vpsubw          " "ymm4, ymm12, ymm4"),
        Q!("    vpsubw          " "ymm8, ymm13, ymm8"),
        Q!("    vpsubw          " "ymm7, ymm14, ymm7"),
        Q!("    vpsubw          " "ymm11, ymm15, ymm11"),
        Q!("    vpmulhw         " "ymm12, ymm10, ymm1"),
        Q!("    vpsraw          " "ymm12, ymm12, 0xa"),
        Q!("    vpmullw         " "ymm12, ymm12, ymm0"),
        Q!("    vpsubw          " "ymm10, ymm10, ymm12"),
        Q!("    vpunpcklqdq     " "ymm9, ymm10, ymm3"),
        Q!("    vpunpckhqdq     " "ymm3, ymm10, ymm3"),
        Q!("    vpunpcklqdq     " "ymm10, ymm6, ymm5"),
        Q!("    vpunpckhqdq     " "ymm5, ymm6, ymm5"),
        Q!("    vpunpcklqdq     " "ymm6, ymm4, ymm8"),
        Q!("    vpunpckhqdq     " "ymm8, ymm4, ymm8"),
        Q!("    vpunpcklqdq     " "ymm4, ymm7, ymm11"),
        Q!("    vpunpckhqdq     " "ymm11, ymm7, ymm11"),
        Q!("    vpermq          " "ymm2, [160 + rsi], 0x4e"),
        Q!("    vpermq          " "ymm7, [192 + rsi], 0x4e"),
        Q!("    vpsubw          " "ymm12, ymm3, ymm9"),
        Q!("    vpaddw          " "ymm9, ymm9, ymm3"),
        Q!("    vpsubw          " "ymm13, ymm5, ymm10"),
        Q!("    vpmullw         " "ymm3, ymm12, ymm2"),
        Q!("    vpaddw          " "ymm10, ymm10, ymm5"),
        Q!("    vpsubw          " "ymm14, ymm8, ymm6"),
        Q!("    vpmullw         " "ymm5, ymm13, ymm2"),
        Q!("    vpaddw          " "ymm6, ymm6, ymm8"),
        Q!("    vpsubw          " "ymm15, ymm11, ymm4"),
        Q!("    vpmullw         " "ymm8, ymm14, ymm2"),
        Q!("    vpaddw          " "ymm4, ymm4, ymm11"),
        Q!("    vpmullw         " "ymm11, ymm15, ymm2"),
        Q!("    vpmulhw         " "ymm12, ymm12, ymm7"),
        Q!("    vpmulhw         " "ymm13, ymm13, ymm7"),
        Q!("    vpmulhw         " "ymm14, ymm14, ymm7"),
        Q!("    vpmulhw         " "ymm15, ymm15, ymm7"),
        Q!("    vpmulhw         " "ymm3, ymm3, ymm0"),
        Q!("    vpmulhw         " "ymm5, ymm5, ymm0"),
        Q!("    vpmulhw         " "ymm8, ymm8, ymm0"),
        Q!("    vpmulhw         " "ymm11, ymm11, ymm0"),
        Q!("    vpsubw          " "ymm3, ymm12, ymm3"),
        Q!("    vpsubw          " "ymm5, ymm13, ymm5"),
        Q!("    vpsubw          " "ymm8, ymm14, ymm8"),
        Q!("    vpsubw          " "ymm11, ymm15, ymm11"),
        Q!("    vpmulhw         " "ymm12, ymm9, ymm1"),
        Q!("    vpsraw          " "ymm12, ymm12, 0xa"),
        Q!("    vpmullw         " "ymm12, ymm12, ymm0"),
        Q!("    vpsubw          " "ymm9, ymm9, ymm12"),
        Q!("    vperm2i128      " "ymm7, ymm9, ymm10, 0x20"),
        Q!("    vperm2i128      " "ymm10, ymm9, ymm10, 0x31"),
        Q!("    vperm2i128      " "ymm9, ymm6, ymm4, 0x20"),
        Q!("    vperm2i128      " "ymm4, ymm6, ymm4, 0x31"),
        Q!("    vperm2i128      " "ymm6, ymm3, ymm5, 0x20"),
        Q!("    vperm2i128      " "ymm5, ymm3, ymm5, 0x31"),
        Q!("    vperm2i128      " "ymm3, ymm8, ymm11, 0x20"),
        Q!("    vperm2i128      " "ymm11, ymm8, ymm11, 0x31"),
        Q!("    vmovdqa         " "ymm2, [96 + rsi]"),
        Q!("    vmovdqa         " "ymm8, [128 + rsi]"),
        Q!("    vpsubw          " "ymm12, ymm10, ymm7"),
        Q!("    vpaddw          " "ymm7, ymm7, ymm10"),
        Q!("    vpsubw          " "ymm13, ymm4, ymm9"),
        Q!("    vpmullw         " "ymm10, ymm12, ymm2"),
        Q!("    vpaddw          " "ymm9, ymm9, ymm4"),
        Q!("    vpsubw          " "ymm14, ymm5, ymm6"),
        Q!("    vpmullw         " "ymm4, ymm13, ymm2"),
        Q!("    vpaddw          " "ymm6, ymm6, ymm5"),
        Q!("    vpsubw          " "ymm15, ymm11, ymm3"),
        Q!("    vpmullw         " "ymm5, ymm14, ymm2"),
        Q!("    vpaddw          " "ymm3, ymm3, ymm11"),
        Q!("    vpmullw         " "ymm11, ymm15, ymm2"),
        Q!("    vpmulhw         " "ymm12, ymm12, ymm8"),
        Q!("    vpmulhw         " "ymm13, ymm13, ymm8"),
        Q!("    vpmulhw         " "ymm14, ymm14, ymm8"),
        Q!("    vpmulhw         " "ymm15, ymm15, ymm8"),
        Q!("    vpmulhw         " "ymm10, ymm10, ymm0"),
        Q!("    vpmulhw         " "ymm4, ymm4, ymm0"),
        Q!("    vpmulhw         " "ymm5, ymm5, ymm0"),
        Q!("    vpmulhw         " "ymm11, ymm11, ymm0"),
        Q!("    vpsubw          " "ymm10, ymm12, ymm10"),
        Q!("    vpsubw          " "ymm4, ymm13, ymm4"),
        Q!("    vpsubw          " "ymm5, ymm14, ymm5"),
        Q!("    vpsubw          " "ymm11, ymm15, ymm11"),
        Q!("    vpmulhw         " "ymm12, ymm7, ymm1"),
        Q!("    vpsraw          " "ymm12, ymm12, 0xa"),
        Q!("    vpmullw         " "ymm12, ymm12, ymm0"),
        Q!("    vpsubw          " "ymm7, ymm7, ymm12"),
        Q!("    vmovdqa         " "[256 + rdi], ymm7"),
        Q!("    vmovdqa         " "[288 + rdi], ymm9"),
        Q!("    vmovdqa         " "[320 + rdi], ymm6"),
        Q!("    vmovdqa         " "[352 + rdi], ymm3"),
        Q!("    vmovdqa         " "[384 + rdi], ymm10"),
        Q!("    vmovdqa         " "[416 + rdi], ymm4"),
        Q!("    vmovdqa         " "[448 + rdi], ymm5"),
        Q!("    vmovdqa         " "[480 + rdi], ymm11"),
        Q!("    vmovdqa         " "ymm4, [rdi]"),
        Q!("    vmovdqa         " "ymm8, [256 + rdi]"),
        Q!("    vmovdqa         " "ymm5, [32 + rdi]"),
        Q!("    vmovdqa         " "ymm9, [288 + rdi]"),
        Q!("    vpbroadcastq    " "ymm2, [64 + rsi]"),
        Q!("    vmovdqa         " "ymm6, [64 + rdi]"),
        Q!("    vmovdqa         " "ymm10, [320 + rdi]"),
        Q!("    vmovdqa         " "ymm7, [96 + rdi]"),
        Q!("    vmovdqa         " "ymm11, [352 + rdi]"),
        Q!("    vpbroadcastq    " "ymm3, [72 + rsi]"),
        Q!("    vpsubw          " "ymm12, ymm8, ymm4"),
        Q!("    vpaddw          " "ymm4, ymm4, ymm8"),
        Q!("    vpsubw          " "ymm13, ymm9, ymm5"),
        Q!("    vpmullw         " "ymm8, ymm12, ymm2"),
        Q!("    vpaddw          " "ymm5, ymm5, ymm9"),
        Q!("    vpsubw          " "ymm14, ymm10, ymm6"),
        Q!("    vpmullw         " "ymm9, ymm13, ymm2"),
        Q!("    vpaddw          " "ymm6, ymm6, ymm10"),
        Q!("    vpsubw          " "ymm15, ymm11, ymm7"),
        Q!("    vpmullw         " "ymm10, ymm14, ymm2"),
        Q!("    vpaddw          " "ymm7, ymm7, ymm11"),
        Q!("    vpmullw         " "ymm11, ymm15, ymm2"),
        Q!("    vpmulhw         " "ymm12, ymm12, ymm3"),
        Q!("    vpmulhw         " "ymm13, ymm13, ymm3"),
        Q!("    vpmulhw         " "ymm14, ymm14, ymm3"),
        Q!("    vpmulhw         " "ymm15, ymm15, ymm3"),
        Q!("    vpmulhw         " "ymm8, ymm8, ymm0"),
        Q!("    vpmulhw         " "ymm9, ymm9, ymm0"),
        Q!("    vpmulhw         " "ymm10, ymm10, ymm0"),
        Q!("    vpmulhw         " "ymm11, ymm11, ymm0"),
        Q!("    vpsubw          " "ymm8, ymm12, ymm8"),
        Q!("    vpsubw          " "ymm9, ymm13, ymm9"),
        Q!("    vpsubw          " "ymm10, ymm14, ymm10"),
        Q!("    vpsubw          " "ymm11, ymm15, ymm11"),
        Q!("    vmovdqa         " "[rdi], ymm4"),
        Q!("    vmovdqa         " "[32 + rdi], ymm5"),
        Q!("    vmovdqa         " "[64 + rdi], ymm6"),
        Q!("    vmovdqa         " "[96 + rdi], ymm7"),
        Q!("    vmovdqa         " "[256 + rdi], ymm8"),
        Q!("    vmovdqa         " "[288 + rdi], ymm9"),
        Q!("    vmovdqa         " "[320 + rdi], ymm10"),
        Q!("    vmovdqa         " "[352 + rdi], ymm11"),
        Q!("    vmovdqa         " "ymm4, [128 + rdi]"),
        Q!("    vmovdqa         " "ymm8, [384 + rdi]"),
        Q!("    vmovdqa         " "ymm5, [160 + rdi]"),
        Q!("    vmovdqa         " "ymm9, [416 + rdi]"),
        Q!("    vpbroadcastq    " "ymm2, [64 + rsi]"),
        Q!("    vmovdqa         " "ymm6, [192 + rdi]"),
        Q!("    vmovdqa         " "ymm10, [448 + rdi]"),
        Q!("    vmovdqa         " "ymm7, [224 + rdi]"),
        Q!("    vmovdqa         " "ymm11, [480 + rdi]"),
        Q!("    vpbroadcastq    " "ymm3, [72 + rsi]"),
        Q!("    vpsubw          " "ymm12, ymm8, ymm4"),
        Q!("    vpaddw          " "ymm4, ymm4, ymm8"),
        Q!("    vpsubw          " "ymm13, ymm9, ymm5"),
        Q!("    vpmullw         " "ymm8, ymm12, ymm2"),
        Q!("    vpaddw          " "ymm5, ymm5, ymm9"),
        Q!("    vpsubw          " "ymm14, ymm10, ymm6"),
        Q!("    vpmullw         " "ymm9, ymm13, ymm2"),
        Q!("    vpaddw          " "ymm6, ymm6, ymm10"),
        Q!("    vpsubw          " "ymm15, ymm11, ymm7"),
        Q!("    vpmullw         " "ymm10, ymm14, ymm2"),
        Q!("    vpaddw          " "ymm7, ymm7, ymm11"),
        Q!("    vpmullw         " "ymm11, ymm15, ymm2"),
        Q!("    vpmulhw         " "ymm12, ymm12, ymm3"),
        Q!("    vpmulhw         " "ymm13, ymm13, ymm3"),
        Q!("    vpmulhw         " "ymm14, ymm14, ymm3"),
        Q!("    vpmulhw         " "ymm15, ymm15, ymm3"),
        Q!("    vpmulhw         " "ymm8, ymm8, ymm0"),
        Q!("    vpmulhw         " "ymm9, ymm9, ymm0"),
        Q!("    vpmulhw         " "ymm10, ymm10, ymm0"),
        Q!("    vpmulhw         " "ymm11, ymm11, ymm0"),
        Q!("    vpsubw          " "ymm8, ymm12, ymm8"),
        Q!("    vpsubw          " "ymm9, ymm13, ymm9"),
        Q!("    vpsubw          " "ymm10, ymm14, ymm10"),
        Q!("    vpsubw          " "ymm11, ymm15, ymm11"),
        Q!("    vmovdqa         " "[128 + rdi], ymm4"),
        Q!("    vmovdqa         " "[160 + rdi], ymm5"),
        Q!("    vmovdqa         " "[192 + rdi], ymm6"),
        Q!("    vmovdqa         " "[224 + rdi], ymm7"),
        Q!("    vmovdqa         " "[384 + rdi], ymm8"),
        Q!("    vmovdqa         " "[416 + rdi], ymm9"),
        Q!("    vmovdqa         " "[448 + rdi], ymm10"),
        Q!("    vmovdqa         " "[480 + rdi], ymm11"),

        inout("rdi") a.as_mut_ptr() => _,
        inout("rsi") qdata.as_ptr() => _,
        // clobbers
        out("rax") _,
        out("zmm0") _,
        out("zmm1") _,
        out("zmm10") _,
        out("zmm11") _,
        out("zmm12") _,
        out("zmm13") _,
        out("zmm14") _,
        out("zmm15") _,
        out("zmm2") _,
        out("zmm3") _,
        out("zmm4") _,
        out("zmm5") _,
        out("zmm6") _,
        out("zmm7") _,
        out("zmm8") _,
        out("zmm9") _,
            )
    };
}