graviola 0.4.0

graviola is a modern, fast cryptography library
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810

// generated source. do not edit.
#![allow(non_upper_case_globals, unused_macros, unused_imports)]
use crate::low::macros::*;

// Copyright (c) 2024 The mlkem-native project authors
// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
// SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT

// ----------------------------------------------------------------------------
// Scalar product of 3-element polynomial vectors in NTT domain, with mulcache
// Inputs a[768], b[768], bt[384] (signed 16-bit words); output r[256] (signed 16-bit words)
//
// The inputs a and b are considered as 3-element vectors of linear
// polynomials in the NTT domain (in Montgomery form), and the bt
// argument an analogous 3-element vector of mulcaches for the bi:
//
//   a0 = a[0..255], a1 = a[256..511], a2 = a[512..767]
//   b0 = b[0..255], b1 = b[256..511], b2 = b[512..767],
//   bt0 = bt[0..127], bt1 = bt[128..255], bt2 = bt[256..383]
//
// Scalar multiplication of those 3-element vectors is performed,
// with base multiplication in Fq[X]/(X^2-zeta^i'), with zeta^i'
// being a power of zeta = 17, with i bit-reversed as used for NTTs,
// making use of the mulcache for optimization.
//
// All input elements are assumed <= 2^12 and the bts are
// assumed to be as computed by mlkem_mulcache_compute.
//
// extern void mlkem_basemul_k3
//      (int16_t r[static 256],const int16_t a[static 768],
//       const int16_t b[static 768], const int16_t bt[static 384]);
//
// Standard x86-64 ABI: RDI = r, RSI = a, RDX = b, RCX = bt
// Microsoft x64 ABI:   RCX = r, RDX = a, R8 = b, R9 = bt
// ----------------------------------------------------------------------------

/// Scalar product of 3-element polynomial vectors in NTT domain, with mulcache
///
/// Inputs a[768], b[768], bt[384] (signed 16-bit words); output r[256] (signed 16-bit words)
///
/// The inputs a and b are considered as 3-element vectors of linear
/// polynomials in the NTT domain (in Montgomery form), and the bt
/// argument an analogous 3-element vector of mulcaches for the bi:
///
///   a0 = a[0..255], a1 = a[256..511], a2 = a[512..767]
///   b0 = b[0..255], b1 = b[256..511], b2 = b[512..767],
///   bt0 = bt[0..127], bt1 = bt[128..255], bt2 = bt[256..383]
///
/// Scalar multiplication of those 3-element vectors is performed,
/// with base multiplication in Fq[X]/(X^2-zeta^i'), with zeta^i'
/// being a power of zeta = 17, with i bit-reversed as used for NTTs,
/// making use of the mulcache for optimization.
///
/// All input elements are assumed <= 2^12 and the bts are
/// assumed to be as computed by mlkem_mulcache_compute.
pub(crate) fn mlkem_basemul_k3(
    r: &mut [i16; 256],
    a: &[i16; 768],
    b: &[i16; 768],
    bt: &[i16; 384],
) {
    // SAFETY: inline assembly. see [crate::low::inline_assembly_safety] for safety info.
    unsafe {
        core::arch::asm!(

        Q!("    endbr64         " ),

        Q!("    mov             " "eax, 0xd010d01"),
        Q!("    vmovd           " "xmm0, eax"),
        Q!("    vpbroadcastd    " "ymm0, xmm0"),
        Q!("    mov             " "eax, 0xf301f301"),
        Q!("    vmovd           " "xmm1, eax"),
        Q!("    vpbroadcastd    " "ymm1, xmm1"),
        Q!("    vmovdqa         " "ymm2, [rsi]"),
        Q!("    vmovdqa         " "ymm3, [32 + rsi]"),
        Q!("    vmovdqa         " "ymm4, [rdx]"),
        Q!("    vmovdqa         " "ymm5, [32 + rdx]"),
        Q!("    vmovdqa         " "ymm6, [rcx]"),
        Q!("    vpmullw         " "ymm13, ymm1, ymm2"),
        Q!("    vpmullw         " "ymm14, ymm1, ymm3"),
        Q!("    vpmullw         " "ymm7, ymm4, ymm13"),
        Q!("    vpmullw         " "ymm9, ymm5, ymm13"),
        Q!("    vpmullw         " "ymm8, ymm6, ymm14"),
        Q!("    vpmullw         " "ymm10, ymm4, ymm14"),
        Q!("    vpmulhw         " "ymm7, ymm0, ymm7"),
        Q!("    vpmulhw         " "ymm9, ymm0, ymm9"),
        Q!("    vpmulhw         " "ymm8, ymm0, ymm8"),
        Q!("    vpmulhw         " "ymm10, ymm0, ymm10"),
        Q!("    vpmulhw         " "ymm11, ymm4, ymm2"),
        Q!("    vpmulhw         " "ymm12, ymm5, ymm2"),
        Q!("    vpmulhw         " "ymm13, ymm6, ymm3"),
        Q!("    vpmulhw         " "ymm14, ymm4, ymm3"),
        Q!("    vpsubw          " "ymm7, ymm11, ymm7"),
        Q!("    vpsubw          " "ymm9, ymm12, ymm9"),
        Q!("    vpsubw          " "ymm8, ymm13, ymm8"),
        Q!("    vpsubw          " "ymm10, ymm14, ymm10"),
        Q!("    vpaddw          " "ymm7, ymm8, ymm7"),
        Q!("    vpaddw          " "ymm9, ymm10, ymm9"),
        Q!("    vmovdqa         " "[rdi], ymm7"),
        Q!("    vmovdqa         " "[32 + rdi], ymm9"),
        Q!("    vmovdqa         " "ymm2, [64 + rsi]"),
        Q!("    vmovdqa         " "ymm3, [96 + rsi]"),
        Q!("    vmovdqa         " "ymm4, [64 + rdx]"),
        Q!("    vmovdqa         " "ymm5, [96 + rdx]"),
        Q!("    vmovdqa         " "ymm6, [32 + rcx]"),
        Q!("    vpmullw         " "ymm13, ymm1, ymm2"),
        Q!("    vpmullw         " "ymm14, ymm1, ymm3"),
        Q!("    vpmullw         " "ymm7, ymm4, ymm13"),
        Q!("    vpmullw         " "ymm9, ymm5, ymm13"),
        Q!("    vpmullw         " "ymm8, ymm6, ymm14"),
        Q!("    vpmullw         " "ymm10, ymm4, ymm14"),
        Q!("    vpmulhw         " "ymm7, ymm0, ymm7"),
        Q!("    vpmulhw         " "ymm9, ymm0, ymm9"),
        Q!("    vpmulhw         " "ymm8, ymm0, ymm8"),
        Q!("    vpmulhw         " "ymm10, ymm0, ymm10"),
        Q!("    vpmulhw         " "ymm11, ymm4, ymm2"),
        Q!("    vpmulhw         " "ymm12, ymm5, ymm2"),
        Q!("    vpmulhw         " "ymm13, ymm6, ymm3"),
        Q!("    vpmulhw         " "ymm14, ymm4, ymm3"),
        Q!("    vpsubw          " "ymm7, ymm11, ymm7"),
        Q!("    vpsubw          " "ymm9, ymm12, ymm9"),
        Q!("    vpsubw          " "ymm8, ymm8, ymm13"),
        Q!("    vpsubw          " "ymm10, ymm14, ymm10"),
        Q!("    vpaddw          " "ymm7, ymm8, ymm7"),
        Q!("    vpaddw          " "ymm9, ymm10, ymm9"),
        Q!("    vmovdqa         " "[64 + rdi], ymm7"),
        Q!("    vmovdqa         " "[96 + rdi], ymm9"),
        Q!("    vmovdqa         " "ymm2, [128 + rsi]"),
        Q!("    vmovdqa         " "ymm3, [160 + rsi]"),
        Q!("    vmovdqa         " "ymm4, [128 + rdx]"),
        Q!("    vmovdqa         " "ymm5, [160 + rdx]"),
        Q!("    vmovdqa         " "ymm6, [64 + rcx]"),
        Q!("    vpmullw         " "ymm13, ymm1, ymm2"),
        Q!("    vpmullw         " "ymm14, ymm1, ymm3"),
        Q!("    vpmullw         " "ymm7, ymm4, ymm13"),
        Q!("    vpmullw         " "ymm9, ymm5, ymm13"),
        Q!("    vpmullw         " "ymm8, ymm6, ymm14"),
        Q!("    vpmullw         " "ymm10, ymm4, ymm14"),
        Q!("    vpmulhw         " "ymm7, ymm0, ymm7"),
        Q!("    vpmulhw         " "ymm9, ymm0, ymm9"),
        Q!("    vpmulhw         " "ymm8, ymm0, ymm8"),
        Q!("    vpmulhw         " "ymm10, ymm0, ymm10"),
        Q!("    vpmulhw         " "ymm11, ymm4, ymm2"),
        Q!("    vpmulhw         " "ymm12, ymm5, ymm2"),
        Q!("    vpmulhw         " "ymm13, ymm6, ymm3"),
        Q!("    vpmulhw         " "ymm14, ymm4, ymm3"),
        Q!("    vpsubw          " "ymm7, ymm11, ymm7"),
        Q!("    vpsubw          " "ymm9, ymm12, ymm9"),
        Q!("    vpsubw          " "ymm8, ymm13, ymm8"),
        Q!("    vpsubw          " "ymm10, ymm14, ymm10"),
        Q!("    vpaddw          " "ymm7, ymm8, ymm7"),
        Q!("    vpaddw          " "ymm9, ymm10, ymm9"),
        Q!("    vmovdqa         " "[128 + rdi], ymm7"),
        Q!("    vmovdqa         " "[160 + rdi], ymm9"),
        Q!("    vmovdqa         " "ymm2, [192 + rsi]"),
        Q!("    vmovdqa         " "ymm3, [224 + rsi]"),
        Q!("    vmovdqa         " "ymm4, [192 + rdx]"),
        Q!("    vmovdqa         " "ymm5, [224 + rdx]"),
        Q!("    vmovdqa         " "ymm6, [96 + rcx]"),
        Q!("    vpmullw         " "ymm13, ymm1, ymm2"),
        Q!("    vpmullw         " "ymm14, ymm1, ymm3"),
        Q!("    vpmullw         " "ymm7, ymm4, ymm13"),
        Q!("    vpmullw         " "ymm9, ymm5, ymm13"),
        Q!("    vpmullw         " "ymm8, ymm6, ymm14"),
        Q!("    vpmullw         " "ymm10, ymm4, ymm14"),
        Q!("    vpmulhw         " "ymm7, ymm0, ymm7"),
        Q!("    vpmulhw         " "ymm9, ymm0, ymm9"),
        Q!("    vpmulhw         " "ymm8, ymm0, ymm8"),
        Q!("    vpmulhw         " "ymm10, ymm0, ymm10"),
        Q!("    vpmulhw         " "ymm11, ymm4, ymm2"),
        Q!("    vpmulhw         " "ymm12, ymm5, ymm2"),
        Q!("    vpmulhw         " "ymm13, ymm6, ymm3"),
        Q!("    vpmulhw         " "ymm14, ymm4, ymm3"),
        Q!("    vpsubw          " "ymm7, ymm11, ymm7"),
        Q!("    vpsubw          " "ymm9, ymm12, ymm9"),
        Q!("    vpsubw          " "ymm8, ymm8, ymm13"),
        Q!("    vpsubw          " "ymm10, ymm14, ymm10"),
        Q!("    vpaddw          " "ymm7, ymm8, ymm7"),
        Q!("    vpaddw          " "ymm9, ymm10, ymm9"),
        Q!("    vmovdqa         " "[192 + rdi], ymm7"),
        Q!("    vmovdqa         " "[224 + rdi], ymm9"),
        Q!("    vmovdqa         " "ymm2, [256 + rsi]"),
        Q!("    vmovdqa         " "ymm3, [288 + rsi]"),
        Q!("    vmovdqa         " "ymm4, [256 + rdx]"),
        Q!("    vmovdqa         " "ymm5, [288 + rdx]"),
        Q!("    vmovdqa         " "ymm6, [128 + rcx]"),
        Q!("    vpmullw         " "ymm13, ymm1, ymm2"),
        Q!("    vpmullw         " "ymm14, ymm1, ymm3"),
        Q!("    vpmullw         " "ymm7, ymm4, ymm13"),
        Q!("    vpmullw         " "ymm9, ymm5, ymm13"),
        Q!("    vpmullw         " "ymm8, ymm6, ymm14"),
        Q!("    vpmullw         " "ymm10, ymm4, ymm14"),
        Q!("    vpmulhw         " "ymm7, ymm0, ymm7"),
        Q!("    vpmulhw         " "ymm9, ymm0, ymm9"),
        Q!("    vpmulhw         " "ymm8, ymm0, ymm8"),
        Q!("    vpmulhw         " "ymm10, ymm0, ymm10"),
        Q!("    vpmulhw         " "ymm11, ymm4, ymm2"),
        Q!("    vpmulhw         " "ymm12, ymm5, ymm2"),
        Q!("    vpmulhw         " "ymm13, ymm6, ymm3"),
        Q!("    vpmulhw         " "ymm14, ymm4, ymm3"),
        Q!("    vpsubw          " "ymm7, ymm11, ymm7"),
        Q!("    vpsubw          " "ymm9, ymm12, ymm9"),
        Q!("    vpsubw          " "ymm8, ymm13, ymm8"),
        Q!("    vpsubw          " "ymm10, ymm14, ymm10"),
        Q!("    vpaddw          " "ymm7, ymm8, ymm7"),
        Q!("    vpaddw          " "ymm9, ymm10, ymm9"),
        Q!("    vmovdqa         " "[256 + rdi], ymm7"),
        Q!("    vmovdqa         " "[288 + rdi], ymm9"),
        Q!("    vmovdqa         " "ymm2, [320 + rsi]"),
        Q!("    vmovdqa         " "ymm3, [352 + rsi]"),
        Q!("    vmovdqa         " "ymm4, [320 + rdx]"),
        Q!("    vmovdqa         " "ymm5, [352 + rdx]"),
        Q!("    vmovdqa         " "ymm6, [160 + rcx]"),
        Q!("    vpmullw         " "ymm13, ymm1, ymm2"),
        Q!("    vpmullw         " "ymm14, ymm1, ymm3"),
        Q!("    vpmullw         " "ymm7, ymm4, ymm13"),
        Q!("    vpmullw         " "ymm9, ymm5, ymm13"),
        Q!("    vpmullw         " "ymm8, ymm6, ymm14"),
        Q!("    vpmullw         " "ymm10, ymm4, ymm14"),
        Q!("    vpmulhw         " "ymm7, ymm0, ymm7"),
        Q!("    vpmulhw         " "ymm9, ymm0, ymm9"),
        Q!("    vpmulhw         " "ymm8, ymm0, ymm8"),
        Q!("    vpmulhw         " "ymm10, ymm0, ymm10"),
        Q!("    vpmulhw         " "ymm11, ymm4, ymm2"),
        Q!("    vpmulhw         " "ymm12, ymm5, ymm2"),
        Q!("    vpmulhw         " "ymm13, ymm6, ymm3"),
        Q!("    vpmulhw         " "ymm14, ymm4, ymm3"),
        Q!("    vpsubw          " "ymm7, ymm11, ymm7"),
        Q!("    vpsubw          " "ymm9, ymm12, ymm9"),
        Q!("    vpsubw          " "ymm8, ymm8, ymm13"),
        Q!("    vpsubw          " "ymm10, ymm14, ymm10"),
        Q!("    vpaddw          " "ymm7, ymm8, ymm7"),
        Q!("    vpaddw          " "ymm9, ymm10, ymm9"),
        Q!("    vmovdqa         " "[320 + rdi], ymm7"),
        Q!("    vmovdqa         " "[352 + rdi], ymm9"),
        Q!("    vmovdqa         " "ymm2, [384 + rsi]"),
        Q!("    vmovdqa         " "ymm3, [416 + rsi]"),
        Q!("    vmovdqa         " "ymm4, [384 + rdx]"),
        Q!("    vmovdqa         " "ymm5, [416 + rdx]"),
        Q!("    vmovdqa         " "ymm6, [192 + rcx]"),
        Q!("    vpmullw         " "ymm13, ymm1, ymm2"),
        Q!("    vpmullw         " "ymm14, ymm1, ymm3"),
        Q!("    vpmullw         " "ymm7, ymm4, ymm13"),
        Q!("    vpmullw         " "ymm9, ymm5, ymm13"),
        Q!("    vpmullw         " "ymm8, ymm6, ymm14"),
        Q!("    vpmullw         " "ymm10, ymm4, ymm14"),
        Q!("    vpmulhw         " "ymm7, ymm0, ymm7"),
        Q!("    vpmulhw         " "ymm9, ymm0, ymm9"),
        Q!("    vpmulhw         " "ymm8, ymm0, ymm8"),
        Q!("    vpmulhw         " "ymm10, ymm0, ymm10"),
        Q!("    vpmulhw         " "ymm11, ymm4, ymm2"),
        Q!("    vpmulhw         " "ymm12, ymm5, ymm2"),
        Q!("    vpmulhw         " "ymm13, ymm6, ymm3"),
        Q!("    vpmulhw         " "ymm14, ymm4, ymm3"),
        Q!("    vpsubw          " "ymm7, ymm11, ymm7"),
        Q!("    vpsubw          " "ymm9, ymm12, ymm9"),
        Q!("    vpsubw          " "ymm8, ymm13, ymm8"),
        Q!("    vpsubw          " "ymm10, ymm14, ymm10"),
        Q!("    vpaddw          " "ymm7, ymm8, ymm7"),
        Q!("    vpaddw          " "ymm9, ymm10, ymm9"),
        Q!("    vmovdqa         " "[384 + rdi], ymm7"),
        Q!("    vmovdqa         " "[416 + rdi], ymm9"),
        Q!("    vmovdqa         " "ymm2, [448 + rsi]"),
        Q!("    vmovdqa         " "ymm3, [480 + rsi]"),
        Q!("    vmovdqa         " "ymm4, [448 + rdx]"),
        Q!("    vmovdqa         " "ymm5, [480 + rdx]"),
        Q!("    vmovdqa         " "ymm6, [224 + rcx]"),
        Q!("    vpmullw         " "ymm13, ymm1, ymm2"),
        Q!("    vpmullw         " "ymm14, ymm1, ymm3"),
        Q!("    vpmullw         " "ymm7, ymm4, ymm13"),
        Q!("    vpmullw         " "ymm9, ymm5, ymm13"),
        Q!("    vpmullw         " "ymm8, ymm6, ymm14"),
        Q!("    vpmullw         " "ymm10, ymm4, ymm14"),
        Q!("    vpmulhw         " "ymm7, ymm0, ymm7"),
        Q!("    vpmulhw         " "ymm9, ymm0, ymm9"),
        Q!("    vpmulhw         " "ymm8, ymm0, ymm8"),
        Q!("    vpmulhw         " "ymm10, ymm0, ymm10"),
        Q!("    vpmulhw         " "ymm11, ymm4, ymm2"),
        Q!("    vpmulhw         " "ymm12, ymm5, ymm2"),
        Q!("    vpmulhw         " "ymm13, ymm6, ymm3"),
        Q!("    vpmulhw         " "ymm14, ymm4, ymm3"),
        Q!("    vpsubw          " "ymm7, ymm11, ymm7"),
        Q!("    vpsubw          " "ymm9, ymm12, ymm9"),
        Q!("    vpsubw          " "ymm8, ymm8, ymm13"),
        Q!("    vpsubw          " "ymm10, ymm14, ymm10"),
        Q!("    vpaddw          " "ymm7, ymm8, ymm7"),
        Q!("    vpaddw          " "ymm9, ymm10, ymm9"),
        Q!("    vmovdqa         " "[448 + rdi], ymm7"),
        Q!("    vmovdqa         " "[480 + rdi], ymm9"),
        Q!("    vmovdqa         " "ymm2, [512 + rsi]"),
        Q!("    vmovdqa         " "ymm3, [544 + rsi]"),
        Q!("    vmovdqa         " "ymm4, [512 + rdx]"),
        Q!("    vmovdqa         " "ymm5, [544 + rdx]"),
        Q!("    vmovdqa         " "ymm6, [256 + rcx]"),
        Q!("    vpmullw         " "ymm13, ymm1, ymm2"),
        Q!("    vpmullw         " "ymm14, ymm1, ymm3"),
        Q!("    vpmullw         " "ymm7, ymm4, ymm13"),
        Q!("    vpmullw         " "ymm9, ymm5, ymm13"),
        Q!("    vpmullw         " "ymm8, ymm6, ymm14"),
        Q!("    vpmullw         " "ymm10, ymm4, ymm14"),
        Q!("    vpmulhw         " "ymm7, ymm0, ymm7"),
        Q!("    vpmulhw         " "ymm9, ymm0, ymm9"),
        Q!("    vpmulhw         " "ymm8, ymm0, ymm8"),
        Q!("    vpmulhw         " "ymm10, ymm0, ymm10"),
        Q!("    vpmulhw         " "ymm11, ymm4, ymm2"),
        Q!("    vpmulhw         " "ymm12, ymm5, ymm2"),
        Q!("    vpmulhw         " "ymm13, ymm6, ymm3"),
        Q!("    vpmulhw         " "ymm14, ymm4, ymm3"),
        Q!("    vpsubw          " "ymm7, ymm11, ymm7"),
        Q!("    vpsubw          " "ymm9, ymm12, ymm9"),
        Q!("    vpsubw          " "ymm8, ymm13, ymm8"),
        Q!("    vpsubw          " "ymm10, ymm14, ymm10"),
        Q!("    vpaddw          " "ymm7, ymm8, ymm7"),
        Q!("    vpaddw          " "ymm9, ymm10, ymm9"),
        Q!("    vmovdqa         " "ymm8, [rdi]"),
        Q!("    vmovdqa         " "ymm10, [32 + rdi]"),
        Q!("    vpaddw          " "ymm7, ymm8, ymm7"),
        Q!("    vpaddw          " "ymm9, ymm10, ymm9"),
        Q!("    vmovdqa         " "[rdi], ymm7"),
        Q!("    vmovdqa         " "[32 + rdi], ymm9"),
        Q!("    vmovdqa         " "ymm2, [576 + rsi]"),
        Q!("    vmovdqa         " "ymm3, [608 + rsi]"),
        Q!("    vmovdqa         " "ymm4, [576 + rdx]"),
        Q!("    vmovdqa         " "ymm5, [608 + rdx]"),
        Q!("    vmovdqa         " "ymm6, [288 + rcx]"),
        Q!("    vpmullw         " "ymm13, ymm1, ymm2"),
        Q!("    vpmullw         " "ymm14, ymm1, ymm3"),
        Q!("    vpmullw         " "ymm7, ymm4, ymm13"),
        Q!("    vpmullw         " "ymm9, ymm5, ymm13"),
        Q!("    vpmullw         " "ymm8, ymm6, ymm14"),
        Q!("    vpmullw         " "ymm10, ymm4, ymm14"),
        Q!("    vpmulhw         " "ymm7, ymm0, ymm7"),
        Q!("    vpmulhw         " "ymm9, ymm0, ymm9"),
        Q!("    vpmulhw         " "ymm8, ymm0, ymm8"),
        Q!("    vpmulhw         " "ymm10, ymm0, ymm10"),
        Q!("    vpmulhw         " "ymm11, ymm4, ymm2"),
        Q!("    vpmulhw         " "ymm12, ymm5, ymm2"),
        Q!("    vpmulhw         " "ymm13, ymm6, ymm3"),
        Q!("    vpmulhw         " "ymm14, ymm4, ymm3"),
        Q!("    vpsubw          " "ymm7, ymm11, ymm7"),
        Q!("    vpsubw          " "ymm9, ymm12, ymm9"),
        Q!("    vpsubw          " "ymm8, ymm8, ymm13"),
        Q!("    vpsubw          " "ymm10, ymm14, ymm10"),
        Q!("    vpaddw          " "ymm7, ymm8, ymm7"),
        Q!("    vpaddw          " "ymm9, ymm10, ymm9"),
        Q!("    vmovdqa         " "ymm8, [64 + rdi]"),
        Q!("    vmovdqa         " "ymm10, [96 + rdi]"),
        Q!("    vpaddw          " "ymm7, ymm8, ymm7"),
        Q!("    vpaddw          " "ymm9, ymm10, ymm9"),
        Q!("    vmovdqa         " "[64 + rdi], ymm7"),
        Q!("    vmovdqa         " "[96 + rdi], ymm9"),
        Q!("    vmovdqa         " "ymm2, [640 + rsi]"),
        Q!("    vmovdqa         " "ymm3, [672 + rsi]"),
        Q!("    vmovdqa         " "ymm4, [640 + rdx]"),
        Q!("    vmovdqa         " "ymm5, [672 + rdx]"),
        Q!("    vmovdqa         " "ymm6, [320 + rcx]"),
        Q!("    vpmullw         " "ymm13, ymm1, ymm2"),
        Q!("    vpmullw         " "ymm14, ymm1, ymm3"),
        Q!("    vpmullw         " "ymm7, ymm4, ymm13"),
        Q!("    vpmullw         " "ymm9, ymm5, ymm13"),
        Q!("    vpmullw         " "ymm8, ymm6, ymm14"),
        Q!("    vpmullw         " "ymm10, ymm4, ymm14"),
        Q!("    vpmulhw         " "ymm7, ymm0, ymm7"),
        Q!("    vpmulhw         " "ymm9, ymm0, ymm9"),
        Q!("    vpmulhw         " "ymm8, ymm0, ymm8"),
        Q!("    vpmulhw         " "ymm10, ymm0, ymm10"),
        Q!("    vpmulhw         " "ymm11, ymm4, ymm2"),
        Q!("    vpmulhw         " "ymm12, ymm5, ymm2"),
        Q!("    vpmulhw         " "ymm13, ymm6, ymm3"),
        Q!("    vpmulhw         " "ymm14, ymm4, ymm3"),
        Q!("    vpsubw          " "ymm7, ymm11, ymm7"),
        Q!("    vpsubw          " "ymm9, ymm12, ymm9"),
        Q!("    vpsubw          " "ymm8, ymm13, ymm8"),
        Q!("    vpsubw          " "ymm10, ymm14, ymm10"),
        Q!("    vpaddw          " "ymm7, ymm8, ymm7"),
        Q!("    vpaddw          " "ymm9, ymm10, ymm9"),
        Q!("    vmovdqa         " "ymm8, [128 + rdi]"),
        Q!("    vmovdqa         " "ymm10, [160 + rdi]"),
        Q!("    vpaddw          " "ymm7, ymm8, ymm7"),
        Q!("    vpaddw          " "ymm9, ymm10, ymm9"),
        Q!("    vmovdqa         " "[128 + rdi], ymm7"),
        Q!("    vmovdqa         " "[160 + rdi], ymm9"),
        Q!("    vmovdqa         " "ymm2, [704 + rsi]"),
        Q!("    vmovdqa         " "ymm3, [736 + rsi]"),
        Q!("    vmovdqa         " "ymm4, [704 + rdx]"),
        Q!("    vmovdqa         " "ymm5, [736 + rdx]"),
        Q!("    vmovdqa         " "ymm6, [352 + rcx]"),
        Q!("    vpmullw         " "ymm13, ymm1, ymm2"),
        Q!("    vpmullw         " "ymm14, ymm1, ymm3"),
        Q!("    vpmullw         " "ymm7, ymm4, ymm13"),
        Q!("    vpmullw         " "ymm9, ymm5, ymm13"),
        Q!("    vpmullw         " "ymm8, ymm6, ymm14"),
        Q!("    vpmullw         " "ymm10, ymm4, ymm14"),
        Q!("    vpmulhw         " "ymm7, ymm0, ymm7"),
        Q!("    vpmulhw         " "ymm9, ymm0, ymm9"),
        Q!("    vpmulhw         " "ymm8, ymm0, ymm8"),
        Q!("    vpmulhw         " "ymm10, ymm0, ymm10"),
        Q!("    vpmulhw         " "ymm11, ymm4, ymm2"),
        Q!("    vpmulhw         " "ymm12, ymm5, ymm2"),
        Q!("    vpmulhw         " "ymm13, ymm6, ymm3"),
        Q!("    vpmulhw         " "ymm14, ymm4, ymm3"),
        Q!("    vpsubw          " "ymm7, ymm11, ymm7"),
        Q!("    vpsubw          " "ymm9, ymm12, ymm9"),
        Q!("    vpsubw          " "ymm8, ymm8, ymm13"),
        Q!("    vpsubw          " "ymm10, ymm14, ymm10"),
        Q!("    vpaddw          " "ymm7, ymm8, ymm7"),
        Q!("    vpaddw          " "ymm9, ymm10, ymm9"),
        Q!("    vmovdqa         " "ymm8, [192 + rdi]"),
        Q!("    vmovdqa         " "ymm10, [224 + rdi]"),
        Q!("    vpaddw          " "ymm7, ymm8, ymm7"),
        Q!("    vpaddw          " "ymm9, ymm10, ymm9"),
        Q!("    vmovdqa         " "[192 + rdi], ymm7"),
        Q!("    vmovdqa         " "[224 + rdi], ymm9"),
        Q!("    vmovdqa         " "ymm2, [768 + rsi]"),
        Q!("    vmovdqa         " "ymm3, [800 + rsi]"),
        Q!("    vmovdqa         " "ymm4, [768 + rdx]"),
        Q!("    vmovdqa         " "ymm5, [800 + rdx]"),
        Q!("    vmovdqa         " "ymm6, [384 + rcx]"),
        Q!("    vpmullw         " "ymm13, ymm1, ymm2"),
        Q!("    vpmullw         " "ymm14, ymm1, ymm3"),
        Q!("    vpmullw         " "ymm7, ymm4, ymm13"),
        Q!("    vpmullw         " "ymm9, ymm5, ymm13"),
        Q!("    vpmullw         " "ymm8, ymm6, ymm14"),
        Q!("    vpmullw         " "ymm10, ymm4, ymm14"),
        Q!("    vpmulhw         " "ymm7, ymm0, ymm7"),
        Q!("    vpmulhw         " "ymm9, ymm0, ymm9"),
        Q!("    vpmulhw         " "ymm8, ymm0, ymm8"),
        Q!("    vpmulhw         " "ymm10, ymm0, ymm10"),
        Q!("    vpmulhw         " "ymm11, ymm4, ymm2"),
        Q!("    vpmulhw         " "ymm12, ymm5, ymm2"),
        Q!("    vpmulhw         " "ymm13, ymm6, ymm3"),
        Q!("    vpmulhw         " "ymm14, ymm4, ymm3"),
        Q!("    vpsubw          " "ymm7, ymm11, ymm7"),
        Q!("    vpsubw          " "ymm9, ymm12, ymm9"),
        Q!("    vpsubw          " "ymm8, ymm13, ymm8"),
        Q!("    vpsubw          " "ymm10, ymm14, ymm10"),
        Q!("    vpaddw          " "ymm7, ymm8, ymm7"),
        Q!("    vpaddw          " "ymm9, ymm10, ymm9"),
        Q!("    vmovdqa         " "ymm8, [256 + rdi]"),
        Q!("    vmovdqa         " "ymm10, [288 + rdi]"),
        Q!("    vpaddw          " "ymm7, ymm8, ymm7"),
        Q!("    vpaddw          " "ymm9, ymm10, ymm9"),
        Q!("    vmovdqa         " "[256 + rdi], ymm7"),
        Q!("    vmovdqa         " "[288 + rdi], ymm9"),
        Q!("    vmovdqa         " "ymm2, [832 + rsi]"),
        Q!("    vmovdqa         " "ymm3, [864 + rsi]"),
        Q!("    vmovdqa         " "ymm4, [832 + rdx]"),
        Q!("    vmovdqa         " "ymm5, [864 + rdx]"),
        Q!("    vmovdqa         " "ymm6, [416 + rcx]"),
        Q!("    vpmullw         " "ymm13, ymm1, ymm2"),
        Q!("    vpmullw         " "ymm14, ymm1, ymm3"),
        Q!("    vpmullw         " "ymm7, ymm4, ymm13"),
        Q!("    vpmullw         " "ymm9, ymm5, ymm13"),
        Q!("    vpmullw         " "ymm8, ymm6, ymm14"),
        Q!("    vpmullw         " "ymm10, ymm4, ymm14"),
        Q!("    vpmulhw         " "ymm7, ymm0, ymm7"),
        Q!("    vpmulhw         " "ymm9, ymm0, ymm9"),
        Q!("    vpmulhw         " "ymm8, ymm0, ymm8"),
        Q!("    vpmulhw         " "ymm10, ymm0, ymm10"),
        Q!("    vpmulhw         " "ymm11, ymm4, ymm2"),
        Q!("    vpmulhw         " "ymm12, ymm5, ymm2"),
        Q!("    vpmulhw         " "ymm13, ymm6, ymm3"),
        Q!("    vpmulhw         " "ymm14, ymm4, ymm3"),
        Q!("    vpsubw          " "ymm7, ymm11, ymm7"),
        Q!("    vpsubw          " "ymm9, ymm12, ymm9"),
        Q!("    vpsubw          " "ymm8, ymm8, ymm13"),
        Q!("    vpsubw          " "ymm10, ymm14, ymm10"),
        Q!("    vpaddw          " "ymm7, ymm8, ymm7"),
        Q!("    vpaddw          " "ymm9, ymm10, ymm9"),
        Q!("    vmovdqa         " "ymm8, [320 + rdi]"),
        Q!("    vmovdqa         " "ymm10, [352 + rdi]"),
        Q!("    vpaddw          " "ymm7, ymm8, ymm7"),
        Q!("    vpaddw          " "ymm9, ymm10, ymm9"),
        Q!("    vmovdqa         " "[320 + rdi], ymm7"),
        Q!("    vmovdqa         " "[352 + rdi], ymm9"),
        Q!("    vmovdqa         " "ymm2, [896 + rsi]"),
        Q!("    vmovdqa         " "ymm3, [928 + rsi]"),
        Q!("    vmovdqa         " "ymm4, [896 + rdx]"),
        Q!("    vmovdqa         " "ymm5, [928 + rdx]"),
        Q!("    vmovdqa         " "ymm6, [448 + rcx]"),
        Q!("    vpmullw         " "ymm13, ymm1, ymm2"),
        Q!("    vpmullw         " "ymm14, ymm1, ymm3"),
        Q!("    vpmullw         " "ymm7, ymm4, ymm13"),
        Q!("    vpmullw         " "ymm9, ymm5, ymm13"),
        Q!("    vpmullw         " "ymm8, ymm6, ymm14"),
        Q!("    vpmullw         " "ymm10, ymm4, ymm14"),
        Q!("    vpmulhw         " "ymm7, ymm0, ymm7"),
        Q!("    vpmulhw         " "ymm9, ymm0, ymm9"),
        Q!("    vpmulhw         " "ymm8, ymm0, ymm8"),
        Q!("    vpmulhw         " "ymm10, ymm0, ymm10"),
        Q!("    vpmulhw         " "ymm11, ymm4, ymm2"),
        Q!("    vpmulhw         " "ymm12, ymm5, ymm2"),
        Q!("    vpmulhw         " "ymm13, ymm6, ymm3"),
        Q!("    vpmulhw         " "ymm14, ymm4, ymm3"),
        Q!("    vpsubw          " "ymm7, ymm11, ymm7"),
        Q!("    vpsubw          " "ymm9, ymm12, ymm9"),
        Q!("    vpsubw          " "ymm8, ymm13, ymm8"),
        Q!("    vpsubw          " "ymm10, ymm14, ymm10"),
        Q!("    vpaddw          " "ymm7, ymm8, ymm7"),
        Q!("    vpaddw          " "ymm9, ymm10, ymm9"),
        Q!("    vmovdqa         " "ymm8, [384 + rdi]"),
        Q!("    vmovdqa         " "ymm10, [416 + rdi]"),
        Q!("    vpaddw          " "ymm7, ymm8, ymm7"),
        Q!("    vpaddw          " "ymm9, ymm10, ymm9"),
        Q!("    vmovdqa         " "[384 + rdi], ymm7"),
        Q!("    vmovdqa         " "[416 + rdi], ymm9"),
        Q!("    vmovdqa         " "ymm2, [960 + rsi]"),
        Q!("    vmovdqa         " "ymm3, [992 + rsi]"),
        Q!("    vmovdqa         " "ymm4, [960 + rdx]"),
        Q!("    vmovdqa         " "ymm5, [992 + rdx]"),
        Q!("    vmovdqa         " "ymm6, [480 + rcx]"),
        Q!("    vpmullw         " "ymm13, ymm1, ymm2"),
        Q!("    vpmullw         " "ymm14, ymm1, ymm3"),
        Q!("    vpmullw         " "ymm7, ymm4, ymm13"),
        Q!("    vpmullw         " "ymm9, ymm5, ymm13"),
        Q!("    vpmullw         " "ymm8, ymm6, ymm14"),
        Q!("    vpmullw         " "ymm10, ymm4, ymm14"),
        Q!("    vpmulhw         " "ymm7, ymm0, ymm7"),
        Q!("    vpmulhw         " "ymm9, ymm0, ymm9"),
        Q!("    vpmulhw         " "ymm8, ymm0, ymm8"),
        Q!("    vpmulhw         " "ymm10, ymm0, ymm10"),
        Q!("    vpmulhw         " "ymm11, ymm4, ymm2"),
        Q!("    vpmulhw         " "ymm12, ymm5, ymm2"),
        Q!("    vpmulhw         " "ymm13, ymm6, ymm3"),
        Q!("    vpmulhw         " "ymm14, ymm4, ymm3"),
        Q!("    vpsubw          " "ymm7, ymm11, ymm7"),
        Q!("    vpsubw          " "ymm9, ymm12, ymm9"),
        Q!("    vpsubw          " "ymm8, ymm8, ymm13"),
        Q!("    vpsubw          " "ymm10, ymm14, ymm10"),
        Q!("    vpaddw          " "ymm7, ymm8, ymm7"),
        Q!("    vpaddw          " "ymm9, ymm10, ymm9"),
        Q!("    vmovdqa         " "ymm8, [448 + rdi]"),
        Q!("    vmovdqa         " "ymm10, [480 + rdi]"),
        Q!("    vpaddw          " "ymm7, ymm8, ymm7"),
        Q!("    vpaddw          " "ymm9, ymm10, ymm9"),
        Q!("    vmovdqa         " "[448 + rdi], ymm7"),
        Q!("    vmovdqa         " "[480 + rdi], ymm9"),
        Q!("    vmovdqa         " "ymm2, [1024 + rsi]"),
        Q!("    vmovdqa         " "ymm3, [1056 + rsi]"),
        Q!("    vmovdqa         " "ymm4, [1024 + rdx]"),
        Q!("    vmovdqa         " "ymm5, [1056 + rdx]"),
        Q!("    vmovdqa         " "ymm6, [512 + rcx]"),
        Q!("    vpmullw         " "ymm13, ymm1, ymm2"),
        Q!("    vpmullw         " "ymm14, ymm1, ymm3"),
        Q!("    vpmullw         " "ymm7, ymm4, ymm13"),
        Q!("    vpmullw         " "ymm9, ymm5, ymm13"),
        Q!("    vpmullw         " "ymm8, ymm6, ymm14"),
        Q!("    vpmullw         " "ymm10, ymm4, ymm14"),
        Q!("    vpmulhw         " "ymm7, ymm0, ymm7"),
        Q!("    vpmulhw         " "ymm9, ymm0, ymm9"),
        Q!("    vpmulhw         " "ymm8, ymm0, ymm8"),
        Q!("    vpmulhw         " "ymm10, ymm0, ymm10"),
        Q!("    vpmulhw         " "ymm11, ymm4, ymm2"),
        Q!("    vpmulhw         " "ymm12, ymm5, ymm2"),
        Q!("    vpmulhw         " "ymm13, ymm6, ymm3"),
        Q!("    vpmulhw         " "ymm14, ymm4, ymm3"),
        Q!("    vpsubw          " "ymm7, ymm11, ymm7"),
        Q!("    vpsubw          " "ymm9, ymm12, ymm9"),
        Q!("    vpsubw          " "ymm8, ymm13, ymm8"),
        Q!("    vpsubw          " "ymm10, ymm14, ymm10"),
        Q!("    vpaddw          " "ymm7, ymm8, ymm7"),
        Q!("    vpaddw          " "ymm9, ymm10, ymm9"),
        Q!("    vmovdqa         " "ymm8, [rdi]"),
        Q!("    vmovdqa         " "ymm10, [32 + rdi]"),
        Q!("    vpaddw          " "ymm7, ymm8, ymm7"),
        Q!("    vpaddw          " "ymm9, ymm10, ymm9"),
        Q!("    vmovdqa         " "[rdi], ymm7"),
        Q!("    vmovdqa         " "[32 + rdi], ymm9"),
        Q!("    vmovdqa         " "ymm2, [1088 + rsi]"),
        Q!("    vmovdqa         " "ymm3, [1120 + rsi]"),
        Q!("    vmovdqa         " "ymm4, [1088 + rdx]"),
        Q!("    vmovdqa         " "ymm5, [1120 + rdx]"),
        Q!("    vmovdqa         " "ymm6, [544 + rcx]"),
        Q!("    vpmullw         " "ymm13, ymm1, ymm2"),
        Q!("    vpmullw         " "ymm14, ymm1, ymm3"),
        Q!("    vpmullw         " "ymm7, ymm4, ymm13"),
        Q!("    vpmullw         " "ymm9, ymm5, ymm13"),
        Q!("    vpmullw         " "ymm8, ymm6, ymm14"),
        Q!("    vpmullw         " "ymm10, ymm4, ymm14"),
        Q!("    vpmulhw         " "ymm7, ymm0, ymm7"),
        Q!("    vpmulhw         " "ymm9, ymm0, ymm9"),
        Q!("    vpmulhw         " "ymm8, ymm0, ymm8"),
        Q!("    vpmulhw         " "ymm10, ymm0, ymm10"),
        Q!("    vpmulhw         " "ymm11, ymm4, ymm2"),
        Q!("    vpmulhw         " "ymm12, ymm5, ymm2"),
        Q!("    vpmulhw         " "ymm13, ymm6, ymm3"),
        Q!("    vpmulhw         " "ymm14, ymm4, ymm3"),
        Q!("    vpsubw          " "ymm7, ymm11, ymm7"),
        Q!("    vpsubw          " "ymm9, ymm12, ymm9"),
        Q!("    vpsubw          " "ymm8, ymm8, ymm13"),
        Q!("    vpsubw          " "ymm10, ymm14, ymm10"),
        Q!("    vpaddw          " "ymm7, ymm8, ymm7"),
        Q!("    vpaddw          " "ymm9, ymm10, ymm9"),
        Q!("    vmovdqa         " "ymm8, [64 + rdi]"),
        Q!("    vmovdqa         " "ymm10, [96 + rdi]"),
        Q!("    vpaddw          " "ymm7, ymm8, ymm7"),
        Q!("    vpaddw          " "ymm9, ymm10, ymm9"),
        Q!("    vmovdqa         " "[64 + rdi], ymm7"),
        Q!("    vmovdqa         " "[96 + rdi], ymm9"),
        Q!("    vmovdqa         " "ymm2, [1152 + rsi]"),
        Q!("    vmovdqa         " "ymm3, [1184 + rsi]"),
        Q!("    vmovdqa         " "ymm4, [1152 + rdx]"),
        Q!("    vmovdqa         " "ymm5, [1184 + rdx]"),
        Q!("    vmovdqa         " "ymm6, [576 + rcx]"),
        Q!("    vpmullw         " "ymm13, ymm1, ymm2"),
        Q!("    vpmullw         " "ymm14, ymm1, ymm3"),
        Q!("    vpmullw         " "ymm7, ymm4, ymm13"),
        Q!("    vpmullw         " "ymm9, ymm5, ymm13"),
        Q!("    vpmullw         " "ymm8, ymm6, ymm14"),
        Q!("    vpmullw         " "ymm10, ymm4, ymm14"),
        Q!("    vpmulhw         " "ymm7, ymm0, ymm7"),
        Q!("    vpmulhw         " "ymm9, ymm0, ymm9"),
        Q!("    vpmulhw         " "ymm8, ymm0, ymm8"),
        Q!("    vpmulhw         " "ymm10, ymm0, ymm10"),
        Q!("    vpmulhw         " "ymm11, ymm4, ymm2"),
        Q!("    vpmulhw         " "ymm12, ymm5, ymm2"),
        Q!("    vpmulhw         " "ymm13, ymm6, ymm3"),
        Q!("    vpmulhw         " "ymm14, ymm4, ymm3"),
        Q!("    vpsubw          " "ymm7, ymm11, ymm7"),
        Q!("    vpsubw          " "ymm9, ymm12, ymm9"),
        Q!("    vpsubw          " "ymm8, ymm13, ymm8"),
        Q!("    vpsubw          " "ymm10, ymm14, ymm10"),
        Q!("    vpaddw          " "ymm7, ymm8, ymm7"),
        Q!("    vpaddw          " "ymm9, ymm10, ymm9"),
        Q!("    vmovdqa         " "ymm8, [128 + rdi]"),
        Q!("    vmovdqa         " "ymm10, [160 + rdi]"),
        Q!("    vpaddw          " "ymm7, ymm8, ymm7"),
        Q!("    vpaddw          " "ymm9, ymm10, ymm9"),
        Q!("    vmovdqa         " "[128 + rdi], ymm7"),
        Q!("    vmovdqa         " "[160 + rdi], ymm9"),
        Q!("    vmovdqa         " "ymm2, [1216 + rsi]"),
        Q!("    vmovdqa         " "ymm3, [1248 + rsi]"),
        Q!("    vmovdqa         " "ymm4, [1216 + rdx]"),
        Q!("    vmovdqa         " "ymm5, [1248 + rdx]"),
        Q!("    vmovdqa         " "ymm6, [608 + rcx]"),
        Q!("    vpmullw         " "ymm13, ymm1, ymm2"),
        Q!("    vpmullw         " "ymm14, ymm1, ymm3"),
        Q!("    vpmullw         " "ymm7, ymm4, ymm13"),
        Q!("    vpmullw         " "ymm9, ymm5, ymm13"),
        Q!("    vpmullw         " "ymm8, ymm6, ymm14"),
        Q!("    vpmullw         " "ymm10, ymm4, ymm14"),
        Q!("    vpmulhw         " "ymm7, ymm0, ymm7"),
        Q!("    vpmulhw         " "ymm9, ymm0, ymm9"),
        Q!("    vpmulhw         " "ymm8, ymm0, ymm8"),
        Q!("    vpmulhw         " "ymm10, ymm0, ymm10"),
        Q!("    vpmulhw         " "ymm11, ymm4, ymm2"),
        Q!("    vpmulhw         " "ymm12, ymm5, ymm2"),
        Q!("    vpmulhw         " "ymm13, ymm6, ymm3"),
        Q!("    vpmulhw         " "ymm14, ymm4, ymm3"),
        Q!("    vpsubw          " "ymm7, ymm11, ymm7"),
        Q!("    vpsubw          " "ymm9, ymm12, ymm9"),
        Q!("    vpsubw          " "ymm8, ymm8, ymm13"),
        Q!("    vpsubw          " "ymm10, ymm14, ymm10"),
        Q!("    vpaddw          " "ymm7, ymm8, ymm7"),
        Q!("    vpaddw          " "ymm9, ymm10, ymm9"),
        Q!("    vmovdqa         " "ymm8, [192 + rdi]"),
        Q!("    vmovdqa         " "ymm10, [224 + rdi]"),
        Q!("    vpaddw          " "ymm7, ymm8, ymm7"),
        Q!("    vpaddw          " "ymm9, ymm10, ymm9"),
        Q!("    vmovdqa         " "[192 + rdi], ymm7"),
        Q!("    vmovdqa         " "[224 + rdi], ymm9"),
        Q!("    vmovdqa         " "ymm2, [1280 + rsi]"),
        Q!("    vmovdqa         " "ymm3, [1312 + rsi]"),
        Q!("    vmovdqa         " "ymm4, [1280 + rdx]"),
        Q!("    vmovdqa         " "ymm5, [1312 + rdx]"),
        Q!("    vmovdqa         " "ymm6, [640 + rcx]"),
        Q!("    vpmullw         " "ymm13, ymm1, ymm2"),
        Q!("    vpmullw         " "ymm14, ymm1, ymm3"),
        Q!("    vpmullw         " "ymm7, ymm4, ymm13"),
        Q!("    vpmullw         " "ymm9, ymm5, ymm13"),
        Q!("    vpmullw         " "ymm8, ymm6, ymm14"),
        Q!("    vpmullw         " "ymm10, ymm4, ymm14"),
        Q!("    vpmulhw         " "ymm7, ymm0, ymm7"),
        Q!("    vpmulhw         " "ymm9, ymm0, ymm9"),
        Q!("    vpmulhw         " "ymm8, ymm0, ymm8"),
        Q!("    vpmulhw         " "ymm10, ymm0, ymm10"),
        Q!("    vpmulhw         " "ymm11, ymm4, ymm2"),
        Q!("    vpmulhw         " "ymm12, ymm5, ymm2"),
        Q!("    vpmulhw         " "ymm13, ymm6, ymm3"),
        Q!("    vpmulhw         " "ymm14, ymm4, ymm3"),
        Q!("    vpsubw          " "ymm7, ymm11, ymm7"),
        Q!("    vpsubw          " "ymm9, ymm12, ymm9"),
        Q!("    vpsubw          " "ymm8, ymm13, ymm8"),
        Q!("    vpsubw          " "ymm10, ymm14, ymm10"),
        Q!("    vpaddw          " "ymm7, ymm8, ymm7"),
        Q!("    vpaddw          " "ymm9, ymm10, ymm9"),
        Q!("    vmovdqa         " "ymm8, [256 + rdi]"),
        Q!("    vmovdqa         " "ymm10, [288 + rdi]"),
        Q!("    vpaddw          " "ymm7, ymm8, ymm7"),
        Q!("    vpaddw          " "ymm9, ymm10, ymm9"),
        Q!("    vmovdqa         " "[256 + rdi], ymm7"),
        Q!("    vmovdqa         " "[288 + rdi], ymm9"),
        Q!("    vmovdqa         " "ymm2, [1344 + rsi]"),
        Q!("    vmovdqa         " "ymm3, [1376 + rsi]"),
        Q!("    vmovdqa         " "ymm4, [1344 + rdx]"),
        Q!("    vmovdqa         " "ymm5, [1376 + rdx]"),
        Q!("    vmovdqa         " "ymm6, [672 + rcx]"),
        Q!("    vpmullw         " "ymm13, ymm1, ymm2"),
        Q!("    vpmullw         " "ymm14, ymm1, ymm3"),
        Q!("    vpmullw         " "ymm7, ymm4, ymm13"),
        Q!("    vpmullw         " "ymm9, ymm5, ymm13"),
        Q!("    vpmullw         " "ymm8, ymm6, ymm14"),
        Q!("    vpmullw         " "ymm10, ymm4, ymm14"),
        Q!("    vpmulhw         " "ymm7, ymm0, ymm7"),
        Q!("    vpmulhw         " "ymm9, ymm0, ymm9"),
        Q!("    vpmulhw         " "ymm8, ymm0, ymm8"),
        Q!("    vpmulhw         " "ymm10, ymm0, ymm10"),
        Q!("    vpmulhw         " "ymm11, ymm4, ymm2"),
        Q!("    vpmulhw         " "ymm12, ymm5, ymm2"),
        Q!("    vpmulhw         " "ymm13, ymm6, ymm3"),
        Q!("    vpmulhw         " "ymm14, ymm4, ymm3"),
        Q!("    vpsubw          " "ymm7, ymm11, ymm7"),
        Q!("    vpsubw          " "ymm9, ymm12, ymm9"),
        Q!("    vpsubw          " "ymm8, ymm8, ymm13"),
        Q!("    vpsubw          " "ymm10, ymm14, ymm10"),
        Q!("    vpaddw          " "ymm7, ymm8, ymm7"),
        Q!("    vpaddw          " "ymm9, ymm10, ymm9"),
        Q!("    vmovdqa         " "ymm8, [320 + rdi]"),
        Q!("    vmovdqa         " "ymm10, [352 + rdi]"),
        Q!("    vpaddw          " "ymm7, ymm8, ymm7"),
        Q!("    vpaddw          " "ymm9, ymm10, ymm9"),
        Q!("    vmovdqa         " "[320 + rdi], ymm7"),
        Q!("    vmovdqa         " "[352 + rdi], ymm9"),
        Q!("    vmovdqa         " "ymm2, [1408 + rsi]"),
        Q!("    vmovdqa         " "ymm3, [1440 + rsi]"),
        Q!("    vmovdqa         " "ymm4, [1408 + rdx]"),
        Q!("    vmovdqa         " "ymm5, [1440 + rdx]"),
        Q!("    vmovdqa         " "ymm6, [704 + rcx]"),
        Q!("    vpmullw         " "ymm13, ymm1, ymm2"),
        Q!("    vpmullw         " "ymm14, ymm1, ymm3"),
        Q!("    vpmullw         " "ymm7, ymm4, ymm13"),
        Q!("    vpmullw         " "ymm9, ymm5, ymm13"),
        Q!("    vpmullw         " "ymm8, ymm6, ymm14"),
        Q!("    vpmullw         " "ymm10, ymm4, ymm14"),
        Q!("    vpmulhw         " "ymm7, ymm0, ymm7"),
        Q!("    vpmulhw         " "ymm9, ymm0, ymm9"),
        Q!("    vpmulhw         " "ymm8, ymm0, ymm8"),
        Q!("    vpmulhw         " "ymm10, ymm0, ymm10"),
        Q!("    vpmulhw         " "ymm11, ymm4, ymm2"),
        Q!("    vpmulhw         " "ymm12, ymm5, ymm2"),
        Q!("    vpmulhw         " "ymm13, ymm6, ymm3"),
        Q!("    vpmulhw         " "ymm14, ymm4, ymm3"),
        Q!("    vpsubw          " "ymm7, ymm11, ymm7"),
        Q!("    vpsubw          " "ymm9, ymm12, ymm9"),
        Q!("    vpsubw          " "ymm8, ymm13, ymm8"),
        Q!("    vpsubw          " "ymm10, ymm14, ymm10"),
        Q!("    vpaddw          " "ymm7, ymm8, ymm7"),
        Q!("    vpaddw          " "ymm9, ymm10, ymm9"),
        Q!("    vmovdqa         " "ymm8, [384 + rdi]"),
        Q!("    vmovdqa         " "ymm10, [416 + rdi]"),
        Q!("    vpaddw          " "ymm7, ymm8, ymm7"),
        Q!("    vpaddw          " "ymm9, ymm10, ymm9"),
        Q!("    vmovdqa         " "[384 + rdi], ymm7"),
        Q!("    vmovdqa         " "[416 + rdi], ymm9"),
        Q!("    vmovdqa         " "ymm2, [1472 + rsi]"),
        Q!("    vmovdqa         " "ymm3, [1504 + rsi]"),
        Q!("    vmovdqa         " "ymm4, [1472 + rdx]"),
        Q!("    vmovdqa         " "ymm5, [1504 + rdx]"),
        Q!("    vmovdqa         " "ymm6, [736 + rcx]"),
        Q!("    vpmullw         " "ymm13, ymm1, ymm2"),
        Q!("    vpmullw         " "ymm14, ymm1, ymm3"),
        Q!("    vpmullw         " "ymm7, ymm4, ymm13"),
        Q!("    vpmullw         " "ymm9, ymm5, ymm13"),
        Q!("    vpmullw         " "ymm8, ymm6, ymm14"),
        Q!("    vpmullw         " "ymm10, ymm4, ymm14"),
        Q!("    vpmulhw         " "ymm7, ymm0, ymm7"),
        Q!("    vpmulhw         " "ymm9, ymm0, ymm9"),
        Q!("    vpmulhw         " "ymm8, ymm0, ymm8"),
        Q!("    vpmulhw         " "ymm10, ymm0, ymm10"),
        Q!("    vpmulhw         " "ymm11, ymm4, ymm2"),
        Q!("    vpmulhw         " "ymm12, ymm5, ymm2"),
        Q!("    vpmulhw         " "ymm13, ymm6, ymm3"),
        Q!("    vpmulhw         " "ymm14, ymm4, ymm3"),
        Q!("    vpsubw          " "ymm7, ymm11, ymm7"),
        Q!("    vpsubw          " "ymm9, ymm12, ymm9"),
        Q!("    vpsubw          " "ymm8, ymm8, ymm13"),
        Q!("    vpsubw          " "ymm10, ymm14, ymm10"),
        Q!("    vpaddw          " "ymm7, ymm8, ymm7"),
        Q!("    vpaddw          " "ymm9, ymm10, ymm9"),
        Q!("    vmovdqa         " "ymm8, [448 + rdi]"),
        Q!("    vmovdqa         " "ymm10, [480 + rdi]"),
        Q!("    vpaddw          " "ymm7, ymm8, ymm7"),
        Q!("    vpaddw          " "ymm9, ymm10, ymm9"),
        Q!("    vmovdqa         " "[448 + rdi], ymm7"),
        Q!("    vmovdqa         " "[480 + rdi], ymm9"),

        inout("rdi") r.as_mut_ptr() => _,
        inout("rsi") a.as_ptr() => _,
        inout("rdx") b.as_ptr() => _,
        inout("rcx") bt.as_ptr() => _,
        // clobbers
        out("rax") _,
        out("zmm0") _,
        out("zmm1") _,
        out("zmm10") _,
        out("zmm11") _,
        out("zmm12") _,
        out("zmm13") _,
        out("zmm14") _,
        out("zmm2") _,
        out("zmm3") _,
        out("zmm4") _,
        out("zmm5") _,
        out("zmm6") _,
        out("zmm7") _,
        out("zmm8") _,
        out("zmm9") _,
            )
    };
}