1 // Copyright 2026 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
4
5 //go:build !purego
6
7 // AES-GCM for loong64.
8 //
9 // LoongArch (LA464/LA664) has no carry-less multiply instruction, so
10 // GHASH is a scalar port of ghash.go: nine 32x32->64 carry-less
11 // multiplications using the BearSSL "bit hole" technique (MULV with
12 // masked inputs), Karatsuba recombination, and polynomial reduction.
13 // AES block encryption uses the same VSHUFB-based S-box as
14 // aes_loong64.s (SUBBYTES/MIXCOLUMNS macros are algorithm-identical;
15 // sbox data is duplicated locally because this is a separate package).
16 //
17 // productTable layout (loong64-specific, 84 bytes used of 256):
18 // +0 h0, h1, h2, h3 (uint32, host order, as in ghash.go)
19 // +16 h0^h1, h2^h3, h0^h2, h1^h3, (h0^h2)^(h1^h3)
20 // +36 MULT128 scratch: six 64-bit intermediate products
21 // (r00, r01 at +36/+44; r10, r11 at +52/+60; r20, r21 at +68/+76)
22 //
23 // enc[] round keys are stored in word order (32-bit BE words), so
24 // aesEnc applies a byteSwap32 VSHUFB to each round key before XOR.
25
26 #include "textflag.h"
27
28 // -----------------------------------------------------------------------
29 // Tables (local to this file)
30 // -----------------------------------------------------------------------
31
32 // sboxG: AES encryption S-box (identical contents to sbox0_0 in
33 // aes_loong64.s).
34 DATA sboxG<>+0x00(SB)/8, $0xc56f6bf27b777c63
35 DATA sboxG<>+0x08(SB)/8, $0x76abd7fe2b670130
36 DATA sboxG<>+0x10(SB)/8, $0xf04759fa7dc982ca
37 DATA sboxG<>+0x18(SB)/8, $0xc072a49cafa2d4ad
38 DATA sboxG<>+0x20(SB)/8, $0xccf73f362693fdb7
39 DATA sboxG<>+0x28(SB)/8, $0x1531d871f1e5a534
40 DATA sboxG<>+0x30(SB)/8, $0x9a059618c323c704
41 DATA sboxG<>+0x38(SB)/8, $0x75b227ebe2801207
42 DATA sboxG<>+0x40(SB)/8, $0xa05a6e1b1a2c8309
43 DATA sboxG<>+0x48(SB)/8, $0x842fe329b3d63b52
44 DATA sboxG<>+0x50(SB)/8, $0x5bb1fc20ed00d153
45 DATA sboxG<>+0x58(SB)/8, $0xcf584c4a39becb6a
46 DATA sboxG<>+0x60(SB)/8, $0x85334d43fbaaefd0
47 DATA sboxG<>+0x68(SB)/8, $0xa89f3c507f02f945
48 DATA sboxG<>+0x70(SB)/8, $0xf5389d928f40a351
49 DATA sboxG<>+0x78(SB)/8, $0xd2f3ff1021dab6bc
50 DATA sboxG<>+0x80(SB)/8, $0x1744975fec130ccd
51 DATA sboxG<>+0x88(SB)/8, $0x73195d643d7ea7c4
52 DATA sboxG<>+0x90(SB)/8, $0x88902a22dc4f8160
53 DATA sboxG<>+0x98(SB)/8, $0xdb0b5ede14b8ee46
54 DATA sboxG<>+0xa0(SB)/8, $0x5c2406490a3a32e0
55 DATA sboxG<>+0xa8(SB)/8, $0x79e4959162acd3c2
56 DATA sboxG<>+0xb0(SB)/8, $0xa94ed58d6d37c8e7
57 DATA sboxG<>+0xb8(SB)/8, $0x08ae7a65eaf4566c
58 DATA sboxG<>+0xc0(SB)/8, $0xc6b4a61c2e2578ba
59 DATA sboxG<>+0xc8(SB)/8, $0x8a8bbd4b1f74dde8
60 DATA sboxG<>+0xd0(SB)/8, $0x0ef6034866b53e70
61 DATA sboxG<>+0xd8(SB)/8, $0x9e1dc186b9573561
62 DATA sboxG<>+0xe0(SB)/8, $0x948ed9691198f8e1
63 DATA sboxG<>+0xe8(SB)/8, $0xdf2855cee9871e9b
64 DATA sboxG<>+0xf0(SB)/8, $0x6842e6bf0d89a18c
65 DATA sboxG<>+0xf8(SB)/8, $0x16bb54b00f2d9941
66 GLOBL sboxG<>(SB), (NOPTR+RODATA), $256
67
68 DATA shiftRowsG<>+0x00(SB)/8, $0x030e09040f0a0500
69 DATA shiftRowsG<>+0x08(SB)/8, $0x0b06010c07020d08
70 GLOBL shiftRowsG<>(SB), (NOPTR+RODATA), $16
71
72 DATA rot1G<>+0x00(SB)/8, $0x0407060500030201
73 DATA rot1G<>+0x08(SB)/8, $0x0c0f0e0d080b0a09
74 GLOBL rot1G<>(SB), (NOPTR+RODATA), $16
75
76 DATA rot2G<>+0x00(SB)/8, $0x0504070601000302
77 DATA rot2G<>+0x08(SB)/8, $0x0d0c0f0e09080b0a
78 GLOBL rot2G<>(SB), (NOPTR+RODATA), $16
79
80 DATA rot3G<>+0x00(SB)/8, $0x0605040702010003
81 DATA rot3G<>+0x08(SB)/8, $0x0e0d0c0f0a09080b
82 GLOBL rot3G<>(SB), (NOPTR+RODATA), $16
83
84 // bit-hole masks for the carry-less multiply (see ghash.go ghashMul).
85 DATA mask1111<>+0(SB)/8, $0x1111111111111111
86 GLOBL mask1111<>(SB), (NOPTR+RODATA), $8
87
88 DATA byteSwap32G<>+0x00(SB)/8, $0x0405060700010203
89 DATA byteSwap32G<>+0x08(SB)/8, $0x0c0d0e0f08090a0b
90 GLOBL byteSwap32G<>(SB), (NOPTR+RODATA), $16
91
92 // -----------------------------------------------------------------------
93 // AES macros (algorithm-identical to aes_loong64.s)
94 // -----------------------------------------------------------------------
95
96 #define XTIME(src, dst, tmp) \
97 VSLLB $1, src, dst; \
98 VSRAB $7, src, tmp; \
99 VANDB $0x1b, tmp, tmp; \
100 VXORV tmp, dst, dst
101
102 #define MIXCOLUMNS(src, out, r1, r2, r3, t1, t2, t3) \
103 VSHUFB r1, src, src, t1; \
104 VXORV src, t1, t2; \
105 VSHUFB r2, src, src, t3; \
106 VXORV t2, t3, out; \
107 VSHUFB r3, src, src, t3; \
108 VXORV out, t3, out; \
109 XTIME(t2, t1, t3); \
110 VXORV src, out, out; \
111 VXORV out, t1, out
112
113 // SUBBYTES implements the AES S-box via 4 table-lookup pairs.
114 // Masks: V31=bit7, V30=bit6, V29=bit5; V26=low5 index.
115 // Each chunk pair produces a result via VSHUFB+VBITSELV:
116 // r01 = chunk0/1, r23 = chunk2/3, r45 = chunk4/5, r67 = chunk6/7
117 // r0123 = M6 ? r23 : r01; r4567 = M6 ? r67 : r45
118 // out = M7 ? r4567 : r0123
119 #define SUBBYTES(src, sbox_ptr, out) \
120 VANDB $0xe0, src, V25; \
121 VSRAB $7, V25, V31; \
122 VSLLB $1, V25, V24; \
123 VSRAB $7, V24, V30; \
124 VSLLB $2, V25, V24; \
125 VSRAB $7, V24, V29; \
126 VANDB $0x1f, src, V26; \
127 VMOVQ 0(sbox_ptr), V16; \
128 VMOVQ 16(sbox_ptr), V17; \
129 VMOVQ 32(sbox_ptr), V18; \
130 VMOVQ 48(sbox_ptr), V19; \
131 VSHUFB V26, V16, V17, V28; \
132 VSHUFB V26, V18, V19, V27; \
133 VBITSELV V29, V27, V28, V20; \
134 VMOVQ 64(sbox_ptr), V16; \
135 VMOVQ 80(sbox_ptr), V17; \
136 VMOVQ 96(sbox_ptr), V18; \
137 VMOVQ 112(sbox_ptr), V19; \
138 VSHUFB V26, V16, V17, V28; \
139 VSHUFB V26, V18, V19, V27; \
140 VBITSELV V29, V27, V28, V21; \
141 VBITSELV V30, V21, V20, V22; \
142 VMOVQ 128(sbox_ptr), V16; \
143 VMOVQ 144(sbox_ptr), V17; \
144 VMOVQ 160(sbox_ptr), V18; \
145 VMOVQ 176(sbox_ptr), V19; \
146 VSHUFB V26, V16, V17, V28; \
147 VSHUFB V26, V18, V19, V27; \
148 VBITSELV V29, V27, V28, V20; \
149 VMOVQ 192(sbox_ptr), V16; \
150 VMOVQ 208(sbox_ptr), V17; \
151 VMOVQ 224(sbox_ptr), V18; \
152 VMOVQ 240(sbox_ptr), V19; \
153 VSHUFB V26, V16, V17, V28; \
154 VSHUFB V26, V18, V19, V27; \
155 VBITSELV V29, V27, V28, V21; \
156 VBITSELV V30, V21, V20, V23; \
157 VBITSELV V31, V23, V22, out
158
159 // -----------------------------------------------------------------------
160 // GHASH scalar macros — direct port of ghash.go.
161 //
162 // GHASHMUL(x, y, out): out = carryless(x,y), x/y are 32-bit values in
163 // scalar regs. Clobbers R13-R21, R24-R27. Port of ghashMul().
164 //
165 // xm[i] = x & (0x1111111111111111 << i) -> R24..R27; ym[i] -> R15..R18.
166 // z[i] = (xm0*ym[i] ^ xm1*ym[i+3] ^ xm2*ym[i+2] ^ xm3*ym[i+1])
167 // & (0x1111.. << i), accumulated into out.
168 // -----------------------------------------------------------------------
169 #define GHASHMUL(xr, yr, out) \
170 MOVV mask1111<>+0(SB), R20; \
171 AND R20, xr, R24; \
172 SLLV $1, R20, R21; AND R21, xr, R25; \
173 SLLV $2, R20, R21; AND R21, xr, R26; \
174 SLLV $3, R20, R21; AND R21, xr, R27; \
175 AND R20, yr, R15; \
176 AND R21, yr, R18; \
177 SLLV $1, R20, R21; AND R21, yr, R16; \
178 SLLV $2, R20, R21; AND R21, yr, R17; \
179 MOVV R20, R19; \
180 MULV R24, R15, out; \
181 MULV R25, R18, R14; XOR R14, out; \
182 MULV R26, R17, R14; XOR R14, out; \
183 MULV R27, R16, R14; XOR R14, out; \
184 AND R19, out, out; \
185 MULV R24, R16, R14; \
186 MULV R25, R15, R13; XOR R13, R14; \
187 MULV R26, R18, R13; XOR R13, R14; \
188 MULV R27, R17, R13; XOR R13, R14; \
189 SLLV $1, R19, R13; AND R13, R14, R14; OR R14, out; \
190 MULV R24, R17, R14; \
191 MULV R25, R16, R13; XOR R13, R14; \
192 MULV R26, R15, R13; XOR R13, R14; \
193 MULV R27, R18, R13; XOR R13, R14; \
194 SLLV $2, R19, R13; AND R13, R14, R14; OR R14, out; \
195 MULV R24, R18, R14; \
196 MULV R25, R17, R13; XOR R13, R14; \
197 MULV R26, R16, R13; XOR R13, R14; \
198 MULV R27, R15, R13; XOR R13, R14; \
199 SLLV $3, R19, R13; AND R13, R14, R14; OR R14, out
200
201 // MULT128: y = y (*) H in GF(2^128). y[0..3] in R8-R11, R7 = productTable.
202 // Clobbers R12-R21, R23-R28, R31. Writes intermediate products to
203 // productTable+36..76. Port of the ghash() block body.
204 //
205 // Three GHASHMUL groups (zLo/zHi/zSum) are spilled to productTable:
206 // group 0 (y0,y1 x h0,h1) -> r00@36, r01@44
207 // group 1 (y2,y3 x h2,h3) -> r10@52, r11@60
208 // group 2 (y0^y2, y1^y3) -> r20@68, r21@76
209 // Then r20 ^= r00^r10, r21 ^= r01^r11, and the halves are folded:
210 // r01 ^= r20 (R13), r10 ^= r21 (R14).
211 // z = r << 1 into R18,R19,R21,R23, followed by the GF(2^128) reduction
212 // (lw ^ lw>>1 ^ lw>>2 ^ lw>>7 and carries <<63|<<62|<<57).
213 // Result: R8,R9 = z2 lo/hi, R10,R11 = z3 lo/hi.
214 #define MULT128() \
215 MOVWU 0(R7), R13; GHASHMUL(R8, R13, R23); \
216 MOVWU 4(R7), R13; GHASHMUL(R9, R13, R28); \
217 MOVWU 16(R7), R14; XOR R9, R8, R13; GHASHMUL(R13, R14, R31); \
218 XOR R23, R31; XOR R28, R31; \
219 SLLV $32, R31, R13; XOR R13, R23; MOVV R23, 36(R7); \
220 SRLV $32, R31, R13; XOR R13, R28; MOVV R28, 44(R7); \
221 MOVWU 8(R7), R13; GHASHMUL(R10, R13, R23); \
222 MOVWU 12(R7), R13; GHASHMUL(R11, R13, R28); \
223 MOVWU 20(R7), R14; XOR R11, R10, R13; GHASHMUL(R13, R14, R31); \
224 XOR R23, R31; XOR R28, R31; \
225 SLLV $32, R31, R13; XOR R13, R23; MOVV R23, 52(R7); \
226 SRLV $32, R31, R13; XOR R13, R28; MOVV R28, 60(R7); \
227 MOVWU 24(R7), R14; XOR R10, R8, R13; GHASHMUL(R13, R14, R23); \
228 MOVWU 28(R7), R14; XOR R11, R9, R13; GHASHMUL(R13, R14, R28); \
229 MOVWU 32(R7), R14; \
230 XOR R10, R8, R13; \
231 XOR R11, R9, R12; \
232 XOR R12, R13; \
233 GHASHMUL(R13, R14, R31); \
234 XOR R23, R31; XOR R28, R31; \
235 SLLV $32, R31, R13; XOR R13, R23; MOVV R23, 68(R7); \
236 SRLV $32, R31, R13; XOR R13, R28; MOVV R28, 76(R7); \
237 MOVV 36(R7), R12; \
238 MOVV 44(R7), R13; \
239 MOVV 52(R7), R14; \
240 MOVV 60(R7), R15; \
241 MOVV 68(R7), R16; \
242 MOVV 76(R7), R17; \
243 XOR R12, R16; XOR R14, R16; \
244 XOR R13, R17; XOR R15, R17; \
245 XOR R16, R13; \
246 XOR R17, R14; \
247 SLLV $1, R12, R18; \
248 SLLV $1, R13, R19; SRLV $63, R12, R20; OR R20, R19; \
249 SLLV $1, R15, R23; SRLV $63, R14, R20; OR R20, R23; \
250 SLLV $1, R14, R21; SRLV $63, R13, R20; OR R20, R21; \
251 SRLV $1, R18, R20; XOR R20, R18, R20; SRLV $2, R18, R24; XOR R24, R20; \
252 SRLV $7, R18, R24; XOR R24, R20; XOR R20, R21; \
253 SLLV $63, R18, R20; SLLV $62, R18, R24; XOR R24, R20; \
254 SLLV $57, R18, R24; XOR R24, R20; XOR R20, R19; \
255 SRLV $1, R19, R20; XOR R20, R19, R20; SRLV $2, R19, R24; XOR R24, R20; \
256 SRLV $7, R19, R24; XOR R24, R20; XOR R20, R23; \
257 SLLV $63, R19, R20; SLLV $62, R19, R24; XOR R24, R20; \
258 SLLV $57, R19, R24; XOR R24, R20; XOR R20, R21; \
259 MOVWU R21, R8; SRLV $32, R21, R9; \
260 MOVWU R23, R10; SRLV $32, R23, R11
261
262 // GHASHSTEP: XOR the 16-byte block at R4 into y and multiply by H.
263 #define GHASHSTEP() \
264 MOVWU 0(R4), R12; REVB2W R12, R12; XOR R12, R11; \
265 MOVWU 4(R4), R12; REVB2W R12, R12; XOR R12, R10; \
266 MOVWU 8(R4), R12; REVB2W R12, R12; XOR R12, R9; \
267 MOVWU 12(R4), R12; REVB2W R12, R12; XOR R12, R8; \
268 MULT128()
269
270 // -----------------------------------------------------------------------
271 // aesEnc: encrypt one block. R4=nr, R5=xk (word order; keys byte-swapped
272 // per round via V9), V0 in/out.
273 // Leaf helper; clobbers R24-R26, V4-V9, V15-V31.
274 // -----------------------------------------------------------------------
275 TEXT aesEnc<>(SB), NOSPLIT, $0-0
276 MOVV $sboxG<>(SB), R24
277 MOVV $shiftRowsG<>(SB), R25
278 VMOVQ (R25), V7
279 MOVV $byteSwap32G<>(SB), R25
280 VMOVQ (R25), V9
281 MOVV $rot1G<>(SB), R25
282 VMOVQ (R25), V4
283 MOVV $rot2G<>(SB), R25
284 VMOVQ (R25), V5
285 MOVV $rot3G<>(SB), R25
286 VMOVQ (R25), V6
287
288 VMOVQ (R5), V8
289 VXORV V8, V0, V0
290 ADDV $16, R5
291
292 MOVV R4, R26
293 ADDV $-1, R26
294 Lgcm_aes_loop:
295 SUBBYTES(V0, R24, V0)
296 VSHUFB V7, V0, V0, V0
297 MIXCOLUMNS(V0, V15, V4, V5, V6, V10, V11, V12)
298 VMOVQ (R5), V8
299 ADDV $16, R5
300 VXORV V8, V15, V0
301 ADDV $-1, R26
302 BNE R26, R0, Lgcm_aes_loop
303
304 SUBBYTES(V0, R24, V0)
305 VSHUFB V7, V0, V0, V0
306 VMOVQ (R5), V8
307 VXORV V8, V0, V0
308 RET
309
310 // -----------------------------------------------------------------------
311 // func gcmAesInit(productTable *[256]byte, ks []uint32)
312 // -----------------------------------------------------------------------
313 TEXT ·gcmAesInit(SB), NOSPLIT, $32-32
314 MOVV R1, 24(R3) // save return address
315 MOVV productTable+0(FP), R7
316 MOVV ks_base+8(FP), R5
317 MOVV ks_len+16(FP), R4
318 SRLV $2, R4, R4
319 ADDV $-1, R4 // nr = ks_len/4 - 1
320
321 // H = AES(key, 0^128)
322 VXORV V0, V0, V0
323 CALL aesEnc<>(SB)
324 VMOVQ V0, 8(R3) // spill H to our stack frame
325
326 // h[3-i] = BEU32(H[i*4:]) -> h0=BEU32(H[12:16]) ... h3=BEU32(H[0:4])
327 MOVWU (8+12)(R3), R8; REVB2W R8, R8 // h0
328 MOVWU (8+8)(R3), R9; REVB2W R9, R9 // h1
329 MOVWU (8+4)(R3), R10; REVB2W R10, R10 // h2
330 MOVWU (8+0)(R3), R11; REVB2W R11, R11 // h3
331 MOVW R8, 0(R7)
332 MOVW R9, 4(R7)
333 MOVW R10, 8(R7)
334 MOVW R11, 12(R7)
335 XOR R9, R8, R12; MOVW R12, 16(R7) // h0^h1
336 XOR R11, R10, R12; MOVW R12, 20(R7) // h2^h3
337 XOR R10, R8, R12; MOVW R12, 24(R7) // h0^h2
338 XOR R11, R9, R12; MOVW R12, 28(R7) // h1^h3
339 MOVW 24(R7), R12
340 MOVW 28(R7), R13
341 XOR R13, R12; MOVW R12, 32(R7) // (h0^h2)^(h1^h3)
342 MOVV 24(R3), R1 // restore return address
343 RET
344
345 // -----------------------------------------------------------------------
346 // func gcmAesData(productTable *[256]byte, data []byte, T *[16]byte)
347 // -----------------------------------------------------------------------
348 TEXT ·gcmAesData(SB), NOSPLIT, $32-40
349 MOVV productTable+0(FP), R7
350 MOVV data_base+8(FP), R4
351 MOVV data_len+16(FP), R6
352 MOVV T+32(FP), R5
353
354 // y[3-i] = BEU32(T[i*4:])
355 MOVWU 12(R5), R8; REVB2W R8, R8
356 MOVWU 8(R5), R9; REVB2W R9, R9
357 MOVWU 4(R5), R10; REVB2W R10, R10
358 MOVWU 0(R5), R11; REVB2W R11, R11
359
360 data_blocks:
361 SGTU $16, R6, R13
362 BNE R13, data_tail
363 SUBV $16, R6
364 GHASHSTEP()
365 ADDV $16, R4
366 JMP data_blocks
367
368 data_tail:
369 BEQ R6, data_done
370 // zero-pad the partial block in our stack frame
371 MOVV R0, 8(R3)
372 MOVV R0, 16(R3)
373 MOVV R3, R14
374 ADDV $8, R14
375 MOVV R4, R15
376 data_tail_copy:
377 MOVB (R15), R13
378 MOVB R13, (R14)
379 ADDV $1, R15
380 ADDV $1, R14
381 ADDV $-1, R6
382 BNE R6, R0, data_tail_copy
383 MOVV R3, R4
384 ADDV $8, R4 // R4 = &partialBlock
385 GHASHSTEP()
386
387 data_done:
388 // T[i*4:] = BEU32(y[3-i])
389 REVB2W R11, R12; MOVW R12, 0(R5)
390 REVB2W R10, R12; MOVW R12, 4(R5)
391 REVB2W R9, R12; MOVW R12, 8(R5)
392 REVB2W R8, R12; MOVW R12, 12(R5)
393 RET
394
395 // -----------------------------------------------------------------------
396 // func gcmAesFinish(productTable *[256]byte, tagMask, T *[16]byte,
397 // pLen, dLen uint64)
398 // -----------------------------------------------------------------------
399 TEXT ·gcmAesFinish(SB), NOSPLIT, $0-40
400 MOVV productTable+0(FP), R7
401 MOVV tagMask+8(FP), R6
402 MOVV T+16(FP), R5
403 MOVV pLen+24(FP), R14
404 MOVV dLen+32(FP), R15
405
406 // lenBlock = BE64(dLen*8) || BE64(pLen*8), matching the amd64
407 // gcmAesFinish convention (dLen = AAD len first; amd64 stores the
408 // same block in its byte-reversed internal representation).
409 // As host u32s: BEU32(blk[0:4]) = dLen*8 >> 32, etc.
410 SLLV $3, R14, R14
411 SLLV $3, R15, R15
412
413 // load y from T (BE word order as in gcmAesData)
414 MOVWU 12(R5), R8; REVB2W R8, R8
415 MOVWU 8(R5), R9; REVB2W R9, R9
416 MOVWU 4(R5), R10; REVB2W R10, R10
417 MOVWU 0(R5), R11; REVB2W R11, R11
418
419 // y ^= lenBlock
420 SRLV $32, R15, R12; XOR R12, R11 // y[3] ^= hi(dLen*8) block[0:4]
421 MOVWU R15, R12; XOR R12, R10 // y[2] ^= lo(dLen*8) block[4:8]
422 SRLV $32, R14, R12; XOR R12, R9 // y[1] ^= hi(pLen*8) block[8:12]
423 MOVWU R14, R12; XOR R12, R8 // y[0] ^= lo(pLen*8) block[12:16]
424
425 MULT128()
426
427 // tag = y ^ tagMask
428 MOVWU 0(R6), R12; REVB2W R12, R12; XOR R12, R11
429 MOVWU 4(R6), R12; REVB2W R12, R12; XOR R12, R10
430 MOVWU 8(R6), R12; REVB2W R12, R12; XOR R12, R9
431 MOVWU 12(R6), R12; REVB2W R12, R12; XOR R12, R8
432
433 REVB2W R11, R12; MOVW R12, 0(R5)
434 REVB2W R10, R12; MOVW R12, 4(R5)
435 REVB2W R9, R12; MOVW R12, 8(R5)
436 REVB2W R8, R12; MOVW R12, 12(R5)
437 RET
438
439 // helper: encrypt blocks, GHASH ciphertext, ctr++
440 // Frame layout (offsets from R3):
441 // 8(R3)=src ptr 16(R3)=dst ptr 24(R3)=remaining len
442 // 32(R3)=ctr ptr 40(R3)=T ptr 48(R3)=ks base
443 // 56(R3)=nr 64(R3)=productTable
444 // 72(R3)-87(R3): 16-byte keystream scratch buffer (tail path)
445 // 88(R3)-103(R3): 16-byte zero-padded GHASH input buffer (tail path)
446 // 120(R3): saved R1 (return address)
447 // y[0..3] stay in R8-R11 across CALL aesEnc<> (aesEnc uses only
448 // R4, R5, R24-R26 and V registers).
449 TEXT gcmCryptEnc<>(SB), NOSPLIT, $128-0
450 MOVV R1, 120(R3)
451 // ---- spill incoming state to frame ----
452 MOVV R4, 8(R3) // src
453 MOVV R5, 16(R3) // dst
454 MOVV R6, 24(R3) // len
455 MOVV R13, 32(R3) // ctr
456 MOVV R14, 40(R3) // T
457 MOVV R16, 48(R3) // ks base
458 MOVV R15, 56(R3) // nr
459 MOVV R7, 64(R3) // productTable
460
461 // y from T (BE word order)
462 MOVWU 12(R14), R8; REVB2W R8, R8
463 MOVWU 8(R14), R9; REVB2W R9, R9
464 MOVWU 4(R14), R10; REVB2W R10, R10
465 MOVWU 0(R14), R11; REVB2W R11, R11
466
467 Lenc_block_loop:
468 MOVV 24(R3), R6
469 SGTU $16, R6, R12 // R12 = (R6 < 16)
470 BEQ R12, R0, Lenc_have_blocks // R6 >= 16, jmp
471 BNE R6, R0, Lenc_tail
472 JMP Lenc_done
473
474 Lenc_have_blocks:
475 // ---- ctr++ FIRST (BE low 32 bits): first block uses inc32(J0) ----
476 MOVV 32(R3), R4
477 MOVWU 12(R4), R12
478 REVB2W R12, R12
479 ADDV $1, R12
480 REVB2W R12, R12
481 MOVW R12, 12(R4)
482
483 // ---- load ctr into V0 (counter block, BE) ----
484 MOVV 32(R3), R4
485 VMOVQ (R4), V0
486
487 // ---- encrypt counter: aesEnc(nr, ks, V0) ----
488 MOVV 56(R3), R4 // nr
489 MOVV 48(R3), R5 // xk
490 CALL aesEnc<>(SB) // V0 = AES(ctr)
491
492 // ---- load plaintext block, XOR with AES(ctr), store ----
493 MOVV 8(R3), R4
494 MOVV 16(R3), R5
495 VMOVQ (R4), V1 // plaintext
496 VXORV V0, V1, V1 // ciphertext
497 VMOVQ V1, (R5) // store to dst
498
499 // ---- GHASH ciphertext block ----
500 MOVV 16(R3), R4 // dst = ciphertext just written
501 MOVV 64(R3), R7 // productTable
502 GHASHSTEP()
503
504 // ---- advance pointers, decrement len ----
505 MOVV 8(R3), R4; ADDV $16, R4; MOVV R4, 8(R3)
506 MOVV 16(R3), R5; ADDV $16, R5; MOVV R5, 16(R3)
507 MOVV 24(R3), R6; ADDV $-16, R6; MOVV R6, 24(R3)
508
509 JMP Lenc_block_loop
510
511 Lenc_tail:
512 // ---- ctr++ first, then encrypt ----
513 MOVV 32(R3), R4
514 MOVWU 12(R4), R12
515 REVB2W R12, R12
516 ADDV $1, R12
517 REVB2W R12, R12
518 MOVW R12, 12(R4)
519
520 // Remaining len is 1..15 (R6 = 24(R3) != 0 and < 16).
521 //
522 // Semantics (must match amd64 encBlocks tail + generic):
523 // 1. ctr++ -- first block uses inc32(J0)
524 // 2. keystream = AES(ctr)
525 // 3. dst[i] = src[i] ^ keystream[i] (i < len, byte-wise)
526 // 4. GHASH(zero-padded ciphertext tail)
527
528 // ---- encrypt counter ----
529 MOVV 32(R3), R4
530 VMOVQ (R4), V0 // V0 = counter block
531 MOVV 56(R3), R4 // nr
532 MOVV 48(R3), R5 // xk
533 CALL aesEnc<>(SB) // V0 = AES(ctr) keystream
534 VMOVQ V0, 72(R3) // stash keystream on stack
535
536 // ---- zero the 16-byte GHASH input buffer at 88(R3) ----
537 MOVV R0, 88(R3)
538 MOVV R0, 96(R3)
539
540 // ---- byte loop: dst[i] = src[i] ^ ks[i]; ghashbuf[i] = dst[i] ----
541 MOVV 8(R3), R4 // src
542 MOVV 16(R3), R5 // dst
543 MOVV 24(R3), R6 // len (1..15)
544 MOVV R3, R16
545 ADDV $72, R16 // keystream buffer
546 MOVV R3, R17
547 ADDV $88, R17 // ghash buffer
548
549 Lenc_tail_xor:
550 MOVB (R4), R12
551 MOVB (R16), R13
552 XOR R13, R12, R12 // ct byte = pt ^ keystream
553 MOVB R12, (R5) // store to dst
554 MOVB R12, (R17) // copy ct byte into ghash buffer
555 ADDV $1, R4
556 ADDV $1, R5
557 ADDV $1, R16
558 ADDV $1, R17
559 ADDV $-1, R6
560 BNE R6, R0, Lenc_tail_xor
561
562 // ---- GHASH the zero-padded ciphertext block ----
563 MOVV R3, R4
564 ADDV $88, R4 // R4 = &ghash buffer
565 MOVV 64(R3), R7 // productTable
566 GHASHSTEP()
567
568 Lenc_done:
569 // write y back to T
570 MOVV 40(R3), R14
571 REVB2W R11, R12; MOVW R12, 0(R14)
572 REVB2W R10, R12; MOVW R12, 4(R14)
573 REVB2W R9, R12; MOVW R12, 8(R14)
574 REVB2W R8, R12; MOVW R12, 12(R14)
575
576 MOVV 120(R3), R1 // restore return address
577 RET
578
579 // helper: decrypt blocks, GHASH ciphertext (src), ctr++
580 // All loop state lives on our stack frame because CALL aesEnc<>
581 // clobbers R4, R5, R24-R26 and vector regs.
582 // Frame layout (offsets from R3):
583 // 8(R3)=src ptr 16(R3)=dst ptr 24(R3)=remaining len
584 // 32(R3)=ctr ptr 40(R3)=T ptr 48(R3)=ks base
585 // 56(R3)=nr 64(R3)=productTable
586 // 72(R3)-87(R3): keystream scratch 88(R3)-103(R3): ghash pad buffer
587 // 120(R3): saved R1 (return address)
588 // y[0..3] stay in R8-R11 across CALL aesEnc<> and are written back to
589 // T before returning (same convention as gcmCryptEnc).
590 TEXT gcmCryptDec<>(SB), NOSPLIT, $128-0
591 MOVV R1, 120(R3) // save return address
592
593 // ---- spill incoming state to frame ----
594 MOVV R4, 8(R3) // src
595 MOVV R5, 16(R3) // dst
596 MOVV R6, 24(R3) // len
597 MOVV R13, 32(R3) // ctr
598 MOVV R14, 40(R3) // T
599 MOVV R16, 48(R3) // ks base
600 MOVV R15, 56(R3) // nr
601 MOVV R7, 64(R3) // productTable
602
603 // y[3-i] = BEU32(T[i*4:])
604 MOVWU 12(R14), R8; REVB2W R8, R8
605 MOVWU 8(R14), R9; REVB2W R9, R9
606 MOVWU 4(R14), R10; REVB2W R10, R10
607 MOVWU 0(R14), R11; REVB2W R11, R11
608
609 Ldec_block_loop:
610 MOVV 24(R3), R6
611 BEQ R6, R0, Ldec_done
612 SGTU $16, R6, R12 // R12 = (R6 < 16)
613 BNE R12, R0, Ldec_tail
614
615 // ---- reload state ----
616 MOVV 8(R3), R4 // src ptr
617 MOVV 64(R3), R7 // productTable
618
619 // ---- GHASH ciphertext (src) BEFORE decrypting ----
620 GHASHSTEP() // reads (R4), y in R8-R11
621
622 // ---- ctr++ (BE low 32 bits): first block uses inc32(J0) ----
623 MOVV 32(R3), R13
624 MOVWU 12(R13), R12
625 REVB2W R12, R12
626 ADDV $1, R12
627 REVB2W R12, R12
628 MOVW R12, 12(R13)
629
630 // ---- encrypt counter -> V0 keystream ----
631 MOVV 56(R3), R4 // nr
632 MOVV 48(R3), R5 // xk
633 VMOVQ (R13), V0 // counter block
634 CALL aesEnc<>(SB) // clobbers R4, R5, R24-R26, V regs
635
636 // ---- dst = src ^ keystream ----
637 MOVV 8(R3), R4
638 VMOVQ (R4), V1
639 VXORV V1, V0, V0
640 MOVV 16(R3), R5
641 VMOVQ V0, (R5)
642
643 // ---- advance pointers, decrement len ----
644 MOVV 8(R3), R4; ADDV $16, R4; MOVV R4, 8(R3)
645 MOVV 16(R3), R5; ADDV $16, R5; MOVV R5, 16(R3)
646 MOVV 24(R3), R6; ADDV $-16, R6; MOVV R6, 24(R3)
647
648 JMP Ldec_block_loop
649
650 Ldec_tail:
651 // 0 < R6 < 16: R6 is at 24(R3)
652 // ---- 1. zero the ghash pad buffer at 88(R3) ----
653 // (the keystream buffer at 72(R3) needs no zeroing: it is fully
654 // overwritten by the VMOVQ below)
655 MOVV R0, 88(R3)
656 MOVV R0, 96(R3)
657
658 // ---- 2. copy R6 ciphertext bytes into pad buffer 88(R3)
659 // AND GHASH them (ciphertext = input for dec) ----
660 MOVV 8(R3), R23 // src tail ptr
661 MOVV R3, R20; ADDV $88, R20 // &pad buffer
662 MOVV 24(R3), R21 // tail len
663 Ldec_tail_copy:
664 MOVB (R23), R12
665 MOVB R12, (R20)
666 ADDV $1, R23; ADDV $1, R20; ADDV $-1, R21
667 BNE R21, R0, Ldec_tail_copy
668
669 // ---- 3. GHASH the zero-padded ciphertext block ----
670 MOVV R3, R4; ADDV $88, R4
671 MOVV 64(R3), R7
672 GHASHSTEP()
673
674 // ---- ctr++ BEFORE encrypt ----
675 MOVV 32(R3), R13
676 MOVWU 12(R13), R12
677 REVB2W R12, R12
678 ADDV $1, R12
679 REVB2W R12, R12
680 MOVW R12, 12(R13)
681
682 // ---- 4. encrypt current ctr -> keystream on stack ----
683 MOVV 56(R3), R4 // nr
684 MOVV 48(R3), R5 // xk
685 VMOVQ (R13), V0
686 CALL aesEnc<>(SB)
687 VMOVQ V0, 72(R3) // keystream buffer
688
689 // ---- 5. dst[i] = src[i] ^ keystream[i], i < R6 ----
690 MOVV 8(R3), R16 // src tail
691 MOVV 16(R3), R17 // dst tail
692 MOVV R3, R20; ADDV $72, R20 // &keystream
693 MOVV 24(R3), R21 // tail len
694 Ldec_tail_xor:
695 MOVB (R16), R12
696 MOVB (R20), R18
697 XOR R18, R12
698 MOVB R12, (R17)
699 ADDV $1, R16
700 ADDV $1, R17
701 ADDV $1, R20
702 ADDV $-1, R21
703 BNE R21, R0, Ldec_tail_xor
704
705 Ldec_done:
706 // write y back to T
707 MOVV 40(R3), R14
708 REVB2W R11, R12; MOVW R12, 0(R14)
709 REVB2W R10, R12; MOVW R12, 4(R14)
710 REVB2W R9, R12; MOVW R12, 8(R14)
711 REVB2W R8, R12; MOVW R12, 12(R14)
712
713 MOVV 120(R3), R1 // restore return address
714 RET
715
716 // -----------------------------------------------------------------------
717 // func gcmAesEnc(productTable *[256]byte, dst, src []byte,
718 // ctr, T *[16]byte, ks []uint32)
719 // -----------------------------------------------------------------------
720 TEXT ·gcmAesEnc(SB), NOSPLIT, $16-96
721 MOVV R1, 8(R3) // save return address
722 MOVV productTable+0(FP), R7
723 MOVV dst_base+8(FP), R5
724 MOVV src_base+32(FP), R4
725 MOVV src_len+40(FP), R6
726 MOVV ctr+56(FP), R13
727 MOVV T+64(FP), R14
728 MOVV ks_base+72(FP), R16
729 MOVV ks_len+80(FP), R15
730 SRLV $2, R15, R15
731 ADDV $-1, R15 // nr = ks_len/4 - 1
732
733 CALL gcmCryptEnc<>(SB)
734
735 MOVV 8(R3), R1 // restore return address
736 RET
737
738 // -----------------------------------------------------------------------
739 // func gcmAesDec(productTable *[256]byte, dst, src []byte,
740 // ctr, T *[16]byte, ks []uint32)
741 // -----------------------------------------------------------------------
742 TEXT ·gcmAesDec(SB), NOSPLIT, $16-96
743 MOVV R1, 8(R3) // save return address
744 MOVV productTable+0(FP), R7
745 MOVV dst_base+8(FP), R5
746 MOVV src_base+32(FP), R4
747 MOVV src_len+40(FP), R6
748 MOVV ctr+56(FP), R13
749 MOVV T+64(FP), R14
750 MOVV ks_base+72(FP), R16
751 MOVV ks_len+80(FP), R15
752 SRLV $2, R15, R15
753 ADDV $-1, R15
754
755 CALL gcmCryptDec<>(SB)
756
757 RET
758
View as plain text