1 // Copyright 2022 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
4
5 #include "go_asm.h"
6 #include "textflag.h"
7
8 // func Cas(ptr *int32, old, new int32) bool
9 // Atomically:
10 // if *ptr == old {
11 // *ptr = new
12 // return true
13 // } else {
14 // return false
15 // }
16 TEXT ·Cas(SB), NOSPLIT, $0-17
17 MOVV ptr+0(FP), R4
18 MOVW old+8(FP), R5
19 MOVW new+12(FP), R6
20
21 MOVBU internal∕cpu·Loong64+const_offsetLOONG64HasLAMCAS(SB), R8
22 BEQ R8, ll_sc
23 MOVV R5, R7 // backup old value
24 AMCASDBW R6, (R4), R5
25 BNE R7, R5, cas_fail0
26 MOVV $1, R4
27 MOVB R4, ret+16(FP)
28 RET
29 cas_fail0:
30 MOVB R0, ret+16(FP)
31 RET
32
33 ll_sc:
34 // Implemented using the ll-sc instruction pair
35 DBAR $0x14 // LoadAcquire barrier
36 cas_again:
37 MOVV R6, R7
38 LL (R4), R8
39 BNE R5, R8, cas_fail1
40 SC R7, (R4)
41 BEQ R7, cas_again
42 MOVV $1, R4
43 MOVB R4, ret+16(FP)
44 DBAR $0x12 // StoreRelease barrier
45 RET
46 cas_fail1:
47 MOVV $0, R4
48 JMP -4(PC)
49
50 // func Cas64(ptr *uint64, old, new uint64) bool
51 // Atomically:
52 // if *ptr == old {
53 // *ptr = new
54 // return true
55 // } else {
56 // return false
57 // }
58 TEXT ·Cas64(SB), NOSPLIT, $0-25
59 MOVV ptr+0(FP), R4
60 MOVV old+8(FP), R5
61 MOVV new+16(FP), R6
62
63 MOVBU internal∕cpu·Loong64+const_offsetLOONG64HasLAMCAS(SB), R8
64 BEQ R8, ll_sc_64
65 MOVV R5, R7 // backup old value
66 AMCASDBV R6, (R4), R5
67 BNE R7, R5, cas64_fail0
68 MOVV $1, R4
69 MOVB R4, ret+24(FP)
70 RET
71 cas64_fail0:
72 MOVB R0, ret+24(FP)
73 RET
74
75 ll_sc_64:
76 // Implemented using the ll-sc instruction pair
77 DBAR $0x14
78 cas64_again:
79 MOVV R6, R7
80 LLV (R4), R8
81 BNE R5, R8, cas64_fail1
82 SCV R7, (R4)
83 BEQ R7, cas64_again
84 MOVV $1, R4
85 MOVB R4, ret+24(FP)
86 DBAR $0x12
87 RET
88 cas64_fail1:
89 MOVV $0, R4
90 JMP -4(PC)
91
92 TEXT ·Casint32(SB),NOSPLIT,$0-17
93 JMP ·Cas(SB)
94
95 TEXT ·Casint64(SB),NOSPLIT,$0-25
96 JMP ·Cas64(SB)
97
98 TEXT ·Casuintptr(SB), NOSPLIT, $0-25
99 JMP ·Cas64(SB)
100
101 TEXT ·CasRel(SB), NOSPLIT, $0-17
102 JMP ·Cas(SB)
103
104 TEXT ·Loaduintptr(SB), NOSPLIT|NOFRAME, $0-16
105 JMP ·Load64(SB)
106
107 TEXT ·Loaduint(SB), NOSPLIT|NOFRAME, $0-16
108 JMP ·Load64(SB)
109
110 TEXT ·Storeuintptr(SB), NOSPLIT, $0-16
111 JMP ·Store64(SB)
112
113 TEXT ·Xadduintptr(SB), NOSPLIT, $0-24
114 JMP ·Xadd64(SB)
115
116 TEXT ·Loadint64(SB), NOSPLIT, $0-16
117 JMP ·Load64(SB)
118
119 TEXT ·Xaddint32(SB),NOSPLIT,$0-20
120 JMP ·Xadd(SB)
121
122 TEXT ·Xaddint64(SB), NOSPLIT, $0-24
123 JMP ·Xadd64(SB)
124
125 // func Casp(ptr *unsafe.Pointer, old, new unsafe.Pointer) bool
126 // Atomically:
127 // if *ptr == old {
128 // *ptr = new
129 // return true
130 // } else {
131 // return false
132 // }
133 TEXT ·Casp1(SB), NOSPLIT, $0-25
134 JMP ·Cas64(SB)
135
136 // uint32 Xadd(uint32 volatile *ptr, int32 delta)
137 // Atomically:
138 // *val += delta;
139 // return *val;
140 TEXT ·Xadd(SB), NOSPLIT, $0-20
141 MOVV ptr+0(FP), R4
142 MOVW delta+8(FP), R5
143 AMADDDBW R5, (R4), R6
144 ADDV R6, R5, R4
145 MOVW R4, ret+16(FP)
146 RET
147
148 // func Xadd64(ptr *uint64, delta int64) uint64
149 TEXT ·Xadd64(SB), NOSPLIT, $0-24
150 MOVV ptr+0(FP), R4
151 MOVV delta+8(FP), R5
152 AMADDDBV R5, (R4), R6
153 ADDV R6, R5, R4
154 MOVV R4, ret+16(FP)
155 RET
156
157 // uint8 Xchg8(ptr *uint8, new uint8)
158 // Atomically:
159 // old := *ptr;
160 // *ptr = new;
161 // return old;
162 TEXT ·Xchg8(SB), NOSPLIT, $0-17
163 MOVV ptr+0(FP), R4
164 MOVBU new+8(FP), R5
165
166 // R6 = ((ptr & 3) * 8)
167 AND $3, R4, R6
168 SLLV $3, R6
169
170 // R7 = ((0xFF) << R6) ^ (-1)
171 MOVV $0xFF, R8
172 SLLV R6, R8, R7
173 XOR $-1, R7
174
175 // R4 = ptr & (~3)
176 MOVV $~3, R8
177 AND R8, R4
178
179 // R5 = ((val) << R6)
180 SLLV R6, R5
181
182 DBAR $0x14 // LoadAcquire barrier
183 _xchg8_again:
184 LL (R4), R8
185 MOVV R8, R9 // backup old val
186 AND R7, R8
187 OR R5, R8
188 SC R8, (R4)
189 BEQ R8, _xchg8_again
190 DBAR $0x12 // StoreRelease barrier
191 SRLV R6, R9, R9
192 MOVBU R9, ret+16(FP)
193 RET
194
195 // func Xchg(ptr *uint32, new uint32) uint32
196 TEXT ·Xchg(SB), NOSPLIT, $0-20
197 MOVV ptr+0(FP), R4
198 MOVW new+8(FP), R5
199 AMSWAPDBW R5, (R4), R6
200 MOVW R6, ret+16(FP)
201 RET
202
203 // func Xchg64(ptr *uint64, new uint64) uint64
204 TEXT ·Xchg64(SB), NOSPLIT, $0-24
205 MOVV ptr+0(FP), R4
206 MOVV new+8(FP), R5
207 AMSWAPDBV R5, (R4), R6
208 MOVV R6, ret+16(FP)
209 RET
210
211 TEXT ·Xchguintptr(SB), NOSPLIT, $0-24
212 JMP ·Xchg64(SB)
213
214 // func Xchgint32(ptr *int32, new int32) int32
215 TEXT ·Xchgint32(SB), NOSPLIT, $0-20
216 JMP ·Xchg(SB)
217
218 // func Xchgint64(ptr *int64, new int64) int64
219 TEXT ·Xchgint64(SB), NOSPLIT, $0-24
220 JMP ·Xchg64(SB)
221
222 TEXT ·StorepNoWB(SB), NOSPLIT, $0-16
223 JMP ·Store64(SB)
224
225 TEXT ·StoreRel(SB), NOSPLIT, $0-12
226 JMP ·Store(SB)
227
228 TEXT ·StoreRel64(SB), NOSPLIT, $0-16
229 JMP ·Store64(SB)
230
231 TEXT ·StoreReluintptr(SB), NOSPLIT, $0-16
232 JMP ·Store64(SB)
233
234 TEXT ·Store8(SB), NOSPLIT, $0-9
235 MOVV ptr+0(FP), R4
236 MOVB val+8(FP), R5
237 // StoreRelease barrier
238 DBAR $0x12
239 MOVB R5, 0(R4)
240 DBAR $0x18
241 RET
242
243 TEXT ·Store(SB), NOSPLIT, $0-12
244 MOVV ptr+0(FP), R4
245 MOVW val+8(FP), R5
246 MOVBU internal∕cpu·Loong64+const_offsetLoong64HasDBAR_HINTS(SB), R6
247 BEQ R6, _variant_
248 // StoreRelease barrier
249 DBAR $0x12
250 MOVW R5, 0(R4)
251 DBAR $0x18
252 RET
253 _variant_:
254 AMSWAPDBW R5, (R4), R0
255 RET
256
257 TEXT ·Store64(SB), NOSPLIT, $0-16
258 MOVV ptr+0(FP), R4
259 MOVV val+8(FP), R5
260 MOVBU internal∕cpu·Loong64+const_offsetLoong64HasDBAR_HINTS(SB), R6
261 BEQ R6, _variant_
262 // StoreRelease barrier
263 DBAR $0x12
264 MOVV R5, 0(R4)
265 DBAR $0x18
266 RET
267 _variant_:
268 AMSWAPDBV R5, (R4), R0
269 RET
270
271 // void Or8(byte volatile*, byte);
272 TEXT ·Or8(SB), NOSPLIT, $0-9
273 MOVV ptr+0(FP), R4
274 MOVBU val+8(FP), R5
275 // R6 = ptr & (~3)
276 MOVV $~3, R6
277 AND R4, R6
278 // R7 = ((ptr & 3) * 8)
279 AND $3, R4, R7
280 SLLV $3, R7
281 // R5 = val << R7
282 SLLV R7, R5
283 AMORDBW R5, (R6), R0
284 RET
285
286 // void And8(byte volatile*, byte);
287 TEXT ·And8(SB), NOSPLIT, $0-9
288 MOVV ptr+0(FP), R4
289 MOVBU val+8(FP), R5
290 // R6 = ptr & (~3)
291 MOVV $~3, R6
292 AND R4, R6
293 // R7 = ((ptr & 3) * 8)
294 AND $3, R4, R7
295 SLLV $3, R7
296 // R5 = ((val ^ 0xFF) << R7) ^ (-1)
297 XOR $255, R5
298 SLLV R7, R5
299 XOR $-1, R5
300 AMANDDBW R5, (R6), R0
301 RET
302
303 // func Or(addr *uint32, v uint32)
304 TEXT ·Or(SB), NOSPLIT, $0-12
305 MOVV ptr+0(FP), R4
306 MOVW val+8(FP), R5
307 AMORDBW R5, (R4), R0
308 RET
309
310 // func And(addr *uint32, v uint32)
311 TEXT ·And(SB), NOSPLIT, $0-12
312 MOVV ptr+0(FP), R4
313 MOVW val+8(FP), R5
314 AMANDDBW R5, (R4), R0
315 RET
316
317 // func Or32(addr *uint32, v uint32) old uint32
318 TEXT ·Or32(SB), NOSPLIT, $0-20
319 MOVV ptr+0(FP), R4
320 MOVW val+8(FP), R5
321 AMORDBW R5, (R4), R6
322 MOVW R6, ret+16(FP)
323 RET
324
325 // func And32(addr *uint32, v uint32) old uint32
326 TEXT ·And32(SB), NOSPLIT, $0-20
327 MOVV ptr+0(FP), R4
328 MOVW val+8(FP), R5
329 AMANDDBW R5, (R4), R6
330 MOVW R6, ret+16(FP)
331 RET
332
333 // func Or64(addr *uint64, v uint64) old uint64
334 TEXT ·Or64(SB), NOSPLIT, $0-24
335 MOVV ptr+0(FP), R4
336 MOVV val+8(FP), R5
337 AMORDBV R5, (R4), R6
338 MOVV R6, ret+16(FP)
339 RET
340
341 // func And64(addr *uint64, v uint64) old uint64
342 TEXT ·And64(SB), NOSPLIT, $0-24
343 MOVV ptr+0(FP), R4
344 MOVV val+8(FP), R5
345 AMANDDBV R5, (R4), R6
346 MOVV R6, ret+16(FP)
347 RET
348
349 // func Anduintptr(addr *uintptr, v uintptr) old uintptr
350 TEXT ·Anduintptr(SB), NOSPLIT, $0-24
351 JMP ·And64(SB)
352
353 // func Oruintptr(addr *uintptr, v uintptr) old uintptr
354 TEXT ·Oruintptr(SB), NOSPLIT, $0-24
355 JMP ·Or64(SB)
356
357 // uint32 internal∕runtime∕atomic·Load(uint32 volatile* ptr)
358 TEXT ·Load(SB),NOSPLIT|NOFRAME,$0-12
359 MOVV ptr+0(FP), R19
360 MOVWU 0(R19), R19
361 DBAR $0x14 // LoadAcquire barrier
362 MOVW R19, ret+8(FP)
363 RET
364
365 // uint8 internal∕runtime∕atomic·Load8(uint8 volatile* ptr)
366 TEXT ·Load8(SB),NOSPLIT|NOFRAME,$0-9
367 MOVV ptr+0(FP), R19
368 MOVBU 0(R19), R19
369 DBAR $0x14
370 MOVB R19, ret+8(FP)
371 RET
372
373 // uint64 internal∕runtime∕atomic·Load64(uint64 volatile* ptr)
374 TEXT ·Load64(SB),NOSPLIT|NOFRAME,$0-16
375 MOVV ptr+0(FP), R19
376 MOVV 0(R19), R19
377 DBAR $0x14
378 MOVV R19, ret+8(FP)
379 RET
380
381 // void *internal∕runtime∕atomic·Loadp(void *volatile *ptr)
382 TEXT ·Loadp(SB),NOSPLIT|NOFRAME,$0-16
383 JMP ·Load64(SB)
384
385 // uint32 internal∕runtime∕atomic·LoadAcq(uint32 volatile* ptr)
386 TEXT ·LoadAcq(SB),NOSPLIT|NOFRAME,$0-12
387 JMP ·Load(SB)
388
389 // uint64 ·LoadAcq64(uint64 volatile* ptr)
390 TEXT ·LoadAcq64(SB),NOSPLIT|NOFRAME,$0-16
391 JMP ·Load64(SB)
392
393 // uintptr ·LoadAcquintptr(uintptr volatile* ptr)
394 TEXT ·LoadAcquintptr(SB),NOSPLIT|NOFRAME,$0-16
395 JMP ·Load64(SB)
396
397
View as plain text