1 // Copyright 2014 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
4
5 // RISC-V's atomic operations have two bits, aq ("acquire") and rl ("release"),
6 // which may be toggled on and off. Their precise semantics are defined in
7 // section 6.3 of the specification, but the basic idea is as follows:
8 //
9 // - If neither aq nor rl is set, the CPU may reorder the atomic arbitrarily.
10 // It guarantees only that it will execute atomically.
11 //
12 // - If aq is set, the CPU may move the instruction backward, but not forward.
13 //
14 // - If rl is set, the CPU may move the instruction forward, but not backward.
15 //
16 // - If both are set, the CPU may not reorder the instruction at all.
17 //
18 // These four modes correspond to other well-known memory models on other CPUs.
19 // On ARM, aq corresponds to a dmb ishst, aq+rl corresponds to a dmb ish. On
20 // Intel, aq corresponds to an lfence, rl to an sfence, and aq+rl to an mfence
21 // (or a lock prefix).
22 //
23 // Go's memory model requires that
24 // - if a read happens after a write, the read must observe the write, and
25 // that
26 // - if a read happens concurrently with a write, the read may observe the
27 // write.
28 // aq is sufficient to guarantee this, so that's what we use here. (This jibes
29 // with ARM, which uses dmb ishst.)
30
31 #include "textflag.h"
32
33 // func Cas(ptr *uint64, old, new uint64) bool
34 // Atomically:
35 // if(*val == old){
36 // *val = new;
37 // return 1;
38 // } else {
39 // return 0;
40 // }
41 TEXT ·Cas(SB), NOSPLIT, $0-17
42 MOV ptr+0(FP), A0
43 MOVW old+8(FP), A1
44 MOVW new+12(FP), A2
45 cas_again:
46 LRW (A0), A3
47 BNE A3, A1, cas_fail
48 SCW A2, (A0), A4
49 BNE A4, ZERO, cas_again
50 MOV $1, A0
51 MOVB A0, ret+16(FP)
52 RET
53 cas_fail:
54 MOV $0, A0
55 MOV A0, ret+16(FP)
56 RET
57
58 // func Cas64(ptr *uint64, old, new uint64) bool
59 TEXT ·Cas64(SB), NOSPLIT, $0-25
60 MOV ptr+0(FP), A0
61 MOV old+8(FP), A1
62 MOV new+16(FP), A2
63 cas_again:
64 LRD (A0), A3
65 BNE A3, A1, cas_fail
66 SCD A2, (A0), A4
67 BNE A4, ZERO, cas_again
68 MOV $1, A0
69 MOVB A0, ret+24(FP)
70 RET
71 cas_fail:
72 MOVB ZERO, ret+24(FP)
73 RET
74
75 // func Load(ptr *uint32) uint32
76 TEXT ·Load(SB),NOSPLIT|NOFRAME,$0-12
77 MOV ptr+0(FP), A0
78 LRW (A0), A0
79 MOVW A0, ret+8(FP)
80 RET
81
82 // func Load8(ptr *uint8) uint8
83 TEXT ·Load8(SB),NOSPLIT|NOFRAME,$0-9
84 MOV ptr+0(FP), A0
85 FENCE
86 MOVBU (A0), A1
87 FENCE
88 MOVB A1, ret+8(FP)
89 RET
90
91 // func Load64(ptr *uint64) uint64
92 TEXT ·Load64(SB),NOSPLIT|NOFRAME,$0-16
93 MOV ptr+0(FP), A0
94 LRD (A0), A0
95 MOV A0, ret+8(FP)
96 RET
97
98 // func Store(ptr *uint32, val uint32)
99 TEXT ·Store(SB), NOSPLIT, $0-12
100 MOV ptr+0(FP), A0
101 MOVW val+8(FP), A1
102 AMOSWAPW A1, (A0), ZERO
103 RET
104
105 // func Store8(ptr *uint8, val uint8)
106 TEXT ·Store8(SB), NOSPLIT, $0-9
107 MOV ptr+0(FP), A0
108 MOVBU val+8(FP), A1
109 FENCE
110 MOVB A1, (A0)
111 FENCE
112 RET
113
114 // func Store64(ptr *uint64, val uint64)
115 TEXT ·Store64(SB), NOSPLIT, $0-16
116 MOV ptr+0(FP), A0
117 MOV val+8(FP), A1
118 AMOSWAPD A1, (A0), ZERO
119 RET
120
121 TEXT ·Casp1(SB), NOSPLIT, $0-25
122 JMP ·Cas64(SB)
123
124 TEXT ·Casint32(SB),NOSPLIT,$0-17
125 JMP ·Cas(SB)
126
127 TEXT ·Casint64(SB),NOSPLIT,$0-25
128 JMP ·Cas64(SB)
129
130 TEXT ·Casuintptr(SB),NOSPLIT,$0-25
131 JMP ·Cas64(SB)
132
133 TEXT ·CasRel(SB), NOSPLIT, $0-17
134 JMP ·Cas(SB)
135
136 TEXT ·Loaduintptr(SB),NOSPLIT,$0-16
137 JMP ·Load64(SB)
138
139 TEXT ·Storeint32(SB),NOSPLIT,$0-12
140 JMP ·Store(SB)
141
142 TEXT ·Storeint64(SB),NOSPLIT,$0-16
143 JMP ·Store64(SB)
144
145 TEXT ·Storeuintptr(SB),NOSPLIT,$0-16
146 JMP ·Store64(SB)
147
148 TEXT ·Loaduint(SB),NOSPLIT,$0-16
149 JMP ·Loaduintptr(SB)
150
151 TEXT ·Loadint32(SB),NOSPLIT,$0-12
152 JMP ·Load(SB)
153
154 TEXT ·Loadint64(SB),NOSPLIT,$0-16
155 JMP ·Load64(SB)
156
157 TEXT ·Xaddint32(SB),NOSPLIT,$0-20
158 JMP ·Xadd(SB)
159
160 TEXT ·Xaddint64(SB),NOSPLIT,$0-24
161 MOV ptr+0(FP), A0
162 MOV delta+8(FP), A1
163 AMOADDD A1, (A0), A0
164 ADD A0, A1, A0
165 MOVW A0, ret+16(FP)
166 RET
167
168 TEXT ·LoadAcq(SB),NOSPLIT|NOFRAME,$0-12
169 JMP ·Load(SB)
170
171 TEXT ·LoadAcq64(SB),NOSPLIT|NOFRAME,$0-16
172 JMP ·Load64(SB)
173
174 TEXT ·LoadAcquintptr(SB),NOSPLIT|NOFRAME,$0-16
175 JMP ·Load64(SB)
176
177 // func Loadp(ptr unsafe.Pointer) unsafe.Pointer
178 TEXT ·Loadp(SB),NOSPLIT,$0-16
179 JMP ·Load64(SB)
180
181 // func StorepNoWB(ptr unsafe.Pointer, val unsafe.Pointer)
182 TEXT ·StorepNoWB(SB), NOSPLIT, $0-16
183 JMP ·Store64(SB)
184
185 TEXT ·StoreRel(SB), NOSPLIT, $0-12
186 JMP ·Store(SB)
187
188 TEXT ·StoreRel64(SB), NOSPLIT, $0-16
189 JMP ·Store64(SB)
190
191 TEXT ·StoreReluintptr(SB), NOSPLIT, $0-16
192 JMP ·Store64(SB)
193
194 // func Xchg(ptr *uint32, new uint32) uint32
195 TEXT ·Xchg(SB), NOSPLIT, $0-20
196 MOV ptr+0(FP), A0
197 MOVW new+8(FP), A1
198 AMOSWAPW A1, (A0), A1
199 MOVW A1, ret+16(FP)
200 RET
201
202 // func Xchg64(ptr *uint64, new uint64) uint64
203 TEXT ·Xchg64(SB), NOSPLIT, $0-24
204 MOV ptr+0(FP), A0
205 MOV new+8(FP), A1
206 AMOSWAPD A1, (A0), A1
207 MOV A1, ret+16(FP)
208 RET
209
210 // Atomically:
211 // *val += delta;
212 // return *val;
213
214 // func Xadd(ptr *uint32, delta int32) uint32
215 TEXT ·Xadd(SB), NOSPLIT, $0-20
216 MOV ptr+0(FP), A0
217 MOVW delta+8(FP), A1
218 AMOADDW A1, (A0), A2
219 ADD A2,A1,A0
220 MOVW A0, ret+16(FP)
221 RET
222
223 // func Xadd64(ptr *uint64, delta int64) uint64
224 TEXT ·Xadd64(SB), NOSPLIT, $0-24
225 MOV ptr+0(FP), A0
226 MOV delta+8(FP), A1
227 AMOADDD A1, (A0), A2
228 ADD A2, A1, A0
229 MOV A0, ret+16(FP)
230 RET
231
232 // func Xadduintptr(ptr *uintptr, delta uintptr) uintptr
233 TEXT ·Xadduintptr(SB), NOSPLIT, $0-24
234 JMP ·Xadd64(SB)
235
236 // func Xchgint32(ptr *int32, new int32) int32
237 TEXT ·Xchgint32(SB), NOSPLIT, $0-20
238 JMP ·Xchg(SB)
239
240 // func Xchgint64(ptr *int64, new int64) int64
241 TEXT ·Xchgint64(SB), NOSPLIT, $0-24
242 JMP ·Xchg64(SB)
243
244 // func Xchguintptr(ptr *uintptr, new uintptr) uintptr
245 TEXT ·Xchguintptr(SB), NOSPLIT, $0-24
246 JMP ·Xchg64(SB)
247
248 // func And8(ptr *uint8, val uint8)
249 TEXT ·And8(SB), NOSPLIT, $0-9
250 MOV ptr+0(FP), A0
251 MOVBU val+8(FP), A1
252 AND $3, A0, A2
253 AND $-4, A0
254 SLL $3, A2
255 XOR $255, A1
256 SLL A2, A1
257 XOR $-1, A1
258 AMOANDW A1, (A0), ZERO
259 RET
260
261 // func Or8(ptr *uint8, val uint8)
262 TEXT ·Or8(SB), NOSPLIT, $0-9
263 MOV ptr+0(FP), A0
264 MOVBU val+8(FP), A1
265 AND $3, A0, A2
266 AND $-4, A0
267 SLL $3, A2
268 SLL A2, A1
269 AMOORW A1, (A0), ZERO
270 RET
271
272 // func And(ptr *uint32, val uint32)
273 TEXT ·And(SB), NOSPLIT, $0-12
274 MOV ptr+0(FP), A0
275 MOVW val+8(FP), A1
276 AMOANDW A1, (A0), ZERO
277 RET
278
279 // func Or(ptr *uint32, val uint32)
280 TEXT ·Or(SB), NOSPLIT, $0-12
281 MOV ptr+0(FP), A0
282 MOVW val+8(FP), A1
283 AMOORW A1, (A0), ZERO
284 RET
285
286 // func Or32(ptr *uint32, val uint32) uint32
287 TEXT ·Or32(SB), NOSPLIT, $0-20
288 MOV ptr+0(FP), A0
289 MOVW val+8(FP), A1
290 AMOORW A1, (A0), A2
291 MOVW A2, ret+16(FP)
292 RET
293
294 // func And32(ptr *uint32, val uint32) uint32
295 TEXT ·And32(SB), NOSPLIT, $0-20
296 MOV ptr+0(FP), A0
297 MOVW val+8(FP), A1
298 AMOANDW A1, (A0), A2
299 MOVW A2, ret+16(FP)
300 RET
301
302 // func Or64(ptr *uint64, val uint64) uint64
303 TEXT ·Or64(SB), NOSPLIT, $0-24
304 MOV ptr+0(FP), A0
305 MOV val+8(FP), A1
306 AMOORD A1, (A0), A2
307 MOV A2, ret+16(FP)
308 RET
309
310 // func And64(ptr *uint64, val uint64) uint64
311 TEXT ·And64(SB), NOSPLIT, $0-24
312 MOV ptr+0(FP), A0
313 MOV val+8(FP), A1
314 AMOANDD A1, (A0), A2
315 MOV A2, ret+16(FP)
316 RET
317
318 // func Anduintptr(ptr *uintptr, val uintptr) uintptr
319 TEXT ·Anduintptr(SB), NOSPLIT, $0-24
320 JMP ·And64(SB)
321
322 // func Oruintptr(ptr *uintptr, val uintptr) uintptr
323 TEXT ·Oruintptr(SB), NOSPLIT, $0-24
324 JMP ·Or64(SB)
325
View as plain text