1 // Copyright 2015 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
4
5 #include "go_asm.h"
6 #include "textflag.h"
7 #include "funcdata.h"
8
9 // bool armcas(int32 *val, int32 old, int32 new)
10 // Atomically:
11 // if(*val == old){
12 // *val = new;
13 // return 1;
14 // }else
15 // return 0;
16 //
17 // To implement ·cas in sys_$GOOS_arm.s
18 // using the native instructions, use:
19 //
20 // TEXT ·cas(SB),NOSPLIT,$0
21 // B ·armcas(SB)
22 //
23 TEXT ·armcas(SB),NOSPLIT,$0-13
24 MOVW ptr+0(FP), R1
25 MOVW old+4(FP), R2
26 MOVW new+8(FP), R3
27 casl:
28 LDREX (R1), R0
29 CMP R0, R2
30 BNE casfail
31
32 #ifndef GOARM_7
33 MOVB internal∕cpu·ARM+const_offsetARMHasV7Atomics(SB), R11
34 CMP $0, R11
35 BEQ 2(PC)
36 #endif
37 DMB MB_ISHST
38
39 STREX R3, (R1), R0
40 CMP $0, R0
41 BNE casl
42 MOVW $1, R0
43
44 #ifndef GOARM_7
45 CMP $0, R11
46 BEQ 2(PC)
47 #endif
48 DMB MB_ISH
49
50 MOVB R0, ret+12(FP)
51 RET
52 casfail:
53 MOVW $0, R0
54 MOVB R0, ret+12(FP)
55 RET
56
57 // stubs
58
59 TEXT ·Loadp(SB),NOSPLIT|NOFRAME,$0-8
60 B ·Load(SB)
61
62 TEXT ·LoadAcq(SB),NOSPLIT|NOFRAME,$0-8
63 B ·Load(SB)
64
65 TEXT ·LoadAcquintptr(SB),NOSPLIT|NOFRAME,$0-8
66 B ·Load(SB)
67
68 TEXT ·Casint32(SB),NOSPLIT,$0-13
69 B ·Cas(SB)
70
71 TEXT ·Casint64(SB),NOSPLIT,$-4-21
72 B ·Cas64(SB)
73
74 TEXT ·Casuintptr(SB),NOSPLIT,$0-13
75 B ·Cas(SB)
76
77 TEXT ·Casp1(SB),NOSPLIT,$0-13
78 B ·Cas(SB)
79
80 TEXT ·CasRel(SB),NOSPLIT,$0-13
81 B ·Cas(SB)
82
83 TEXT ·Loadint32(SB),NOSPLIT,$0-8
84 B ·Load(SB)
85
86 TEXT ·Loadint64(SB),NOSPLIT,$-4-12
87 B ·Load64(SB)
88
89 TEXT ·Loaduintptr(SB),NOSPLIT,$0-8
90 B ·Load(SB)
91
92 TEXT ·Loaduint(SB),NOSPLIT,$0-8
93 B ·Load(SB)
94
95 TEXT ·Storeint32(SB),NOSPLIT,$0-8
96 B ·Store(SB)
97
98 TEXT ·Storeint64(SB),NOSPLIT,$0-12
99 B ·Store64(SB)
100
101 TEXT ·Storeuintptr(SB),NOSPLIT,$0-8
102 B ·Store(SB)
103
104 TEXT ·StorepNoWB(SB),NOSPLIT,$0-8
105 B ·Store(SB)
106
107 TEXT ·StoreRel(SB),NOSPLIT,$0-8
108 B ·Store(SB)
109
110 TEXT ·StoreReluintptr(SB),NOSPLIT,$0-8
111 B ·Store(SB)
112
113 TEXT ·Xaddint32(SB),NOSPLIT,$0-12
114 B ·Xadd(SB)
115
116 TEXT ·Xaddint64(SB),NOSPLIT,$-4-20
117 B ·Xadd64(SB)
118
119 TEXT ·Xadduintptr(SB),NOSPLIT,$0-12
120 B ·Xadd(SB)
121
122 TEXT ·Xchgint32(SB),NOSPLIT,$0-12
123 B ·Xchg(SB)
124
125 TEXT ·Xchgint64(SB),NOSPLIT,$-4-20
126 B ·Xchg64(SB)
127
128 // 64-bit atomics
129 // The native ARM implementations use LDREXD/STREXD, which are
130 // available on ARMv6k or later. We use them only on ARMv7.
131 // On older ARM, we use Go implementations which simulate 64-bit
132 // atomics with locks.
133 TEXT armCas64<>(SB),NOSPLIT,$0-21
134 // addr is already in R1
135 MOVW old_lo+4(FP), R2
136 MOVW old_hi+8(FP), R3
137 MOVW new_lo+12(FP), R4
138 MOVW new_hi+16(FP), R5
139 cas64loop:
140 LDREXD (R1), R6 // loads R6 and R7
141 CMP R2, R6
142 BNE cas64fail
143 CMP R3, R7
144 BNE cas64fail
145
146 DMB MB_ISHST
147
148 STREXD R4, (R1), R0 // stores R4 and R5
149 CMP $0, R0
150 BNE cas64loop
151 MOVW $1, R0
152
153 DMB MB_ISH
154
155 MOVBU R0, swapped+20(FP)
156 RET
157 cas64fail:
158 MOVW $0, R0
159 MOVBU R0, swapped+20(FP)
160 RET
161
162 TEXT armXadd64<>(SB),NOSPLIT,$0-20
163 // addr is already in R1
164 MOVW delta_lo+4(FP), R2
165 MOVW delta_hi+8(FP), R3
166
167 add64loop:
168 LDREXD (R1), R4 // loads R4 and R5
169 ADD.S R2, R4
170 ADC R3, R5
171
172 DMB MB_ISHST
173
174 STREXD R4, (R1), R0 // stores R4 and R5
175 CMP $0, R0
176 BNE add64loop
177
178 DMB MB_ISH
179
180 MOVW R4, new_lo+12(FP)
181 MOVW R5, new_hi+16(FP)
182 RET
183
184 TEXT armXchg64<>(SB),NOSPLIT,$0-20
185 // addr is already in R1
186 MOVW new_lo+4(FP), R2
187 MOVW new_hi+8(FP), R3
188
189 swap64loop:
190 LDREXD (R1), R4 // loads R4 and R5
191
192 DMB MB_ISHST
193
194 STREXD R2, (R1), R0 // stores R2 and R3
195 CMP $0, R0
196 BNE swap64loop
197
198 DMB MB_ISH
199
200 MOVW R4, old_lo+12(FP)
201 MOVW R5, old_hi+16(FP)
202 RET
203
204 TEXT armLoad64<>(SB),NOSPLIT,$0-12
205 // addr is already in R1
206
207 LDREXD (R1), R2 // loads R2 and R3
208 DMB MB_ISH
209
210 MOVW R2, val_lo+4(FP)
211 MOVW R3, val_hi+8(FP)
212 RET
213
214 TEXT armStore64<>(SB),NOSPLIT,$0-12
215 // addr is already in R1
216 MOVW val_lo+4(FP), R2
217 MOVW val_hi+8(FP), R3
218
219 store64loop:
220 LDREXD (R1), R4 // loads R4 and R5
221
222 DMB MB_ISHST
223
224 STREXD R2, (R1), R0 // stores R2 and R3
225 CMP $0, R0
226 BNE store64loop
227
228 DMB MB_ISH
229 RET
230
231 TEXT armAnd8<>(SB),NOSPLIT,$0-5
232 // addr is already in R1
233 MOVB v+4(FP), R2
234
235 and8loop:
236 LDREXB (R1), R6
237
238 DMB MB_ISHST
239
240 AND R2, R6
241 STREXB R6, (R1), R0
242 CMP $0, R0
243 BNE and8loop
244
245 DMB MB_ISH
246
247 RET
248
249 TEXT armOr8<>(SB),NOSPLIT,$0-5
250 // addr is already in R1
251 MOVB v+4(FP), R2
252
253 or8loop:
254 LDREXB (R1), R6
255
256 DMB MB_ISHST
257
258 ORR R2, R6
259 STREXB R6, (R1), R0
260 CMP $0, R0
261 BNE or8loop
262
263 DMB MB_ISH
264
265 RET
266
267 TEXT armXchg8<>(SB),NOSPLIT,$0-9
268 // addr is already in R1
269 MOVB v+4(FP), R2
270 xchg8loop:
271 LDREXB (R1), R6
272
273 DMB MB_ISHST
274
275 STREXB R2, (R1), R0
276 CMP $0, R0
277 BNE xchg8loop
278
279 DMB MB_ISH
280
281 MOVB R6, ret+8(FP)
282 RET
283
284 // The following functions all panic if their address argument isn't
285 // 8-byte aligned. Since we're calling back into Go code to do this,
286 // we have to cooperate with stack unwinding. In the normal case, the
287 // functions tail-call into the appropriate implementation, which
288 // means they must not open a frame. Hence, when they go down the
289 // panic path, at that point they push the LR to create a real frame
290 // (they don't need to pop it because panic won't return; however, we
291 // do need to set the SP delta back).
292
293 // Check if R1 is 8-byte aligned, panic if not.
294 // Clobbers R2.
295 #define CHECK_ALIGN \
296 AND.S $7, R1, R2 \
297 BEQ 4(PC) \
298 MOVW.W R14, -4(R13) /* prepare a real frame */ \
299 BL ·panicUnaligned(SB) \
300 ADD $4, R13 /* compensate SP delta */
301
302 TEXT ·Cas64(SB),NOSPLIT,$-4-21
303 NO_LOCAL_POINTERS
304 MOVW addr+0(FP), R1
305 CHECK_ALIGN
306
307 #ifndef GOARM_7
308 MOVB internal∕cpu·ARM+const_offsetARMHasV7Atomics(SB), R11
309 CMP $1, R11
310 BEQ 2(PC)
311 JMP ·goCas64(SB)
312 #endif
313 JMP armCas64<>(SB)
314
315 TEXT ·Xadd64(SB),NOSPLIT,$-4-20
316 NO_LOCAL_POINTERS
317 MOVW addr+0(FP), R1
318 CHECK_ALIGN
319
320 #ifndef GOARM_7
321 MOVB internal∕cpu·ARM+const_offsetARMHasV7Atomics(SB), R11
322 CMP $1, R11
323 BEQ 2(PC)
324 JMP ·goXadd64(SB)
325 #endif
326 JMP armXadd64<>(SB)
327
328 TEXT ·Xchg64(SB),NOSPLIT,$-4-20
329 NO_LOCAL_POINTERS
330 MOVW addr+0(FP), R1
331 CHECK_ALIGN
332
333 #ifndef GOARM_7
334 MOVB internal∕cpu·ARM+const_offsetARMHasV7Atomics(SB), R11
335 CMP $1, R11
336 BEQ 2(PC)
337 JMP ·goXchg64(SB)
338 #endif
339 JMP armXchg64<>(SB)
340
341 TEXT ·Load64(SB),NOSPLIT,$-4-12
342 NO_LOCAL_POINTERS
343 MOVW addr+0(FP), R1
344 CHECK_ALIGN
345
346 #ifndef GOARM_7
347 MOVB internal∕cpu·ARM+const_offsetARMHasV7Atomics(SB), R11
348 CMP $1, R11
349 BEQ 2(PC)
350 JMP ·goLoad64(SB)
351 #endif
352 JMP armLoad64<>(SB)
353
354 TEXT ·Store64(SB),NOSPLIT,$-4-12
355 NO_LOCAL_POINTERS
356 MOVW addr+0(FP), R1
357 CHECK_ALIGN
358
359 #ifndef GOARM_7
360 MOVB internal∕cpu·ARM+const_offsetARMHasV7Atomics(SB), R11
361 CMP $1, R11
362 BEQ 2(PC)
363 JMP ·goStore64(SB)
364 #endif
365 JMP armStore64<>(SB)
366
367 TEXT ·And8(SB),NOSPLIT,$-4-5
368 NO_LOCAL_POINTERS
369 MOVW addr+0(FP), R1
370
371 // Uses STREXB/LDREXB that is armv6k or later.
372 // For simplicity we only enable this on armv7.
373 #ifndef GOARM_7
374 MOVB internal∕cpu·ARM+const_offsetARMHasV7Atomics(SB), R11
375 CMP $1, R11
376 BEQ 2(PC)
377 JMP ·goAnd8(SB)
378 #endif
379 JMP armAnd8<>(SB)
380
381 TEXT ·Or8(SB),NOSPLIT,$-4-5
382 NO_LOCAL_POINTERS
383 MOVW addr+0(FP), R1
384
385 // Uses STREXB/LDREXB that is armv6k or later.
386 // For simplicity we only enable this on armv7.
387 #ifndef GOARM_7
388 MOVB internal∕cpu·ARM+const_offsetARMHasV7Atomics(SB), R11
389 CMP $1, R11
390 BEQ 2(PC)
391 JMP ·goOr8(SB)
392 #endif
393 JMP armOr8<>(SB)
394
395 TEXT ·Xchg8(SB),NOSPLIT,$-4-9
396 NO_LOCAL_POINTERS
397 MOVW addr+0(FP), R1
398
399 // Uses STREXB/LDREXB that is armv6k or later.
400 // For simplicity we only enable this on armv7.
401 #ifndef GOARM_7
402 MOVB internal∕cpu·ARM+const_offsetARMHasV7Atomics(SB), R11
403 CMP $1, R11
404 BEQ 2(PC)
405 JMP ·goXchg8(SB)
406 #endif
407 JMP armXchg8<>(SB)
408
View as plain text