2 * Copyright 2007-2008 Analog Devices Inc.
3 * Philippe Gerum <rpm@xenomai.org>
5 * Licensed under the GPL-2 or later.
8 #include <linux/linkage.h>
9 #include <asm/blackfin.h>
10 #include <asm/cache.h>
11 #include <asm/asm-offsets.h>
12 #include <asm/rwlock.h>
17 .macro coreslot_loadaddr reg:req
18 \reg\().l = _corelock;
19 \reg\().h = _corelock;
22 .macro safe_testset addr:req, scratch:req
33 * r0 = address of atomic data to flush and invalidate (32bit).
35 * Clear interrupts and return the old mask.
36 * We assume that no atomic data can span cachelines.
47 if cc jump .Ldone_corelock;
52 /* flush core internal write buffer before invalidate dcache */
57 ENDPROC(_get_core_lock)
60 * r0 = address of atomic data in uncacheable memory region (32bit).
62 * Clear interrupts and return the old mask.
66 ENTRY(_get_core_lock_noflush)
69 .Lretry_corelock_noflush:
71 if cc jump .Ldone_corelock_noflush;
73 jump .Lretry_corelock_noflush
74 .Ldone_corelock_noflush:
76 * SMP kgdb runs into dead loop without NOP here, when one core
77 * single steps over get_core_lock_noflush and the other executes
78 * get_core_lock as a slave node.
83 ENDPROC(_get_core_lock_noflush)
86 * r0 = interrupt mask to restore.
87 * r1 = address of atomic data to flush and invalidate (32bit).
89 * Interrupts are masked on entry (see _get_core_lock).
93 /* Write-through cache assumed, so no flush needed here. */
100 ENDPROC(_put_core_lock)
102 #ifdef __ARCH_SYNC_CORE_DCACHE
104 ENTRY(___raw_smp_mark_barrier_asm)
110 call _get_core_lock_noflush;
113 * Calculate current core mask
120 * Set bit of other cores in barrier mask. Don't change current core bit.
122 p1.l = _barrier_mask;
123 p1.h = _barrier_mask;
141 ENDPROC(___raw_smp_mark_barrier_asm)
143 ENTRY(___raw_smp_check_barrier_asm)
149 call _get_core_lock_noflush;
152 * Calculate current core mask
159 * Clear current core bit in barrier mask if it is set.
161 p1.l = _barrier_mask;
162 p1.h = _barrier_mask;
175 * Invalidate the entire D-cache of current core.
178 call _resync_core_dcache
190 ENDPROC(___raw_smp_check_barrier_asm)
194 * r1 = address of atomic data
196 * Clobbers: r2:0, p1:0
198 _start_lock_coherent:
206 * Determine whether the atomic data was previously
207 * owned by another CPU (=r6).
215 r1 >>= 28; /* CPU fingerprints are stored in the high nibble. */
223 * Release the core lock now, but keep IRQs disabled while we are
224 * performing the remaining housekeeping chores for the current CPU.
226 coreslot_loadaddr p0;
231 * If another CPU has owned the same atomic section before us,
232 * then our D-cached copy of the shared data protected by the
233 * current spin/write_lock may be obsolete.
236 if cc jump .Lcache_synced
239 * Invalidate the entire D-cache of the current core.
242 call _resync_core_dcache
254 * r1 = address of atomic data
256 * Clobbers: r2:0, p1:0
271 #endif /* __ARCH_SYNC_CORE_DCACHE */
274 * r0 = &spinlock->lock
276 * Clobbers: r3:0, p1:0
278 ENTRY(___raw_spin_is_locked_asm)
283 cc = bittst( r3, 0 );
290 ENDPROC(___raw_spin_is_locked_asm)
293 * r0 = &spinlock->lock
295 * Clobbers: r3:0, p1:0
297 ENTRY(___raw_spin_lock_asm)
304 cc = bittst( r2, 0 );
305 if cc jump .Lbusy_spinlock
306 #ifdef __ARCH_SYNC_CORE_DCACHE
308 bitset ( r2, 0 ); /* Raise the lock bit. */
310 call _start_lock_coherent
320 /* We don't touch the atomic area if busy, so that flush
321 will behave like nop in _put_core_lock. */
325 jump .Lretry_spinlock
326 ENDPROC(___raw_spin_lock_asm)
329 * r0 = &spinlock->lock
331 * Clobbers: r3:0, p1:0
333 ENTRY(___raw_spin_trylock_asm)
339 cc = bittst( r3, 0 );
340 if cc jump .Lfailed_trylock
341 #ifdef __ARCH_SYNC_CORE_DCACHE
342 bitset ( r3, 0 ); /* Raise the lock bit. */
344 call _start_lock_coherent
358 ENDPROC(___raw_spin_trylock_asm)
361 * r0 = &spinlock->lock
363 * Clobbers: r2:0, p1:0
365 ENTRY(___raw_spin_unlock_asm)
373 #ifdef __ARCH_SYNC_CORE_DCACHE
374 call _end_lock_coherent
380 ENDPROC(___raw_spin_unlock_asm)
385 * Clobbers: r2:0, p1:0
387 ENTRY(___raw_read_lock_asm)
396 if cc jump .Lrdlock_failed
398 #ifdef __ARCH_SYNC_CORE_DCACHE
399 call _start_lock_coherent
417 if cc jump .Lrdlock_wait;
419 ENDPROC(___raw_read_lock_asm)
424 * Clobbers: r3:0, p1:0
426 ENTRY(___raw_read_trylock_asm)
432 if cc jump .Lfailed_tryrdlock;
436 #ifdef __ARCH_SYNC_CORE_DCACHE
437 call _start_lock_coherent
450 ENDPROC(___raw_read_trylock_asm)
455 * Note: Processing controlled by a reader lock should not have
456 * any side-effect on cache issues with the other core, so we
457 * just release the core lock and exit (no _end_lock_coherent).
459 * Clobbers: r3:0, p1:0
461 ENTRY(___raw_read_unlock_asm)
472 ENDPROC(___raw_read_unlock_asm)
477 * Clobbers: r3:0, p1:0
479 ENTRY(___raw_write_lock_asm)
481 r3.l = lo(RW_LOCK_BIAS);
482 r3.h = hi(RW_LOCK_BIAS);
488 #ifdef __ARCH_SYNC_CORE_DCACHE
496 if !cc jump .Lwrlock_wait
499 #ifdef __ARCH_SYNC_CORE_DCACHE
500 call _start_lock_coherent
514 #ifdef __ARCH_SYNC_CORE_DCACHE
519 if !cc jump .Lwrlock_wait;
521 ENDPROC(___raw_write_lock_asm)
526 * Clobbers: r3:0, p1:0
528 ENTRY(___raw_write_trylock_asm)
533 r2.l = lo(RW_LOCK_BIAS);
534 r2.h = hi(RW_LOCK_BIAS);
536 if !cc jump .Lfailed_trywrlock;
537 #ifdef __ARCH_SYNC_CORE_DCACHE
545 #ifdef __ARCH_SYNC_CORE_DCACHE
546 call _start_lock_coherent
560 ENDPROC(___raw_write_trylock_asm)
565 * Clobbers: r3:0, p1:0
567 ENTRY(___raw_write_unlock_asm)
569 r3.l = lo(RW_LOCK_BIAS);
570 r3.h = hi(RW_LOCK_BIAS);
577 #ifdef __ARCH_SYNC_CORE_DCACHE
578 call _end_lock_coherent
584 ENDPROC(___raw_write_unlock_asm)
590 * Add a signed value to a 32bit word and return the new value atomically.
591 * Clobbers: r3:0, p1:0
593 ENTRY(___raw_atomic_update_asm)
606 ENDPROC(___raw_atomic_update_asm)
612 * Clear the mask bits from a 32bit word and return the old 32bit value
614 * Clobbers: r3:0, p1:0
616 ENTRY(___raw_atomic_clear_asm)
630 ENDPROC(___raw_atomic_clear_asm)
636 * Set the mask bits into a 32bit word and return the old 32bit value
638 * Clobbers: r3:0, p1:0
640 ENTRY(___raw_atomic_set_asm)
654 ENDPROC(___raw_atomic_set_asm)
660 * XOR the mask bits with a 32bit word and return the old 32bit value
662 * Clobbers: r3:0, p1:0
664 ENTRY(___raw_atomic_xor_asm)
678 ENDPROC(___raw_atomic_xor_asm)
684 * Perform a logical AND between the mask bits and a 32bit word, and
685 * return the masked value. We need this on this architecture in
686 * order to invalidate the local cache before testing.
688 * Clobbers: r3:0, p1:0
690 ENTRY(___raw_atomic_test_asm)
693 r1 = -L1_CACHE_BYTES;
696 /* flush core internal write buffer before invalidate dcache */
703 ENDPROC(___raw_atomic_test_asm)
709 * Swap *ptr with value and return the old 32bit value atomically.
710 * Clobbers: r3:0, p1:0
712 #define __do_xchg(src, dst) \
716 call _get_core_lock; \
721 call _put_core_lock; \
726 ENTRY(___raw_xchg_1_asm)
727 __do_xchg(b[p1] (z), b[p1])
728 ENDPROC(___raw_xchg_1_asm)
730 ENTRY(___raw_xchg_2_asm)
731 __do_xchg(w[p1] (z), w[p1])
732 ENDPROC(___raw_xchg_2_asm)
734 ENTRY(___raw_xchg_4_asm)
735 __do_xchg([p1], [p1])
736 ENDPROC(___raw_xchg_4_asm)
743 * Swap *ptr with new if *ptr == old and return the previous *ptr
746 * Clobbers: r3:0, p1:0
748 #define __do_cmpxchg(src, dst) \
754 call _get_core_lock; \
761 call _put_core_lock; \
767 ENTRY(___raw_cmpxchg_1_asm)
768 __do_cmpxchg(b[p1] (z), b[p1])
769 ENDPROC(___raw_cmpxchg_1_asm)
771 ENTRY(___raw_cmpxchg_2_asm)
772 __do_cmpxchg(w[p1] (z), w[p1])
773 ENDPROC(___raw_cmpxchg_2_asm)
775 ENTRY(___raw_cmpxchg_4_asm)
776 __do_cmpxchg([p1], [p1])
777 ENDPROC(___raw_cmpxchg_4_asm)
783 * Set a bit in a 32bit word and return the old 32bit value atomically.
784 * Clobbers: r3:0, p1:0
786 ENTRY(___raw_bit_set_asm)
790 jump ___raw_atomic_set_asm
791 ENDPROC(___raw_bit_set_asm)
797 * Clear a bit in a 32bit word and return the old 32bit value atomically.
798 * Clobbers: r3:0, p1:0
800 ENTRY(___raw_bit_clear_asm)
804 jump ___raw_atomic_clear_asm
805 ENDPROC(___raw_bit_clear_asm)
811 * Toggle a bit in a 32bit word and return the old 32bit value atomically.
812 * Clobbers: r3:0, p1:0
814 ENTRY(___raw_bit_toggle_asm)
818 jump ___raw_atomic_xor_asm
819 ENDPROC(___raw_bit_toggle_asm)
825 * Test-and-set a bit in a 32bit word and return the old bit value atomically.
826 * Clobbers: r3:0, p1:0
828 ENTRY(___raw_bit_test_set_asm)
831 call ___raw_bit_set_asm
842 ENDPROC(___raw_bit_test_set_asm)
848 * Test-and-clear a bit in a 32bit word and return the old bit value atomically.
849 * Clobbers: r3:0, p1:0
851 ENTRY(___raw_bit_test_clear_asm)
854 call ___raw_bit_clear_asm
865 ENDPROC(___raw_bit_test_clear_asm)
871 * Test-and-toggle a bit in a 32bit word,
872 * and return the old bit value atomically.
873 * Clobbers: r3:0, p1:0
875 ENTRY(___raw_bit_test_toggle_asm)
878 call ___raw_bit_toggle_asm
889 ENDPROC(___raw_bit_test_toggle_asm)
895 * Test a bit in a 32bit word and return its value.
896 * We need this on this architecture in order to invalidate
897 * the local cache before testing.
899 * Clobbers: r3:0, p1:0
901 ENTRY(___raw_bit_test_asm)
905 jump ___raw_atomic_test_asm
906 ENDPROC(___raw_bit_test_asm)
911 * Fetch and return an uncached 32bit value.
913 * Clobbers: r2:0, p1:0
915 ENTRY(___raw_uncached_fetch_asm)
917 r1 = -L1_CACHE_BYTES;
920 /* flush core internal write buffer before invalidate dcache */
926 ENDPROC(___raw_uncached_fetch_asm)