kernel/sched: Use new barrier and spin APIs

The switch_handle field in the thread struct is used as an atomic flag
between CPUs in SMP, and has been known for a long time to technically
require memory barriers for correct operation.  We have an API for
that now, so put them in:

* The code immediately before arch_switch() needs a write barrier to
  ensure that thread state written by the scheduler is seen to happen
  before the outgoing thread is flagged with a valid switch handle.

* The loop in z_sched_switch_spin() needs a read barrier at the end,
  to make sure the calling context doesn't load state from before the
  other CPU stored the switch handle.

Also, that same spot in switch_spin was spinning with interrupts held,
which means it needs a call to arch_spin_relax() to avoid a FPU state
deadlock on some architectures.

Signed-off-by: Andy Ross <[email protected]>
(cherry picked from commit c3046f4)

Author: andyross

kernel/include/kswap.h

@@ -7,6 +7,7 @@
 #define ZEPHYR_KERNEL_INCLUDE_KSWAP_H_
 
 #include <ksched.h>
+#include <sys/atomic.h>
 #include <spinlock.h>
 #include <kernel_arch_func.h>
 
@@ -48,19 +49,25 @@ void z_smp_release_global_lock(struct k_thread *thread);
  * treat this because the scheduler lock can't be released by the
  * switched-to thread, which is going to (obviously) be running its
  * own code and doesn't know it was switched out.
- *
- * Note: future SMP architectures may need a fence/barrier or cache
- * invalidation here.  Current ones don't, and sadly Zephyr doesn't
- * have a framework for that yet.
  */
 static inline void z_sched_switch_spin(struct k_thread *thread)
 {
 #ifdef CONFIG_SMP
 	volatile void **shp = (void *)&thread->switch_handle;
 
 	while (*shp == NULL) {
-		k_busy_wait(1);
+		__asm__ __volatile__ (""::: "memory");
 	}
+	/* Read barrier: don't allow any subsequent loads in the
+	 * calling code to reorder before we saw switch_handle go
+	 * non-null.
+	 */
+#if defined(__GNUC__)
+			/* GCC-ism */
+			__atomic_thread_fence(__ATOMIC_SEQ_CST);
+#else
+			atomic_thread_fence(memory_order_seq_cst);
+#endif
 #endif
 }
 
@@ -152,8 +159,17 @@ static ALWAYS_INLINE unsigned int do_swap(unsigned int key,
 		void *newsh = new_thread->switch_handle;
 
 		if (IS_ENABLED(CONFIG_SMP)) {
-			/* Active threads MUST have a null here */
+			/* Active threads must have a null here.  And
+			 * it must be seen before the scheduler lock
+			 * is released!
+			 */
 			new_thread->switch_handle = NULL;
+#if defined(__GNUC__)
+			/* GCC-ism */
+			__atomic_thread_fence(__ATOMIC_SEQ_CST);
+#else
+			atomic_thread_fence(memory_order_seq_cst);
+#endif
 		}
 		k_spin_release(&sched_spinlock);
 		arch_switch(newsh, &old_thread->switch_handle);