Add support for emergency spill slots.

- Create a new kind of stack slot: emergency_slot.
- Add a get_emergency_slot() method which finds a suitable emergency
  slot given a list of slots already in use.
- Use emergency spill slots when schedule_moves needs them.

Author: Jakob Stoklund Olesen

filetests/parser/tiny.cton

@@ -117,6 +117,7 @@ function %stack() {
     ss2 = local 4
     ss3 = incoming_arg 4, offset 8
     ss4 = outgoing_arg 4
+    ss5 = emergency_slot 4
 
 ebb0:
     v1 = stack_load.i32 ss10
@@ -129,6 +130,7 @@ ebb0:
 ; nextln:    $ss2 = local 4
 ; nextln:    $ss3 = incoming_arg 4, offset 8
 ; nextln:    $ss4 = outgoing_arg 4
+; nextln:    $ss5 = emergency_slot 4
 
 ; check: ebb0:
 ; nextln: $v1 = stack_load.i32 $ss10

filetests/regalloc/schedule-moves.cton

@@ -1,12 +1,11 @@
 test compile
-set is_64bit=1
 isa intel haswell
 
 function %pr165() native {
 ebb0:
-    v0 = iconst.i64 0x0102_0304_f1f2_f3f4
-    v1 = iconst.i64 0x1102_0304_f1f2_f3f4
-    v2 = iconst.i64 0x2102_0304_f1f2_f3f4
+    v0 = iconst.i32 0x0102_0304
+    v1 = iconst.i32 0x1102_0304
+    v2 = iconst.i32 0x2102_0304
     v20 = ishl v1, v0
     v21 = ishl v2, v0
     v22 = sshr v1, v0
@@ -17,3 +16,24 @@ ebb0:
     istore8 v1, v0+0x2710
     return
 }
+
+; Same as above, but use so many registers that spilling is required.
+; Note: This is also a candidate for using xchg instructions.
+function %emergency_spill() native {
+ebb0:
+    v0 = iconst.i32 0x0102_0304
+    v1 = iconst.i32 0x1102_0304
+    v2 = iconst.i32 0x2102_0304
+    v3 = iconst.i32 0x3102_0304
+    v4 = iconst.i32 0x4102_0304
+    v20 = ishl v1, v0
+    v21 = ishl v2, v3
+    v22 = sshr v1, v0
+    v23 = sshr v2, v0
+    v24 = ushr v1, v0
+    v25 = ushr v2, v0
+    istore8 v0, v1+0x2710
+    istore8 v1, v0+0x2710
+    istore8 v3, v4+0x2710
+    return
+}

lib/cretonne/src/ir/stackslot.rs

@@ -5,6 +5,7 @@
 
 use entity::{PrimaryMap, Keys};
 use ir::{Type, StackSlot};
+use packed_option::PackedOption;
 use std::fmt;
 use std::ops::Index;
 use std::str::FromStr;
@@ -44,6 +45,12 @@ pub enum StackSlotKind {
     /// stack slots are used to represent individual arguments in the outgoing call frame. These
     /// stack slots are only valid while setting up a call.
     OutgoingArg,
+
+    /// An emergency spill slot.
+    ///
+    /// Emergency slots are allocated late when the register's constraint solver needs extra space
+    /// to shuffle registers around. The are only used briefly, and can be reused.
+    EmergencySlot,
 }
 
 impl FromStr for StackSlotKind {
@@ -56,6 +63,7 @@ impl FromStr for StackSlotKind {
             "spill_slot" => Ok(SpillSlot),
             "incoming_arg" => Ok(IncomingArg),
             "outgoing_arg" => Ok(OutgoingArg),
+            "emergency_slot" => Ok(EmergencySlot),
             _ => Err(()),
         }
     }
@@ -69,6 +77,7 @@ impl fmt::Display for StackSlotKind {
             SpillSlot => "spill_slot",
             IncomingArg => "incoming_arg",
             OutgoingArg => "outgoing_arg",
+            EmergencySlot => "emergency_slot",
         })
     }
 }
@@ -135,6 +144,9 @@ pub struct StackSlots {
     /// All the outgoing stack slots, ordered by offset.
     outgoing: Vec<StackSlot>,
 
+    /// All the emergency slots.
+    emergency: Vec<StackSlot>,
+
     /// The total size of the stack frame.
     ///
     /// This is the distance from the stack pointer in the current function to the stack pointer in
@@ -152,6 +164,7 @@ impl StackSlots {
         StackSlots {
             slots: PrimaryMap::new(),
             outgoing: Vec::new(),
+            emergency: Vec::new(),
             frame_size: None,
         }
     }
@@ -160,6 +173,7 @@ impl StackSlots {
     pub fn clear(&mut self) {
         self.slots.clear();
         self.outgoing.clear();
+        self.emergency.clear();
         self.frame_size = None;
     }
 
@@ -243,6 +257,43 @@ impl StackSlots {
         self.outgoing.insert(inspos, ss);
         ss
     }
+
+    /// Get an emergency spill slot that can be used to store a `ty` value.
+    ///
+    /// This may allocate a new slot, or it may reuse an existing emergency spill slot, excluding
+    /// any slots in the `in_use` list.
+    pub fn get_emergency_slot(
+        &mut self,
+        ty: Type,
+        in_use: &[PackedOption<StackSlot>],
+    ) -> StackSlot {
+        let size = ty.bytes();
+
+        // Find the smallest existing slot that can fit the type.
+        if let Some(&ss) = self.emergency
+            .iter()
+            .filter(|&&ss| self[ss].size >= size && !in_use.contains(&ss.into()))
+            .min_by_key(|&&ss| self[ss].size)
+        {
+            return ss;
+        }
+
+        // Alternatively, use the largest available slot and make it larger.
+        if let Some(&ss) = self.emergency
+            .iter()
+            .filter(|&&ss| !in_use.contains(&ss.into()))
+            .max_by_key(|&&ss| self[ss].size)
+        {
+            self.slots[ss].size = size;
+            return ss;
+        }
+
+        // No existing slot found. Make one and insert it into `emergency`.
+        let data = StackSlotData::new(StackSlotKind::EmergencySlot, size);
+        let ss = self.slots.push(data);
+        self.emergency.push(ss);
+        ss
+    }
 }
 
 #[cfg(test)]
@@ -304,4 +355,31 @@ mod tests {
         assert_eq!(slot2.alignment(16), 8);
         assert_eq!(slot2.alignment(32), 8);
     }
+
+    #[test]
+    fn emergency() {
+        let mut sss = StackSlots::new();
+
+        let ss0 = sss.get_emergency_slot(types::I32, &[]);
+        assert_eq!(sss[ss0].size, 4);
+
+        // When a smaller size is requested, we should simply get the same slot back.
+        assert_eq!(sss.get_emergency_slot(types::I8, &[]), ss0);
+        assert_eq!(sss[ss0].size, 4);
+        assert_eq!(sss.get_emergency_slot(types::F32, &[]), ss0);
+        assert_eq!(sss[ss0].size, 4);
+
+        // Ask for a larger size and the slot should grow.
+        assert_eq!(sss.get_emergency_slot(types::F64, &[]), ss0);
+        assert_eq!(sss[ss0].size, 8);
+
+        // When one slot is in use, we should get a new one.
+        let ss1 = sss.get_emergency_slot(types::I32, &[None.into(), ss0.into()]);
+        assert_eq!(sss[ss0].size, 8);
+        assert_eq!(sss[ss1].size, 4);
+
+        // Now we should get the smallest fit of the two available slots.
+        assert_eq!(sss.get_emergency_slot(types::F32, &[]), ss1);
+        assert_eq!(sss.get_emergency_slot(types::F64, &[]), ss0);
+    }
 }

lib/cretonne/src/regalloc/coloring.rs

@@ -48,6 +48,7 @@ use ir::{Ebb, Inst, Value, Function, ValueLoc, SigRef};
 use ir::{InstBuilder, ArgumentType, ArgumentLoc};
 use isa::{RegUnit, RegClass, RegInfo, regs_overlap};
 use isa::{TargetIsa, EncInfo, RecipeConstraints, OperandConstraint, ConstraintKind};
+use packed_option::PackedOption;
 use regalloc::RegDiversions;
 use regalloc::affinity::Affinity;
 use regalloc::allocatable_set::AllocatableSet;
@@ -733,14 +734,47 @@ impl<'a> Context<'a> {
     fn shuffle_inputs(&mut self, regs: &mut AllocatableSet) {
         use regalloc::solver::Move::*;
 
-        self.solver.schedule_moves(regs);
+        let spills = self.solver.schedule_moves(regs);
+
+        // The move operations returned by `schedule_moves` refer to emergency spill slots by
+        // consecutive indexes starting from 0. Map these to real stack slots.
+        // It is very unlikely (impossible?) that we would need more than one spill per top-level
+        // register class, so avoid allocation by using a fixed array here.
+        let mut slot = [PackedOption::default(); 8];
+        assert!(spills <= slot.len(), "Too many spills ({})", spills);
+
         for m in self.solver.moves() {
             match *m {
                 Reg { value, from, to, .. } => {
                     self.divert.regmove(value, from, to);
                     self.cur.ins().regmove(value, from, to);
                 }
-                Spill { .. } | Fill { .. } => unimplemented!(),
+                Spill {
+                    value,
+                    from,
+                    to_slot,
+                    ..
+                } => {
+                    debug_assert_eq!(slot[to_slot].expand(), None, "Overwriting slot in use");
+                    let ss = self.cur.func.stack_slots.get_emergency_slot(
+                        self.cur.func.dfg.value_type(value),
+                        &slot[0..spills],
+                    );
+                    slot[to_slot] = ss.into();
+                    self.divert.regspill(value, from, ss);
+                    self.cur.ins().regspill(value, from, ss);
+                }
+                Fill {
+                    value,
+                    from_slot,
+                    to,
+                    ..
+                } => {
+                    // These slots are single use, so mark `ss` as available again.
+                    let ss = slot[from_slot].take().expect("Using unallocated slot");
+                    self.divert.regfill(value, ss, to);
+                    self.cur.ins().regfill(value, ss, to);
+                }
             }
         }
     }

lib/cretonne/src/regalloc/solver.rs

@@ -523,7 +523,13 @@ impl Solver {
     /// In either case, `to` will not be available for variables on the input side of the
     /// instruction.
     pub fn reassign_in(&mut self, value: Value, rc: RegClass, from: RegUnit, to: RegUnit) {
-        dbg!("reassign_in({}:{}, %{} -> %{})", value, rc, from, to);
+        dbg!(
+            "reassign_in({}:{}, {} -> {})",
+            value,
+            rc,
+            rc.info.display_regunit(from),
+            rc.info.display_regunit(to)
+        );
         debug_assert!(!self.inputs_done);
         if self.regs_in.is_avail(rc, from) {
             // It looks like `value` was already removed from the register set. It must have been
@@ -826,7 +832,9 @@ impl Solver {
             Move::with_assignment,
         ));
 
-        dbg!("collect_moves: {}", DisplayList(&self.moves));
+        if !(self.moves.is_empty()) {
+            dbg!("collect_moves: {}", DisplayList(&self.moves));
+        }
     }
 
     /// Try to schedule a sequence of `regmove` instructions that will shuffle registers into

lib/cretonne/src/stack_layout.rs

@@ -56,7 +56,9 @@ pub fn layout_stack(frame: &mut StackSlots, alignment: StackSize) -> Result<Stac
                 )?;
                 outgoing_max = max(outgoing_max, offset);
             }
-            StackSlotKind::SpillSlot | StackSlotKind::Local => {
+            StackSlotKind::SpillSlot |
+            StackSlotKind::Local |
+            StackSlotKind::EmergencySlot => {
                 // Determine the smallest alignment of any local or spill slot.
                 min_align = slot.alignment(min_align);
             }

misc/vim/syntax/cton.vim

@@ -14,7 +14,7 @@ endif
 syn spell notoplevel
 
 syn keyword ctonHeader test isa set
-syn keyword ctonDecl function jump_table incoming_arg outgoing_arg spill_slot local
+syn keyword ctonDecl function jump_table incoming_arg outgoing_arg spill_slot local emergency_slot
 syn keyword ctonFilecheck check sameln nextln unordered not regex contained
 
 syn match ctonType  /\<[bif]\d\+\(x\d\+\)\?\>/