[vm/ffi] Document shadow space and restore RSP

Split off from https://dart-review.googlesource.com/c/sdk/+/140290.
We start using the stack after the C call, so RSP needs to be restored.

Issue #36730.

TEST=All existing FFI tests on windows.

Change-Id: Idcaff9007c1b6f42b95288e39470b3d6cca2ef56
Cq-Include-Trybots: luci.dart.try:vm-kernel-win-debug-x64-try
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/170425
Reviewed-by: Martin Kustermann <[email protected]>
Commit-Queue: Daco Harkes <[email protected]>

Author: dcharkes

runtime/vm/compiler/assembler/assembler_x64.cc

@@ -1671,19 +1671,27 @@ void Assembler::LeaveCallRuntimeFrame() {
   LeaveStubFrame();
 }
 
-void Assembler::CallCFunction(Register reg) {
+void Assembler::CallCFunction(Register reg, bool restore_rsp) {
   // Reserve shadow space for outgoing arguments.
   if (CallingConventions::kShadowSpaceBytes != 0) {
     subq(RSP, Immediate(CallingConventions::kShadowSpaceBytes));
   }
   call(reg);
+  // Restore stack.
+  if (restore_rsp && CallingConventions::kShadowSpaceBytes != 0) {
+    addq(RSP, Immediate(CallingConventions::kShadowSpaceBytes));
+  }
 }
-void Assembler::CallCFunction(Address address) {
+void Assembler::CallCFunction(Address address, bool restore_rsp) {
   // Reserve shadow space for outgoing arguments.
   if (CallingConventions::kShadowSpaceBytes != 0) {
     subq(RSP, Immediate(CallingConventions::kShadowSpaceBytes));
   }
   call(address);
+  // Restore stack.
+  if (restore_rsp && CallingConventions::kShadowSpaceBytes != 0) {
+    addq(RSP, Immediate(CallingConventions::kShadowSpaceBytes));
+  }
 }
 
 void Assembler::CallRuntime(const RuntimeEntry& entry,

runtime/vm/compiler/assembler/assembler_x64.h

@@ -829,9 +829,9 @@ class Assembler : public AssemblerBase {
   void CallRuntime(const RuntimeEntry& entry, intptr_t argument_count);
 
   // Call runtime function. Reserves shadow space on the stack before calling
-  // if platform ABI requires that. Does not restore RSP after the call itself.
-  void CallCFunction(Register reg);
-  void CallCFunction(Address address);
+  // if platform ABI requires that.
+  void CallCFunction(Register reg, bool restore_rsp = false);
+  void CallCFunction(Address address, bool restore_rsp = false);
 
   void ExtractClassIdFromTags(Register result, Register tags);
   void ExtractInstanceSizeFromTags(Register result, Register tags);

runtime/vm/compiler/backend/il_x64.cc

@@ -1082,7 +1082,7 @@ void FfiCallInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
     __ TransitionGeneratedToNative(target_address, FPREG, TMP,
                                    /*enter_safepoint=*/true);
 
-    __ CallCFunction(target_address);
+    __ CallCFunction(target_address, /*restore_rsp=*/true);
 
     // Update information in the thread object and leave the safepoint.
     __ TransitionNativeToGenerated(/*leave_safepoint=*/true);

runtime/vm/compiler/stub_code_compiler_x64.cc

@@ -282,7 +282,7 @@ void StubCodeCompiler::GenerateCallNativeThroughSafepointStub(
                                  /*enter_safepoint=*/true);
 
   __ popq(R12);
-  __ CallCFunction(RBX);
+  __ CallCFunction(RBX, /*restore_rsp=*/true);
 
   __ TransitionNativeToGenerated(/*leave_safepoint=*/true);
 

runtime/vm/constants_x64.h

@@ -304,6 +304,16 @@ class CallingConventions {
   // same time? (Windows no, rest yes)
   static const bool kArgumentIntRegXorFpuReg = true;
 
+  // > The x64 Application Binary Interface (ABI) uses a four-register
+  // > fast-call calling convention by default. Space is allocated on the call
+  // > stack as a shadow store for callees to save those registers.
+  // https://docs.microsoft.com/en-us/cpp/build/x64-calling-convention?view=msvc-160
+  //
+  // The caller allocates this space. The caller should also reclaim this space
+  // after the call to restore the stack to its original state if needed.
+  //
+  // This is also known as home space.
+  // https://devblogs.microsoft.com/oldnewthing/20160623-00/?p=93735
   static const intptr_t kShadowSpaceBytes = 4 * kWordSize;
 
   static const intptr_t kVolatileCpuRegisters =