Merge pull request #533 from java-native-access/531-callback-cc-from-options

[531] Fix stdcall callback calling convention

Author: twall

CHANGES.md

@@ -25,6 +25,7 @@ Bug Fixes
 ---------
 * [#549](https://github.com/java-native-access/jna/pull/549): Fixed bug in types derived from XID - [@twall](https://github.com/twall).
 * [#536](https://github.com/java-native-access/jna/pull/536): Fixed bug in determining the Library and options associated with types defined outside of a Library - [@twall](https://github.com/twall).
+* [#531](https://github.com/java-native-access/jna/pull/531): Ensure direct-mapped callbacks use the right calling convention - [@twall](https://github.com/twall).
 
 Release 4.2.1
 =============

src/com/sun/jna/CallbackReference.java

@@ -39,7 +39,9 @@ class CallbackReference extends WeakReference {
     static final Map callbackMap = new WeakHashMap();
     static final Map directCallbackMap = new WeakHashMap();
     static final Map pointerCallbackMap = new WeakHashMap();
+    // Track memory allocations associated with this closure (usually String args)
     static final Map allocations = new WeakHashMap();
+    // Global map of allocated closures to facilitate centralized cleanup
     private static final Map allocatedMemory = Collections.synchronizedMap(new WeakHashMap());
 
     private static final Method PROXY_CALLBACK_METHOD;
@@ -146,9 +148,11 @@ private static Callback getCallback(Class type, Pointer p, boolean direct) {
     // Keep a reference to the proxy to avoid premature GC of it
     CallbackProxy proxy;
     Method method;
+    int callingConvention;
     private CallbackReference(Callback callback, int callingConvention, boolean direct) {
         super(callback);
         TypeMapper mapper = Native.getTypeMapper(callback.getClass());
+        this.callingConvention = callingConvention;
         Class[] nativeParamTypes;
         Class returnType;
 
@@ -180,6 +184,7 @@ private CallbackReference(Callback callback, int callingConvention, boolean dire
         }
 
         String encoding = Native.getStringEncoding(callback.getClass());
+        long peer = 0;
         if (direct) {
             method = getCallbackMethod(callback);
             nativeParamTypes = method.getParameterTypes();
@@ -188,12 +193,10 @@ private CallbackReference(Callback callback, int callingConvention, boolean dire
             if (callback instanceof DLLCallback) {
                 flags |= Native.CB_OPTION_IN_DLL;
             }
-            long peer = Native.createNativeCallback(callback, method,
-                                                    nativeParamTypes, returnType,
-                                                    callingConvention, flags,
-                                                    encoding);
-            cbstruct = peer != 0 ? new Pointer(peer) : null;
-            allocatedMemory.put(this, new WeakReference(this));
+            peer = Native.createNativeCallback(callback, method,
+                                               nativeParamTypes, returnType,
+                                               callingConvention, flags,
+                                               encoding);
         }
         else {
             if (callback instanceof CallbackProxy) {
@@ -236,12 +239,13 @@ private CallbackReference(Callback callback, int callingConvention, boolean dire
             }
             int flags = callback instanceof DLLCallback
                 ? Native.CB_OPTION_IN_DLL : 0;
-            long peer = Native.createNativeCallback(proxy, PROXY_CALLBACK_METHOD,  
-                                                    nativeParamTypes, returnType,
-                                                    callingConvention, flags,
-                                                    encoding);
-            cbstruct = peer != 0 ? new Pointer(peer) : null;
+            peer = Native.createNativeCallback(proxy, PROXY_CALLBACK_METHOD,  
+                                               nativeParamTypes, returnType,
+                                               callingConvention, flags,
+                                               encoding);
         }
+        cbstruct = peer != 0 ? new Pointer(peer) : null;
+        allocatedMemory.put(this, new WeakReference(this));
     }
 
     private Class getNativeType(Class cls) {
@@ -404,8 +408,13 @@ private static Pointer getFunctionPointer(Callback cb, boolean direct) {
         if ((fp = getNativeFunctionPointer(cb)) != null) {
             return fp;
         }
+        Map options = Native.getLibraryOptions(cb.getClass());
         int callingConvention = cb instanceof AltCallingConvention
-            ? Function.ALT_CONVENTION : Function.C_CONVENTION;
+            ? Function.ALT_CONVENTION
+            : (options != null && options.containsKey(Library.OPTION_CALLING_CONVENTION)
+               ? ((Integer)options.get(Library.OPTION_CALLING_CONVENTION)).intValue()
+               : Function.C_CONVENTION);
+
         Map map = direct ? directCallbackMap : callbackMap;
         synchronized(callbackMap) {
             CallbackReference cbref = (CallbackReference)map.get(cb);

test/com/sun/jna/CallbacksTest.java

@@ -1439,6 +1439,54 @@ public void callback() {
         }
     }
 
+    public interface TaggedCallingConventionTestLibrary extends Library, AltCallingConvention {
+        interface TestCallbackTagged extends Callback, AltCallingConvention {
+            void invoke();
+        }
+    }
+
+    public void testCallingConventionFromInterface() {
+        TaggedCallingConventionTestLibrary lib = (TaggedCallingConventionTestLibrary)
+            Native.loadLibrary("testlib", TaggedCallingConventionTestLibrary.class);
+        TaggedCallingConventionTestLibrary.TestCallbackTagged cb = new TaggedCallingConventionTestLibrary.TestCallbackTagged() {
+            public void invoke() { }
+        };
+        try {
+            Pointer p = CallbackReference.getFunctionPointer(cb);
+            CallbackReference ref = (CallbackReference)CallbackReference.callbackMap.get(cb);
+            assertNotNull("CallbackReference not found", ref);
+            assertEquals("Tag-based calling convention not applied", Function.ALT_CONVENTION, ref.callingConvention);
+        }
+        catch (IllegalArgumentException e) {
+            // Alt convention not supported
+        }
+    }
+
+    public interface OptionCallingConventionTestLibrary extends Library {
+        interface TestCallback extends Callback {
+            void invoke();
+        }
+    }
+
+    public void testCallingConventionFromOptions() {
+        Map options = new HashMap();
+        options.put(Library.OPTION_CALLING_CONVENTION, Function.ALT_CONVENTION);
+        OptionCallingConventionTestLibrary lib = (OptionCallingConventionTestLibrary)
+            Native.loadLibrary("testlib", OptionCallingConventionTestLibrary.class, options);
+        OptionCallingConventionTestLibrary.TestCallback cb = new OptionCallingConventionTestLibrary.TestCallback() {
+            public void invoke() { }
+        };
+        try {
+            Pointer p = CallbackReference.getFunctionPointer(cb);
+            CallbackReference ref = (CallbackReference)CallbackReference.callbackMap.get(cb);
+            assertNotNull("CallbackReference not found", ref);
+            assertEquals("Option-based calling convention not applied", Function.ALT_CONVENTION, ref.callingConvention);
+        }
+        catch(IllegalArgumentException e) {
+            // Alt convention not supported
+        }
+    }
+
     public static void main(java.lang.String[] argList) {
         junit.textui.TestRunner.run(CallbacksTest.class);
     }