Adreno, PowerVR support for Android

Adreno support
PowerVR support
i915 - extended supported event set

perf stacks collection on Linux
dtrace stacks collection on OSX
tasks from samples reconstruction

python auto instrumentation extended with memory collection
sea.py made python 2/3 compatible
bin folder location is normalized between installed and developer versions
installer cmake script part made cleared
changed approach to Win symbols unwinding (made more efficient)
added note about cmake to readme
tests updated to collect stacks
collection with stacks skips aliens by default (non-system-wide)
added runtool execution tracing (for dbugging purposes)
internal API: overlapped tasks support
secure compilation flags
verbose compilation mode for buildall.py
added include and lib folders to installer
bugfixes

Author: araud

CMakeLists.txt

@@ -19,6 +19,7 @@ cmake_minimum_required(VERSION 2.8)
 
 project(IntelSEAPI)
 
+set(CMAKE_MACOSX_RPATH 1)
 OPTION(JDK "Enable Java build")
 OPTION(FORCE_32 "Force a 32bit compile on 64bit" OFF)
 
@@ -63,7 +64,7 @@ foreach (Flag ${Flags})
     message(STATUS "${Flag} = ${${Flag}}")
 endforeach()
 
-set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/../bin)
+set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/../../bin)
 
 set(EXECUTABLE_OUTPUT_PATH ${CMAKE_RUNTIME_OUTPUT_DIRECTORY})
 set(LIBRARY_OUTPUT_PATH ${CMAKE_RUNTIME_OUTPUT_DIRECTORY})
@@ -99,36 +100,157 @@ else()
     endif()
 endif()
 
-add_subdirectory(sea_itt_lib)
-add_subdirectory(ittnotify)
-
-TARGET_LINK_LIBRARIES(IntelSEAPI ittnotify)
 
-target_include_directories(IntelSEAPI
-    PUBLIC ittnotify/include
-)
 
 if (WIN32)
     TARGET_LINK_LIBRARIES(IntelSEAPI)
-    if(NOT ARCH_64 AND CMAKE_BUILD_TYPE STREQUAL "Debug") #Compiler Automated Instrumentation
-        set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /Gh /GH")
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /Zi /MP")
+    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /MP")
+
+    #SDL:
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /GS /sdl")
+    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /GS /sdl")
+    set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} /NXCompat /DynamicBase")
+    set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} /NXCompat /DynamicBase")
+    if(NOT ARCH_64)
+        set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} /SafeSEH")
+        set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} /SafeSEH")
     endif()
-    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /Zi") #in release as well
+
     set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} /DEBUG") #in release as well
     set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} /DEBUG") #in release as well
 elseif (APPLE)
     TARGET_LINK_LIBRARIES(IntelSEAPI dl)
-    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -stdlib=libc++ -std=c++11 -fPIC -Wno-unused-parameter")
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -stdlib=libc++ -std=c++11 -fPIC -Wno-unused-parameter -fstack-protector-strong -fPIE -O2 -D_FORTIFY_SOURCE=2 -Wformat -Wformat-security")
+    set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fPIE -Wl")
+    set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fPIE -Wl")
     set(CMAKE_OSX_ARCHITECTURES "i386;x86_64")
 elseif (ANDROID_NDK)
     TARGET_LINK_LIBRARIES(IntelSEAPI dl)
-    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -fPIE -pthread")
-    set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fPIE -pie -Wl,--no-undefined")
-    set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fPIE -pie")
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -fPIE -pthread -fstack-protector-strong -fPIE -O2 -D_FORTIFY_SOURCE=2 -Wformat -Wformat-security")
+    set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fPIE -pie -Wl,--no-undefined -z noexecstack -z relro -z now")
+    set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fPIE -pie -Wl,--no-undefined -z noexecstack -z relro -z now")
+    if (ARM AND ARCH_64)
+        add_definitions(-DITT_ARCH=ITT_ARCH_PPC64)
+    endif()
 else()
     TARGET_LINK_LIBRARIES(IntelSEAPI dl)
-    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -fPIC -pthread")
-    set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -pie -Wl,--no-undefined")
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -fPIC -pthread -fstack-protector-strong -fPIE -O2 -D_FORTIFY_SOURCE=2 -Wformat -Wformat-security")
+    set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -pie -Wl,--no-undefined -z noexecstack -z relro -z now")
+    set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fPIE -pie -Wl,--no-undefined -z noexecstack -z relro -z now")
+endif()
+
+add_subdirectory(ittnotify)
+add_subdirectory(sea_itt_lib)
+if (WIN32)
+    if(NOT ARCH_64 AND CMAKE_BUILD_TYPE STREQUAL "Debug") #Compiler Automated Instrumentation
+        set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /Gh /GH")
+    endif()
+else()
     #SET_TARGET_PROPERTIES(IntelSEAPI PROPERTIES COMPILE_FLAGS "-finstrument-functions")
 endif()
 
+TARGET_LINK_LIBRARIES(IntelSEAPI ittnotify)
+
+target_include_directories(IntelSEAPI
+    PUBLIC ittnotify/include
+)
+
+################################################### INSTALLER #########################################################
+
+set(CPACK_PACKAGE_NAME "IntelSEAPI")
+set(CPACK_PACKAGE_VENDOR "Intel")
+set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Intel(R) Single Event API")
+set(CPACK_PACKAGE_VERSION "1.0.0")
+set(CPACK_PACKAGE_INSTALL_DIRECTORY "IntelSEAPI")
+set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Intel(R) Single Event API")
+set(CPACK_PACKAGE_DESCRIPTION_FILE "${PROJECT_SOURCE_DIR}/README.txt")
+set(CPACK_RESOURCE_FILE_LICENSE "${PROJECT_SOURCE_DIR}/sea_itt_lib/Copyright.txt")
+set(CPACK_RESOURCE_FILE_README "${PROJECT_SOURCE_DIR}/README.txt")
+set(CPACK_RESOURCE_FILE_WELCOME "${PROJECT_SOURCE_DIR}/README.txt")
+
+install(
+    FILES
+        ${PROJECT_SOURCE_DIR}/README.txt
+    DESTINATION .
+)
+
+install(
+    DIRECTORY "${PROJECT_SOURCE_DIR}/ittnotify/include"
+    DESTINATION .
+)
+
+install(
+    FILES
+        "${PROJECT_SOURCE_DIR}/itt_notify.hpp"
+    DESTINATION include
+)
+
+set(SEA_PROJECT_BIN ${PROJECT_SOURCE_DIR}/bin)
+
+function(InstallWildCard what where)
+    install(
+        CODE "
+            file(GLOB SEA_INSTALL_FILES ${what})
+            file(INSTALL DESTINATION \"\${CMAKE_INSTALL_PREFIX}/${where}\" TYPE FILE FILES \${SEA_INSTALL_FILES})
+            string(REGEX REPLACE \";\" \"\\n\\t\\t\" SEA_INSTALL_FILES_MSG \"\${SEA_INSTALL_FILES}\")
+            message(\"\\n###\tInstall ${what} to ${where}:\\n\\t\\t\${SEA_INSTALL_FILES_MSG}\")
+        "
+    )
+endfunction()
+
+InstallWildCard("${PROJECT_SOURCE_DIR}/runtool/*.py" "runtool")
+InstallWildCard("${PROJECT_SOURCE_DIR}/runtool/importers/*.py" "runtool/importers")
+InstallWildCard("${PROJECT_SOURCE_DIR}/runtool/exporters/*.py" "runtool/exporters")
+InstallWildCard("${PROJECT_SOURCE_DIR}/runtool/collectors/*.py" "runtool/collectors")
+InstallWildCard("${PROJECT_SOURCE_DIR}/runtool/decoders/*.py" "runtool/decoders")
+
+if (WIN32)
+    InstallWildCard("${SEA_PROJECT_BIN}/*ittnotify*.lib" "lib")
+    InstallWildCard("${SEA_PROJECT_BIN}/*.dll" "bin")
+    InstallWildCard("${SEA_PROJECT_BIN}/*.exe" "bin")
+    InstallWildCard("${SEA_PROJECT_BIN}/*.pdb" "bin")
+
+    install(
+        FILES
+            sea_itt_lib/IntelSEAPI.man
+            sea_itt_lib/IntelSEAPI.wprp
+            sea_itt_lib/IntelSEAPI_roi.xml
+            sea_itt_lib/IntelSEAPI.wpaProfile
+            sea_itt_lib/register.bat
+        DESTINATION ETW
+    )
+
+    set(CPACK_GENERATOR NSIS)
+
+    #run at exit:
+    set(CPACK_NSIS_EXECUTABLES_DIRECTORY "ETW")
+    set(CPACK_NSIS_MUI_FINISHPAGE_RUN "register.bat")
+    set(CPACK_NSIS_DISPLAY_NAME "Intel(R) Single Event API")
+
+    set(CPACK_NSIS_CONTACT "[email protected]")
+
+    SET(CPACK_NSIS_INSTALL_ROOT "c:\\\\Intel")
+
+else()
+
+    set(CPACK_BUNDLE_NAME "IntelSEAPI")
+    set(CPACK_GENERATOR STGZ)
+    install(
+        DIRECTORY "${SEA_PROJECT_BIN}"
+        DESTINATION .
+        PATTERN "*.a" EXCLUDE
+    )
+
+    InstallWildCard("${SEA_PROJECT_BIN}/*ittnotify*.a" "lib")
+
+    if(APPLE)
+        install(
+            FILES
+                "${PROJECT_SOURCE_DIR}/sea_itt_lib/IntelSEAPI.instrument"
+            DESTINATION dtrace
+        )
+    endif()
+endif()
+
+include(CPack)

README.txt

@@ -22,10 +22,11 @@ After your code is instrumented with itt, to load up the library follow these st
 OR you can use sea_runtool.py, see examples in test_<OS>.<bat/sh>
 
 Bulding:
+    Make sure you have cmake (https://cmake.org) in PATH
     All platforms except Android:
         >> python buildall.py -i
         this will produce installer
-        on Windows requires Visual Studio 2013 and NSIS (http://nsis.sourceforge.net/) installed
+        on Windows requires Visual Studio 2013 and NSIS (http://nsis.sourceforge.net) installed
         for Yocto just run this script in the Yocto build environment
     Android:
         requires ANDROID_NDK to be set in environment to the Android NDK path
@@ -54,7 +55,7 @@ Open Source Intel® SEAPI currently supports these formats:
     To enable: use systrace from AndroidStudio/Eclipse.
 
 * Json google trace format - All platforms: https://www.chromium.org/developers/how-tos/trace-event-profiling-tool
-    Cons: no correlation with OSX system events (yet)
+    Cons: no concerns, excellent viewer
     Pros: any platform; Corellation with ftrace (Android, Yocto, Linux), ETW (Windows). Supported: object state tracing, counters, tasks (sync and async) - immediate and with clock domains...
     To enable set environment variable INTEL_SEA_SAVE_TO=<any path>/<trace name>
     Use runtool to transform the SEA directory into json format with next command:

buildall.py

@@ -33,7 +33,7 @@ def get_share_folder():
 
 
 def run_shell(cmd):
-    print "\n>>", cmd
+    print("\n>>", cmd)
     os.system(cmd)
 
 
@@ -47,14 +47,14 @@ def replace_in_file(file, what_by):
 
 
 def get_yocto():
-    if not os.environ.has_key('CXX'):
+    if 'CXX' not in os.environ:
         return None
     cxx = os.environ['CXX']
     if '-poky' not in cxx:
         return None
     if '-m32' in cxx:
-        return {'bits':'32'}
-    return {'bits':'64'}
+        return {'bits': '32'}
+    return {'bits': '64'}
 
 
 def GetJDKPath():
@@ -64,7 +64,7 @@ def GetJDKPath():
         try:
             aKey = _winreg.OpenKey(_winreg.HKEY_LOCAL_MACHINE, path, 0, _winreg.KEY_READ | _winreg.KEY_WOW64_64KEY)
         except WindowsError:
-            print "No key:", path
+            print("No key:", path)
             return None
         subkeys = []
         try:
@@ -75,22 +75,22 @@ def GetJDKPath():
         except WindowsError:
             pass
         if not subkeys:
-            print "No subkeys for:", path
+            print("No subkeys for:", path)
             return None
         subkeys.sort()
         path += "\\" + subkeys[-1]
         try:
             aKey = _winreg.OpenKey(_winreg.HKEY_LOCAL_MACHINE, path, 0, _winreg.KEY_READ | _winreg.KEY_WOW64_64KEY)
             return _winreg.QueryValueEx(aKey, "JavaHome")[0]
         except WindowsError:
-            print "No value for:", path
+            print("No value for:", path)
             return None
     else:
         path, err = subprocess.Popen("which javah", shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
         if err or not path:
             return None
         if sys.platform == 'darwin':
-            path = subprocess.check_output("/usr/libexec/java_home").split('\n')[0]
+            path = subprocess.check_output("/usr/libexec/java_home").decode("utf-8").split('\n')[0]
             return path if os.path.exists(path) else None
         else:
             matches = []
@@ -161,7 +161,7 @@ def get_vs_versions():
         if int(version) >= 12:
             versions.append(version)
     if not versions:
-        print "No Visual Studio version found"
+        print("No Visual Studio version found")
     return sorted(versions)
 
 
@@ -175,6 +175,7 @@ def main():
     parser.add_argument("-a", "--android", action="store_true")
     parser.add_argument("--arm", action="store_true")
     parser.add_argument("-c", "--clean", action="store_true")
+    parser.add_argument("-v", "--verbose", action="store_true")
     if sys.platform == 'win32' and vs_versions:
         parser.add_argument("--vs", choices=vs_versions, default=vs_versions[0])
     args = parser.parse_args()
@@ -185,27 +186,31 @@ def main():
     else:
         if not yocto:
             target_bits = ['64']
-            if (sys.platform != 'darwin') or args.android: #on MAC OSX we produce FAT library including both 32 and 64 bits
+            if (sys.platform != 'darwin') or args.android:  # on MAC OSX we produce FAT library including both 32 and 64 bits
                 target_bits.append('32')
         else:
             target_bits = [yocto['bits']]
 
-    print "target_bits", target_bits
+    print("target_bits", target_bits)
 
     jdk_path = GetJDKPath()
-    print "Found JDK:", jdk_path
+    print("Found JDK:", jdk_path)
 
     work_dir = os.getcwd()
-    print work_dir
-    for bits in target_bits: #create separate build dirs
+    print(work_dir)
+    if args.clean:
+        bin_dir = os.path.join(work_dir, 'bin')
+        if os.path.exists(bin_dir):
+            shutil.rmtree(bin_dir)
+    for bits in target_bits:  # create separate build dirs
         work_folder = os.path.join(work_dir, "build_" + ("android" if args.android else "yocto" if yocto else sys.platform.replace('32', "")), bits)
         already_there = os.path.exists(work_folder)
         if already_there and args.clean:
             shutil.rmtree(work_folder)
             already_there = False
         if not already_there:
             os.makedirs(work_folder)
-        print work_folder
+        print("work_folder: ", work_folder)
         os.chdir(work_folder)
 
         if args.android:
@@ -221,11 +226,12 @@ def main():
                     ("-DANDROID_NDK=%s" % (os.environ['ANDROID_NDK'])),
                     ("-DCMAKE_BUILD_TYPE=%s" % ("Debug" if args.debug else "Release")),
                     ('-DANDROID_ABI="%s"' % abi),
-                    (('-DJDK="%s"' % jdk_path) if jdk_path else "")
+                    (('-DJDK="%s"' % jdk_path) if jdk_path else ""),
+                    ('-DCMAKE_VERBOSE_MAKEFILE:BOOL=ON' if args.verbose else '')
                 ])))
                 run_shell('cmake --build .')
             else:
-                print "Set ANDROID_NDK environment to build Android!"
+                print("Set ANDROID_NDK environment to build Android!")
             continue
         if sys.platform == 'win32':
             if vs_versions:
@@ -238,22 +244,24 @@ def main():
             ("-DFORCE_32=ON" if bits == '32' else ""),
             ("-DCMAKE_BUILD_TYPE=Debug" if args.debug else ""),
             ("-DYOCTO=1" if yocto else ""),
-            (('-DJDK="%s"' % jdk_path) if jdk_path else "")
+            (('-DJDK="%s"' % jdk_path) if jdk_path else ""),
+            ('-DCMAKE_VERBOSE_MAKEFILE:BOOL=ON' if args.verbose else '')
         ])))
         if sys.platform == 'win32':
             install = args.install and bits == target_bits[-1]
-            target_project = 'PACKAGE' if install else 'ALL_BUILD' #making install only on last config, to pack them all
+            target_project = 'PACKAGE' if install else 'ALL_BUILD'  # making install only on last config, to pack them all
             run_shell('cmake --build . --config %s --target %s' % ('Debug' if args.debug else 'Release', target_project))
             if install:
                 run_shell(r'echo f | xcopy "IntelSEAPI*.exe" "%s\IntelSEAPI.exe" /F' % get_share_folder())
         else:
             import glob
-            run_shell('make -j4')
+            run_shell('cmake --build . --config %s' % ('Debug' if args.debug else 'Release'))
             if not args.install or ('linux' in sys.platform and bits == '64'):
-                continue #don't pack on first round, instead on the second pass collect all
-            run_shell('cpack "%s"' % work_dir)
+                continue  # don't pack on first round, instead on the second pass collect all
+            run_shell('cmake --build . --config %s --target package' % ('Debug' if args.debug else 'Release'))
+
             installer = glob.glob(os.path.join(work_folder, "IntelSEAPI*.sh"))[0]
-            print installer
+            print(installer)
             if sys.platform == 'darwin':
                 replace_in_file(installer, [
                     ('toplevel="`pwd`"', 'toplevel="/Applications"'),