Draw/Queue/CPU ratio and GPU/CPU Bound detection

araud · araud · commit 848d255a44c9 · 2017-02-17T17:23:50.000+03:00
Python 3 Support for buildall.py
Exporters API:
	setting multi-value for counter
	frame lanes support
	process labels
	metadata
Tasks from stacks algorithm is refactored
Handling of Adreno format deviations
Better ETW process names handling
More correct ETW VSync handling
Fixes
diff --git a/buildall.py b/buildall.py
@@ -16,6 +16,7 @@
 #
 #**********************************************************************************************************************************************************************************************************************************************************************************************
 
+from __future__ import print_function
 import os
 import sys
 import shutil
@@ -56,54 +57,12 @@ def get_yocto():
         return {'bits': '32'}
     return {'bits': '64'}
 
-
-def GetJDKPath():
-    if sys.platform == 'win32':
-        import _winreg
-        path = "SOFTWARE\\JavaSoft\\Java Development Kit"
-        try:
-            aKey = _winreg.OpenKey(_winreg.HKEY_LOCAL_MACHINE, path, 0, _winreg.KEY_READ | _winreg.KEY_WOW64_64KEY)
-        except WindowsError:
-            print("No key:", path)
-            return None
-        subkeys = []
-        try:
-            i = 0
-            while True:
-                subkeys.append(_winreg.EnumKey(aKey, i))
-                i += 1
-        except WindowsError:
-            pass
-        if not subkeys:
-            print("No subkeys for:", path)
-            return None
-        subkeys.sort()
-        path += "\\" + subkeys[-1]
-        try:
-            aKey = _winreg.OpenKey(_winreg.HKEY_LOCAL_MACHINE, path, 0, _winreg.KEY_READ | _winreg.KEY_WOW64_64KEY)
-            return _winreg.QueryValueEx(aKey, "JavaHome")[0]
-        except WindowsError:
-            print("No value for:", path)
-            return None
-    else:
-        path, err = subprocess.Popen("which javah", shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
-        if err or not path:
-            return None
-        if sys.platform == 'darwin':
-            path = subprocess.check_output("/usr/libexec/java_home").decode("utf-8").split('\n')[0]
-            return path if os.path.exists(path) else None
-        else:
-            matches = []
-            for root, dirnames, filenames in os.walk('/usr/lib/jvm'):
-                for filename in fnmatch.filter(filenames, 'jni.h'):
-                    matches.append(os.path.join(root, filename))
-            if not matches:
-                return None
-            return os.path.split(matches[0])[0]
-
 if sys.platform == 'win32':
     def read_registry(path, depth=0xFFFFFFFF, statics={}):
-        import _winreg
+        try:
+            import _winreg
+        except ImportError:
+            import winreg as _winreg
         parts = path.split('\\')
         hub = parts[0]
         path = '\\'.join(parts[1:])
@@ -148,13 +107,37 @@ def enum_nodes(curpath, level):
         return enum_nodes(path, depth)
 
 
+def GetJDKPath():
+    if sys.platform == 'win32':
+        bush = read_registry(r'HKLM\SOFTWARE\JavaSoft\Java Development Kit')
+        subkeys = sorted([key for key in bush if 'JavaHome' in bush[key]])
+        if subkeys:
+            return bush[subkeys[-1]]['JavaHome']
+        return None
+    else:
+        path, err = subprocess.Popen("which javah", shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
+        if err or not path:
+            return None
+        if sys.platform == 'darwin':
+            path = subprocess.check_output("/usr/libexec/java_home").decode("utf-8").split('\n')[0]
+            return path if os.path.exists(path) else None
+        else:
+            matches = []
+            for root, dirnames, filenames in os.walk('/usr/lib/jvm'):
+                for filename in fnmatch.filter(filenames, 'jni.h'):
+                    matches.append(os.path.join(root, filename))
+            if not matches:
+                return None
+            return os.path.split(matches[0])[0]
+
+
 def get_vs_versions():
     if sys.platform != 'win32':
         return []
     bush = read_registry(r'HKLM\SOFTWARE\Microsoft\VisualStudio', 2)
 
     versions = []
-    for key, val in bush.iteritems():
+    for key in bush:
         if '.' not in key:
             continue
         version = key.split('.')[0]
diff --git a/runtool/collectors/ftrace.py b/runtool/collectors/ftrace.py
@@ -164,7 +164,7 @@ def start(self):
             self.perf_file = os.path.join(self.args.output, 'perf-%s.data' % (self.args.cuts[0] if self.args.cuts else '0'))
             if os.path.exists(self.perf_file):
                 os.remove(self.perf_file)
-            cmd = 'perf record -a -g -o "%s" -a --pid=%s' % (self.perf_file, self.args.target)
+            cmd = 'perf record -a -g -o "%s" --pid=%s' % (self.perf_file, self.args.target)
             self.perf_proc = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, preexec_fn=os.setpgrp)
 
     def copy_from_target(self, what, where):
diff --git a/runtool/collectors/osx.py b/runtool/collectors/osx.py
@@ -183,12 +183,10 @@ def stop(self, wait=True):
         self.log("pid: %s" % str(self.pid))
         if not self.pid:
             return []
-        children = list(self.get_pid_children(self.pid))
-        for pid in children:
+        dtrace_pids = [self.pid] + list(self.get_pid_children(self.pid))
+        for pid in dtrace_pids:  # FIXME: check if it has parent pid as well. Looks as we kill only children.
             self.execute("sudo -A kill -2 %d" % pid, env=os.environ)
-        print pid, str(children)
-        os.waitpid(self.pid, 0)
-        for pid in children:
+        for pid in dtrace_pids:
             try:
                 os.waitpid(pid, 0)
             except:
diff --git a/runtool/collectors/win.py b/runtool/collectors/win.py
@@ -228,7 +228,7 @@ def __init__(self, args):
         Collector.__init__(self, args)
         wpr = WPRCollector.detect()
         self.xperf = os.path.normpath(os.path.join(os.path.dirname(wpr), 'xperf')) if wpr else None
-        if not os.path.exists(self.xperf):
+        if not self.xperf or not os.path.exists(self.xperf):
             variants = self.detect_instances('xperf')
             if variants:
                 self.xperf = variants[0]  # TODO: select by higher version
diff --git a/runtool/decoders/Adreno.py b/runtool/decoders/Adreno.py
@@ -15,7 +15,15 @@ def __init__(self, args, callbacks):
 
     @staticmethod
     def parse_args(args):
-        return dict(pair.split('=') for pair in args.split())
+        res = {None:[]}
+        no_pair = []
+        for chunk in args.split():
+            if '=' in chunk:
+                key, val = chunk.split('=')
+                res[key] = val
+            else:
+                res[None].append(chunk)
+        return res
 
     def gpu_queue(self, ctx, inflight, timestamp):
         state = self.ctx.setdefault(ctx, {'inflight': inflight, 'queued': None})
@@ -123,20 +131,34 @@ def handle_record(self, proc, pid, tid, cpu, flags, timestamp, name, args):
         elif name in ['kgsl_pwr_set_state', 'kgsl_pwr_request_state']:
             args = self.parse_args(args)
             thread = self.gpu.thread(-1)
-            thread.counter('POWER_STATE').set_value(timestamp, 0 if args['state'] == 'NAP' else 1)
+            if 'state' in args:
+                state = 0 if args['state'] == 'NAP' else 1
+            else:
+                state = 0 if 'NAP' in args[None] else 1
+            thread.counter('POWER_STATE').set_value(timestamp, state)
         elif name == 'kgsl_gpubusy':
             args = self.parse_args(args)
             thread = self.gpu.thread(-1)
             thread.counter('busy').set_value(timestamp, int(args['busy']))
             thread.counter('elapsed').set_value(timestamp, int(args['elapsed']))
+        elif name == 'kgsl_tz_params':
+            args = self.parse_args(args)
+            thread = self.gpu.thread(-1)
+            thread.counter('total_time').set_value(timestamp, int(args['total_time']))
+            thread.counter('busy_time').set_value(timestamp, int(args['busy_time']))
+            thread.counter('idle_time').set_value(timestamp, int(args['idle_time']))
         elif name == 'kgsl_a3xx_irq_status':
             args = self.parse_args(args)
             thread = self.gpu.thread(-1)
             thread.object(0, args['d_name'], 'Adreno').snapshot(timestamp, args)
         elif name in ['kgsl_bus', 'kgsl_rail', 'kgsl_irq', 'kgsl_clk']:
             args = self.parse_args(args)
             thread = self.gpu.thread(-1)
-            thread.counter(name.split('_')[1].upper()).set_value(timestamp, 1 if args['flag'] == 'on' else 0)
+            if 'flag' in args:
+                state = 1 if args['flag'] == 'on' else 0
+            else:
+                state = 1 if 'on' in args[None] else 0
+            thread.counter(name.split('_')[1].upper()).set_value(timestamp, state)
         elif name in ['kgsl_register_event', 'kgsl_fire_event', 'kgsl_regwrite', 'kgsl_issueibcmds', 'kgsl_active_count']:
             args = self.parse_args(args)
             self.callbacks.process(pid).thread(tid).marker('thread', name, 'Adreno').set(timestamp, args=args)
diff --git a/runtool/exporters/ChromeTracing.py b/runtool/exporters/ChromeTracing.py
@@ -50,6 +50,7 @@ def __init__(self, args, tree):
         self.frames = {}
         self.samples = []
         self.last_task = None
+        self.metadata = {}
         if self.args.trace:
             for trace in self.args.trace:
                 if trace.endswith(".etl"):
@@ -209,13 +210,17 @@ def calc_time_sync(time_sync, series=True):
 
     def global_metadata(self, data):
         if data['str'] == "__process__":  # this is the very first record in the trace
-            if data.has_key('data'):
+            if 'data' in data:
                 self.file.write(
                     '{"name": "process_name", "ph":"M", "pid":%d, "tid":%d, "args": {"name":"%s"}},\n' % (int(data['pid']), int(data['tid']), data['data'].replace("\\", "\\\\").encode('utf-8'))
                 )
-            if data.has_key('delta'):
+            if 'delta' in data:
                 self.file.write(
-                    '{"name": "process_sort_index", "ph":"M", "pid":%d, "tid":%s, "args": {"sort_index":%d}},\n' % (data['pid'], data['tid'], data['delta'])
+                    '{"name": "process_sort_index", "ph":"M", "pid":%d, "tid":%s, "args": {"sort_index":%d}},\n' % (data['pid'], data['tid'], abs(data['delta']) if abs(data['delta']) > 100 else data['delta'])
+                )
+            if 'labels' in data and data['labels']:
+                self.file.write(
+                    '{"name": "process_labels", "ph":"M", "pid":%d, "tid":%s, "args": {"labels":"%s"}},\n' % (data['pid'], data['tid'], ','.join(data['labels']))
                 )
             if data['tid'] >= 0 and not self.tree['threads'].has_key('%d,%d' % (data['pid'], data['tid'])):  # marking the main thread
                 self.file.write(
@@ -225,10 +230,12 @@ def global_metadata(self, data):
             self.file.write(
                 '{"name": "thread_name", "ph":"M", "pid":%d, "tid":%d, "args": {"name":"%s"}},\n' % (int(data['pid']), int(data['tid']), data['data'].replace("\\", "\\\\").encode('utf-8'))
             )
-            if data.has_key('delta'):
+            if 'delta' in data:
                 self.file.write(
-                    '{"name": "thread_sort_index", "ph":"M", "pid":%d, "tid":%s, "args": {"sort_index":%d}},\n' % (data['pid'], data['tid'], data['delta'])
+                    '{"name": "thread_sort_index", "ph":"M", "pid":%d, "tid":%s, "args": {"sort_index":%d}},\n' % (data['pid'], data['tid'], abs(data['delta']) if abs(data['delta']) > 100 else data['delta'])
                 )
+        else:
+            self.metadata.setdefault(data['str'], []).append(data['data'])
 
     def relation(self, data, head, tail):
         if not head or not tail:
@@ -266,12 +273,14 @@ def complete_task(self, type, begin, end):
             self.last_task = (type, begin, end)
         assert (GoogleTrace.Phase.has_key(type))
         if begin['type'] == 7:  # frame_begin
-            begin['id'] = begin['tid'] if begin.has_key('tid') else 0  # Async events are groupped by cat & id
+            if 'id' not in begin:
+                begin['id'] = id(begin)  # Async events are groupped by cat & id
             res = self.format_task('b', 'frame', begin, {})
-            res += [',\n']
-            end_begin = begin.copy()
-            end_begin['time'] = end['time']
-            res += self.format_task('e', 'frame', end_begin, {})
+            if end:
+                res += [',\n']
+                end_begin = begin.copy()
+                end_begin['time'] = end['time'] - 1000
+                res += self.format_task('e', 'frame', end_begin, {})
         else:
             res = self.format_task(GoogleTrace.Phase[type], type, begin, end)
 
@@ -331,7 +340,7 @@ def handle_stack(self, task, stack, name='stack'):
 
     def format_task(self, phase, type, begin, end):
         res = []
-        res.append('{"ph":"%s"' % (phase))
+        res.append('{"ph":"%s"' % phase)
         res.append(', "pid":%(pid)d' % begin)
         if begin.has_key('tid'):
             res.append(', "tid":%(tid)d' % begin)
@@ -374,7 +383,7 @@ def format_task(self, phase, type, begin, end):
         res.append(', "cat":"%s"' % (begin['domain']))
 
         if 'id' in begin:
-            res.append(', "id":%s' % (begin['id']))
+            res.append(', "id":"%s"' % str(begin['id']))
         if type in ['task']:
             dur = self.convert_time(end['time']) - self.convert_time(begin['time'])
             if dur < self.args.min_dur:
@@ -392,7 +401,8 @@ def format_task(self, phase, type, begin, end):
             args["__file__"] = begin["__file__"]
             args["__line__"] = begin["__line__"]
         if 'counter' == type:
-            args[name] = begin['delta']
+            if 'delta' in begin:  # multi-counter is passed as named sub-counters dict
+                args[name] = begin['delta']
         if 'memory' in begin:
             total = 0
             breakdown = {}
@@ -426,23 +436,31 @@ def remove_last(self, count):
         self.file.truncate()
 
     def finish(self, intermediate=False):
-        if self.samples and not intermediate:
-            self.remove_last(2)
-            self.file.write('], "stackFrames": {\n')
-            for id, frame in self.frames.iteritems():
-                self.file.write('"%s": %s,\n' % (id, json.dumps(frame)))
-            if self.frames:  # deleting last two symbols from the file as we can't leave comma at the end due to json restrictions
-                self.remove_last(2)
-            self.file.write('\n}, "samples": [\n')
-            for sample in self.samples:
-                self.file.write(json.dumps(sample) + ',\n')
-            if self.samples:   # deleting last two symbols from the file as we can't leave comma at the end due to json restrictions
-                self.remove_last(2)
-            self.file.write('\n]}')
-            self.samples = []
-            self.frames = {}
-        else:
-            self.file.write('{}]}')
+        self.remove_last(2)  # remove trailing ,\n
+        if not intermediate:
+            if self.samples:
+                self.file.write('], "stackFrames": {\n')
+                for id, frame in self.frames.iteritems():
+                    self.file.write('"%s": %s,\n' % (id, json.dumps(frame)))
+                if self.frames:  # deleting last two symbols from the file as we can't leave comma at the end due to json restrictions
+                    self.remove_last(2)
+                self.file.write('\n}, "samples": [\n')
+                for sample in self.samples:
+                    self.file.write(json.dumps(sample) + ',\n')
+                if self.samples:   # deleting last two symbols from the file as we can't leave comma at the end due to json restrictions
+                    self.remove_last(2)
+                self.samples = []
+                self.frames = {}
+            if self.metadata:
+                self.file.write('\n],\n')
+                for key, value in self.metadata.iteritems():
+                    self.file.write('"%s": %s,\n' % (key, json.dumps(value[0] if len(value) == 1 else value)))
+                self.remove_last(2)  # remove trailing ,\n
+                self.file.write('\n}')
+                self.file.close()
+                return
+
+        self.file.write('\n]}')
         self.file.close()
 
     @staticmethod
diff --git a/runtool/importers/etw.py b/runtool/importers/etw.py
diff --git a/runtool/sea.py b/runtool/sea.py
diff --git a/runtool/sea_runtool.py b/runtool/sea_runtool.py
