Improve Android helper scripts (#41)

Improve the Android helper scripts to perform more setup to 
support development.

* Server script can now automatically setup adb reverse.
* Server script now optionally captures metadata traces for 
   Mali timeline layer.
* Install script can now automatically capture logcat logs.
* Install script can now automatically capture Perfetto traces 
   for Mali timeline layer.

lgl_android_install.py

@@ -97,13 +97,16 @@
 '''
 
 import argparse
+import atexit
 import json
 import os
+import shutil
+import subprocess as sp
 import sys
 from typing import Optional
 
 from lglpy.android.adb import ADBConnect
-from lglpy.android.utils import AndroidUtils
+from lglpy.android.utils import AndroidUtils, NamedTempFile
 from lglpy.android.filesystem import AndroidFilesystem
 from lglpy.ui import console
 
@@ -406,6 +409,142 @@ def disable_layers(conn: ADBConnect) -> bool:
     return s1 and s2 and s3
 
 
+def configure_logcat(conn: ADBConnect, output_path: str) -> None:
+    '''
+    Clear logcat and then pipe new logs to a file.
+
+    Does not error on failure, but will print a warning.
+
+    Args:
+        conn: The adb connection.
+        output_path: The desired output file path.
+    '''
+    # Delete the file to avoid user reading stale logs
+    if os.path.exists(output_path):
+        os.remove(output_path)
+
+    # Pipe adb to file using an async command to avoid losing logs
+    # Do NOT use shell=True with a > redirect for this - you cannot easily kill
+    # the child on Windows and the log file ends up with an active reference
+    try:
+        conn.adb('logcat', '-c')
+        handle = open(output_path, 'w', encoding='utf-8')
+        child = conn.adb_async('logcat', filex=handle)
+
+        # Register a cleanup handler to kill the child
+        def cleanup(child_process):
+            child_process.kill()
+
+        atexit.register(cleanup, child)
+
+    except sp.CalledProcessError:
+        print('WARNING: Cannot enable logcat recording')
+
+
+def configure_perfetto(
+        conn: ADBConnect, output_path: str) -> Optional[tuple[str, str]]:
+    '''
+    Configure Perfetto traced recording for the given ADB connection.
+
+    Args:
+        conn: The adb connection, requires package to be set.
+        output_path: The desired output file path.
+
+    Returns:
+        PID of the Perfetto process and config file name on success, None on
+        failure.
+    '''
+    assert conn.package, \
+        'Cannot use configure_perfetto() without package'
+
+    # Populate the Perfetto template file
+    output_file = os.path.basename(output_path)
+
+    base_dir = os.path.dirname(__file__)
+    template_path = os.path.join(base_dir, 'lglpy', 'android', 'perfetto.cfg')
+
+    with open(template_path, 'r', encoding='utf-8') as handle:
+        template = handle.read()
+        template = template.replace('{{PACKAGE}}', conn.package)
+
+    try:
+        with NamedTempFile('.cfg') as config_path:
+            # Write a file we can push
+            with open(config_path, 'w', encoding='utf-8') as handle:
+                handle.write(template)
+
+            # Push the file where Perfetto traced can access it
+            config_file = os.path.basename(config_path)
+            conn.adb('push', config_path, '/data/misc/perfetto-configs/')
+
+            # Enable Perfetto traced and the render stages profiler
+            AndroidUtils.set_property(
+                conn, 'persist.traced.enable', '1')
+            AndroidUtils.set_property(
+                conn, 'debug.graphics.gpu.profiler.perfetto', '1')
+
+            # Start Perfetto traced
+            output = conn.adb_run(
+                        'perfetto',
+                        '--background', '--txt',
+                        '-c', f'/data/misc/perfetto-configs/{config_file}',
+                        '-o', f'/data/misc/perfetto-traces/{output_file}')
+
+            pid = output.strip()
+            return (pid, config_file)
+
+    except sp.CalledProcessError:
+        print('ERROR: Cannot enable Perfetto recording')
+        return None
+
+
+def cleanup_perfetto(
+        conn: ADBConnect, output_path: str, pid: str,
+        config_file: str) -> None:
+    '''
+    Cleanup Perfetto traced recording for the given ADB connection.
+
+    Args:
+        conn: The adb connection, requires package to be set.
+        output_path: The desired output file path.
+        pid: The Perfetto process pid.
+        config_file: The Perfetto config path on the device.
+
+    Returns:
+        PID of the Perfetto process and config file name on success, None on
+        failure.
+    '''
+    # Compute the various paths we need
+    output_file = os.path.basename(output_path)
+    output_dir = os.path.dirname(output_path)
+    if not output_dir:
+        output_dir = '.'
+
+    data_file = f'/data/misc/perfetto-traces/{output_file}'
+    config_file = f'/data/misc/perfetto-configs/{config_file}'
+
+    try:
+        # Stop Perfetto recording
+        # TODO: This doesn't work on a user build phone, is there another way?
+        # conn.adb_run('kill', '-TERM', pid)
+
+        # Download the recording data file
+        conn.adb('pull', data_file, output_dir)
+
+        # Remove the device-side files
+        conn.adb_run('rm', data_file)
+        conn.adb_run('rm', config_file)
+
+        # Disable Perfetto traced and the render stages profiler
+        AndroidUtils.set_property(
+            conn, 'persist.traced.enable', '0')
+        AndroidUtils.set_property(
+            conn, 'debug.graphics.gpu.profiler.perfetto', '0')
+
+    except sp.CalledProcessError:
+        print('ERROR: Cannot disable Perfetto recording')
+
+
 def parse_cli() -> argparse.Namespace:
     '''
     Parse the command line.
@@ -430,6 +569,15 @@ def parse_cli() -> argparse.Namespace:
     parser.add_argument(
         '--symbols', '-S', action='store_true', default=False,
         help='use to install layers with unstripped symbols')
+
+    parser.add_argument(
+        '--logcat', type=str, default=None,
+        help='file path to save logcat to after a run')
+
+    parser.add_argument(
+        '--perfetto', type=str, default=None,
+        help='file path to save Perfetto trace to after run')
+
     return parser.parse_args()
 
 
@@ -486,18 +634,30 @@ def main() -> int:
         print(f'    - {layer.name}')
     print()
 
-    input('Press any key to uninstall all layers')
+    # Enable logcat
+    if args.logcat:
+        configure_logcat(conn, args.logcat)
+
+    # Enable Perfetto trace
+    if args.perfetto:
+        perfetto_conf = configure_perfetto(conn, args.perfetto)
+
+    input('Press any key when finished to uninstall all layers')
+
+    # Disable Perfetto trace
+    if args.perfetto and perfetto_conf:
+        cleanup_perfetto(conn, args.perfetto, *perfetto_conf)
 
     # Disable layers
     if not disable_layers(conn):
         print('ERROR: Layer disable on device failed')
-        return 7
+        return 10
 
     # Remove files
     for layer in layers:
         if not uninstall_layer_binary(conn, layer):
             print('ERROR: Layer uninstall from device failed')
-            return 8
+            return 11
 
     return 0
 

lgl_host_server.py

@@ -34,10 +34,12 @@
 '''
 
 import argparse
+import subprocess as sp
 import sys
 import threading
 from typing import Any
 
+from lglpy.android.adb import ADBConnect
 from lglpy.comms import server
 from lglpy.comms import service_gpu_timeline
 from lglpy.comms import service_test
@@ -54,9 +56,17 @@ def parse_cli() -> argparse.Namespace:
     parser = argparse.ArgumentParser()
 
     parser.add_argument(
-        '--test', '-T', action='store_true', default=False,
+        '--test', action='store_true', default=False,
         help='enable the communications unit test helper service')
 
+    parser.add_argument(
+        '--android-port', '-A', type=int, default=62142,
+        help='enable adb reverse on the specified port for network comms')
+
+    parser.add_argument(
+        '--timeline', '-T', type=str, default=None,
+        help='file path to save timeline metadata to to after a run')
+
     return parser.parse_args()
 
 
@@ -69,8 +79,20 @@ def main() -> int:
     '''
     args = parse_cli()
 
+    # Configure Android adb reverse on the specified port
+    conn = ADBConnect()
+    try:
+        conn.adb(
+            'reverse',
+            'localabstract:lglcomms',
+            f'tcp:{args.android_port}')
+
+    except sp.CalledProcessError:
+        print('ERROR: Could not setup Android network comms')
+        return 1
+
     # Create a server instance
-    svr = server.CommsServer(63412)
+    svr = server.CommsServer(args.android_port)
 
     # Register all the services with it
     print('Registering host services:')
@@ -86,9 +108,11 @@ def main() -> int:
     endpoint_id = svr.register_endpoint(service)
     print(f'  - [{endpoint_id}] = {service.get_service_name()}')
 
-    service = service_gpu_timeline.GPUTimelineService()
-    endpoint_id = svr.register_endpoint(service)
-    print(f'  - [{endpoint_id}] = {service.get_service_name()}')
+    if args.timeline:
+        service = service_gpu_timeline.GPUTimelineService(args.timeline)
+        endpoint_id = svr.register_endpoint(service)
+        print(f'  - [{endpoint_id}] = {service.get_service_name()}')
+
     print()
 
     # Start it running

lglpy/android/adb.py

@@ -176,7 +176,8 @@ def adb(self, *args: str, text: bool = True, shell: bool = False,
         return rep.stdout
 
     def adb_async(self, *args: str, text: bool = True, shell: bool = False,
-                  quote: bool = False, pipe: bool = False) -> sp.Popen:
+                  quote: bool = False, pipe: bool = False,
+                  filex=None) -> sp.Popen:
         '''
         Call adb to asynchronously run a command, without waiting for it to
         complete.
@@ -196,7 +197,8 @@ def adb_async(self, *args: str, text: bool = True, shell: bool = False,
             text: True if output is text, False if binary
             shell: True if this should invoke via host shell, False if direct.
             quote: True if arguments are quoted, False if unquoted.
-            pipe: True if child stdout is collected, False if discarded.
+            pipe: True if child stdout is collected to pipe, else False.
+            filex: True if child stdout is collected to file, else False.
 
         Returns:
             The process handle.
@@ -205,7 +207,13 @@ def adb_async(self, *args: str, text: bool = True, shell: bool = False,
             CalledProcessError: The invoked call failed.
         '''
         # Setup the configuration
-        output = sp.PIPE if pipe else sp.DEVNULL
+        output = sp.DEVNULL
+
+        if pipe:
+            output = sp.PIPE
+
+        if filex:
+            output = filex
 
         # Build the command list
         commands = self.get_base_command(args)

lglpy/android/perfetto.cfg

@@ -0,0 +1,28 @@
+buffers {
+  size_kb: 65536
+  fill_policy: RING_BUFFER
+}
+
+data_sources: {
+  config {
+    name: "linux.process_stats"
+  }
+}
+
+data_sources: {
+  config {
+    name: "gpu.renderstages"
+  }
+  producer_name_filter: "{{PACKAGE}}"
+}
+
+data_sources: {
+  config {
+    name: "android.surfaceflinger.frametimeline"
+  }
+}
+
+duration_ms: 60000
+write_into_file: true
+file_write_period_ms: 1000
+max_file_size_bytes: 536870912

lglpy/android/test.py

@@ -27,7 +27,6 @@
 installed to be connected to the host PC with an authorized adb connection.
 '''
 
-import contextlib
 import os
 import re
 import shutil
@@ -37,7 +36,7 @@
 import unittest
 
 from .adb import ADBConnect
-from .utils import AndroidUtils
+from .utils import AndroidUtils, NamedTempFile
 from .filesystem import AndroidFilesystem
 
 
@@ -58,30 +57,6 @@ def get_script_relative_path(file_name: str) -> str:
     return os.path.join(dir_name, file_name)
 
 
-@contextlib.contextmanager
-def NamedTempFile():  # pylint: disable=invalid-name
-    '''
-    Creates a context managed temporary file that can be used with external
-    subprocess.
-
-    On context entry this yields the file name, on exit it deletes the file.
-
-    Yields:
-        The name of the temporary file.
-    '''
-    name = None
-
-    try:
-        f = tempfile.NamedTemporaryFile(delete=False)
-        name = f.name
-        f.close()
-        yield name
-
-    finally:
-        if name:
-            os.unlink(name)
-
-
 class AndroidTestNoDevice(unittest.TestCase):
     '''
     This set of tests validates execution of global commands that can run