Memfault BORT SDK 5.0.0 (Build 2450655)

CHANGELOG.md

@@ -1,5 +1,57 @@
 # Memfault Bort Changelog
 
+## v5.0.0 - September 12, 2024
+
+### :boom: Breaking Changes
+
+- Bort has been moved from the `system` partition to the `system-ext` partition.
+  This enables the use of Generic System Images.
+- The AOSP patches to `product/mainline.mk`/`mainline_system.mk` (Android 11)
+  and `product/generic_system.mk` (Android 12+) have been removed. These were
+  required to whitelist Bort in the `system` partition. These were the only
+  required AOSP patches if using Android 9+, meaning that all remaining AOSP
+  patches are now optional, to enhance Bort functionality (except if using
+  Android 8).
+- To support this change, we have forked the Bort SDK because of
+  backwards-incompatible `bp` file changes required to support `system-ext`. We
+  will continue to support Android 8-10 on the `8-10` branch, and the `master`
+  branch will support android versions 11+. No other work is required other than
+  making sure to check out the correct branch - the AOSP build will fail if
+  using the wrong branch.
+- You will need to remove the aforementioned patch, if already applied and
+  committed to your AOSP repo.
+
+### :rocket: New Features
+
+- Added the ability for Bort to upload any DropBoxManager tags, without the need
+  to add specific processing for each in the SDK. This is configurable from the
+  dashboard - by default we will start collecting `SYSTEM_AUDIT`, `SYSTEM_BOOT`,
+  `SYSTEM_FSCK`, `SYSTEM_RECOVERY_LOG`, `SYSTEM_RESTART` entries, which can be
+  viewed on the Device Timeline.
+
+### :construction: Fixes
+
+Bug fixes in the Bort SDK are now broken out into their own section in this
+changelog.
+
+- Fixed a bug where Bort would drop reboot events after a factory reset, if
+  `RUNTIME_ENABLE_REQUIRED=true` and Bort was not enabled at the time of boot.
+  Bort will now upload the latest reboot when it is enabled, if not already
+  processed.
+
+### :chart_with_upwards_trend: Improvements
+
+- Store the software version alongside a metric report, so that metrics will not
+  be associated with the wrong software version in the case of an OTA update.
+- Also, collect metrics immediately after boot, if an OTA update was just
+  installed - reducing the potential for metrics to be assigned to the wrong
+  software version.
+
+### :house: Internal
+
+- Refactored HRT file generation.
+- Switched to using the Room gradle plugin.
+
 ## v4.19.0 - August 13, 2024
 
 ### :rocket: New Features

MemfaultDumpstateRunner/Android.bp

@@ -28,4 +28,5 @@ cc_binary {
         "liblog",
     ],
     init_rc: ["memfault_init.rc"],
+    system_ext_specific: true,
 }

MemfaultDumpstateRunner/memfault_init.rc

@@ -23,7 +23,7 @@ service memfault_dumpstatez /system/bin/dumpstate -S -d -z -q \
 # into Bort's filesystem sandboxed location. See MemfaultDumpstateRunner.cpp for
 # the source code of the program itself.
 
-service memfault_dumpstate_runner /system/bin/MemfaultDumpstateRunner
+service memfault_dumpstate_runner /system_ext/bin/MemfaultDumpstateRunner
     class core
     disabled
     oneshot

MemfaultDumpster/Android.bp

@@ -13,4 +13,5 @@ cc_library_shared {
         "com/memfault/dumpster/IDumpsterBasicCommandListener.aidl",
         "com/memfault/dumpster/IDumpster.aidl",
     ],
+    system_ext_specific: true,
 }

MemfaultDumpster/Android.mk

@@ -27,4 +27,5 @@ LOCAL_SHARED_LIBRARIES := \
 LOCAL_PROTOC_OPTIMIZE_TYPE := full
 LOCAL_STATIC_LIBRARIES := liblog
 LOCAL_INIT_RC := memfault_dumpster.rc
+LOCAL_SYSTEM_EXT_MODULE := true
 include $(BUILD_EXECUTABLE)

MemfaultDumpster/memfault_dumpster.rc

@@ -6,5 +6,5 @@ on post-fs-data
     mkdir /data/system/MemfaultDumpster/ 0700 root root
     wait /data/system/MemfaultDumpster/
 
-service memfault_dumpster /system/bin/MemfaultDumpster
+service memfault_dumpster /system_ext/bin/MemfaultDumpster
     class core

MemfaultPackages/Android.mk

@@ -2,6 +2,12 @@ LOCAL_PATH := $(call my-dir)
 
 include $(LOCAL_PATH)/bort_src_gen.mk
 
+ifeq ($(shell test $(PLATFORM_SDK_VERSION) -ge 30 && echo true),true)
+  _BORT_ETC_PATH := $(TARGET_OUT_SYSTEM_EXT_ETC)
+else
+  _BORT_ETC_PATH := $(TARGET_OUT_ETC)
+endif
+
 
 # Check that ensures MemfaultBort.x509.pem matches signature of MemfaultBort.apk
 ################################################################################
@@ -31,7 +37,7 @@ include $(CLEAR_VARS)
 
 LOCAL_MODULE := com.memfault.bort.xml
 LOCAL_MODULE_CLASS := ETC
-LOCAL_MODULE_PATH := $(TARGET_OUT_ETC)/permissions
+LOCAL_MODULE_PATH := $(_BORT_ETC_PATH)/permissions
 BORT_XML_TARGET := $(LOCAL_MODULE_PATH)/$(LOCAL_MODULE)
 $(call bort_src_gen,$(LOCAL_PATH)/com.memfault.bort.xml.in,$(BORT_XML_TARGET))
 LOCAL_ADDITIONAL_DEPENDENCIES := $(BORT_XML_TARGET)
@@ -57,6 +63,7 @@ LOCAL_REQUIRED_MODULES := com.memfault.bort.xml CheckMemfaultBortSignature
 
 # The priv-app folder
 TARGET_OUT_DATA_APPS_PRIVILEGED := $(TARGET_OUT_DATA)/priv-app
+LOCAL_SYSTEM_EXT_MODULE := true
 include $(BUILD_PREBUILT)
 
 # MemfaultBortOta.apk
@@ -66,7 +73,7 @@ include $(CLEAR_VARS)
 
 LOCAL_MODULE := com.memfault.bort.ota.xml
 LOCAL_MODULE_CLASS := ETC
-LOCAL_MODULE_PATH := $(TARGET_OUT_ETC)/permissions
+LOCAL_MODULE_PATH := $(_BORT_ETC_PATH)/permissions
 BORT_XML_TARGET := $(LOCAL_MODULE_PATH)/$(LOCAL_MODULE)
 $(call bort_src_gen,$(LOCAL_PATH)/com.memfault.bort.ota.xml.in,$(BORT_XML_TARGET))
 LOCAL_ADDITIONAL_DEPENDENCIES := $(BORT_XML_TARGET)
@@ -92,6 +99,7 @@ LOCAL_REQUIRED_MODULES := com.memfault.bort.ota.xml CheckMemfaultBortOtaSignatur
 
 # The priv-app folder
 TARGET_OUT_DATA_APPS_PRIVILEGED := $(TARGET_OUT_DATA)/priv-app
+LOCAL_SYSTEM_EXT_MODULE := true
 include $(BUILD_PREBUILT)
 
 
@@ -101,7 +109,7 @@ include $(BUILD_PREBUILT)
 include $(CLEAR_VARS)
 LOCAL_MODULE := com.memfault.usagereporter.xml
 LOCAL_MODULE_CLASS := ETC
-LOCAL_MODULE_PATH := $(TARGET_OUT_ETC)/permissions
+LOCAL_MODULE_PATH := $(_BORT_ETC_PATH)/permissions
 USAGE_REPORTER_XML_TARGET := $(LOCAL_MODULE_PATH)/$(LOCAL_MODULE)
 $(call bort_src_gen,$(LOCAL_PATH)/com.memfault.usagereporter.xml.in,$(USAGE_REPORTER_XML_TARGET))
 LOCAL_ADDITIONAL_DEPENDENCIES := $(USAGE_REPORTER_XML_TARGET)
@@ -126,6 +134,7 @@ LOCAL_REQUIRED_MODULES := com.memfault.usagereporter.xml
 
 # The priv-app folder
 TARGET_OUT_DATA_APPS_PRIVILEGED := $(TARGET_OUT_DATA)/priv-app
+LOCAL_SYSTEM_EXT_MODULE := true
 include $(BUILD_PREBUILT)
 
 include $(LOCAL_PATH)/reporting-libs/reporting-lib-java/Android.mk

MemfaultPackages/bort-shared/src/main/java/com/memfault/bort/DataScrubber.kt

@@ -5,15 +5,19 @@ import kotlinx.serialization.Serializable
 import org.kocakosm.jblake2.Blake2b
 import java.lang.StringBuilder
 
+fun interface LineScrubbingCleaners {
+    operator fun invoke(): List<LineScrubbingCleaner>
+}
+
 interface LineScrubbingCleaner {
     fun clean(line: String): String
 }
 
 class DataScrubber(
-    private val cleaners: List<LineScrubbingCleaner> = listOf(),
+    private val cleaners: LineScrubbingCleaners,
     private val hash: (line: String) -> String = ::blake2b,
 ) {
-    operator fun invoke(line: String): String = cleaners.fold(line) { data, matcher ->
+    operator fun invoke(line: String): String = cleaners().fold(line) { data, matcher ->
         matcher.clean(data)
     }
 

MemfaultPackages/bort-shared/src/main/java/com/memfault/bort/DeviceInfo.kt

@@ -35,9 +35,11 @@ data class DeviceInfo(
                     "${getBuildFingerprint()}::${props[settings.androidBuildVersionKey] ?: "unknown"}"
             }
             return DeviceInfo(
-                settings.overriddenSerialNumber ?: props[settings.androidSerialNumberKey] ?: getFallbackAndroidId(),
-                hardwareVersionFromSettingsAndSystemProperties(settings, props),
-                softwareVersion,
+                deviceSerial = settings.overriddenSerialNumber
+                    ?: props[settings.androidSerialNumberKey]
+                    ?: getFallbackAndroidId(),
+                hardwareVersion = hardwareVersionFromSettingsAndSystemProperties(settings, props),
+                softwareVersion = softwareVersion,
             )
         }
 

MemfaultPackages/bort-shared/src/main/java/com/memfault/bort/DeviceInfoProvider.kt

@@ -24,18 +24,19 @@ class RealDeviceInfoProvider @Inject constructor(
 ) : DeviceInfoProvider {
     private val deviceInfo = CachedAsyncProperty {
         DeviceInfo.fromSettingsAndSystemProperties(
-            lastSettings,
-            dumpsterClient.getprop() ?: emptyMap(),
-            application,
+            settings = lastSettings,
+            props = dumpsterClient.getprop() ?: emptyMap(),
+            context = application,
         )
     }
     private var lastSettings = deviceInfoSettings.asParams(overrideSerial)
     private val mutex = Mutex()
 
     override suspend fun getDeviceInfo(): DeviceInfo = mutex.withLock {
-        if (lastSettings != deviceInfoSettings.asParams(overrideSerial)) {
+        val newSettings = deviceInfoSettings.asParams(overrideSerial)
+        if (lastSettings != newSettings) {
             Logger.d("Invalidating deviceInfo")
-            lastSettings = deviceInfoSettings.asParams(overrideSerial)
+            lastSettings = newSettings
             deviceInfo.invalidate()
         }
         deviceInfo.get()

MemfaultPackages/bort-shared/src/main/java/com/memfault/bort/process/ProcessExecutor.kt

@@ -1,9 +1,12 @@
 package com.memfault.bort.process
 
+import com.memfault.bort.BasicCommandTimeout
 import com.memfault.bort.IO
 import com.memfault.bort.shared.Logger
 import kotlinx.coroutines.runInterruptible
+import kotlinx.coroutines.time.withTimeoutOrNull
 import kotlinx.coroutines.withContext
+import kotlinx.coroutines.withTimeoutOrNull
 import java.io.InputStream
 import javax.inject.Inject
 import kotlin.coroutines.CoroutineContext
@@ -12,22 +15,27 @@ import kotlin.coroutines.cancellation.CancellationException
 class ProcessExecutor
 @Inject constructor(
     @IO private val ioDispatcher: CoroutineContext,
+    @BasicCommandTimeout private val basicCommandTimeout: Long,
 ) {
     suspend fun <T> execute(
         command: List<String>,
         handleLines: (InputStream) -> T,
     ): T? = withContext(ioDispatcher) {
         Logger.d("ProcessExecutor: executing $command")
-        val process = ProcessBuilder(command).start()
+        val process = runCatching {
+            ProcessBuilder(command).start()
+        }.getOrNull() ?: return@withContext null
 
         try {
             val output = handleLines(process.inputStream)
 
-            runInterruptible {
-                val exitCode = process.waitFor()
-                Logger.d("ProcessExecutor: $command exit code $exitCode")
-                if (exitCode != 0) return@runInterruptible null
-                output
+            withTimeoutOrNull(basicCommandTimeout) {
+                runInterruptible {
+                    val exitCode = process.waitFor()
+                    Logger.d("ProcessExecutor: $command exit code $exitCode")
+                    if (exitCode != 0) return@runInterruptible null
+                    output
+                }
             }
         } catch (e: CancellationException) {
             Logger.d("ProcessExecutor", e)

MemfaultPackages/bort-shared/src/main/java/com/memfault/bort/settings/FetchedSettings.kt

@@ -121,6 +121,15 @@ data class FetchedSettings(
     @Serializable(with = DurationAsMillisecondsLong::class)
     val dropBoxPollingInterval: BoxedDuration = 15.minutes.boxed(),
 
+    @SerialName("drop_box.other_tags")
+    val dropBoxOtherTags: Set<String> = setOf(
+        "SYSTEM_AUDIT",
+        "SYSTEM_BOOT",
+        "SYSTEM_FSCK",
+        "SYSTEM_RECOVERY_LOG",
+        "SYSTEM_RESTART",
+    ),
+
     @SerialName("drop_box.java_exceptions.rate_limiting_settings")
     val dropBoxJavaExceptionsRateLimitingSettings: RateLimitingSettings,
 
@@ -154,6 +163,13 @@ data class FetchedSettings(
         maxBuckets = 1,
     ),
 
+    @SerialName("drop_box.other_tags.rate_limiting_settings")
+    val dropBoxOtherTagsRateLimitingSettings: RateLimitingSettings = RateLimitingSettings(
+        defaultCapacity = 15,
+        defaultPeriod = 60.minutes.boxed(),
+        maxBuckets = 1,
+    ),
+
     @SerialName("file_upload_holding_area.max_stored_events_of_interest")
     val fileUploadHoldingAreaMaxStoredEventsOfInterest: Int,
 
@@ -276,6 +292,10 @@ data class FetchedSettings(
     @SerialName("metrics.data_source_enabled")
     val metricsDataSourceEnabled: Boolean,
 
+    @SerialName("metrics.polling_interval_ms")
+    @Serializable(with = DurationAsMillisecondsLong::class)
+    val metricsPollingInterval: BoxedDuration = 5.minutes.boxed(),
+
     @SerialName("metrics.properties_use_service")
     val metricsPropertiesUseService: Boolean = true,
 

MemfaultPackages/bort/build.gradle

@@ -1,5 +1,3 @@
-import static com.memfault.bort.buildsrc.BortSettingsDownloaderKt.getBortSettingsAssetsPath
-
 apply plugin: 'com.android.application'
 apply plugin: 'com.squareup.anvil'
 apply plugin: 'dagger.hilt.android.plugin'
@@ -8,6 +6,7 @@ apply plugin: 'kotlin-kapt'
 apply plugin: 'kotlinx-serialization'
 apply from: rootProject.file("properties.gradle")
 apply plugin: 'common-android'
+apply plugin: 'androidx.room'
 
 def keystoreProperties = loadBortKeystoreProperties()
 def keystoreFilePath = getKeystoreFilePath()
@@ -33,12 +32,6 @@ android {
         buildConfigField "Boolean", "BORT_LITE", bortProperty("BORT_LITE") ?: "false"
 
         testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner"
-
-        javaCompileOptions {
-          annotationProcessorOptions {
-            arguments = ["room.schemaLocation": "$projectDir/schemas".toString()]
-          }
-        }
     }
 
     signingConfigs {
@@ -65,6 +58,10 @@ android {
     }
 }
 
+room {
+    schemaDirectory "$projectDir/schemas"
+}
+
 kapt {
     correctErrorTypes true
     arguments {