Skip to content

Commit

Permalink
Read HighVolume & Persist directories of logarchive in sysdiagnose to…
Browse files Browse the repository at this point in the history 
… fix Alex' issue.
  • Loading branch information
@lukbukkit
lukbukkit committed Jun 16, 2024
1 parent def1fa5 commit e547dea
Show file tree
Hide file tree
Showing 6 changed files with 43 additions and 40 deletions.
55 changes: 29 additions & 26 deletions CellGuard/CellGuard/System Diagnoses/LogArchiveReader.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ struct LogArchiveReader {
private static let rustApp = RustApp.init()
public static var logParseProgress: (() -> Void)?

static func importInBackground(url: URL, highVolumeSpeedup: Bool, progress: @escaping LogProgressFunc, completion: @escaping (Result<ImportResult, Error>) -> ()) {
static func importInBackground(url: URL, speedup: Bool, progress: @escaping LogProgressFunc, completion: @escaping (Result<ImportResult, Error>) -> ()) {
// It's crucial that the task has not the lowest priority, otherwise the process is very slloooowww
Task(priority: TaskPriority.high) {
PortStatus.importActive.store(true, ordering: .relaxed)
Expand All @@ -69,7 +69,7 @@ struct LogArchiveReader {
}
let result = Result.init {
do {
return try LogArchiveReader().read(url: url, highVolumeSpeedup: highVolumeSpeedup, rust: rustApp, progress: localProgress)
return try LogArchiveReader().read(url: url, speedup: speedup, rust: rustApp, progress: localProgress)
} catch {
logger.warning("Failed to read sysdiagnose \(url): \(error)")
throw error
Expand All @@ -86,8 +86,8 @@ struct LogArchiveReader {

private let fileManager = FileManager.default

func read(url: URL, highVolumeSpeedup: Bool, rust: RustApp, progress: @escaping LogProgressFunc) throws -> ImportResult {
Self.logger.debug("Reading log archive at \(url) with HighVolume speedup = \(highVolumeSpeedup)")
func read(url: URL, speedup: Bool, rust: RustApp, progress: @escaping LogProgressFunc) throws -> ImportResult {
Self.logger.debug("Reading log archive at \(url) with speedup = \(speedup)")
progress(.unarchiving, 0, 0)
let tmpDir: URL
do {
Expand All @@ -107,23 +107,23 @@ struct LogArchiveReader {

progress(.extractingTar, 0, 0)
var logArchive: URL
var highVolumeSpeedup = highVolumeSpeedup
var speedup = speedup
var fileCountTraceV3 = 0
do {
logArchive = try extractLogArchive(tmpDir: tmpDir, tmpTarFile: tmpTarFile, highVolumeSpeedup: highVolumeSpeedup)
fileCountTraceV3 = try countLogArchiveFiles(logArchiveDir: logArchive, highVolumeSpeedup: highVolumeSpeedup)
logArchive = try extractLogArchive(tmpDir: tmpDir, tmpTarFile: tmpTarFile, speedup: speedup)
fileCountTraceV3 = try countLogArchiveFiles(logArchiveDir: logArchive, speedup: speedup)

// If the speed-up is enabled, check if the log archive yields HighVolume tracev3 files.
// If the speed-up is enabled, check if the log archive has Persist or HighVolume tracev3 files.
// If not, we can't apply the speed-up and have to scan all tracev3 files.
if highVolumeSpeedup && fileCountTraceV3 == 0 {
Self.logger.debug("Disables HighVolume speed-up as there is no HighVolume directory")
highVolumeSpeedup = false
if speedup && fileCountTraceV3 == 0 {
Self.logger.debug("Disables Persist & HighVolume speed-up as there is none the relevant directories")
speedup = false
// Redo the extraction as we need all tracev3 files for parsing
logArchive = try extractLogArchive(tmpDir: tmpDir, tmpTarFile: tmpTarFile, highVolumeSpeedup: highVolumeSpeedup)
fileCountTraceV3 = try countLogArchiveFiles(logArchiveDir: logArchive, highVolumeSpeedup: highVolumeSpeedup)
logArchive = try extractLogArchive(tmpDir: tmpDir, tmpTarFile: tmpTarFile, speedup: speedup)
fileCountTraceV3 = try countLogArchiveFiles(logArchiveDir: logArchive, speedup: speedup)
// Disable the speed-up for future invocations as future sysdiagnoses recorded with the device will also not support the speed-up.
// The user can always re-enable the speed-up in the settings.
UserDefaults.standard.setValue(false, forKey: UserDefaultsKeys.highVolumeSpeedup.rawValue)
UserDefaults.standard.setValue(false, forKey: UserDefaultsKeys.logArchiveSpeedup.rawValue)
}

try fileManager.removeItem(at: tmpTarFile)
Expand All @@ -144,7 +144,7 @@ struct LogArchiveReader {
// Can we do the same? -> Yes, we already doing this in Rust.
// The function `output` in the file `src/csv_parser.rs` filters log entries based on their subsystem and content.
// Therefore, we have to modify the function if we want to analyze log entries of different subsystems.
csvFile = try parseLogArchive(tmpDir: tmpDir, logArchiveDir: logArchive, highVolumeSpeedup: highVolumeSpeedup, rust: rust)
csvFile = try parseLogArchive(tmpDir: tmpDir, logArchiveDir: logArchive, speedup: speedup, rust: rust)

Self.logParseProgress = nil
try fileManager.removeItem(at: logArchive)
Expand Down Expand Up @@ -198,7 +198,7 @@ struct LogArchiveReader {
return tmpTarFile
}

private func extractLogArchive(tmpDir: URL, tmpTarFile: URL, highVolumeSpeedup: Bool) throws -> URL {
private func extractLogArchive(tmpDir: URL, tmpTarFile: URL, speedup: Bool) throws -> URL {
Self.logger.debug("Extracting tar contents")
let fileHandle = try FileHandle(forReadingFrom: tmpTarFile)
defer { try? fileHandle.close() }
Expand All @@ -222,15 +222,16 @@ struct LogArchiveReader {
let path = nameComponents[logArchiveIndex...nameComponents.count-1]
.reduce(tmpDir) { $0.appendingPathComponent(String($1)) }

// Only extract HighVolume/*.travev3 files if the speedup is enabled as other tracev3 files are not read.
if highVolumeSpeedup && path.lastPathComponent.hasSuffix(".tracev3") && !path.pathComponents.contains("HighVolume") {
Self.logger.debug("Skipping extraction from TAR due to HighVolume speed up: \(path)")
// Only extract HighVolume/*.travev3 & Persist/*.tracev3 files if the speedup is enabled as other tracev3 files are not read.
if speedup && path.lastPathComponent.hasSuffix(".tracev3") &&
!(path.pathComponents.contains("HighVolume") || path.pathComponents.contains("Persist")) {
Self.logger.debug("Skipping extraction from tar due to speed up: \(path)")
return
}

try fileManager.createDirectory(at: path.deletingLastPathComponent(), withIntermediateDirectories: true)
try entry.data?.write(to: path)
Self.logger.debug("Extracting from TAR: \(path)")
Self.logger.debug("Extracting from tar: \(path)")
}
}

Expand Down Expand Up @@ -261,7 +262,7 @@ struct LogArchiveReader {
return logArchiveDir
}

private func countLogArchiveFiles(logArchiveDir: URL, highVolumeSpeedup: Bool) throws -> Int {
private func countLogArchiveFiles(logArchiveDir: URL, speedup: Bool) throws -> Int {
// See: https://stackoverflow.com/a/41979314
let resourceKeys : [URLResourceKey] = [.isDirectoryKey]
guard let enumerator = fileManager.enumerator(at: logArchiveDir, includingPropertiesForKeys: resourceKeys) else {
Expand All @@ -277,9 +278,11 @@ struct LogArchiveReader {
}
// We search for files which will be parsed by the library
if fileUrl.lastPathComponent.hasSuffix(".tracev3") {
if highVolumeSpeedup && fileUrl.pathComponents.contains("HighVolume") {
count += 1
} else if !highVolumeSpeedup {
if speedup {
if fileUrl.pathComponents.contains("HighVolume") || fileUrl.pathComponents.contains("Persist") {
count += 1
}
} else if !speedup {
count += 1
}
}
Expand All @@ -288,15 +291,15 @@ struct LogArchiveReader {
return count
}

private func parseLogArchive(tmpDir: URL, logArchiveDir: URL, highVolumeSpeedup: Bool, rust: RustApp) throws -> URL {
private func parseLogArchive(tmpDir: URL, logArchiveDir: URL, speedup: Bool, rust: RustApp) throws -> URL {
Self.logger.debug("Parsing extracted log archive using macos-unifiedlogs")

// Define the path of the output file
let outFile = tmpDir.appendingPathComponent("system_logs", conformingTo: .commaSeparatedText)

// Call the native macos-unifiedlogs via swift-bridge
// It the returns the total number of parsed log lines
_ = rust.parse_system_log(logArchiveDir.path, outFile.path, highVolumeSpeedup)
_ = rust.parse_system_log(logArchiveDir.path, outFile.path, speedup)

return outFile
}
Expand Down
6 changes: 3 additions & 3 deletions CellGuard/CellGuard/User Interface/Settings/AdvancedSettingsView.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ struct AdvancedSettingsView: View {

@AppStorage(UserDefaultsKeys.showTrackingMarker.rawValue) var showTrackingMarker: Bool = false
@AppStorage(UserDefaultsKeys.appMode.rawValue) var appMode: DataCollectionMode = .none
@AppStorage(UserDefaultsKeys.highVolumeSpeedup.rawValue) var highVolumeSpeedup: Bool = true
@AppStorage(UserDefaultsKeys.logArchiveSpeedup.rawValue) var logArchiveSpeedup: Bool = true

private var dataCollectionFooter: String {
var text = "The data collection mode determines if and how CellGuard collects data. "
Expand All @@ -37,8 +37,8 @@ struct AdvancedSettingsView: View {
}
}

Section(header: Text("HighVolume Log Speedup"), footer: Text("Only scan certain log files from sysdiagnoses to speed up their import. Will be automatically disabled if not applicable for your system.")) {
Toggle("Enable Speedup", isOn: $highVolumeSpeedup)
Section(header: Text("Logarchive Import Speedup"), footer: Text("Only scan certain log files from sysdiagnoses to speed up their import. Will be automatically disabled if not applicable for your system.")) {
Toggle("Enable Speedup", isOn: $logArchiveSpeedup)
}

Section(header: Text("Location"), footer: Text("Show iOS' background indicator to quickly access CellGuard.")) {
Expand Down
4 changes: 2 additions & 2 deletions CellGuard/CellGuard/User Interface/Settings/ImportView.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,7 @@ struct ImportView: View {

@State private var importNotices: [ImportNotice] = []

@AppStorage(UserDefaultsKeys.highVolumeSpeedup.rawValue) private var highVolumeSpeedup = true
@AppStorage(UserDefaultsKeys.logArchiveSpeedup.rawValue) private var logArchiveSpeedup = true

init() {
self.fileUrlFixed = false
Expand Down Expand Up @@ -257,7 +257,7 @@ struct ImportView: View {
}
break
case .sysdiagnose:
LogArchiveReader.importInBackground(url: url, highVolumeSpeedup: highVolumeSpeedup) { phase, currentProgress, totalProgress in
LogArchiveReader.importInBackground(url: url, speedup: logArchiveSpeedup) { phase, currentProgress, totalProgress in
let progress = Float(currentProgress) / Float(totalProgress)
switch (phase) {
case .unarchiving:
Expand Down
2 changes: 1 addition & 1 deletion CellGuard/CellGuard/UserDefaultsKeys.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ enum UserDefaultsKeys: String {
case showTrackingMarker
case lastExportDate
case appMode
case highVolumeSpeedup
case logArchiveSpeedup
case study
case importedCellNumber
}
Expand Down
12 changes: 6 additions & 6 deletions src/csv_parser.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,14 +53,14 @@ pub fn parse_log_archive(path: &str, output_path: &str, high_volume_speedup: boo

// Use the provided strings, shared strings, timesync data to parse the Unified Log data at provided path.
// Currently, expect the path to follow macOS log collect structure.
// If high_volume = true, only scan tracev3 files in the HighVolume folder as they are the only ones containing interesting cellular logs.
// If speedup = true, only scan tracev3 files in the Persist & HighVolume folders as they are the only ones containing interesting cellular logs.
fn parse_trace_files(
string_results: &[UUIDText],
shared_strings_results: &[SharedCacheStrings],
timesync_data: &[TimesyncBoot],
path: &str,
output_path: &str,
high_volume_speedup: bool,
speedup: bool,
) -> u32 {
// We need to persist the Oversize log entries (they contain large strings that don't fit in normal log entries)
// Some log entries have Oversize strings located in different tracev3 files.
Expand All @@ -80,7 +80,7 @@ fn parse_trace_files(
archive_path.push("Persist");

let mut log_count: usize = 0;
if archive_path.exists() && !high_volume_speedup {
if archive_path.exists() {
let paths = fs::read_dir(&archive_path).unwrap();

// Loop through all tracev3 files in Persist directory
Expand Down Expand Up @@ -121,7 +121,7 @@ fn parse_trace_files(
archive_path.pop();
archive_path.push("Special");

if archive_path.exists() && !high_volume_speedup {
if archive_path.exists() && !speedup {
let paths = fs::read_dir(&archive_path).unwrap();

// Loop through all tracev3 files in Special directory
Expand Down Expand Up @@ -161,7 +161,7 @@ fn parse_trace_files(
archive_path.pop();
archive_path.push("Signpost");

if archive_path.exists() && !high_volume_speedup {
if archive_path.exists() && !speedup {
let paths = fs::read_dir(&archive_path).unwrap();

// Loop through all tracev3 files in Signpost directory
Expand Down Expand Up @@ -238,7 +238,7 @@ fn parse_trace_files(
archive_path.push("logdata.LiveData.tracev3");

// Check if livedata exists. We only have it if 'log collect' was used
if archive_path.exists() && !high_volume_speedup {
if archive_path.exists() && !speedup {
println!("Parsing: logdata.LiveData.tracev3");
let mut log_data = parse_log(&archive_path.display().to_string()).unwrap();
log_data.oversize.append(&mut oversize_strings.oversize);
Expand Down
4 changes: 2 additions & 2 deletions src/lib.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,8 @@ impl RustApp {
RustApp {}
}

fn parse_system_log(&self, input: &str, output: &str, high_volume_speedup: bool) -> u32 {
fn parse_system_log(&self, input: &str, output: &str, speedup: bool) -> u32 {
csv_parser::output_header(output).unwrap();
return csv_parser::parse_log_archive(input, output, high_volume_speedup);
return csv_parser::parse_log_archive(input, output, speedup);
}
}

0 comments on commit e547dea

Please sign in to comment.