From 7bfcaf989fdee76f146ec907e18c2908fa447bde Mon Sep 17 00:00:00 2001 From: Ivan Dlugos Date: Wed, 11 Oct 2023 22:36:35 +0200 Subject: [PATCH 01/47] chore: flutter symbol collector CLI tool --- scripts/flutter_symbol_collector/.gitignore | 3 + scripts/flutter_symbol_collector/README.md | 4 + .../analysis_options.yaml | 30 +++++++ .../bin/flutter_symbol_collector.dart | 5 ++ .../lib/flutter_symbol_collector.dart | 2 + .../lib/src/downloader.dart | 60 ++++++++++++++ .../lib/src/flutter_version.dart | 14 ++++ .../lib/src/symbol_resolver.dart | 48 ++++++++++++ scripts/flutter_symbol_collector/pubspec.yaml | 18 +++++ .../test/downloader_test.dart | 78 +++++++++++++++++++ 10 files changed, 262 insertions(+) create mode 100644 scripts/flutter_symbol_collector/.gitignore create mode 100644 scripts/flutter_symbol_collector/README.md create mode 100644 scripts/flutter_symbol_collector/analysis_options.yaml create mode 100644 scripts/flutter_symbol_collector/bin/flutter_symbol_collector.dart create mode 100644 scripts/flutter_symbol_collector/lib/flutter_symbol_collector.dart create mode 100644 scripts/flutter_symbol_collector/lib/src/downloader.dart create mode 100644 scripts/flutter_symbol_collector/lib/src/flutter_version.dart create mode 100644 scripts/flutter_symbol_collector/lib/src/symbol_resolver.dart create mode 100644 scripts/flutter_symbol_collector/pubspec.yaml create mode 100644 scripts/flutter_symbol_collector/test/downloader_test.dart diff --git a/scripts/flutter_symbol_collector/.gitignore b/scripts/flutter_symbol_collector/.gitignore new file mode 100644 index 0000000000..3a85790408 --- /dev/null +++ b/scripts/flutter_symbol_collector/.gitignore @@ -0,0 +1,3 @@ +# https://dart.dev/guides/libraries/private-files +# Created by `dart pub` +.dart_tool/ diff --git a/scripts/flutter_symbol_collector/README.md b/scripts/flutter_symbol_collector/README.md new file mode 100644 index 0000000000..55c60bca27 --- /dev/null +++ b/scripts/flutter_symbol_collector/README.md @@ -0,0 +1,4 @@ +# Flutter symbol collector + +This is an internal tool to collect Flutter debug symbols and upload them to Sentry. +This application is not intended for public usage - we're uploading the symbols in CI automatically so you don't have to. diff --git a/scripts/flutter_symbol_collector/analysis_options.yaml b/scripts/flutter_symbol_collector/analysis_options.yaml new file mode 100644 index 0000000000..dee8927aaf --- /dev/null +++ b/scripts/flutter_symbol_collector/analysis_options.yaml @@ -0,0 +1,30 @@ +# This file configures the static analysis results for your project (errors, +# warnings, and lints). +# +# This enables the 'recommended' set of lints from `package:lints`. +# This set helps identify many issues that may lead to problems when running +# or consuming Dart code, and enforces writing Dart using a single, idiomatic +# style and format. +# +# If you want a smaller set of lints you can change this to specify +# 'package:lints/core.yaml'. These are just the most critical lints +# (the recommended set includes the core lints). +# The core lints are also what is used by pub.dev for scoring packages. + +include: package:lints/recommended.yaml + +# Uncomment the following section to specify additional rules. + +# linter: +# rules: +# - camel_case_types + +# analyzer: +# exclude: +# - path/to/excluded/files/** + +# For more information about the core and recommended set of lints, see +# https://dart.dev/go/core-lints + +# For additional information about configuring this file, see +# https://dart.dev/guides/language/analysis-options diff --git a/scripts/flutter_symbol_collector/bin/flutter_symbol_collector.dart b/scripts/flutter_symbol_collector/bin/flutter_symbol_collector.dart new file mode 100644 index 0000000000..2a14ea89e2 --- /dev/null +++ b/scripts/flutter_symbol_collector/bin/flutter_symbol_collector.dart @@ -0,0 +1,5 @@ +import 'package:flutter_symbol_collector/flutter_symbol_collector.dart'; + +void main(List arguments) { + final downloader = FlutterSymbolDownloader(); +} diff --git a/scripts/flutter_symbol_collector/lib/flutter_symbol_collector.dart b/scripts/flutter_symbol_collector/lib/flutter_symbol_collector.dart new file mode 100644 index 0000000000..9dea2540d7 --- /dev/null +++ b/scripts/flutter_symbol_collector/lib/flutter_symbol_collector.dart @@ -0,0 +1,2 @@ +export 'src/downloader.dart'; +export 'src/flutter_version.dart'; diff --git a/scripts/flutter_symbol_collector/lib/src/downloader.dart b/scripts/flutter_symbol_collector/lib/src/downloader.dart new file mode 100644 index 0000000000..0fe98439a2 --- /dev/null +++ b/scripts/flutter_symbol_collector/lib/src/downloader.dart @@ -0,0 +1,60 @@ +import 'package:flutter_symbol_collector/src/flutter_version.dart'; +import 'package:flutter_symbol_collector/src/symbol_resolver.dart'; +import 'package:github/github.dart'; +import 'package:gcloud/storage.dart'; +import 'package:http/http.dart'; +import 'package:logging/logging.dart'; +import 'package:path/path.dart' as path; + +class FlutterSymbolDownloader { + late final Logger _log; + final _github = GitHub(); + late final _flutterRepo = RepositorySlug('flutter', 'flutter'); + late final _symbolsBucket = + Storage(Client(), '').bucket('flutter_infra_release'); + + FlutterSymbolDownloader({Logger? logger}) { + _log = logger ?? Logger.root; + } + + Stream listFlutterVersions() => _github.repositories + .listTags(_flutterRepo, perPage: 30) + .map((t) => FlutterVersion(t.name)); + + Future> listSymbolArchives(FlutterVersion version) async { + // example: https://console.cloud.google.com/storage/browser/flutter_infra_release/flutter/9064459a8b0dcd32877107f6002cc429a71659d1 + final prefix = 'flutter/${await version.getEngineVersion()}/'; + + late final List resolvers; + if (version.tagName.startsWith('3.')) { + resolvers = [ + IosSymbolResolver(_symbolsBucket, prefix), + MacOSSymbolResolver(_symbolsBucket, prefix) + ]; + } else { + _log.warning('No symbol resolvers registered for ${version.tagName}'); + return []; + } + + assert(resolvers.isNotEmpty); + final archives = List.empty(growable: true); + for (var resolver in resolvers) { + final files = await resolver.listArchives(); + if (files.isEmpty) { + _log.warning( + 'Flutter ${version.tagName}: no debug symbols found by ${resolver.runtimeType}'); + } else { + _log.fine( + 'Flutter ${version.tagName}: ${resolver.runtimeType} found debug symbols: ${files.map((v) => path.basename(v))}'); + archives.addAll(files); + } + } + + return archives; + } + + Stream> download( + String path, + ) => + _symbolsBucket.read(path); +} diff --git a/scripts/flutter_symbol_collector/lib/src/flutter_version.dart b/scripts/flutter_symbol_collector/lib/src/flutter_version.dart new file mode 100644 index 0000000000..32e5775789 --- /dev/null +++ b/scripts/flutter_symbol_collector/lib/src/flutter_version.dart @@ -0,0 +1,14 @@ +import 'package:http/http.dart' as http; + +class FlutterVersion { + final String tagName; + + FlutterVersion(this.tagName); + + bool get isPreRelease => tagName.endsWith('.pre'); + + Future getEngineVersion() => http + .get(Uri.https('raw.githubusercontent.com', + 'flutter/flutter/$tagName/bin/internal/engine.version')) + .then((value) => value.body.trim()); +} diff --git a/scripts/flutter_symbol_collector/lib/src/symbol_resolver.dart b/scripts/flutter_symbol_collector/lib/src/symbol_resolver.dart new file mode 100644 index 0000000000..ea00fca3bf --- /dev/null +++ b/scripts/flutter_symbol_collector/lib/src/symbol_resolver.dart @@ -0,0 +1,48 @@ +import 'package:gcloud/storage.dart'; + +abstract class SymbolResolver { + final String _prefix; + final Bucket _bucket; + final _resolvedFiles = List.empty(growable: true); + + SymbolResolver(this._bucket, String prefix) + : _prefix = prefix.endsWith('/') + ? prefix.substring(0, prefix.length - 1) + : prefix; + + Future tryResolve(String path) async { + path = '$_prefix/$path'; + final matches = await _bucket + .list(prefix: path) + .where((v) => v.isObject) + .where((v) => v.name == path) // because it's a prefix search + .map((v) => v.name) + .toList(); + if (matches.isNotEmpty) { + _resolvedFiles.add(matches.single); + } + } + + Future> listArchives(); +} + +class IosSymbolResolver extends SymbolResolver { + IosSymbolResolver(super.bucket, super.prefix); + + @override + Future> listArchives() async { + await tryResolve('ios-release/Flutter.dSYM.zip'); + return _resolvedFiles; + } +} + +class MacOSSymbolResolver extends SymbolResolver { + MacOSSymbolResolver(super.bucket, super.prefix); + + @override + Future> listArchives() async { + // darwin-x64-release directory contains a fat (arm64+x86_64) binary. + await tryResolve('darwin-x64-release/FlutterMacOS.dSYM.zip'); + return _resolvedFiles; + } +} diff --git a/scripts/flutter_symbol_collector/pubspec.yaml b/scripts/flutter_symbol_collector/pubspec.yaml new file mode 100644 index 0000000000..b8c649a007 --- /dev/null +++ b/scripts/flutter_symbol_collector/pubspec.yaml @@ -0,0 +1,18 @@ +name: flutter_symbol_collector +description: Internal tool to collect Flutter debug symbols and upload them to Sentry +publish_to: none + +environment: + sdk: ^3.0.0 + +dependencies: + gcloud: ^0.8.11 + github: ^9.19.0 + http: ^1.1.0 + logging: ^1.2.0 + path: ^1.8.3 + + +dev_dependencies: + lints: ^2.0.0 + test: ^1.21.0 diff --git a/scripts/flutter_symbol_collector/test/downloader_test.dart b/scripts/flutter_symbol_collector/test/downloader_test.dart new file mode 100644 index 0000000000..eb622ecb7e --- /dev/null +++ b/scripts/flutter_symbol_collector/test/downloader_test.dart @@ -0,0 +1,78 @@ +import 'package:flutter_symbol_collector/flutter_symbol_collector.dart'; +import 'package:logging/logging.dart'; +import 'package:test/test.dart'; + +void main() { + Logger.root.level = Level.ALL; + late FlutterSymbolDownloader sut; + + setUp(() { + sut = FlutterSymbolDownloader(); + }); + + test('$FlutterVersion.isPrerelease()', () async { + expect(FlutterVersion('v1.16.3').isPreRelease, false); + expect(FlutterVersion('v1.16.3.pre').isPreRelease, true); + expect(FlutterVersion('3.16.0-9.0').isPreRelease, false); + expect(FlutterVersion('3.16.0-9.0.pre').isPreRelease, true); + }); + + test('listFlutterVersions() returns a stable list', () async { + final versions = await sut.listFlutterVersions().take(3).toList(); + expect(versions.map((v) => v.tagName), + equals(['v1.16.3', 'v1.16.2', 'v1.16.1'])); + }); + + test('listFlutterVersions() fetches items across multiple API page requests', + () async { + // the page size defaults to 30 at the moment, see listFlutterVersions() + final versions = await sut.listFlutterVersions().take(105).toList(); + expect(versions.length, equals(105)); + }); + + test('Engine versions match expected values', () async { + final versions = await sut.listFlutterVersions().take(3).toList(); + final engines = List.empty(growable: true); + for (var v in versions) { + engines.add("${v.tagName} => ${await v.getEngineVersion()}"); + } + expect( + engines, + equals([ + 'v1.16.3 => b2bdeb3f0f1683f3e0562f491b5e316240dfbc2c', + 'v1.16.2 => 2d42c74a348d98d2fd372a91953c104e58f185cd', + 'v1.16.1 => 216c420a2c06e5266a60a768b3fd0b660551cc9c' + ])); + }); + + test('listSymbolArchives() supports expected platforms', () async { + final archives = await sut.listSymbolArchives(FlutterVersion('3.13.4')); + const prefix = 'flutter/9064459a8b0dcd32877107f6002cc429a71659d1'; + expect( + archives, + equals([ + '$prefix/ios-release/Flutter.dSYM.zip', + '$prefix/darwin-x64-release/FlutterMacOS.dSYM.zip' + ])); + }); + + test('listSymbolArchives() supports expected platforms', () async { + final archives = await sut.listSymbolArchives(FlutterVersion('3.13.4')); + const prefix = 'flutter/9064459a8b0dcd32877107f6002cc429a71659d1'; + expect( + archives, + equals([ + '$prefix/ios-release/Flutter.dSYM.zip', + '$prefix/darwin-x64-release/FlutterMacOS.dSYM.zip' + ])); + }); + + test('download() downloads the file', () async { + // No need to download a large archive, just some small file to test this. + final content = await sut + .download('test.txt') + .map(String.fromCharCodes) + .reduce((a, b) => '$a$b'); + expect(content, equals('test\n')); + }); +} From d1313bdf34c74e344b7c7225ab11cf21fba4310e Mon Sep 17 00:00:00 2001 From: Ivan Dlugos Date: Thu, 12 Oct 2023 08:36:54 +0200 Subject: [PATCH 02/47] renames --- .../bin/flutter_symbol_collector.dart | 2 +- .../lib/flutter_symbol_collector.dart | 2 +- ...ol_resolver.dart => flutter_symbol_resolver.dart} | 8 ++++---- .../{downloader.dart => flutter_symbol_source.dart} | 8 ++++---- ...der_test.dart => flutter_symbol_source_test.dart} | 11 ++--------- .../test/flutter_version_test..dart | 12 ++++++++++++ 6 files changed, 24 insertions(+), 19 deletions(-) rename scripts/flutter_symbol_collector/lib/src/{symbol_resolver.dart => flutter_symbol_resolver.dart} (84%) rename scripts/flutter_symbol_collector/lib/src/{downloader.dart => flutter_symbol_source.dart} (90%) rename scripts/flutter_symbol_collector/test/{downloader_test.dart => flutter_symbol_source_test.dart} (86%) create mode 100644 scripts/flutter_symbol_collector/test/flutter_version_test..dart diff --git a/scripts/flutter_symbol_collector/bin/flutter_symbol_collector.dart b/scripts/flutter_symbol_collector/bin/flutter_symbol_collector.dart index 2a14ea89e2..b18031f19b 100644 --- a/scripts/flutter_symbol_collector/bin/flutter_symbol_collector.dart +++ b/scripts/flutter_symbol_collector/bin/flutter_symbol_collector.dart @@ -1,5 +1,5 @@ import 'package:flutter_symbol_collector/flutter_symbol_collector.dart'; void main(List arguments) { - final downloader = FlutterSymbolDownloader(); + final source = FlutterSymbolSource(); } diff --git a/scripts/flutter_symbol_collector/lib/flutter_symbol_collector.dart b/scripts/flutter_symbol_collector/lib/flutter_symbol_collector.dart index 9dea2540d7..16f6899c79 100644 --- a/scripts/flutter_symbol_collector/lib/flutter_symbol_collector.dart +++ b/scripts/flutter_symbol_collector/lib/flutter_symbol_collector.dart @@ -1,2 +1,2 @@ -export 'src/downloader.dart'; +export 'src/flutter_symbol_source.dart'; export 'src/flutter_version.dart'; diff --git a/scripts/flutter_symbol_collector/lib/src/symbol_resolver.dart b/scripts/flutter_symbol_collector/lib/src/flutter_symbol_resolver.dart similarity index 84% rename from scripts/flutter_symbol_collector/lib/src/symbol_resolver.dart rename to scripts/flutter_symbol_collector/lib/src/flutter_symbol_resolver.dart index ea00fca3bf..a790490be1 100644 --- a/scripts/flutter_symbol_collector/lib/src/symbol_resolver.dart +++ b/scripts/flutter_symbol_collector/lib/src/flutter_symbol_resolver.dart @@ -1,11 +1,11 @@ import 'package:gcloud/storage.dart'; -abstract class SymbolResolver { +abstract class FlutterSymbolResolver { final String _prefix; final Bucket _bucket; final _resolvedFiles = List.empty(growable: true); - SymbolResolver(this._bucket, String prefix) + FlutterSymbolResolver(this._bucket, String prefix) : _prefix = prefix.endsWith('/') ? prefix.substring(0, prefix.length - 1) : prefix; @@ -26,7 +26,7 @@ abstract class SymbolResolver { Future> listArchives(); } -class IosSymbolResolver extends SymbolResolver { +class IosSymbolResolver extends FlutterSymbolResolver { IosSymbolResolver(super.bucket, super.prefix); @override @@ -36,7 +36,7 @@ class IosSymbolResolver extends SymbolResolver { } } -class MacOSSymbolResolver extends SymbolResolver { +class MacOSSymbolResolver extends FlutterSymbolResolver { MacOSSymbolResolver(super.bucket, super.prefix); @override diff --git a/scripts/flutter_symbol_collector/lib/src/downloader.dart b/scripts/flutter_symbol_collector/lib/src/flutter_symbol_source.dart similarity index 90% rename from scripts/flutter_symbol_collector/lib/src/downloader.dart rename to scripts/flutter_symbol_collector/lib/src/flutter_symbol_source.dart index 0fe98439a2..941df45225 100644 --- a/scripts/flutter_symbol_collector/lib/src/downloader.dart +++ b/scripts/flutter_symbol_collector/lib/src/flutter_symbol_source.dart @@ -1,19 +1,19 @@ import 'package:flutter_symbol_collector/src/flutter_version.dart'; -import 'package:flutter_symbol_collector/src/symbol_resolver.dart'; +import 'package:flutter_symbol_collector/src/flutter_symbol_resolver.dart'; import 'package:github/github.dart'; import 'package:gcloud/storage.dart'; import 'package:http/http.dart'; import 'package:logging/logging.dart'; import 'package:path/path.dart' as path; -class FlutterSymbolDownloader { +class FlutterSymbolSource { late final Logger _log; final _github = GitHub(); late final _flutterRepo = RepositorySlug('flutter', 'flutter'); late final _symbolsBucket = Storage(Client(), '').bucket('flutter_infra_release'); - FlutterSymbolDownloader({Logger? logger}) { + FlutterSymbolSource({Logger? logger}) { _log = logger ?? Logger.root; } @@ -25,7 +25,7 @@ class FlutterSymbolDownloader { // example: https://console.cloud.google.com/storage/browser/flutter_infra_release/flutter/9064459a8b0dcd32877107f6002cc429a71659d1 final prefix = 'flutter/${await version.getEngineVersion()}/'; - late final List resolvers; + late final List resolvers; if (version.tagName.startsWith('3.')) { resolvers = [ IosSymbolResolver(_symbolsBucket, prefix), diff --git a/scripts/flutter_symbol_collector/test/downloader_test.dart b/scripts/flutter_symbol_collector/test/flutter_symbol_source_test.dart similarity index 86% rename from scripts/flutter_symbol_collector/test/downloader_test.dart rename to scripts/flutter_symbol_collector/test/flutter_symbol_source_test.dart index eb622ecb7e..b9b72105b6 100644 --- a/scripts/flutter_symbol_collector/test/downloader_test.dart +++ b/scripts/flutter_symbol_collector/test/flutter_symbol_source_test.dart @@ -4,17 +4,10 @@ import 'package:test/test.dart'; void main() { Logger.root.level = Level.ALL; - late FlutterSymbolDownloader sut; + late FlutterSymbolSource sut; setUp(() { - sut = FlutterSymbolDownloader(); - }); - - test('$FlutterVersion.isPrerelease()', () async { - expect(FlutterVersion('v1.16.3').isPreRelease, false); - expect(FlutterVersion('v1.16.3.pre').isPreRelease, true); - expect(FlutterVersion('3.16.0-9.0').isPreRelease, false); - expect(FlutterVersion('3.16.0-9.0.pre').isPreRelease, true); + sut = FlutterSymbolSource(); }); test('listFlutterVersions() returns a stable list', () async { diff --git a/scripts/flutter_symbol_collector/test/flutter_version_test..dart b/scripts/flutter_symbol_collector/test/flutter_version_test..dart new file mode 100644 index 0000000000..024505742c --- /dev/null +++ b/scripts/flutter_symbol_collector/test/flutter_version_test..dart @@ -0,0 +1,12 @@ +import 'package:flutter_symbol_collector/flutter_symbol_collector.dart'; +import 'package:logging/logging.dart'; +import 'package:test/test.dart'; + +void main() { + test('$FlutterVersion.isPrerelease()', () async { + expect(FlutterVersion('v1.16.3').isPreRelease, false); + expect(FlutterVersion('v1.16.3.pre').isPreRelease, true); + expect(FlutterVersion('3.16.0-9.0').isPreRelease, false); + expect(FlutterVersion('3.16.0-9.0.pre').isPreRelease, true); + }); +} From 7a0c73ebf41e2809638dcadc5839185506820889 Mon Sep 17 00:00:00 2001 From: Ivan Dlugos Date: Thu, 12 Oct 2023 11:00:50 +0200 Subject: [PATCH 03/47] symbol collector CLI integration --- .../lib/src/symbol_collector_cli.dart | 96 +++++++++++++++++++ scripts/flutter_symbol_collector/pubspec.yaml | 5 + .../flutter_symbol_collector/test/common.dart | 8 ++ .../test/flutter_symbol_source_test.dart | 5 +- .../test/flutter_version_test..dart | 1 - .../test/symbol_collector_cli_test.dart | 42 ++++++++ 6 files changed, 154 insertions(+), 3 deletions(-) create mode 100644 scripts/flutter_symbol_collector/lib/src/symbol_collector_cli.dart create mode 100644 scripts/flutter_symbol_collector/test/common.dart create mode 100644 scripts/flutter_symbol_collector/test/symbol_collector_cli_test.dart diff --git a/scripts/flutter_symbol_collector/lib/src/symbol_collector_cli.dart b/scripts/flutter_symbol_collector/lib/src/symbol_collector_cli.dart new file mode 100644 index 0000000000..9311d073ae --- /dev/null +++ b/scripts/flutter_symbol_collector/lib/src/symbol_collector_cli.dart @@ -0,0 +1,96 @@ +import 'dart:io'; + +import 'package:archive/archive.dart'; +import 'package:archive/archive_io.dart'; +import 'package:file/file.dart'; +import 'package:http/http.dart' as http; +import 'package:logging/logging.dart'; +import 'package:meta/meta.dart'; +import 'package:platform/platform.dart'; +import 'package:posix/posix.dart' as posix; + +class SymbolCollectorCli { + late final Logger _log = Logger.root; + late bool _isExecutable; + + // https://github.com/getsentry/symbol-collector/releases + @visibleForTesting + static const version = '1.12.0'; + + @visibleForTesting + late final String cli; + + @visibleForTesting + static Platform platform = LocalPlatform(); + + SymbolCollectorCli._(); + + // Downloads the CLI to the given temporary directory and prepares it for use. + static Future setup(Directory tempDir) async { + late final String platformIdentifier; + final executableName = 'symbol-collector'; + + if (platform.isLinux) { + platformIdentifier = 'linux-x64'; + } else if (platform.isMacOS) { + platformIdentifier = 'osx-x64'; + } else { + throw UnsupportedError( + 'Cannot run symbol-collector CLI on this platform - there\'s no binary available at this time.'); + } + + final self = SymbolCollectorCli._(); + + self._log.fine( + 'Downloading symbol-collector CLI v$version for $platformIdentifier'); + final zipData = await http.readBytes(Uri.parse( + 'https://github.com/getsentry/symbol-collector/releases/download/$version/symbolcollector-console-$platformIdentifier.zip')); + self._log.fine( + 'Download successful, received ${zipData.length} bytes; extracting the archive'); + + final archive = ZipDecoder().decodeBytes(zipData); + final stream = OutputStream(); + archive.single.writeContent(stream); + stream.flush(); + + await tempDir.create(); + final executableFile = await tempDir.childFile(executableName).create(); + self.cli = executableFile.path; + + executableFile.writeAsBytes(stream.getBytes(), flush: true); + self._log.fine( + 'Symbol-collector CLI extracted to ${executableFile.path}: ${await executableFile.length()} bytes'); + self._isExecutable = platform.isWindows; + return self; + } + + void _makeExecutable() { + if (!_isExecutable) { + _isExecutable = true; + if (LocalPlatform().operatingSystem == platform.operatingSystem) { + if (platform.isLinux || platform.isMacOS) { + _log.fine('Making Symbol-collector CLI executable (chmod +x)'); + + posix.chmod(cli, '0666'); + } + } else { + _log.warning( + 'Symbol-collector CLI has been run with a platform that is not the current OS platform.' + 'This should only be done in tests because we can\'t execute the downloaded program'); + } + } + } + + Future getVersion() => _execute(['--version', '-h']); + + Future _execute(List arguments) async { + var result = await Process.run(cli, arguments); + if (result.exitCode != 0) { + _log.shout( + 'Symbol-collector CLI failed to execute $arguments with exit code ${result.exitCode}.'); + _log.shout('Stderr: ${result.stderr}'); + _log.shout('Stdout: ${result.stdout}'); + } + return result.stdout; + } +} diff --git a/scripts/flutter_symbol_collector/pubspec.yaml b/scripts/flutter_symbol_collector/pubspec.yaml index b8c649a007..2285b6a22b 100644 --- a/scripts/flutter_symbol_collector/pubspec.yaml +++ b/scripts/flutter_symbol_collector/pubspec.yaml @@ -6,11 +6,16 @@ environment: sdk: ^3.0.0 dependencies: + archive: ^3.4.6 + file: ^7.0.0 gcloud: ^0.8.11 github: ^9.19.0 http: ^1.1.0 logging: ^1.2.0 + meta: ^1.11.0 path: ^1.8.3 + platform: ^3.1.3 + posix: ^5.0.0 dev_dependencies: diff --git a/scripts/flutter_symbol_collector/test/common.dart b/scripts/flutter_symbol_collector/test/common.dart new file mode 100644 index 0000000000..4d4a42ff08 --- /dev/null +++ b/scripts/flutter_symbol_collector/test/common.dart @@ -0,0 +1,8 @@ +import 'package:logging/logging.dart'; + +void setupLogging() { + Logger.root.level = Level.ALL; + Logger.root.onRecord.listen((record) { + print('${record.level.name}: ${record.time}: ${record.message}'); + }); +} diff --git a/scripts/flutter_symbol_collector/test/flutter_symbol_source_test.dart b/scripts/flutter_symbol_collector/test/flutter_symbol_source_test.dart index b9b72105b6..83efa35fd8 100644 --- a/scripts/flutter_symbol_collector/test/flutter_symbol_source_test.dart +++ b/scripts/flutter_symbol_collector/test/flutter_symbol_source_test.dart @@ -1,9 +1,10 @@ import 'package:flutter_symbol_collector/flutter_symbol_collector.dart'; -import 'package:logging/logging.dart'; import 'package:test/test.dart'; +import 'common.dart'; + void main() { - Logger.root.level = Level.ALL; + setupLogging(); late FlutterSymbolSource sut; setUp(() { diff --git a/scripts/flutter_symbol_collector/test/flutter_version_test..dart b/scripts/flutter_symbol_collector/test/flutter_version_test..dart index 024505742c..5786292dc0 100644 --- a/scripts/flutter_symbol_collector/test/flutter_version_test..dart +++ b/scripts/flutter_symbol_collector/test/flutter_version_test..dart @@ -1,5 +1,4 @@ import 'package:flutter_symbol_collector/flutter_symbol_collector.dart'; -import 'package:logging/logging.dart'; import 'package:test/test.dart'; void main() { diff --git a/scripts/flutter_symbol_collector/test/symbol_collector_cli_test.dart b/scripts/flutter_symbol_collector/test/symbol_collector_cli_test.dart new file mode 100644 index 0000000000..522a2e5f73 --- /dev/null +++ b/scripts/flutter_symbol_collector/test/symbol_collector_cli_test.dart @@ -0,0 +1,42 @@ +import 'package:file/file.dart'; +import 'package:file/memory.dart'; +import 'package:flutter_symbol_collector/src/symbol_collector_cli.dart'; +import 'package:platform/platform.dart'; +import 'package:test/test.dart'; + +import 'common.dart'; + +void main() { + setupLogging(); + late FileSystem fs; + + setUp(() { + fs = MemoryFileSystem.test(); + }); + + group('setup() downloads CLI on', () { + for (final platform in [Platform.macOS, Platform.linux]) { + test(platform, () async { + const path = 'temp/symbol-collector'; + + // make sure the file is overwritten if there's an older version + await fs + .file(path) + .create(recursive: true) + .then((file) => file.writeAsString('foo')); + expect(fs.file(path).lengthSync(), equals(3)); + + SymbolCollectorCli.platform = FakePlatform(operatingSystem: platform); + final sut = await SymbolCollectorCli.setup(fs.directory('temp')); + expect(sut.cli, equals(path)); + expect(fs.file(path).existsSync(), isTrue); + expect(fs.file(path).lengthSync(), greaterThan(1000000)); + }); + } + }); + + test('exec() works', () async { + final sut = await SymbolCollectorCli.setup(fs.currentDirectory); + expect(sut.getVersion(), startsWith('${SymbolCollectorCli.version}+')); + }, skip: LocalPlatform().isWindows); +} From ac1c1c0bc4252771544d66217787d0e53a5b53ff Mon Sep 17 00:00:00 2001 From: Ivan Dlugos Date: Thu, 12 Oct 2023 11:37:07 +0200 Subject: [PATCH 04/47] fixup getVersion() --- .../analysis_options.yaml | 33 +++---------------- .../lib/src/symbol_collector_cli.dart | 9 ++--- .../test/symbol_collector_cli_test.dart | 18 ++++++++-- 3 files changed, 25 insertions(+), 35 deletions(-) diff --git a/scripts/flutter_symbol_collector/analysis_options.yaml b/scripts/flutter_symbol_collector/analysis_options.yaml index dee8927aaf..9fe3182f8d 100644 --- a/scripts/flutter_symbol_collector/analysis_options.yaml +++ b/scripts/flutter_symbol_collector/analysis_options.yaml @@ -1,30 +1,7 @@ -# This file configures the static analysis results for your project (errors, -# warnings, and lints). -# -# This enables the 'recommended' set of lints from `package:lints`. -# This set helps identify many issues that may lead to problems when running -# or consuming Dart code, and enforces writing Dart using a single, idiomatic -# style and format. -# -# If you want a smaller set of lints you can change this to specify -# 'package:lints/core.yaml'. These are just the most critical lints -# (the recommended set includes the core lints). -# The core lints are also what is used by pub.dev for scoring packages. - include: package:lints/recommended.yaml -# Uncomment the following section to specify additional rules. - -# linter: -# rules: -# - camel_case_types - -# analyzer: -# exclude: -# - path/to/excluded/files/** - -# For more information about the core and recommended set of lints, see -# https://dart.dev/go/core-lints - -# For additional information about configuring this file, see -# https://dart.dev/guides/language/analysis-options +linter: + rules: + prefer_relative_imports: true + unnecessary_brace_in_string_interps: true + unawaited_futures: true diff --git a/scripts/flutter_symbol_collector/lib/src/symbol_collector_cli.dart b/scripts/flutter_symbol_collector/lib/src/symbol_collector_cli.dart index 9311d073ae..f62cb99eea 100644 --- a/scripts/flutter_symbol_collector/lib/src/symbol_collector_cli.dart +++ b/scripts/flutter_symbol_collector/lib/src/symbol_collector_cli.dart @@ -57,21 +57,21 @@ class SymbolCollectorCli { final executableFile = await tempDir.childFile(executableName).create(); self.cli = executableFile.path; - executableFile.writeAsBytes(stream.getBytes(), flush: true); + await executableFile.writeAsBytes(stream.getBytes(), flush: true); self._log.fine( 'Symbol-collector CLI extracted to ${executableFile.path}: ${await executableFile.length()} bytes'); self._isExecutable = platform.isWindows; return self; } - void _makeExecutable() { + void _ensureIsExecutable() { if (!_isExecutable) { _isExecutable = true; if (LocalPlatform().operatingSystem == platform.operatingSystem) { if (platform.isLinux || platform.isMacOS) { _log.fine('Making Symbol-collector CLI executable (chmod +x)'); - posix.chmod(cli, '0666'); + posix.chmod(cli, '0700'); } } else { _log.warning( @@ -84,6 +84,7 @@ class SymbolCollectorCli { Future getVersion() => _execute(['--version', '-h']); Future _execute(List arguments) async { + _ensureIsExecutable(); var result = await Process.run(cli, arguments); if (result.exitCode != 0) { _log.shout( @@ -91,6 +92,6 @@ class SymbolCollectorCli { _log.shout('Stderr: ${result.stderr}'); _log.shout('Stdout: ${result.stdout}'); } - return result.stdout; + return result.stdout.trim(); } } diff --git a/scripts/flutter_symbol_collector/test/symbol_collector_cli_test.dart b/scripts/flutter_symbol_collector/test/symbol_collector_cli_test.dart index 522a2e5f73..cecbd7ebed 100644 --- a/scripts/flutter_symbol_collector/test/symbol_collector_cli_test.dart +++ b/scripts/flutter_symbol_collector/test/symbol_collector_cli_test.dart @@ -1,4 +1,5 @@ import 'package:file/file.dart'; +import 'package:file/local.dart'; import 'package:file/memory.dart'; import 'package:flutter_symbol_collector/src/symbol_collector_cli.dart'; import 'package:platform/platform.dart'; @@ -35,8 +36,19 @@ void main() { } }); - test('exec() works', () async { - final sut = await SymbolCollectorCli.setup(fs.currentDirectory); - expect(sut.getVersion(), startsWith('${SymbolCollectorCli.version}+')); + group('execute', () { + final tmpDir = LocalFileSystem() + .systemTempDirectory + .createTempSync('symbol_collector_test'); + late final SymbolCollectorCli sut; + + setUp(() async => sut = await SymbolCollectorCli.setup(tmpDir)); + tearDown(() => tmpDir.delete(recursive: true)); + + test('getVersion()', () async { + final versionString = await sut.getVersion(); + expect(versionString, startsWith('${SymbolCollectorCli.version}+')); + expect(versionString.split("\n").length, equals(1)); + }); }, skip: LocalPlatform().isWindows); } From 7218df6b9046ae4b2b160918e44b98b66fdfeef5 Mon Sep 17 00:00:00 2001 From: Ivan Dlugos Date: Thu, 12 Oct 2023 12:53:55 +0200 Subject: [PATCH 05/47] collector upload --- .../lib/src/flutter_symbol_source.dart | 5 +- .../lib/src/symbol_collector_cli.dart | 58 ++++++++++++++++--- ...n_test..dart => flutter_version_test.dart} | 0 .../test/symbol_collector_cli_test.dart | 19 +++++- 4 files changed, 70 insertions(+), 12 deletions(-) rename scripts/flutter_symbol_collector/test/{flutter_version_test..dart => flutter_version_test.dart} (100%) diff --git a/scripts/flutter_symbol_collector/lib/src/flutter_symbol_source.dart b/scripts/flutter_symbol_collector/lib/src/flutter_symbol_source.dart index 941df45225..ae9278d64f 100644 --- a/scripts/flutter_symbol_collector/lib/src/flutter_symbol_source.dart +++ b/scripts/flutter_symbol_collector/lib/src/flutter_symbol_source.dart @@ -1,11 +1,12 @@ -import 'package:flutter_symbol_collector/src/flutter_version.dart'; -import 'package:flutter_symbol_collector/src/flutter_symbol_resolver.dart'; import 'package:github/github.dart'; import 'package:gcloud/storage.dart'; import 'package:http/http.dart'; import 'package:logging/logging.dart'; import 'package:path/path.dart' as path; +import 'flutter_version.dart'; +import 'flutter_symbol_resolver.dart'; + class FlutterSymbolSource { late final Logger _log; final _github = GitHub(); diff --git a/scripts/flutter_symbol_collector/lib/src/symbol_collector_cli.dart b/scripts/flutter_symbol_collector/lib/src/symbol_collector_cli.dart index f62cb99eea..8ad81cfd1a 100644 --- a/scripts/flutter_symbol_collector/lib/src/symbol_collector_cli.dart +++ b/scripts/flutter_symbol_collector/lib/src/symbol_collector_cli.dart @@ -1,3 +1,4 @@ +import 'dart:convert'; import 'dart:io'; import 'package:archive/archive.dart'; @@ -8,6 +9,9 @@ import 'package:logging/logging.dart'; import 'package:meta/meta.dart'; import 'package:platform/platform.dart'; import 'package:posix/posix.dart' as posix; +import 'package:path/path.dart' as path; + +import 'flutter_version.dart'; class SymbolCollectorCli { late final Logger _log = Logger.root; @@ -83,15 +87,55 @@ class SymbolCollectorCli { Future getVersion() => _execute(['--version', '-h']); + Future upload( + Directory dir, Platform symbolsPlatform, FlutterVersion flutterVersion, + {bool dryRun = false}) { + final type = symbolsPlatform.operatingSystem; + return _execute([ + '--upload', + 'directory', + '--path', + dir.path, + '--batch-type', + type, + '--bundle-id', + 'flutter-${flutterVersion.tagName}-$type', + '--server-endpoint', + 'https://symbol-collector.services.sentry.io/', + ]); + } + Future _execute(List arguments) async { _ensureIsExecutable(); - var result = await Process.run(cli, arguments); - if (result.exitCode != 0) { - _log.shout( - 'Symbol-collector CLI failed to execute $arguments with exit code ${result.exitCode}.'); - _log.shout('Stderr: ${result.stderr}'); - _log.shout('Stdout: ${result.stdout}'); + + _log.fine('Executing ${path.basename(cli)} $arguments'); + final process = await Process.start(cli, arguments); + + final output = StringBuffer(); + handleOutput(Level level, String message) { + message.trimRight().split('\n').forEach((s) => _log.log(level, ' $s')); + output.write(message); + } + + final pipes = [ + process.stdout + .transform(utf8.decoder) + .forEach((s) => handleOutput(Level.FINER, s)), + process.stderr + .transform(utf8.decoder) + .forEach((s) => handleOutput(Level.SEVERE, s)) + ]; + + final exitCode = await process.exitCode; + await Future.wait(pipes); + final strOutput = output.toString().trimRight(); + if (exitCode != 0) { + throw Exception('Symbol-collector CLI failed with exit code $exitCode.'); + } else if (strOutput.contains('Exception:')) { + // see https://github.com/getsentry/symbol-collector/issues/162 + throw Exception('Symbol-collector CLI failed with an exception.'); } - return result.stdout.trim(); + + return strOutput; } } diff --git a/scripts/flutter_symbol_collector/test/flutter_version_test..dart b/scripts/flutter_symbol_collector/test/flutter_version_test.dart similarity index 100% rename from scripts/flutter_symbol_collector/test/flutter_version_test..dart rename to scripts/flutter_symbol_collector/test/flutter_version_test.dart diff --git a/scripts/flutter_symbol_collector/test/symbol_collector_cli_test.dart b/scripts/flutter_symbol_collector/test/symbol_collector_cli_test.dart index cecbd7ebed..aef11629e1 100644 --- a/scripts/flutter_symbol_collector/test/symbol_collector_cli_test.dart +++ b/scripts/flutter_symbol_collector/test/symbol_collector_cli_test.dart @@ -1,6 +1,7 @@ import 'package:file/file.dart'; import 'package:file/local.dart'; import 'package:file/memory.dart'; +import 'package:flutter_symbol_collector/src/flutter_version.dart'; import 'package:flutter_symbol_collector/src/symbol_collector_cli.dart'; import 'package:platform/platform.dart'; import 'package:test/test.dart'; @@ -46,9 +47,21 @@ void main() { tearDown(() => tmpDir.delete(recursive: true)); test('getVersion()', () async { - final versionString = await sut.getVersion(); - expect(versionString, startsWith('${SymbolCollectorCli.version}+')); - expect(versionString.split("\n").length, equals(1)); + final output = await sut.getVersion(); + expect(output, startsWith('${SymbolCollectorCli.version}+')); + expect(output.split("\n").length, equals(1)); + }); + + test('upload()', () async { + final uploadDir = LocalFileSystem() + .systemTempDirectory + .createTempSync('symbol_collector_upload_test'); + try { + await sut.upload( + uploadDir, LocalPlatform(), FlutterVersion('v0.0.0-test')); + } finally { + uploadDir.deleteSync(); + } }); }, skip: LocalPlatform().isWindows); } From d3941818f59cad11b610c6129d901673540f6c96 Mon Sep 17 00:00:00 2001 From: Ivan Dlugos Date: Thu, 12 Oct 2023 21:44:32 +0200 Subject: [PATCH 06/47] collect platform info with symbol archive --- .../lib/src/flutter_symbol_resolver.dart | 20 ++++++++++++++----- .../lib/src/flutter_symbol_source.dart | 12 +++++------ .../lib/src/symbol_archive.dart | 8 ++++++++ .../test/flutter_symbol_source_test.dart | 17 +++------------- 4 files changed, 31 insertions(+), 26 deletions(-) create mode 100644 scripts/flutter_symbol_collector/lib/src/symbol_archive.dart diff --git a/scripts/flutter_symbol_collector/lib/src/flutter_symbol_resolver.dart b/scripts/flutter_symbol_collector/lib/src/flutter_symbol_resolver.dart index a790490be1..889cf515dd 100644 --- a/scripts/flutter_symbol_collector/lib/src/flutter_symbol_resolver.dart +++ b/scripts/flutter_symbol_collector/lib/src/flutter_symbol_resolver.dart @@ -1,9 +1,13 @@ import 'package:gcloud/storage.dart'; +import 'package:platform/platform.dart'; + +import 'symbol_archive.dart'; abstract class FlutterSymbolResolver { final String _prefix; final Bucket _bucket; - final _resolvedFiles = List.empty(growable: true); + final _resolvedFiles = List.empty(growable: true); + Platform get platform; FlutterSymbolResolver(this._bucket, String prefix) : _prefix = prefix.endsWith('/') @@ -19,18 +23,21 @@ abstract class FlutterSymbolResolver { .map((v) => v.name) .toList(); if (matches.isNotEmpty) { - _resolvedFiles.add(matches.single); + _resolvedFiles.add(SymbolArchive(matches.single, platform)); } } - Future> listArchives(); + Future> listArchives(); } class IosSymbolResolver extends FlutterSymbolResolver { IosSymbolResolver(super.bucket, super.prefix); @override - Future> listArchives() async { + final platform = FakePlatform(operatingSystem: Platform.iOS); + + @override + Future> listArchives() async { await tryResolve('ios-release/Flutter.dSYM.zip'); return _resolvedFiles; } @@ -40,7 +47,10 @@ class MacOSSymbolResolver extends FlutterSymbolResolver { MacOSSymbolResolver(super.bucket, super.prefix); @override - Future> listArchives() async { + final platform = FakePlatform(operatingSystem: Platform.macOS); + + @override + Future> listArchives() async { // darwin-x64-release directory contains a fat (arm64+x86_64) binary. await tryResolve('darwin-x64-release/FlutterMacOS.dSYM.zip'); return _resolvedFiles; diff --git a/scripts/flutter_symbol_collector/lib/src/flutter_symbol_source.dart b/scripts/flutter_symbol_collector/lib/src/flutter_symbol_source.dart index ae9278d64f..cb3ac67d2a 100644 --- a/scripts/flutter_symbol_collector/lib/src/flutter_symbol_source.dart +++ b/scripts/flutter_symbol_collector/lib/src/flutter_symbol_source.dart @@ -6,6 +6,7 @@ import 'package:path/path.dart' as path; import 'flutter_version.dart'; import 'flutter_symbol_resolver.dart'; +import 'symbol_archive.dart'; class FlutterSymbolSource { late final Logger _log; @@ -22,7 +23,7 @@ class FlutterSymbolSource { .listTags(_flutterRepo, perPage: 30) .map((t) => FlutterVersion(t.name)); - Future> listSymbolArchives(FlutterVersion version) async { + Future> listSymbolArchives(FlutterVersion version) async { // example: https://console.cloud.google.com/storage/browser/flutter_infra_release/flutter/9064459a8b0dcd32877107f6002cc429a71659d1 final prefix = 'flutter/${await version.getEngineVersion()}/'; @@ -38,7 +39,7 @@ class FlutterSymbolSource { } assert(resolvers.isNotEmpty); - final archives = List.empty(growable: true); + final archives = List.empty(growable: true); for (var resolver in resolvers) { final files = await resolver.listArchives(); if (files.isEmpty) { @@ -46,7 +47,7 @@ class FlutterSymbolSource { 'Flutter ${version.tagName}: no debug symbols found by ${resolver.runtimeType}'); } else { _log.fine( - 'Flutter ${version.tagName}: ${resolver.runtimeType} found debug symbols: ${files.map((v) => path.basename(v))}'); + 'Flutter ${version.tagName}: ${resolver.runtimeType} found debug symbols: ${files.map((v) => path.basename(v.path))}'); archives.addAll(files); } } @@ -54,8 +55,5 @@ class FlutterSymbolSource { return archives; } - Stream> download( - String path, - ) => - _symbolsBucket.read(path); + Stream> download(String path) => _symbolsBucket.read(path); } diff --git a/scripts/flutter_symbol_collector/lib/src/symbol_archive.dart b/scripts/flutter_symbol_collector/lib/src/symbol_archive.dart new file mode 100644 index 0000000000..99382018d9 --- /dev/null +++ b/scripts/flutter_symbol_collector/lib/src/symbol_archive.dart @@ -0,0 +1,8 @@ +import 'package:platform/platform.dart'; + +class SymbolArchive { + final String path; + final Platform platform; + + SymbolArchive(this.path, this.platform); +} diff --git a/scripts/flutter_symbol_collector/test/flutter_symbol_source_test.dart b/scripts/flutter_symbol_collector/test/flutter_symbol_source_test.dart index 83efa35fd8..a2e4c1f9f3 100644 --- a/scripts/flutter_symbol_collector/test/flutter_symbol_source_test.dart +++ b/scripts/flutter_symbol_collector/test/flutter_symbol_source_test.dart @@ -43,21 +43,10 @@ void main() { final archives = await sut.listSymbolArchives(FlutterVersion('3.13.4')); const prefix = 'flutter/9064459a8b0dcd32877107f6002cc429a71659d1'; expect( - archives, + archives.map((v) => '${v.platform.operatingSystem} - ${v.path}'), equals([ - '$prefix/ios-release/Flutter.dSYM.zip', - '$prefix/darwin-x64-release/FlutterMacOS.dSYM.zip' - ])); - }); - - test('listSymbolArchives() supports expected platforms', () async { - final archives = await sut.listSymbolArchives(FlutterVersion('3.13.4')); - const prefix = 'flutter/9064459a8b0dcd32877107f6002cc429a71659d1'; - expect( - archives, - equals([ - '$prefix/ios-release/Flutter.dSYM.zip', - '$prefix/darwin-x64-release/FlutterMacOS.dSYM.zip' + 'ios - $prefix/ios-release/Flutter.dSYM.zip', + 'macos - $prefix/darwin-x64-release/FlutterMacOS.dSYM.zip' ])); }); From 5e963aa74441eb84462b7eece1ba632c69edb80a Mon Sep 17 00:00:00 2001 From: Ivan Dlugos Date: Thu, 12 Oct 2023 22:43:06 +0200 Subject: [PATCH 07/47] download and extract zip archives --- .../lib/src/flutter_symbol_source.dart | 69 ++++++++++++++++++- .../lib/src/symbol_collector_cli.dart | 2 +- .../flutter_symbol_collector/test/common.dart | 3 +- .../test/flutter_symbol_source_test.dart | 18 +++++ 4 files changed, 89 insertions(+), 3 deletions(-) diff --git a/scripts/flutter_symbol_collector/lib/src/flutter_symbol_source.dart b/scripts/flutter_symbol_collector/lib/src/flutter_symbol_source.dart index cb3ac67d2a..a4de5cac02 100644 --- a/scripts/flutter_symbol_collector/lib/src/flutter_symbol_source.dart +++ b/scripts/flutter_symbol_collector/lib/src/flutter_symbol_source.dart @@ -1,3 +1,8 @@ +import 'dart:typed_data'; + +import 'package:archive/archive.dart'; +import 'package:archive/archive_io.dart'; +import 'package:file/file.dart'; import 'package:github/github.dart'; import 'package:gcloud/storage.dart'; import 'package:http/http.dart'; @@ -55,5 +60,67 @@ class FlutterSymbolSource { return archives; } - Stream> download(String path) => _symbolsBucket.read(path); + // Streams the remote file contents. + Stream> download(String filePath) => _symbolsBucket.read(filePath); + + // Downloads the remote file to the given target directory or if it's an + //archive, extracts the content instead. + Future downloadAndExtractTo(Directory target, String filePath) async { + await target.create(recursive: true); + + if (path.extension(filePath) == '.zip') { + try { + final buffer = BytesBuilder(); + await download(filePath).forEach(buffer.add); + final archive = ZipDecoder().decodeBytes(buffer.toBytes()); + buffer.clear(); + + // For all of the entries in the archive + for (var entry in archive.files) { + // Make sure we don't have any zip-slip issues. + final entryPath = path.normalize(entry.name); + + if (!path + .normalize(target.childFile(entryPath).path) + .startsWith(target.path)) { + throw Exception( + 'Invalid ZIP entry path (looks like a zip-slip issue): ${entry.name}'); + } + + // If it's a file and not a directory + if (entry.isFile) { + final file = await target + .childFile(path.normalize(entryPath)) + .create(exclusive: true); + + // Note: package:archive doesn't support extracting directly to an + // IOSink. See https://github.com/brendan-duncan/archive/issues/12 + final stream = OutputStream(); + entry.writeContent(stream, freeMemory: true); + stream.flush(); + + await file.writeAsBytes(stream.getBytes(), flush: true); + } else { + await target.childDirectory(entryPath).create(recursive: true); + } + } + } catch (e, trace) { + _log.warning('Failed to download $filePath to $target', e, trace); + } + } else { + final file = await target + .childFile(path.basename(filePath)) + .create(exclusive: true); + final sink = file.openWrite(); + try { + await sink.addStream(download(filePath)); + await sink.flush(); + await sink.close(); + } catch (e, trace) { + _log.warning('Failed to download $filePath to $target', e, trace); + await sink.close(); + await file.delete(); + } + } + } } diff --git a/scripts/flutter_symbol_collector/lib/src/symbol_collector_cli.dart b/scripts/flutter_symbol_collector/lib/src/symbol_collector_cli.dart index 8ad81cfd1a..e3bec5e6ac 100644 --- a/scripts/flutter_symbol_collector/lib/src/symbol_collector_cli.dart +++ b/scripts/flutter_symbol_collector/lib/src/symbol_collector_cli.dart @@ -54,7 +54,7 @@ class SymbolCollectorCli { final archive = ZipDecoder().decodeBytes(zipData); final stream = OutputStream(); - archive.single.writeContent(stream); + archive.single.writeContent(stream, freeMemory: true); stream.flush(); await tempDir.create(); diff --git a/scripts/flutter_symbol_collector/test/common.dart b/scripts/flutter_symbol_collector/test/common.dart index 4d4a42ff08..22d1877435 100644 --- a/scripts/flutter_symbol_collector/test/common.dart +++ b/scripts/flutter_symbol_collector/test/common.dart @@ -3,6 +3,7 @@ import 'package:logging/logging.dart'; void setupLogging() { Logger.root.level = Level.ALL; Logger.root.onRecord.listen((record) { - print('${record.level.name}: ${record.time}: ${record.message}'); + print('${record.level.name}: ${record.time}: ${record.message}' + '${record.error == null ? '' : ': ${record.error}'}'); }); } diff --git a/scripts/flutter_symbol_collector/test/flutter_symbol_source_test.dart b/scripts/flutter_symbol_collector/test/flutter_symbol_source_test.dart index a2e4c1f9f3..84473a1f72 100644 --- a/scripts/flutter_symbol_collector/test/flutter_symbol_source_test.dart +++ b/scripts/flutter_symbol_collector/test/flutter_symbol_source_test.dart @@ -1,3 +1,5 @@ +import 'package:file/file.dart'; +import 'package:file/memory.dart'; import 'package:flutter_symbol_collector/flutter_symbol_collector.dart'; import 'package:test/test.dart'; @@ -5,9 +7,11 @@ import 'common.dart'; void main() { setupLogging(); + late FileSystem fs; late FlutterSymbolSource sut; setUp(() { + fs = MemoryFileSystem.test(); sut = FlutterSymbolSource(); }); @@ -58,4 +62,18 @@ void main() { .reduce((a, b) => '$a$b'); expect(content, equals('test\n')); }); + + test('downloadAndExtractTo() downloads a plain file', () async { + await sut.downloadAndExtractTo(fs.currentDirectory, 'test.txt'); + expect(fs.isFileSync('test.txt'), isTrue); + expect(fs.file('test.txt').readAsStringSync(), equals('test\n')); + }); + + test('downloadAndExtractTo() extracts a zip file', () async { + await sut.downloadAndExtractTo(fs.currentDirectory, + 'flutter/0005149dca9b248663adcde4bdd7c6c915a76584/sky_engine.zip'); + expect(fs.isDirectorySync('sky_engine'), isTrue); + expect(fs.file('sky_engine/README.md').readAsStringSync(), + startsWith('Flutter Engine')); + }); } From c4aa8a81a7ed77d1ae2cae744acf2f8ded580b6a Mon Sep 17 00:00:00 2001 From: Ivan Dlugos Date: Thu, 12 Oct 2023 22:58:19 +0200 Subject: [PATCH 08/47] download via cli bin --- scripts/flutter_symbol_collector/.gitignore | 2 ++ .../bin/flutter_symbol_collector.dart | 35 ++++++++++++++++++- .../lib/src/flutter_symbol_source.dart | 16 +++++---- 3 files changed, 46 insertions(+), 7 deletions(-) diff --git a/scripts/flutter_symbol_collector/.gitignore b/scripts/flutter_symbol_collector/.gitignore index 3a85790408..2b68fd4595 100644 --- a/scripts/flutter_symbol_collector/.gitignore +++ b/scripts/flutter_symbol_collector/.gitignore @@ -1,3 +1,5 @@ # https://dart.dev/guides/libraries/private-files # Created by `dart pub` .dart_tool/ + +.temp \ No newline at end of file diff --git a/scripts/flutter_symbol_collector/bin/flutter_symbol_collector.dart b/scripts/flutter_symbol_collector/bin/flutter_symbol_collector.dart index b18031f19b..d9261576f5 100644 --- a/scripts/flutter_symbol_collector/bin/flutter_symbol_collector.dart +++ b/scripts/flutter_symbol_collector/bin/flutter_symbol_collector.dart @@ -1,5 +1,38 @@ +import 'dart:io'; + +import 'package:file/local.dart'; import 'package:flutter_symbol_collector/flutter_symbol_collector.dart'; +import 'package:logging/logging.dart'; + +late final source = FlutterSymbolSource(); +late final fs = LocalFileSystem(); +late final tempDir = fs.currentDirectory.childDirectory('.temp'); void main(List arguments) { - final source = FlutterSymbolSource(); + Logger.root.level = Level.ALL; + Logger.root.onRecord.listen((record) { + print('${record.level.name}: ${record.time}: ${record.message}' + '${record.error == null ? '' : ': ${record.error}'}'); + }); + + // source + // .listFlutterVersions() + // .where((version) => !version.isPreRelease) + // .where((version) => version.tagName.startsWith('3.')) + // .forEach(processFlutterVerion); + + processFlutterVerion(FlutterVersion('3.13.7')); +} + +Future processFlutterVerion(FlutterVersion version) async { + Logger.root.info('Processing Flutter ${version.tagName}'); + final engineVersion = await version.getEngineVersion(); + Logger.root.info('Engine version: $engineVersion'); + + final archives = await source.listSymbolArchives(version); + final dir = tempDir.childDirectory(version.tagName); + for (final archive in archives) { + final archiveDir = dir.childDirectory(archive.platform.operatingSystem); + await source.downloadAndExtractTo(archiveDir, archive.path); + } } diff --git a/scripts/flutter_symbol_collector/lib/src/flutter_symbol_source.dart b/scripts/flutter_symbol_collector/lib/src/flutter_symbol_source.dart index a4de5cac02..32ec2d97a9 100644 --- a/scripts/flutter_symbol_collector/lib/src/flutter_symbol_source.dart +++ b/scripts/flutter_symbol_collector/lib/src/flutter_symbol_source.dart @@ -52,7 +52,7 @@ class FlutterSymbolSource { 'Flutter ${version.tagName}: no debug symbols found by ${resolver.runtimeType}'); } else { _log.fine( - 'Flutter ${version.tagName}: ${resolver.runtimeType} found debug symbols: ${files.map((v) => path.basename(v.path))}'); + 'Flutter ${version.tagName}: ${resolver.runtimeType} found ${files.length} debug symbols: ${files.map((v) => path.basename(v.path))}'); archives.addAll(files); } } @@ -61,20 +61,23 @@ class FlutterSymbolSource { } // Streams the remote file contents. - Stream> download(String filePath) => _symbolsBucket.read(filePath); + Stream> download(String filePath) { + _log.fine('Downloading $filePath'); + return _symbolsBucket.read(filePath); + } // Downloads the remote file to the given target directory or if it's an //archive, extracts the content instead. Future downloadAndExtractTo(Directory target, String filePath) async { - await target.create(recursive: true); - if (path.extension(filePath) == '.zip') { + target = await target.childDirectory(filePath).create(recursive: true); try { final buffer = BytesBuilder(); await download(filePath).forEach(buffer.add); final archive = ZipDecoder().decodeBytes(buffer.toBytes()); buffer.clear(); + _log.fine('Extracting $filePath to $target'); // For all of the entries in the archive for (var entry in archive.files) { // Make sure we don't have any zip-slip issues. @@ -108,9 +111,10 @@ class FlutterSymbolSource { _log.warning('Failed to download $filePath to $target', e, trace); } } else { + _log.fine('Downloading $filePath to $target'); final file = await target - .childFile(path.basename(filePath)) - .create(exclusive: true); + .childFile(filePath) + .create(recursive: true, exclusive: true); final sink = file.openWrite(); try { await sink.addStream(download(filePath)); From a7a80eac75c3e32902d9e5f3417e2e245cf10a94 Mon Sep 17 00:00:00 2001 From: Ivan Dlugos Date: Thu, 12 Oct 2023 23:10:04 +0200 Subject: [PATCH 09/47] upload via symbol-collector cli --- .../bin/flutter_symbol_collector.dart | 16 +++++++++------- .../lib/flutter_symbol_collector.dart | 1 + .../lib/src/symbol_collector_cli.dart | 6 +++--- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/scripts/flutter_symbol_collector/bin/flutter_symbol_collector.dart b/scripts/flutter_symbol_collector/bin/flutter_symbol_collector.dart index d9261576f5..e34abfe4e9 100644 --- a/scripts/flutter_symbol_collector/bin/flutter_symbol_collector.dart +++ b/scripts/flutter_symbol_collector/bin/flutter_symbol_collector.dart @@ -1,27 +1,28 @@ -import 'dart:io'; - import 'package:file/local.dart'; import 'package:flutter_symbol_collector/flutter_symbol_collector.dart'; import 'package:logging/logging.dart'; -late final source = FlutterSymbolSource(); -late final fs = LocalFileSystem(); -late final tempDir = fs.currentDirectory.childDirectory('.temp'); +final source = FlutterSymbolSource(); +final fs = LocalFileSystem(); +final tempDir = fs.currentDirectory.childDirectory('.temp'); +late final SymbolCollectorCli collector; -void main(List arguments) { +void main(List arguments) async { Logger.root.level = Level.ALL; Logger.root.onRecord.listen((record) { print('${record.level.name}: ${record.time}: ${record.message}' '${record.error == null ? '' : ': ${record.error}'}'); }); + collector = await SymbolCollectorCli.setup(tempDir); + // source // .listFlutterVersions() // .where((version) => !version.isPreRelease) // .where((version) => version.tagName.startsWith('3.')) // .forEach(processFlutterVerion); - processFlutterVerion(FlutterVersion('3.13.7')); + await processFlutterVerion(FlutterVersion('3.13.7')); } Future processFlutterVerion(FlutterVersion version) async { @@ -34,5 +35,6 @@ Future processFlutterVerion(FlutterVersion version) async { for (final archive in archives) { final archiveDir = dir.childDirectory(archive.platform.operatingSystem); await source.downloadAndExtractTo(archiveDir, archive.path); + await collector.upload(archiveDir, archive.platform, version); } } diff --git a/scripts/flutter_symbol_collector/lib/flutter_symbol_collector.dart b/scripts/flutter_symbol_collector/lib/flutter_symbol_collector.dart index 16f6899c79..3d54e0d8e5 100644 --- a/scripts/flutter_symbol_collector/lib/flutter_symbol_collector.dart +++ b/scripts/flutter_symbol_collector/lib/flutter_symbol_collector.dart @@ -1,2 +1,3 @@ export 'src/flutter_symbol_source.dart'; export 'src/flutter_version.dart'; +export 'src/symbol_collector_cli.dart'; diff --git a/scripts/flutter_symbol_collector/lib/src/symbol_collector_cli.dart b/scripts/flutter_symbol_collector/lib/src/symbol_collector_cli.dart index e3bec5e6ac..ccb0323cb4 100644 --- a/scripts/flutter_symbol_collector/lib/src/symbol_collector_cli.dart +++ b/scripts/flutter_symbol_collector/lib/src/symbol_collector_cli.dart @@ -18,13 +18,13 @@ class SymbolCollectorCli { late bool _isExecutable; // https://github.com/getsentry/symbol-collector/releases - @visibleForTesting + @internal static const version = '1.12.0'; - @visibleForTesting + @internal late final String cli; - @visibleForTesting + @internal static Platform platform = LocalPlatform(); SymbolCollectorCli._(); From 667c144fbfff4d2b61ef16d46bbcfa7a092d9e93 Mon Sep 17 00:00:00 2001 From: Ivan Dlugos Date: Fri, 13 Oct 2023 12:45:57 +0200 Subject: [PATCH 10/47] add symbol collector CI --- .github/workflows/flutter-symbols.yml | 28 +++++++++++++ .../bin/flutter_symbol_collector.dart | 42 +++++++++++++++---- .../lib/src/flutter_symbol_source.dart | 15 ++++--- scripts/flutter_symbol_collector/pubspec.yaml | 1 + 4 files changed, 72 insertions(+), 14 deletions(-) create mode 100644 .github/workflows/flutter-symbols.yml diff --git a/.github/workflows/flutter-symbols.yml b/.github/workflows/flutter-symbols.yml new file mode 100644 index 0000000000..7227f445d3 --- /dev/null +++ b/.github/workflows/flutter-symbols.yml @@ -0,0 +1,28 @@ +name: Flutter symbols collection +on: + push: # Temporary so we get the workflow on the PR branch + workflow_dispatch: + inputs: + flutter_version: + description: Flutter version, can be either a specific version (3.17.0) or a wildcard (3.2.*) + required: false + type: string + +jobs: + run: + runs-on: ubuntu-latest + timeout-minutes: 60 + defaults: + run: + working-directory: scripts/flutter_symbol_collector + steps: + - uses: actions/checkout@v3 + + - uses: dart-lang/setup-dart@6a218f2413a3e78e9087f638a238f6b40893203d # pin@v1 + + - run: dart pub get + + # - run: dart run bin/flutter_symbol_collector.dart --version=${{ inputs.flutter_version }} + - run: dart run bin/flutter_symbol_collector.dart --version=3.13.7 + env: + GITHUB_TOKEN: ${{ github.token }} diff --git a/scripts/flutter_symbol_collector/bin/flutter_symbol_collector.dart b/scripts/flutter_symbol_collector/bin/flutter_symbol_collector.dart index e34abfe4e9..1f43aeba05 100644 --- a/scripts/flutter_symbol_collector/bin/flutter_symbol_collector.dart +++ b/scripts/flutter_symbol_collector/bin/flutter_symbol_collector.dart @@ -1,8 +1,14 @@ +import 'package:args/args.dart'; import 'package:file/local.dart'; import 'package:flutter_symbol_collector/flutter_symbol_collector.dart'; +import 'package:github/github.dart'; import 'package:logging/logging.dart'; -final source = FlutterSymbolSource(); +const githubToken = String.fromEnvironment('GITHUB_TOKEN'); +final source = FlutterSymbolSource( + githubAuth: githubToken.isEmpty + ? Authentication.anonymous() + : Authentication.withToken(githubToken)); final fs = LocalFileSystem(); final tempDir = fs.currentDirectory.childDirectory('.temp'); late final SymbolCollectorCli collector; @@ -14,15 +20,35 @@ void main(List arguments) async { '${record.error == null ? '' : ': ${record.error}'}'); }); - collector = await SymbolCollectorCli.setup(tempDir); + final parser = ArgParser()..addOption('version', defaultsTo: ''); + final args = parser.parse(arguments); + final argVersion = args['version'] as String; - // source - // .listFlutterVersions() - // .where((version) => !version.isPreRelease) - // .where((version) => version.tagName.startsWith('3.')) - // .forEach(processFlutterVerion); + collector = await SymbolCollectorCli.setup(tempDir); - await processFlutterVerion(FlutterVersion('3.13.7')); + // If a specific version was given, run just for this version. + if (argVersion.isNotEmpty && + !argVersion.contains('*') && + argVersion.split('.').length == 3) { + Logger.root.info('Running for a single flutter version: $argVersion'); + await processFlutterVerion(FlutterVersion(argVersion)); + } else { + // Otherwise, walk all the versions and run for the matching ones. + final versionRegex = RegExp(argVersion.isEmpty + ? '.*' + : '^${argVersion.replaceAll('.', '\\.').replaceAll('*', '.+')}\$'); + Logger.root.info('Running for all Flutter versions matching $versionRegex'); + final versions = await source + .listFlutterVersions() + .where((v) => !v.isPreRelease) + .where((v) => versionRegex.hasMatch(v.tagName)) + .toList(); + Logger.root.info( + 'Found ${versions.length} Flutter versions matching $versionRegex'); + for (var version in versions) { + await processFlutterVerion(version); + } + } } Future processFlutterVerion(FlutterVersion version) async { diff --git a/scripts/flutter_symbol_collector/lib/src/flutter_symbol_source.dart b/scripts/flutter_symbol_collector/lib/src/flutter_symbol_source.dart index 32ec2d97a9..778ea18837 100644 --- a/scripts/flutter_symbol_collector/lib/src/flutter_symbol_source.dart +++ b/scripts/flutter_symbol_collector/lib/src/flutter_symbol_source.dart @@ -3,7 +3,7 @@ import 'dart:typed_data'; import 'package:archive/archive.dart'; import 'package:archive/archive_io.dart'; import 'package:file/file.dart'; -import 'package:github/github.dart'; +import 'package:github/github.dart' as github; import 'package:gcloud/storage.dart'; import 'package:http/http.dart'; import 'package:logging/logging.dart'; @@ -15,14 +15,17 @@ import 'symbol_archive.dart'; class FlutterSymbolSource { late final Logger _log; - final _github = GitHub(); - late final _flutterRepo = RepositorySlug('flutter', 'flutter'); + final github.GitHub _github; + late final _flutterRepo = github.RepositorySlug('flutter', 'flutter'); late final _symbolsBucket = Storage(Client(), '').bucket('flutter_infra_release'); - FlutterSymbolSource({Logger? logger}) { - _log = logger ?? Logger.root; - } + FlutterSymbolSource( + {Logger? logger, + github.Authentication githubAuth = + const github.Authentication.anonymous()}) + : _log = logger ?? Logger.root, + _github = github.GitHub(auth: githubAuth); Stream listFlutterVersions() => _github.repositories .listTags(_flutterRepo, perPage: 30) diff --git a/scripts/flutter_symbol_collector/pubspec.yaml b/scripts/flutter_symbol_collector/pubspec.yaml index 2285b6a22b..0464e16e0d 100644 --- a/scripts/flutter_symbol_collector/pubspec.yaml +++ b/scripts/flutter_symbol_collector/pubspec.yaml @@ -7,6 +7,7 @@ environment: dependencies: archive: ^3.4.6 + args: ^2.4.2 file: ^7.0.0 gcloud: ^0.8.11 github: ^9.19.0 From 5390bcffe6adb03d9b95a8e360c9ba31705c1209 Mon Sep 17 00:00:00 2001 From: Ivan Dlugos Date: Fri, 13 Oct 2023 13:01:06 +0200 Subject: [PATCH 11/47] extract inner zip files --- .../bin/flutter_symbol_collector.dart | 5 +- .../lib/src/flutter_symbol_source.dart | 76 +++++++++++-------- 2 files changed, 47 insertions(+), 34 deletions(-) diff --git a/scripts/flutter_symbol_collector/bin/flutter_symbol_collector.dart b/scripts/flutter_symbol_collector/bin/flutter_symbol_collector.dart index 1f43aeba05..7aca8ef4ae 100644 --- a/scripts/flutter_symbol_collector/bin/flutter_symbol_collector.dart +++ b/scripts/flutter_symbol_collector/bin/flutter_symbol_collector.dart @@ -59,8 +59,7 @@ Future processFlutterVerion(FlutterVersion version) async { final archives = await source.listSymbolArchives(version); final dir = tempDir.childDirectory(version.tagName); for (final archive in archives) { - final archiveDir = dir.childDirectory(archive.platform.operatingSystem); - await source.downloadAndExtractTo(archiveDir, archive.path); - await collector.upload(archiveDir, archive.platform, version); + await source.downloadAndExtractTo(dir, archive.path); + await collector.upload(dir, archive.platform, version); } } diff --git a/scripts/flutter_symbol_collector/lib/src/flutter_symbol_source.dart b/scripts/flutter_symbol_collector/lib/src/flutter_symbol_source.dart index 778ea18837..64ffd806fd 100644 --- a/scripts/flutter_symbol_collector/lib/src/flutter_symbol_source.dart +++ b/scripts/flutter_symbol_collector/lib/src/flutter_symbol_source.dart @@ -73,45 +73,20 @@ class FlutterSymbolSource { //archive, extracts the content instead. Future downloadAndExtractTo(Directory target, String filePath) async { if (path.extension(filePath) == '.zip') { - target = await target.childDirectory(filePath).create(recursive: true); + target = await target + .childDirectory(path.withoutExtension(filePath)) + .create(recursive: true); try { final buffer = BytesBuilder(); await download(filePath).forEach(buffer.add); final archive = ZipDecoder().decodeBytes(buffer.toBytes()); buffer.clear(); - _log.fine('Extracting $filePath to $target'); - // For all of the entries in the archive - for (var entry in archive.files) { - // Make sure we don't have any zip-slip issues. - final entryPath = path.normalize(entry.name); - - if (!path - .normalize(target.childFile(entryPath).path) - .startsWith(target.path)) { - throw Exception( - 'Invalid ZIP entry path (looks like a zip-slip issue): ${entry.name}'); - } - - // If it's a file and not a directory - if (entry.isFile) { - final file = await target - .childFile(path.normalize(entryPath)) - .create(exclusive: true); - - // Note: package:archive doesn't support extracting directly to an - // IOSink. See https://github.com/brendan-duncan/archive/issues/12 - final stream = OutputStream(); - entry.writeContent(stream, freeMemory: true); - stream.flush(); - - await file.writeAsBytes(stream.getBytes(), flush: true); - } else { - await target.childDirectory(entryPath).create(recursive: true); - } - } + await _extractZip(target, archive); } catch (e, trace) { _log.warning('Failed to download $filePath to $target', e, trace); + // Remove the directory so that we don't leave a partial extraction. + await target.delete(recursive: true); } } else { _log.fine('Downloading $filePath to $target'); @@ -130,4 +105,43 @@ class FlutterSymbolSource { } } } + + Future _extractZip(Directory target, Archive archive) async { + for (var entry in archive.files) { + // Make sure we don't have any zip-slip issues. + final entryPath = path.normalize(entry.name); + if (!path + .normalize(target.childFile(entryPath).path) + .startsWith(target.path)) { + throw Exception( + 'Invalid ZIP entry path (looks like a zip-slip issue): ${entry.name}'); + } + + if (!entry.isFile) { + // If it's a directory - create it. + await target.childDirectory(entryPath).create(recursive: true); + } else { + // Note: package:archive doesn't support extracting directly to an + // IOSink. See https://github.com/brendan-duncan/archive/issues/12 + final stream = OutputStream(); + entry.writeContent(stream, freeMemory: true); + stream.flush(); + + // If it's an inner ZIP archive - extract it recursively. + if (path.extension(entryPath) == '.zip') { + final innerArchive = ZipDecoder().decodeBytes(stream.getBytes()); + stream.clear(); + final innerTarget = target + .childDirectory(path.withoutExtension(path.normalize(entryPath))); + _log.fine('Extracting inner archive $entryPath to $innerTarget'); + await _extractZip(innerTarget, innerArchive); + } else { + final file = await target + .childFile(path.normalize(entryPath)) + .create(exclusive: true); + await file.writeAsBytes(stream.getBytes(), flush: true); + } + } + } + } } From 2bc4ed7d68b29388f9a7fe665037205b852afd8a Mon Sep 17 00:00:00 2001 From: Ivan Dlugos Date: Fri, 13 Oct 2023 13:10:44 +0200 Subject: [PATCH 12/47] test symbol collector in CI --- .github/workflows/flutter-symbols.yml | 19 ++++++++++++++++--- .../bin/flutter_symbol_collector.dart | 3 +-- .../lib/src/flutter_symbol_source.dart | 2 +- .../lib/src/flutter_version.dart | 10 +++++----- .../test/flutter_symbol_source_test.dart | 2 +- 5 files changed, 24 insertions(+), 12 deletions(-) diff --git a/.github/workflows/flutter-symbols.yml b/.github/workflows/flutter-symbols.yml index 7227f445d3..831fdac319 100644 --- a/.github/workflows/flutter-symbols.yml +++ b/.github/workflows/flutter-symbols.yml @@ -8,13 +8,26 @@ on: required: false type: string +defaults: + run: + working-directory: scripts/flutter_symbol_collector + jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - uses: dart-lang/setup-dart@6a218f2413a3e78e9087f638a238f6b40893203d # pin@v1 + + - run: dart pub get + + - run: dart test + run: + needs: [test] runs-on: ubuntu-latest timeout-minutes: 60 - defaults: - run: - working-directory: scripts/flutter_symbol_collector steps: - uses: actions/checkout@v3 diff --git a/scripts/flutter_symbol_collector/bin/flutter_symbol_collector.dart b/scripts/flutter_symbol_collector/bin/flutter_symbol_collector.dart index 7aca8ef4ae..73d4ec7147 100644 --- a/scripts/flutter_symbol_collector/bin/flutter_symbol_collector.dart +++ b/scripts/flutter_symbol_collector/bin/flutter_symbol_collector.dart @@ -53,8 +53,7 @@ void main(List arguments) async { Future processFlutterVerion(FlutterVersion version) async { Logger.root.info('Processing Flutter ${version.tagName}'); - final engineVersion = await version.getEngineVersion(); - Logger.root.info('Engine version: $engineVersion'); + Logger.root.info('Engine version: ${await version.engineVersion}'); final archives = await source.listSymbolArchives(version); final dir = tempDir.childDirectory(version.tagName); diff --git a/scripts/flutter_symbol_collector/lib/src/flutter_symbol_source.dart b/scripts/flutter_symbol_collector/lib/src/flutter_symbol_source.dart index 64ffd806fd..414443d8d3 100644 --- a/scripts/flutter_symbol_collector/lib/src/flutter_symbol_source.dart +++ b/scripts/flutter_symbol_collector/lib/src/flutter_symbol_source.dart @@ -33,7 +33,7 @@ class FlutterSymbolSource { Future> listSymbolArchives(FlutterVersion version) async { // example: https://console.cloud.google.com/storage/browser/flutter_infra_release/flutter/9064459a8b0dcd32877107f6002cc429a71659d1 - final prefix = 'flutter/${await version.getEngineVersion()}/'; + final prefix = 'flutter/${await version.engineVersion}/'; late final List resolvers; if (version.tagName.startsWith('3.')) { diff --git a/scripts/flutter_symbol_collector/lib/src/flutter_version.dart b/scripts/flutter_symbol_collector/lib/src/flutter_version.dart index 32e5775789..899534df4b 100644 --- a/scripts/flutter_symbol_collector/lib/src/flutter_version.dart +++ b/scripts/flutter_symbol_collector/lib/src/flutter_version.dart @@ -3,12 +3,12 @@ import 'package:http/http.dart' as http; class FlutterVersion { final String tagName; - FlutterVersion(this.tagName); - - bool get isPreRelease => tagName.endsWith('.pre'); - - Future getEngineVersion() => http + late final engineVersion = http .get(Uri.https('raw.githubusercontent.com', 'flutter/flutter/$tagName/bin/internal/engine.version')) .then((value) => value.body.trim()); + + FlutterVersion(this.tagName); + + bool get isPreRelease => tagName.endsWith('.pre'); } diff --git a/scripts/flutter_symbol_collector/test/flutter_symbol_source_test.dart b/scripts/flutter_symbol_collector/test/flutter_symbol_source_test.dart index 84473a1f72..f4c14b2fcc 100644 --- a/scripts/flutter_symbol_collector/test/flutter_symbol_source_test.dart +++ b/scripts/flutter_symbol_collector/test/flutter_symbol_source_test.dart @@ -32,7 +32,7 @@ void main() { final versions = await sut.listFlutterVersions().take(3).toList(); final engines = List.empty(growable: true); for (var v in versions) { - engines.add("${v.tagName} => ${await v.getEngineVersion()}"); + engines.add("${v.tagName} => ${await v.engineVersion}"); } expect( engines, From ed79ef47128109d6852d33e18717b3d80670dda0 Mon Sep 17 00:00:00 2001 From: Ivan Dlugos Date: Fri, 13 Oct 2023 13:17:21 +0200 Subject: [PATCH 13/47] fix tests --- .../test/symbol_collector_cli_test.dart | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/flutter_symbol_collector/test/symbol_collector_cli_test.dart b/scripts/flutter_symbol_collector/test/symbol_collector_cli_test.dart index aef11629e1..763481b03e 100644 --- a/scripts/flutter_symbol_collector/test/symbol_collector_cli_test.dart +++ b/scripts/flutter_symbol_collector/test/symbol_collector_cli_test.dart @@ -43,8 +43,8 @@ void main() { .createTempSync('symbol_collector_test'); late final SymbolCollectorCli sut; - setUp(() async => sut = await SymbolCollectorCli.setup(tmpDir)); - tearDown(() => tmpDir.delete(recursive: true)); + setUpAll(() async => sut = await SymbolCollectorCli.setup(tmpDir)); + tearDownAll(() => tmpDir.delete(recursive: true)); test('getVersion()', () async { final output = await sut.getVersion(); From ff91b356dc38f120be0f84ff06a6097c569f811f Mon Sep 17 00:00:00 2001 From: Ivan Dlugos Date: Fri, 13 Oct 2023 13:19:58 +0200 Subject: [PATCH 14/47] chore: update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3629b48f9c..74a6b7e08d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ ### Features - Breadcrumbs for file I/O operations ([#1649](https://github.com/getsentry/sentry-dart/pull/1649)) +- Better Flutter framework stack traces - we now collect Flutter framework debug symbols for iOS, macOS and Android automatically on the Sentry server ([#1673](https://github.com/getsentry/sentry-dart/pull/1673)) ### Dependencies From 795e7a4141ae5e59ddc991bee17d8c97748c686f Mon Sep 17 00:00:00 2001 From: Ivan Dlugos Date: Fri, 13 Oct 2023 14:04:55 +0200 Subject: [PATCH 15/47] fix tests --- .../lib/src/flutter_symbol_source.dart | 24 ++++++++++--------- .../lib/src/flutter_version.dart | 2 ++ .../lib/src/symbol_archive.dart | 2 ++ .../test/flutter_symbol_source_test.dart | 8 +++---- 4 files changed, 21 insertions(+), 15 deletions(-) diff --git a/scripts/flutter_symbol_collector/lib/src/flutter_symbol_source.dart b/scripts/flutter_symbol_collector/lib/src/flutter_symbol_source.dart index 414443d8d3..c8e675c898 100644 --- a/scripts/flutter_symbol_collector/lib/src/flutter_symbol_source.dart +++ b/scripts/flutter_symbol_collector/lib/src/flutter_symbol_source.dart @@ -63,14 +63,14 @@ class FlutterSymbolSource { return archives; } - // Streams the remote file contents. + /// Streams the remote file contents. Stream> download(String filePath) { _log.fine('Downloading $filePath'); return _symbolsBucket.read(filePath); } - // Downloads the remote file to the given target directory or if it's an - //archive, extracts the content instead. + /// Downloads the remote [filePath] to the given [target] directory. + /// If it's an archive, extracts the content instead. Future downloadAndExtractTo(Directory target, String filePath) async { if (path.extension(filePath) == '.zip') { target = await target @@ -109,9 +109,8 @@ class FlutterSymbolSource { Future _extractZip(Directory target, Archive archive) async { for (var entry in archive.files) { // Make sure we don't have any zip-slip issues. - final entryPath = path.normalize(entry.name); - if (!path - .normalize(target.childFile(entryPath).path) + final entryPath = _pathNormalize(entry.name); + if (!_pathNormalize(target.childFile(entryPath).path) .startsWith(target.path)) { throw Exception( 'Invalid ZIP entry path (looks like a zip-slip issue): ${entry.name}'); @@ -131,17 +130,20 @@ class FlutterSymbolSource { if (path.extension(entryPath) == '.zip') { final innerArchive = ZipDecoder().decodeBytes(stream.getBytes()); stream.clear(); - final innerTarget = target - .childDirectory(path.withoutExtension(path.normalize(entryPath))); + final innerTarget = + target.childDirectory(path.withoutExtension(entryPath)); _log.fine('Extracting inner archive $entryPath to $innerTarget'); await _extractZip(innerTarget, innerArchive); } else { - final file = await target - .childFile(path.normalize(entryPath)) - .create(exclusive: true); + final file = + await target.childFile(entryPath).create(exclusive: true); + _log.finer('Writing $file: ${stream.length} bytes'); await file.writeAsBytes(stream.getBytes(), flush: true); } } } } + + String _pathNormalize(String p) => + path.normalize(p).replaceAll(path.separator, '/'); } diff --git a/scripts/flutter_symbol_collector/lib/src/flutter_version.dart b/scripts/flutter_symbol_collector/lib/src/flutter_version.dart index 899534df4b..6461a4c1fe 100644 --- a/scripts/flutter_symbol_collector/lib/src/flutter_version.dart +++ b/scripts/flutter_symbol_collector/lib/src/flutter_version.dart @@ -1,5 +1,7 @@ import 'package:http/http.dart' as http; +import 'package:meta/meta.dart'; +@immutable class FlutterVersion { final String tagName; diff --git a/scripts/flutter_symbol_collector/lib/src/symbol_archive.dart b/scripts/flutter_symbol_collector/lib/src/symbol_archive.dart index 99382018d9..f4681e44be 100644 --- a/scripts/flutter_symbol_collector/lib/src/symbol_archive.dart +++ b/scripts/flutter_symbol_collector/lib/src/symbol_archive.dart @@ -1,5 +1,7 @@ import 'package:platform/platform.dart'; +import 'package:meta/meta.dart'; +@immutable class SymbolArchive { final String path; final Platform platform; diff --git a/scripts/flutter_symbol_collector/test/flutter_symbol_source_test.dart b/scripts/flutter_symbol_collector/test/flutter_symbol_source_test.dart index f4c14b2fcc..bd6f25ec94 100644 --- a/scripts/flutter_symbol_collector/test/flutter_symbol_source_test.dart +++ b/scripts/flutter_symbol_collector/test/flutter_symbol_source_test.dart @@ -70,10 +70,10 @@ void main() { }); test('downloadAndExtractTo() extracts a zip file', () async { - await sut.downloadAndExtractTo(fs.currentDirectory, - 'flutter/0005149dca9b248663adcde4bdd7c6c915a76584/sky_engine.zip'); - expect(fs.isDirectorySync('sky_engine'), isTrue); - expect(fs.file('sky_engine/README.md').readAsStringSync(), + const path = 'flutter/0005149dca9b248663adcde4bdd7c6c915a76584'; + await sut.downloadAndExtractTo(fs.currentDirectory, '$path/sky_engine.zip'); + expect(fs.isDirectorySync('$path/sky_engine/sky_engine'), isTrue); + expect(fs.file('$path/sky_engine/sky_engine/README.md').readAsStringSync(), startsWith('Flutter Engine')); }); } From 816ae3e20033810b7e44d9eada7a9f3a8f335cc7 Mon Sep 17 00:00:00 2001 From: Ivan Dlugos Date: Fri, 13 Oct 2023 17:09:11 +0200 Subject: [PATCH 16/47] fix tests --- .../test/symbol_collector_cli_test.dart | 25 +++++++++++-------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/scripts/flutter_symbol_collector/test/symbol_collector_cli_test.dart b/scripts/flutter_symbol_collector/test/symbol_collector_cli_test.dart index 763481b03e..dbd9a50e12 100644 --- a/scripts/flutter_symbol_collector/test/symbol_collector_cli_test.dart +++ b/scripts/flutter_symbol_collector/test/symbol_collector_cli_test.dart @@ -10,13 +10,13 @@ import 'common.dart'; void main() { setupLogging(); - late FileSystem fs; - - setUp(() { - fs = MemoryFileSystem.test(); - }); group('setup() downloads CLI on', () { + late FileSystem fs; + + setUp(() { + fs = MemoryFileSystem.test(); + }); for (final platform in [Platform.macOS, Platform.linux]) { test(platform, () async { const path = 'temp/symbol-collector'; @@ -28,11 +28,16 @@ void main() { .then((file) => file.writeAsString('foo')); expect(fs.file(path).lengthSync(), equals(3)); - SymbolCollectorCli.platform = FakePlatform(operatingSystem: platform); - final sut = await SymbolCollectorCli.setup(fs.directory('temp')); - expect(sut.cli, equals(path)); - expect(fs.file(path).existsSync(), isTrue); - expect(fs.file(path).lengthSync(), greaterThan(1000000)); + final originalPlatform = SymbolCollectorCli.platform; + try { + SymbolCollectorCli.platform = FakePlatform(operatingSystem: platform); + final sut = await SymbolCollectorCli.setup(fs.directory('temp')); + expect(sut.cli, equals(path)); + expect(fs.file(path).existsSync(), isTrue); + expect(fs.file(path).lengthSync(), greaterThan(1000000)); + } finally { + SymbolCollectorCli.platform = originalPlatform; + } }); } }); From 5170a4646beba7a39617cfdfeffad0f955fd3805 Mon Sep 17 00:00:00 2001 From: Ivan Dlugos Date: Fri, 13 Oct 2023 17:09:40 +0200 Subject: [PATCH 17/47] upload android symbols --- .../lib/src/flutter_symbol_resolver.dart | 15 +++++++++++++++ .../lib/src/flutter_symbol_source.dart | 4 +++- .../test/flutter_symbol_source_test.dart | 4 +++- 3 files changed, 21 insertions(+), 2 deletions(-) diff --git a/scripts/flutter_symbol_collector/lib/src/flutter_symbol_resolver.dart b/scripts/flutter_symbol_collector/lib/src/flutter_symbol_resolver.dart index 889cf515dd..93d9b7e02f 100644 --- a/scripts/flutter_symbol_collector/lib/src/flutter_symbol_resolver.dart +++ b/scripts/flutter_symbol_collector/lib/src/flutter_symbol_resolver.dart @@ -56,3 +56,18 @@ class MacOSSymbolResolver extends FlutterSymbolResolver { return _resolvedFiles; } } + +class AndroidSymbolResolver extends FlutterSymbolResolver { + final String architecture; + + AndroidSymbolResolver(super.bucket, super.prefix, this.architecture); + + @override + final platform = FakePlatform(operatingSystem: Platform.android); + + @override + Future> listArchives() async { + await tryResolve('android-$architecture-release/symbols.zip'); + return _resolvedFiles; + } +} diff --git a/scripts/flutter_symbol_collector/lib/src/flutter_symbol_source.dart b/scripts/flutter_symbol_collector/lib/src/flutter_symbol_source.dart index c8e675c898..fff836b6ed 100644 --- a/scripts/flutter_symbol_collector/lib/src/flutter_symbol_source.dart +++ b/scripts/flutter_symbol_collector/lib/src/flutter_symbol_source.dart @@ -39,7 +39,9 @@ class FlutterSymbolSource { if (version.tagName.startsWith('3.')) { resolvers = [ IosSymbolResolver(_symbolsBucket, prefix), - MacOSSymbolResolver(_symbolsBucket, prefix) + MacOSSymbolResolver(_symbolsBucket, prefix), + AndroidSymbolResolver(_symbolsBucket, prefix, 'arm'), + AndroidSymbolResolver(_symbolsBucket, prefix, 'arm64') ]; } else { _log.warning('No symbol resolvers registered for ${version.tagName}'); diff --git a/scripts/flutter_symbol_collector/test/flutter_symbol_source_test.dart b/scripts/flutter_symbol_collector/test/flutter_symbol_source_test.dart index bd6f25ec94..9153ac3128 100644 --- a/scripts/flutter_symbol_collector/test/flutter_symbol_source_test.dart +++ b/scripts/flutter_symbol_collector/test/flutter_symbol_source_test.dart @@ -50,7 +50,9 @@ void main() { archives.map((v) => '${v.platform.operatingSystem} - ${v.path}'), equals([ 'ios - $prefix/ios-release/Flutter.dSYM.zip', - 'macos - $prefix/darwin-x64-release/FlutterMacOS.dSYM.zip' + 'macos - $prefix/darwin-x64-release/FlutterMacOS.dSYM.zip', + 'android - flutter/9064459a8b0dcd32877107f6002cc429a71659d1/android-arm-release/symbols.zip', + 'android - flutter/9064459a8b0dcd32877107f6002cc429a71659d1/android-arm64-release/symbols.zip' ])); }); From caa9b62e1aec18c582e1e72dd09478ff98ff6b3c Mon Sep 17 00:00:00 2001 From: Ivan Dlugos Date: Fri, 13 Oct 2023 17:17:50 +0200 Subject: [PATCH 18/47] fix duplicate upload --- .../bin/flutter_symbol_collector.dart | 5 +++-- .../lib/src/symbol_collector_cli.dart | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/scripts/flutter_symbol_collector/bin/flutter_symbol_collector.dart b/scripts/flutter_symbol_collector/bin/flutter_symbol_collector.dart index 73d4ec7147..7c65f1f360 100644 --- a/scripts/flutter_symbol_collector/bin/flutter_symbol_collector.dart +++ b/scripts/flutter_symbol_collector/bin/flutter_symbol_collector.dart @@ -58,7 +58,8 @@ Future processFlutterVerion(FlutterVersion version) async { final archives = await source.listSymbolArchives(version); final dir = tempDir.childDirectory(version.tagName); for (final archive in archives) { - await source.downloadAndExtractTo(dir, archive.path); - await collector.upload(dir, archive.platform, version); + final archiveDir = dir.childDirectory(archive.platform.operatingSystem); + await source.downloadAndExtractTo(archiveDir, archive.path); + await collector.upload(archiveDir, archive.platform, version); } } diff --git a/scripts/flutter_symbol_collector/lib/src/symbol_collector_cli.dart b/scripts/flutter_symbol_collector/lib/src/symbol_collector_cli.dart index ccb0323cb4..3b386fb456 100644 --- a/scripts/flutter_symbol_collector/lib/src/symbol_collector_cli.dart +++ b/scripts/flutter_symbol_collector/lib/src/symbol_collector_cli.dart @@ -108,7 +108,7 @@ class SymbolCollectorCli { Future _execute(List arguments) async { _ensureIsExecutable(); - _log.fine('Executing ${path.basename(cli)} $arguments'); + _log.fine('Executing ${path.basename(cli)} ${arguments.join(' ')}'); final process = await Process.start(cli, arguments); final output = StringBuffer(); From 4724c814bebf9faccd4421726ed0caf0647e0a17 Mon Sep 17 00:00:00 2001 From: Ivan Dlugos Date: Tue, 24 Oct 2023 20:02:04 +0200 Subject: [PATCH 19/47] upload symbols for all of 3.13.* --- .github/workflows/flutter-symbols.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/flutter-symbols.yml b/.github/workflows/flutter-symbols.yml index 831fdac319..b42fc76442 100644 --- a/.github/workflows/flutter-symbols.yml +++ b/.github/workflows/flutter-symbols.yml @@ -36,6 +36,6 @@ jobs: - run: dart pub get # - run: dart run bin/flutter_symbol_collector.dart --version=${{ inputs.flutter_version }} - - run: dart run bin/flutter_symbol_collector.dart --version=3.13.7 + - run: dart run bin/flutter_symbol_collector.dart --version=3.13.* env: GITHUB_TOKEN: ${{ github.token }} From b55becbe55661a4fac85037eac98ad0666a21e11 Mon Sep 17 00:00:00 2001 From: Ivan Dlugos Date: Tue, 24 Oct 2023 21:22:52 +0200 Subject: [PATCH 20/47] cache previous symbol collector successful uploads --- .github/workflows/flutter-symbols.yml | 15 ++++++-- scripts/flutter_symbol_collector/.gitignore | 3 +- .../bin/flutter_symbol_collector.dart | 28 +++++++++++++-- .../lib/src/flutter_symbol_source.dart | 7 +++- .../lib/src/symbol_collector_cli.dart | 34 +++++++++++-------- 5 files changed, 66 insertions(+), 21 deletions(-) diff --git a/.github/workflows/flutter-symbols.yml b/.github/workflows/flutter-symbols.yml index b42fc76442..068dd061f6 100644 --- a/.github/workflows/flutter-symbols.yml +++ b/.github/workflows/flutter-symbols.yml @@ -1,6 +1,6 @@ name: Flutter symbols collection on: - push: # Temporary so we get the workflow on the PR branch + push: # TODO REMOVE workflow_dispatch: inputs: flutter_version: @@ -27,7 +27,6 @@ jobs: run: needs: [test] runs-on: ubuntu-latest - timeout-minutes: 60 steps: - uses: actions/checkout@v3 @@ -35,7 +34,17 @@ jobs: - run: dart pub get + - uses: actions/download-artifact@v3 + with: + name: flutter-symbol-collector-database + path: scripts/flutter_symbol_collector/.successful + # - run: dart run bin/flutter_symbol_collector.dart --version=${{ inputs.flutter_version }} - - run: dart run bin/flutter_symbol_collector.dart --version=3.13.* + - run: dart run bin/flutter_symbol_collector.dart --version=3.13.8 env: GITHUB_TOKEN: ${{ github.token }} + + - uses: actions/upload-artifact@v3 + with: + name: flutter-symbol-collector-database + path: scripts/flutter_symbol_collector/.successful diff --git a/scripts/flutter_symbol_collector/.gitignore b/scripts/flutter_symbol_collector/.gitignore index 2b68fd4595..0aa8e5e343 100644 --- a/scripts/flutter_symbol_collector/.gitignore +++ b/scripts/flutter_symbol_collector/.gitignore @@ -2,4 +2,5 @@ # Created by `dart pub` .dart_tool/ -.temp \ No newline at end of file +.temp +.successful diff --git a/scripts/flutter_symbol_collector/bin/flutter_symbol_collector.dart b/scripts/flutter_symbol_collector/bin/flutter_symbol_collector.dart index 7c65f1f360..ad203dec8b 100644 --- a/scripts/flutter_symbol_collector/bin/flutter_symbol_collector.dart +++ b/scripts/flutter_symbol_collector/bin/flutter_symbol_collector.dart @@ -11,6 +11,7 @@ final source = FlutterSymbolSource( : Authentication.withToken(githubToken)); final fs = LocalFileSystem(); final tempDir = fs.currentDirectory.childDirectory('.temp'); +final successDir = fs.currentDirectory.childDirectory('.successful'); late final SymbolCollectorCli collector; void main(List arguments) async { @@ -25,6 +26,7 @@ void main(List arguments) async { final argVersion = args['version'] as String; collector = await SymbolCollectorCli.setup(tempDir); + successDir.createSync(recursive: true); // If a specific version was given, run just for this version. if (argVersion.isNotEmpty && @@ -52,14 +54,36 @@ void main(List arguments) async { } Future processFlutterVerion(FlutterVersion version) async { + if (bool.hasEnvironment('CI')) { + print('::group::Processing Flutter ${version.tagName}'); + } Logger.root.info('Processing Flutter ${version.tagName}'); Logger.root.info('Engine version: ${await version.engineVersion}'); final archives = await source.listSymbolArchives(version); final dir = tempDir.childDirectory(version.tagName); + final sdir = successDir.childDirectory(version.tagName.toLowerCase()); for (final archive in archives) { + // Later, we'll write create an empty file to mark this as successful. + final sFile = sdir.childFile(archive.path.toLowerCase()); + if (sFile.existsSync()) { + Logger.root + .info('Skipping ${archive.path} - already processed successfully'); + continue; + } + final archiveDir = dir.childDirectory(archive.platform.operatingSystem); - await source.downloadAndExtractTo(archiveDir, archive.path); - await collector.upload(archiveDir, archive.platform, version); + if (!await source.downloadAndExtractTo(archiveDir, archive.path)) { + continue; + } + if (!await collector.upload(archiveDir, archive.platform, version)) { + continue; + } + + sFile.createSync(recursive: true); + } + + if (bool.hasEnvironment('CI')) { + print('::endgroup::'); } } diff --git a/scripts/flutter_symbol_collector/lib/src/flutter_symbol_source.dart b/scripts/flutter_symbol_collector/lib/src/flutter_symbol_source.dart index fff836b6ed..aca3a1e187 100644 --- a/scripts/flutter_symbol_collector/lib/src/flutter_symbol_source.dart +++ b/scripts/flutter_symbol_collector/lib/src/flutter_symbol_source.dart @@ -31,6 +31,7 @@ class FlutterSymbolSource { .listTags(_flutterRepo, perPage: 30) .map((t) => FlutterVersion(t.name)); + /// Returns false as the first record value in case there was any error fetching the symbol archives. Future> listSymbolArchives(FlutterVersion version) async { // example: https://console.cloud.google.com/storage/browser/flutter_infra_release/flutter/9064459a8b0dcd32877107f6002cc429a71659d1 final prefix = 'flutter/${await version.engineVersion}/'; @@ -73,7 +74,8 @@ class FlutterSymbolSource { /// Downloads the remote [filePath] to the given [target] directory. /// If it's an archive, extracts the content instead. - Future downloadAndExtractTo(Directory target, String filePath) async { + /// returns `true` if the file was downloaded and extracted successfully. + Future downloadAndExtractTo(Directory target, String filePath) async { if (path.extension(filePath) == '.zip') { target = await target .childDirectory(path.withoutExtension(filePath)) @@ -89,6 +91,7 @@ class FlutterSymbolSource { _log.warning('Failed to download $filePath to $target', e, trace); // Remove the directory so that we don't leave a partial extraction. await target.delete(recursive: true); + return false; } } else { _log.fine('Downloading $filePath to $target'); @@ -104,8 +107,10 @@ class FlutterSymbolSource { _log.warning('Failed to download $filePath to $target', e, trace); await sink.close(); await file.delete(); + return false; } } + return true; } Future _extractZip(Directory target, Archive archive) async { diff --git a/scripts/flutter_symbol_collector/lib/src/symbol_collector_cli.dart b/scripts/flutter_symbol_collector/lib/src/symbol_collector_cli.dart index 3b386fb456..5448834e0c 100644 --- a/scripts/flutter_symbol_collector/lib/src/symbol_collector_cli.dart +++ b/scripts/flutter_symbol_collector/lib/src/symbol_collector_cli.dart @@ -87,22 +87,28 @@ class SymbolCollectorCli { Future getVersion() => _execute(['--version', '-h']); - Future upload( + Future upload( Directory dir, Platform symbolsPlatform, FlutterVersion flutterVersion, - {bool dryRun = false}) { + {bool dryRun = false}) async { final type = symbolsPlatform.operatingSystem; - return _execute([ - '--upload', - 'directory', - '--path', - dir.path, - '--batch-type', - type, - '--bundle-id', - 'flutter-${flutterVersion.tagName}-$type', - '--server-endpoint', - 'https://symbol-collector.services.sentry.io/', - ]); + try { + await _execute([ + '--upload', + 'directory', + '--path', + dir.path, + '--batch-type', + type, + '--bundle-id', + 'flutter-${flutterVersion.tagName}-$type', + '--server-endpoint', + 'https://symbol-collector.services.sentry.io/', + ]); + } catch (e) { + _log.warning('Failed to upload symbols from ${dir.path}', e); + return false; + } + return true; } Future _execute(List arguments) async { From cb298d6cc1c404f8d7d5f6be9045396fc9a87f68 Mon Sep 17 00:00:00 2001 From: Ivan Dlugos Date: Tue, 24 Oct 2023 21:30:03 +0200 Subject: [PATCH 21/47] fix artifacts --- .github/workflows/flutter-symbols.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/flutter-symbols.yml b/.github/workflows/flutter-symbols.yml index 068dd061f6..4c253d4bfb 100644 --- a/.github/workflows/flutter-symbols.yml +++ b/.github/workflows/flutter-symbols.yml @@ -35,6 +35,7 @@ jobs: - run: dart pub get - uses: actions/download-artifact@v3 + continue-on-error: true with: name: flutter-symbol-collector-database path: scripts/flutter_symbol_collector/.successful @@ -45,6 +46,7 @@ jobs: GITHUB_TOKEN: ${{ github.token }} - uses: actions/upload-artifact@v3 + if: always() with: name: flutter-symbol-collector-database path: scripts/flutter_symbol_collector/.successful From 57efb36e1288200c1a49a8f19459363c2d241fd5 Mon Sep 17 00:00:00 2001 From: Ivan Dlugos Date: Tue, 24 Oct 2023 21:36:52 +0200 Subject: [PATCH 22/47] we need to use cache --- .github/workflows/flutter-symbols.yml | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/.github/workflows/flutter-symbols.yml b/.github/workflows/flutter-symbols.yml index 4c253d4bfb..144dc908c6 100644 --- a/.github/workflows/flutter-symbols.yml +++ b/.github/workflows/flutter-symbols.yml @@ -34,19 +34,12 @@ jobs: - run: dart pub get - - uses: actions/download-artifact@v3 - continue-on-error: true + - uses: actions/cache@v3 with: - name: flutter-symbol-collector-database + key: flutter-symbol-collector-database path: scripts/flutter_symbol_collector/.successful # - run: dart run bin/flutter_symbol_collector.dart --version=${{ inputs.flutter_version }} - run: dart run bin/flutter_symbol_collector.dart --version=3.13.8 env: GITHUB_TOKEN: ${{ github.token }} - - - uses: actions/upload-artifact@v3 - if: always() - with: - name: flutter-symbol-collector-database - path: scripts/flutter_symbol_collector/.successful From 8d686cb9bb7d7bfdecfee8b4a434e0335268f297 Mon Sep 17 00:00:00 2001 From: Ivan Dlugos Date: Tue, 24 Oct 2023 21:47:02 +0200 Subject: [PATCH 23/47] cron --- .github/workflows/flutter-symbols.yml | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/.github/workflows/flutter-symbols.yml b/.github/workflows/flutter-symbols.yml index 144dc908c6..f506fd7c85 100644 --- a/.github/workflows/flutter-symbols.yml +++ b/.github/workflows/flutter-symbols.yml @@ -1,12 +1,15 @@ name: Flutter symbols collection on: - push: # TODO REMOVE + push: + schedule: + - cron: "10 */6 * * *" # every six hours workflow_dispatch: inputs: flutter_version: description: Flutter version, can be either a specific version (3.17.0) or a wildcard (3.2.*) required: false type: string + default: "3.*.*" defaults: run: @@ -34,12 +37,18 @@ jobs: - run: dart pub get - - uses: actions/cache@v3 + - uses: actions/cache/restore@v3 with: key: flutter-symbol-collector-database path: scripts/flutter_symbol_collector/.successful # - run: dart run bin/flutter_symbol_collector.dart --version=${{ inputs.flutter_version }} - - run: dart run bin/flutter_symbol_collector.dart --version=3.13.8 + - run: dart run bin/flutter_symbol_collector.dart --version=3.13.8 ; exit 1 env: GITHUB_TOKEN: ${{ github.token }} + + - uses: actions/cache/save@v3 + if: always() + with: + key: flutter-symbol-collector-database + path: scripts/flutter_symbol_collector/.successful From bdfcfdced4d0b37dbbc456b06bcfa2fd47880fc4 Mon Sep 17 00:00:00 2001 From: Ivan Dlugos Date: Tue, 24 Oct 2023 21:50:05 +0200 Subject: [PATCH 24/47] test --- .github/workflows/flutter-symbols.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/flutter-symbols.yml b/.github/workflows/flutter-symbols.yml index f506fd7c85..3a0c78f8e3 100644 --- a/.github/workflows/flutter-symbols.yml +++ b/.github/workflows/flutter-symbols.yml @@ -43,7 +43,7 @@ jobs: path: scripts/flutter_symbol_collector/.successful # - run: dart run bin/flutter_symbol_collector.dart --version=${{ inputs.flutter_version }} - - run: dart run bin/flutter_symbol_collector.dart --version=3.13.8 ; exit 1 + - run: dart run bin/flutter_symbol_collector.dart --version=3.13.7 ; exit 1 env: GITHUB_TOKEN: ${{ github.token }} From cee0761425d0be9ef6f8a5fd5f9dcb51e77f1a20 Mon Sep 17 00:00:00 2001 From: Ivan Dlugos Date: Tue, 24 Oct 2023 21:55:44 +0200 Subject: [PATCH 25/47] use artifacts --- .github/workflows/flutter-symbols.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/flutter-symbols.yml b/.github/workflows/flutter-symbols.yml index 3a0c78f8e3..ea6c7b6e13 100644 --- a/.github/workflows/flutter-symbols.yml +++ b/.github/workflows/flutter-symbols.yml @@ -37,9 +37,10 @@ jobs: - run: dart pub get - - uses: actions/cache/restore@v3 + - uses: actions/download-artifact@v3 + continue-on-error: true with: - key: flutter-symbol-collector-database + name: flutter-symbol-collector-database path: scripts/flutter_symbol_collector/.successful # - run: dart run bin/flutter_symbol_collector.dart --version=${{ inputs.flutter_version }} @@ -47,8 +48,8 @@ jobs: env: GITHUB_TOKEN: ${{ github.token }} - - uses: actions/cache/save@v3 + - uses: actions/upload-artifact@v3 if: always() with: - key: flutter-symbol-collector-database + name: flutter-symbol-collector-database path: scripts/flutter_symbol_collector/.successful From 611f636e3c56a7c9c40f3a432c39d08a60f4db31 Mon Sep 17 00:00:00 2001 From: Ivan Dlugos Date: Tue, 24 Oct 2023 22:04:11 +0200 Subject: [PATCH 26/47] run for all v3 flutter versions --- .github/workflows/flutter-symbols.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/flutter-symbols.yml b/.github/workflows/flutter-symbols.yml index ea6c7b6e13..a7f1f1f069 100644 --- a/.github/workflows/flutter-symbols.yml +++ b/.github/workflows/flutter-symbols.yml @@ -1,6 +1,6 @@ name: Flutter symbols collection on: - push: + push: # TODO REMOVE schedule: - cron: "10 */6 * * *" # every six hours workflow_dispatch: @@ -43,8 +43,7 @@ jobs: name: flutter-symbol-collector-database path: scripts/flutter_symbol_collector/.successful - # - run: dart run bin/flutter_symbol_collector.dart --version=${{ inputs.flutter_version }} - - run: dart run bin/flutter_symbol_collector.dart --version=3.13.7 ; exit 1 + - run: dart run bin/flutter_symbol_collector.dart --version=${{ inputs.flutter_version || '3.*.*' }} env: GITHUB_TOKEN: ${{ github.token }} From 321198c41a8c8e9de5d1fa8e67c901c72ffaa9ab Mon Sep 17 00:00:00 2001 From: Ivan Dlugos Date: Tue, 24 Oct 2023 22:14:27 +0200 Subject: [PATCH 27/47] fixup changelog --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 74a6b7e08d..66ac1854af 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## Unreleased + +### Features + +- Better Flutter framework stack traces - we now collect Flutter framework debug symbols for iOS, macOS and Android automatically on the Sentry server ([#1673](https://github.com/getsentry/sentry-dart/pull/1673)) + ## 7.11.0 ### Fixes From 6834bc8f897857d7574b49c54322d1cc6fc105f0 Mon Sep 17 00:00:00 2001 From: Ivan Dlugos Date: Wed, 25 Oct 2023 07:20:05 +0200 Subject: [PATCH 28/47] fixup changelog --- CHANGELOG.md | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 66ac1854af..d33691189d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,7 +15,6 @@ ### Features - Breadcrumbs for file I/O operations ([#1649](https://github.com/getsentry/sentry-dart/pull/1649)) -- Better Flutter framework stack traces - we now collect Flutter framework debug symbols for iOS, macOS and Android automatically on the Sentry server ([#1673](https://github.com/getsentry/sentry-dart/pull/1673)) ### Dependencies From dfb8fee2a7a6b8b4539af28579e04c002dabccab Mon Sep 17 00:00:00 2001 From: Ivan Dlugos Date: Wed, 25 Oct 2023 08:42:37 +0200 Subject: [PATCH 29/47] refactor: move status cache to a separate file --- .../bin/flutter_symbol_collector.dart | 30 +++++----- .../lib/flutter_symbol_collector.dart | 1 + .../lib/src/status_cache.dart | 56 +++++++++++++++++++ 3 files changed, 70 insertions(+), 17 deletions(-) create mode 100644 scripts/flutter_symbol_collector/lib/src/status_cache.dart diff --git a/scripts/flutter_symbol_collector/bin/flutter_symbol_collector.dart b/scripts/flutter_symbol_collector/bin/flutter_symbol_collector.dart index ad203dec8b..834ea0bf5e 100644 --- a/scripts/flutter_symbol_collector/bin/flutter_symbol_collector.dart +++ b/scripts/flutter_symbol_collector/bin/flutter_symbol_collector.dart @@ -5,13 +5,13 @@ import 'package:github/github.dart'; import 'package:logging/logging.dart'; const githubToken = String.fromEnvironment('GITHUB_TOKEN'); -final source = FlutterSymbolSource( - githubAuth: githubToken.isEmpty - ? Authentication.anonymous() - : Authentication.withToken(githubToken)); +final githubAuth = githubToken.isEmpty + ? Authentication.anonymous() + : Authentication.withToken(githubToken); +final source = FlutterSymbolSource(githubAuth: githubAuth); final fs = LocalFileSystem(); final tempDir = fs.currentDirectory.childDirectory('.temp'); -final successDir = fs.currentDirectory.childDirectory('.successful'); +final stateCache = DirectoryStatusCache(fs.currentDirectory.childDirectory('.successful')); late final SymbolCollectorCli collector; void main(List arguments) async { @@ -26,7 +26,6 @@ void main(List arguments) async { final argVersion = args['version'] as String; collector = await SymbolCollectorCli.setup(tempDir); - successDir.createSync(recursive: true); // If a specific version was given, run just for this version. if (argVersion.isNotEmpty && @@ -62,25 +61,22 @@ Future processFlutterVerion(FlutterVersion version) async { final archives = await source.listSymbolArchives(version); final dir = tempDir.childDirectory(version.tagName); - final sdir = successDir.childDirectory(version.tagName.toLowerCase()); for (final archive in archives) { - // Later, we'll write create an empty file to mark this as successful. - final sFile = sdir.childFile(archive.path.toLowerCase()); - if (sFile.existsSync()) { + final status = await stateCache.getStatus(archive); + if (status == SymbolArchiveStatus.success) { Logger.root .info('Skipping ${archive.path} - already processed successfully'); continue; } final archiveDir = dir.childDirectory(archive.platform.operatingSystem); - if (!await source.downloadAndExtractTo(archiveDir, archive.path)) { - continue; - } - if (!await collector.upload(archiveDir, archive.platform, version)) { - continue; + if (await source.downloadAndExtractTo(archiveDir, archive.path)) { + if (await collector.upload(archiveDir, archive.platform, version)) { + await stateCache.setStatus(archive, SymbolArchiveStatus.success); + continue; + } } - - sFile.createSync(recursive: true); + await stateCache.setStatus(archive, SymbolArchiveStatus.error); } if (bool.hasEnvironment('CI')) { diff --git a/scripts/flutter_symbol_collector/lib/flutter_symbol_collector.dart b/scripts/flutter_symbol_collector/lib/flutter_symbol_collector.dart index 3d54e0d8e5..92baabf813 100644 --- a/scripts/flutter_symbol_collector/lib/flutter_symbol_collector.dart +++ b/scripts/flutter_symbol_collector/lib/flutter_symbol_collector.dart @@ -1,3 +1,4 @@ export 'src/flutter_symbol_source.dart'; export 'src/flutter_version.dart'; export 'src/symbol_collector_cli.dart'; +export 'src/status_cache.dart'; diff --git a/scripts/flutter_symbol_collector/lib/src/status_cache.dart b/scripts/flutter_symbol_collector/lib/src/status_cache.dart new file mode 100644 index 0000000000..8a280f5817 --- /dev/null +++ b/scripts/flutter_symbol_collector/lib/src/status_cache.dart @@ -0,0 +1,56 @@ +import 'package:file/file.dart'; +import 'package:github/github.dart' as github; +import 'package:logging/logging.dart'; + +import 'symbol_archive.dart'; + +enum SymbolArchiveStatus { + /// The archive has been successfully processed. + success, + + /// The archive has been processed but there was an error. + error, + + /// The archive hasn't been processed yet + pending, +} + +/// Stores and retrieves information about symbol processing status. +abstract class SymbolArchiveStatusCache { + Future setStatus(SymbolArchive archive, SymbolArchiveStatus status); + Future getStatus(SymbolArchive archive); +} + +/// Stores information about symbol processing status in a local directory. +class DirectoryStatusCache implements SymbolArchiveStatusCache { + final Directory _dir; + + DirectoryStatusCache(this._dir) { + _dir.createSync(recursive: true); + } + + File _statusFile(SymbolArchive archive) => + _dir.childFile(archive.path.toLowerCase()); + + @override + Future getStatus(SymbolArchive archive) async { + final file = _statusFile(archive); + if (!await file.exists()) { + return SymbolArchiveStatus.pending; + } + return file.readAsString().then((value) { + switch (value) { + case 'success': + return SymbolArchiveStatus.success; + case 'error': + return SymbolArchiveStatus.error; + default: + throw StateError('Unknown status: $value'); + } + }); + } + + @override + Future setStatus(SymbolArchive archive, SymbolArchiveStatus status) => + _statusFile(archive).writeAsString(status.toString()); +} From deca8fce88737df0397f61ceac0250f0392718db Mon Sep 17 00:00:00 2001 From: Ivan Dlugos Date: Wed, 25 Oct 2023 08:56:43 +0200 Subject: [PATCH 30/47] change artifact action to download from previous runs --- .github/workflows/flutter-symbols.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/flutter-symbols.yml b/.github/workflows/flutter-symbols.yml index a7f1f1f069..ca8b931a6e 100644 --- a/.github/workflows/flutter-symbols.yml +++ b/.github/workflows/flutter-symbols.yml @@ -37,13 +37,17 @@ jobs: - run: dart pub get - - uses: actions/download-artifact@v3 + # We use a custom action that downloads artifacts from the previous run. + - uses: dawidd6/action-download-artifact@268677152d06ba59fcec7a7f0b5d961b6ccd7e1e # pin@v2.28.0 continue-on-error: true with: + workflow_conclusion: "" # i.e. any conclusion, even failures and timeouts + if_no_artifact_found: warn name: flutter-symbol-collector-database path: scripts/flutter_symbol_collector/.successful - - run: dart run bin/flutter_symbol_collector.dart --version=${{ inputs.flutter_version || '3.*.*' }} + - run: dart run bin/flutter_symbol_collector.dart --version=${{ inputs.flutter_version || '3.13.*' }} + timeout-minutes: 300 env: GITHUB_TOKEN: ${{ github.token }} From 12f603366100304645691da93467a19f66a87458 Mon Sep 17 00:00:00 2001 From: Ivan Dlugos Date: Wed, 25 Oct 2023 09:04:14 +0200 Subject: [PATCH 31/47] fix status cache --- .../flutter_symbol_collector/lib/src/status_cache.dart | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/scripts/flutter_symbol_collector/lib/src/status_cache.dart b/scripts/flutter_symbol_collector/lib/src/status_cache.dart index 8a280f5817..4500d896b4 100644 --- a/scripts/flutter_symbol_collector/lib/src/status_cache.dart +++ b/scripts/flutter_symbol_collector/lib/src/status_cache.dart @@ -51,6 +51,10 @@ class DirectoryStatusCache implements SymbolArchiveStatusCache { } @override - Future setStatus(SymbolArchive archive, SymbolArchiveStatus status) => - _statusFile(archive).writeAsString(status.toString()); + Future setStatus( + SymbolArchive archive, SymbolArchiveStatus status) async { + final file = _statusFile(archive); + await file.create(recursive: true); + await file.writeAsString(status.toString()); + } } From 0dc96c33b36d3dbfd12010dadf6ba4517ea2bcaa Mon Sep 17 00:00:00 2001 From: Ivan Dlugos Date: Wed, 25 Oct 2023 09:16:10 +0200 Subject: [PATCH 32/47] fix status cache --- .../lib/flutter_symbol_collector.dart | 1 + .../lib/src/status_cache.dart | 5 ++- .../test/status_cache_test.dart | 43 +++++++++++++++++++ 3 files changed, 47 insertions(+), 2 deletions(-) create mode 100644 scripts/flutter_symbol_collector/test/status_cache_test.dart diff --git a/scripts/flutter_symbol_collector/lib/flutter_symbol_collector.dart b/scripts/flutter_symbol_collector/lib/flutter_symbol_collector.dart index 92baabf813..a972b43f75 100644 --- a/scripts/flutter_symbol_collector/lib/flutter_symbol_collector.dart +++ b/scripts/flutter_symbol_collector/lib/flutter_symbol_collector.dart @@ -2,3 +2,4 @@ export 'src/flutter_symbol_source.dart'; export 'src/flutter_version.dart'; export 'src/symbol_collector_cli.dart'; export 'src/status_cache.dart'; +export 'src/symbol_archive.dart'; diff --git a/scripts/flutter_symbol_collector/lib/src/status_cache.dart b/scripts/flutter_symbol_collector/lib/src/status_cache.dart index 4500d896b4..a9973e2607 100644 --- a/scripts/flutter_symbol_collector/lib/src/status_cache.dart +++ b/scripts/flutter_symbol_collector/lib/src/status_cache.dart @@ -45,7 +45,8 @@ class DirectoryStatusCache implements SymbolArchiveStatusCache { case 'error': return SymbolArchiveStatus.error; default: - throw StateError('Unknown status: $value'); + Logger.root.warning('Unknown status \'$value\' in $file'); + return SymbolArchiveStatus.error; } }); } @@ -55,6 +56,6 @@ class DirectoryStatusCache implements SymbolArchiveStatusCache { SymbolArchive archive, SymbolArchiveStatus status) async { final file = _statusFile(archive); await file.create(recursive: true); - await file.writeAsString(status.toString()); + await file.writeAsString(status.name); } } diff --git a/scripts/flutter_symbol_collector/test/status_cache_test.dart b/scripts/flutter_symbol_collector/test/status_cache_test.dart new file mode 100644 index 0000000000..f8b929db79 --- /dev/null +++ b/scripts/flutter_symbol_collector/test/status_cache_test.dart @@ -0,0 +1,43 @@ +import 'package:file/file.dart'; +import 'package:file/memory.dart'; +import 'package:flutter_symbol_collector/flutter_symbol_collector.dart'; +import 'package:platform/platform.dart'; +import 'package:test/test.dart'; + +import 'common.dart'; + +void main() { + setupLogging(); + + group('DirectoryStatusCache', () { + late FileSystem fs; + late SymbolArchiveStatusCache sut; + final archive = SymbolArchive('path/to/archive.zip', LocalPlatform()); + + setUp(() { + fs = MemoryFileSystem.test(); + sut = DirectoryStatusCache(fs.currentDirectory); + }); + + test('retrieve unprocessed file', () async { + expect(await sut.getStatus(archive), SymbolArchiveStatus.pending); + }); + + test('store and retrieve error', () async { + await sut.setStatus(archive, SymbolArchiveStatus.error); + expect(await sut.getStatus(archive), SymbolArchiveStatus.error); + }); + + test('store and retrieve success', () async { + await sut.setStatus(archive, SymbolArchiveStatus.success); + expect(await sut.getStatus(archive), SymbolArchiveStatus.success); + }); + + test('store, overwrite and retrieve', () async { + await sut.setStatus(archive, SymbolArchiveStatus.error); + expect(await sut.getStatus(archive), SymbolArchiveStatus.error); + await sut.setStatus(archive, SymbolArchiveStatus.success); + expect(await sut.getStatus(archive), SymbolArchiveStatus.success); + }); + }); +} From 38751294fae9f3cab9daaf674000c159d60defff Mon Sep 17 00:00:00 2001 From: Ivan Dlugos Date: Wed, 25 Oct 2023 09:29:28 +0200 Subject: [PATCH 33/47] register symbol collector CLI updater --- .github/workflows/update-deps.yml | 9 +++++++++ scripts/update-symbol-collector.sh | 28 ++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+) create mode 100755 scripts/update-symbol-collector.sh diff --git a/.github/workflows/update-deps.yml b/.github/workflows/update-deps.yml index 953c63cf49..d7eaf21a58 100644 --- a/.github/workflows/update-deps.yml +++ b/.github/workflows/update-deps.yml @@ -35,3 +35,12 @@ jobs: changelog-entry: false secrets: api-token: ${{ secrets.CI_DEPLOY_KEY }} + + symbol-collector: + uses: getsentry/github-workflows/.github/workflows/updater.yml@v2 + with: + path: scripts/update-symbol-collector.sh + name: Symbol collector CLI + changelog-entry: false + secrets: + api-token: ${{ secrets.CI_DEPLOY_KEY }} diff --git a/scripts/update-symbol-collector.sh b/scripts/update-symbol-collector.sh new file mode 100755 index 0000000000..0df03b9434 --- /dev/null +++ b/scripts/update-symbol-collector.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +set -euo pipefail + +cd $(dirname "$0")/flutter_symbol_collector +file='lib/src/symbol_collector_cli.dart' +content=$(cat $file) +regex="(static const version = )'([0-9\.]+)'" +if ! [[ $content =~ $regex ]]; then + echo "Failed to find the plugin version in $file" + exit 1 +fi + +case $1 in +get-version) + echo ${BASH_REMATCH[2]} + ;; +get-repo) + echo "https://github.com/getsentry/symbol-collector.git" + ;; +set-version) + newValue="${BASH_REMATCH[1]}'$2'" + echo "${content/${BASH_REMATCH[0]}/$newValue}" >$file + ;; +*) + echo "Unknown argument $1" + exit 1 + ;; +esac From 6ba64f607cc1087b0a65a332c04c917e57334f2d Mon Sep 17 00:00:00 2001 From: Ivan Dlugos Date: Wed, 25 Oct 2023 09:31:05 +0200 Subject: [PATCH 34/47] log status change --- scripts/flutter_symbol_collector/lib/src/status_cache.dart | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/flutter_symbol_collector/lib/src/status_cache.dart b/scripts/flutter_symbol_collector/lib/src/status_cache.dart index a9973e2607..94501cb3d3 100644 --- a/scripts/flutter_symbol_collector/lib/src/status_cache.dart +++ b/scripts/flutter_symbol_collector/lib/src/status_cache.dart @@ -54,6 +54,7 @@ class DirectoryStatusCache implements SymbolArchiveStatusCache { @override Future setStatus( SymbolArchive archive, SymbolArchiveStatus status) async { + Logger.root.info('Setting ${archive.path} status to ${status.name}'); final file = _statusFile(archive); await file.create(recursive: true); await file.writeAsString(status.name); From d352c73378a22f8274948aba84a8823274ff400c Mon Sep 17 00:00:00 2001 From: Ivan Dlugos Date: Wed, 25 Oct 2023 10:27:05 +0200 Subject: [PATCH 35/47] fix status caching --- .github/workflows/flutter-symbols.yml | 13 +++----- .../bin/flutter_symbol_collector.dart | 10 +++--- .../lib/src/status_cache.dart | 25 +++++++------- .../test/status_cache_test.dart | 33 ++++++++++++++----- 4 files changed, 49 insertions(+), 32 deletions(-) diff --git a/.github/workflows/flutter-symbols.yml b/.github/workflows/flutter-symbols.yml index ca8b931a6e..76e4d270d0 100644 --- a/.github/workflows/flutter-symbols.yml +++ b/.github/workflows/flutter-symbols.yml @@ -37,16 +37,13 @@ jobs: - run: dart pub get - # We use a custom action that downloads artifacts from the previous run. - - uses: dawidd6/action-download-artifact@268677152d06ba59fcec7a7f0b5d961b6ccd7e1e # pin@v2.28.0 + # This downloads the database from the latest successful run. + - run: gh run download --name 'flutter-symbol-collector-database' continue-on-error: true - with: - workflow_conclusion: "" # i.e. any conclusion, even failures and timeouts - if_no_artifact_found: warn - name: flutter-symbol-collector-database - path: scripts/flutter_symbol_collector/.successful + env: + GITHUB_TOKEN: ${{ github.token }} - - run: dart run bin/flutter_symbol_collector.dart --version=${{ inputs.flutter_version || '3.13.*' }} + - run: dart run bin/flutter_symbol_collector.dart --version=${{ inputs.flutter_version || '3.13.1' }} timeout-minutes: 300 env: GITHUB_TOKEN: ${{ github.token }} diff --git a/scripts/flutter_symbol_collector/bin/flutter_symbol_collector.dart b/scripts/flutter_symbol_collector/bin/flutter_symbol_collector.dart index 834ea0bf5e..3170eaabc4 100644 --- a/scripts/flutter_symbol_collector/bin/flutter_symbol_collector.dart +++ b/scripts/flutter_symbol_collector/bin/flutter_symbol_collector.dart @@ -11,7 +11,8 @@ final githubAuth = githubToken.isEmpty final source = FlutterSymbolSource(githubAuth: githubAuth); final fs = LocalFileSystem(); final tempDir = fs.currentDirectory.childDirectory('.temp'); -final stateCache = DirectoryStatusCache(fs.currentDirectory.childDirectory('.successful')); +final stateCache = + DirectoryStatusCache(fs.currentDirectory.childDirectory('.successful')); late final SymbolCollectorCli collector; void main(List arguments) async { @@ -62,7 +63,7 @@ Future processFlutterVerion(FlutterVersion version) async { final archives = await source.listSymbolArchives(version); final dir = tempDir.childDirectory(version.tagName); for (final archive in archives) { - final status = await stateCache.getStatus(archive); + final status = await stateCache.getStatus(version, archive); if (status == SymbolArchiveStatus.success) { Logger.root .info('Skipping ${archive.path} - already processed successfully'); @@ -72,11 +73,12 @@ Future processFlutterVerion(FlutterVersion version) async { final archiveDir = dir.childDirectory(archive.platform.operatingSystem); if (await source.downloadAndExtractTo(archiveDir, archive.path)) { if (await collector.upload(archiveDir, archive.platform, version)) { - await stateCache.setStatus(archive, SymbolArchiveStatus.success); + await stateCache.setStatus( + version, archive, SymbolArchiveStatus.success); continue; } } - await stateCache.setStatus(archive, SymbolArchiveStatus.error); + await stateCache.setStatus(version, archive, SymbolArchiveStatus.error); } if (bool.hasEnvironment('CI')) { diff --git a/scripts/flutter_symbol_collector/lib/src/status_cache.dart b/scripts/flutter_symbol_collector/lib/src/status_cache.dart index 94501cb3d3..4a97f53ce9 100644 --- a/scripts/flutter_symbol_collector/lib/src/status_cache.dart +++ b/scripts/flutter_symbol_collector/lib/src/status_cache.dart @@ -1,7 +1,7 @@ import 'package:file/file.dart'; -import 'package:github/github.dart' as github; import 'package:logging/logging.dart'; +import 'flutter_version.dart'; import 'symbol_archive.dart'; enum SymbolArchiveStatus { @@ -17,8 +17,10 @@ enum SymbolArchiveStatus { /// Stores and retrieves information about symbol processing status. abstract class SymbolArchiveStatusCache { - Future setStatus(SymbolArchive archive, SymbolArchiveStatus status); - Future getStatus(SymbolArchive archive); + Future setStatus(FlutterVersion version, SymbolArchive archive, + SymbolArchiveStatus status); + Future getStatus( + FlutterVersion version, SymbolArchive archive); } /// Stores information about symbol processing status in a local directory. @@ -29,12 +31,13 @@ class DirectoryStatusCache implements SymbolArchiveStatusCache { _dir.createSync(recursive: true); } - File _statusFile(SymbolArchive archive) => - _dir.childFile(archive.path.toLowerCase()); + File _statusFile(FlutterVersion version, SymbolArchive archive) => + _dir.childFile('${version.tagName}/${archive.path.toLowerCase()}.status'); @override - Future getStatus(SymbolArchive archive) async { - final file = _statusFile(archive); + Future getStatus( + FlutterVersion version, SymbolArchive archive) async { + final file = _statusFile(version, archive); if (!await file.exists()) { return SymbolArchiveStatus.pending; } @@ -52,10 +55,10 @@ class DirectoryStatusCache implements SymbolArchiveStatusCache { } @override - Future setStatus( - SymbolArchive archive, SymbolArchiveStatus status) async { - Logger.root.info('Setting ${archive.path} status to ${status.name}'); - final file = _statusFile(archive); + Future setStatus(FlutterVersion version, SymbolArchive archive, + SymbolArchiveStatus status) async { + final file = _statusFile(version, archive); + Logger.root.info('Setting ${file.path} status to ${status.name}'); await file.create(recursive: true); await file.writeAsString(status.name); } diff --git a/scripts/flutter_symbol_collector/test/status_cache_test.dart b/scripts/flutter_symbol_collector/test/status_cache_test.dart index f8b929db79..dc643ef9c0 100644 --- a/scripts/flutter_symbol_collector/test/status_cache_test.dart +++ b/scripts/flutter_symbol_collector/test/status_cache_test.dart @@ -12,6 +12,7 @@ void main() { group('DirectoryStatusCache', () { late FileSystem fs; late SymbolArchiveStatusCache sut; + final version = FlutterVersion('1.2.3'); final archive = SymbolArchive('path/to/archive.zip', LocalPlatform()); setUp(() { @@ -20,24 +21,38 @@ void main() { }); test('retrieve unprocessed file', () async { - expect(await sut.getStatus(archive), SymbolArchiveStatus.pending); + expect( + await sut.getStatus(version, archive), SymbolArchiveStatus.pending); }); test('store and retrieve error', () async { - await sut.setStatus(archive, SymbolArchiveStatus.error); - expect(await sut.getStatus(archive), SymbolArchiveStatus.error); + await sut.setStatus(version, archive, SymbolArchiveStatus.error); + expect(await sut.getStatus(version, archive), SymbolArchiveStatus.error); }); test('store and retrieve success', () async { - await sut.setStatus(archive, SymbolArchiveStatus.success); - expect(await sut.getStatus(archive), SymbolArchiveStatus.success); + await sut.setStatus(version, archive, SymbolArchiveStatus.success); + expect( + await sut.getStatus(version, archive), SymbolArchiveStatus.success); }); test('store, overwrite and retrieve', () async { - await sut.setStatus(archive, SymbolArchiveStatus.error); - expect(await sut.getStatus(archive), SymbolArchiveStatus.error); - await sut.setStatus(archive, SymbolArchiveStatus.success); - expect(await sut.getStatus(archive), SymbolArchiveStatus.success); + await sut.setStatus(version, archive, SymbolArchiveStatus.error); + expect(await sut.getStatus(version, archive), SymbolArchiveStatus.error); + await sut.setStatus(version, archive, SymbolArchiveStatus.success); + expect( + await sut.getStatus(version, archive), SymbolArchiveStatus.success); + }); + + test('various flutter versions are independent', () async { + await sut.setStatus( + FlutterVersion('1.2.3'), archive, SymbolArchiveStatus.success); + await sut.setStatus( + FlutterVersion('5.6.7'), archive, SymbolArchiveStatus.error); + expect(await sut.getStatus(FlutterVersion('1.2.3'), archive), + SymbolArchiveStatus.success); + expect(await sut.getStatus(FlutterVersion('5.6.7'), archive), + SymbolArchiveStatus.error); }); }); } From 6d163a6955249acb5a3a64cba6f921afa107af8c Mon Sep 17 00:00:00 2001 From: Ivan Dlugos Date: Wed, 25 Oct 2023 10:49:29 +0200 Subject: [PATCH 36/47] fix --- .github/workflows/flutter-symbols.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/flutter-symbols.yml b/.github/workflows/flutter-symbols.yml index 76e4d270d0..5e58a48193 100644 --- a/.github/workflows/flutter-symbols.yml +++ b/.github/workflows/flutter-symbols.yml @@ -38,7 +38,10 @@ jobs: - run: dart pub get # This downloads the database from the latest successful run. - - run: gh run download --name 'flutter-symbol-collector-database' + - name: Download status of previously processed files + run: | + gh run download --name 'flutter-symbol-collector-database' --dir .successful + find ./.successful/ -name '*.status' continue-on-error: true env: GITHUB_TOKEN: ${{ github.token }} From b7e47198c4a1b0b92bcf8afee0b3d1b432cb47be Mon Sep 17 00:00:00 2001 From: Ivan Dlugos Date: Wed, 25 Oct 2023 10:57:43 +0200 Subject: [PATCH 37/47] process all flutter v3 subversions --- .github/workflows/flutter-symbols.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/flutter-symbols.yml b/.github/workflows/flutter-symbols.yml index 5e58a48193..29efb215a6 100644 --- a/.github/workflows/flutter-symbols.yml +++ b/.github/workflows/flutter-symbols.yml @@ -46,7 +46,7 @@ jobs: env: GITHUB_TOKEN: ${{ github.token }} - - run: dart run bin/flutter_symbol_collector.dart --version=${{ inputs.flutter_version || '3.13.1' }} + - run: dart run bin/flutter_symbol_collector.dart --version=${{ inputs.flutter_version || '3.*.*' }} timeout-minutes: 300 env: GITHUB_TOKEN: ${{ github.token }} From 2edb9be13af3473b7d75e6c69398dfec5d0d2d11 Mon Sep 17 00:00:00 2001 From: Ivan Dlugos Date: Wed, 25 Oct 2023 13:07:25 +0200 Subject: [PATCH 38/47] try to fix memory usage --- .github/workflows/flutter-symbols.yml | 8 +++++--- .../bin/flutter_symbol_collector.dart | 4 ++++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/flutter-symbols.yml b/.github/workflows/flutter-symbols.yml index 29efb215a6..ab35c49453 100644 --- a/.github/workflows/flutter-symbols.yml +++ b/.github/workflows/flutter-symbols.yml @@ -17,7 +17,7 @@ defaults: jobs: test: - runs-on: ubuntu-latest + runs-on: macos-13 # macOS 13 runner has 4 CPUs and 14 GB of RAM steps: - uses: actions/checkout@v3 @@ -37,7 +37,6 @@ jobs: - run: dart pub get - # This downloads the database from the latest successful run. - name: Download status of previously processed files run: | gh run download --name 'flutter-symbol-collector-database' --dir .successful @@ -46,7 +45,10 @@ jobs: env: GITHUB_TOKEN: ${{ github.token }} - - run: dart run bin/flutter_symbol_collector.dart --version=${{ inputs.flutter_version || '3.*.*' }} + # An AOT compiled app should have slightly lower memory footprint than JIT. + - run: | + dart compile exe bin/flutter_symbol_collector.dart + ./bin/flutter_symbol_collector.exe --version=${{ inputs.flutter_version || '3.*.*' }} timeout-minutes: 300 env: GITHUB_TOKEN: ${{ github.token }} diff --git a/scripts/flutter_symbol_collector/bin/flutter_symbol_collector.dart b/scripts/flutter_symbol_collector/bin/flutter_symbol_collector.dart index 3170eaabc4..b1813695c9 100644 --- a/scripts/flutter_symbol_collector/bin/flutter_symbol_collector.dart +++ b/scripts/flutter_symbol_collector/bin/flutter_symbol_collector.dart @@ -81,6 +81,10 @@ Future processFlutterVerion(FlutterVersion version) async { await stateCache.setStatus(version, archive, SymbolArchiveStatus.error); } + if (await dir.exists()) { + await dir.delete(recursive: true); + } + if (bool.hasEnvironment('CI')) { print('::endgroup::'); } From adf8f60d8ac4fede155c83d5014d7437b2e0c06e Mon Sep 17 00:00:00 2001 From: Ivan Dlugos Date: Wed, 25 Oct 2023 14:35:38 +0200 Subject: [PATCH 39/47] cleanup earlier --- .../bin/flutter_symbol_collector.dart | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/scripts/flutter_symbol_collector/bin/flutter_symbol_collector.dart b/scripts/flutter_symbol_collector/bin/flutter_symbol_collector.dart index b1813695c9..9b51c8fb11 100644 --- a/scripts/flutter_symbol_collector/bin/flutter_symbol_collector.dart +++ b/scripts/flutter_symbol_collector/bin/flutter_symbol_collector.dart @@ -71,14 +71,20 @@ Future processFlutterVerion(FlutterVersion version) async { } final archiveDir = dir.childDirectory(archive.platform.operatingSystem); - if (await source.downloadAndExtractTo(archiveDir, archive.path)) { - if (await collector.upload(archiveDir, archive.platform, version)) { - await stateCache.setStatus( - version, archive, SymbolArchiveStatus.success); - continue; + try { + if (await source.downloadAndExtractTo(archiveDir, archive.path)) { + if (await collector.upload(archiveDir, archive.platform, version)) { + await stateCache.setStatus( + version, archive, SymbolArchiveStatus.success); + continue; + } + } + await stateCache.setStatus(version, archive, SymbolArchiveStatus.error); + } finally { + if (await archiveDir.exists()) { + await archiveDir.delete(recursive: true); } } - await stateCache.setStatus(version, archive, SymbolArchiveStatus.error); } if (await dir.exists()) { From 6529c4bc3f5d7f1837bf5b151eb3070fccd446b6 Mon Sep 17 00:00:00 2001 From: Ivan Dlugos Date: Wed, 25 Oct 2023 18:35:24 +0200 Subject: [PATCH 40/47] roll back changes after figuring out the issue is with symbol collector CLI --- .github/workflows/flutter-symbols.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/flutter-symbols.yml b/.github/workflows/flutter-symbols.yml index ab35c49453..b23559b5d1 100644 --- a/.github/workflows/flutter-symbols.yml +++ b/.github/workflows/flutter-symbols.yml @@ -17,7 +17,7 @@ defaults: jobs: test: - runs-on: macos-13 # macOS 13 runner has 4 CPUs and 14 GB of RAM + runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 @@ -45,10 +45,7 @@ jobs: env: GITHUB_TOKEN: ${{ github.token }} - # An AOT compiled app should have slightly lower memory footprint than JIT. - - run: | - dart compile exe bin/flutter_symbol_collector.dart - ./bin/flutter_symbol_collector.exe --version=${{ inputs.flutter_version || '3.*.*' }} + - run: dart run bin/flutter_symbol_collector.dart --version=${{ inputs.flutter_version || '3.*.*' }} timeout-minutes: 300 env: GITHUB_TOKEN: ${{ github.token }} From 6f1fc8da11e526b80c0a6b61f34aeb004c49f58f Mon Sep 17 00:00:00 2001 From: Ivan Dlugos Date: Thu, 26 Oct 2023 07:46:00 +0200 Subject: [PATCH 41/47] update symbol-collector to the latest version --- .../flutter_symbol_collector/lib/src/symbol_collector_cli.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/flutter_symbol_collector/lib/src/symbol_collector_cli.dart b/scripts/flutter_symbol_collector/lib/src/symbol_collector_cli.dart index 5448834e0c..b0ff4918d3 100644 --- a/scripts/flutter_symbol_collector/lib/src/symbol_collector_cli.dart +++ b/scripts/flutter_symbol_collector/lib/src/symbol_collector_cli.dart @@ -19,7 +19,7 @@ class SymbolCollectorCli { // https://github.com/getsentry/symbol-collector/releases @internal - static const version = '1.12.0'; + static const version = '1.12.1'; @internal late final String cli; From 7b9fb06fdd5535a861df2620943d035f5ac66ea9 Mon Sep 17 00:00:00 2001 From: Ivan Dlugos Date: Thu, 26 Oct 2023 10:39:57 +0200 Subject: [PATCH 42/47] rename .successful to .cache --- .github/workflows/flutter-symbols.yml | 11 ++++++----- scripts/flutter_symbol_collector/.gitignore | 2 +- .../bin/flutter_symbol_collector.dart | 2 +- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/.github/workflows/flutter-symbols.yml b/.github/workflows/flutter-symbols.yml index b23559b5d1..bd1cd62aa1 100644 --- a/.github/workflows/flutter-symbols.yml +++ b/.github/workflows/flutter-symbols.yml @@ -37,10 +37,10 @@ jobs: - run: dart pub get - - name: Download status of previously processed files + - name: Download status cache of previously processed files run: | - gh run download --name 'flutter-symbol-collector-database' --dir .successful - find ./.successful/ -name '*.status' + gh run download --name 'flutter-symbol-collector-database' --dir .cache + grep -r "" .cache continue-on-error: true env: GITHUB_TOKEN: ${{ github.token }} @@ -50,8 +50,9 @@ jobs: env: GITHUB_TOKEN: ${{ github.token }} - - uses: actions/upload-artifact@v3 + - name: Upload updated status cache of processed files + uses: actions/upload-artifact@v3 if: always() with: name: flutter-symbol-collector-database - path: scripts/flutter_symbol_collector/.successful + path: scripts/flutter_symbol_collector/.cache diff --git a/scripts/flutter_symbol_collector/.gitignore b/scripts/flutter_symbol_collector/.gitignore index 0aa8e5e343..d0e14b73aa 100644 --- a/scripts/flutter_symbol_collector/.gitignore +++ b/scripts/flutter_symbol_collector/.gitignore @@ -3,4 +3,4 @@ .dart_tool/ .temp -.successful +.cache diff --git a/scripts/flutter_symbol_collector/bin/flutter_symbol_collector.dart b/scripts/flutter_symbol_collector/bin/flutter_symbol_collector.dart index 9b51c8fb11..cb464d75c6 100644 --- a/scripts/flutter_symbol_collector/bin/flutter_symbol_collector.dart +++ b/scripts/flutter_symbol_collector/bin/flutter_symbol_collector.dart @@ -12,7 +12,7 @@ final source = FlutterSymbolSource(githubAuth: githubAuth); final fs = LocalFileSystem(); final tempDir = fs.currentDirectory.childDirectory('.temp'); final stateCache = - DirectoryStatusCache(fs.currentDirectory.childDirectory('.successful')); + DirectoryStatusCache(fs.currentDirectory.childDirectory('.cache')); late final SymbolCollectorCli collector; void main(List arguments) async { From e7a20f3bcf098debf5edaa0de510f8cabb3079b4 Mon Sep 17 00:00:00 2001 From: Ivan Dlugos Date: Thu, 26 Oct 2023 10:53:56 +0200 Subject: [PATCH 43/47] don't use version in the status cache --- .../bin/flutter_symbol_collector.dart | 7 ++-- .../lib/src/status_cache.dart | 22 +++++-------- .../test/status_cache_test.dart | 33 +++++-------------- 3 files changed, 21 insertions(+), 41 deletions(-) diff --git a/scripts/flutter_symbol_collector/bin/flutter_symbol_collector.dart b/scripts/flutter_symbol_collector/bin/flutter_symbol_collector.dart index cb464d75c6..a2957e15fb 100644 --- a/scripts/flutter_symbol_collector/bin/flutter_symbol_collector.dart +++ b/scripts/flutter_symbol_collector/bin/flutter_symbol_collector.dart @@ -63,7 +63,7 @@ Future processFlutterVerion(FlutterVersion version) async { final archives = await source.listSymbolArchives(version); final dir = tempDir.childDirectory(version.tagName); for (final archive in archives) { - final status = await stateCache.getStatus(version, archive); + final status = await stateCache.getStatus(archive); if (status == SymbolArchiveStatus.success) { Logger.root .info('Skipping ${archive.path} - already processed successfully'); @@ -74,12 +74,11 @@ Future processFlutterVerion(FlutterVersion version) async { try { if (await source.downloadAndExtractTo(archiveDir, archive.path)) { if (await collector.upload(archiveDir, archive.platform, version)) { - await stateCache.setStatus( - version, archive, SymbolArchiveStatus.success); + await stateCache.setStatus(archive, SymbolArchiveStatus.success); continue; } } - await stateCache.setStatus(version, archive, SymbolArchiveStatus.error); + await stateCache.setStatus(archive, SymbolArchiveStatus.error); } finally { if (await archiveDir.exists()) { await archiveDir.delete(recursive: true); diff --git a/scripts/flutter_symbol_collector/lib/src/status_cache.dart b/scripts/flutter_symbol_collector/lib/src/status_cache.dart index 4a97f53ce9..6c456b73c8 100644 --- a/scripts/flutter_symbol_collector/lib/src/status_cache.dart +++ b/scripts/flutter_symbol_collector/lib/src/status_cache.dart @@ -1,7 +1,6 @@ import 'package:file/file.dart'; import 'package:logging/logging.dart'; -import 'flutter_version.dart'; import 'symbol_archive.dart'; enum SymbolArchiveStatus { @@ -17,10 +16,8 @@ enum SymbolArchiveStatus { /// Stores and retrieves information about symbol processing status. abstract class SymbolArchiveStatusCache { - Future setStatus(FlutterVersion version, SymbolArchive archive, - SymbolArchiveStatus status); - Future getStatus( - FlutterVersion version, SymbolArchive archive); + Future setStatus(SymbolArchive archive, SymbolArchiveStatus status); + Future getStatus(SymbolArchive archive); } /// Stores information about symbol processing status in a local directory. @@ -31,13 +28,12 @@ class DirectoryStatusCache implements SymbolArchiveStatusCache { _dir.createSync(recursive: true); } - File _statusFile(FlutterVersion version, SymbolArchive archive) => - _dir.childFile('${version.tagName}/${archive.path.toLowerCase()}.status'); + File _statusFile(SymbolArchive archive) => + _dir.childFile('${archive.path.toLowerCase()}.status'); @override - Future getStatus( - FlutterVersion version, SymbolArchive archive) async { - final file = _statusFile(version, archive); + Future getStatus(SymbolArchive archive) async { + final file = _statusFile(archive); if (!await file.exists()) { return SymbolArchiveStatus.pending; } @@ -55,9 +51,9 @@ class DirectoryStatusCache implements SymbolArchiveStatusCache { } @override - Future setStatus(FlutterVersion version, SymbolArchive archive, - SymbolArchiveStatus status) async { - final file = _statusFile(version, archive); + Future setStatus( + SymbolArchive archive, SymbolArchiveStatus status) async { + final file = _statusFile(archive); Logger.root.info('Setting ${file.path} status to ${status.name}'); await file.create(recursive: true); await file.writeAsString(status.name); diff --git a/scripts/flutter_symbol_collector/test/status_cache_test.dart b/scripts/flutter_symbol_collector/test/status_cache_test.dart index dc643ef9c0..f8b929db79 100644 --- a/scripts/flutter_symbol_collector/test/status_cache_test.dart +++ b/scripts/flutter_symbol_collector/test/status_cache_test.dart @@ -12,7 +12,6 @@ void main() { group('DirectoryStatusCache', () { late FileSystem fs; late SymbolArchiveStatusCache sut; - final version = FlutterVersion('1.2.3'); final archive = SymbolArchive('path/to/archive.zip', LocalPlatform()); setUp(() { @@ -21,38 +20,24 @@ void main() { }); test('retrieve unprocessed file', () async { - expect( - await sut.getStatus(version, archive), SymbolArchiveStatus.pending); + expect(await sut.getStatus(archive), SymbolArchiveStatus.pending); }); test('store and retrieve error', () async { - await sut.setStatus(version, archive, SymbolArchiveStatus.error); - expect(await sut.getStatus(version, archive), SymbolArchiveStatus.error); + await sut.setStatus(archive, SymbolArchiveStatus.error); + expect(await sut.getStatus(archive), SymbolArchiveStatus.error); }); test('store and retrieve success', () async { - await sut.setStatus(version, archive, SymbolArchiveStatus.success); - expect( - await sut.getStatus(version, archive), SymbolArchiveStatus.success); + await sut.setStatus(archive, SymbolArchiveStatus.success); + expect(await sut.getStatus(archive), SymbolArchiveStatus.success); }); test('store, overwrite and retrieve', () async { - await sut.setStatus(version, archive, SymbolArchiveStatus.error); - expect(await sut.getStatus(version, archive), SymbolArchiveStatus.error); - await sut.setStatus(version, archive, SymbolArchiveStatus.success); - expect( - await sut.getStatus(version, archive), SymbolArchiveStatus.success); - }); - - test('various flutter versions are independent', () async { - await sut.setStatus( - FlutterVersion('1.2.3'), archive, SymbolArchiveStatus.success); - await sut.setStatus( - FlutterVersion('5.6.7'), archive, SymbolArchiveStatus.error); - expect(await sut.getStatus(FlutterVersion('1.2.3'), archive), - SymbolArchiveStatus.success); - expect(await sut.getStatus(FlutterVersion('5.6.7'), archive), - SymbolArchiveStatus.error); + await sut.setStatus(archive, SymbolArchiveStatus.error); + expect(await sut.getStatus(archive), SymbolArchiveStatus.error); + await sut.setStatus(archive, SymbolArchiveStatus.success); + expect(await sut.getStatus(archive), SymbolArchiveStatus.success); }); }); } From 946137245429def7ab9961bf5de6b13275685793 Mon Sep 17 00:00:00 2001 From: Ivan Dlugos Date: Thu, 26 Oct 2023 11:41:49 +0200 Subject: [PATCH 44/47] remove temp code --- .github/workflows/flutter-symbols.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/flutter-symbols.yml b/.github/workflows/flutter-symbols.yml index bd1cd62aa1..58341c32f1 100644 --- a/.github/workflows/flutter-symbols.yml +++ b/.github/workflows/flutter-symbols.yml @@ -1,6 +1,5 @@ name: Flutter symbols collection on: - push: # TODO REMOVE schedule: - cron: "10 */6 * * *" # every six hours workflow_dispatch: From 986f954e73c1a0df04315a8b6c32ce42f34765db Mon Sep 17 00:00:00 2001 From: Ivan Dlugos Date: Thu, 26 Oct 2023 12:12:05 +0200 Subject: [PATCH 45/47] run cron every hour --- .github/workflows/flutter-symbols.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/flutter-symbols.yml b/.github/workflows/flutter-symbols.yml index 58341c32f1..057240d94a 100644 --- a/.github/workflows/flutter-symbols.yml +++ b/.github/workflows/flutter-symbols.yml @@ -1,7 +1,8 @@ name: Flutter symbols collection on: schedule: - - cron: "10 */6 * * *" # every six hours + # Run once an hour. It takes just a couple of minutes because of status caching. + - cron: "10 * * * *" workflow_dispatch: inputs: flutter_version: From 9b74649db1cc4b04e2bfc7956fe6599bef4fe147 Mon Sep 17 00:00:00 2001 From: Ivan Dlugos Date: Fri, 27 Oct 2023 08:43:55 +0200 Subject: [PATCH 46/47] update symbol collector issue link --- .../flutter_symbol_collector/lib/src/symbol_collector_cli.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/flutter_symbol_collector/lib/src/symbol_collector_cli.dart b/scripts/flutter_symbol_collector/lib/src/symbol_collector_cli.dart index b0ff4918d3..d825c0c89f 100644 --- a/scripts/flutter_symbol_collector/lib/src/symbol_collector_cli.dart +++ b/scripts/flutter_symbol_collector/lib/src/symbol_collector_cli.dart @@ -138,7 +138,7 @@ class SymbolCollectorCli { if (exitCode != 0) { throw Exception('Symbol-collector CLI failed with exit code $exitCode.'); } else if (strOutput.contains('Exception:')) { - // see https://github.com/getsentry/symbol-collector/issues/162 + // see https://github.com/getsentry/symbol-collector/issues/167 throw Exception('Symbol-collector CLI failed with an exception.'); } From 0d8652b1f274eca32c7e205491796afbabb0942e Mon Sep 17 00:00:00 2001 From: Ivan Dlugos Date: Mon, 30 Oct 2023 12:13:58 +0100 Subject: [PATCH 47/47] minor fixes --- .../bin/flutter_symbol_collector.dart | 6 +++--- .../lib/src/symbol_collector_cli.dart | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/flutter_symbol_collector/bin/flutter_symbol_collector.dart b/scripts/flutter_symbol_collector/bin/flutter_symbol_collector.dart index a2957e15fb..424134eea7 100644 --- a/scripts/flutter_symbol_collector/bin/flutter_symbol_collector.dart +++ b/scripts/flutter_symbol_collector/bin/flutter_symbol_collector.dart @@ -33,7 +33,7 @@ void main(List arguments) async { !argVersion.contains('*') && argVersion.split('.').length == 3) { Logger.root.info('Running for a single flutter version: $argVersion'); - await processFlutterVerion(FlutterVersion(argVersion)); + await processFlutterVersion(FlutterVersion(argVersion)); } else { // Otherwise, walk all the versions and run for the matching ones. final versionRegex = RegExp(argVersion.isEmpty @@ -48,12 +48,12 @@ void main(List arguments) async { Logger.root.info( 'Found ${versions.length} Flutter versions matching $versionRegex'); for (var version in versions) { - await processFlutterVerion(version); + await processFlutterVersion(version); } } } -Future processFlutterVerion(FlutterVersion version) async { +Future processFlutterVersion(FlutterVersion version) async { if (bool.hasEnvironment('CI')) { print('::group::Processing Flutter ${version.tagName}'); } diff --git a/scripts/flutter_symbol_collector/lib/src/symbol_collector_cli.dart b/scripts/flutter_symbol_collector/lib/src/symbol_collector_cli.dart index d825c0c89f..0f4c97d6a6 100644 --- a/scripts/flutter_symbol_collector/lib/src/symbol_collector_cli.dart +++ b/scripts/flutter_symbol_collector/lib/src/symbol_collector_cli.dart @@ -70,13 +70,13 @@ class SymbolCollectorCli { void _ensureIsExecutable() { if (!_isExecutable) { - _isExecutable = true; if (LocalPlatform().operatingSystem == platform.operatingSystem) { if (platform.isLinux || platform.isMacOS) { _log.fine('Making Symbol-collector CLI executable (chmod +x)'); posix.chmod(cli, '0700'); } + _isExecutable = true; } else { _log.warning( 'Symbol-collector CLI has been run with a platform that is not the current OS platform.'