benchmark: maintain no. of parallel requests in throughput benchmark (#17)

Author: unsuitable001

README.md

@@ -2,8 +2,6 @@
 
 This package binds to Cronet's [native API](https://chromium.googlesource.com/chromium/src/+/master/components/cronet/native/test_instructions.md) to expose them in Dart.
 
-This is an HTTP Client Package with [almost](dart_io_comparison.md#api-comparison) the same API as `dart:io`. By comparison, `package:cronet` is capable of serving [about 4 to 5 times](dart_io_comparison.md#throughput-parallel-requests) more parallel network requests than `dart:io` and on par in making sequential network requests.
-
 This is a [GSoC 2021 project](https://summerofcode.withgoogle.com/projects/#4757095741652992).
 
 ## Supported Platforms

benchmark/latency.dart

@@ -20,7 +20,7 @@ abstract class LatencyBenchmark {
     await run();
   }
 
-  static Future<double> measureFor(Function f, Duration duration) async {
+  Future<double> measureFor(Function f, Duration duration) async {
     var durationInMilliseconds = duration.inMilliseconds;
     var iter = 0;
     var watch = Stopwatch();

benchmark/run_all.dart

@@ -118,12 +118,13 @@ void main(List<String> args) async {
   print('| AOT           | ${aotCronetLatency.toStringAsFixed(3)} ms   |'
       ' ${aotDartIOLatency.toStringAsFixed(3)} ms   |');
   print('\nThroughput Test Results (Duration: ${duration.inSeconds}s)');
-  print('| Mode          | package:cronet  | dart:io        |');
-  print('| :-----------: |:--------------: | :-----------:  |');
-  print('| JIT           | ${jitCronetThroughput[1]} out of'
-      ' ${jitCronetThroughput[0]}  | ${jitDartIOThroughput[1]} out of'
-      ' ${jitDartIOThroughput[0]}  |');
-  print('| AOT           | ${aotCronetThroughput[1]} out of'
-      ' ${aotCronetThroughput[0]} | ${aotDartIOThroughput[1]} out of'
-      ' ${aotDartIOThroughput[0]} |');
+  print('Considering the best appearing value only');
+  print('| Mode | package:cronet             | dart:io                   |');
+  print('| :--: |:-------------------------: | :----------------------:  |');
+  print('| JIT  | ${jitCronetThroughput[1]} (Parallel Requests:'
+      ' ${jitCronetThroughput[0]})| ${jitDartIOThroughput[1]}'
+      ' (Parallel Requests: ${jitDartIOThroughput[0]})|');
+  print('| AOT  | ${aotCronetThroughput[1]} (Parallel Requests: '
+      ' ${aotCronetThroughput[0]})| ${aotDartIOThroughput[1]}'
+      ' (Parallel Requests: ${aotDartIOThroughput[0]})|');
 }

benchmark/throughput.dart

@@ -11,36 +11,47 @@ import 'package:cronet/cronet.dart';
 
 abstract class ThroughputBenchmark {
   final String url;
-  final int spawnThreshold;
+  final int parallelLimit;
   final Duration duration;
 
   Future<void> run();
   void setup();
   void teardown();
   Future<void> warmup();
 
-  ThroughputBenchmark(this.url, this.spawnThreshold, this.duration);
+  ThroughputBenchmark(this.url, this.parallelLimit, this.duration);
 
-  static Future<List<int>> measureFor(
-      Future<void> Function() f, Duration duration, int maxSpawn) async {
+  Future<List<int>> measureFor(
+      Future<void> Function() f, Duration duration, int parallelLimit) async {
     var durationInMicroseconds = duration.inMicroseconds;
     var inTimeReturns = 0;
     var lateReturns = 0;
+    var spawned = 0;
     var watch = Stopwatch();
     final completer = Completer<List<int>>();
-    watch.start();
-    for (int i = 0; i < maxSpawn; i++) {
-      f().then((_) {
-        if (watch.elapsedMicroseconds < durationInMicroseconds) {
-          inTimeReturns++;
-        } else {
-          watch.stop();
+
+    FutureOr<void> _measureAndRespawn(void _) {
+      if (watch.elapsedMicroseconds < durationInMicroseconds) {
+        inTimeReturns++;
+        f().then(_measureAndRespawn).onError((error, stackTrace) {
           lateReturns++;
+        });
+        spawned++;
+      } else {
+        watch.stop();
+        lateReturns++;
+        if (inTimeReturns + lateReturns == spawned) {
+          completer.complete([parallelLimit, inTimeReturns]);
         }
-        if (inTimeReturns + lateReturns == maxSpawn) {
-          completer.complete([maxSpawn, inTimeReturns]);
-        }
-      }).onError((error, stackTrace) {});
+      }
+    }
+
+    watch.start();
+    for (int i = 0; i < parallelLimit; i++) {
+      f().then(_measureAndRespawn).onError((error, stackTrace) {
+        lateReturns++;
+      });
+      spawned++;
     }
     return completer.future;
   }
@@ -58,12 +69,12 @@ abstract class ThroughputBenchmark {
   Future<List<int>> report() async {
     var maxReturn = 0;
     var throughput = [0, 0];
-    // Run the benchmark for 1, 2, 4...spawnThreshold.
+    // Run the benchmark for 1, 2, 4...parallelLimit.
     for (int currentThreshold = 1;
-        currentThreshold <= spawnThreshold;
+        currentThreshold <= parallelLimit;
         currentThreshold *= 2) {
       final res = await measure(currentThreshold);
-      print('$runtimeType: Total Spawned: ${res[0]},'
+      print('$runtimeType: Parallel Requests: ${res[0]},'
           ' Returned in time: ${res[1]}.');
       if (res[1] > maxReturn) {
         maxReturn = res[1];

dart_io_comparison.md

@@ -33,6 +33,14 @@ Payload: Lorem Ipsum Text
 | JIT           | 1.807 ms       | **1.801 ms**   |
 | AOT           | 1.441 ms       | **1.049 ms**   |
 
+Server: HTTP/2 example.com Server
+Payload: example.com index.html page
+
+| Mode          | package:cronet | dart:io        |
+| :-----------: |:-------------: | :------------: |
+| JIT           | **90.696 ms**  | 104.150 ms     |
+| AOT           | **89.348 ms**  | 104.050 ms     |
+
 Server: HTTP/2 Google Server
 Payload: Google Chrome Debian Package
 
@@ -43,24 +51,11 @@ Payload: Google Chrome Debian Package
 
 ### Throughput (Parallel Requests)
 
-_Contents written as, x concurrent requests succesfully completed out of y requests spawned within 1 second._
-
-Server: HTTP/2 Local Caddy Server
-Payload: example.org 's index.html page
-
-| Mode          | package:cronet     | dart:io         |
-| :-----------: |:------------------:| :--------------:|
-| JIT           |**2982 out of 4096**| 512 out of 512  |
-| AOT           |**2883 out of 4096**| 512 out of 512  |
-
-*`dart:io`'s successful requests went down to 1 when more than 512 requests are spawned.*
-
 Server: HTTP/2 example.com Server
 Payload: example.com index.html page
 
-| Mode          | package:cronet     | dart:io        |
-| :-----------: |:------------------:| :-------------:|
-| JIT           |**178 out of 512**  | 39 out of 128  |
-| AOT           |**214 out of 512**  | 49 out of 128  |
-
-*These are the best results. `dart:io`'s successful requests went down to rapidly to 0 as we reach 512 requests mark.*
+Considering the best appearing value only
+| Mode | package:cronet               | dart:io                      |
+| :--: |:-------------------------:   | :----------------------:     |
+| JIT  | 855 (Parallel Requests: 256) | 1078 (Parallel Requests: 256)|
+| AOT  | 789 (Parallel Requests: 128) | 1306 (Parallel Requests: 512)|