"Fossies" - the Fresh Open Source Software Archive

Member "flutter-3.7.0/dev/devicelab/lib/framework/ab.dart" (24 Jan 2023, 10664 Bytes) of package /linux/misc/flutter-3.7.0.tar.gz:


As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) Dart source code syntax highlighting (style: standard) with prefixed line numbers and code folding option. Alternatively you can here view or download the uninterpreted source code file.

    1 // Copyright 2014 The Flutter Authors. All rights reserved.
    2 // Use of this source code is governed by a BSD-style license that can be
    3 // found in the LICENSE file.
    4 
    5 import 'dart:math' as math;
    6 
    7 import 'task_result.dart';
    8 
    9 const String kBenchmarkTypeKeyName = 'benchmark_type';
   10 const String kBenchmarkVersionKeyName = 'version';
   11 const String kLocalEngineKeyName = 'local_engine';
   12 const String kTaskNameKeyName = 'task_name';
   13 const String kRunStartKeyName = 'run_start';
   14 const String kRunEndKeyName = 'run_end';
   15 const String kAResultsKeyName = 'default_results';
   16 const String kBResultsKeyName = 'local_engine_results';
   17 
   18 const String kBenchmarkResultsType = 'A/B summaries';
   19 const String kBenchmarkABVersion = '1.0';
   20 
   21 enum FieldJustification { LEFT, RIGHT, CENTER }
   22 
   23 /// Collects data from an A/B test and produces a summary for human evaluation.
   24 ///
   25 /// See [printSummary] for more.
   26 class ABTest {
   27   ABTest(this.localEngine, this.taskName)
   28       : runStart = DateTime.now(),
   29         _aResults = <String, List<double>>{},
   30         _bResults = <String, List<double>>{};
   31 
   32   ABTest.fromJsonMap(Map<String, dynamic> jsonResults)
   33       : localEngine = jsonResults[kLocalEngineKeyName] as String,
   34         taskName = jsonResults[kTaskNameKeyName] as String,
   35         runStart = DateTime.parse(jsonResults[kRunStartKeyName] as String),
   36         _runEnd = DateTime.parse(jsonResults[kRunEndKeyName] as String),
   37         _aResults = _convertFrom(jsonResults[kAResultsKeyName] as Map<String, dynamic>),
   38         _bResults = _convertFrom(jsonResults[kBResultsKeyName] as Map<String, dynamic>);
   39 
   40   final String localEngine;
   41   final String taskName;
   42   final DateTime runStart;
   43   DateTime? _runEnd;
   44   DateTime? get runEnd => _runEnd;
   45 
   46   final Map<String, List<double>> _aResults;
   47   final Map<String, List<double>> _bResults;
   48 
   49   static Map<String, List<double>> _convertFrom(dynamic results) {
   50     final Map<String, dynamic> resultMap = results as Map<String, dynamic>;
   51     return <String, List<double>> {
   52       for (String key in resultMap.keys)
   53         key: (resultMap[key] as List<dynamic>).cast<double>(),
   54     };
   55   }
   56 
   57   /// Adds the result of a single A run of the benchmark.
   58   ///
   59   /// The result may contain multiple score keys.
   60   ///
   61   /// [result] is expected to be a serialization of [TaskResult].
   62   void addAResult(TaskResult result) {
   63     if (_runEnd != null) {
   64       throw StateError('Cannot add results to ABTest after it is finalized');
   65     }
   66     _addResult(result, _aResults);
   67   }
   68 
   69   /// Adds the result of a single B run of the benchmark.
   70   ///
   71   /// The result may contain multiple score keys.
   72   ///
   73   /// [result] is expected to be a serialization of [TaskResult].
   74   void addBResult(TaskResult result) {
   75     if (_runEnd != null) {
   76       throw StateError('Cannot add results to ABTest after it is finalized');
   77     }
   78     _addResult(result, _bResults);
   79   }
   80 
   81   void finalize() {
   82     _runEnd = DateTime.now();
   83   }
   84 
   85   Map<String, dynamic> get jsonMap => <String, dynamic>{
   86     kBenchmarkTypeKeyName:     kBenchmarkResultsType,
   87     kBenchmarkVersionKeyName:  kBenchmarkABVersion,
   88     kLocalEngineKeyName:       localEngine,
   89     kTaskNameKeyName:          taskName,
   90     kRunStartKeyName:          runStart.toIso8601String(),
   91     kRunEndKeyName:            runEnd!.toIso8601String(),
   92     kAResultsKeyName:          _aResults,
   93     kBResultsKeyName:          _bResults,
   94   };
   95 
   96   static void updateColumnLengths(List<int> lengths, List<String?> results) {
   97     for (int column = 0; column < lengths.length; column++) {
   98       if (results[column] != null) {
   99         lengths[column] = math.max(lengths[column], results[column]?.length ?? 0);
  100       }
  101     }
  102   }
  103 
  104   static void formatResult(StringBuffer buffer,
  105                            List<int> lengths,
  106                            List<FieldJustification> aligns,
  107                            List<String?> values) {
  108     for (int column = 0; column < lengths.length; column++) {
  109       final int len = lengths[column];
  110       String? value = values[column];
  111       if (value == null) {
  112         value = ''.padRight(len);
  113       } else {
  114         switch (aligns[column]) {
  115           case FieldJustification.LEFT:
  116             value = value.padRight(len);
  117             break;
  118           case FieldJustification.RIGHT:
  119             value = value.padLeft(len);
  120             break;
  121           case FieldJustification.CENTER:
  122             value = value.padLeft((len + value.length) ~/2);
  123             value = value.padRight(len);
  124             break;
  125         }
  126       }
  127       if (column > 0) {
  128         value = value.padLeft(len+1);
  129       }
  130       buffer.write(value);
  131     }
  132     buffer.writeln();
  133   }
  134 
  135   /// Returns the summary as a tab-separated spreadsheet.
  136   ///
  137   /// This value can be copied straight to a Google Spreadsheet for further analysis.
  138   String asciiSummary() {
  139     final Map<String, _ScoreSummary> summariesA = _summarize(_aResults);
  140     final Map<String, _ScoreSummary> summariesB = _summarize(_bResults);
  141 
  142     final List<List<String?>> tableRows = <List<String?>>[
  143       for (final String scoreKey in <String>{...summariesA.keys, ...summariesB.keys})
  144         <String?>[
  145           scoreKey,
  146           summariesA[scoreKey]?.averageString, summariesA[scoreKey]?.noiseString,
  147           summariesB[scoreKey]?.averageString, summariesB[scoreKey]?.noiseString,
  148           summariesA[scoreKey]?.improvementOver(summariesB[scoreKey]),
  149         ],
  150     ];
  151 
  152     final List<String> titles = <String>[
  153       'Score',
  154       'Average A', '(noise)',
  155       'Average B', '(noise)',
  156       'Speed-up',
  157     ];
  158     final List<FieldJustification> alignments = <FieldJustification>[
  159       FieldJustification.LEFT,
  160       FieldJustification.RIGHT, FieldJustification.LEFT,
  161       FieldJustification.RIGHT, FieldJustification.LEFT,
  162       FieldJustification.CENTER,
  163     ];
  164 
  165     final List<int> lengths = List<int>.filled(6, 0);
  166     updateColumnLengths(lengths, titles);
  167     for (final List<String?> row in tableRows) {
  168       updateColumnLengths(lengths, row);
  169     }
  170 
  171     final StringBuffer buffer = StringBuffer();
  172     formatResult(buffer, lengths,
  173         <FieldJustification>[
  174           FieldJustification.CENTER,
  175           ...alignments.skip(1),
  176         ], titles);
  177     for (final List<String?> row in tableRows) {
  178       formatResult(buffer, lengths, alignments, row);
  179     }
  180 
  181     return buffer.toString();
  182   }
  183 
  184   /// Returns unprocessed data collected by the A/B test formatted as
  185   /// a tab-separated spreadsheet.
  186   String rawResults() {
  187     final StringBuffer buffer = StringBuffer();
  188     for (final String scoreKey in _allScoreKeys) {
  189       buffer.writeln('$scoreKey:');
  190       buffer.write('  A:\t');
  191       if (_aResults.containsKey(scoreKey)) {
  192         for (final double score in _aResults[scoreKey]!) {
  193           buffer.write('${score.toStringAsFixed(2)}\t');
  194         }
  195       } else {
  196         buffer.write('N/A');
  197       }
  198       buffer.writeln();
  199 
  200       buffer.write('  B:\t');
  201       if (_bResults.containsKey(scoreKey)) {
  202         for (final double score in _bResults[scoreKey]!) {
  203           buffer.write('${score.toStringAsFixed(2)}\t');
  204         }
  205       } else {
  206         buffer.write('N/A');
  207       }
  208       buffer.writeln();
  209     }
  210     return buffer.toString();
  211   }
  212 
  213   Set<String> get _allScoreKeys {
  214     return <String>{
  215       ..._aResults.keys,
  216       ..._bResults.keys,
  217     };
  218   }
  219 
  220   /// Returns the summary as a tab-separated spreadsheet.
  221   ///
  222   /// This value can be copied straight to a Google Spreadsheet for further analysis.
  223   String printSummary() {
  224     final Map<String, _ScoreSummary> summariesA = _summarize(_aResults);
  225     final Map<String, _ScoreSummary> summariesB = _summarize(_bResults);
  226 
  227     final StringBuffer buffer = StringBuffer(
  228       'Score\tAverage A (noise)\tAverage B (noise)\tSpeed-up\n',
  229     );
  230 
  231     for (final String scoreKey in _allScoreKeys) {
  232       final _ScoreSummary? summaryA = summariesA[scoreKey];
  233       final _ScoreSummary? summaryB = summariesB[scoreKey];
  234       buffer.write('$scoreKey\t');
  235 
  236       if (summaryA != null) {
  237         buffer.write('${summaryA.averageString} ${summaryA.noiseString}\t');
  238       } else {
  239         buffer.write('\t');
  240       }
  241 
  242       if (summaryB != null) {
  243         buffer.write('${summaryB.averageString} ${summaryB.noiseString}\t');
  244       } else {
  245         buffer.write('\t');
  246       }
  247 
  248       if (summaryA != null && summaryB != null) {
  249         buffer.write('${summaryA.improvementOver(summaryB)}\t');
  250       }
  251 
  252       buffer.writeln();
  253     }
  254 
  255     return buffer.toString();
  256   }
  257 }
  258 
  259 class _ScoreSummary {
  260   _ScoreSummary({
  261     required this.average,
  262     required this.noise,
  263   });
  264 
  265   /// Average (arithmetic mean) of a series of values collected by a benchmark.
  266   final double average;
  267 
  268   /// The noise (standard deviation divided by [average]) in the collected
  269   /// values.
  270   final double noise;
  271 
  272   String get averageString => average.toStringAsFixed(2);
  273   String get noiseString => '(${_ratioToPercent(noise)})';
  274 
  275   String improvementOver(_ScoreSummary? other) {
  276     return other == null ? '' : '${(average / other.average).toStringAsFixed(2)}x';
  277   }
  278 }
  279 
  280 void _addResult(TaskResult result, Map<String, List<double>> results) {
  281   for (final String scoreKey in result.benchmarkScoreKeys ?? <String>[]) {
  282     final double score = (result.data![scoreKey] as num).toDouble();
  283     results.putIfAbsent(scoreKey, () => <double>[]).add(score);
  284   }
  285 }
  286 
  287 Map<String, _ScoreSummary> _summarize(Map<String, List<double>> results) {
  288   return results.map<String, _ScoreSummary>((String scoreKey, List<double> values) {
  289     final double average = _computeAverage(values);
  290     return MapEntry<String, _ScoreSummary>(scoreKey, _ScoreSummary(
  291       average: average,
  292       // If the average is zero, the benchmark got the perfect score with no noise.
  293       noise: average > 0
  294         ? _computeStandardDeviationForPopulation(values) / average
  295         : 0.0,
  296     ));
  297   });
  298 }
  299 
  300 /// Computes the arithmetic mean (or average) of given [values].
  301 double _computeAverage(Iterable<double> values) {
  302   final double sum = values.reduce((double a, double b) => a + b);
  303   return sum / values.length;
  304 }
  305 
  306 /// Computes population standard deviation.
  307 ///
  308 /// Unlike sample standard deviation, which divides by N - 1, this divides by N.
  309 ///
  310 /// See also:
  311 ///
  312 /// * https://en.wikipedia.org/wiki/Standard_deviation
  313 double _computeStandardDeviationForPopulation(Iterable<double> population) {
  314   final double mean = _computeAverage(population);
  315   final double sumOfSquaredDeltas = population.fold<double>(
  316     0.0,
  317     (double previous, num value) => previous += math.pow(value - mean, 2),
  318   );
  319   return math.sqrt(sumOfSquaredDeltas / population.length);
  320 }
  321 
  322 String _ratioToPercent(double value) {
  323   return '${(value * 100).toStringAsFixed(2)}%';
  324 }