"Fossies" - the Fresh Open Source Software Archive 
Member "flutter-3.7.0/dev/devicelab/lib/framework/ab.dart" (24 Jan 2023, 10664 Bytes) of package /linux/misc/flutter-3.7.0.tar.gz:
As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) Dart source code syntax highlighting (style:
standard) with prefixed line numbers and
code folding option.
Alternatively you can here
view or
download the uninterpreted source code file.
1 // Copyright 2014 The Flutter Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 import 'dart:math' as math;
6
7 import 'task_result.dart';
8
9 const String kBenchmarkTypeKeyName = 'benchmark_type';
10 const String kBenchmarkVersionKeyName = 'version';
11 const String kLocalEngineKeyName = 'local_engine';
12 const String kTaskNameKeyName = 'task_name';
13 const String kRunStartKeyName = 'run_start';
14 const String kRunEndKeyName = 'run_end';
15 const String kAResultsKeyName = 'default_results';
16 const String kBResultsKeyName = 'local_engine_results';
17
18 const String kBenchmarkResultsType = 'A/B summaries';
19 const String kBenchmarkABVersion = '1.0';
20
21 enum FieldJustification { LEFT, RIGHT, CENTER }
22
23 /// Collects data from an A/B test and produces a summary for human evaluation.
24 ///
25 /// See [printSummary] for more.
26 class ABTest {
27 ABTest(this.localEngine, this.taskName)
28 : runStart = DateTime.now(),
29 _aResults = <String, List<double>>{},
30 _bResults = <String, List<double>>{};
31
32 ABTest.fromJsonMap(Map<String, dynamic> jsonResults)
33 : localEngine = jsonResults[kLocalEngineKeyName] as String,
34 taskName = jsonResults[kTaskNameKeyName] as String,
35 runStart = DateTime.parse(jsonResults[kRunStartKeyName] as String),
36 _runEnd = DateTime.parse(jsonResults[kRunEndKeyName] as String),
37 _aResults = _convertFrom(jsonResults[kAResultsKeyName] as Map<String, dynamic>),
38 _bResults = _convertFrom(jsonResults[kBResultsKeyName] as Map<String, dynamic>);
39
40 final String localEngine;
41 final String taskName;
42 final DateTime runStart;
43 DateTime? _runEnd;
44 DateTime? get runEnd => _runEnd;
45
46 final Map<String, List<double>> _aResults;
47 final Map<String, List<double>> _bResults;
48
49 static Map<String, List<double>> _convertFrom(dynamic results) {
50 final Map<String, dynamic> resultMap = results as Map<String, dynamic>;
51 return <String, List<double>> {
52 for (String key in resultMap.keys)
53 key: (resultMap[key] as List<dynamic>).cast<double>(),
54 };
55 }
56
57 /// Adds the result of a single A run of the benchmark.
58 ///
59 /// The result may contain multiple score keys.
60 ///
61 /// [result] is expected to be a serialization of [TaskResult].
62 void addAResult(TaskResult result) {
63 if (_runEnd != null) {
64 throw StateError('Cannot add results to ABTest after it is finalized');
65 }
66 _addResult(result, _aResults);
67 }
68
69 /// Adds the result of a single B run of the benchmark.
70 ///
71 /// The result may contain multiple score keys.
72 ///
73 /// [result] is expected to be a serialization of [TaskResult].
74 void addBResult(TaskResult result) {
75 if (_runEnd != null) {
76 throw StateError('Cannot add results to ABTest after it is finalized');
77 }
78 _addResult(result, _bResults);
79 }
80
81 void finalize() {
82 _runEnd = DateTime.now();
83 }
84
85 Map<String, dynamic> get jsonMap => <String, dynamic>{
86 kBenchmarkTypeKeyName: kBenchmarkResultsType,
87 kBenchmarkVersionKeyName: kBenchmarkABVersion,
88 kLocalEngineKeyName: localEngine,
89 kTaskNameKeyName: taskName,
90 kRunStartKeyName: runStart.toIso8601String(),
91 kRunEndKeyName: runEnd!.toIso8601String(),
92 kAResultsKeyName: _aResults,
93 kBResultsKeyName: _bResults,
94 };
95
96 static void updateColumnLengths(List<int> lengths, List<String?> results) {
97 for (int column = 0; column < lengths.length; column++) {
98 if (results[column] != null) {
99 lengths[column] = math.max(lengths[column], results[column]?.length ?? 0);
100 }
101 }
102 }
103
104 static void formatResult(StringBuffer buffer,
105 List<int> lengths,
106 List<FieldJustification> aligns,
107 List<String?> values) {
108 for (int column = 0; column < lengths.length; column++) {
109 final int len = lengths[column];
110 String? value = values[column];
111 if (value == null) {
112 value = ''.padRight(len);
113 } else {
114 switch (aligns[column]) {
115 case FieldJustification.LEFT:
116 value = value.padRight(len);
117 break;
118 case FieldJustification.RIGHT:
119 value = value.padLeft(len);
120 break;
121 case FieldJustification.CENTER:
122 value = value.padLeft((len + value.length) ~/2);
123 value = value.padRight(len);
124 break;
125 }
126 }
127 if (column > 0) {
128 value = value.padLeft(len+1);
129 }
130 buffer.write(value);
131 }
132 buffer.writeln();
133 }
134
135 /// Returns the summary as a tab-separated spreadsheet.
136 ///
137 /// This value can be copied straight to a Google Spreadsheet for further analysis.
138 String asciiSummary() {
139 final Map<String, _ScoreSummary> summariesA = _summarize(_aResults);
140 final Map<String, _ScoreSummary> summariesB = _summarize(_bResults);
141
142 final List<List<String?>> tableRows = <List<String?>>[
143 for (final String scoreKey in <String>{...summariesA.keys, ...summariesB.keys})
144 <String?>[
145 scoreKey,
146 summariesA[scoreKey]?.averageString, summariesA[scoreKey]?.noiseString,
147 summariesB[scoreKey]?.averageString, summariesB[scoreKey]?.noiseString,
148 summariesA[scoreKey]?.improvementOver(summariesB[scoreKey]),
149 ],
150 ];
151
152 final List<String> titles = <String>[
153 'Score',
154 'Average A', '(noise)',
155 'Average B', '(noise)',
156 'Speed-up',
157 ];
158 final List<FieldJustification> alignments = <FieldJustification>[
159 FieldJustification.LEFT,
160 FieldJustification.RIGHT, FieldJustification.LEFT,
161 FieldJustification.RIGHT, FieldJustification.LEFT,
162 FieldJustification.CENTER,
163 ];
164
165 final List<int> lengths = List<int>.filled(6, 0);
166 updateColumnLengths(lengths, titles);
167 for (final List<String?> row in tableRows) {
168 updateColumnLengths(lengths, row);
169 }
170
171 final StringBuffer buffer = StringBuffer();
172 formatResult(buffer, lengths,
173 <FieldJustification>[
174 FieldJustification.CENTER,
175 ...alignments.skip(1),
176 ], titles);
177 for (final List<String?> row in tableRows) {
178 formatResult(buffer, lengths, alignments, row);
179 }
180
181 return buffer.toString();
182 }
183
184 /// Returns unprocessed data collected by the A/B test formatted as
185 /// a tab-separated spreadsheet.
186 String rawResults() {
187 final StringBuffer buffer = StringBuffer();
188 for (final String scoreKey in _allScoreKeys) {
189 buffer.writeln('$scoreKey:');
190 buffer.write(' A:\t');
191 if (_aResults.containsKey(scoreKey)) {
192 for (final double score in _aResults[scoreKey]!) {
193 buffer.write('${score.toStringAsFixed(2)}\t');
194 }
195 } else {
196 buffer.write('N/A');
197 }
198 buffer.writeln();
199
200 buffer.write(' B:\t');
201 if (_bResults.containsKey(scoreKey)) {
202 for (final double score in _bResults[scoreKey]!) {
203 buffer.write('${score.toStringAsFixed(2)}\t');
204 }
205 } else {
206 buffer.write('N/A');
207 }
208 buffer.writeln();
209 }
210 return buffer.toString();
211 }
212
213 Set<String> get _allScoreKeys {
214 return <String>{
215 ..._aResults.keys,
216 ..._bResults.keys,
217 };
218 }
219
220 /// Returns the summary as a tab-separated spreadsheet.
221 ///
222 /// This value can be copied straight to a Google Spreadsheet for further analysis.
223 String printSummary() {
224 final Map<String, _ScoreSummary> summariesA = _summarize(_aResults);
225 final Map<String, _ScoreSummary> summariesB = _summarize(_bResults);
226
227 final StringBuffer buffer = StringBuffer(
228 'Score\tAverage A (noise)\tAverage B (noise)\tSpeed-up\n',
229 );
230
231 for (final String scoreKey in _allScoreKeys) {
232 final _ScoreSummary? summaryA = summariesA[scoreKey];
233 final _ScoreSummary? summaryB = summariesB[scoreKey];
234 buffer.write('$scoreKey\t');
235
236 if (summaryA != null) {
237 buffer.write('${summaryA.averageString} ${summaryA.noiseString}\t');
238 } else {
239 buffer.write('\t');
240 }
241
242 if (summaryB != null) {
243 buffer.write('${summaryB.averageString} ${summaryB.noiseString}\t');
244 } else {
245 buffer.write('\t');
246 }
247
248 if (summaryA != null && summaryB != null) {
249 buffer.write('${summaryA.improvementOver(summaryB)}\t');
250 }
251
252 buffer.writeln();
253 }
254
255 return buffer.toString();
256 }
257 }
258
259 class _ScoreSummary {
260 _ScoreSummary({
261 required this.average,
262 required this.noise,
263 });
264
265 /// Average (arithmetic mean) of a series of values collected by a benchmark.
266 final double average;
267
268 /// The noise (standard deviation divided by [average]) in the collected
269 /// values.
270 final double noise;
271
272 String get averageString => average.toStringAsFixed(2);
273 String get noiseString => '(${_ratioToPercent(noise)})';
274
275 String improvementOver(_ScoreSummary? other) {
276 return other == null ? '' : '${(average / other.average).toStringAsFixed(2)}x';
277 }
278 }
279
280 void _addResult(TaskResult result, Map<String, List<double>> results) {
281 for (final String scoreKey in result.benchmarkScoreKeys ?? <String>[]) {
282 final double score = (result.data![scoreKey] as num).toDouble();
283 results.putIfAbsent(scoreKey, () => <double>[]).add(score);
284 }
285 }
286
287 Map<String, _ScoreSummary> _summarize(Map<String, List<double>> results) {
288 return results.map<String, _ScoreSummary>((String scoreKey, List<double> values) {
289 final double average = _computeAverage(values);
290 return MapEntry<String, _ScoreSummary>(scoreKey, _ScoreSummary(
291 average: average,
292 // If the average is zero, the benchmark got the perfect score with no noise.
293 noise: average > 0
294 ? _computeStandardDeviationForPopulation(values) / average
295 : 0.0,
296 ));
297 });
298 }
299
300 /// Computes the arithmetic mean (or average) of given [values].
301 double _computeAverage(Iterable<double> values) {
302 final double sum = values.reduce((double a, double b) => a + b);
303 return sum / values.length;
304 }
305
306 /// Computes population standard deviation.
307 ///
308 /// Unlike sample standard deviation, which divides by N - 1, this divides by N.
309 ///
310 /// See also:
311 ///
312 /// * https://en.wikipedia.org/wiki/Standard_deviation
313 double _computeStandardDeviationForPopulation(Iterable<double> population) {
314 final double mean = _computeAverage(population);
315 final double sumOfSquaredDeltas = population.fold<double>(
316 0.0,
317 (double previous, num value) => previous += math.pow(value - mean, 2),
318 );
319 return math.sqrt(sumOfSquaredDeltas / population.length);
320 }
321
322 String _ratioToPercent(double value) {
323 return '${(value * 100).toStringAsFixed(2)}%';
324 }