[Perl] DAQ system for the FAPG.
refactor remove server-side smoothing and simplify series API
Remove the median-window smoothing pass (minute_trends, median) from the series endpoint. The server now only aggregates rollup data into display buckets by averaging, returning a minimal generic payload. Server changes: - Remove smooth parameter, its validation, and response metadata (smooth, band, gap_seconds) - Simplify aggregate_series to average rollup values per bucket - Rename response: readings → points, lower → min, upper → max - Drop per-point unit/probe fields (redundant with top-level) Client changes: - Remove smooth=1 from series API requests - Read data.points instead of data.series - Use min/max field names throughout chart rendering Tests: - Remove smooth-specific test cases and seed data - Update assertions to new response shape
Changed files
roles/dashboard/lib/FAPG/DAQ/Dashboard/Controller/Reading.pm
@@ -75,18 +75,12 @@
75
75
76
76
my $timeframe = $self->param('timeframe') // 'day';
77
77
my $range = $self->param('range') // 1;
78
Removed:
my $smooth = $self->param('smooth') // 0;
79
78
80
79
return $self->render(
81
80
status => 400,
82
81
json => { error => 'Range must be an integer from 1 to 24', },
83
82
) if $range !~ /\A\d+\z/ || $range < 1 || $range > $MAX_SERIES_RANGE;
84
83
85
Removed:
return $self->render(
86
Removed:
status => 400,
87
Removed:
json => { error => 'Smooth must be either 0 or 1', },
88
Removed:
) if $smooth !~ /\A[01]\z/;
89
Removed:
90
84
my $window = series_window( $timeframe, $range );
91
85
92
86
return $self->render(
@@ -94,13 +88,12 @@
94
88
json => { error => "Unknown timeframe: $timeframe", },
95
89
) unless defined $window;
96
90
97
Removed:
my $unit = ( grep { $_->{key} eq $probe } $self->probes->@* )[0]{unit};
98
91
my $rollup_seconds
99
92
= $window->{bucket_stride} >= 3_600 ? 3_600 : $MINUTE_SECONDS;
100
93
my $now = time;
101
94
my $expires_epoch
102
95
= int( $now / $rollup_seconds ) * $rollup_seconds + $rollup_seconds;
103
Removed:
my $cache_key = join q{:}, $probe, $timeframe, $range, $smooth,
96
Added:
my $cache_key = join q{:}, $probe, $timeframe, $range,
104
97
$window->{start_epoch}, $window->{end_epoch};
105
98
my $cached = $SERIES_CACHE{$cache_key};
106
99
@@ -115,9 +108,9 @@
115
108
SELECT
116
109
bucket_epoch AS minute_epoch,
117
110
sample_count AS count,
118
Removed:
value_total / sample_count AS median,
119
Removed:
value_min AS lower,
120
Removed:
value_max AS upper,
111
Added:
value_total / sample_count AS avg_value,
112
Added:
value_min,
113
Added:
value_max,
121
114
max_gap_seconds,
122
115
first_sample_epoch - LAG(last_sample_epoch) OVER (
123
116
ORDER BY bucket_epoch
@@ -131,9 +124,9 @@
131
124
SELECT
132
125
minute_epoch,
133
126
count,
134
Removed:
median,
135
Removed:
lower,
136
Removed:
upper,
127
Added:
avg_value,
128
Added:
value_min,
129
Added:
value_max,
137
130
CASE
138
131
WHEN max_gap_seconds > CAST(? AS INTEGER)
139
132
OR gap_seconds > CAST(? AS INTEGER) THEN 1
@@ -150,20 +143,13 @@
150
143
$SERIES_GAP_SECONDS,
151
144
)->hashes->to_array;
152
145
153
Removed:
my $points = aggregate_series( $rollups, $window, $probe, $unit, $smooth,
154
Removed:
$rollup_seconds );
146
Added:
my $points = aggregate_series( $rollups, $window );
155
147
156
148
my $payload = {
157
149
probe => $probe,
158
150
timeframe => $timeframe,
159
151
range => 0 + $range,
160
Removed:
smooth => 0 + $smooth,
161
Removed:
band => {
162
Removed:
lower_statistic => 'minimum',
163
Removed:
upper_statistic => 'maximum',
164
Removed:
},
165
Removed:
gap_seconds => $SERIES_GAP_SECONDS,
166
Removed:
readings => $points,
152
Added:
points => $points,
167
153
};
168
154
169
155
prune_series_cache($now);
@@ -303,10 +289,8 @@
303
289
$window->{start_epoch} + ( $i * $window->{bucket_stride} )
304
290
),
305
291
value => undef,
306
Removed:
lower => undef,
307
Removed:
upper => undef,
308
Removed:
unit => undef,
309
Removed:
probe => undef,
292
Added:
min => undef,
293
Added:
max => undef,
310
294
count => 0,
311
295
gap => 0,
312
296
};
@@ -315,16 +299,12 @@
315
299
return \@points;
316
300
}
317
301
318
Removed:
sub aggregate_series ( $minutes, $window, $probe, $unit, $smooth,
319
Removed:
$rollup_seconds )
320
Removed:
{
302
Added:
sub aggregate_series ( $rollups, $window ) {
321
303
my $points = series_points($window);
322
Removed:
my $trends = minute_trends( $minutes, $smooth, $rollup_seconds );
323
304
my @values;
324
305
325
Removed:
for my $i ( 0 .. $minutes->$#* ) {
326
Removed:
my $minute = $minutes->[$i];
327
Removed:
my $epoch = 0 + $minute->{minute_epoch};
306
Added:
for my $rollup ( @$rollups ) {
307
Added:
my $epoch = 0 + $rollup->{minute_epoch};
328
308
next
329
309
if $epoch < $window->{start_epoch}
330
310
|| $epoch >= $window->{end_epoch};
@@ -333,13 +313,15 @@
333
313
( $epoch - $window->{start_epoch} ) / $window->{bucket_stride} );
334
314
my $point = $points->[$point_index];
335
315
336
Removed:
push $values[$point_index]->@*, $trends->[$i];
337
Removed:
$point->{lower} = 0 + $minute->{lower}
338
Removed:
if !defined $point->{lower} || $minute->{lower} < $point->{lower};
339
Removed:
$point->{upper} = 0 + $minute->{upper}
340
Removed:
if !defined $point->{upper} || $minute->{upper} > $point->{upper};
341
Removed:
$point->{count} += 0 + $minute->{count};
342
Removed:
$point->{gap} = 1 if $minute->{gap};
316
Added:
push $values[$point_index]->@*, 0 + $rollup->{avg_value};
317
Added:
$point->{min} = 0 + $rollup->{value_min}
318
Added:
if !defined $point->{min}
319
Added:
|| $rollup->{value_min} < $point->{min};
320
Added:
$point->{max} = 0 + $rollup->{value_max}
321
Added:
if !defined $point->{max}
322
Added:
|| $rollup->{value_max} > $point->{max};
323
Added:
$point->{count} += 0 + $rollup->{count};
324
Added:
$point->{gap} = 1 if $rollup->{gap};
343
325
}
344
326
345
327
for my $i ( 0 .. $points->$#* ) {
@@ -350,64 +332,9 @@
350
332
$total += $_ for @$point_values;
351
333
352
334
$points->[$i]{value} = $total / @$point_values;
353
Removed:
$points->[$i]{unit} = $unit;
354
Removed:
$points->[$i]{probe} = $probe;
355
335
}
356
336
357
337
return $points;
358
Removed:
}
359
Removed:
360
Removed:
sub minute_trends ( $minutes, $smooth, $rollup_seconds ) {
361
Removed:
my @trends = map { 0 + $_->{median} } @$minutes;
362
Removed:
return \@trends if !$smooth;
363
Removed:
364
Removed:
my $segment_start = 0;
365
Removed:
366
Removed:
while ( $segment_start <= $minutes->$#* ) {
367
Removed:
my $segment_end = $segment_start;
368
Removed:
369
Removed:
while ( $segment_end < $minutes->$#* ) {
370
Removed:
my $next = $segment_end + 1;
371
Removed:
last if $minutes->[$next]{gap};
372
Removed:
last
373
Removed:
if $minutes->[$next]{minute_epoch}
374
Removed:
- $minutes->[$segment_end]{minute_epoch} > $rollup_seconds;
375
Removed:
$segment_end = $next;
376
Removed:
}
377
Removed:
378
Removed:
for my $i ( $segment_start .. $segment_end ) {
379
Removed:
my ( $first, $last );
380
Removed:
381
Removed:
if ( $i == $segment_start && $segment_end - $segment_start >= 2 )
382
Removed:
{
383
Removed:
$first = $i;
384
Removed:
$last = $i + 2;
385
Removed:
}
386
Removed:
elsif ( $i < $segment_end ) {
387
Removed:
$first = $i > $segment_start ? $i - 1 : $i;
388
Removed:
$last = $i + 1;
389
Removed:
}
390
Removed:
else {
391
Removed:
$first = $i - 2 > $segment_start ? $i - 2 : $segment_start;
392
Removed:
$last = $i;
393
Removed:
}
394
Removed:
395
Removed:
$trends[$i] = median(
396
Removed:
[ map { 0 + $minutes->[$_]{median} } $first .. $last ] );
397
Removed:
}
398
Removed:
399
Removed:
$segment_start = $segment_end + 1;
400
Removed:
}
401
Removed:
402
Removed:
return \@trends;
403
Removed:
}
404
Removed:
405
Removed:
sub median ($values) {
406
Removed:
my @sorted = sort { $a <=> $b } @$values;
407
Removed:
my $middle = int( @sorted / 2 );
408
Removed:
409
Removed:
return $sorted[$middle] if @sorted % 2;
410
Removed:
return ( $sorted[ $middle - 1 ] + $sorted[$middle] ) / 2;
411
338
}
412
339
413
340
sub utc_timestamp ($epoch) {
roles/dashboard/public/js/dashboard/api.js
@@ -24,8 +24,7 @@
24
24
export async function fetchProbeData(probe, timeframe, range, includeReadings = false, signal = null) {
25
25
const seriesParams = new URLSearchParams({
26
26
timeframe,
27
Removed:
range: String(range),
28
Removed:
smooth: "1"
27
Added:
range: String(range)
29
28
});
30
29
const readingsParams = new URLSearchParams({ limit: "10" });
31
30
const options = signal ? { signal } : {};
@@ -38,7 +37,7 @@
38
37
]);
39
38
40
39
return {
41
Removed:
series: series.readings || [],
40
Added:
points: series.points || [],
42
41
readings: readings.readings || []
43
42
};
44
43
}
roles/dashboard/public/js/dashboard/charts.js
@@ -35,21 +35,21 @@
35
35
}
36
36
37
37
function chartSeries(rows, timeframe) {
38
Removed:
const series = { labels: [], timestamps: [], lower: [], upper: [], trend: [], gapFill: [] };
38
Added:
const series = { labels: [], timestamps: [], min: [], max: [], trend: [], gapFill: [] };
39
39
40
40
rows.forEach(row => {
41
41
if (row.gap && series.labels.length) {
42
42
series.labels.push("");
43
43
series.timestamps.push(null);
44
Removed:
series.lower.push(null);
45
Removed:
series.upper.push(null);
44
Added:
series.min.push(null);
45
Added:
series.max.push(null);
46
46
series.trend.push(null);
47
47
}
48
48
49
49
series.labels.push(formatChartTime(row.timestamp, timeframe));
50
50
series.timestamps.push(row.timestamp);
51
Removed:
series.lower.push(row.lower === null ? null : Number(row.lower));
52
Removed:
series.upper.push(row.upper === null ? null : Number(row.upper));
51
Added:
series.min.push(row.min === null ? null : Number(row.min));
52
Added:
series.max.push(row.max === null ? null : Number(row.max));
53
53
series.trend.push(row.value === null ? null : Number(row.value));
54
54
});
55
55
@@ -135,7 +135,7 @@
135
135
datasets: [
136
136
{
137
137
label: "Raw lower percentile",
138
Removed:
data: series.lower,
138
Added:
data: series.min,
139
139
borderColor: TRANSPARENT_CHART_COLOR,
140
140
backgroundColor: TRANSPARENT_CHART_COLOR,
141
141
pointRadius: 0,
@@ -143,7 +143,7 @@
143
143
},
144
144
{
145
145
label: "Observed range (minimum–maximum)",
146
Removed:
data: series.upper,
146
Added:
data: series.max,
147
147
borderColor: TRANSPARENT_CHART_COLOR,
148
148
backgroundColor: RAW_BAND_COLOR,
149
149
pointRadius: 0,
@@ -227,14 +227,14 @@
227
227
hasMessageTable,
228
228
signal
229
229
);
230
Removed:
const series = chartSeries(data.series, timeframe.selected());
230
Added:
const series = chartSeries(data.points, timeframe.selected());
231
231
const latest = [...data.readings]
232
232
.reverse()
233
233
.find(row => row.value !== null && Number.isFinite(Number(row.value)));
234
Removed:
const unit = [...data.series].reverse().find(row => row.unit)?.unit || "";
234
Added:
const unit = [...data.readings].reverse().find(row => row.unit)?.unit || "";
235
235
const yBounds = yAxisBounds(probe, [
236
Removed:
...series.lower,
237
Removed:
...series.upper,
236
Added:
...series.min,
237
Added:
...series.max,
238
238
...series.trend,
239
239
...series.gapFill
240
240
]);
@@ -246,8 +246,8 @@
246
246
const chart = charts.get(probe);
247
247
chart.data.labels = series.labels;
248
248
chart.data.timestamps = series.timestamps;
249
Removed:
chart.data.datasets[0].data = series.lower;
250
Removed:
chart.data.datasets[1].data = series.upper;
249
Added:
chart.data.datasets[0].data = series.min;
250
Added:
chart.data.datasets[1].data = series.max;
251
251
chart.data.datasets[2].data = series.trend;
252
252
chart.data.datasets[2].label = trendLabel;
253
253
chart.data.datasets[2].tooltipUnit = unit;
roles/dashboard/t/04-readings-series.t
@@ -12,25 +12,20 @@
12
12
my $t = test_app();
13
13
14
14
seed_series_readings($t);
15
Removed:
seed_smoothing_readings($t);
16
15
seed_gap_readings($t);
17
16
18
17
$t->get_ok('/api/readings/ec/series?timeframe=hour')
19
18
->status_is(200)
20
Removed:
->json_is( '/probe' => 'ec' )
21
Removed:
->json_is( '/timeframe' => 'hour' )
22
Removed:
->json_is( '/smooth' => 0 )
23
Removed:
->json_is( '/band/lower_statistic' => 'minimum' )
24
Removed:
->json_is( '/band/upper_statistic' => 'maximum' )
25
Removed:
->json_is( '/gap_seconds' => 30 )
26
Removed:
->json_hasnt('/readings/30');
27
Removed:
assert_series_reading( $t, 1050, 1000, 1100, 2 );
19
Added:
->json_is( '/probe' => 'ec' )
20
Added:
->json_is( '/timeframe' => 'hour' )
21
Added:
->json_hasnt('/points/30');
22
Added:
assert_series_point( $t, 1050, 1000, 1100, 2 );
28
23
29
24
$t->get_ok('/api/readings/ec/series?timeframe=day')
30
25
->status_is(200)
31
26
->header_is( 'X-FAPG-Chart-Cache' => 'MISS' )
32
Removed:
->json_hasnt('/readings/48');
33
Removed:
assert_series_reading( $t, 1050, 1000, 1100, 2 );
27
Added:
->json_hasnt('/points/48');
28
Added:
assert_series_point( $t, 1050, 1000, 1100, 2 );
34
29
35
30
$t->get_ok('/api/readings/ec/series?timeframe=day')
36
31
->status_is(200)
@@ -40,62 +35,40 @@
40
35
$t->get_ok('/api/readings/ec/series?timeframe=month')
41
36
->status_is(200)
42
37
->json_is( '/timeframe' => 'month' )
43
Removed:
->json_hasnt('/readings/60');
44
Removed:
assert_series_reading( $t, 1050, 1000, 1100, 2 );
38
Added:
->json_hasnt('/points/60');
39
Added:
assert_series_point( $t, 1050, 1000, 1100, 2 );
45
40
46
41
$t->get_ok('/api/readings/ec/series?timeframe=year')
47
42
->status_is(200)
48
Removed:
->json_hasnt('/readings/73');
49
Removed:
assert_series_reading( $t, 1050, 1000, 1100, 2 );
43
Added:
->json_hasnt('/points/73');
44
Added:
assert_series_point( $t, 1050, 1000, 1100, 2 );
50
45
51
46
$t->get_ok('/api/readings/ec/series?timeframe=week&range=2')
52
47
->status_is(200)
53
48
->json_is( '/range' => 2 )
54
Removed:
->json_hasnt('/readings/56');
55
Removed:
assert_series_reading( $t, 1050, 1000, 1100, 2 );
49
Added:
->json_hasnt('/points/56');
50
Added:
assert_series_point( $t, 1050, 1000, 1100, 2 );
56
51
57
Removed:
$t->get_ok('/api/readings/orp/series?timeframe=hour')
58
Removed:
->status_is(200)
59
Removed:
->json_is( '/smooth' => 0 );
60
Removed:
ok( scalar(
61
Removed:
grep { defined $_->{value} && $_->{value} > 0 }
62
Removed:
$t->tx->res->json->{readings}->@*
63
Removed:
),
64
Removed:
'unsmoothed series retains the observed value'
65
Removed:
);
66
Removed:
67
Removed:
$t->get_ok('/api/readings/orp/series?timeframe=hour&smooth=1')
68
Removed:
->status_is(200)
69
Removed:
->json_is( '/smooth' => 1 );
70
Removed:
ok( scalar( grep { defined $_->{value} } $t->tx->res->json->{readings}->@* ),
71
Removed:
'smoothed series contains readings'
72
Removed:
);
73
Removed:
74
Removed:
$t->get_ok('/api/readings/do/series?timeframe=hour&smooth=1')->status_is(200);
75
Removed:
ok( scalar( grep { $_->{gap} } $t->tx->res->json->{readings}->@* ),
52
Added:
$t->get_ok('/api/readings/do/series?timeframe=hour')->status_is(200);
53
Added:
ok( scalar( grep { $_->{gap} } $t->tx->res->json->{points}->@* ),
76
54
'series marks a data gap' );
77
55
78
56
$t->get_ok('/api/readings/ec/series?timeframe=century')->status_is(400);
79
57
$t->get_ok('/api/readings/ec/series?timeframe=week&range=0')->status_is(400);
80
58
$t->get_ok('/api/readings/ec/series?timeframe=week&range=25')->status_is(400);
81
Removed:
$t->get_ok('/api/readings/ec/series?timeframe=week&smooth=yes')
82
Removed:
->status_is(400);
83
59
84
60
done_testing;
85
61
86
Removed:
sub assert_series_reading {
87
Removed:
my ( $test, $value, $lower, $upper, $count ) = @_;
62
Added:
sub assert_series_point {
63
Added:
my ( $test, $value, $min, $max, $count ) = @_;
88
64
my @populated
89
Removed:
= grep { defined $_->{value} } $test->tx->res->json->{readings}->@*;
65
Added:
= grep { defined $_->{value} } $test->tx->res->json->{points}->@*;
90
66
91
67
is( scalar @populated, 1, 'one series bucket is populated' );
92
68
is( $populated[0]{value}, $value, 'series bucket has expected value' );
93
Removed:
is( $populated[0]{lower},
94
Removed:
$lower, 'series bucket has expected lower band' );
95
Removed:
is( $populated[0]{upper},
96
Removed:
$upper, 'series bucket has expected upper band' );
97
Removed:
is( $populated[0]{count},
98
Removed:
$count, 'series bucket has expected sample count' );
69
Added:
is( $populated[0]{min}, $min, 'series bucket has expected min' );
70
Added:
is( $populated[0]{max}, $max, 'series bucket has expected max' );
71
Added:
is( $populated[0]{count}, $count, 'series bucket has expected sample count' );
99
72
100
73
return;
101
74
}
@@ -130,42 +103,6 @@
130
103
1100,
131
104
'µS/cm',
132
105
);
133
Removed:
134
Removed:
return;
135
Removed:
}
136
Removed:
137
Removed:
sub seed_smoothing_readings {
138
Removed:
my ($t) = @_;
139
Removed:
my $bucket = minute_floor(time);
140
Removed:
my $db = $t->app->sqlite->db;
141
Removed:
142
Removed:
my @minute_values = ( 0, 0, 70, 0, 0, 70, 70, 0, 0 );
143
Removed:
my @readings;
144
Removed:
145
Removed:
for my $minute ( 0 .. $#minute_values ) {
146
Removed:
push @readings, map {
147
Removed:
[ $bucket - ( @minute_values * 60 ) + ( $minute * 60 ) + $_,
148
Removed:
$minute_values[$minute]
149
Removed:
]
150
Removed:
} map { $_ * 6 } 0 .. 9;
151
Removed:
}
152
Removed:
153
Removed:
for my $reading (@readings) {
154
Removed:
my $timestamp = utc_timestamp( $reading->[0] );
155
Removed:
156
Removed:
$db->query(
157
Removed:
q{
158
Removed:
INSERT INTO readings (received_at, timestamp, node, probe, value, unit)
159
Removed:
VALUES (?, ?, ?, ?, ?, ?)
160
Removed:
},
161
Removed:
$timestamp,
162
Removed:
$timestamp,
163
Removed:
'fapg-daq-zero-orp-01',
164
Removed:
'orp',
165
Removed:
$reading->[1],
166
Removed:
'mV',
167
Removed:
);
168
Removed:
}
169
106
170
107
return;
171
108
}