[Perl] DAQ system for the FAPG.
feat add robust probe smoothing
Changed files
roles/dashboard/lib/FAPG/DAQ/Dashboard/Controller/Reading.pm
@@ -3,200 +3,235 @@
3
3
package FAPG::DAQ::Dashboard::Controller::Reading;
4
4
use Mojo::Base 'Mojolicious::Controller', -signatures;
5
5
6
Removed:
use POSIX qw(strftime);
7
Removed:
use Time::Local qw(timegm);
6
Added:
use POSIX qw(strftime);
8
7
9
8
my $STATUS_REACHABLE_SECONDS = 120;
10
9
my $MAX_SERIES_RANGE = 24;
10
Added:
my $SERIES_GAP_SECONDS = 30;
11
Added:
my $MINUTE_SECONDS = 60;
11
12
my %SERIES_TIMEFRAMES = (
12
Removed:
hour => {
13
Removed:
bucket_count => 60,
14
Removed:
bucket_sql => '%Y-%m-%dT%H:%M:00Z',
15
Removed:
bucket_stride => 60,
16
Removed:
floor_to => 'minute',
17
Removed:
},
18
Removed:
day => {
19
Removed:
bucket_count => 24,
20
Removed:
bucket_sql => '%Y-%m-%dT%H:00:00Z',
21
Removed:
bucket_stride => 60 * 60,
22
Removed:
floor_to => 'hour',
23
Removed:
},
24
Removed:
week => {
25
Removed:
bucket_count => 7,
26
Removed:
bucket_sql => '%Y-%m-%dT00:00:00Z',
27
Removed:
bucket_stride => 24 * 60 * 60,
28
Removed:
floor_to => 'day',
29
Removed:
},
30
Removed:
month => {
31
Removed:
bucket_count => 30,
32
Removed:
bucket_sql => '%Y-%m-%dT00:00:00Z',
33
Removed:
bucket_stride => 24 * 60 * 60,
34
Removed:
floor_to => 'day',
35
Removed:
},
36
Removed:
year => {
37
Removed:
bucket_count => 12,
38
Removed:
bucket_sql => '%Y-%m-01T00:00:00Z',
39
Removed:
bucket_stride => undef,
40
Removed:
floor_to => 'month',
41
Removed:
},
13
Added:
hour => {
14
Added:
span_seconds => 60 * 60,
15
Added:
bucket_stride => 60,
16
Added:
},
17
Added:
day => {
18
Added:
span_seconds => 24 * 60 * 60,
19
Added:
bucket_stride => 15 * 60,
20
Added:
},
21
Added:
week => {
22
Added:
span_seconds => 7 * 24 * 60 * 60,
23
Added:
bucket_stride => 60 * 60,
24
Added:
},
25
Added:
month => {
26
Added:
span_seconds => 30 * 24 * 60 * 60,
27
Added:
bucket_stride => 6 * 60 * 60,
28
Added:
},
29
Added:
year => {
30
Added:
span_seconds => 365 * 24 * 60 * 60,
31
Added:
bucket_stride => 24 * 60 * 60,
32
Added:
},
42
33
);
43
34
44
35
sub list ($self) {
45
Removed:
my $probe = $self->param('probe') // '';
36
Added:
my $probe = $self->param('probe') // '';
46
37
47
Removed:
my %known = map { $_->{key} => 1 } $self->probes->@*;
38
Added:
my %known = map { $_->{key} => 1 } $self->probes->@*;
48
39
49
Removed:
return $self->render(
50
Removed:
status => 404,
51
Removed:
json => { error => "Unknown probe type: $probe", },
52
Removed:
) unless $known{$probe};
40
Added:
return $self->render(
41
Added:
status => 404,
42
Added:
json => { error => "Unknown probe type: $probe", },
43
Added:
) unless $known{$probe};
53
44
54
Removed:
my $limit = $self->param('limit') // 300;
45
Added:
my $limit = $self->param('limit') // 300;
55
46
56
Removed:
$limit = 300 unless $limit =~ /^\d+$/;
57
Removed:
$limit = 2_000 if $limit > 2_000;
47
Added:
$limit = 300 unless $limit =~ /^\d+$/;
48
Added:
$limit = 2_000 if $limit > 2_000;
58
49
59
Removed:
my @where = ('probe = ?');
60
Removed:
my @bind = ($probe);
61
Removed:
my $since = $self->param('since');
50
Added:
my @where = ('probe = ?');
51
Added:
my @bind = ($probe);
52
Added:
my $since = $self->param('since');
62
53
63
Removed:
if ( defined $since
64
Removed:
&& $since =~ /\A\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z\z/ )
65
Removed:
{
66
Removed:
push @where, 'COALESCE(received_at, timestamp) >= ?';
67
Removed:
push @bind, $since;
68
Removed:
}
54
Added:
if ( defined $since
55
Added:
&& $since =~ /\A\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z\z/ )
56
Added:
{
57
Added:
push @where, 'COALESCE(timestamp, received_at) >= ?';
58
Added:
push @bind, $since;
59
Added:
}
69
60
70
Removed:
my $rows = $self->sqlite->db->query(
71
Removed:
q{
61
Added:
my $rows = $self->sqlite->db->query(
62
Added:
q{
72
63
SELECT timestamp, received_at, node, probe, value, unit
73
64
FROM readings
74
65
WHERE
75
66
}
76
Removed:
. join( "\n AND ", @where ) . q{
77
Removed:
ORDER BY COALESCE(received_at, timestamp) DESC, id DESC
67
Added:
. join( "\n AND ", @where ) . q{
68
Added:
ORDER BY COALESCE(timestamp, received_at) DESC, id DESC
78
69
LIMIT ?
79
70
},
80
Removed:
@bind,
81
Removed:
$limit,
82
Removed:
)->hashes->to_array;
71
Added:
@bind,
72
Added:
$limit,
73
Added:
)->hashes->to_array;
83
74
84
Removed:
$rows = [ reverse $rows->@* ];
75
Added:
$rows = [ reverse $rows->@* ];
85
76
86
Removed:
$self->render(
87
Removed:
json => {
88
Removed:
probe => $probe,
89
Removed:
readings => $rows,
90
Removed:
},
91
Removed:
);
77
Added:
$self->render(
78
Added:
json => {
79
Added:
probe => $probe,
80
Added:
readings => $rows,
81
Added:
},
82
Added:
);
92
83
}
93
84
94
85
sub series ($self) {
95
Removed:
my $probe = $self->param('probe') // '';
86
Added:
my $probe = $self->param('probe') // '';
96
87
97
Removed:
my %known = map { $_->{key} => 1 } $self->probes->@*;
88
Added:
my %known = map { $_->{key} => 1 } $self->probes->@*;
98
89
99
Removed:
return $self->render(
100
Removed:
status => 404,
101
Removed:
json => { error => "Unknown probe type: $probe", },
102
Removed:
) unless $known{$probe};
90
Added:
return $self->render(
91
Added:
status => 404,
92
Added:
json => { error => "Unknown probe type: $probe", },
93
Added:
) unless $known{$probe};
103
94
104
Removed:
my $timeframe = $self->param('timeframe') // 'day';
105
Removed:
my $range = $self->param('range') // 1;
106
Removed:
my $smooth = $self->param('smooth') // 0;
95
Added:
my $timeframe = $self->param('timeframe') // 'day';
96
Added:
my $range = $self->param('range') // 1;
97
Added:
my $smooth = $self->param('smooth') // 0;
107
98
108
Removed:
return $self->render(
109
Removed:
status => 400,
110
Removed:
json => { error => 'Range must be an integer from 1 to 24', },
111
Removed:
) if $range !~ /\A\d+\z/ || $range < 1 || $range > $MAX_SERIES_RANGE;
99
Added:
return $self->render(
100
Added:
status => 400,
101
Added:
json => { error => 'Range must be an integer from 1 to 24', },
102
Added:
) if $range !~ /\A\d+\z/ || $range < 1 || $range > $MAX_SERIES_RANGE;
112
103
113
Removed:
return $self->render(
114
Removed:
status => 400,
115
Removed:
json => { error => 'Smooth must be either 0 or 1', },
116
Removed:
) if $smooth !~ /\A[01]\z/;
104
Added:
return $self->render(
105
Added:
status => 400,
106
Added:
json => { error => 'Smooth must be either 0 or 1', },
107
Added:
) if $smooth !~ /\A[01]\z/;
117
108
118
Removed:
my $window = series_window( $timeframe, $range );
109
Added:
my $window = series_window( $timeframe, $range );
119
110
120
Removed:
return $self->render(
121
Removed:
status => 400,
122
Removed:
json => { error => "Unknown timeframe: $timeframe", },
123
Removed:
) unless defined $window;
111
Added:
return $self->render(
112
Added:
status => 400,
113
Added:
json => { error => "Unknown timeframe: $timeframe", },
114
Added:
) unless defined $window;
124
115
125
Removed:
my $unit = ( grep { $_->{key} eq $probe } $self->probes->@* )[0]{unit};
126
Removed:
my $points = series_points($window);
127
Removed:
my %index = map { $points->[$_]{timestamp} => $_ } 0 .. $points->$#*;
128
Removed:
129
Removed:
my $chart_value_sql = $smooth
130
Removed:
? q{AVG(value) OVER (
131
Removed:
ORDER BY COALESCE(received_at, timestamp), id
132
Removed:
ROWS BETWEEN 3 PRECEDING AND 3 FOLLOWING
133
Removed:
)}
134
Removed:
: 'value';
135
Removed:
136
Removed:
my $rows = $self->sqlite->db->query(
137
Removed:
q{
138
Removed:
WITH raw_series AS (
116
Added:
my $unit = ( grep { $_->{key} eq $probe } $self->probes->@* )[0]{unit};
117
Added:
my $minutes = $self->sqlite->db->query(
118
Added:
q{
119
Added:
WITH samples AS (
139
120
SELECT
140
121
id,
141
Removed:
COALESCE(received_at, timestamp) AS sample_at,
142
Removed:
}
143
Removed:
. $chart_value_sql . q{ AS chart_value
122
Added:
CAST(strftime('%s', COALESCE(timestamp, received_at)) AS INTEGER)
123
Added:
AS sample_epoch,
124
Added:
CAST(value AS REAL) AS value
144
125
FROM readings
145
126
WHERE probe = ?
146
Removed:
AND COALESCE(received_at, timestamp) >= ?
147
Removed:
AND COALESCE(received_at, timestamp) < ?
127
Added:
AND value IS NOT NULL
128
Added:
AND COALESCE(timestamp, received_at) >= ?
129
Added:
AND COALESCE(timestamp, received_at) < ?
130
Added:
),
131
Added:
ordered AS (
132
Added:
SELECT
133
Added:
id,
134
Added:
sample_epoch,
135
Added:
value,
136
Added:
CAST(sample_epoch / 60 AS INTEGER) * 60 AS minute_epoch,
137
Added:
sample_epoch - LAG(sample_epoch) OVER (
138
Added:
ORDER BY sample_epoch, id
139
Added:
) AS gap_seconds
140
Added:
FROM samples
141
Added:
),
142
Added:
ranked AS (
143
Added:
SELECT
144
Added:
*,
145
Added:
ROW_NUMBER() OVER (
146
Added:
PARTITION BY minute_epoch
147
Added:
ORDER BY value, id
148
Added:
) AS value_rank,
149
Added:
COUNT(*) OVER (
150
Added:
PARTITION BY minute_epoch
151
Added:
) AS value_count
152
Added:
FROM ordered
148
153
)
149
154
SELECT
150
Removed:
strftime(?, sample_at) AS bucket,
151
Removed:
AVG(chart_value) AS value,
152
Removed:
COUNT(*) AS count
153
Removed:
FROM raw_series
154
Removed:
GROUP BY bucket
155
Removed:
ORDER BY bucket
155
Added:
minute_epoch,
156
Added:
COUNT(*) AS count,
157
Added:
AVG(
158
Added:
CASE
159
Added:
WHEN value_rank IN (
160
Added:
CAST((value_count + 1) / 2 AS INTEGER),
161
Added:
CAST((value_count + 2) / 2 AS INTEGER)
162
Added:
) THEN value
163
Added:
END
164
Added:
) AS median,
165
Added:
MAX(
166
Added:
CASE
167
Added:
WHEN value_rank = CASE
168
Added:
WHEN value_count >= 5 THEN MAX(
169
Added:
2,
170
Added:
CAST((value_count + 9) / 10 AS INTEGER)
171
Added:
)
172
Added:
ELSE 1
173
Added:
END THEN value
174
Added:
END
175
Added:
) AS lower,
176
Added:
MAX(
177
Added:
CASE
178
Added:
WHEN value_rank = CASE
179
Added:
WHEN value_count >= 5 THEN MIN(
180
Added:
value_count - 1,
181
Added:
CAST((9 * value_count + 9) / 10 AS INTEGER)
182
Added:
)
183
Added:
ELSE value_count
184
Added:
END THEN value
185
Added:
END
186
Added:
) AS upper,
187
Added:
MAX(
188
Added:
CASE
189
Added:
WHEN gap_seconds > CAST(? AS INTEGER) THEN 1
190
Added:
ELSE 0
191
Added:
END
192
Added:
) AS gap
193
Added:
FROM ranked
194
Added:
GROUP BY minute_epoch
195
Added:
ORDER BY minute_epoch
156
196
},
157
Removed:
$probe,
158
Removed:
$window->{start_iso},
159
Removed:
$window->{end_iso},
160
Removed:
$window->{bucket_sql},
161
Removed:
)->hashes->to_array;
197
Added:
$probe,
198
Added:
utc_timestamp( $window->{start_epoch} - ( 2 * $MINUTE_SECONDS ) ),
199
Added:
$window->{end_iso},
200
Added:
$SERIES_GAP_SECONDS,
201
Added:
)->hashes->to_array;
162
202
163
Removed:
for my $row ( $rows->@* ) {
164
Removed:
my $point_index = $index{ $row->{bucket} };
165
Removed:
next if !defined $point_index;
203
Added:
my $points
204
Added:
= aggregate_series( $minutes, $window, $probe, $unit, $smooth );
166
205
167
Removed:
$points->[$point_index] = {
168
Removed:
timestamp => $row->{bucket},
169
Removed:
value => 0 + $row->{value},
170
Removed:
unit => $unit,
171
Removed:
probe => $probe,
172
Removed:
count => 0 + $row->{count},
173
Removed:
};
174
Removed:
}
175
Removed:
176
Removed:
$self->render(
177
Removed:
json => {
178
Removed:
probe => $probe,
179
Removed:
timeframe => $timeframe,
180
Removed:
range => 0 + $range,
181
Removed:
smooth => 0 + $smooth,
182
Removed:
readings => $points,
183
Removed:
},
184
Removed:
);
206
Added:
$self->render(
207
Added:
json => {
208
Added:
probe => $probe,
209
Added:
timeframe => $timeframe,
210
Added:
range => 0 + $range,
211
Added:
smooth => 0 + $smooth,
212
Added:
band => {
213
Added:
lower_percentile => 10,
214
Added:
upper_percentile => 90,
215
Added:
},
216
Added:
gap_seconds => $SERIES_GAP_SECONDS,
217
Added:
readings => $points,
218
Added:
},
219
Added:
);
185
220
}
186
221
187
222
sub status ($self) {
188
Removed:
my $probe = $self->param('probe') // '';
223
Added:
my $probe = $self->param('probe') // '';
189
224
190
Removed:
my %known = map { $_->{key} => 1 } $self->status_items->@*;
225
Added:
my %known = map { $_->{key} => 1 } $self->status_items->@*;
191
226
192
Removed:
return $self->render(
193
Removed:
status => 404,
194
Removed:
json => { error => "Unknown status item: $probe", },
195
Removed:
) unless $known{$probe};
227
Added:
return $self->render(
228
Added:
status => 404,
229
Added:
json => { error => "Unknown status item: $probe", },
230
Added:
) unless $known{$probe};
196
231
197
Removed:
my $row = eval {
198
Removed:
$self->sqlite->db->query(
199
Removed:
q{
232
Added:
my $row = eval {
233
Added:
$self->sqlite->db->query(
234
Added:
q{
200
235
SELECT timestamp, received_at, node, probe, status, message, error, device
201
236
FROM node_status
202
237
WHERE probe = ?
@@ -204,158 +239,170 @@
204
239
ORDER BY COALESCE(timestamp, received_at) DESC, id DESC
205
240
LIMIT 1
206
241
},
207
Removed:
$probe,
208
Removed:
)->hash;
209
Removed:
};
242
Added:
$probe,
243
Added:
)->hash;
244
Added:
};
210
245
211
Removed:
$row = undef if $@;
212
Removed:
mark_unreachable_if_stale($row) if $row;
246
Added:
$row = undef if $@;
247
Added:
mark_unreachable_if_stale($row) if $row;
213
248
214
Removed:
$self->render(
215
Removed:
json => {
216
Removed:
probe => $probe,
217
Removed:
status => $row,
218
Removed:
},
219
Removed:
);
249
Added:
$self->render(
250
Added:
json => {
251
Added:
probe => $probe,
252
Added:
status => $row,
253
Added:
},
254
Added:
);
220
255
}
221
256
222
257
sub mark_unreachable_if_stale ($row) {
223
Removed:
my $received_at = $row->{received_at} // $row->{timestamp};
224
Removed:
return if !defined $received_at || $received_at ge reachable_since();
258
Added:
my $received_at = $row->{received_at} // $row->{timestamp};
259
Added:
return if !defined $received_at || $received_at ge reachable_since();
225
260
226
Removed:
$row->{status} = 'unreachable';
227
Removed:
$row->{message} = "No status received in the last $STATUS_REACHABLE_SECONDS seconds";
261
Added:
$row->{status} = 'unreachable';
262
Added:
$row->{message}
263
Added:
= "No status received in the last $STATUS_REACHABLE_SECONDS seconds";
228
264
}
229
265
230
266
sub reachable_since {
231
Removed:
return strftime( '%Y-%m-%dT%H:%M:%SZ', gmtime( time - $STATUS_REACHABLE_SECONDS ) );
267
Added:
return strftime( '%Y-%m-%dT%H:%M:%SZ',
268
Added:
gmtime( time - $STATUS_REACHABLE_SECONDS ) );
232
269
}
233
270
234
271
sub series_window ( $timeframe, $range ) {
235
Removed:
my $config = $SERIES_TIMEFRAMES{$timeframe} or return undef;
236
Removed:
my @now = gmtime;
237
Removed:
my ( $sec, $min, $hour, $mday, $mon, $year ) = @now[ 0 .. 5 ];
272
Added:
my $config = $SERIES_TIMEFRAMES{$timeframe} or return undef;
273
Added:
my $bucket_stride = $config->{bucket_stride} * $range;
274
Added:
my $span_seconds = $config->{span_seconds} * $range;
275
Added:
my $end_epoch
276
Added:
= int( time / $bucket_stride ) * $bucket_stride + $bucket_stride;
277
Added:
my $start_epoch = $end_epoch - $span_seconds;
238
278
239
Removed:
if ( $config->{floor_to} eq 'minute' ) {
240
Removed:
my $end_epoch = timegm( 0, $min, $hour, $mday, $mon, $year ) + 60;
241
Removed:
242
279
return {
243
Removed:
bucket_count => $config->{bucket_count} * $range,
244
Removed:
bucket_sql => $config->{bucket_sql},
245
Removed:
bucket_stride => $config->{bucket_stride},
246
Removed:
start_epoch => $end_epoch - ( $config->{bucket_count} * $range * $config->{bucket_stride} ),
247
Removed:
end_epoch => $end_epoch,
248
Removed:
start_iso => utc_timestamp(
249
Removed:
$end_epoch - ( $config->{bucket_count} * $range * $config->{bucket_stride} )
250
Removed:
),
251
Removed:
end_iso => utc_timestamp($end_epoch),
280
Added:
bucket_count => int( $span_seconds / $bucket_stride ),
281
Added:
bucket_stride => $bucket_stride,
282
Added:
start_epoch => $start_epoch,
283
Added:
end_epoch => $end_epoch,
284
Added:
start_iso => utc_timestamp($start_epoch),
285
Added:
end_iso => utc_timestamp($end_epoch),
252
286
};
253
Removed:
}
287
Added:
}
254
288
255
Removed:
if ( $config->{floor_to} eq 'hour' ) {
256
Removed:
my $end_epoch = timegm( 0, 0, $hour, $mday, $mon, $year ) + ( 60 * 60 );
289
Added:
sub series_points ($window) {
290
Added:
my @points;
257
291
258
Removed:
return {
259
Removed:
bucket_count => $config->{bucket_count} * $range,
260
Removed:
bucket_sql => $config->{bucket_sql},
261
Removed:
bucket_stride => $config->{bucket_stride},
262
Removed:
start_epoch => $end_epoch - ( $config->{bucket_count} * $range * $config->{bucket_stride} ),
263
Removed:
end_epoch => $end_epoch,
264
Removed:
start_iso => utc_timestamp(
265
Removed:
$end_epoch - ( $config->{bucket_count} * $range * $config->{bucket_stride} )
266
Removed:
),
267
Removed:
end_iso => utc_timestamp($end_epoch),
268
Removed:
};
269
Removed:
}
292
Added:
for my $i ( 0 .. $window->{bucket_count} - 1 ) {
293
Added:
push @points,
294
Added:
{
295
Added:
timestamp => utc_timestamp(
296
Added:
$window->{start_epoch} + ( $i * $window->{bucket_stride} )
297
Added:
),
298
Added:
value => undef,
299
Added:
lower => undef,
300
Added:
upper => undef,
301
Added:
unit => undef,
302
Added:
probe => undef,
303
Added:
count => 0,
304
Added:
gap => 0,
305
Added:
};
306
Added:
}
270
307
271
Removed:
if ( $config->{floor_to} eq 'day' ) {
272
Removed:
my $end_epoch = timegm( 0, 0, 0, $mday, $mon, $year ) + ( 24 * 60 * 60 );
308
Added:
return \@points;
309
Added:
}
273
310
274
Removed:
return {
275
Removed:
bucket_count => $config->{bucket_count} * $range,
276
Removed:
bucket_sql => $config->{bucket_sql},
277
Removed:
bucket_stride => $config->{bucket_stride},
278
Removed:
start_epoch => $end_epoch - ( $config->{bucket_count} * $range * $config->{bucket_stride} ),
279
Removed:
end_epoch => $end_epoch,
280
Removed:
start_iso => utc_timestamp(
281
Removed:
$end_epoch - ( $config->{bucket_count} * $range * $config->{bucket_stride} )
282
Removed:
),
283
Removed:
end_iso => utc_timestamp($end_epoch),
284
Removed:
};
285
Removed:
}
311
Added:
sub aggregate_series ( $minutes, $window, $probe, $unit, $smooth ) {
312
Added:
my $points = series_points($window);
313
Added:
my $trends = minute_trends( $minutes, $smooth );
314
Added:
my @values;
286
315
287
Removed:
if ( $config->{floor_to} eq 'month' ) {
288
Removed:
my ( $start_year, $start_month ) =
289
Removed:
normalize_month( $year, $mon - ( $config->{bucket_count} * $range - 1 ) );
290
Removed:
my ( $end_year, $end_month ) = normalize_month( $year, $mon + 1 );
316
Added:
for my $i ( 0 .. $minutes->$#* ) {
317
Added:
my $minute = $minutes->[$i];
318
Added:
my $epoch = 0 + $minute->{minute_epoch};
319
Added:
next
320
Added:
if $epoch < $window->{start_epoch}
321
Added:
|| $epoch >= $window->{end_epoch};
291
322
292
Removed:
return {
293
Removed:
bucket_count => $config->{bucket_count} * $range,
294
Removed:
bucket_sql => $config->{bucket_sql},
295
Removed:
bucket_stride => $config->{bucket_stride},
296
Removed:
start_epoch => timegm( 0, 0, 0, 1, $start_month, $start_year ),
297
Removed:
end_epoch => timegm( 0, 0, 0, 1, $end_month, $end_year ),
298
Removed:
start_iso => utc_timestamp( timegm( 0, 0, 0, 1, $start_month, $start_year ) ),
299
Removed:
end_iso => utc_timestamp( timegm( 0, 0, 0, 1, $end_month, $end_year ) ),
300
Removed:
};
301
Removed:
}
323
Added:
my $point_index = int(
324
Added:
( $epoch - $window->{start_epoch} ) / $window->{bucket_stride} );
325
Added:
my $point = $points->[$point_index];
302
326
303
Removed:
return undef;
304
Removed:
}
327
Added:
push $values[$point_index]->@*, $trends->[$i];
328
Added:
$point->{lower} = 0 + $minute->{lower}
329
Added:
if !defined $point->{lower} || $minute->{lower} < $point->{lower};
330
Added:
$point->{upper} = 0 + $minute->{upper}
331
Added:
if !defined $point->{upper} || $minute->{upper} > $point->{upper};
332
Added:
$point->{count} += 0 + $minute->{count};
333
Added:
$point->{gap} = 1 if $minute->{gap};
334
Added:
}
305
335
306
Removed:
sub series_points ($window) {
307
Removed:
my @points;
336
Added:
for my $i ( 0 .. $points->$#* ) {
337
Added:
my $point_values = $values[$i] // [];
338
Added:
next if !@$point_values;
308
339
309
Removed:
if ( defined $window->{bucket_stride} ) {
310
Removed:
for my $i ( 0 .. $window->{bucket_count} - 1 ) {
311
Removed:
push @points,
312
Removed:
{
313
Removed:
timestamp => utc_timestamp( $window->{start_epoch} + ( $i * $window->{bucket_stride} ) ),
314
Removed:
value => undef,
315
Removed:
unit => undef,
316
Removed:
probe => undef,
317
Removed:
count => 0,
318
Removed:
};
340
Added:
my $total = 0;
341
Added:
$total += $_ for @$point_values;
342
Added:
343
Added:
$points->[$i]{value} = $total / @$point_values;
344
Added:
$points->[$i]{unit} = $unit;
345
Added:
$points->[$i]{probe} = $probe;
319
346
}
320
347
321
Removed:
return \@points;
322
Removed:
}
348
Added:
return $points;
349
Added:
}
323
350
324
Removed:
my @cursor = gmtime( $window->{start_epoch} );
325
Removed:
my ( $year, $month ) = ( $cursor[5], $cursor[4] );
351
Added:
sub minute_trends ( $minutes, $smooth ) {
352
Added:
my @trends = map { 0 + $_->{median} } @$minutes;
353
Added:
return \@trends if !$smooth;
326
354
327
Removed:
for my $i ( 0 .. $window->{bucket_count} - 1 ) {
328
Removed:
my ( $bucket_year, $bucket_month ) = normalize_month( $year, $month + $i );
355
Added:
my $segment_start = 0;
329
356
330
Removed:
push @points,
331
Removed:
{
332
Removed:
timestamp => utc_timestamp( timegm( 0, 0, 0, 1, $bucket_month, $bucket_year ) ),
333
Removed:
value => undef,
334
Removed:
unit => undef,
335
Removed:
probe => undef,
336
Removed:
count => 0,
337
Removed:
};
338
Removed:
}
357
Added:
while ( $segment_start <= $minutes->$#* ) {
358
Added:
my $segment_end = $segment_start;
339
359
340
Removed:
return \@points;
341
Removed:
}
360
Added:
while ( $segment_end < $minutes->$#* ) {
361
Added:
my $next = $segment_end + 1;
362
Added:
last if $minutes->[$next]{gap};
363
Added:
last
364
Added:
if $minutes->[$next]{minute_epoch}
365
Added:
- $minutes->[$segment_end]{minute_epoch} > $MINUTE_SECONDS;
366
Added:
$segment_end = $next;
367
Added:
}
342
368
343
Removed:
sub normalize_month ( $year, $month ) {
344
Removed:
while ( $month < 0 ) {
345
Removed:
$month += 12;
346
Removed:
$year--;
347
Removed:
}
369
Added:
for my $i ( $segment_start .. $segment_end ) {
370
Added:
my ( $first, $last );
348
371
349
Removed:
while ( $month > 11 ) {
350
Removed:
$month -= 12;
351
Removed:
$year++;
352
Removed:
}
372
Added:
if ( $i == $segment_start && $segment_end - $segment_start >= 2 )
373
Added:
{
374
Added:
$first = $i;
375
Added:
$last = $i + 2;
376
Added:
}
377
Added:
elsif ( $i < $segment_end ) {
378
Added:
$first = $i > $segment_start ? $i - 1 : $i;
379
Added:
$last = $i + 1;
380
Added:
}
381
Added:
else {
382
Added:
$first = $i - 2 > $segment_start ? $i - 2 : $segment_start;
383
Added:
$last = $i;
384
Added:
}
353
385
354
Removed:
return ( $year, $month );
386
Added:
$trends[$i] = median(
387
Added:
[ map { 0 + $minutes->[$_]{median} } $first .. $last ] );
388
Added:
}
389
Added:
390
Added:
$segment_start = $segment_end + 1;
391
Added:
}
392
Added:
393
Added:
return \@trends;
355
394
}
356
395
396
Added:
sub median ($values) {
397
Added:
my @sorted = sort { $a <=> $b } @$values;
398
Added:
my $middle = int( @sorted / 2 );
399
Added:
400
Added:
return $sorted[$middle] if @sorted % 2;
401
Added:
return ( $sorted[ $middle - 1 ] + $sorted[$middle] ) / 2;
402
Added:
}
403
Added:
357
404
sub utc_timestamp ($epoch) {
358
Removed:
return strftime( '%Y-%m-%dT%H:%M:%SZ', gmtime $epoch );
405
Added:
return strftime( '%Y-%m-%dT%H:%M:%SZ', gmtime $epoch );
359
406
}
360
407
361
408
1;
roles/dashboard/public/js/dashboard.js
@@ -61,6 +61,8 @@
61
61
};
62
62
const MAX_TIMEFRAME_RANGE = 24;
63
63
const PRIMARY_CHART_COLOR = "#276749";
64
Added:
const RAW_BAND_COLOR = "rgba(39, 103, 73, 0.18)";
65
Added:
const TRANSPARENT_CHART_COLOR = "rgba(39, 103, 73, 0)";
64
66
let activeTimeframe = DEFAULT_TIMEFRAME;
65
67
let expandedTimeframe = null;
66
68
let smoothingEnabled = true;
@@ -249,6 +251,31 @@
249
251
};
250
252
}
251
253
254
Added:
function chartSeries(rows) {
255
Added:
const series = {
256
Added:
labels: [],
257
Added:
lower: [],
258
Added:
upper: [],
259
Added:
trend: []
260
Added:
};
261
Added:
262
Added:
rows.forEach(row => {
263
Added:
if (row.gap && series.labels.length) {
264
Added:
series.labels.push("");
265
Added:
series.lower.push(null);
266
Added:
series.upper.push(null);
267
Added:
series.trend.push(null);
268
Added:
}
269
Added:
270
Added:
series.labels.push(labelForRow(row));
271
Added:
series.lower.push(row.lower === null ? null : Number(row.lower));
272
Added:
series.upper.push(row.upper === null ? null : Number(row.upper));
273
Added:
series.trend.push(row.value === null ? null : Number(row.value));
274
Added:
});
275
Added:
276
Added:
return series;
277
Added:
}
278
Added:
252
279
function statusLabel(state) {
253
280
return {
254
281
online: "Online",
@@ -662,11 +689,16 @@
662
689
const rows = seriesPayload.readings || [];
663
690
const readings = readingsPayload.readings || [];
664
691
665
Removed:
const labels = rows.map(labelForRow);
666
Removed:
const values = rows.map(row => row.value === null ? null : Number(row.value));
667
Removed:
const latest = [...readings].reverse().find(row => Number.isFinite(Number(row.value)));
668
Removed:
const unit = rows.length ? rows[rows.length - 1].unit : "";
669
Removed:
const yBounds = yAxisBounds(probe, values);
692
Added:
const series = chartSeries(rows);
693
Added:
const latest = [...readings]
694
Added:
.reverse()
695
Added:
.find(row => row.value !== null && Number.isFinite(Number(row.value)));
696
Added:
const unit = [...rows].reverse().find(row => row.unit)?.unit || "";
697
Added:
const yBounds = yAxisBounds(
698
Added:
probe,
699
Added:
[...series.lower, ...series.upper, ...series.trend]
700
Added:
);
701
Added:
const trendLabel = `${probe} ${unit ? `(${unit})` : ""}${smoothingEnabled ? " trend" : " median"}`;
670
702
671
703
if (!charts.has(probe)) {
672
704
const ctx = chartElement(probe);
@@ -674,17 +706,36 @@
674
706
const chart = new Chart(ctx, {
675
707
type: "line",
676
708
data: {
677
Removed:
labels,
678
Removed:
datasets: [{
679
Removed:
label: `${probe} ${unit ? `(${unit})` : ""}${smoothingEnabled ? " smoothed" : ""}`,
680
Removed:
data: values,
681
Removed:
tension: 0.2,
682
Removed:
borderColor: PRIMARY_CHART_COLOR,
683
Removed:
backgroundColor: PRIMARY_CHART_COLOR,
684
Removed:
pointBackgroundColor: PRIMARY_CHART_COLOR,
685
Removed:
pointRadius: 3,
686
Removed:
pointHoverRadius: 5
687
Removed:
}]
709
Added:
labels: series.labels,
710
Added:
datasets: [
711
Added:
{
712
Added:
label: "Raw lower percentile",
713
Added:
data: series.lower,
714
Added:
borderColor: TRANSPARENT_CHART_COLOR,
715
Added:
backgroundColor: TRANSPARENT_CHART_COLOR,
716
Added:
pointRadius: 0,
717
Added:
pointHoverRadius: 0
718
Added:
},
719
Added:
{
720
Added:
label: "Raw variability (10th–90th percentile)",
721
Added:
data: series.upper,
722
Added:
borderColor: TRANSPARENT_CHART_COLOR,
723
Added:
backgroundColor: RAW_BAND_COLOR,
724
Added:
pointRadius: 0,
725
Added:
pointHoverRadius: 0,
726
Added:
fill: "-1"
727
Added:
},
728
Added:
{
729
Added:
label: trendLabel,
730
Added:
data: series.trend,
731
Added:
tension: 0.2,
732
Added:
borderColor: PRIMARY_CHART_COLOR,
733
Added:
backgroundColor: PRIMARY_CHART_COLOR,
734
Added:
pointBackgroundColor: PRIMARY_CHART_COLOR,
735
Added:
pointRadius: 3,
736
Added:
pointHoverRadius: 5
737
Added:
}
738
Added:
]
688
739
},
689
740
options: {
690
741
responsive: true,
@@ -703,7 +754,10 @@
703
754
},
704
755
plugins: {
705
756
legend: {
706
Removed:
display: true
757
Added:
display: true,
758
Added:
labels: {
759
Added:
filter: item => item.datasetIndex !== 0
760
Added:
}
707
761
},
708
762
tooltip: {
709
763
mode: "index",
@@ -717,9 +771,11 @@
717
771
} else {
718
772
const chart = charts.get(probe);
719
773
720
Removed:
chart.data.labels = labels;
721
Removed:
chart.data.datasets[0].data = values;
722
Removed:
chart.data.datasets[0].label = `${probe} ${unit ? `(${unit})` : ""}${smoothingEnabled ? " smoothed" : ""}`;
774
Added:
chart.data.labels = series.labels;
775
Added:
chart.data.datasets[0].data = series.lower;
776
Added:
chart.data.datasets[1].data = series.upper;
777
Added:
chart.data.datasets[2].data = series.trend;
778
Added:
chart.data.datasets[2].label = trendLabel;
723
779
chart.options.scales.x.ticks.maxTicksLimit = TIMEFRAMES[selectedTimeframe()].ticks;
724
780
chart.options.scales.y.min = yBounds.min;
725
781
chart.options.scales.y.max = yBounds.max;
roles/dashboard/t/02-readings-api.t
@@ -18,12 +18,12 @@
18
18
->json_is( '/readings/0/value' => 7.12 )
19
19
->json_hasnt('/readings/1');
20
20
21
Removed:
$t->get_ok('/api/readings/ph?since=2026-07-05T12:00:05Z')
21
Added:
$t->get_ok('/api/readings/ph?since=2026-07-05T11:59:55Z')
22
22
->status_is(200)
23
23
->json_is( '/readings/0/value' => 7.12 )
24
24
->json_hasnt('/readings/1');
25
25
26
Removed:
$t->get_ok('/api/readings/ph?since=2026-07-05T12:00:15Z')
26
Added:
$t->get_ok('/api/readings/ph?since=2026-07-05T12:00:05Z')
27
27
->status_is(200)
28
28
->json_hasnt('/readings/0');
29
29
roles/dashboard/t/04-readings-series.t
@@ -13,59 +13,74 @@
13
13
14
14
seed_series_readings($t);
15
15
seed_smoothing_readings($t);
16
Added:
seed_gap_readings($t);
16
17
17
18
$t->get_ok('/api/readings/ec/series?timeframe=hour')
18
19
->status_is(200)
19
Removed:
->json_is( '/probe' => 'ec' )
20
Removed:
->json_is( '/timeframe' => 'hour' )
21
Removed:
->json_is( '/readings/58/value' => 1050 )
22
Removed:
->json_is( '/readings/58/count' => 2 )
20
Added:
->json_is( '/probe' => 'ec' )
21
Added:
->json_is( '/timeframe' => 'hour' )
22
Added:
->json_is( '/readings/58/value' => 1050 )
23
Added:
->json_is( '/readings/58/lower' => 1000 )
24
Added:
->json_is( '/readings/58/upper' => 1100 )
25
Added:
->json_is( '/readings/58/count' => 2 )
26
Added:
->json_is( '/band/lower_percentile' => 10 )
27
Added:
->json_is( '/band/upper_percentile' => 90 )
28
Added:
->json_is( '/gap_seconds' => 30 )
23
29
->json_hasnt('/readings/60');
24
30
25
31
$t->get_ok('/api/readings/ec/series?timeframe=day')
26
32
->status_is(200)
27
Removed:
->json_is( '/readings/23/value' => 1050 )
28
Removed:
->json_is( '/readings/23/count' => 2 )
29
Removed:
->json_hasnt('/readings/24');
33
Added:
->json_is( '/readings/95/value' => 1050 )
34
Added:
->json_is( '/readings/95/count' => 2 )
35
Added:
->json_hasnt('/readings/96');
30
36
31
37
$t->get_ok('/api/readings/ec/series?timeframe=month')
32
38
->status_is(200)
33
Removed:
->json_is( '/timeframe' => 'month' )
34
Removed:
->json_is( '/readings/29/value' => 1050 )
35
Removed:
->json_is( '/readings/29/count' => 2 )
36
Removed:
->json_hasnt('/readings/30');
39
Added:
->json_is( '/timeframe' => 'month' )
40
Added:
->json_is( '/readings/119/value' => 1050 )
41
Added:
->json_is( '/readings/119/count' => 2 )
42
Added:
->json_hasnt('/readings/120');
37
43
38
44
$t->get_ok('/api/readings/ec/series?timeframe=year')
39
45
->status_is(200)
40
Removed:
->json_is( '/readings/11/value' => 1050 )
41
Removed:
->json_is( '/readings/11/count' => 2 )
42
Removed:
->json_hasnt('/readings/12');
46
Added:
->json_is( '/readings/364/value' => 1050 )
47
Added:
->json_is( '/readings/364/count' => 2 )
48
Added:
->json_hasnt('/readings/365');
43
49
44
50
$t->get_ok('/api/readings/ec/series?timeframe=week&range=2')
45
51
->status_is(200)
46
Removed:
->json_is( '/range' => 2 )
47
Removed:
->json_is( '/readings/13/value' => 1050 )
48
Removed:
->json_is( '/readings/13/count' => 2 )
49
Removed:
->json_hasnt('/readings/14');
52
Added:
->json_is( '/range' => 2 )
53
Added:
->json_is( '/readings/167/value' => 1050 )
54
Added:
->json_is( '/readings/167/count' => 2 )
55
Added:
->json_hasnt('/readings/168');
50
56
51
57
$t->get_ok('/api/readings/orp/series?timeframe=hour&smooth=0')
52
58
->status_is(200)
53
59
->json_is( '/smooth' => 0 )
54
Removed:
->json_is( '/readings/56/value' => 0 )
55
Removed:
->json_is( '/readings/57/value' => 70 )
56
Removed:
->json_is( '/readings/58/value' => 0 );
60
Added:
->json_is( '/readings/52/value' => 70 )
61
Added:
->json_is( '/readings/52/lower' => 70 )
62
Added:
->json_is( '/readings/52/upper' => 70 )
63
Added:
->json_is( '/readings/55/value' => 70 )
64
Added:
->json_is( '/readings/56/value' => 70 );
57
65
58
66
$t->get_ok('/api/readings/orp/series?timeframe=hour&smooth=1')
59
67
->status_is(200)
60
68
->json_is( '/smooth' => 1 )
61
Removed:
->json_is( '/readings/56/value' => 5 )
62
Removed:
->json_is( '/readings/57/value' => 10 )
63
Removed:
->json_is( '/readings/58/value' => 5 );
69
Added:
->json_is( '/readings/52/value' => 0 )
70
Added:
->json_is( '/readings/52/upper' => 70 )
71
Added:
->json_is( '/readings/55/value' => 70 )
72
Added:
->json_is( '/readings/56/value' => 70 )
73
Added:
->json_is( '/readings/58/value' => 0 );
64
74
75
Added:
$t->get_ok('/api/readings/do/series?timeframe=hour&smooth=1')
76
Added:
->status_is(200)
77
Added:
->json_is( '/readings/57/gap' => 1 );
78
Added:
65
79
$t->get_ok('/api/readings/ec/series?timeframe=century')->status_is(400);
66
80
$t->get_ok('/api/readings/ec/series?timeframe=week&range=0')->status_is(400);
67
81
$t->get_ok('/api/readings/ec/series?timeframe=week&range=25')->status_is(400);
68
Removed:
$t->get_ok('/api/readings/ec/series?timeframe=week&smooth=yes')->status_is(400);
82
Added:
$t->get_ok('/api/readings/ec/series?timeframe=week&smooth=yes')
83
Added:
->status_is(400);
69
84
70
85
done_testing;
71
86
@@ -104,34 +119,64 @@
104
119
}
105
120
106
121
sub seed_smoothing_readings {
107
Removed:
my ($t) = @_;
108
Removed:
my $bucket = minute_floor(time);
109
Removed:
my $db = $t->app->sqlite->db;
122
Added:
my ($t) = @_;
123
Added:
my $bucket = minute_floor(time);
124
Added:
my $db = $t->app->sqlite->db;
110
125
111
Removed:
my @readings = (
112
Removed:
( map { [ $bucket - 180 + $_, 0 ] } 10 .. 15 ),
113
Removed:
[ $bucket - 120 + 10, 70 ],
114
Removed:
( map { [ $bucket - 60 + $_, 0 ] } 10 .. 15 ),
115
Removed:
);
126
Added:
my @minute_values = ( 0, 0, 70, 0, 0, 70, 70, 0, 0 );
127
Added:
my @readings;
116
128
117
Removed:
for my $reading (@readings) {
118
Removed:
my $timestamp = utc_timestamp( $reading->[0] );
129
Added:
for my $minute ( 0 .. $#minute_values ) {
130
Added:
push @readings, map {
131
Added:
[ $bucket - ( @minute_values * 60 ) + ( $minute * 60 ) + $_,
132
Added:
$minute_values[$minute]
133
Added:
]
134
Added:
} map { $_ * 6 } 0 .. 9;
135
Added:
}
119
136
120
Removed:
$db->query(
121
Removed:
q{
137
Added:
for my $reading (@readings) {
138
Added:
my $timestamp = utc_timestamp( $reading->[0] );
139
Added:
140
Added:
$db->query(
141
Added:
q{
122
142
INSERT INTO readings (received_at, timestamp, node, probe, value, unit)
123
143
VALUES (?, ?, ?, ?, ?, ?)
124
144
},
125
Removed:
$timestamp,
126
Removed:
$timestamp,
127
Removed:
'fapg-daq-zero-orp-01',
128
Removed:
'orp',
129
Removed:
$reading->[1],
130
Removed:
'mV',
131
Removed:
);
132
Removed:
}
145
Added:
$timestamp,
146
Added:
$timestamp,
147
Added:
'fapg-daq-zero-orp-01',
148
Added:
'orp',
149
Added:
$reading->[1],
150
Added:
'mV',
151
Added:
);
152
Added:
}
133
153
134
Removed:
return;
154
Added:
return;
155
Added:
}
156
Added:
157
Added:
sub seed_gap_readings {
158
Added:
my ($t) = @_;
159
Added:
my $bucket = minute_floor(time);
160
Added:
my $db = $t->app->sqlite->db;
161
Added:
162
Added:
for my $offset ( 0, 6, 12, 18, 24, 60, 66, 72, 78, 84 ) {
163
Added:
my $timestamp = utc_timestamp( $bucket - 180 + $offset );
164
Added:
165
Added:
$db->query(
166
Added:
q{
167
Added:
INSERT INTO readings (received_at, timestamp, node, probe, value, unit)
168
Added:
VALUES (?, ?, ?, ?, ?, ?)
169
Added:
},
170
Added:
$timestamp,
171
Added:
$timestamp,
172
Added:
'fapg-daq-zero-do-01',
173
Added:
'do',
174
Added:
8,
175
Added:
'mg/L',
176
Added:
);
177
Added:
}
178
Added:
179
Added:
return;
135
180
}
136
181
137
182
sub utc_timestamp {