[Perl] DAQ system for the FAPG.

dashboard: add fixed series API for probe charts

Commit
26b32cc51d242860bd8c6054f182f9f7fff869a5
Author
Marius Peter <dev@marius-peter.com>
Author date
Committer
Marius Peter <dev@marius-peter.com>
Committer date
Changed files
roles/dashboard/lib/FAPG/DAQ/Dashboard.pm
index e194e5e6..4ebf704d 100644..100644
@@ -74,6 +74,7 @@
74 74 $r->get('/')->to('dashboard#index');
75 75 $r->get('/graphs/:probe')->to('dashboard#graph');
76 76
77 Added: $r->get('/api/readings/:probe/series')->to('Reading#series');
77 78 $r->get('/api/readings/:probe')->to('Reading#list');
78 79 $r->get('/api/status/:probe')->to('Reading#status');
79 80 }
roles/dashboard/lib/FAPG/DAQ/Dashboard/Controller/Reading.pm
index b6c4a412..49acb106 100644..100644
@@ -3,9 +3,36 @@
3 3 package FAPG::DAQ::Dashboard::Controller::Reading;
4 4 use Mojo::Base 'Mojolicious::Controller', -signatures;
5 5
6 Removed: use POSIX qw(strftime);
6 Added: use POSIX qw(strftime);
7 Added: use Time::Local qw(timegm);
7 8
8 9 my $STATUS_REACHABLE_SECONDS = 120;
10 Added: my %SERIES_TIMEFRAMES = (
11 Added: hour => {
12 Added: bucket_count => 60,
13 Added: bucket_sql => '%Y-%m-%dT%H:%M:00Z',
14 Added: bucket_stride => 60,
15 Added: floor_to => 'minute',
16 Added: },
17 Added: day => {
18 Added: bucket_count => 24,
19 Added: bucket_sql => '%Y-%m-%dT%H:00:00Z',
20 Added: bucket_stride => 60 * 60,
21 Added: floor_to => 'hour',
22 Added: },
23 Added: week => {
24 Added: bucket_count => 7,
25 Added: bucket_sql => '%Y-%m-%dT00:00:00Z',
26 Added: bucket_stride => 24 * 60 * 60,
27 Added: floor_to => 'day',
28 Added: },
29 Added: year => {
30 Added: bucket_count => 12,
31 Added: bucket_sql => '%Y-%m-01T00:00:00Z',
32 Added: bucket_stride => undef,
33 Added: floor_to => 'month',
34 Added: },
35 Added: );
9 36
10 37 sub list ($self) {
11 38 my $probe = $self->param('probe') // '';
@@ -57,6 +84,69 @@
57 84 );
58 85 }
59 86
87 Added: sub series ($self) {
88 Added: my $probe = $self->param('probe') // '';
89 Added:
90 Added: my %known = map { $_->{key} => 1 } $self->probes->@*;
91 Added:
92 Added: return $self->render(
93 Added: status => 404,
94 Added: json => { error => "Unknown probe type: $probe", },
95 Added: ) unless $known{$probe};
96 Added:
97 Added: my $timeframe = $self->param('timeframe') // 'day';
98 Added: my $window = series_window($timeframe);
99 Added:
100 Added: return $self->render(
101 Added: status => 400,
102 Added: json => { error => "Unknown timeframe: $timeframe", },
103 Added: ) unless defined $window;
104 Added:
105 Added: my $unit = ( grep { $_->{key} eq $probe } $self->probes->@* )[0]{unit};
106 Added: my $points = series_points($window);
107 Added: my %index = map { $points->[$_]{timestamp} => $_ } 0 .. $points->$#*;
108 Added:
109 Added: my $rows = $self->sqlite->db->query(
110 Added: q{
111 Added: SELECT
112 Added: strftime(?, COALESCE(received_at, timestamp)) AS bucket,
113 Added: AVG(value) AS value,
114 Added: COUNT(*) AS count
115 Added: FROM readings
116 Added: WHERE probe = ?
117 Added: AND COALESCE(received_at, timestamp) >= ?
118 Added: AND COALESCE(received_at, timestamp) < ?
119 Added: GROUP BY bucket
120 Added: ORDER BY bucket
121 Added: },
122 Added: $window->{bucket_sql},
123 Added: $probe,
124 Added: $window->{start_iso},
125 Added: $window->{end_iso},
126 Added: )->hashes->to_array;
127 Added:
128 Added: for my $row ( $rows->@* ) {
129 Added: my $point_index = $index{ $row->{bucket} };
130 Added: next if !defined $point_index;
131 Added:
132 Added: $points->[$point_index] = {
133 Added: timestamp => $row->{bucket},
134 Added: value => 0 + $row->{value},
135 Added: unit => $unit,
136 Added: probe => $probe,
137 Added: count => 0 + $row->{count},
138 Added: };
139 Added: }
140 Added:
141 Added: $self->render(
142 Added: json => {
143 Added: probe => $probe,
144 Added: timeframe => $timeframe,
145 Added: readings => $points,
146 Added: },
147 Added: );
148 Added: }
149 Added:
60 150 sub status ($self) {
61 151 my $probe = $self->param('probe') // '';
62 152
@@ -104,6 +194,149 @@
104 194 sub reachable_since {
105 195 return strftime( '%Y-%m-%dT%H:%M:%SZ',
106 196 gmtime( time - $STATUS_REACHABLE_SECONDS ) );
197 Added: }
198 Added:
199 Added: sub series_window ($timeframe) {
200 Added: my $config = $SERIES_TIMEFRAMES{$timeframe} or return undef;
201 Added: my @now = gmtime;
202 Added: my ( $sec, $min, $hour, $mday, $mon, $year ) = @now[ 0 .. 5 ];
203 Added:
204 Added: if ( $config->{floor_to} eq 'minute' ) {
205 Added: my $end_epoch = timegm( 0, $min, $hour, $mday, $mon, $year ) + 60;
206 Added:
207 Added: return {
208 Added: bucket_count => $config->{bucket_count},
209 Added: bucket_sql => $config->{bucket_sql},
210 Added: bucket_stride => $config->{bucket_stride},
211 Added: start_epoch => $end_epoch
212 Added: - ( $config->{bucket_count} * $config->{bucket_stride} ),
213 Added: end_epoch => $end_epoch,
214 Added: start_iso => utc_timestamp(
215 Added: $end_epoch
216 Added: - ( $config->{bucket_count} * $config->{bucket_stride} )
217 Added: ),
218 Added: end_iso => utc_timestamp($end_epoch),
219 Added: };
220 Added: }
221 Added:
222 Added: if ( $config->{floor_to} eq 'hour' ) {
223 Added: my $end_epoch
224 Added: = timegm( 0, 0, $hour, $mday, $mon, $year ) + ( 60 * 60 );
225 Added:
226 Added: return {
227 Added: bucket_count => $config->{bucket_count},
228 Added: bucket_sql => $config->{bucket_sql},
229 Added: bucket_stride => $config->{bucket_stride},
230 Added: start_epoch => $end_epoch
231 Added: - ( $config->{bucket_count} * $config->{bucket_stride} ),
232 Added: end_epoch => $end_epoch,
233 Added: start_iso => utc_timestamp(
234 Added: $end_epoch
235 Added: - ( $config->{bucket_count} * $config->{bucket_stride} )
236 Added: ),
237 Added: end_iso => utc_timestamp($end_epoch),
238 Added: };
239 Added: }
240 Added:
241 Added: if ( $config->{floor_to} eq 'day' ) {
242 Added: my $end_epoch
243 Added: = timegm( 0, 0, 0, $mday, $mon, $year ) + ( 24 * 60 * 60 );
244 Added:
245 Added: return {
246 Added: bucket_count => $config->{bucket_count},
247 Added: bucket_sql => $config->{bucket_sql},
248 Added: bucket_stride => $config->{bucket_stride},
249 Added: start_epoch => $end_epoch
250 Added: - ( $config->{bucket_count} * $config->{bucket_stride} ),
251 Added: end_epoch => $end_epoch,
252 Added: start_iso => utc_timestamp(
253 Added: $end_epoch
254 Added: - ( $config->{bucket_count} * $config->{bucket_stride} )
255 Added: ),
256 Added: end_iso => utc_timestamp($end_epoch),
257 Added: };
258 Added: }
259 Added:
260 Added: if ( $config->{floor_to} eq 'month' ) {
261 Added: my ( $start_year, $start_month )
262 Added: = normalize_month( $year, $mon - 11 );
263 Added: my ( $end_year, $end_month ) = normalize_month( $year, $mon + 1 );
264 Added:
265 Added: return {
266 Added: bucket_count => $config->{bucket_count},
267 Added: bucket_sql => $config->{bucket_sql},
268 Added: bucket_stride => $config->{bucket_stride},
269 Added: start_epoch => timegm( 0, 0, 0, 1, $start_month, $start_year ),
270 Added: end_epoch => timegm( 0, 0, 0, 1, $end_month, $end_year ),
271 Added: start_iso => utc_timestamp(
272 Added: timegm( 0, 0, 0, 1, $start_month, $start_year )
273 Added: ),
274 Added: end_iso =>
275 Added: utc_timestamp( timegm( 0, 0, 0, 1, $end_month, $end_year ) ),
276 Added: };
277 Added: }
278 Added:
279 Added: return undef;
280 Added: }
281 Added:
282 Added: sub series_points ($window) {
283 Added: my @points;
284 Added:
285 Added: if ( defined $window->{bucket_stride} ) {
286 Added: for my $i ( 0 .. $window->{bucket_count} - 1 ) {
287 Added: push @points,
288 Added: {
289 Added: timestamp => utc_timestamp(
290 Added: $window->{start_epoch} + ( $i * $window->{bucket_stride} )
291 Added: ),
292 Added: value => undef,
293 Added: unit => undef,
294 Added: probe => undef,
295 Added: count => 0,
296 Added: };
297 Added: }
298 Added:
299 Added: return \@points;
300 Added: }
301 Added:
302 Added: my @cursor = gmtime( $window->{start_epoch} );
303 Added: my ( $year, $month ) = ( $cursor[5], $cursor[4] );
304 Added:
305 Added: for my $i ( 0 .. $window->{bucket_count} - 1 ) {
306 Added: my ( $bucket_year, $bucket_month )
307 Added: = normalize_month( $year, $month + $i );
308 Added:
309 Added: push @points,
310 Added: {
311 Added: timestamp => utc_timestamp(
312 Added: timegm( 0, 0, 0, 1, $bucket_month, $bucket_year )
313 Added: ),
314 Added: value => undef,
315 Added: unit => undef,
316 Added: probe => undef,
317 Added: count => 0,
318 Added: };
319 Added: }
320 Added:
321 Added: return \@points;
322 Added: }
323 Added:
324 Added: sub normalize_month ( $year, $month ) {
325 Added: while ( $month < 0 ) {
326 Added: $month += 12;
327 Added: $year--;
328 Added: }
329 Added:
330 Added: while ( $month > 11 ) {
331 Added: $month -= 12;
332 Added: $year++;
333 Added: }
334 Added:
335 Added: return ( $year, $month );
336 Added: }
337 Added:
338 Added: sub utc_timestamp ($epoch) {
339 Added: return strftime( '%Y-%m-%dT%H:%M:%SZ', gmtime $epoch );
107 340 }
108 341
109 342 1;
roles/dashboard/public/js/dashboard.js
index fe82fc14..f8a77840 100644..100644
@@ -166,11 +166,6 @@
166 166 return TIMEFRAMES[activeTimeframe] ? activeTimeframe : DEFAULT_TIMEFRAME;
167 167 }
168 168
169 Removed: function timeframeSince() {
170 Removed: const timeframe = TIMEFRAMES[selectedTimeframe()];
171 Removed: return new Date(Date.now() - timeframe.ms).toISOString().replace(/\.\d{3}Z$/, "Z");
172 Removed: }
173 Removed:
174 169 function formatChartTime(timestamp) {
175 170 const date = new Date(timestamp);
176 171 const timeframe = selectedTimeframe();
@@ -198,7 +193,7 @@
198 193
199 194 return date.toLocaleDateString([], {
200 195 month: "short",
201 Removed: day: "numeric"
196 Added: year: "numeric"
202 197 });
203 198 }
204 199
@@ -629,22 +624,34 @@
629 624 const status = statusElement(probe);
630 625
631 626 try {
632 Removed: const params = new URLSearchParams({
633 Removed: limit: "2000",
634 Removed: since: timeframeSince()
627 Added: const seriesParams = new URLSearchParams({
628 Added: timeframe: selectedTimeframe()
635 629 });
636 Removed: const response = await fetch(`/api/readings/${probe}?${params}`);
630 Added: const readingsParams = new URLSearchParams({
631 Added: limit: "10"
632 Added: });
633 Added: const [seriesResponse, readingsResponse] = await Promise.all([
634 Added: fetch(`/api/readings/${probe}/series?${seriesParams}`),
635 Added: fetch(`/api/readings/${probe}?${readingsParams}`)
636 Added: ]);
637 637
638 Removed: if (!response.ok) {
639 Removed: throw new Error(`HTTP ${response.status}`);
638 Added: if (!seriesResponse.ok) {
639 Added: throw new Error(`HTTP ${seriesResponse.status}`);
640 640 }
641 641
642 Removed: const payload = await response.json();
643 Removed: const rows = payload.readings || [];
642 Added: if (!readingsResponse.ok) {
643 Added: throw new Error(`HTTP ${readingsResponse.status}`);
644 Added: }
644 645
646 Added: const seriesPayload = await seriesResponse.json();
647 Added: const readingsPayload = await readingsResponse.json();
648 Added: const rows = seriesPayload.readings || [];
649 Added: const readings = readingsPayload.readings || [];
650 Added:
645 651 const labels = rows.map(labelForRow);
646 652 const rawValues = rows.map(row => Number(row.value));
647 653 const values = chartValues(rawValues);
654 Added: const latest = [...readings].reverse().find(row => Number.isFinite(Number(row.value)));
648 655 const unit = rows.length ? rows[rows.length - 1].unit : "";
649 656 const yBounds = yAxisBounds(probe, values);
650 657
@@ -702,17 +709,16 @@
702 709 chart.update();
703 710 }
704 711
705 Removed: if (rows.length) {
706 Removed: const latest = rows[rows.length - 1];
712 Added: if (latest) {
707 713 const age = ageMs(latest.timestamp);
708 714
709 715 status.textContent =
710 Removed: `Latest: ${latest.value} ${latest.unit || ""} from ${latest.node} at ${formatDateTime(latest.timestamp)} (${formatAge(age)})`;
716 Added: `Latest: ${latest.value} ${latest.unit || ""} at ${formatDateTime(latest.timestamp)} (${formatAge(age)})`;
711 717 } else {
712 718 status.textContent = "No readings found for this probe yet.";
713 719 }
714 720
715 Removed: renderMessageTable(probe, rows);
721 Added: renderMessageTable(probe, readings);
716 722 } catch (error) {
717 723 status.textContent = `Could not load ${probe} readings: ${error.message}`;
718 724 renderMessageTable(probe, []);
roles/dashboard/t/04-readings-series.t
index 00000000..4afcade1 000000..100644
@@ -0,0 +1,85 @@
1 Added: # -*- mode: cperl; -*-
2 Added:
3 Added: use Mojo::Base -strict;
4 Added:
5 Added: use Test2::V0;
6 Added:
7 Added: use FindBin;
8 Added: use lib "${FindBin::Bin}/../lib/";
9 Added: use lib "${FindBin::Bin}/lib/";
10 Added: use Dashboard::Test qw(test_app);
11 Added:
12 Added: my $t = test_app();
13 Added:
14 Added: seed_series_readings($t);
15 Added:
16 Added: $t->get_ok('/api/readings/ec/series?timeframe=hour')
17 Added: ->status_is(200)
18 Added: ->json_is( '/probe' => 'ec' )
19 Added: ->json_is( '/timeframe' => 'hour' )
20 Added: ->json_is( '/readings/58/value' => 1050 )
21 Added: ->json_is( '/readings/58/count' => 2 )
22 Added: ->json_hasnt('/readings/60');
23 Added:
24 Added: $t->get_ok('/api/readings/ec/series?timeframe=day')
25 Added: ->status_is(200)
26 Added: ->json_is( '/readings/23/value' => 1050 )
27 Added: ->json_is( '/readings/23/count' => 2 )
28 Added: ->json_hasnt('/readings/24');
29 Added:
30 Added: $t->get_ok('/api/readings/ec/series?timeframe=year')
31 Added: ->status_is(200)
32 Added: ->json_is( '/readings/11/value' => 1050 )
33 Added: ->json_is( '/readings/11/count' => 2 )
34 Added: ->json_hasnt('/readings/12');
35 Added:
36 Added: $t->get_ok('/api/readings/ec/series?timeframe=century')->status_is(400);
37 Added:
38 Added: done_testing;
39 Added:
40 Added: sub seed_series_readings {
41 Added: my ($t) = @_;
42 Added: my $bucket = minute_floor(time);
43 Added: my $db = $t->app->sqlite->db;
44 Added:
45 Added: $db->query(
46 Added: q{
47 Added: INSERT INTO readings (received_at, timestamp, node, probe, value, unit)
48 Added: VALUES (?, ?, ?, ?, ?, ?)
49 Added: },
50 Added: utc_timestamp( $bucket - 50 ),
51 Added: utc_timestamp( $bucket - 50 ),
52 Added: 'fapg-daq-zero-ec-01',
53 Added: 'ec',
54 Added: 1000,
55 Added: 'µS/cm',
56 Added: );
57 Added:
58 Added: $db->query(
59 Added: q{
60 Added: INSERT INTO readings (received_at, timestamp, node, probe, value, unit)
61 Added: VALUES (?, ?, ?, ?, ?, ?)
62 Added: },
63 Added: utc_timestamp( $bucket - 40 ),
64 Added: utc_timestamp( $bucket - 40 ),
65 Added: 'fapg-daq-zero-ec-01',
66 Added: 'ec',
67 Added: 1100,
68 Added: 'µS/cm',
69 Added: );
70 Added:
71 Added: return;
72 Added: }
73 Added:
74 Added: sub utc_timestamp {
75 Added: my ($epoch) = @_;
76 Added: require POSIX;
77 Added: return POSIX::strftime( '%Y-%m-%dT%H:%M:%SZ', gmtime $epoch );
78 Added: }
79 Added:
80 Added: sub minute_floor {
81 Added: my ($epoch) = @_;
82 Added: require Time::Local;
83 Added: my ( $sec, $min, $hour, $mday, $mon, $year ) = gmtime($epoch);
84 Added: return Time::Local::timegm( 0, $min, $hour, $mday, $mon, $year );
85 Added: }