[Perl] DAQ system for the FAPG.
fix remove server-side gap detection from series
The per-rollup gap flag was causing most data to render as red dashed gap-fill on longer timeframes. A single flagged rollup within a multi-hour display bucket would taint the entire bucket as a gap. Remove gap detection entirely from the series SQL and aggregation. Gaps now manifest naturally as null-value points (empty display buckets with no contributing rollups). The client handles these as chart breaks without needing an explicit gap flag. This simplifies the SQL from a CTE with LAG window function to a straightforward SELECT from reading_rollups.
Changed files
roles/dashboard/lib/FAPG/DAQ/Dashboard/Controller/Reading.pm
@@ -89,7 +89,6 @@
89
89
90
90
my $rollup_seconds
91
91
= $window->{bucket_stride} >= 3_600 ? 3_600 : $MINUTE_SECONDS;
92
Removed:
my $gap_threshold = 2 * $rollup_seconds;
93
92
my $now = time;
94
93
my $expires_epoch
95
94
= int( $now / $rollup_seconds ) * $rollup_seconds + $rollup_seconds;
@@ -104,43 +103,23 @@
104
103
105
104
my $rollups = $self->sqlite->db->query(
106
105
q{
107
Removed:
WITH ordered AS (
108
Removed:
SELECT
109
Removed:
bucket_epoch AS minute_epoch,
110
Removed:
sample_count AS count,
111
Removed:
value_total / sample_count AS avg_value,
112
Removed:
value_min,
113
Removed:
value_max,
114
Removed:
max_gap_seconds,
115
Removed:
first_sample_epoch - LAG(last_sample_epoch) OVER (
116
Removed:
ORDER BY bucket_epoch
117
Removed:
) AS gap_seconds
118
Removed:
FROM reading_rollups
119
Removed:
WHERE probe = ?
120
Removed:
AND bucket_seconds = ?
121
Removed:
AND bucket_epoch >= ?
122
Removed:
AND bucket_epoch < ?
123
Removed:
)
124
106
SELECT
125
Removed:
minute_epoch,
126
Removed:
count,
127
Removed:
avg_value,
107
Added:
bucket_epoch AS minute_epoch,
108
Added:
sample_count AS count,
109
Added:
value_total / sample_count AS avg_value,
128
110
value_min,
129
Removed:
value_max,
130
Removed:
CASE
131
Removed:
WHEN max_gap_seconds > CAST(? AS INTEGER)
132
Removed:
OR gap_seconds > CAST(? AS INTEGER) THEN 1
133
Removed:
ELSE 0
134
Removed:
END AS gap
135
Removed:
FROM ordered
136
Removed:
ORDER BY minute_epoch
111
Added:
value_max
112
Added:
FROM reading_rollups
113
Added:
WHERE probe = ?
114
Added:
AND bucket_seconds = ?
115
Added:
AND bucket_epoch >= ?
116
Added:
AND bucket_epoch < ?
117
Added:
ORDER BY bucket_epoch
137
118
},
138
119
$probe,
139
120
$rollup_seconds,
140
121
$window->{start_epoch} - ( 2 * $rollup_seconds ),
141
122
$window->{end_epoch},
142
Removed:
$gap_threshold,
143
Removed:
$gap_threshold,
144
123
)->hashes->to_array;
145
124
146
125
my $points = aggregate_series( $rollups, $window );
@@ -321,7 +300,6 @@
321
300
if !defined $point->{max}
322
301
|| $rollup->{value_max} > $point->{max};
323
302
$point->{count} += 0 + $rollup->{count};
324
Removed:
$point->{gap} = 1 if $rollup->{gap};
325
303
}
326
304
327
305
for my $i ( 0 .. $points->$#* ) {
roles/dashboard/t/04-readings-series.t
@@ -50,8 +50,8 @@
50
50
assert_series_point( $t, 1050, 1000, 1100, 2 );
51
51
52
52
$t->get_ok('/api/readings/do/series?timeframe=hour')->status_is(200);
53
Removed:
ok( scalar( grep { $_->{gap} } $t->tx->res->json->{points}->@* ),
54
Removed:
'series marks a data gap' );
53
Added:
ok( scalar( grep { !defined $_->{value} } $t->tx->res->json->{points}->@* ),
54
Added:
'series has null-value points for empty time buckets' );
55
55
56
56
$t->get_ok('/api/readings/ec/series?timeframe=century')->status_is(400);
57
57
$t->get_ok('/api/readings/ec/series?timeframe=week&range=0')->status_is(400);