[Perl] DAQ system for the FAPG.
fix backfill extended weather columns for existing data
Detect when solar_radiation is NULL and re-fetch the full year via archive API. Existing rows are updated via INSERT OR REPLACE with the new column values.
Changed files
roles/dashboard/lib/FAPG/DAQ/Dashboard/Model/Weather.pm
@@ -143,4 +143,12 @@
143
143
return $row->{n} // 0;
144
144
}
145
145
146
Added:
sub has_extended_data ($self) {
147
Added:
my $row = $self->sqlite->db->query(
148
Added:
q{SELECT COUNT(*) AS n FROM weather_hourly
149
Added:
WHERE solar_radiation IS NOT NULL}
150
Added:
)->hash;
151
Added:
return ( $row->{n} // 0 ) > 0;
152
Added:
}
153
Added:
146
154
1;
roles/dashboard/script/fetch-weather
@@ -33,20 +33,24 @@
33
33
34
34
print "Weather ingestion fetched $fetched points and stored $stored points\n";
35
35
36
Removed:
# Backfill history if the database has less than ~365 days of data.
37
Removed:
# Fetches in 3-month chunks to avoid exceeding reasonable response sizes.
38
Removed:
# The archive API counts separately from the forecast API quota.
39
Removed:
my $one_year_ago = time - ( 365 * 24 * 60 * 60 );
40
Removed:
my $oldest = $app->weather->oldest_epoch;
36
Added:
# Backfill history if the database has less than ~365 days of data
37
Added:
# or if the extended columns (solar, cloud, soil, pressure) are empty.
38
Added:
my $one_year_ago = time - ( 365 * 24 * 60 * 60 );
39
Added:
my $oldest = $app->weather->oldest_epoch;
40
Added:
my $needs_history = !defined $oldest || $oldest > $one_year_ago;
41
Added:
my $needs_extended = !$app->weather->has_extended_data;
41
42
42
Removed:
if ( !defined $oldest || $oldest > $one_year_ago ) {
43
Added:
if ( $needs_history || $needs_extended ) {
43
44
require POSIX;
44
45
my $target_start = $one_year_ago;
45
46
my $end_epoch
46
Removed:
= defined $oldest ? $oldest - 3600 : time - ( 7 * 24 * 60 * 60 );
47
Added:
= $needs_extended
48
Added:
? time - ( 7 * 24 * 60 * 60 )
49
Added:
: ( defined $oldest ? $oldest - 3600 : time - ( 7 * 24 * 60 * 60 ) );
47
50
my $chunk_days = 90;
48
51
49
Removed:
print "Backfilling weather history...\n";
52
Added:
print "Backfilling weather history"
53
Added:
. ( $needs_extended ? " (extended columns)" : "" ) . "...\n";
50
54
51
55
my $cursor = $target_start;
52
56
while ( $cursor < $end_epoch ) {