[Perl] DAQ system for the FAPG.
feat backfill one year of weather history
Use Open-Meteo archive API to fetch historical data in 90-day chunks. Runs automatically on first execution when DB has less than a year of data. Includes 1s sleep between chunks to respect API rate limits.
Changed files
roles/dashboard/lib/FAPG/DAQ/Dashboard/Model/Weather.pm
@@ -131,4 +131,16 @@
131
131
)->hash;
132
132
}
133
133
134
Added:
sub oldest_epoch ($self) {
135
Added:
my $row = $self->sqlite->db->query(
136
Added:
q{SELECT MIN(epoch) AS epoch FROM weather_hourly})->hash;
137
Added:
return $row ? $row->{epoch} : return;
138
Added:
}
139
Added:
140
Added:
sub count ($self) {
141
Added:
my $row = $self->sqlite->db->query(
142
Added:
q{SELECT COUNT(*) AS n FROM weather_hourly})->hash;
143
Added:
return $row->{n} // 0;
144
Added:
}
145
Added:
134
146
1;
roles/dashboard/lib/FAPG/DAQ/Dashboard/Service/WeatherFetcher.pm
@@ -11,17 +11,20 @@
11
11
has 'log' => sub { Mojo::Log->new };
12
12
13
13
use constant BASE_URL => 'https://api.open-meteo.com/v1/meteofrance';
14
Added:
use constant ARCHIVE_BASE_URL =>
15
Added:
'https://archive-api.open-meteo.com/v1/archive';
14
16
17
Added:
my @HOURLY_VARS = qw(
18
Added:
temperature_2m relative_humidity_2m rain
19
Added:
shortwave_radiation cloud_cover surface_pressure
20
Added:
);
21
Added:
15
22
sub fetch ($self) {
16
23
my $url = Mojo::URL->new(BASE_URL);
17
24
$url->query(
18
Removed:
latitude => $self->latitude,
19
Removed:
longitude => $self->longitude,
20
Removed:
hourly => join( ',',
21
Removed:
'temperature_2m', 'relative_humidity_2m',
22
Removed:
'rain', 'shortwave_radiation',
23
Removed:
'cloud_cover', 'soil_temperature_6cm',
24
Removed:
'surface_pressure', ),
25
Added:
latitude => $self->latitude,
26
Added:
longitude => $self->longitude,
27
Added:
hourly => join( ',', @HOURLY_VARS, 'soil_temperature_6cm' ),
25
28
daily => 'sunrise,sunset',
26
29
past_days => 7,
27
30
forecast_days => 2,
@@ -41,9 +44,36 @@
41
44
return $self->_parse_response($json);
42
45
}
43
46
44
Removed:
sub _parse_response ( $self, $json ) {
47
Added:
sub fetch_history ( $self, $start_date, $end_date ) {
48
Added:
my $url = Mojo::URL->new(ARCHIVE_BASE_URL);
49
Added:
$url->query(
50
Added:
latitude => $self->latitude,
51
Added:
longitude => $self->longitude,
52
Added:
hourly => join( ',', @HOURLY_VARS, 'soil_temperature_0_to_7cm' ),
53
Added:
daily => 'sunrise,sunset',
54
Added:
start_date => $start_date,
55
Added:
end_date => $end_date,
56
Added:
timezone => 'UTC',
57
Added:
);
58
Added:
59
Added:
my $tx = $self->ua->get($url);
60
Added:
61
Added:
if ( my $err = $tx->error ) {
62
Added:
my $msg = $err->{message} || "HTTP $err->{code}";
63
Added:
die "Weather history fetch failed: $msg";
64
Added:
}
65
Added:
66
Added:
my $json = $tx->result->json;
67
Added:
die 'Weather history fetch returned an invalid response'
68
Added:
if !$json || ref $json->{hourly} ne 'HASH';
69
Added:
return $self->_parse_response( $json,
70
Added:
soil_key => 'soil_temperature_0_to_7cm' );
71
Added:
}
72
Added:
73
Added:
sub _parse_response ( $self, $json, %opts ) {
45
74
return [] unless $json && $json->{hourly};
46
75
76
Added:
my $soil_key = $opts{soil_key} // 'soil_temperature_6cm';
47
77
my $hourly = $json->{hourly};
48
78
my $times = $hourly->{time} || [];
49
79
my $temps = $hourly->{temperature_2m} || [];
@@ -51,7 +81,7 @@
51
81
my $rains = $hourly->{rain} || [];
52
82
my $radiation = $hourly->{shortwave_radiation} || [];
53
83
my $clouds = $hourly->{cloud_cover} || [];
54
Removed:
my $soil_temp = $hourly->{soil_temperature_6cm} || [];
84
Added:
my $soil_temp = $hourly->{$soil_key} || [];
55
85
my $pressure = $hourly->{surface_pressure} || [];
56
86
57
87
# Build a date→sunrise/sunset lookup from daily data
roles/dashboard/script/fetch-weather
@@ -32,4 +32,50 @@
32
32
}
33
33
34
34
print "Weather ingestion fetched $fetched points and stored $stored points\n";
35
Added:
36
Added:
# Backfill history if the database has less than ~365 days of data.
37
Added:
# Fetches in 3-month chunks to avoid exceeding reasonable response sizes.
38
Added:
# The archive API counts separately from the forecast API quota.
39
Added:
my $one_year_ago = time - ( 365 * 24 * 60 * 60 );
40
Added:
my $oldest = $app->weather->oldest_epoch;
41
Added:
42
Added:
if ( !defined $oldest || $oldest > $one_year_ago ) {
43
Added:
require POSIX;
44
Added:
my $target_start = $one_year_ago;
45
Added:
my $end_epoch
46
Added:
= defined $oldest ? $oldest - 3600 : time - ( 7 * 24 * 60 * 60 );
47
Added:
my $chunk_days = 90;
48
Added:
49
Added:
print "Backfilling weather history...\n";
50
Added:
51
Added:
my $cursor = $target_start;
52
Added:
while ( $cursor < $end_epoch ) {
53
Added:
my $chunk_end = $cursor + ( $chunk_days * 24 * 60 * 60 );
54
Added:
$chunk_end = $end_epoch if $chunk_end > $end_epoch;
55
Added:
56
Added:
my $start_date = POSIX::strftime( '%Y-%m-%d', gmtime($cursor) );
57
Added:
my $end_date = POSIX::strftime( '%Y-%m-%d', gmtime($chunk_end) );
58
Added:
59
Added:
my $ok = eval {
60
Added:
my $rows = $app->weather_fetcher->fetch_history( $start_date,
61
Added:
$end_date );
62
Added:
my $n = $app->weather->store_batch($rows) // 0;
63
Added:
print " $start_date to $end_date: $n points\n";
64
Added:
1;
65
Added:
};
66
Added:
67
Added:
if ( !$ok ) {
68
Added:
my $err = $@ || 'Unknown error';
69
Added:
chomp $err;
70
Added:
print STDERR " Backfill chunk $start_date failed: $err\n";
71
Added:
last;
72
Added:
}
73
Added:
74
Added:
$cursor = $chunk_end + ( 24 * 60 * 60 );
75
Added:
sleep 1; # Be polite to the archive API
76
Added:
}
77
Added:
78
Added:
print "Backfill complete.\n";
79
Added:
}
80
Added:
35
81
exit 0;