1 #!/usr/bin/env perl 2 3 use strict; 4 use warnings; 5 6 use Mojo::File qw(curfile); 7 use lib curfile->dirname->sibling('lib')->to_string; 8 9 use FAPG::DAQ::Dashboard; 10 11 my $app = FAPG::DAQ::Dashboard->new; 12 my $weather_config = $app->config->{weather} // {}; 13 14 if ( !$weather_config->{enabled} ) { 15 print "Weather ingestion is disabled in dashboard.yml\n"; 16 exit 0; 17 } 18 19 my ( $fetched, $stored ); 20 my $success = eval { 21 my $rows = $app->weather_fetcher->fetch; 22 $fetched = scalar @$rows; 23 $stored = $app->weather->store_batch($rows) // 0; 24 1; 25 }; 26 27 if ( !$success ) { 28 my $error = $@ || 'Unknown weather ingestion failure'; 29 chomp $error; 30 print STDERR "$error\n"; 31 exit 1; 32 } 33 34 print "Weather ingestion fetched $fetched points and stored $stored points\n"; 35 36 # Backfill history if the database has less than ~365 days of data 37 # or if the extended columns (solar, cloud, soil, pressure) are empty. 38 my $one_year_ago = time - ( 365 * 24 * 60 * 60 ); 39 my $oldest = $app->weather->oldest_epoch; 40 my $needs_history = !defined $oldest || $oldest > $one_year_ago; 41 my $needs_extended = !$app->weather->has_extended_data; 42 43 if ( $needs_history || $needs_extended ) { 44 require POSIX; 45 my $target_start = $one_year_ago; 46 my $end_epoch 47 = $needs_extended 48 ? time - ( 7 * 24 * 60 * 60 ) 49 : ( defined $oldest ? $oldest - 3600 : time - ( 7 * 24 * 60 * 60 ) ); 50 my $chunk_days = 90; 51 52 print "Backfilling weather history" 53 . ( $needs_extended ? " (extended columns)" : "" ) . "...\n"; 54 55 my $cursor = $target_start; 56 while ( $cursor < $end_epoch ) { 57 my $chunk_end = $cursor + ( $chunk_days * 24 * 60 * 60 ); 58 $chunk_end = $end_epoch if $chunk_end > $end_epoch; 59 60 my $start_date = POSIX::strftime( '%Y-%m-%d', gmtime($cursor) ); 61 my $end_date = POSIX::strftime( '%Y-%m-%d', gmtime($chunk_end) ); 62 63 my $ok = eval { 64 my $rows = $app->weather_fetcher->fetch_history( $start_date, 65 $end_date ); 66 my $n = $app->weather->store_batch($rows) // 0; 67 print " $start_date to $end_date: $n points\n"; 68 1; 69 }; 70 71 if ( !$ok ) { 72 my $err = $@ || 'Unknown error'; 73 chomp $err; 74 print STDERR " Backfill chunk $start_date failed: $err\n"; 75 last; 76 } 77 78 $cursor = $chunk_end + ( 24 * 60 * 60 ); 79 sleep 1; # Be polite to the archive API 80 } 81 82 print "Backfill complete.\n"; 83 } 84 85 exit 0; 86