Carbon intensity data caching

setup.php

@@ -213,6 +213,11 @@ function plugin_carbon_check_prerequisites()
         $prerequisitesSuccess = false;
     }
 
+    if ($DB->use_timezones !== true) {
+        echo "Enable timezones support<br>";
+        $prerequisitesSuccess = false;
+    }
+
     if (getenv('CI') === false) {
         // only when not under test
         $version_string = $DB->getVersion();

src/CarbonIntensity.php

@@ -214,7 +214,7 @@ public function getFirstKnownDate(string $zone_name, string $source_name): ?Date
      */
     public function downloadOneZone(ClientInterface $data_source, string $zone_name, int $limit = 0, ?ProgressBar $progress_bar = null): int
     {
-        $start_date = $this->getDownloadStartDate($zone_name, $data_source);
+        $start_date = $this->getDownloadStartDate();
 
         $total_count = 0;
 
@@ -257,34 +257,15 @@ public function downloadOneZone(ClientInterface $data_source, string $zone_name,
             }
         }
 
-        $first_known_intensity_date = $this->getFirstKnownDate($zone_name, $data_source->getSourceName());
-        $incremental = false;
-        if ($first_known_intensity_date !== null) {
-            $incremental = ($start_date >= $first_known_intensity_date);
-        }
-        if ($first_known_intensity_date !== null && $first_known_intensity_date <= $data_source->getHardStartDate()) {
-            // Cannot download older data than absolute start date of the source, then switch to incremetal mode
-            $incremental = true;
-        }
-        if ($incremental) {
-            $start_date = max($data_source->getMaxIncrementalAge(), $this->getLastKnownDate($zone_name, $data_source->getSourceName()));
-            $start_date = $start_date->add(new DateInterval('PT1H'));
-            $count = $data_source->incrementalDownload($zone_name, $start_date, $this, $limit);
-            $total_count += $count;
-            return $total_count;
-        }
-
-        return $total_count;
+        return 0;
     }
 
     /**
      * Get the oldest date where data are required
      *
-     * @param string                   $zone_name   ignored for now; zone to examine
-     * @param ClientInterface          $data_source ignored for now; data source
      * @return DateTimeImmutable|null
      */
-    public function getDownloadStartDate(string $zone_name, ClientInterface $data_source): ?DateTimeImmutable
+    public function getDownloadStartDate(): ?DateTimeImmutable
     {
         // Get the default oldest date od data to download
         $start_date = new DateTime(self::MIN_HISTORY_LENGTH);

src/Command/CollectCarbonIntensityCommand.php

@@ -46,6 +46,7 @@
 use Symfony\Component\Console\Helper\QuestionHelper;
 use Symfony\Component\Console\Input\InputArgument;
 use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Input\InputOption;
 use Symfony\Component\Console\Output\OutputInterface;
 use Symfony\Component\Console\Question\ChoiceQuestion;
 use Symfony\Component\Console\Question\Question;
@@ -68,6 +69,7 @@ protected function configure()
            ->setDescription(__('Read carbon dioxyde intensity from external sources', 'carbon'))
            ->addArgument('source', InputArgument::REQUIRED, '')
            ->addArgument('zone', InputArgument::REQUIRED, '')
+           ->addOption('cache', null, InputOption::VALUE_NEGATABLE, 'Use cache. Cache is not read is disabled, but still fed by requests.')
            ;
     }
 
@@ -136,6 +138,7 @@ protected function execute(InputInterface $input, OutputInterface $output)
         $this->source_id = $data_source->getID();
 
         $zone_code = $input->getArgument('zone');
+        $use_cache = $input->getOption('cache');
         $zone = new Zone();
         $zone->getFromDBByCrit(['name' => $this->zones[$zone_code]]);
         $carbon_intensity = new CarbonIntensity();
@@ -165,11 +168,14 @@ protected function execute(InputInterface $input, OutputInterface $output)
         if ($this->client === null) {
             $this->client = ClientFactory::createByName($source_name);
         }
+        if ($use_cache === false) {
+            $this->client->disableCache();
+        }
 
         $carbon_intensity->downloadOneZone($this->client, $this->zones[$zone_code], 0, new ProgressBar($this->output));
 
         // Find start and stop dates to cover
-        $start_date = $carbon_intensity->getDownloadStartDate($this->zones[$zone_code], $this->client);
+        $start_date = $carbon_intensity->getDownloadStartDate();
         $gaps = $carbon_intensity->findGaps($this->source_id, $zone->getID(), $start_date);
 
         // Count the hours not covered by any sample

src/DataSource/CarbonIntensity/AbstractClient.php

@@ -48,6 +48,8 @@ abstract class AbstractClient implements ClientInterface
 {
     protected int $step;
 
+    protected bool $use_cache = true;
+
     abstract public function getSourceName(): string;
 
     abstract public function getDataInterval(): string;
@@ -77,6 +79,11 @@ abstract public function fetchDay(DateTimeImmutable $day, string $zone): array;
      */
     abstract public function fetchRange(DateTimeImmutable $start, DateTimeImmutable $stop, string $zone): array;
 
+    public function disableCache()
+    {
+        $this->use_cache = false;
+    }
+
     /**
      * Key of the configuration value that indicates if the full download is complete
      *
@@ -157,8 +164,17 @@ public function getZones(array $crit = []): array
 
     public function fullDownload(string $zone, DateTimeImmutable $start_date, DateTimeImmutable $stop_date, CarbonIntensity $intensity, int $limit = 0, ?ProgressBar $progress_bar = null): int
     {
+        if ($start_date >= $stop_date) {
+            return 0;
+        }
+        // Round start date to beginning of month
+        $start_date = $start_date->setTime(0, 0, 0, 0)->setDate((int) $start_date->format('Y'), (int) $start_date->format('m'), 1);
+        $stop_date = $stop_date->setTime(0, 0, 0, 0)->setDate((int) $stop_date->format('Y'), (int) $stop_date->format('m'), 1);
         $count = 0;
         $saved = 0;
+        if ($start_date == $stop_date) {
+            $stop_date = $stop_date->add(new DateInterval('P1M'));
+        }
 
         /**
          * Huge quantity of SQL queries will be executed
@@ -171,9 +187,18 @@ public function fullDownload(string $zone, DateTimeImmutable $start_date, DateTi
             // enpty string in PHPUnit environment
             $memory_limit = null;
         }
-        foreach ($this->sliceDateRangeByMonth($start_date, $stop_date) as $slice) {
+
+        // Traverse each month from start_date to end_date
+        $current_date = DateTime::createFromImmutable($start_date);
+        while ($current_date < $stop_date) {
+            $next_month = clone $current_date;
+            $next_month->add(new DateInterval('P1M'));
             try {
-                $data = $this->fetchRange($slice['start'], $slice['stop'], $zone);
+                $data = $this->fetchRange(
+                    DateTimeImmutable::createFromMutable($current_date),
+                    DateTimeImmutable::createFromMutable($next_month),
+                    $zone
+                );
             } catch (AbortException $e) {
                 break;
             }
@@ -194,8 +219,8 @@ public function fullDownload(string $zone, DateTimeImmutable $start_date, DateTi
                 // 8 MB memory left, emergency exit
                 return $saved > 0 ? $count : -$count;
             }
+            $current_date = $next_month;
         }
-
         return $saved > 0 ? $count : -$count;
     }
 
@@ -211,6 +236,10 @@ public function incrementalDownload(string $zone, DateTimeImmutable $start_date,
             } catch (AbortException $e) {
                 throw $e;
             }
+            $data = $this->formatOutput($data, $this->step);
+            if (!isset($data[$zone])) {
+                continue;
+            }
             $saved = $intensity->save($zone, $this->getSourceName(), $data[$zone]);
             $count += abs($saved);
             if ($limit > 0 && $count >= $limit) {
@@ -278,10 +307,9 @@ protected function sliceDateRangeByMonth(DateTimeImmutable $start, DateTimeImmut
      */
     protected function sliceDateRangeByDay(DateTimeImmutable $start, DateTimeImmutable $stop)
     {
-        $real_start = $start;
         $real_stop = $stop->setTime((int) $stop->format('H'), 0, 0);
 
-        $current_date = DateTime::createFromImmutable($real_start);
+        $current_date = DateTime::createFromImmutable($start);
         while ($current_date <= $real_stop) {
             yield DateTimeImmutable::createFromMutable($current_date);
             $current_date->add(new DateInterval('P1D'));

src/DataSource/CarbonIntensity/ClientInterface.php

@@ -129,6 +129,13 @@ public function getSourceName(): string;
      */
     public function createZones(): int;
 
+    /**
+     * Disable caching of data source
+     *
+     * @return void
+     */
+    public function disableCache();
+
     /**
      * Get the absolute starrt date of data from the source
      *

src/DataSource/CarbonIntensity/ElectricityMaps/Client.php

@@ -41,7 +41,6 @@
 use GlpiPlugin\Carbon\Source;
 use GlpiPlugin\Carbon\Zone;
 use GlpiPlugin\Carbon\Source_Zone;
-use Config as GlpiConfig;
 use GLPIKey;
 use GlpiPlugin\Carbon\DataSource\CarbonIntensity\AbortException;
 use GlpiPlugin\Carbon\DataSource\CarbonIntensity\AbstractClient;
@@ -264,16 +263,45 @@ public function fetchDay(DateTimeImmutable $day, string $zone): array
 
     public function fetchRange(DateTimeImmutable $start, DateTimeImmutable $stop, string $zone): array
     {
-
         // TODO: get zones from GLPI locations
         $params = [
             'zone' => $zone,
         ];
 
+        $base_path = GLPI_PLUGIN_DOC_DIR . '/carbon/carbon_intensity/' . $this->getSourceName() . '/' . $zone;
+        $cache_file = $this->getCacheFilename(
+            $base_path,
+            $start,
+            $stop
+        );
+        // If cached file exists, use it
+        if (file_exists($cache_file)) {
+            $full_response = json_decode(file_get_contents($cache_file), true);
+            return $full_response;
+        }
+        @mkdir(dirname($cache_file), 0755, true);
+
+        // Set timezone to +00:00 and extend range by 12 hours on each side
+        $request_start = $start->setTimezone(new DateTimeZone('+0000'))->sub(new DateInterval('PT12H'));
+        $request_stop = $stop->setTimezone(new DateTimeZone('+0000'))->add(new DateInterval('PT12H'));
         $this->step = 60;
 
-        $response = $this->client->request('GET', $this->base_url . self::PAST_URL, ['query' => $params]);
-        if (!$response) {
+        $step = new DateInterval('P10D');
+        $full_response = [];
+        $current_date = DateTime::createFromImmutable($request_start);
+        while ($current_date < $request_stop) {
+            $response = $this->client->request('GET', $this->base_url . self::PAST_URL, ['query' => $params]);
+            if (!$full_response) {
+                $full_response = $response;
+            } else {
+                $full_response['data'] = array_merge($full_response['data'], $response['data']);
+            }
+            $current_date->add($step);
+            if ($current_date > $request_stop) {
+                $current_date = $request_stop;
+            }
+        }
+        if (!$full_response) {
             return [];
         }
 
@@ -286,7 +314,9 @@ public function fetchRange(DateTimeImmutable $start, DateTimeImmutable $stop, st
             return [];
         }
 
-        return $$response['history'];
+        $json = json_encode($full_response);
+        file_put_contents($cache_file, $json);
+        return $full_response['data'];
     }
 
     protected function formatOutput(array $response, int $step): array
@@ -369,4 +399,14 @@ protected function sliceDateRangeByDay(DateTimeImmutable $start, DateTimeImmutab
             $current_date->setTime(0, 0, 0);
         }
     }
+
+    protected function getCacheFilename(string $base_dir, DateTimeImmutable $start, DateTimeImmutable $end): string
+    {
+        return sprintf(
+            '%s/%s_%s.json',
+            $base_dir,
+            $start->format('Y-m-d'),
+            $end->format('Y-m-d')
+        );
+    }
 }