diff --git a/src/CarbonEmission.php b/src/CarbonEmission.php index b8dabee7..2033f37a 100644 --- a/src/CarbonEmission.php +++ b/src/CarbonEmission.php @@ -38,6 +38,7 @@ use Entity; use Location; use CommonDBTM; +use DBmysqlIterator; class CarbonEmission extends CommonDBChild { @@ -178,9 +179,9 @@ public function rawSearchOptions() * @param integer $id * @param DateTimeInterface|null $start * @param DateTimeInterface|null $stop - * @return array + * @return DBmysqlIterator */ - public function findGaps(string $itemtype, int $id, ?DateTimeInterface $start, ?DateTimeInterface $stop = null): array + public function findGaps(string $itemtype, int $id, ?DateTimeInterface $start, ?DateTimeInterface $stop = null): DBmysqlIterator { $criteria = [ 'itemtype' => $itemtype, diff --git a/src/CarbonIntensity.php b/src/CarbonIntensity.php index a319562e..8f3c9051 100644 --- a/src/CarbonIntensity.php +++ b/src/CarbonIntensity.php @@ -38,6 +38,7 @@ use DateTimeImmutable; use DateTimeInterface; use DBmysql; +use DBmysqlIterator; use GlpiPlugin\Carbon\Source; use GlpiPlugin\Carbon\Zone; use Glpi\DBAL\QueryParam; @@ -394,9 +395,9 @@ public function save(string $zone_name, string $source_name, array $data): int * @param integer $zone_id * @param DateTimeInterface $start * @param DateTimeInterface|null $stop - * @return array + * @return DBmysqlIterator */ - public function findGaps(int $source_id, int $zone_id, DateTimeInterface $start, ?DateTimeInterface $stop = null): array + public function findGaps(int $source_id, int $zone_id, DateTimeInterface $start, ?DateTimeInterface $stop = null): DBmysqlIterator { $criteria = [ Source::getForeignKeyField() => $source_id, diff --git a/src/Impact/Embodied/Boavizta/Computer.php b/src/Impact/Embodied/Boavizta/Computer.php index 85ee6c89..f395664e 100644 --- a/src/Impact/Embodied/Boavizta/Computer.php +++ b/src/Impact/Embodied/Boavizta/Computer.php @@ -195,7 +195,7 @@ protected function analyzeHardware(): array ); if ($device_hard_drive_type !== false && $device_hard_drive_type->fields['name'] === 'removable') { // Ignore removable storage (USB sticks, ...) - continue; + break; } $interface_type = new InterfaceType(); $interface_type->getFromDB($device_hard_drive->fields['interfacetypes_id']); diff --git a/src/Toolbox.php b/src/Toolbox.php index bd03445e..df245fa0 100644 --- a/src/Toolbox.php +++ b/src/Toolbox.php @@ -37,11 +37,13 @@ use DateTimeImmutable; use DateTimeInterface; use DBmysql; +use DBmysqlIterator; use Glpi\Dashboard\Dashboard as GlpiDashboard; use Infocom; use Location; use Glpi\DBAL\QueryExpression; use Glpi\DBAL\QuerySubQuery; +use Glpi\DBAL\QueryUnion; use Mexitek\PHPColors\Color; class Toolbox @@ -460,17 +462,17 @@ public static function isLocationExistForZone(string $name): bool /** * Gets date intervals where data are missing in a table * To use with Mysql 8.0+ or MariaDB 10.2+ + * Gaps are expressed as intervals like [X; Y[ + * X is the 1st missing row, and Y is the first existing row which ends the gap * - * @see https://bertwagner.com/posts/gaps-and-islands/ - * - * @param string $table Table to search for gaps - * @param DateTimeInterface $start Start date to search - * @param DateInterval $interval Interval between each data sample (do not use intervals in months or years) - * @param DateTimeInterface|null $stop Stop date to search - * @param array $criteria Criterias for the SQL query - * @return array list of gaps + * @param string $table Table to search for gaps + * @param DateTimeInterface $start Start date to search + * @param DateInterval $interval Interval between each data sample (do not use intervals in months or years) + * @param DateTimeInterface|null $stop Stop date to search + * @param array $criteria Criterias for the SQL query + * @return DBmysqlIterator list of gaps */ - public static function findTemporalGapsInTable(string $table, DateTimeInterface $start, DateInterval $interval, ?DateTimeInterface $stop = null, array $criteria = []) + public static function findTemporalGapsInTable(string $table, DateTimeInterface $start, DateInterval $interval, ?DateTimeInterface $stop = null, array $criteria = []): DBmysqlIterator { /** @var DBmysql $DB */ global $DB; @@ -478,98 +480,88 @@ public static function findTemporalGapsInTable(string $table, DateTimeInterface if ($interval->m !== 0 || $interval->y !== 0) { throw new \InvalidArgumentException('Interval must be in days, hours, minutes or seconds'); } - // $interval_in_seconds = $interval->s + $interval->i * 60 + $interval->h * 3600 + $interval->d * 86400; + if ($stop === null) { + // Assume stop date is yesterday at midnight + $stop = new DateTime('yesterday midnight'); + } $sql_interval = self::dateIntervalToMySQLInterval($interval); + $start_string = $start->format('Y-m-d H:i:s'); + $stop_string = $stop->format('Y-m-d H:i:s'); // Get start date as unix timestamp - $boundaries[] = new QueryExpression('`date` >= "' . $start->format('Y-m-d H:i:s') . '"'); + $boundaries[] = new QueryExpression('`date` >= "' . $start_string . '"'); + $boundaries[] = new QueryExpression('`date` < "' . $stop_string . '"'); - // get stop date as unix timestamp - if ($stop === null) { - // Assume stop date is yesterday at midnight - $stop = new DateTime(); - $stop->setTime(0, 0, 0); - $stop->sub(new DateInterval('P1D')); - } - $boundaries[] = new QueryExpression('`date` <= "' . $stop->format('Y-m-d H:i:s') . '"'); + $common_criterias = array_merge($boundaries, $criteria); - // prepare sub query to get start and end date of an atomic date range - // An atomic date range is set to 1 hour - // To reduce problems with DST, we use the unix timestamp of the date - $atomic_ranges_subquery = new QuerySubQuery([ + $records_query = new QuerySubQuery([ 'SELECT' => [ - new QueryExpression('`date` as `start_date`'), - new QueryExpression("DATE_ADD(`date`, $sql_interval) as `end_date`"), + 'date', + new QueryExpression('LAG(`date`) OVER (ORDER BY `date`) AS `prev_date`') ], - 'FROM' => $table, - 'WHERE' => $criteria + $boundaries, - ], 'atomic_ranges'); - - // For each atomic date range, find the end date of previous atomic date range - $groups_subquery = new QuerySubQuery([ - 'SELECT' => [ - new QueryExpression('ROW_NUMBER() OVER (ORDER BY `start_date`, `end_date`) AS `row_number`'), - 'start_date', - 'end_date', - new QueryExpression('LAG(`end_date`, 1) OVER (ORDER BY `start_date`, `end_date`) AS `previous_end_date`') + 'FROM' => $table, + 'WHERE' => $common_criterias, + ], 'records'); + + $request = new QueryUnion([ + // Internal gaps (between existing records in the requred interval) + [ + 'SELECT' => [ + new QueryExpression('`prev_date` + ' . $sql_interval . ' AS `start`'), + 'date AS `end`' + ], + 'FROM' => $records_query, + 'WHERE' => array_merge($boundaries, [ + 'NOT' => ['prev_date' => null], + // new QueryExpression('TIMESTAMPDIFF(SECOND, `records`.`prev_date`, `records`.`date`) > ' . $interval_in_seconds) + new QueryExpression('DATE_ADD(`records`.`prev_date`, ' . $sql_interval . ') < `date`') + ]), ], - 'FROM' => $atomic_ranges_subquery - ], 'groups'); - // For each atomic date range, find if it is the start of an island - $islands_subquery = new QuerySubQuery([ - 'SELECT' => [ - '*', - new QueryExpression('SUM(CASE WHEN `groups`.`previous_end_date` >= `start_date` THEN 0 ELSE 1 END) OVER (ORDER BY `groups`.`row_number`) AS `ìsland_id`') + // Gap before the beginning of the serie + [ + 'SELECT' => [ + new queryExpression('\'' . $start_string . '\' AS start'), + new queryExpression('MIN(`date`) AS `end`') + ], + 'FROM' => $table, + 'WHERE' => $common_criterias, + 'HAVING' => [ + new QueryExpression("'" . $start_string . "' < MIN(`date`)") + ], ], - 'FROM' => $groups_subquery - ], 'islands'); - $request = [ - 'SELECT' => [ - 'MIN' => 'start_date as island_start_date', - 'MAX' => 'end_date as island_end_date', + // Gap after the end of the serie + [ + 'SELECT' => [ + new queryExpression('MAX(`date`) + ' . $sql_interval . ' AS `start`'), + new queryExpression('\'' . $stop_string . '\' AS `end`'), + ], + 'FROM' => $table, + 'WHERE' => $common_criterias, + 'HAVING' => [ + new QueryExpression("DATE_SUB('" . $stop_string . "', " . $sql_interval . ") > MAX(`date`)") + ], ], - 'FROM' => $islands_subquery, - 'GROUPBY' => ['ìsland_id'], - 'ORDER' => ['island_start_date'] - ]; - $result = $DB->request($request); - if ($result->count() === 0) { - // No island at all, the whole range is a gap - return [ - [ - 'start' => $start->format('Y-m-d H:i:s'), - 'end' => $stop->format('Y-m-d H:i:s'), - ] - ]; - } - - // Find gaps from islands - $expected_start_date = $start; - $gaps = []; - foreach ($result as $row) { - if ($expected_start_date < new DateTimeImmutable($row['island_start_date'])) { - // The current island starts after the expected start date - // Then there is a gap - $gaps[] = [ - 'start' => $expected_start_date->format('Y-m-d H:i:s'), - 'end' => $row['island_start_date'], - ]; - } - $expected_start_date = new DateTimeImmutable($row['island_end_date']); - } - if ($expected_start_date < $stop) { - // The last island ends before the stop date - // Then there is a gap - $gaps[] = [ - 'start' => $expected_start_date->format('Y-m-d H:i:s'), - 'end' => $stop->format('Y-m-d H:i:s'), - ]; - } - - return $gaps; + // No record between the boundaries + [ + 'SELECT' => [ + new QueryExpression('\'' . $start_string . '\' AS `start`'), + new QueryExpression('\'' . $stop_string . '\' AS `end`'), + ], + 'FROM' => $table, + 'WHERE' => $common_criterias, + 'HAVING' => [ + new QueryExpression('COUNT(*) = 0'), + ], + ] + ], true); + + return $DB->request([ + 'FROM' => $request, + 'ORDER' => 'start' + ]); } /** diff --git a/tests/units/CarbonEmissionTest.php b/tests/units/CarbonEmissionTest.php index feaf73c0..a348e127 100644 --- a/tests/units/CarbonEmissionTest.php +++ b/tests/units/CarbonEmissionTest.php @@ -84,7 +84,8 @@ public function testFindGaps($timezone) $start_date = DateTimeImmutable::createFromFormat('Y-m-d H:i:s', $start_date); $stop_date = DateTimeImmutable::createFromFormat('Y-m-d H:i:s', $stop_date); - $gaps = $instance->findGaps($itemtype, $asset->getID(), $start_date, $stop_date); + $result = $instance->findGaps($itemtype, $asset->getID(), $start_date, $stop_date); + $result = iterator_to_array($result); $expected = [ [ 'start' => '2019-01-01 00:00:00', @@ -95,7 +96,7 @@ public function testFindGaps($timezone) 'end' => '2023-12-31 00:00:00', ], ]; - $this->assertEquals($expected, $gaps); + $this->assertEquals($expected, $result); // Create a gap $gap_start = '2020-04-05 00:00:00'; @@ -106,7 +107,8 @@ public function testFindGaps($timezone) ['date' => ['<=', $gap_end]], ]); - $gaps = $instance->findGaps($itemtype, $asset->getID(), $start_date, $stop_date); + $result = $instance->findGaps($itemtype, $asset->getID(), $start_date, $stop_date); + $result = iterator_to_array($result); $expected = [ [ 'start' => '2019-01-01 00:00:00', @@ -121,7 +123,7 @@ public function testFindGaps($timezone) 'end' => '2023-12-31 00:00:00', ], ]; - $this->assertEquals($expected, $gaps); + $this->assertEquals($expected, $result); // Create an other gap $gap_start_2 = '2020-06-20 00:00:00'; @@ -131,7 +133,8 @@ public function testFindGaps($timezone) ['date' => ['>=', $gap_start_2]], ['date' => ['<=', $gap_end_2]], ]); - $gaps = $instance->findGaps($itemtype, $asset->getID(), $start_date, $stop_date); + $result = $instance->findGaps($itemtype, $asset->getID(), $start_date, $stop_date); + $result = iterator_to_array($result); $expected = [ [ 'start' => '2019-01-01 00:00:00', @@ -150,6 +153,6 @@ public function testFindGaps($timezone) 'end' => '2023-12-31 00:00:00', ], ]; - $this->assertEquals($expected, $gaps); + $this->assertEquals($expected, $result); } } diff --git a/tests/units/CarbonIntensityTest.php b/tests/units/CarbonIntensityTest.php index eaf4e8cd..b87120bd 100644 --- a/tests/units/CarbonIntensityTest.php +++ b/tests/units/CarbonIntensityTest.php @@ -173,8 +173,9 @@ public function testFindGaps() } $carbon_intensity = new CarbonIntensity(); - $output = $carbon_intensity->findGaps($source->getID(), $zone->getID(), $start_date, $end_date); - $this->assertEquals([], $output); + $result = $carbon_intensity->findGaps($source->getID(), $zone->getID(), $start_date, $end_date); + $result = iterator_to_array($result); + $this->assertEquals([], $result); // delete some samples at the beginning $delete_before_date = new DateTime('2024-01-03 12:00:00'); @@ -184,13 +185,14 @@ public function testFindGaps() 'date' => ['<', $delete_before_date->format('Y-m-d H:i:s')], ]); - $output = $carbon_intensity->findGaps($source->getID(), $zone->getID(), $start_date, $end_date); + $result = $carbon_intensity->findGaps($source->getID(), $zone->getID(), $start_date, $end_date); + $result = iterator_to_array($result); $this->assertEquals([ [ 'start' => $start_date->format('Y-m-d H:i:s'), 'end' => $delete_before_date->format('Y-m-d H:i:s'), ], - ], $output); + ], $result); // delete some samples at the end $delete_after_date = new DateTime('2024-02-17 09:00:00'); @@ -199,7 +201,8 @@ public function testFindGaps() 'plugin_carbon_zones_id' => $zone->getID(), 'date' => ['>=', $delete_after_date->format('Y-m-d H:i:s')], ]); - $output = $carbon_intensity->findGaps($source->getID(), $zone->getID(), $start_date, $end_date); + $result = $carbon_intensity->findGaps($source->getID(), $zone->getID(), $start_date, $end_date); + $result = iterator_to_array($result); $this->assertEquals([ [ 'start' => $start_date->format('Y-m-d H:i:s'), @@ -209,7 +212,7 @@ public function testFindGaps() 'start' => $delete_after_date->format('Y-m-d H:i:s'), 'end' => $end_date->format('Y-m-d H:i:s'), ], - ], $output); + ], $result); // delete some samples in the middle $delete_middle_start_date = new DateTime('2024-01-29 06:00:00'); @@ -222,7 +225,8 @@ public function testFindGaps() ['date' => ['<', $delete_middle_end_date->format('Y-m-d H:i:s')]], ] ]); - $output = $carbon_intensity->findGaps($source->getID(), $zone->getID(), $start_date, $end_date); + $result = $carbon_intensity->findGaps($source->getID(), $zone->getID(), $start_date, $end_date); + $result = iterator_to_array($result); $this->assertEquals([ [ 'start' => $start_date->format('Y-m-d H:i:s'), @@ -236,7 +240,7 @@ public function testFindGaps() 'start' => $delete_after_date->format('Y-m-d H:i:s'), 'end' => $end_date->format('Y-m-d H:i:s'), ], - ], $output); + ], $result); // restore the deleted samples at the beginning $cursor_date = clone $start_date; @@ -250,7 +254,8 @@ public function testFindGaps() $cursor_date->modify('+1 hour'); } - $output = $carbon_intensity->findGaps($source->getID(), $zone->getID(), $start_date, $end_date); + $result = $carbon_intensity->findGaps($source->getID(), $zone->getID(), $start_date, $end_date); + $result = iterator_to_array($result); $this->assertEquals([ [ 'start' => $delete_middle_start_date->format('Y-m-d H:i:s'), @@ -260,7 +265,7 @@ public function testFindGaps() 'start' => $delete_after_date->format('Y-m-d H:i:s'), 'end' => $end_date->format('Y-m-d H:i:s'), ], - ], $output); + ], $result); // restore the deleted samples at the middle $cursor_date = clone $delete_middle_start_date; @@ -274,13 +279,14 @@ public function testFindGaps() $cursor_date->modify('+1 hour'); } - $output = $carbon_intensity->findGaps($source->getID(), $zone->getID(), $start_date, $end_date); + $result = $carbon_intensity->findGaps($source->getID(), $zone->getID(), $start_date, $end_date); + $result = iterator_to_array($result); $this->assertEquals([ [ 'start' => $delete_after_date->format('Y-m-d H:i:s'), 'end' => $end_date->format('Y-m-d H:i:s'), ], - ], $output); + ], $result); } public function testGetDownloadStartDate() diff --git a/tests/units/ToolboxTest.php b/tests/units/ToolboxTest.php index 92d91491..ca19e9a9 100644 --- a/tests/units/ToolboxTest.php +++ b/tests/units/ToolboxTest.php @@ -36,6 +36,9 @@ use DateInterval; use DateTime; use DateTimeImmutable; +use GlpiPlugin\Carbon\CarbonIntensity; +use GlpiPlugin\Carbon\Source; +use GlpiPlugin\Carbon\Source_Zone; use GlpiPlugin\Carbon\Zone; use GlpiPlugin\Carbon\Tests\DbTestCase; use GlpiPlugin\Carbon\Toolbox; @@ -258,4 +261,159 @@ public function testDateIntervalToMySQLInterval() $result = Toolbox::dateIntervalToMySQLInterval($interval); $this->assertEquals('INTERVAL 3 YEAR + INTERVAL 40 MINUTE', $result); } + + + /** + * Undocumented function + * + * @return void + */ + public function testFindTemporalGapsInTable() + { + $table = getTableForItemType(CarbonIntensity::class); + $source = $this->createItem(Source::class); + $zone = $this->createItem(Zone::class); + $criterias = [ + getForeignKeyFieldForItemType(Source::class) => $source->getID(), + getForeignKeyFieldForItemType(Zone::class) => $zone->getID(), + ]; + $source_zone = $this->createItem(Source_Zone::class, $criterias); + + // Test when no record exists in the requested interval + $result = Toolbox::findTemporalGapsInTable( + $table, + new DateTime('2020-01-01 00:00:00'), + new DateInterval('PT1H'), + new DateTime('2020-06-01 00:00:00'), + $criterias + ); + $result = iterator_to_array($result); + $expected = [ + [ + 'start' => '2020-01-01 00:00:00', + 'end' => '2020-06-01 00:00:00' + ] + ]; + $this->assertEquals($expected, $result); + + // Test when there is a record matching the beginning of the interval, + // but none matching the end + $this->createItem(CarbonIntensity::class, [ + 'date' => '2020-01-01 00:00:00' + ] + $criterias); + $result = Toolbox::findTemporalGapsInTable( + $table, + new DateTime('2020-01-01 00:00:00'), + new DateInterval('PT1H'), + new DateTime('2020-06-01 00:00:00'), + $criterias + ); + $result = iterator_to_array($result); + $expected = [ + [ + 'start' => '2020-01-01 01:00:00', + 'end' => '2020-06-01 00:00:00' + ] + ]; + $this->assertEquals($expected, $result); + + // Test when there is a record matching the end of the interval, + // but none matching the beginning + $result = Toolbox::findTemporalGapsInTable( + $table, + new DateTime('2019-01-01 00:00:00'), + new DateInterval('PT1H'), + new DateTime('2020-01-01 00:00:00'), + $criterias + ); + $result = iterator_to_array($result); + $expected = [ + [ + 'start' => '2019-01-01 00:00:00', + 'end' => '2020-01-01 00:00:00' + ] + ]; + $this->assertEquals($expected, $result); + + // Test when there is a record in the requesterd interval + $result = Toolbox::findTemporalGapsInTable( + $table, + new DateTime('2019-01-01 00:00:00'), + new DateInterval('PT1H'), + new DateTime('2021-01-01 00:00:00'), + $criterias + ); + $result = iterator_to_array($result); + $expected = [ + [ + 'start' => '2019-01-01 00:00:00', + 'end' => '2020-01-01 00:00:00' + ], + [ + 'start' => '2020-01-01 01:00:00', + 'end' => '2021-01-01 00:00:00' + ], + ]; + $this->assertEquals($expected, $result); + + // Test when there is are 2 non consecutive records in the requesterd interval + $this->createItem(CarbonIntensity::class, [ + 'date' => '2020-06-01 00:00:00' + ] + $criterias); + $result = Toolbox::findTemporalGapsInTable( + $table, + new DateTime('2019-01-01 00:00:00'), + new DateInterval('PT1H'), + new DateTime('2021-01-01 00:00:00'), + $criterias + ); + $result = iterator_to_array($result); + $expected = [ + [ + 'start' => '2019-01-01 00:00:00', + 'end' => '2020-01-01 00:00:00' + ], + [ + 'start' => '2020-01-01 01:00:00', + 'end' => '2020-06-01 00:00:00' + ], + [ + 'start' => '2020-06-01 01:00:00', + 'end' => '2021-01-01 00:00:00' + ], + ]; + $this->assertEquals($expected, $result); + + // Test when there is are 2 consecutive records + // in 2 separated groups in the requesterd interval + $this->createItem(CarbonIntensity::class, [ + 'date' => '2020-01-01 01:00:00' + ] + $criterias); + $this->createItem(CarbonIntensity::class, [ + 'date' => '2020-06-01 01:00:00' + ] + $criterias); + $result = Toolbox::findTemporalGapsInTable( + $table, + new DateTime('2019-01-01 00:00:00'), + new DateInterval('PT1H'), + new DateTime('2021-01-01 00:00:00'), + $criterias + ); + $result = iterator_to_array($result); + $expected = [ + [ + 'start' => '2019-01-01 00:00:00', + 'end' => '2020-01-01 00:00:00' + ], + [ + 'start' => '2020-01-01 02:00:00', + 'end' => '2020-06-01 00:00:00' + ], + [ + 'start' => '2020-06-01 02:00:00', + 'end' => '2021-01-01 00:00:00' + ], + ]; + $this->assertEquals($expected, $result); + } }