introduce data aggregation routine

We now aggregate statistics data in the cleanup routine preserving
only distinct entries for each date/referrer/target combination with
the sum of all hits.

Author: stklcode

inc/class-statify-cron.php

@@ -38,9 +38,75 @@ public static function cleanup_data() {
 			)
 		);
 
+		// Aggregate.
+		self::aggregate_data();
+
 		// Optimize DB.
 		$wpdb->query(
 			"OPTIMIZE TABLE `$wpdb->statify`"
 		);
 	}
+
+	/**
+	 * Aggregate data in database.
+	 *
+	 * @since 1.9
+	 */
+	public static function aggregate_data() {
+		global $wpdb;
+
+		// Get date of last aggregation.
+		if ( isset( self::$_options['last_aggregation'] ) ) {
+			// Value saved, use it.
+			$start = self::$_options['last_aggregation'];
+		} else {
+			// No? We need to clean up all data. Let's determine the oldest data in the database.
+			$start = $wpdb->get_col( "SELECT MIN(`created`) FROM `$wpdb->statify`" );
+			$start = $start[0];
+		}
+
+		if ( is_null( $start ) ) {
+			// No data available, i.e not cleaned up yet and no data in database.
+			return;
+		}
+
+		$now  = new DateTime();
+		$date = new DateTime( $start );
+
+		// Iterate over every day from start (inclusive) til now.
+		while ( $date < $now ) {
+			$agg = $wpdb->get_results(
+				$wpdb->prepare(
+					"SELECT `created`, `referrer`, `target`, SUM(`hits`) as `hits` FROM `$wpdb->statify` WHERE `created` = %s GROUP BY `created`, `referrer`, `target`",
+					$date->format( 'Y-m-d' )
+				),
+				ARRAY_A
+			);
+
+			// Remove non-aggregated data and insert aggregates within one transaction.
+			$wpdb->query( 'START TRANSACTION' );
+			$res = $wpdb->query(
+				$wpdb->prepare(
+					"DELETE FROM `$wpdb->statify` WHERE `created` = %s",
+					$date->format( 'Y-m-d' )
+				)
+			);
+			if ( false !== $res ) {
+				foreach ( $agg as $a ) {
+					if ( false === $wpdb->insert( $wpdb->statify, $a ) ) {
+						$wpdb->query( 'ROLLBACK' );
+						break;
+					}
+				}
+			}
+			$wpdb->query( 'COMMIT' );
+
+			// Continue with next day.
+			$date->modify( '+1 day' );
+		}
+
+		// Remember last aggregation date.
+		self::$_options['last_aggregation'] = $now->format( 'Y-m-d' );
+		update_option( 'statify', self::$_options );
+	}
 }

tests/test-cron.php

@@ -73,4 +73,51 @@ public function test_cronjob() {
 			$this->assertEquals( 2, $v['count'], 'Unexpected visit count' );
 		}
 	}
+
+	/**
+	 * Test Statify Cron Job execution.
+	 *
+	 * @runInSeparateProcess Must not preserve global constant.
+	 * @preserveGlobalState disabled
+	 */
+	public function test_aggregation() {
+		global $wpdb;
+
+		// Insert test data: 2 days with 3 and 4 distinct combinations of referrer and target.
+		$date  = new DateTime();
+		$this->insert_test_data( $date->format( 'Y-m-d' ), '', '', 2 );
+		$this->insert_test_data( $date->format( 'Y-m-d' ), 'https://statify.pluginkollektiv.org/', '/', 3 );
+		$this->insert_test_data( $date->format( 'Y-m-d' ), 'https://statify.pluginkollektiv.org/', '/test/', 4 );
+		$this->insert_test_data( $date->format( 'Y-m-d' ), 'https://pluginkollektiv.org/', '/', 5 );
+		$date->modify( '-1 days' );
+		$this->insert_test_data( $date->format( 'Y-m-d' ), 'https://statify.pluginkollektiv.org/', '/', 4 );
+		$this->insert_test_data( $date->format( 'Y-m-d' ), 'https://statify.pluginkollektiv.org/', '/test/', 3 );
+		$this->insert_test_data( $date->format( 'Y-m-d' ), 'https://pluginkollektiv.org/', '/', 2 );
+
+		// Get baseline.
+		$this->assertEquals( 23, $wpdb->get_var( "SELECT COUNT(*) FROM `$wpdb->statify`" ), 'Unexpected number of entries before aggregation' );
+		$stats = $this->get_stats();
+
+		// Trigger aggregation.
+		Statify_Cron::aggregate_data();
+
+		// Verify results.
+		$this->assertEquals( 7, $wpdb->get_var( "SELECT COUNT(*) FROM `$wpdb->statify`" ), 'Unexpected number of entries after aggregation' );
+		$stats2 = $this->get_stats();
+		$this->assertEquals( $stats, $stats2, 'Statistics data should be the same after aggregation' );
+		// Check one single row explicitly.
+		$this->assertEquals(
+			3,
+			$wpdb->get_var(
+				$wpdb->prepare(
+					"SELECT hits FROM `$wpdb->statify` WHERE created = %s AND referrer = %s AND target = %s",
+					$date->format( 'Y-m-d' ),
+					'https://statify.pluginkollektiv.org/',
+					'/test/'
+				)
+			),
+			'Unexpected hit count after aggregation'
+		);
+
+	}
 }

tests/test-tracking.php

@@ -361,10 +361,11 @@ function ( $data, $id ) use ( &$capture ) {
 		$this->assertNotNull( $stats['visits'][0]['count'], 'Request not tracked' );
 		$this->assertNotEmpty( $capture, 'Hook stativy__visit_saved has not fired' );
 		$this->assertTrue( is_numeric( $capture['id'] ) && $capture['id'] > 0, 'unexpected entry ID' );
-		$this->assertCount( 3, $capture['data'], 'unexpected number of data fields' );
+		$this->assertCount( 4, $capture['data'], 'unexpected number of data fields' );
 		$this->assertEquals( ( new DateTime() )->format( 'Y-m-d' ), $capture['data']['created'], 'unexpected creation date' );
 		$this->assertEquals( 'https://statify.pluginkollektiv.org/', $capture['data']['referrer'], 'unexpected referrer' );
 		$this->assertEquals( '/page', $capture['data']['target'], 'unexpected target' );
+		$this->assertEquals( 1, $capture['data']['hits'], 'unexpected hits' );
 	}
 
 	/**