Merge pull request #2706 from pi-hole/new/earliest_timestamp

Get earliest query timestamp from database

Author: DL6ER

src/api/docs/content/specs/info.yaml

@@ -994,8 +994,20 @@ components:
                   example: "pihole"
         queries:
           type: integer
-          description: Number of queries in long-term database
+          description: Number of queries in in-memory database
           example: 536956
+        earliest_timestamp:
+          type: number
+          description: Unix timestamp of the earliest query in the in-memory database (Defaults to 0.0 if no queries are stored in memory)
+          example: 1611742120.5
+        queries_disk:
+          type: integer
+          description: Number of queries in on-disk database
+          example: 1234567
+        earliest_timestamp_disk:
+          type: number
+          description: Unix timestamp of the earliest query in the on-disk database (Defaults to 0.0 if no queries are stored on disk)
+          example: 1608450520.3
         sqlite_version:
           type: string
           description: Version of embedded SQLite3 engine

src/api/info.c

@@ -146,9 +146,17 @@ int api_info_database(struct ftl_conn *api)
 	JSON_ADD_ITEM_TO_OBJECT(owner, "group", group);
 	JSON_ADD_ITEM_TO_OBJECT(json, "owner", owner);
 
-	// Add number of queries in on-disk database
-	const int queries_in_database = get_number_of_queries_in_DB(NULL, "query_storage");
+	// Add number of queries and earliest timestamp in in-memory database
+	double earliest_timestamp_mem = 0.0;
+	const int queries_in_database = get_number_of_queries_in_DB(NULL, "query_storage", &earliest_timestamp_mem);
 	JSON_ADD_NUMBER_TO_OBJECT(json, "queries", queries_in_database);
+	JSON_ADD_NUMBER_TO_OBJECT(json, "earliest_timestamp", earliest_timestamp_mem);
+
+	// Add number of queries and earliest timestamp in on-disk database
+	double earliest_timestamp_disk = 0.0;
+	const int queries_in_disk_database = get_number_of_queries_in_DB(NULL, "disk.query_storage", &earliest_timestamp_disk);
+	JSON_ADD_NUMBER_TO_OBJECT(json, "queries_disk", queries_in_disk_database);
+	JSON_ADD_NUMBER_TO_OBJECT(json, "earliest_timestamp_disk", earliest_timestamp_disk);
 
 	// Add SQLite library version
 	JSON_REF_STR_IN_OBJECT(json, "sqlite_version", get_sqlite3_version());

src/database/query-table.c

@@ -368,7 +368,7 @@ static bool get_memdb_size(sqlite3 *db, size_t *memsize, int *queries)
 	*memsize = page_count * page_size;
 
 	// Get number of queries in the memory table
-	if((*queries = get_number_of_queries_in_DB(db, "query_storage")) == DB_FAILED)
+	if((*queries = get_number_of_queries_in_DB(db, "query_storage", NULL)) == DB_FAILED)
 		return false;
 
 	return true;
@@ -506,21 +506,29 @@ bool detach_database(sqlite3* db, const char **message, const char *alias)
 
 // Get number of queries either in the temp or in the on-diks database
 // This routine is used by the API routines.
-int get_number_of_queries_in_DB(sqlite3 *db, const char *tablename)
+int get_number_of_queries_in_DB(sqlite3 *db, const char *tablename, double *earliest_timestamp)
 {
 	int rc = 0, num = 0;
 	sqlite3_stmt *stmt = NULL;
 
-	// Count number of rows
-	const size_t buflen = 42 + strlen(tablename);
-	char *querystr = calloc(buflen, sizeof(char));
-	snprintf(querystr, buflen, "SELECT COUNT(*) FROM %s", tablename);
-
 	// The database pointer may be NULL, meaning we want the memdb
 	if(db == NULL)
 		db = get_memdb();
 
-	// PRAGMA page_size
+	// Build query string based on whether we need the earliest timestamp too
+	const size_t buflen = 37 + strlen(tablename);
+	char *querystr = calloc(buflen, sizeof(char));
+	if(earliest_timestamp != NULL)
+	{
+		// Get both count and earliest timestamp
+		snprintf(querystr, buflen, "SELECT COUNT(*), MIN(timestamp) FROM %s", tablename);
+	}
+	else
+	{
+		// Get only count
+		snprintf(querystr, buflen, "SELECT COUNT(*) FROM %s", tablename);
+	}
+
 	rc = sqlite3_prepare_v2(db, querystr, -1, &stmt, NULL);
 	if( rc != SQLITE_OK )
 	{
@@ -532,7 +540,13 @@ int get_number_of_queries_in_DB(sqlite3 *db, const char *tablename)
 	}
 	rc = sqlite3_step(stmt);
 	if( rc == SQLITE_ROW )
+	{
+		// Get count from first column
 		num = sqlite3_column_int(stmt, 0);
+		// Get timestamp from second column if requested
+		if(earliest_timestamp != NULL && sqlite3_column_type(stmt, 1) != SQLITE_NULL)
+			*earliest_timestamp = sqlite3_column_double(stmt, 1);
+	}
 	sqlite3_finalize(stmt);
 	free(querystr);
 
@@ -617,8 +631,8 @@ bool import_queries_from_disk(void)
 	}
 
 	// Get number of queries on disk before detaching
-	disk_db_num = get_number_of_queries_in_DB(memdb, "disk.query_storage");
-	mem_db_num = get_number_of_queries_in_DB(memdb, "query_storage");
+	disk_db_num = get_number_of_queries_in_DB(memdb, "disk.query_storage", NULL);
+	mem_db_num = get_number_of_queries_in_DB(memdb, "query_storage", NULL);
 
 	log_info("Imported %u queries from the on-disk database (it has %u rows)", mem_db_num, disk_db_num);
 
@@ -710,7 +724,7 @@ bool export_queries_to_disk(const bool final)
 		}
 
 		// Update number of queries in the disk database
-		disk_db_num = get_number_of_queries_in_DB(memdb, "disk.query_storage");
+		disk_db_num = get_number_of_queries_in_DB(memdb, "disk.query_storage", NULL);
 	}
 
 	// Export linking tables and current AUTOINCREMENT values to the disk database
@@ -778,7 +792,7 @@ bool delete_old_queries_from_db(const bool use_memdb, const double mintime)
 		        mintime, sqlite3_errstr(rc));
 
 	// Update number of queries in in-memory database
-	const int new_num = get_number_of_queries_in_DB(NULL, "query_storage");
+	const int new_num = get_number_of_queries_in_DB(NULL, "query_storage", NULL);
 	log_debug(DEBUG_GC, "delete_old_queries_from_db(): Deleted %i (%u) queries, new number of queries in memory: %i",
 	          sqlite3_changes(db), (mem_db_num - new_num), new_num);
 	mem_db_num = new_num;
@@ -1738,7 +1752,7 @@ bool queries_to_database(void)
 	}
 
 	// Update number of queries in in-memory database
-	mem_db_num = get_number_of_queries_in_DB(NULL, "query_storage");
+	mem_db_num = get_number_of_queries_in_DB(NULL, "query_storage", NULL);
 
 	if(config.debug.database.v.b && updated + added > 0)
 	{

src/database/query-table.h

@@ -113,7 +113,7 @@ void close_memory_database(void);
 bool import_queries_from_disk(void);
 bool attach_database(sqlite3* db, const char **message, const char *path, const char *alias);
 bool detach_database(sqlite3* db, const char **message, const char *alias);
-int get_number_of_queries_in_DB(sqlite3 *db, const char *tablename);
+int get_number_of_queries_in_DB(sqlite3 *db, const char *tablename, double *earliest_timestamp);
 bool export_queries_to_disk(const bool final);
 bool delete_old_queries_from_db(const bool use_memdb, const double mintime);
 bool add_additional_info_column(sqlite3 *db);