Improve and document test suites

dutow · dutow · commit 5e68f838f531 · 2023-07-10T08:10:20.000+02:00
This commit improves the 007 prove test: PG oredering is either case sensitive or case insensitive depending on system settings.
The testcase uses ordering based on the query string, with some uppercase and some lowercase queries, causing resulting in different outputs on different systems.
This change modifies the ordering to use lower(query) to ensure consistent results everywhere.

It also adds a TESTING markdown file describing how to run the test suite when building pg ang pgsm from source.
diff --git a/.github/workflows/postgresql-11-build.yml b/.github/workflows/postgresql-11-build.yml
@@ -22,7 +22,8 @@ jobs:
             libipc-run-perl -y docbook-xsl docbook-xsl libxml2 libxml2-utils \
             libxml2-dev libxslt-dev xsltproc libkrb5-dev libldap2-dev \
             libsystemd-dev gettext tcl-dev libperl-dev pkg-config clang-11 \
-            llvm-11 llvm-11-dev libselinux1-dev python3-dev uuid-dev liblz4-dev
+            llvm-11 llvm-11-dev libselinux1-dev python3-dev uuid-dev liblz4-dev \
+            libtext-diff-perl
           sudo rm -rf /var/lib/postgresql /var/log/postgresql /etc/postgresql \
             /usr/lib/postgresql /usr/include/postgresql /usr/share/postgresql \
             /etc/postgresql
diff --git a/.github/workflows/postgresql-11-ppg-package.yml b/.github/workflows/postgresql-11-ppg-package.yml
@@ -18,7 +18,7 @@ jobs:
             postgresql postgresql*
           sudo apt-get install -y libreadline6-dev systemtap-sdt-dev \
             zlib1g-dev libssl-dev libpam0g-dev python3-dev bison flex \
-            libipc-run-perl wget
+            libipc-run-perl wget libtext-diff-perl
           sudo rm -rf /var/lib/postgresql /var/log/postgresql /etc/postgresql \
             /usr/lib/postgresql /usr/include/postgresql /usr/share/postgresql \
             /etc/postgresql
diff --git a/.github/workflows/postgresql-12-build.yml b/.github/workflows/postgresql-12-build.yml
@@ -22,7 +22,8 @@ jobs:
            libipc-run-perl docbook-xsl docbook-xsl libxml2 libxml2-utils \
            libxml2-dev libxslt-dev xsltproc libkrb5-dev libldap2-dev \
            libsystemd-dev gettext tcl-dev libperl-dev pkg-config clang-11 \
-           llvm-11 llvm-11-dev libselinux1-dev python3-dev uuid-dev liblz4-dev
+           llvm-11 llvm-11-dev libselinux1-dev python3-dev uuid-dev liblz4-dev \
+           libtext-diff-perl
           sudo rm -rf /var/lib/postgresql /var/log/postgresql /etc/postgresql \
            /usr/lib/postgresql /usr/include/postgresql /usr/share/postgresql \
            /etc/postgresql
diff --git a/.github/workflows/postgresql-12-pgdg-package.yml b/.github/workflows/postgresql-12-pgdg-package.yml
@@ -18,7 +18,7 @@ jobs:
             postgresql postgresql*
           sudo apt-get install -y libreadline6-dev systemtap-sdt-dev \
             zlib1g-dev libssl-dev libpam0g-dev python3-dev bison flex \
-            libipc-run-perl wget -y
+            libipc-run-perl wget -y libtext-diff-perl
           sudo rm -rf /var/lib/postgresql /var/log/postgresql /etc/postgresql \
             /usr/lib/postgresql /usr/include/postgresql /usr/share/postgresql \
             /etc/postgresql
diff --git a/.github/workflows/postgresql-12-ppg-package.yml b/.github/workflows/postgresql-12-ppg-package.yml
@@ -18,10 +18,7 @@ jobs:
             postgresql postgresql*
           sudo apt-get install -y libreadline6-dev systemtap-sdt-dev \
             zlib1g-dev libssl-dev libpam0g-dev python3-dev bison flex \
-            libipc-run-perl wget
-          sudo apt-get install -y libreadline6-dev systemtap-sdt-dev \
-            zlib1g-dev libssl-dev libpam0g-dev python3-dev bison flex \
-            libipc-run-perl wget
+            libipc-run-perl wget libtext-diff-perl
           sudo rm -rf /var/lib/postgresql /var/log/postgresql /etc/postgresql \
             /usr/lib/postgresql /usr/include/postgresql /usr/share/postgresql \
             /etc/postgresql
diff --git a/.github/workflows/postgresql-13-build.yml b/.github/workflows/postgresql-13-build.yml
@@ -22,7 +22,8 @@ jobs:
             libipc-run-perl -y docbook-xsl docbook-xsl libxml2 libxml2-utils \
             libxml2-dev libxslt-dev xsltproc libkrb5-dev libldap2-dev \
             libsystemd-dev gettext tcl-dev libperl-dev pkg-config clang-11 \
-            llvm-11 llvm-11-dev libselinux1-dev python3-dev uuid-dev liblz4-dev
+            llvm-11 llvm-11-dev libselinux1-dev python3-dev uuid-dev liblz4-dev \
+            libtext-diff-perl
           sudo rm -rf /var/lib/postgresql /var/log/postgresql /etc/postgresql \
            /usr/lib/postgresql /usr/include/postgresql /usr/share/postgresql \
            /etc/postgresql
diff --git a/.github/workflows/postgresql-13-pgdg-package.yml b/.github/workflows/postgresql-13-pgdg-package.yml
@@ -18,7 +18,7 @@ jobs:
             postgresql postgresql*
           sudo apt-get install -y libreadline6-dev systemtap-sdt-dev \
             zlib1g-dev libssl-dev libpam0g-dev python3-dev bison flex \
-            libipc-run-perl wget -y
+            libipc-run-perl wget -y libtext-diff-perl
           sudo rm -rf /var/lib/postgresql /var/log/postgresql /etc/postgresql \
             /usr/lib/postgresql /usr/include/postgresql /usr/share/postgresql \
             /etc/postgresql
diff --git a/.github/workflows/postgresql-13-ppg-package.yml b/.github/workflows/postgresql-13-ppg-package.yml
@@ -18,7 +18,7 @@ jobs:
             postgresql postgresql*
           sudo apt-get install -y libreadline6-dev systemtap-sdt-dev \
             zlib1g-dev libssl-dev libpam0g-dev python3-dev bison flex \
-            libipc-run-perl wget
+            libipc-run-perl wget libtext-diff-perl
           sudo rm -rf /var/lib/postgresql /var/log/postgresql /etc/postgresql \
             /usr/lib/postgresql /usr/include/postgresql /usr/share/postgresql \
             /etc/postgresql
diff --git a/.github/workflows/postgresql-14-build.yml b/.github/workflows/postgresql-14-build.yml
@@ -23,7 +23,7 @@ jobs:
             libxml2-dev libxslt-dev xsltproc libkrb5-dev libldap2-dev \
             libsystemd-dev gettext tcl-dev libperl-dev pkg-config clang-11 \
             llvm-11 llvm-11-dev libselinux1-dev python3-dev \
-            uuid-dev liblz4-dev
+            uuid-dev liblz4-dev libtext-diff-perl
           sudo rm -rf /var/lib/postgresql /var/log/postgresql /etc/postgresql \
            /usr/lib/postgresql /usr/include/postgresql /usr/share/postgresql \
            /etc/postgresql
diff --git a/.github/workflows/postgresql-14-pgdg-package.yml b/.github/workflows/postgresql-14-pgdg-package.yml
@@ -17,7 +17,8 @@ jobs:
           sudo apt purge postgresql-client-common postgresql-common \
             postgresql postgresql*
           sudo apt-get install -y libreadline6-dev systemtap-sdt-dev wget \
-            zlib1g-dev libssl-dev libpam0g-dev bison flex libipc-run-perl
+            zlib1g-dev libssl-dev libpam0g-dev bison flex libipc-run-perl \
+            libtext-diff-perl
           sudo rm -rf /var/lib/postgresql /var/log/postgresql /etc/postgresql \
             /usr/lib/postgresql /usr/include/postgresql /usr/share/postgresql \
             /etc/postgresql
diff --git a/.github/workflows/postgresql-14-ppg-package.yml b/.github/workflows/postgresql-14-ppg-package.yml
@@ -18,7 +18,7 @@ jobs:
             postgresql postgresql*
           sudo apt-get install -y libreadline6-dev systemtap-sdt-dev \
             zlib1g-dev libssl-dev libpam0g-dev python3-dev bison flex \
-            libipc-run-perl wget 
+            libipc-run-perl wget libtext-diff-perl
           sudo rm -rf /var/lib/postgresql /var/log/postgresql /etc/postgresql \
             /usr/lib/postgresql /usr/include/postgresql /usr/share/postgresql \
             /etc/postgresql
diff --git a/.github/workflows/postgresql-15-build.yml b/.github/workflows/postgresql-15-build.yml
@@ -23,7 +23,7 @@ jobs:
             libxml2-dev libxslt-dev xsltproc libkrb5-dev libldap2-dev \
             libsystemd-dev gettext tcl-dev libperl-dev pkg-config clang-11 \
             llvm-11 llvm-11-dev libselinux1-dev python3-dev \
-            uuid-dev liblz4-dev
+            uuid-dev liblz4-dev libtext-diff-perl
           sudo rm -rf /var/lib/postgresql /var/log/postgresql /etc/postgresql \
            /usr/lib/postgresql /usr/include/postgresql /usr/share/postgresql \
            /etc/postgresql
diff --git a/.github/workflows/postgresql-15-pgdg-package.yml b/.github/workflows/postgresql-15-pgdg-package.yml
@@ -17,7 +17,8 @@ jobs:
           sudo apt purge postgresql-client-common postgresql-common \
             postgresql postgresql*
           sudo apt-get install -y libreadline6-dev systemtap-sdt-dev wget \
-            zlib1g-dev libssl-dev libpam0g-dev bison flex libipc-run-perl
+            zlib1g-dev libssl-dev libpam0g-dev bison flex libipc-run-perl \
+            libtext-diff-perl
           sudo rm -rf /var/lib/postgresql /var/log/postgresql /etc/postgresql \
             /usr/lib/postgresql /usr/include/postgresql /usr/share/postgresql \
             /etc/postgresql
diff --git a/.github/workflows/postgresql-15-ppg-package.yml b/.github/workflows/postgresql-15-ppg-package.yml
@@ -18,7 +18,7 @@ jobs:
             postgresql postgresql*
           sudo apt-get install -y libreadline6-dev systemtap-sdt-dev \
             zlib1g-dev libssl-dev libpam0g-dev python3-dev bison flex \
-            libipc-run-perl wget 
+            libipc-run-perl wget libtext-diff-perl
           sudo rm -rf /var/lib/postgresql /var/log/postgresql /etc/postgresql \
             /usr/lib/postgresql /usr/include/postgresql /usr/share/postgresql \
             /etc/postgresql
diff --git a/TESTING.md b/TESTING.md
@@ -0,0 +1,31 @@
+# How to run the tests
+
+This document describes how to run the tests suites when building pg & pgsm from source.
+
+The steps can also be adapted when postgres is installed using a package manager, in that case the relevant directories will be in their usual global locations (`/usr/lib/postgresql`)
+
+## pg_regress
+
+1. setup and start a postgres cluster
+2. create a database named "contrib\_regression"
+3. Install the pg\_stat\_statements extension
+4. Run the tests using (with the working directory as the pgsm git repository):
+
+   ```
+   <PG_BUILD_DIR>/src/test/regress/pg_regress --bindir <INST_DIR>/bin --temp-config pg_stat_monitor.conf --inputdir regression/ --dbname=contrib_regression basic version guc pgsm_query_id functions counters relations database error_insert application_name application_name_unique top_query cmd_type error rows tags user
+   ```
+
+# prove
+
+1. setup and start postgres cluster
+2. create a database named "contrib\_regression"
+3. Install the pg\_stat\_monitor and pg\_stat\_statements extensions
+4. When executing the tests multiple times, delete the `tmp_check` directory between runs
+5. When executing the tests and encountering failures, databases created by some tests might remain in the cluster.
+   Connect to the cluster, and delete these databases if they cause errors.
+6. Run the tests using (with the working directory as the pgsm git repository):
+
+   ```
+   prove -I <PG_SRC_DIR>/src/src/test/perl/ -I . t/*.pl
+   ```
+
diff --git a/t/007_settings_pgsm_query_shared_buffer.pl b/t/007_settings_pgsm_query_shared_buffer.pl
@@ -59,7 +59,7 @@
 print " out: $out \n";
 ok($cmdret == 0, "Run pgbench");
 
-($cmdret, $stdout, $stderr) = $node->psql('postgres', 'SELECT datname, substr(query,0,150) AS query, SUM(calls) AS calls FROM pg_stat_monitor GROUP BY datname, query ORDER BY datname, query, calls DESC Limit 20;', extra_params => ['-a', '-Pformat=aligned','-Ptuples_only=off']);
+($cmdret, $stdout, $stderr) = $node->psql('postgres', 'SELECT datname, substr(query,0,150) AS query, SUM(calls) AS calls FROM pg_stat_monitor GROUP BY datname, query ORDER BY datname, lower(query), calls DESC Limit 20;', extra_params => ['-a', '-Pformat=aligned','-Ptuples_only=off']);
 ok($cmdret == 0, "SELECT XXX FROM pg_stat_monitor");
 PGSM::append_to_file($stdout);
 
@@ -82,7 +82,7 @@
 print " out: $out \n";
 ok($cmdret == 0, "Run pgbench");
 
-($cmdret, $stdout, $stderr) = $node->psql('postgres', 'SELECT datname, substr(query,0,150) AS query, SUM(calls) AS calls FROM pg_stat_monitor GROUP BY datname, query ORDER BY datname, query, calls DESC Limit 20;', extra_params => ['-a', '-Pformat=aligned','-Ptuples_only=off']);
+($cmdret, $stdout, $stderr) = $node->psql('postgres', 'SELECT datname, substr(query,0,150) AS query, SUM(calls) AS calls FROM pg_stat_monitor GROUP BY datname, query ORDER BY datname, lower(query), calls DESC Limit 20;', extra_params => ['-a', '-Pformat=aligned','-Ptuples_only=off']);
 ok($cmdret == 0, "SELECT XXX FROM pg_stat_monitor");
 PGSM::append_to_file($stdout);
 
@@ -105,7 +105,7 @@
 print " out: $out \n";
 ok($cmdret == 0, "Run pgbench");
 
-($cmdret, $stdout, $stderr) = $node->psql('postgres', 'SELECT datname, substr(query,0,150) AS query, SUM(calls) AS calls FROM pg_stat_monitor GROUP BY datname, query ORDER BY datname, query, calls DESC Limit 20;', extra_params => ['-a', '-Pformat=aligned','-Ptuples_only=off']);
+($cmdret, $stdout, $stderr) = $node->psql('postgres', 'SELECT datname, substr(query,0,150) AS query, SUM(calls) AS calls FROM pg_stat_monitor GROUP BY datname, query ORDER BY datname, lower(query), calls DESC Limit 20;', extra_params => ['-a', '-Pformat=aligned','-Ptuples_only=off']);
 ok($cmdret == 0, "SELECT XXX FROM pg_stat_monitor");
 PGSM::append_to_file($stdout);
 
diff --git a/t/expected/007_settings_pgsm_query_shared_buffer.out b/t/expected/007_settings_pgsm_query_shared_buffer.out
diff --git a/t/expected/007_settings_pgsm_query_shared_buffer.out.15 b/t/expected/007_settings_pgsm_query_shared_buffer.out.15
diff --git a/t/pgsm.pm b/t/pgsm.pm
