feat: Oracle script enhancements and DMA_MANUAL_ID additions (#300)

* Change collect-data.sh to use named parameters instead of positional.
Add column DMA_MANUAL_ID

* Add hard coded file names to prevent empty files when no rows returned.

* Fixed column name spelling

* Update documentation.
Add help text to collect_data.sh
Change collection of non-cumulative stats.

* Retain ability to run locally while connecting '/ as sysdba'.

* Reset version prior to merge.

* Version bump

* Fix header on compressbytype

Author: wpuziewicz

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 4.3.20
+current_version = 4.3.21
 commit = False
 tag = False
 

docs/user_guide/oracle/collection_scripts.md

@@ -51,6 +51,7 @@ Execute grants_wrapper.sql. You will be prompted for the name of a database user
 (Note that input is case-sensitive and must match the username created above) to be granted
 privileges on the objects required for data collection.
 You will also be prompted whether or not to allow access to the AWR data.
+Access will be granted to Statspack tables if they are present.
 
 ```shell
 SQL> @grants_wrapper.sql
@@ -60,37 +61,69 @@ SQL> Please enter Y or N to allow or disallow use of the Tuning and Diagnostic P
 
 The grant_wrapper script will grant privileges required and will output a list of what has been granted.
 
-Launch the collection script:
+Launch the collection script: (Note that the parameter names have changed from earlier versions of the collector)
 
-- Pass connect string as input to this script and either UseDiagnostics or NoDiagnostics to match the permissions granted. (see below for example)
 - NOTE: If this is an Oracle RAC and/or PDB environment you just need to run it once per database. No need to run in each PDB or in each Oracle RAC instance.
   - If you are licensed for the Oracle Tuning and Diagnostics packs, pass the parameter UseDiagnostics to use the AWR data.
   - If you are NOT licensed for the Oracle Tuning and Diagnostics packs, pass the parameter NoDiagnostics to exclude the AWR data. The script will attempt to use STATSPACK data if available.
 
+  - Parameters
+```
+ Connection definition must one of:
+    {
+       --connectionStr       Oracle EasyConnect string formatted as {user}/{password}@//{db host}:{listener port}/{service name} 
+     or
+       --hostName            Database server hostname
+       --port                Listener port
+       --databaseService     Database service name
+       --collectionUserName  Database username 
+       --collectionUserPass  Database password
+    }
+ Performance statistics source
+     --statsSrc              Required. Must be one of AWR, STATSPACK, NONE
+```
+
+
 To use the licensed Oracle Tuning and Diagnostics pack data:
 
 ```shell
-./collect-data.sh {user}/{password}@//{db host/scan address}/{service name} UseDiagnostics
+./collect-data.sh --connectionStr {user}/{password}@//{db host}:{listener port}/{service name} --statsSrc AWR
+or
+./collect-data.sh --collectionUserName {user} --collectionUserPass {password} --hostName {db host} --port {listener port} --databaseService {service name} --statsSrc AWR
+
+ex:
+  
+./collect-data.sh --connectionStr MyUser/MyPassword@//dbhost.company.com:1521/MyDbName.company.com --statsSrc AWR
+or
+./collect-data.sh --collectionUserName MyUser --collectionUserPass MyPassword --hostName dbhost.company.com --port 1521 --databaseService MyDbName.company.com --statsSrc AWR
 ```
 
 OR
 To avoid using the licensed Oracle Tuning and Diagnostics pack data:
 
 ```shell
-./collect-data.sh {user}/{password}@//{db host/scan address}/{service name} NoDiagnostics
+./collect-data.sh --connectionStr {user}/{password}@//{db hosti}:{listener port}/{service name} --statsSrc STATSPACK
+or
+./collect-data.sh --collectionUserName {user} --collectionUserPass {password} --hostName {db host} --port {listener port} --databaseService {service name} --statsSrc STATSPACK
+
+ex:
+  
+./collect-data.sh --connectionStr MyUser/MyPassword@//dbhost.company.com:1521/MyDbName.company.com --statsSrc STATSPACK
+or
+./collect-data.sh --collectionUserName MyUser --collectionUserPass MyPassword --hostName dbhost.company.com --port 1521 --databaseService MyDbName.company.com --statsSrc STATSPACK
 ```
 
 Collections can be run as SYS if needed by setting ORACLE_SID and running on the database host:
 
 ```shell
-./collect-data.sh '/ as sysdba' UseDiagnostics
+./collect-data.sh '/ as sysdba' --statsSrc AWR 
 ```
 
 OR
 To avoid using the licensed Oracle Tuning and Diagnostics pack data:
 
 ```shell
-./collect-data.sh '/ as sysdba' NoDiagnostics
+./collect-data.sh '/ as sysdba' --statsSrc STATSPACK
 ```
 
 ## Upload Collections

scripts/collector/oracle/collect-data.sh

@@ -16,7 +16,7 @@
 
 ### Setup directories needed for execution
 #############################################################################
-OpVersion="4.3.20"
+OpVersion="4.3.21"
 dbmajor=""
 
 LOCALE=$(echo $LANG | cut -d '.' -f 1)
@@ -100,6 +100,7 @@ function executeOP {
 connectString="$1"
 OpVersion=$2
 DiagPack=$(echo $3 | tr [[:upper:]] [[:lower:]])
+collectionTag="${4}"
 
 if ! [ -x "$(command -v ${SQLPLUS})" ]; then
   echo "Could not find ${SQLPLUS} command. Source in environment and try again"
@@ -110,7 +111,7 @@ fi
 
 ${SQLPLUS} -s /nolog << EOF
 connect ${connectString}
-@${SQL_DIR}/op_collect.sql ${OpVersion} ${SQL_DIR} ${DiagPack} ${V_TAG} ${SQLOUTPUT_DIR}
+@${SQL_DIR}/op_collect.sql ${OpVersion} ${SQL_DIR} ${DiagPack} ${V_TAG} ${SQLOUTPUT_DIR} "${collectionTag}"
 exit;
 EOF
 
@@ -250,26 +251,121 @@ fi
 
 }
 
+
+function printUsage()
+{
+echo " Usage:"
+echo "  Parameters"
+echo ""
+echo "  Connection definition must one of:"
+echo "      {"
+echo "        --connectionStr       Oracle EasyConnect string formatted as {user}/{password}@//{db host}:{listener port}/{service name}"
+echo "       or"
+echo "        --hostName            Database server host name"
+echo "        --port                Database Listener port"
+echo "        --databaseService     Database service name"
+echo "        --collectionUserName  Database user name"
+echo "        --collectionUserPass  Database password"
+echo "      }"
+echo "  Performance statistics source"
+echo "      --statsSrc              Required. Must be one of AWR, STATSPACK, NONE"
+echo
+echo
+echo " Example:"
+echo
+echo
+echo "  ./collect-data.sh --connectionStr {user}/{password}@//{db host}:{listener port}/{service name} --statsSrc AWR"
+echo " or"
+echo "  ./collect-data.sh --collectionUserName {user} --collectionUserPass {password} --hostName {db host} --port {listener port} --databaseService {service name} --statsSrc AWR"
+
+}
 ### Validate input
-#############################################################################
 
-if [[  $# -ne 2  || (  "$2" != "UseDiagnostics" && "$2" != "NoDiagnostics" ) ]]
+hostName=""
+port=""
+databaseService=""
+collectionUserName=""
+collectionUserPass=""
+dbType=""
+statsSrc=""
+connStr=""
+collectionTag=""
+
+ if [[ $(($# & 1)) == 1 ]] ;
  then
-  echo 
-  echo "You must indicate whether or not to use the Diagnostics Pack views."
-  echo "If this database is licensed to use the Diagnostics pack:"
-  echo "  $0 $1 UseDiagnostics"
-  echo " "
-  echo "If this database is NOT licensed to use the Diagnostics pack:"
-  echo "  $0 $1 NoDiagnostics"
-  echo " "
-  exit 1
-fi
+  echo "Invalid number of parameters "
+  printUsage
+  exit 
+ fi
+
+ while (( "$#" )); do
+	 if   [[ "$1" == "--hostName" ]];           then hostName="${2}"
+	 elif [[ "$1" == "--port" ]];               then port="${2}"
+	 elif [[ "$1" == "--databaseService" ]];    then databaseService="${2}"
+	 elif [[ "$1" == "--collectionUserName" ]]; then collectionUserName="${2}"
+	 elif [[ "$1" == "--collectionUserPass" ]]; then collectionUserPass="${2}"
+	 elif [[ "$1" == "--dbType" ]];             then dbType=$(echo "${2}" | tr '[:upper:]' '[:lower:]')
+	 elif [[ "$1" == "--statsSrc" ]];           then statsSrc=$(echo "${2}" | tr '[:upper:]' '[:lower:]')
+	 elif [[ "$1" == "--connectionStr" ]];      then connStr="${2}"
+	 elif [[ "$1" == "--collectionTag" ]];      then collectionTag="${2}"
+	 else
+		 echo "Unknown parameter ${1}"
+		 printUsage
+		 exit
+	 fi
+	 shift 2
+ done
+
+
+ if [[ "${dbType}" != "oracle" ]] ; then
+	 dbType="oracle"
+ fi
+
+ if [[ "${statsSrc}" = "awr" ]]; then
+          DIAGPACKACCESS="UseDiagnostics"
+ elif [[ "${stasSrc}" = "statspack" ]] ; then
+          DIAGPACKACCESS="NoDiagnostics"
+ else 
+	 echo No performance data will be collected.
+         DIAGPACKACCESS="nostatspack"
+ fi
+
+ if [[ "${connStr}" == "" ]] ; then 
+	 if [[ "${hostName}" != "" && "${port}" != "" && "${databaseService}" != "" && "${collectionUserName}" != "" && "${collectionUserPass}" != "" ]] ; then
+		 connStr="${collectionUserName}/${collectionUserPass}@//${hostName}:${port}/${databaseService}"
+		 echo Got Connection ${connStr}
+	 else
+		 echo "Connection information incomplete"
+		 printUsage
+		 exit
+	 fi
+ fi
+
+ if [[ "${collectionTag}" != "" ]]; then
+	 collectionTag=$(echo "${collectionTag}" | iconv -t ascii//TRANSLIT | sed -E -e 's/[^[:alnum:]]+/-/g' -e 's/^-+|-+$//g' | tr '[:upper:]' '[:lower:]' | cut -c 1-100)
+ else collectionTag='NA'
+ fi
+
+
+#############################################################################
+#
+#if [[  $# -ne 2  || (  "$2" != "UseDiagnostics" && "$2" != "NoDiagnostics" ) ]]
+# then
+#  echo 
+#  echo "You must indicate whether or not to use the Diagnostics Pack views."
+#  echo "If this database is licensed to use the Diagnostics pack:"
+#  echo "  $0 $1 UseDiagnostics"
+#  echo " "
+#  echo "If this database is NOT licensed to use the Diagnostics pack:"
+#  echo "  $0 $1 NoDiagnostics"
+#  echo " "
+#  exit 1
+#fi
 
 # MAIN
 #############################################################################
 
-connectString="$1"
+connectString="${connStr}"
 sqlcmd_result=$(checkVersion "${connectString}" "${OpVersion}" | $GREP DMAFILETAG | cut -d '~' -f 2)
 if [[ "${sqlcmd_result}" = "" ]];
 then
@@ -279,7 +375,7 @@ fi
 
 retval=$?
 
-DIAGPACKACCESS="$2"
+# DIAGPACKACCESS="$2"
 
 extractorVersion="$(getVersion)"
 
@@ -309,7 +405,7 @@ if [ $retval -eq 0 ]; then
       fi  
     fi
     V_TAG="$(echo ${sqlcmd_result} | cut -d '|' -f2).csv"; export V_TAG
-    executeOP "${connectString}" ${OpVersion} ${DIAGPACKACCESS}
+    executeOP "${connectString}" ${OpVersion} ${DIAGPACKACCESS} "${collectionTag}"
     retval=$?
     if [ $retval -ne 0 ]; then
       createErrorLog  $(echo ${V_TAG} | sed 's/.csv//g')

scripts/collector/oracle/sql/extracts/9i_cpucoresusage.sql

@@ -15,5 +15,5 @@ limitations under the License.
 */
 -- This file intentionally left empty
 spool &outputdir/opdb__cpucoresusage__&v_tag
-prompt PKEY|DT|CPU_COUNT|CPU_CORE_COUNT|CPU_SOCKET_COUNT|DMA_SOURCE_ID
+prompt PKEY|DT|CPU_COUNT|CPU_CORE_COUNT|CPU_SOCKET_COUNT|DMA_SOURCE_ID|DMA_MANUAL_ID
 spool off

scripts/collector/oracle/sql/extracts/9i_dbfeatures.sql

@@ -15,5 +15,5 @@ limitations under the License.
 */
 -- This file intentionally left empty
 spool &outputdir/opdb__dbfeatures__&v_tag
-prompt PKEY|CON_ID|NAME|CURRE|DETECTED_USAGES|TOTAL_SAMPLES|FIRST_USAGE|LAST_USAGE|AUX_COUNT|DMA_SOURCE_ID
+prompt PKEY|CON_ID|NAME|CURRE|DETECTED_USAGES|TOTAL_SAMPLES|FIRST_USAGE|LAST_USAGE|AUX_COUNT|DMA_SOURCE_ID|DMA_MANUAL_ID
 spool off

scripts/collector/oracle/sql/extracts/9i_dbhwmarkstatistics.sql

@@ -15,5 +15,5 @@ limitations under the License.
 */
 -- This file intentionally left empty
 spool &outputdir/opdb__dbhwmarkstatistics__&v_tag
-prompt PKEY|DESCRIPTION|HIGHWATER|LAST_VALUE|CON_ID|DMA_SOURCE_ID
+prompt PKEY|DESCRIPTION|HIGHWATER|LAST_VALUE|CON_ID|DMA_SOURCE_ID|DMA_MANUAL_ID
 spool off

scripts/collector/oracle/sql/extracts/archlogs.sql

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 spool &outputdir/opdb__archlogs__&v_tag
-
+prompt PKEY|LOG_START_DATE|HO|THREAD_NUM|DEST_ID|CNT|MBYTES|DMA_SOURCE_ID|DMA_MANUAL_ID
 SELECT '&&v_host'
        || '_'
        || '&&v_dbname'
@@ -26,7 +26,7 @@ SELECT '&&v_host'
        dest_id, 
        count(1) AS CNT, 
        round(sum(blocks * block_size)/1024/1024) as mbytes,
-       '&v_dma_source_id' AS DMA_SOURCE_ID
+       '&v_dma_source_id' AS DMA_SOURCE_ID, chr(39) || '&v_collectionTag' || chr(39) AS DMA_MANUAL_ID
 FROM gv$archived_log
 WHERE first_time >= trunc(sysdate) - '&&dtrange'
 GROUP BY trunc(first_time), thread#, to_char(first_time, 'HH24'), dest_id

scripts/collector/oracle/sql/extracts/awr/awrhistcmdtypes.sql

@@ -15,7 +15,7 @@ limitations under the License.
 */
 
 spool &outputdir/opdb__awrhistcmdtypes__&v_tag
-
+prompt PKEY|CON_ID|HH|COMMAND_TYPE|CNT|AVG_BUFFER_GETS|AVG_ELASPED_TIME|AVG_ROWS_PROCESSED|AVG_EXECUTIONS|AVG_CPU_TIME|AVG_IOWAIT|AVG_CLWAIT|AVG_APWAIT|AVG_CCWAIT|AVG_PLSEXEC_TIME|COMMAND_NAME|DMA_SOURCE_ID|DMA_MANUAL_ID
 WITH vcmdtype AS(
 SELECT '&&v_host'
        || '_'
@@ -61,6 +61,6 @@ GROUP  BY '&&v_host'
 SELECT pkey , con_id , hh24 , command_type , cnt , avg_buffer_gets , avg_elasped_time ,
        avg_rows_processed , avg_executions , avg_cpu_time , avg_iowait , avg_clwait ,
        avg_apwait , avg_ccwait , avg_plsexec_time, command_name,
-       '&v_dma_source_id' AS DMA_SOURCE_ID
+       '&v_dma_source_id' AS DMA_SOURCE_ID, chr(39) || '&v_collectionTag' || chr(39) AS DMA_MANUAL_ID
 FROM vcmdtype;
 spool off

scripts/collector/oracle/sql/extracts/awr/awrhistosstat.sql

@@ -13,9 +13,9 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 */
-
+set echo on
 spool &outputdir/opdb__awrhistosstat__&v_tag
-
+prompt PKEY|DBID|INSTANCE_NUMBER|HH|STAT_NAME|HH24_TOTAL_SECS|CUMULATIVE_VALUE|AVG_VALUE|MODE_VALUE|MEDIAN_VALUE|PERC50|PERC75|PERC90|PERC95|PERC100|MIN_VALUE|MAX_VALUE|SUM_VALUE|COUNT|DMA_SOURCE_ID|DMA_MANUAL_ID
 WITH v_osstat_all
      AS (SELECT os.dbid,
                      os.instance_number,
@@ -51,14 +51,17 @@ WITH v_osstat_all
                        TO_CHAR(os.begin_interval_time, 'hh24'), os.stat_name) AS
                      "PERC100"
               FROM (SELECT snap.begin_interval_time, snap.end_interval_time, s.*,
-                    NVL(DECODE(GREATEST(value, NVL(LAG(value)
-                    OVER (
-                    PARTITION BY s.dbid, s.instance_number, s.stat_name
-                    ORDER BY s.snap_id), 0)), value, value - LAG(value)
-                       OVER (
-                       PARTITION BY s.dbid, s.instance_number, s.stat_name
-                       ORDER BY s.snap_id),
-                    0), 0) AS delta_value
+		    CASE WHEN s.stat_name IN ('IDLE_TIME' ,'BUSY_TIME' ,'USER_TIME' ,'SYS_TIME' ,'IOWAIT_TIME' ,'NICE_TIME' ,'RSRC_MGR_CPU_WAIT_TIME' ,'VM_IN_BYTES' ,'VM_OUT_BYTES')
+		       	THEN
+			    NVL(DECODE(GREATEST(value, NVL(LAG(value)
+			    OVER (
+			    PARTITION BY s.dbid, s.instance_number, s.stat_name
+			    ORDER BY s.snap_id), 0)), value, value - LAG(value)
+			       OVER (
+			       PARTITION BY s.dbid, s.instance_number, s.stat_name
+			       ORDER BY s.snap_id),
+			    0), 0)
+			ELSE s.value END AS delta_value
                     FROM &v_tblprefix._hist_osstat s
                          inner join &v_tblprefix._hist_snapshot snap
                          ON s.snap_id = snap.snap_id
@@ -103,6 +106,6 @@ GROUP  BY '&&v_host'
 SELECT pkey , dbid , instance_number , hh24 , stat_name , hh24_total_secs ,
        cumulative_value , avg_value , mode_value , median_value , PERC50 , PERC75 , PERC90 , PERC95 , PERC100 ,
 	     min_value , max_value , sum_value , count,
-	       '&v_dma_source_id' AS DMA_SOURCE_ID
+	       '&v_dma_source_id' AS DMA_SOURCE_ID, chr(39) || '&v_collectionTag' || chr(39) AS DMA_MANUAL_ID
 FROM vossummary;
 spool off

scripts/collector/oracle/sql/extracts/awr/awrhistsysmetrichist.sql

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 spool &outputdir/opdb__awrhistsysmetrichist__&v_tag
-
+prompt PKEY|DBID|INSTANCE_NUMBER|HOUR|METRIC_NAME|METRIC_UNIT|AVG_VALUE|MODE_VALUE|MEDIAN_VALUE|MIN_VALUE|MAX_VALUE|SUM_VALUE|PERC50|PERC75|PERC90|PERC95|PERC100|DMA_SOURCE_ID|DMA_MANUAL_ID
 WITH vsysmetric AS (
 SELECT '&&v_host'
        || '_'
@@ -66,6 +66,6 @@ ORDER  BY hsm.dbid,
 SELECT pkey , dbid , instance_number , hour , metric_name ,
        metric_unit , avg_value , mode_value , median_value , min_value , max_value ,
 	   sum_value , PERC50 , PERC75 , PERC90 , PERC95 , PERC100,
-	       '&v_dma_source_id' AS DMA_SOURCE_ID
+	       '&v_dma_source_id' AS DMA_SOURCE_ID, chr(39) || '&v_collectionTag' || chr(39) AS DMA_MANUAL_ID
 FROM vsysmetric;
 spool off