Merge pull request #381 from telefonicaid/fix/metrics_by_service_and_subservice

Fix/metrics by service and subservice

Author: fgalan

CONFIG.md

@@ -241,7 +241,7 @@ IOTAGENT = {
 
 ## Metrics
 
-There are an option, that is disabled by default, to obtain a extended metrics report. This is still experimental due to enable it could imply decrease performance.
+There are an option, that is disabled by default, to obtain a extended metrics report. This is still experimental due to enable it could imply decrease performance. And implies configure uWSGI to use just one thread by each process (i.e. enabling THREADS = 1 and PROCESS = 8) instead default uWSGI used by orchestrator
 [Extended Metrics](https://orchestrator2.docs.apiary.io/#reference/orchestrator/metrics)
 
 ```

DOCKER.md

@@ -149,7 +149,7 @@ Additionally, the following environment variables are available for orchestrator
 | PEP_PASSWORD                | PEP.password              | pep                       |
 | IOTAGENT_USER               | IOTAGENT.user             | iotagent                  |
 | IOTAGENT_PASSWORD           | IOTAGENT.password         | iotagent                  |
-
+| ORC_EXTENDED_METRICS        | ORC_EXTENDED_METRICS      | false  (experimental)     |
 
 
 ## Build the image

bin/orchestrator-entrypoint.sh

@@ -67,7 +67,7 @@
 [[ "${IOTAGENT_USER}" == "" ]] && export IOTAGENT_USER=iotagent
 [[ "${IOTAGENT_PASSWORD}" == "" ]] && export IOTAGENT_PASSWORD=iotagent
 
-
+[[ "${ORC_EXTENDED_METRICS}" == "" ]] && export ORC_EXTENDED_METRICS=false
 
 while [[ $# -gt 0 ]]; do
     PARAM=`echo $1`
@@ -304,7 +304,6 @@ sed -i ':a;N;$!ba;s/IOTAGENT = {[A-Za-z0-9,=@.\-\/\"\n: ]*}/IOTAGENT = { \
 }/g' /opt/orchestrator/settings/dev.py
 
 
-
 if [ "$DEBUG_LEVEL" ]; then
 echo "
 LOGGING['handlers']['console']['level'] = '$DEBUG_LEVEL'
@@ -315,6 +314,12 @@ LOGGING['loggers']['orchestrator_core']['level'] = '$DEBUG_LEVEL'
 fi
 
 
+if [ "$ORC_EXTENDED_METRICS" == "true" ]; then
+echo "
+ORC_EXTENDED_METRICS = True
+" >> /opt/orchestrator/settings/dev.py
+fi
+
 # Wait until Keystone and Keypass are up
 while ! nc -zvw10 $KEYSTONE_HOST $KEYSTONE_PORT ; do sleep 10; done
 while ! nc -zvw10 $KEYPASS_HOST $KEYPASS_PORT ; do sleep 10; done

src/orchestrator/api/stats.py

@@ -29,6 +29,7 @@
 import requests
 import logging
 import time
+import http.client
 from settings import dev as settings
 
 class RestOperations(object):
@@ -271,17 +272,26 @@ def collectOutgoingMetrics(self, service_start, data_request, headers_request, r
             transactionError = False
             if response.code not in [200, 201, 204]:
                 transactionError = True
-            data_response = response.msg
             if transactionError:
-                self.sum["outgoingTransactions"] += 1
-            else:
                 self.sum["outgoingTransactionErrors"] += 1
-            self.sum["outgoingTransactionRequestSize"] += len(json.dumps(data_request)) + len(str(headers_request))
-            # Check headers
-            self.sum["outgoingTransactionResponseSize"] += len(json.dumps(data_response)) + len(str(response.headers.headers)) if 'headers' in response and 'headers' in response.headers else 0
+            else:
+                self.sum["outgoingTransactions"] += 1
+
+            try:
+                self.sum["outgoingTransactionRequestSize"] += len(str(data_request)) + len(str(headers_request))
+            except Exception as ex:
+                self.logger.warn("ERROR collecting outgoingTransactionRequestSize: %s data_request %s headers_request %s", ex, str(data_request), str(headers_request))
+            try:
+                if isinstance(response, http.client.HTTPResponse) and hasattr(response, 'getheader'):
+                    inc = response.getheader("Content-Length")
+                    if inc is not None:
+                        self.sum["outgoingTransactionResponseSize"] += int(inc)
+            except Exception as ex:
+                self.logger.warn("ERROR collecting outgoingTransactionResponseSize: %s response %s", ex, str(response))
             self.sum["serviceTimeTotal"] += (service_stop - service_start)
+
         except Exception as ex:
-            self.logger.warn("ERROR collecting outgoing metrics %s", ex)
+            self.logger.warn("ERROR collecting outgoing metrics: %s", ex)
 
     def getOutgoingMetrics(self):
         return self.sum

src/orchestrator/common/util.py

@@ -23,6 +23,7 @@
 #
 import logging
 import uuid
+from functools import reduce
 
 from orchestrator.core.keystone import IdMKeystoneOperations as IdMOperations
 from orchestrator.core.keypass import AccCKeypassOperations as AccCOperations
@@ -252,15 +253,14 @@ def ensure_subservice_name(self, USER_TOKEN, SERVICE_ID, SUBSERVICE_ID,
 
 
     def get_extended_token(self, USER_TOKEN):
-        token_extended = USER_TOKEN
+        token_extended = {
+            "token": USER_TOKEN
+        }
         if USER_TOKEN:
             try:
                 token_detail = self.idm.getTokenDetail(USER_TOKEN)
+                token_extended["user"] = token_detail['token']['user']['name']
 
-                token_extended  = {
-                    "token": USER_TOKEN,
-                    "user": token_detail['token']['user']['name']
-                }
                 # Include service scope if available
                 if 'domain' in token_detail['token']['user']:
                     token_extended['domain'] = \
@@ -273,10 +273,8 @@ def get_extended_token(self, USER_TOKEN):
 
             except Exception as ex:
                 # Probably expired?
-                token_extended  = {
-                    "token": USER_TOKEN,
-                    "error": str(ex)
-                }
+                token_extended["error"] = str(ex)
+
         return token_extended
 
     def collectComponentMetrics(self):
@@ -290,9 +288,9 @@ def collectComponentMetrics(self):
             all.append(self.cb.CBRestOperations.getOutgoingMetrics())
             all.append(self.perseo.PerseoRestOperations.getOutgoingMetrics())
             # TODO: Take care of the following operation takes too much time
-            self.sum = reduce(lambda x, y: dict((k, v + y[k]) for k, v in x.iteritems()), all)
+            self.sum = reduce(lambda x, y: {k: v + y[k] for k, v in x.items()}, all)
         except Exception as ex:
-            self.logger.error("ERROR collecting component metrics %s", ex)
+            self.logger.error("ERROR collecting component metrics: %s", ex)
 
     def getFlowMetrics(self):
         return self.sum

src/orchestrator/core/flow/base.py

@@ -17,14 +17,6 @@
 
 os.environ.setdefault("DJANGO_SETTINGS_MODULE", "orchestrator.settings.dev")
 
-from multiprocessing import Manager
-manager = Manager()
-shared_data = manager.dict()
-
-from django.conf import settings
-settings.SHARED_DATA = shared_data
-
-
 # This application object is used by any WSGI server configured to use this
 # file. This includes Django's development server, if the WSGI_APPLICATION
 # setting points here.