enable task retry recording

Author: a114j0y

lib/recorder.cpp

@@ -10,7 +10,7 @@ const std::string Recorder::REC_START = "|recording started";
 const std::string Recorder::SWSS_FNAME = "swss.rec";
 const std::string Recorder::SAIREDIS_FNAME = "sairedis.rec";
 const std::string Recorder::RESPPUB_FNAME = "responsepublisher.rec";
-
+const std::string Recorder::RETRY_FNAME = "retry.rec";
 
 Recorder& Recorder::Instance()
 {
@@ -19,6 +19,17 @@ Recorder& Recorder::Instance()
 }
 
 
+RetryRec::RetryRec() 
+{
+    /* Set Default values */
+    setRecord(true);
+    setRotate(false);
+    setLocation(Recorder::DEFAULT_DIR);
+    setFileName(Recorder::RETRY_FNAME);
+    setName("Retry");
+}
+
+
 SwSSRec::SwSSRec() 
 {
     /* Set Default values */

lib/recorder.h

@@ -48,6 +48,11 @@ class RecWriter : public RecBase {
     std::string fname;
 };
 
+class RetryRec : public RecWriter {
+public:
+    RetryRec();
+};
+
 class SwSSRec : public RecWriter {
 public:
     SwSSRec();
@@ -73,12 +78,14 @@ class Recorder {
     static const std::string SWSS_FNAME;
     static const std::string SAIREDIS_FNAME;
     static const std::string RESPPUB_FNAME;
+    static const std::string RETRY_FNAME;
 
     Recorder() = default;
     /* Individual Handlers */
     SwSSRec swss;
     SaiRedisRec sairedis;
     ResPubRec respub;
+    RetryRec retry;
 };
 
 }

orchagent/main.cpp

@@ -62,6 +62,7 @@ extern bool gIsNatSupported;
 #define SAIREDIS_RECORD_ENABLE 0x1
 #define SWSS_RECORD_ENABLE (0x1 << 1)
 #define RESPONSE_PUBLISHER_RECORD_ENABLE (0x1 << 2)
+#define RETRY_RECORD_ENABLE (0x1 << 3)
 
 /* orchagent heart beat message interval */
 #define HEART_BEAT_INTERVAL_MSECS_DEFAULT 10 * 1000
@@ -109,6 +110,7 @@ void sighup_handler(int signo)
     /*
      * Don't do any logging since they are using mutexes.
      */
+    Recorder::Instance().retry.setRotate(true);
     Recorder::Instance().swss.setRotate(true);
     Recorder::Instance().sairedis.setRotate(true);
     Recorder::Instance().respub.setRotate(true);
@@ -361,6 +363,7 @@ int main(int argc, char **argv)
     string record_location = Recorder::DEFAULT_DIR;
     string swss_rec_filename = Recorder::SWSS_FNAME;
     string sairedis_rec_filename = Recorder::SAIREDIS_FNAME;
+    string retry_rec_filename = Recorder::RETRY_FNAME;
     string zmq_server_address = "tcp://127.0.0.1:" + to_string(ORCH_ZMQ_PORT);
     string vrf;
     bool   enable_zmq = false;
@@ -395,7 +398,7 @@ int main(int argc, char **argv)
             // Disable all recordings if atoi() fails i.e. returns 0 due to
             // invalid command line argument.
             record_type = atoi(optarg);
-            if (record_type < 0 || record_type > 7) 
+            if (record_type < 0 || record_type > 15) 
             {
                 usage();
                 exit(EXIT_FAILURE);
@@ -522,6 +525,13 @@ int main(int argc, char **argv)
     Recorder::Instance().respub.setFileName(responsepublisher_rec_filename);
     Recorder::Instance().respub.startRec(false);
 
+    Recorder::Instance().retry.setRecord(
+        (record_type & RETRY_RECORD_ENABLE) == RETRY_RECORD_ENABLE
+    );
+    Recorder::Instance().retry.setLocation(record_location);
+    Recorder::Instance().retry.setFileName(retry_rec_filename);
+    Recorder::Instance().retry.startRec(true);
+
     // Instantiate database connectors
     DBConnector appl_db("APPL_DB", 0);
     DBConnector config_db("CONFIG_DB", 0);

orchagent/orch.cpp

@@ -168,10 +168,12 @@ ConsumerBase* Orch::getConsumerBase(const std::string &executorName)
 }
 
 void ConsumerBase::addToRetry(const Task &task, const Constraint &cst) {
+    Recorder::Instance().retry.record(dumpTuple(task).append(CACHE));
     getOrch()->getRetryCache(getName())->cache_failed_task(task, cst);
 }
 
 void Orch::addToRetry(const std::string &executorName, const Task &task, const Constraint &cst) {
+    Recorder::Instance().retry.record(getConsumerBase(executorName)->dumpTuple(task).append(CACHE));
     getRetryCache(executorName)->cache_failed_task(task, cst); 
 }
 
@@ -216,8 +218,11 @@ void ConsumerBase::addToSync(const KeyOpFieldsValuesTuple &entry, bool onRetry)
     string key = kfvKey(entry);
     string op  = kfvOp(entry);
 
-    /* Record incoming tasks */
-    Recorder::Instance().swss.record(dumpTuple(entry));
+    if (!onRetry)
+        /* Record incoming tasks */
+        Recorder::Instance().swss.record(dumpTuple(entry));
+    else
+        Recorder::Instance().retry.record(dumpTuple(entry).append(DECACHE));
 
     auto retryCache = getOrch()->getRetryCache(getName());
 

orchagent/orch.h

@@ -272,6 +272,9 @@ typedef enum
 typedef std::pair<swss::DBConnector *, std::string> TableConnector;
 typedef std::pair<swss::DBConnector *, std::vector<std::string>> TablesConnector;
 
+#define CACHE "| ++++ |"
+#define DECACHE "| ---- |"
+
 
 class Orch
 {

orchagent/retrycache.h

@@ -6,6 +6,8 @@
 #include <sys/time.h>
 #include "timestamp.h"
 #include <fstream>
+#include <sstream> 
+#include "recorder.h"
 
 using namespace swss;
 
@@ -19,6 +21,19 @@ enum ConstraintType
     RETRY_CST_ECMP          // ecmp resources exhausted
 };
 
+
+inline std::ostream& operator<<(std::ostream& os, ConstraintType t) {
+    switch(t) {
+        case ConstraintType::RETRY_CST_DUMMY:   return os << "RETRY_CST_DUMMY";
+        case ConstraintType::RETRY_CST_NHG:   return os << "RETRY_CST_NHG";
+        case ConstraintType::RETRY_CST_NHG_REF:   return os << "RETRY_CST_NHG_REF";
+        case ConstraintType::RETRY_CST_PIC: return os << "RETRY_CST_PIC";
+        case ConstraintType::RETRY_CST_PIC_REF:  return os << "RETRY_CST_PIC_REF";
+        case ConstraintType::RETRY_CST_ECMP:   return os << "RETRY_CST_ECMP";
+        default:           return os << "UNKNOWN";
+    }
+}
+
 using ConstraintData = std::string;
 using Constraint = std::pair<ConstraintType, ConstraintData>;
 
@@ -28,6 +43,11 @@ inline Constraint make_constraint(ConstraintType type, ConstraintData data = "")
     return {type, data};
 }
 
+template<typename T, typename U>
+std::ostream& operator<<(std::ostream& os, const std::pair<T, U>& p) {
+    return os << "(" << p.first << ", " << p.second << ")";
+}
+
 typedef swss::KeyOpFieldsValuesTuple Task;
 typedef std::pair<Constraint, Task> FailedTask;
 typedef std::unordered_map<std::string, FailedTask> RetryMap;
@@ -80,6 +100,9 @@ class RetryCache
         else
         {
             m_resolvedConstraints.emplace(cst.first, cst.second);
+            std::stringstream ss;
+            ss << cst << " resolution notified -> " << m_retryKeys[cst].size() << " task(s)";
+            Recorder::Instance().retry.record(ss.str());
         }
     }
 
@@ -144,15 +167,22 @@ class RetryCache
             }
         }
 
-       if (keys.empty()) {
+        std::stringstream ss;
+        ss << cst << " | " << m_executorName << " | " << tasks->size() << " retried";
+
+        if (keys.empty()) {
             m_retryKeys.erase(cst);
             m_resolvedConstraints.erase(cst);
+        } else {
+            ss << " (rest:" << keys.size() << ")";
         }
 
+        Recorder::Instance().retry.record(ss.str());
+
         return tasks;
     }
 };
 
 typedef std::unordered_map<std::string, std::shared_ptr<RetryCache>> RetryCacheMap;
 
-#endif /* SWSS_RETRY_CACHE_H */
+#endif /* SWSS_RETRY_CACHE_H */