avoid OA crashes for all cases

Signed-off-by: Prabhat Aravind <paravind@microsoft.com>

Author: prabhataravind

orchagent/orch.h

@@ -66,6 +66,15 @@ typedef enum
     task_duplicated
 } task_process_status;
 
+
+typedef enum
+{
+    oper_create,
+    oper_remove,
+    oper_set,
+    oper_get
+} sai_oper_type_t
+
 typedef struct
 {
     // m_objsDependingOnMe stores names (without table name) of all objects depending on the current obj

orchagent/saihelper.cpp

@@ -90,6 +90,7 @@ sai_stp_api_t*                      sai_stp_api;
 
 extern sai_object_id_t gSwitchId;
 extern bool gTraditionalFlexCounter;
+extern event_handle_t g_events_handle;
 
 vector<sai_object_id_t> gGearboxOids;
 
@@ -532,100 +533,23 @@ task_process_status handleSaiCreateStatus(sai_api_t api, sai_status_t status, vo
      *          in each orch.
      *       3. Take the type of sai api into consideration.
      */
-    switch (api)
-    {
-        case SAI_API_FDB:
-            switch (status)
-            {
-                case SAI_STATUS_SUCCESS:
-                    SWSS_LOG_WARN("SAI_STATUS_SUCCESS is not expected in handleSaiCreateStatus");
-                    return task_success;
-                case SAI_STATUS_ITEM_ALREADY_EXISTS:
-                    /*
-                     *  In FDB creation, there are scenarios where the hardware learns an FDB entry before orchagent.
-                     *  In such cases, the FDB SAI creation would report the status of SAI_STATUS_ITEM_ALREADY_EXISTS,
-                     *  and orchagent should ignore the error and treat it as entry was explicitly created.
-                     */
-                    return task_success;
-                default:
-                    SWSS_LOG_ERROR("Encountered failure in create operation, exiting orchagent, SAI API: %s, status: %s",
-                                sai_serialize_api(api).c_str(), sai_serialize_status(status).c_str());
-                    handleSaiFailure(true);
-                    break;
-            }
-            break;
-        case SAI_API_HOSTIF:
-            switch (status)
-            {
-                case SAI_STATUS_SUCCESS:
-                    return task_success;
-                case SAI_STATUS_FAILURE:
-                    /*
-                     * Host interface maybe failed due to lane not available.
-                     * In some scenarios, like SONiC virtual machine, the invalid lane may be not enabled by VM configuration,
-                     * So just ignore the failure and report an error log.
-                     */
-                    return task_ignore;
-                default:
-                    SWSS_LOG_ERROR("Encountered failure in create operation, exiting orchagent, SAI API: %s, status: %s",
-                                sai_serialize_api(api).c_str(), sai_serialize_status(status).c_str());
-                    handleSaiFailure(true);
-                    break;
-            }
-            break;
-        case SAI_API_ROUTE:
-            switch (status)
-            {
-                case SAI_STATUS_SUCCESS:
-                    SWSS_LOG_WARN("SAI_STATUS_SUCCESS is not expected in handleSaiCreateStatus");
-                    return task_success;
-                case SAI_STATUS_ITEM_ALREADY_EXISTS:
-                case SAI_STATUS_NOT_EXECUTED:
-                    /* With VNET routes, the same route can be learned via multiple
-                    sources, like via BGP. Handle this gracefully */
-                    return task_success;
-                case SAI_STATUS_TABLE_FULL:
-                    return task_need_retry;
-                default:
-                    SWSS_LOG_ERROR("Encountered failure in create operation, exiting orchagent, SAI API: %s, status: %s",
-                                sai_serialize_api(api).c_str(), sai_serialize_status(status).c_str());
-                    handleSaiFailure(true);
-                    break;
-            }
-            break;
-        case SAI_API_NEIGHBOR:
-        case SAI_API_NEXT_HOP:
-        case SAI_API_NEXT_HOP_GROUP:
-            switch(status)
-            {
-                case SAI_STATUS_SUCCESS:
-                    SWSS_LOG_WARN("SAI_STATUS_SUCCESS is not expected in handleSaiCreateStatus");
-                    return task_success;
-                case SAI_STATUS_ITEM_ALREADY_EXISTS:
-                    return task_success;
-                case SAI_STATUS_TABLE_FULL:
-                    return task_need_retry;
-                default:
-                    SWSS_LOG_ERROR("Encountered failure in create operation, exiting orchagent, SAI API: %s, status: %s",
-                                sai_serialize_api(api).c_str(), sai_serialize_status(status).c_str());
-                    handleSaiFailure(true);
-                    break;
-            }
-            break;
+     switch (status)
+     {
+        case SAI_STATUS_SUCCESS:
+            SWSS_LOG_WARN("SAI_STATUS_SUCCESS is not expected in handleSaiCreateStatus");
+            return task_success;
+        case SAI_STATUS_ITEM_ALREADY_EXISTS:
+            return task_success;
+        case SAI_STATUS_INSUFFICIENT_RESOURCES:
+        case SAI_STATUS_TABLE_FULL:
+        case SAI_STATUS_NO_MEMORY:
+        case SAI_STATUS_NV_STORAGE_FULL:
+            return task_need_retry;
         default:
-            switch (status)
-            {
-                case SAI_STATUS_SUCCESS:
-                    SWSS_LOG_WARN("SAI_STATUS_SUCCESS is not expected in handleSaiCreateStatus");
-                    return task_success;
-                default:
-                    SWSS_LOG_ERROR("Encountered failure in create operation, exiting orchagent, SAI API: %s, status: %s",
-                                sai_serialize_api(api).c_str(), sai_serialize_status(status).c_str());
-                    handleSaiFailure(true);
-                    break;
-            }
-    }
-    return task_need_retry;
+            handleSaiFailure(api, "create", status);
+            break;
+      }
+      return task_need_retry;
 }
 
 task_process_status handleSaiSetStatus(sai_api_t api, sai_status_t status, void *context)
@@ -641,67 +565,24 @@ task_process_status handleSaiSetStatus(sai_api_t api, sai_status_t status, void
      *          in each orch.
      *       3. Take the type of sai api into consideration.
      */
-    if (status == SAI_STATUS_SUCCESS)
-    {
-        SWSS_LOG_WARN("SAI_STATUS_SUCCESS is not expected in handleSaiSetStatus");
-        return task_success;
-    }
-
-    switch (api)
+    switch (status)
     {
-        case SAI_API_PORT:
-            switch (status)
-            {
-                case SAI_STATUS_INVALID_ATTR_VALUE_0:
-                    /*
-                     * If user gives an invalid attribute value, no need to retry or exit orchagent, just fail the current task
-                     * and let user correct the configuration.
-                     */
-                    SWSS_LOG_ERROR("Encountered SAI_STATUS_INVALID_ATTR_VALUE_0 in set operation, task failed, SAI API: %s, status: %s",
-                            sai_serialize_api(api).c_str(), sai_serialize_status(status).c_str());
-                    return task_failed;
-                default:
-                    SWSS_LOG_ERROR("Encountered failure in set operation, exiting orchagent, SAI API: %s, status: %s",
-                            sai_serialize_api(api).c_str(), sai_serialize_status(status).c_str());
-                    handleSaiFailure(true);
-                    break;
-            }
-            break;
-        case SAI_API_TUNNEL:
-            switch (status)
-            {
-                case SAI_STATUS_ATTR_NOT_SUPPORTED_0:
-                    SWSS_LOG_ERROR("Encountered SAI_STATUS_ATTR_NOT_SUPPORTED_0 in set operation, task failed, SAI API: %s, status: %s",
-                            sai_serialize_api(api).c_str(), sai_serialize_status(status).c_str());
-                    return task_failed;
-                default:
-                    SWSS_LOG_ERROR("Encountered failure in set operation, exiting orchagent, SAI API: %s, status: %s",
-                            sai_serialize_api(api).c_str(), sai_serialize_status(status).c_str());
-                    handleSaiFailure(true);
-                    break;
-            }
-            break;
-        case SAI_API_BUFFER:
-            switch (status)
-            {
-                case SAI_STATUS_INSUFFICIENT_RESOURCES:
-                    SWSS_LOG_ERROR("Encountered SAI_STATUS_INSUFFICIENT_RESOURCES in set operation, task failed, SAI API: %s, status: %s",
-                                   sai_serialize_api(api).c_str(), sai_serialize_status(status).c_str());
-                    return task_failed;
-                default:
-                    SWSS_LOG_ERROR("Encountered failure in set operation, exiting orchagent, SAI API: %s, status: %s",
-                            sai_serialize_api(api).c_str(), sai_serialize_status(status).c_str());
-                    handleSaiFailure(true);
-                    break;
-            }
-            break;
+        case SAI_STATUS_SUCCESS:
+            SWSS_LOG_WARN("SAI_STATUS_SUCCESS is not expected in handleSaiSetStatus");
+            return task_success;
+        case SAI_STATUS_ITEM_ALREADY_EXISTS:
+        /* Bulked calls could return this value */
+        case SAI_STATUS_NOT_EXECUTED:
+            return task_success;
+        case SAI_STATUS_INSUFFICIENT_RESOURCES:
+        case SAI_STATUS_TABLE_FULL:
+        case SAI_STATUS_NO_MEMORY:
+        case SAI_STATUS_NV_STORAGE_FULL:
+            return task_need_retry;
         default:
-            SWSS_LOG_ERROR("Encountered failure in set operation, exiting orchagent, SAI API: %s, status: %s",
-                        sai_serialize_api(api).c_str(), sai_serialize_status(status).c_str());
-            handleSaiFailure(true);
+            handleSaiFailure(api, oper_set, status);
             break;
     }
-
     return task_need_retry;
 }
 
@@ -719,55 +600,18 @@ task_process_status handleSaiRemoveStatus(sai_api_t api, sai_status_t status, vo
      *          in each orch.
      *       3. Take the type of sai api into consideration.
      */
-    switch (api)
+    switch (status)
     {
-        case SAI_API_ROUTE:
-            switch (status)
-            {
-                case SAI_STATUS_SUCCESS:
-                    SWSS_LOG_WARN("SAI_STATUS_SUCCESS is not expected in handleSaiRemoveStatus");
-                    return task_success;
-                case SAI_STATUS_ITEM_NOT_FOUND:
-                case SAI_STATUS_NOT_EXECUTED:
-                    /* When the same route is learned via multiple sources,
-                       there can be a duplicate remove operation. Handle this gracefully */
-                    return task_success;
-                default:
-                    SWSS_LOG_ERROR("Encountered failure in remove operation, exiting orchagent, SAI API: %s, status: %s",
-                                sai_serialize_api(api).c_str(), sai_serialize_status(status).c_str());
-                    handleSaiFailure(true);
-                    break;
-            }
-            break;
-        case SAI_API_NEIGHBOR:
-        case SAI_API_NEXT_HOP:
-        case SAI_API_NEXT_HOP_GROUP:
-            switch (status)
-            {
-                case SAI_STATUS_SUCCESS:
-                    SWSS_LOG_WARN("SAI_STATUS_SUCCESS is not expected in handleSaiRemoveStatus");
-                    return task_success;
-                case SAI_STATUS_ITEM_NOT_FOUND:
-                    return task_success;
-                default:
-                    SWSS_LOG_ERROR("Encountered failure in remove operation, exiting orchagent, SAI API: %s, status: %s",
-                                sai_serialize_api(api).c_str(), sai_serialize_status(status).c_str());
-                    handleSaiFailure(true);
-                    break;
-            }
-            break;
+        case SAI_STATUS_SUCCESS:
+            SWSS_LOG_WARN("SAI_STATUS_SUCCESS is not expected in handleSaiRemoveStatus");
+            return task_success;
+        case SAI_STATUS_NOT_EXECUTED:
+        /* When the same route is learned via multiple sources,
+           there can be a duplicate remove operation. Handle this gracefully */
+            return task_success;
         default:
-            switch (status)
-            {
-                case SAI_STATUS_SUCCESS:
-                    SWSS_LOG_WARN("SAI_STATUS_SUCCESS is not expected in handleSaiRemoveStatus");
-                    return task_success;
-                default:
-                    SWSS_LOG_ERROR("Encountered failure in remove operation, exiting orchagent, SAI API: %s, status: %s",
-                        sai_serialize_api(api).c_str(), sai_serialize_status(status).c_str());
-                    handleSaiFailure(true);
-                    break;
-            }
+            handleSaiFailure(api, oper_remove, status);
+            break;
     }
     return task_need_retry;
 }
@@ -791,12 +635,10 @@ task_process_status handleSaiGetStatus(sai_api_t api, sai_status_t status, void
             SWSS_LOG_WARN("SAI_STATUS_SUCCESS is not expected in handleSaiGetStatus");
             return task_success;
         case SAI_STATUS_NOT_IMPLEMENTED:
-            SWSS_LOG_ERROR("Encountered failure in get operation due to the function is not implemented, exiting orchagent, SAI API: %s",
-                        sai_serialize_api(api).c_str());
             throw std::logic_error("SAI get function not implemented");
         default:
-            SWSS_LOG_ERROR("Encountered failure in get operation, SAI API: %s, status: %s",
-                        sai_serialize_api(api).c_str(), sai_serialize_status(status).c_str());
+            handleSaiFailure(api, oper_get, status);
+            break;
     }
     return task_failed;
 }
@@ -820,11 +662,23 @@ bool parseHandleSaiStatusFailure(task_process_status status)
     return true;
 }
 
-/* Handling SAI failure. Request redis to invoke SAI failure dump and abort if set*/
-void handleSaiFailure(bool abort_on_failure)
+/* Handling SAI failure. Request redis to invoke SAI failure dump */
+void handleSaiFailure(sai_api_t api, string oper, sai_status_t status)
 {
     SWSS_LOG_ENTER();
 
+    string s_api = sai_serialize_api(api);
+    string s_status = sai_serialize_status(status);
+    SWSS_LOG_ERROR("Encountered failure in %s operation, SAI API: %s, status: %s",
+                        oper.c_str(), s_api.c_str(), s_status.c_str());
+
+    // Publish a structured syslog event
+    event_params_t params = {
+        { "operation", oper },
+        { "api", s_api },
+        { "status", s_status }};
+    event_publish(g_events_handle, "sai-operation-failure", &params);
+
     sai_attribute_t attr;
 
     attr.id = SAI_REDIS_SWITCH_ATTR_NOTIFY_SYNCD;
@@ -834,13 +688,8 @@ void handleSaiFailure(bool abort_on_failure)
     {
         SWSS_LOG_ERROR("Failed to take sai failure dump %d", status);
     }
-    if (abort_on_failure)
-    {
-        abort();
-    }
 }
 
-
 static inline void initSaiRedisCounterEmptyParameter(sai_s8_list_t &sai_s8_list)
 {
     sai_s8_list.list = nullptr;
@@ -1140,7 +989,7 @@ std::vector<sai_stat_id_t> queryAvailableCounterStats(const sai_object_type_t ob
 
     if (!info)
     {
-        SWSS_LOG_ERROR("Metadata info query failed, invalid object: %d", object_type);
+        SWSS_LOG_WARN("Metadata info query failed, invalid object: %d", object_type);
         return stat_list;
     }