Merge branch 'feature/add_ota_resumption' into 'master'

ota: Add resumption ota feature

See merge request app-frameworks/esp-rainmaker!577

Author: shahpiyushv

components/esp_rainmaker/CHANGELOG.md

@@ -1,5 +1,15 @@
 # Changelog
 
+## 1.7.2
+
+### New Feature
+
+- Add RainMaker OTA resumption feature. The OTA process will resume downloading from the last position instead of restarting the download.
+
+OTA will check whether the MD5 values of the two consecutive files are the same. If the same, the OTA will continue; if they are different, the OTA will restart.
+Therefore, cloud support for delivering the file MD5 value is required (supported in backend version 3.0.0 and above).
+In addition, this feature requires IDF version 5.5 and above.
+
 ## 1.7.1
 
 ### New Feature

components/esp_rainmaker/Kconfig.projbuild

@@ -393,6 +393,13 @@ menu "ESP RainMaker Config"
             help
                 Delay (in minutes) before re-fetching OTA details after all retry attempts fail (for OTA using topics).
 
+        config ESP_RMAKER_HTTP_OTA_RESUMPTION
+            bool "Enable HTTP OTA resumption"
+            default y
+            depends on ESP_RMAKER_OTA_USE_HTTPS
+            help
+                If you enable this, the OTA process will resume downloading from the last position instead of restarting the download.
+
         choice ESP_RMAKER_OTA_TYPE
             prompt "OTA Update Protocol Type"
             default ESP_RMAKER_OTA_USE_HTTPS

components/esp_rainmaker/idf_component.yml

@@ -1,5 +1,5 @@
 ## IDF Component Manager Manifest File
-version: "1.7.1"
+version: "1.7.2"
 description: ESP RainMaker firmware agent
 url: https://github.com/espressif/esp-rainmaker/tree/master/components/esp_rainmaker
 repository: https://github.com/espressif/esp-rainmaker.git

components/esp_rainmaker/include/esp_rmaker_ota.h

@@ -77,6 +77,9 @@ typedef struct {
     int filesize;
     /** The firmware version of the OTA image **/
     char *fw_version;
+    /** The MD5 of the OTA File. Can be NULL if the MD5 isn't received from
+     * the ESP RainMaker Cloud */
+    char *file_md5;
     /** The OTA Job ID received from cloud **/
     char *ota_job_id;
     /** The server certificate passed in esp_rmaker_enable_ota() */

components/esp_rainmaker/src/ota/esp_rmaker_https_ota.c

@@ -10,6 +10,7 @@
 #include <esp_wifi_types.h>
 #include <esp_wifi.h>
 #include <errno.h>
+#include <nvs.h>
 #include <esp_https_ota.h>
 
 #if CONFIG_BT_ENABLED
@@ -51,6 +52,83 @@ const char *ESP_RMAKER_OTA_DEFAULT_SERVER_CERT = esp_rmaker_ota_def_cert;
 
 #define ESP_RMAKER_HTTPS_OTA_TIMEOUT_MS 5000
 
+#ifdef CONFIG_ESP_RMAKER_HTTP_OTA_RESUMPTION
+#if ESP_IDF_VERSION >= ESP_IDF_VERSION_VAL(5, 5, 0)
+#define RMAKER_OTA_HTTP_OTA_RESUMPTION
+#else
+#warning "HTTP OTA resumption, needs IDF version >= 5.5.0"
+#endif
+#endif
+
+#ifdef RMAKER_OTA_HTTP_OTA_RESUMPTION
+#define RMAKER_OTA_WRITTEN_LENGTH_NVS_NAME  "ota_writen"
+#define RMAKER_OTA_FILE_MD5_NVS_NAME  "ota_file_md5"
+static esp_err_t esp_rmaker_https_ota_get_len_and_md5_from_nvs(uint32_t *written_len, char **file_md5)
+{
+    *written_len = 0;
+    *file_md5 = NULL;
+    nvs_handle handle;
+    esp_err_t err = nvs_open_from_partition(ESP_RMAKER_NVS_PART_NAME, RMAKER_OTA_NVS_NAMESPACE, NVS_READWRITE, &handle);
+    if (err == ESP_OK) {
+        uint32_t len = 0;
+        err = nvs_get_u32(handle, RMAKER_OTA_WRITTEN_LENGTH_NVS_NAME, &len);
+        if (err == ESP_OK) {
+            *written_len = len;
+            size_t file_md5_len = 0;
+            err = nvs_get_str(handle, RMAKER_OTA_FILE_MD5_NVS_NAME, NULL, &file_md5_len);
+            if (err == ESP_OK) {
+                *file_md5 = MEM_CALLOC_EXTRAM(1, file_md5_len + 1);
+                if (!*file_md5) {
+                    err = ESP_ERR_NO_MEM;
+                } else {
+                    err = nvs_get_str(handle, RMAKER_OTA_FILE_MD5_NVS_NAME, *file_md5, &file_md5_len);
+                    if (err == ESP_OK) {
+                        (*file_md5)[file_md5_len] = '\0';
+                    } else {
+                        free(*file_md5);
+                        *file_md5 = NULL;
+                    }
+                }
+            }
+        }
+        nvs_close(handle);
+    }
+    return err;
+}
+
+static esp_err_t esp_rmaker_https_ota_set_len_and_md5_to_nvs(uint32_t written_len, char *file_md5)
+{
+    nvs_handle handle;
+    esp_err_t err = nvs_open_from_partition(ESP_RMAKER_NVS_PART_NAME, RMAKER_OTA_NVS_NAMESPACE, NVS_READWRITE, &handle);
+    if (err == ESP_OK) {
+        err = nvs_set_u32(handle, RMAKER_OTA_WRITTEN_LENGTH_NVS_NAME, written_len);
+        if (err == ESP_OK) {
+            if (file_md5) {
+                err = nvs_set_str(handle, RMAKER_OTA_FILE_MD5_NVS_NAME, file_md5);
+            }
+            if (err == ESP_OK) {
+                nvs_commit(handle);
+            }
+        }
+        nvs_close(handle);
+    }
+    return err;
+}
+
+static esp_err_t esp_rmaker_https_ota_cleanup_ota_cfg_from_nvs(void)
+{
+    nvs_handle handle;
+    esp_err_t err = nvs_open_from_partition(ESP_RMAKER_NVS_PART_NAME, RMAKER_OTA_NVS_NAMESPACE, NVS_READWRITE, &handle);
+    if (err == ESP_OK) {
+        nvs_erase_key(handle, RMAKER_OTA_WRITTEN_LENGTH_NVS_NAME);
+        nvs_erase_key(handle, RMAKER_OTA_FILE_MD5_NVS_NAME);
+        nvs_commit(handle);
+        nvs_close(handle);
+    }
+    return err;
+}
+#endif
+
 static esp_err_t esp_rmaker_ota_use_https(esp_rmaker_ota_handle_t ota_handle, esp_rmaker_ota_data_t *ota_data, char *err_desc, size_t err_desc_size)
 {
     int buffer_size_tx = DEF_HTTP_TX_BUFFER_SIZE;
@@ -85,6 +163,33 @@ static esp_err_t esp_rmaker_ota_use_https(esp_rmaker_ota_handle_t ota_handle, es
     esp_https_ota_config_t ota_config = {
         .http_config = &config,
     };
+#ifdef RMAKER_OTA_HTTP_OTA_RESUMPTION
+    /* Check if file md5 is present and match with the one in the ota_data, if yes, resume the OTA;
+    otherwise, start from the beginning and set the written length 0 and file md5 to NVS */
+    if (ota_data->file_md5) {
+        char *file_md5 = NULL;
+        uint32_t written_len = 0;
+        bool resume_ota = false;
+        if (esp_rmaker_https_ota_get_len_and_md5_from_nvs(&written_len, &file_md5) == ESP_OK) {
+            if (strncmp(file_md5, ota_data->file_md5, strlen(ota_data->file_md5)) != 0) {
+                ESP_LOGW(TAG, "File MD5 mismatch, seems a new firmware, not resuming OTA");
+            } else {
+                resume_ota = true;
+                ota_config.ota_resumption = true;
+                ota_config.ota_image_bytes_written = written_len;
+            }
+            free(file_md5);
+        }
+        if (!resume_ota) {
+            /* Start from the beginning and set the written length 0 and file md5 to NVS */
+            if (esp_rmaker_https_ota_set_len_and_md5_to_nvs(0, ota_data->file_md5) != ESP_OK) {
+                ESP_LOGE(TAG, "Failed to set written length 0 and file MD5 to NVS");
+            }
+        }
+    } else {
+        ESP_LOGW(TAG, "File MD5 not present, not resuming OTA because can not verify the already downloaded firmware is the same as the new firmware, please upgrade the backend to support it");
+    }
+#endif
     /* Using a warning just to highlight the message */
     ESP_LOGW(TAG, "Starting OTA. This may take time.");
     esp_https_ota_handle_t https_ota_handle = NULL;
@@ -158,6 +263,15 @@ static esp_err_t esp_rmaker_ota_use_https(esp_rmaker_ota_handle_t ota_handle, es
             ESP_LOGI(TAG, "Image bytes read: %d", esp_https_ota_get_image_len_read(https_ota_handle));
             count = 0;
         }
+#ifdef RMAKER_OTA_HTTP_OTA_RESUMPTION
+        /* if file md5 is present, save the written length to NVS */
+        if (ota_data->file_md5) {
+            /* file md5 is present, only set the written length to NVS */
+            if (esp_rmaker_https_ota_set_len_and_md5_to_nvs(esp_https_ota_get_image_len_read(https_ota_handle), NULL) != ESP_OK) {
+                ESP_LOGE(TAG, "Failed to save OTA written length to NVS");
+            }
+        }
+#endif
 #ifdef CONFIG_ESP_RMAKER_OTA_PROGRESS_SUPPORT
         int image_size = esp_https_ota_get_image_size(https_ota_handle);
         int read_size = esp_https_ota_get_image_len_read(https_ota_handle);
@@ -186,7 +300,11 @@ static esp_err_t esp_rmaker_ota_use_https(esp_rmaker_ota_handle_t ota_handle, es
         err = ESP_FAIL;
         goto ota_end;
     }
-
+#ifdef RMAKER_OTA_HTTP_OTA_RESUMPTION
+    if (esp_rmaker_https_ota_cleanup_ota_cfg_from_nvs() != ESP_OK) {
+        ESP_LOGE(TAG, "Failed to cleanup OTA config from NVS");
+    }
+#endif
     /* Report completion before finishing */
     esp_rmaker_ota_report_status(ota_handle, OTA_STATUS_IN_PROGRESS, "Firmware Image download complete");
 

components/esp_rainmaker/src/ota/esp_rmaker_ota.c

@@ -148,6 +148,7 @@ void esp_rmaker_ota_common_cb(void *priv)
 #endif
         .filesize = ota->filesize,
         .fw_version = ota->fw_version,
+        .file_md5 = ota->file_md5,
         .ota_job_id = (char *)ota->transient_priv,
         .server_cert = ota->server_cert,
         .priv = ota->priv,

components/esp_rainmaker/src/ota/esp_rmaker_ota_internal.h

@@ -33,6 +33,7 @@ typedef struct {
     const char *server_cert;
     char *url;
     char *fw_version;
+    char *file_md5;
 #ifdef CONFIG_ESP_RMAKER_OTA_USE_MQTT
     char *stream_id;
 #endif

components/esp_rainmaker/src/ota/esp_rmaker_ota_using_topics.c

@@ -195,6 +195,10 @@ void esp_rmaker_ota_finish_using_topics(esp_rmaker_ota_t *ota)
         free(ota->fw_version);
         ota->fw_version = NULL;
     }
+    if (ota->file_md5) {
+        free(ota->file_md5);
+        ota->file_md5 = NULL;
+    }
     ota->ota_in_progress = false;
 }
 static void ota_url_handler(const char *topic, void *payload, size_t payload_len, void *priv_data)
@@ -215,12 +219,13 @@ static void ota_url_handler(const char *topic, void *payload, size_t payload_len
        {
        "ota_job_id": "<ota_job_id>",
        "url": "<fw_url>",
+       "file_md5": "<file_md5>",
        "fw_version": "<fw_version>",
        "filesize": <size_in_bytes>
        }
     */
     jparse_ctx_t jctx;
-    char *url = NULL, *ota_job_id = NULL, *fw_version = NULL;
+    char *url = NULL, *ota_job_id = NULL, *fw_version = NULL, *file_md5 = NULL;
 #ifdef CONFIG_ESP_RMAKER_OTA_USE_MQTT
     char *stream_id = NULL;
 #endif
@@ -270,7 +275,19 @@ static void ota_url_handler(const char *topic, void *payload, size_t payload_len
     }
     json_obj_get_string(&jctx, "url", url, len);
     ESP_LOGI(TAG, "URL: %s", url);
-
+    len = 0;
+    ret = json_obj_get_strlen(&jctx, "file_md5", &len);
+    if (ret == ESP_OK) {
+        len++; /* Increment for NULL character */
+        file_md5 = MEM_CALLOC_EXTRAM(1, len);
+        if (!file_md5) {
+            ESP_LOGE(TAG, "Aborted. File MD5 memory allocation failed");
+            esp_rmaker_ota_report_status(ota_handle, OTA_STATUS_FAILED, "Aborted. File MD5 memory allocation failed");
+            goto end;
+        }
+        json_obj_get_string(&jctx, "file_md5", file_md5, len);
+        ESP_LOGI(TAG, "File MD5: %s", file_md5);
+    }
 #ifdef CONFIG_ESP_RMAKER_OTA_USE_MQTT
     len = 0;
     ret = json_obj_get_strlen(&jctx, "stream_id", &len);
@@ -330,6 +347,7 @@ static void ota_url_handler(const char *topic, void *payload, size_t payload_len
     ota->stream_id = stream_id;
 #endif
     ota->fw_version = fw_version;
+    ota->file_md5 = file_md5;
     ota->filesize = filesize;
     ota->ota_in_progress = true;
     if (esp_rmaker_work_queue_add_task(esp_rmaker_ota_common_cb, ota) != ESP_OK) {
@@ -348,6 +366,9 @@ static void ota_url_handler(const char *topic, void *payload, size_t payload_len
     if (fw_version) {
         free(fw_version);
     }
+    if (file_md5) {
+        free(file_md5);
+    }
     esp_rmaker_ota_finish_using_topics(ota);
     json_parse_end(&jctx);
     return;