deep copy state dict for checkpoint

Summary:
deep copy the state dict for sending checkpoint because if the replica moves to the next step, the state dict can change before the checkpoint is sent

Author: tushar00jain

torchft/manager.py

@@ -26,6 +26,7 @@
 """
 
 import concurrent.futures
+import copy
 import logging
 import os
 import socket
@@ -646,7 +647,7 @@ def _async_quorum(
                             self._checkpoint_transport.send_checkpoint(
                                 dst_ranks=quorum.recover_dst_replica_ranks,
                                 step=max_step,
-                                state_dict=self._manager_state_dict(),
+                                state_dict=copy.deepcopy(self._manager_state_dict()),
                                 timeout=self._timeout,
                             )