| | |
| | | for iepoch in range(start_epoch, trainer_options.max_epoch + 1): |
| | | if iepoch != start_epoch: |
| | | logging.info( |
| | | "{}/{}epoch started. Estimated time to finish: {}".format( |
| | | "{}/{}epoch started. Estimated time to finish: {} hours".format( |
| | | iepoch, |
| | | trainer_options.max_epoch, |
| | | humanfriendly.format_timespan( |
| | | (time.perf_counter() - start_time) |
| | | / (iepoch - start_epoch) |
| | | * (trainer_options.max_epoch - iepoch + 1) |
| | | ), |
| | | (time.perf_counter() - start_time) / 3600.0 / (iepoch - start_epoch) * ( |
| | | trainer_options.max_epoch - iepoch + 1), |
| | | ) |
| | | ) |
| | | else: |
| | |
| | | ], |
| | | "scaler": scaler.state_dict() if scaler is not None else None, |
| | | "ema_model": model.encoder.ema.model.state_dict() |
| | | if hasattr(model.encoder, "ema") and model.encoder.ema is not None else None, |
| | | if hasattr(model, "encoder") and hasattr(model.encoder, "ema") and model.encoder.ema is not None else None, |
| | | }, |
| | | buffer, |
| | | ) |