|
@@ -428,9 +428,7 @@ class BaseAPI:
|
|
|
|
|
|
|
|
if use_vdl:
|
|
if use_vdl:
|
|
|
# VisualDL component
|
|
# VisualDL component
|
|
|
- log_writer = LogWriter(vdl_logdir, sync_cycle=20)
|
|
|
|
|
- train_step_component = OrderedDict()
|
|
|
|
|
- eval_component = OrderedDict()
|
|
|
|
|
|
|
+ log_writer = LogWriter(vdl_logdir)
|
|
|
|
|
|
|
|
thresh = 0.0001
|
|
thresh = 0.0001
|
|
|
if early_stop:
|
|
if early_stop:
|
|
@@ -468,13 +466,7 @@ class BaseAPI:
|
|
|
|
|
|
|
|
if use_vdl:
|
|
if use_vdl:
|
|
|
for k, v in step_metrics.items():
|
|
for k, v in step_metrics.items():
|
|
|
- if k not in train_step_component.keys():
|
|
|
|
|
- with log_writer.mode('Each_Step_while_Training'
|
|
|
|
|
- ) as step_logger:
|
|
|
|
|
- train_step_component[
|
|
|
|
|
- k] = step_logger.scalar(
|
|
|
|
|
- 'Training: {}'.format(k))
|
|
|
|
|
- train_step_component[k].add_record(num_steps, v)
|
|
|
|
|
|
|
+ log_writer.add_scalar('Metrics/Training(Step): {}'.format(k), v, num_steps)
|
|
|
|
|
|
|
|
# 估算剩余时间
|
|
# 估算剩余时间
|
|
|
avg_step_time = np.mean(time_stat)
|
|
avg_step_time = np.mean(time_stat)
|
|
@@ -535,12 +527,7 @@ class BaseAPI:
|
|
|
if isinstance(v, np.ndarray):
|
|
if isinstance(v, np.ndarray):
|
|
|
if v.size > 1:
|
|
if v.size > 1:
|
|
|
continue
|
|
continue
|
|
|
- if k not in eval_component:
|
|
|
|
|
- with log_writer.mode('Each_Epoch_on_Eval_Data'
|
|
|
|
|
- ) as eval_logger:
|
|
|
|
|
- eval_component[k] = eval_logger.scalar(
|
|
|
|
|
- 'Evaluation: {}'.format(k))
|
|
|
|
|
- eval_component[k].add_record(i + 1, v)
|
|
|
|
|
|
|
+ log_writer.add_scalar("Metrics/Eval(Epoch): {}".format(k), v, i+1)
|
|
|
self.save_model(save_dir=current_save_dir)
|
|
self.save_model(save_dir=current_save_dir)
|
|
|
time_eval_one_epoch = time.time() - eval_epoch_start_time
|
|
time_eval_one_epoch = time.time() - eval_epoch_start_time
|
|
|
eval_epoch_start_time = time.time()
|
|
eval_epoch_start_time = time.time()
|