|
|
@@ -170,11 +170,85 @@ def load_pdparams(exe, main_prog, model_dir):
|
|
|
len(vars_to_load), model_dir))
|
|
|
|
|
|
|
|
|
-def load_pretrain_weights(exe, main_prog, weights_dir, fuse_bn=False):
|
|
|
+def is_persistable(var):
|
|
|
+ import paddle.fluid as fluid
|
|
|
+ from paddle.fluid.proto.framework_pb2 import VarType
|
|
|
+
|
|
|
+ if var.desc.type() == fluid.core.VarDesc.VarType.FEED_MINIBATCH or \
|
|
|
+ var.desc.type() == fluid.core.VarDesc.VarType.FETCH_LIST or \
|
|
|
+ var.desc.type() == fluid.core.VarDesc.VarType.READER:
|
|
|
+ return False
|
|
|
+ return var.persistable
|
|
|
+
|
|
|
+
|
|
|
+def is_belong_to_optimizer(var):
|
|
|
+ import paddle.fluid as fluid
|
|
|
+ from paddle.fluid.proto.framework_pb2 import VarType
|
|
|
+
|
|
|
+ if not (isinstance(var, fluid.framework.Parameter)
|
|
|
+ or var.desc.need_check_feed()):
|
|
|
+ return is_persistable(var)
|
|
|
+ return False
|
|
|
+
|
|
|
+
|
|
|
+def load_pdopt(exe, main_prog, model_dir):
|
|
|
+ import paddle.fluid as fluid
|
|
|
+
|
|
|
+ optimizer_var_list = list()
|
|
|
+ vars_to_load = list()
|
|
|
+ import pickle
|
|
|
+ with open(osp.join(model_dir, 'model.pdopt'), 'rb') as f:
|
|
|
+ opt_dict = pickle.load(f) if six.PY2 else pickle.load(
|
|
|
+ f, encoding='latin1')
|
|
|
+ optimizer_var_list = list(
|
|
|
+ filter(is_belong_to_optimizer, main_prog.list_vars()))
|
|
|
+ exception_message = "the training process can not be resumed due to optimizer set now and last time is different. Recommend to use `pretrain_weights` instead of `resume_checkpoint`"
|
|
|
+ if len(optimizer_var_list) > 0:
|
|
|
+ for var in optimizer_var_list:
|
|
|
+ if var.name not in opt_dict:
|
|
|
+ raise Exception(
|
|
|
+ "{} is not in saved paddlex optimizer, {}".format(
|
|
|
+ var.name, exception_message))
|
|
|
+ if var.shape != opt_dict[var.name].shape:
|
|
|
+ raise Exception(
|
|
|
+ "Shape of optimizer variable {} doesn't match.(Last: {}, Now: {}), {}"
|
|
|
+ .format(var.name, opt_dict[var.name].shape,
|
|
|
+ var.shape), exception_message)
|
|
|
+ optimizer_varname_list = [var.name for var in optimizer_var_list]
|
|
|
+ for k, v in opt_dict.items():
|
|
|
+ if k not in optimizer_varname_list:
|
|
|
+ raise Exception(
|
|
|
+ "{} in saved paddlex optimizer is not in the model, {}".
|
|
|
+ format(k, exception_message))
|
|
|
+ fluid.io.set_program_state(main_prog, opt_dict)
|
|
|
+
|
|
|
+ if len(optimizer_var_list) == 0:
|
|
|
+ raise Exception(
|
|
|
+ "There is no optimizer parameters in the model, please set the optimizer!"
|
|
|
+ )
|
|
|
+ else:
|
|
|
+ logging.info(
|
|
|
+ "There are {} optimizer parameters in {} are loaded.".format(
|
|
|
+ len(optimizer_var_list), model_dir))
|
|
|
+
|
|
|
+
|
|
|
+def load_pretrain_weights(exe,
|
|
|
+ main_prog,
|
|
|
+ weights_dir,
|
|
|
+ fuse_bn=False,
|
|
|
+ resume=False):
|
|
|
if not osp.exists(weights_dir):
|
|
|
raise Exception("Path {} not exists.".format(weights_dir))
|
|
|
if osp.exists(osp.join(weights_dir, "model.pdparams")):
|
|
|
- return load_pdparams(exe, main_prog, weights_dir)
|
|
|
+ load_pdparams(exe, main_prog, weights_dir)
|
|
|
+ if resume:
|
|
|
+ if osp.exists(osp.join(weights_dir, "model.pdopt")):
|
|
|
+ load_pdopt(exe, main_prog, weights_dir)
|
|
|
+ else:
|
|
|
+ raise Exception(
|
|
|
+ "Optimizer file {} does not exist. Stop resumming training. Recommend to use `pretrain_weights` instead of `resume_checkpoint`"
|
|
|
+ .format(osp.join(weights_dir, "model.pdopt")))
|
|
|
+ return
|
|
|
import paddle.fluid as fluid
|
|
|
vars_to_load = list()
|
|
|
for var in main_prog.list_vars():
|
|
|
@@ -209,6 +283,45 @@ def load_pretrain_weights(exe, main_prog, weights_dir, fuse_bn=False):
|
|
|
len(vars_to_load), weights_dir))
|
|
|
if fuse_bn:
|
|
|
fuse_bn_weights(exe, main_prog, weights_dir)
|
|
|
+ if resume:
|
|
|
+ exception_message = "the training process can not be resumed due to optimizer set now and last time is different. Recommend to use `pretrain_weights` instead of `resume_checkpoint`"
|
|
|
+ optimizer_var_list = list(
|
|
|
+ filter(is_belong_to_optimizer, main_prog.list_vars()))
|
|
|
+ if len(optimizer_var_list) > 0:
|
|
|
+ for var in optimizer_var_list:
|
|
|
+ if not osp.exists(osp.join(weights_dir, var.name)):
|
|
|
+ raise Exception(
|
|
|
+ "Optimizer parameter {} doesn't exist, {}".format(
|
|
|
+ osp.join(weights_dir, var.name),
|
|
|
+ exception_message))
|
|
|
+ pretrained_shape = parse_param_file(
|
|
|
+ osp.join(weights_dir, var.name))
|
|
|
+ actual_shape = tuple(var.shape)
|
|
|
+ if pretrained_shape != actual_shape:
|
|
|
+ raise Exception(
|
|
|
+ "Shape of optimizer variable {} doesn't match.(Last: {}, Now: {}), {}"
|
|
|
+ .format(var.name, opt_dict[var.name].shape,
|
|
|
+ var.shape), exception_message)
|
|
|
+ optimizer_varname_list = [var.name for var in optimizer_var_list]
|
|
|
+ if os.exists(osp.join(weights_dir, 'learning_rate')
|
|
|
+ ) and 'learning_rate' not in optimizer_varname_list:
|
|
|
+ raise Exception(
|
|
|
+ "Optimizer parameter {}/learning_rate is not in the model, {}"
|
|
|
+ .format(weights_dir, exception_message))
|
|
|
+ fluid.io.load_vars(
|
|
|
+ executor=exe,
|
|
|
+ dirname=weights_dir,
|
|
|
+ main_program=main_prog,
|
|
|
+ vars=optimizer_var_list)
|
|
|
+
|
|
|
+ if len(optimizer_var_list) == 0:
|
|
|
+ raise Exception(
|
|
|
+ "There is no optimizer parameters in the model, please set the optimizer!"
|
|
|
+ )
|
|
|
+ else:
|
|
|
+ logging.info(
|
|
|
+ "There are {} optimizer parameters in {} are loaded.".format(
|
|
|
+ len(optimizer_var_list), weights_dir))
|
|
|
|
|
|
|
|
|
class EarlyStop:
|
FlyingQianMM