rm train deamon

paddlex/modules/anomaly_detection/trainer.py

@@ -18,7 +18,7 @@ import glob
 from pathlib import Path
 import lazy_paddle as paddle
 
-from ..base import BaseTrainer, BaseTrainDeamon
+from ..base import BaseTrainer
 from ...utils.config import AttrDict
 from .model_list import MODELS
 
@@ -28,17 +28,6 @@ class UadTrainer(BaseTrainer):
 
     entities = MODELS
 
-    def build_deamon(self, config: AttrDict) -> "SegTrainDeamon":
-        """build deamon thread for saving training outputs timely
-
-        Args:
-            config (AttrDict): PaddleX pipeline config, which is loaded from pipeline yaml file.
-
-        Returns:
-            SegTrainDeamon: the training deamon thread object for saving training outputs timely.
-        """
-        return SegTrainDeamon(config)
-
     def update_config(self):
         """update training config"""
         self.pdx_config.update_dataset(self.global_config.dataset_dir, "SegDataset")
@@ -81,156 +70,3 @@ class UadTrainer(BaseTrainer):
             train_args["save_interval"] = self.train_config.eval_interval
         train_args["dy2st"] = self.train_config.get("dy2st", False)
         return train_args
-
-
-class SegTrainDeamon(BaseTrainDeamon):
-    """SegTrainResultDemon"""
-
-    last_k = 1
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-
-    def get_the_pdparams_suffix(self):
-        """get the suffix of pdparams file"""
-        return "pdparams"
-
-    def get_the_pdema_suffix(self):
-        """get the suffix of pdema file"""
-        return "pdema"
-
-    def get_the_pdopt_suffix(self):
-        """get the suffix of pdopt file"""
-        return "pdopt"
-
-    def get_the_pdstates_suffix(self):
-        """get the suffix of pdstates file"""
-        return "pdstates"
-
-    def get_ith_ckp_prefix(self, epoch_id):
-        """get the prefix of the epoch_id checkpoint file"""
-        return f"iter_{epoch_id}/model"
-
-    def get_best_ckp_prefix(self):
-        """get the prefix of the best checkpoint file"""
-        return "best_model/model"
-
-    def get_score(self, pdstates_path):
-        """get the score by pdstates file"""
-        if not Path(pdstates_path).exists():
-            return 0
-        return paddle.load(pdstates_path)["mIoU"]
-
-    def get_epoch_id_by_pdparams_prefix(self, pdparams_dir):
-        """get the epoch_id by pdparams file"""
-        return int(pdparams_dir.parent.name.split("_")[-1])
-
-    def update_result(self, result, train_output):
-        """update every result"""
-        train_output = Path(train_output).resolve()
-        config_path = train_output.joinpath("config.yaml").resolve()
-        if not config_path.exists():
-            return result
-
-        model_name = result["model_name"]
-        if (
-            model_name in self.config_recorder
-            and self.config_recorder[model_name] != config_path
-        ):
-            result["models"] = self.init_model_pkg()
-        result["config"] = config_path
-        self.config_recorder[model_name] = config_path
-
-        result["visualdl_log"] = self.update_vdl_log(train_output)
-        result["label_dict"] = self.update_label_dict(train_output)
-
-        model = self.get_model(result["model_name"], config_path)
-
-        params_path_list = list(
-            train_output.glob(
-                ".".join(
-                    [self.get_ith_ckp_prefix("[0-9]*"), self.get_the_pdparams_suffix()]
-                )
-            )
-        )
-        iter_ids = []
-        for params_path in params_path_list:
-            iter_id = self.get_epoch_id_by_pdparams_prefix(params_path)
-            iter_ids.append(iter_id)
-        iter_ids.sort()
-        # TODO(gaotingquan): how to avoid that the latest ckp files is being saved
-        # epoch_ids = epoch_ids[:-1]
-        for i in range(1, self.last_k + 1):
-            if len(iter_ids) < i:
-                break
-            self.update_models(
-                result,
-                model,
-                train_output,
-                f"last_{i}",
-                self.get_ith_ckp_prefix(iter_ids[-i]),
-            )
-        self.update_models(
-            result, model, train_output, "best", self.get_best_ckp_prefix()
-        )
-        return result
-
-    def update_models(self, result, model, train_output, model_key, ckp_prefix):
-        """update info of the models to be saved"""
-        pdparams = train_output.joinpath(
-            ".".join([ckp_prefix, self.get_the_pdparams_suffix()])
-        )
-        if pdparams.exists():
-            recorder_key = f"{train_output.name}_{model_key}"
-            if (
-                model_key != "best"
-                and recorder_key in self.model_recorder
-                and self.model_recorder[recorder_key] == pdparams
-            ):
-                return
-
-            self.model_recorder[recorder_key] = pdparams
-
-            pdema = ""
-            pdema_suffix = self.get_the_pdema_suffix()
-            if pdema_suffix:
-                pdema = pdparams.parents[1].joinpath(
-                    ".".join([ckp_prefix, pdema_suffix])
-                )
-                if not pdema.exists():
-                    pdema = ""
-
-            pdopt = ""
-            pdopt_suffix = self.get_the_pdopt_suffix()
-            if pdopt_suffix:
-                pdopt = pdparams.parents[1].joinpath(
-                    ".".join([ckp_prefix, pdopt_suffix])
-                )
-                if not pdopt.exists():
-                    pdopt = ""
-
-            pdstates = ""
-            pdstates_suffix = self.get_the_pdstates_suffix()
-            if pdstates_suffix:
-                pdstates = pdparams.parents[1].joinpath(
-                    ".".join([ckp_prefix, pdstates_suffix])
-                )
-                if not pdstates.exists():
-                    pdstates = ""
-
-            score = self.get_score(Path(pdstates).resolve().as_posix())
-
-            result["models"][model_key] = {
-                "score": score,
-                "pdparams": pdparams,
-                "pdema": pdema,
-                "pdopt": pdopt,
-                "pdstates": pdstates,
-            }
-
-            self.update_inference_model(
-                model,
-                pdparams,
-                train_output.joinpath(f"{ckp_prefix}"),
-                result["models"][model_key],
-            )

paddlex/modules/base/__init__.py

@@ -13,7 +13,7 @@
 # limitations under the License.
 
 from .dataset_checker import build_dataset_checker, BaseDatasetChecker
-from .trainer import build_trainer, BaseTrainer, BaseTrainDeamon
+from .trainer import build_trainer, BaseTrainer
 from .evaluator import build_evaluater, BaseEvaluator
 from .exportor import build_exportor, BaseExportor
 from .predictor import (

paddlex/modules/base/trainer/__init__.py

@@ -14,4 +14,3 @@
 
 
 from .trainer import build_trainer, BaseTrainer
-from .train_deamon import BaseTrainDeamon

paddlex/modules/base/trainer/trainer.py

@@ -51,7 +51,6 @@ class BaseTrainer(ABC, metaclass=AutoRegisterABCMetaClass):
         self.train_config = config.Train
         self.benchmark_config = config.get("Benchmark", None)
 
-        self.deamon = self.build_deamon(self.config)
         self.pdx_config, self.pdx_model = build_model(self.global_config.model)
 
     def train(self, *args, **kwargs):
@@ -68,8 +67,6 @@ class BaseTrainer(ABC, metaclass=AutoRegisterABCMetaClass):
         ), f"Encountered an unexpected error({train_result.returncode}) in \
 training!"
 
-        self.deamon.stop()
-
     def dump_config(self, config_file_path: str = None):
         """dump the config
 
@@ -96,11 +93,6 @@ training!"
         )
 
     @abstractmethod
-    def build_deamon(self):
-        """build deamon thread for saving training outputs timely"""
-        raise NotImplementedError
-
-    @abstractmethod
     def update_config(self):
         """update training config"""
         raise NotImplementedError

paddlex/modules/image_classification/trainer.py

@@ -17,7 +17,7 @@ import shutil
 import lazy_paddle as paddle
 from pathlib import Path
 
-from ..base import BaseTrainer, BaseTrainDeamon
+from ..base import BaseTrainer
 from .model_list import MODELS
 from ...utils.config import AttrDict
 
@@ -36,17 +36,6 @@ class ClsTrainer(BaseTrainer):
         dst_label_dict_path = Path(self.global_config.output).joinpath("label_dict.txt")
         shutil.copyfile(src_label_dict_path, dst_label_dict_path)
 
-    def build_deamon(self, config: AttrDict) -> "ClsTrainDeamon":
-        """build deamon thread for saving training outputs timely
-
-        Args:
-            config (AttrDict): PaddleX pipeline config, which is loaded from pipeline yaml file.
-
-        Returns:
-            ClsTrainDeamon: the training deamon thread object for saving training outputs timely.
-        """
-        return ClsTrainDeamon(config)
-
     def update_config(self):
         """update training config"""
         if self.train_config.log_interval:
@@ -92,51 +81,3 @@ class ClsTrainer(BaseTrainer):
             train_args["resume_path"] = self.train_config.resume_path
         train_args["dy2st"] = self.train_config.get("dy2st", False)
         return train_args
-
-
-class ClsTrainDeamon(BaseTrainDeamon):
-    """ClsTrainResultDemon"""
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-
-    def get_the_pdparams_suffix(self):
-        """get the suffix of pdparams file"""
-        return "pdparams"
-
-    def get_the_pdema_suffix(self):
-        """get the suffix of pdema file"""
-        return "pdema"
-
-    def get_the_pdopt_suffix(self):
-        """get the suffix of pdopt file"""
-        return "pdopt"
-
-    def get_the_pdstates_suffix(self):
-        """get the suffix of pdstates file"""
-        return "pdstates"
-
-    def get_ith_ckp_prefix(self, epoch_id):
-        """get the prefix of the epoch_id checkpoint file"""
-        return f"epoch_{epoch_id}"
-
-    def get_best_ckp_prefix(self):
-        """get the prefix of the best checkpoint file"""
-        return "best_model"
-
-    def get_score(self, pdstates_path):
-        """get the score by pdstates file"""
-        if not Path(pdstates_path).exists():
-            return 0
-        return paddle.load(pdstates_path)["metric"]
-
-    def get_epoch_id_by_pdparams_prefix(self, pdparams_prefix):
-        """get the epoch_id by pdparams file"""
-        return int(pdparams_prefix.split("_")[-1])
-
-    def update_label_dict(self, train_output):
-        """update label dict"""
-        dict_path = train_output.joinpath("label_dict.txt")
-        if not dict_path.exists():
-            return ""
-        return dict_path

paddlex/modules/multilabel_classification/trainer.py

@@ -17,7 +17,7 @@ import shutil
 import lazy_paddle as paddle
 from pathlib import Path
 
-from ..base import BaseTrainer, BaseTrainDeamon
+from ..base import BaseTrainer
 from .model_list import MODELS
 from ...utils.config import AttrDict
 
@@ -36,17 +36,6 @@ class MLClsTrainer(BaseTrainer):
         dst_label_dict_path = Path(self.global_config.output).joinpath("label_dict.txt")
         shutil.copyfile(src_label_dict_path, dst_label_dict_path)
 
-    def build_deamon(self, config: AttrDict) -> "ClsTrainDeamon":
-        """build deamon thread for saving training outputs timely
-
-        Args:
-            config (AttrDict): PaddleX pipeline config, which is loaded from pipeline yaml file.
-
-        Returns:
-            ClsTrainDeamon: the training deamon thread object for saving training outputs timely.
-        """
-        return ClsTrainDeamon(config)
-
     def update_config(self):
         """update training config"""
         if self.train_config.log_interval:
@@ -95,51 +84,3 @@ class MLClsTrainer(BaseTrainer):
             train_args["resume_path"] = self.train_config.resume_path
         train_args["dy2st"] = self.train_config.get("dy2st", False)
         return train_args
-
-
-class ClsTrainDeamon(BaseTrainDeamon):
-    """ClsTrainResultDemon"""
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-
-    def get_the_pdparams_suffix(self):
-        """get the suffix of pdparams file"""
-        return "pdparams"
-
-    def get_the_pdema_suffix(self):
-        """get the suffix of pdema file"""
-        return "pdema"
-
-    def get_the_pdopt_suffix(self):
-        """get the suffix of pdopt file"""
-        return "pdopt"
-
-    def get_the_pdstates_suffix(self):
-        """get the suffix of pdstates file"""
-        return "pdstates"
-
-    def get_ith_ckp_prefix(self, epoch_id):
-        """get the prefix of the epoch_id checkpoint file"""
-        return f"epoch_{epoch_id}"
-
-    def get_best_ckp_prefix(self):
-        """get the prefix of the best checkpoint file"""
-        return "best_model"
-
-    def get_score(self, pdstates_path):
-        """get the score by pdstates file"""
-        if not Path(pdstates_path).exists():
-            return 0
-        return paddle.load(pdstates_path)["metric"]
-
-    def get_epoch_id_by_pdparams_prefix(self, pdparams_prefix):
-        """get the epoch_id by pdparams file"""
-        return int(pdparams_prefix.split("_")[-1])
-
-    def update_label_dict(self, train_output):
-        """update label dict"""
-        dict_path = train_output.joinpath("label_dict.txt")
-        if not dict_path.exists():
-            return ""
-        return dict_path

paddlex/modules/object_detection/trainer.py

@@ -16,7 +16,7 @@
 from pathlib import Path
 import lazy_paddle as paddle
 
-from ..base import BaseTrainer, BaseTrainDeamon
+from ..base import BaseTrainer
 from ...utils.config import AttrDict
 from ...utils import logging
 from .model_list import MODELS
@@ -27,17 +27,6 @@ class DetTrainer(BaseTrainer):
 
     entities = MODELS
 
-    def build_deamon(self, config: AttrDict) -> "DetTrainDeamon":
-        """build deamon thread for saving training outputs timely
-
-        Args:
-            config (AttrDict): PaddleX pipeline config, which is loaded from pipeline yaml file.
-
-        Returns:
-            DetTrainDeamon: the training deamon thread object for saving training outputs timely.
-        """
-        return DetTrainDeamon(config)
-
     def _update_dataset(self):
         """update dataset settings"""
         self.pdx_config.update_dataset(self.global_config.dataset_dir, "COCODetDataset")
@@ -95,44 +84,3 @@ class DetTrainer(BaseTrainer):
             train_args["resume_path"] = self.train_config.resume_path
         train_args["dy2st"] = self.train_config.get("dy2st", False)
         return train_args
-
-
-class DetTrainDeamon(BaseTrainDeamon):
-    """DetTrainResultDemon"""
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-
-    def get_the_pdparams_suffix(self):
-        """get the suffix of pdparams file"""
-        return "pdparams"
-
-    def get_the_pdema_suffix(self):
-        """get the suffix of pdema file"""
-        return "pdema"
-
-    def get_the_pdopt_suffix(self):
-        """get the suffix of pdopt file"""
-        return "pdopt"
-
-    def get_the_pdstates_suffix(self):
-        """get the suffix of pdstates file"""
-        return "pdstates"
-
-    def get_ith_ckp_prefix(self, epoch_id):
-        """get the prefix of the epoch_id checkpoint file"""
-        return f"{epoch_id}"
-
-    def get_best_ckp_prefix(self):
-        """get the prefix of the best checkpoint file"""
-        return "best_model"
-
-    def get_score(self, pdstates_path):
-        """get the score by pdstates file"""
-        if not Path(pdstates_path).exists():
-            return 0
-        return paddle.load(pdstates_path)["metric"]
-
-    def get_epoch_id_by_pdparams_prefix(self, pdparams_prefix):
-        """get the epoch_id by pdparams file"""
-        return int(pdparams_prefix)

paddlex/modules/semantic_segmentation/trainer.py

@@ -18,7 +18,7 @@ import glob
 from pathlib import Path
 import lazy_paddle as paddle
 
-from ..base import BaseTrainer, BaseTrainDeamon
+from ..base import BaseTrainer
 from ...utils.config import AttrDict
 from .model_list import MODELS
 
@@ -28,17 +28,6 @@ class SegTrainer(BaseTrainer):
 
     entities = MODELS
 
-    def build_deamon(self, config: AttrDict) -> "SegTrainDeamon":
-        """build deamon thread for saving training outputs timely
-
-        Args:
-            config (AttrDict): PaddleX pipeline config, which is loaded from pipeline yaml file.
-
-        Returns:
-            SegTrainDeamon: the training deamon thread object for saving training outputs timely.
-        """
-        return SegTrainDeamon(config)
-
     def update_config(self):
         """update training config"""
         self.pdx_config.update_dataset(self.global_config.dataset_dir, "SegDataset")
@@ -81,156 +70,3 @@ class SegTrainer(BaseTrainer):
             train_args["save_interval"] = self.train_config.eval_interval
         train_args["dy2st"] = self.train_config.get("dy2st", False)
         return train_args
-
-
-class SegTrainDeamon(BaseTrainDeamon):
-    """SegTrainResultDemon"""
-
-    last_k = 1
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-
-    def get_the_pdparams_suffix(self):
-        """get the suffix of pdparams file"""
-        return "pdparams"
-
-    def get_the_pdema_suffix(self):
-        """get the suffix of pdema file"""
-        return "pdema"
-
-    def get_the_pdopt_suffix(self):
-        """get the suffix of pdopt file"""
-        return "pdopt"
-
-    def get_the_pdstates_suffix(self):
-        """get the suffix of pdstates file"""
-        return "pdstates"
-
-    def get_ith_ckp_prefix(self, epoch_id):
-        """get the prefix of the epoch_id checkpoint file"""
-        return f"iter_{epoch_id}/model"
-
-    def get_best_ckp_prefix(self):
-        """get the prefix of the best checkpoint file"""
-        return "best_model/model"
-
-    def get_score(self, pdstates_path):
-        """get the score by pdstates file"""
-        if not Path(pdstates_path).exists():
-            return 0
-        return paddle.load(pdstates_path)["mIoU"]
-
-    def get_epoch_id_by_pdparams_prefix(self, pdparams_dir):
-        """get the epoch_id by pdparams file"""
-        return int(pdparams_dir.parent.name.split("_")[-1])
-
-    def update_result(self, result, train_output):
-        """update every result"""
-        train_output = Path(train_output).resolve()
-        config_path = train_output.joinpath("config.yaml").resolve()
-        if not config_path.exists():
-            return result
-
-        model_name = result["model_name"]
-        if (
-            model_name in self.config_recorder
-            and self.config_recorder[model_name] != config_path
-        ):
-            result["models"] = self.init_model_pkg()
-        result["config"] = config_path
-        self.config_recorder[model_name] = config_path
-
-        result["visualdl_log"] = self.update_vdl_log(train_output)
-        result["label_dict"] = self.update_label_dict(train_output)
-
-        model = self.get_model(result["model_name"], config_path)
-
-        params_path_list = list(
-            train_output.glob(
-                ".".join(
-                    [self.get_ith_ckp_prefix("[0-9]*"), self.get_the_pdparams_suffix()]
-                )
-            )
-        )
-        iter_ids = []
-        for params_path in params_path_list:
-            iter_id = self.get_epoch_id_by_pdparams_prefix(params_path)
-            iter_ids.append(iter_id)
-        iter_ids.sort()
-        # TODO(gaotingquan): how to avoid that the latest ckp files is being saved
-        # epoch_ids = epoch_ids[:-1]
-        for i in range(1, self.last_k + 1):
-            if len(iter_ids) < i:
-                break
-            self.update_models(
-                result,
-                model,
-                train_output,
-                f"last_{i}",
-                self.get_ith_ckp_prefix(iter_ids[-i]),
-            )
-        self.update_models(
-            result, model, train_output, "best", self.get_best_ckp_prefix()
-        )
-        return result
-
-    def update_models(self, result, model, train_output, model_key, ckp_prefix):
-        """update info of the models to be saved"""
-        pdparams = train_output.joinpath(
-            ".".join([ckp_prefix, self.get_the_pdparams_suffix()])
-        )
-        if pdparams.exists():
-            recorder_key = f"{train_output.name}_{model_key}"
-            if (
-                model_key != "best"
-                and recorder_key in self.model_recorder
-                and self.model_recorder[recorder_key] == pdparams
-            ):
-                return
-
-            self.model_recorder[recorder_key] = pdparams
-
-            pdema = ""
-            pdema_suffix = self.get_the_pdema_suffix()
-            if pdema_suffix:
-                pdema = pdparams.parents[1].joinpath(
-                    ".".join([ckp_prefix, pdema_suffix])
-                )
-                if not pdema.exists():
-                    pdema = ""
-
-            pdopt = ""
-            pdopt_suffix = self.get_the_pdopt_suffix()
-            if pdopt_suffix:
-                pdopt = pdparams.parents[1].joinpath(
-                    ".".join([ckp_prefix, pdopt_suffix])
-                )
-                if not pdopt.exists():
-                    pdopt = ""
-
-            pdstates = ""
-            pdstates_suffix = self.get_the_pdstates_suffix()
-            if pdstates_suffix:
-                pdstates = pdparams.parents[1].joinpath(
-                    ".".join([ckp_prefix, pdstates_suffix])
-                )
-                if not pdstates.exists():
-                    pdstates = ""
-
-            score = self.get_score(Path(pdstates).resolve().as_posix())
-
-            result["models"][model_key] = {
-                "score": score,
-                "pdparams": pdparams,
-                "pdema": pdema,
-                "pdopt": pdopt,
-                "pdstates": pdstates,
-            }
-
-            self.update_inference_model(
-                model,
-                pdparams,
-                train_output.joinpath(f"{ckp_prefix}"),
-                result["models"][model_key],
-            )

paddlex/modules/table_recognition/trainer.py

@@ -17,7 +17,7 @@ import os
 from pathlib import Path
 import lazy_paddle as paddle
 
-from ..base import BaseTrainer, BaseTrainDeamon
+from ..base import BaseTrainer
 from ...utils.config import AttrDict
 from .model_list import MODELS
 
@@ -27,17 +27,6 @@ class TableRecTrainer(BaseTrainer):
 
     entities = MODELS
 
-    def build_deamon(self, config: AttrDict) -> "TableRecTrainDeamon":
-        """build deamon thread for saving training outputs timely
-
-        Args:
-            config (AttrDict): PaddleX pipeline config, which is loaded from pipeline yaml file.
-
-        Returns:
-            TableRecTrainDeamon: the training deamon thread object for saving training outputs timely.
-        """
-        return TableRecTrainDeamon(config)
-
     def update_config(self):
         """update training config"""
         if self.train_config.log_interval:
@@ -80,44 +69,3 @@ class TableRecTrainer(BaseTrainer):
             "device": self.get_device(),
             "dy2st": self.train_config.get("dy2st", False),
         }
-
-
-class TableRecTrainDeamon(BaseTrainDeamon):
-    """TableRecTrainDeamon"""
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-
-    def get_the_pdparams_suffix(self):
-        """get the suffix of pdparams file"""
-        return "pdparams"
-
-    def get_the_pdema_suffix(self):
-        """get the suffix of pdema file"""
-        return "pdema"
-
-    def get_the_pdopt_suffix(self):
-        """get the suffix of pdopt file"""
-        return "pdopt"
-
-    def get_the_pdstates_suffix(self):
-        """get the suffix of pdstates file"""
-        return "states"
-
-    def get_ith_ckp_prefix(self, epoch_id):
-        """get the prefix of the epoch_id checkpoint file"""
-        return f"iter_epoch_{epoch_id}"
-
-    def get_best_ckp_prefix(self):
-        """get the prefix of the best checkpoint file"""
-        return "best_accuracy"
-
-    def get_score(self, pdstates_path):
-        """get the score by pdstates file"""
-        if not Path(pdstates_path).exists():
-            return 0
-        return paddle.load(pdstates_path)["best_model_dict"]["acc"]
-
-    def get_epoch_id_by_pdparams_prefix(self, pdparams_prefix):
-        """get the epoch_id by pdparams file"""
-        return int(pdparams_prefix.split(".")[0].split("_")[-1])

paddlex/modules/text_detection/trainer.py

@@ -17,7 +17,7 @@ import os
 from pathlib import Path
 import lazy_paddle as paddle
 
-from ..base import BaseTrainer, BaseTrainDeamon
+from ..base import BaseTrainer
 from ...utils.config import AttrDict
 from .model_list import MODELS
 
@@ -27,17 +27,6 @@ class TextDetTrainer(BaseTrainer):
 
     entities = MODELS
 
-    def build_deamon(self, config: AttrDict) -> "TextDetTrainDeamon":
-        """build deamon thread for saving training outputs timely
-
-        Args:
-            config (AttrDict): PaddleX pipeline config, which is loaded from pipeline yaml file.
-
-        Returns:
-            TextDetTrainDeamon: the training deamon thread object for saving training outputs timely.
-        """
-        return TextDetTrainDeamon(config)
-
     def update_config(self):
         """update training config"""
         if self.train_config.log_interval:
@@ -78,44 +67,3 @@ class TextDetTrainer(BaseTrainer):
             "device": self.get_device(),
             "dy2st": self.train_config.get("dy2st", False),
         }
-
-
-class TextDetTrainDeamon(BaseTrainDeamon):
-    """TableRecTrainDeamon"""
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-
-    def get_the_pdparams_suffix(self):
-        """get the suffix of pdparams file"""
-        return "pdparams"
-
-    def get_the_pdema_suffix(self):
-        """get the suffix of pdema file"""
-        return "pdema"
-
-    def get_the_pdopt_suffix(self):
-        """get the suffix of pdopt file"""
-        return "pdopt"
-
-    def get_the_pdstates_suffix(self):
-        """get the suffix of pdstates file"""
-        return "states"
-
-    def get_ith_ckp_prefix(self, epoch_id):
-        """get the prefix of the epoch_id checkpoint file"""
-        return f"iter_epoch_{epoch_id}"
-
-    def get_best_ckp_prefix(self):
-        """get the prefix of the best checkpoint file"""
-        return "best_accuracy"
-
-    def get_score(self, pdstates_path):
-        """get the score by pdstates file"""
-        if not Path(pdstates_path).exists():
-            return 0
-        return paddle.load(pdstates_path)["best_model_dict"]["hmean"]
-
-    def get_epoch_id_by_pdparams_prefix(self, pdparams_prefix):
-        """get the epoch_id by pdparams file"""
-        return int(pdparams_prefix.split(".")[0].split("_")[-1])

paddlex/modules/text_recognition/trainer.py

@@ -18,7 +18,7 @@ import shutil
 from pathlib import Path
 import lazy_paddle as paddle
 
-from ..base import BaseTrainer, BaseTrainDeamon
+from ..base import BaseTrainer
 from ...utils.config import AttrDict
 from .model_list import MODELS
 
@@ -37,17 +37,6 @@ class TextRecTrainer(BaseTrainer):
         dst_label_dict_path = Path(self.global_config.output).joinpath("label_dict.txt")
         shutil.copyfile(src_label_dict_path, dst_label_dict_path)
 
-    def build_deamon(self, config: AttrDict) -> "TextRecTrainDeamon":
-        """build deamon thread for saving training outputs timely
-
-        Args:
-            config (AttrDict): PaddleX pipeline config, which is loaded from pipeline yaml file.
-
-        Returns:
-            TextRecTrainDeamon: the training deamon thread object for saving training outputs timely.
-        """
-        return TextRecTrainDeamon(config)
-
     def update_config(self):
         """update training config"""
         if self.train_config.log_interval:
@@ -112,47 +101,3 @@ class TextRecTrainer(BaseTrainer):
             "device": self.get_device(),
             "dy2st": self.train_config.get("dy2st", False),
         }
-
-
-class TextRecTrainDeamon(BaseTrainDeamon):
-    """TableRecTrainDeamon"""
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-
-    def get_the_pdparams_suffix(self):
-        """get the suffix of pdparams file"""
-        return "pdparams"
-
-    def get_the_pdema_suffix(self):
-        """get the suffix of pdema file"""
-        return "pdema"
-
-    def get_the_pdopt_suffix(self):
-        """get the suffix of pdopt file"""
-        return "pdopt"
-
-    def get_the_pdstates_suffix(self):
-        """get the suffix of pdstates file"""
-        return "states"
-
-    def get_ith_ckp_prefix(self, epoch_id):
-        """get the prefix of the epoch_id checkpoint file"""
-        return f"iter_epoch_{epoch_id}"
-
-    def get_best_ckp_prefix(self):
-        """get the prefix of the best checkpoint file"""
-        return "best_accuracy"
-
-    def get_score(self, pdstates_path):
-        """get the score by pdstates file"""
-        if not Path(pdstates_path).exists():
-            return 0
-        if self.global_config["model"] == "LaTeX_OCR_rec":
-            return paddle.load(pdstates_path)["best_model_dict"]["exp_rate"]
-        else:
-            return paddle.load(pdstates_path)["best_model_dict"]["acc"]
-
-    def get_epoch_id_by_pdparams_prefix(self, pdparams_prefix):
-        """get the epoch_id by pdparams file"""
-        return int(pdparams_prefix.split("_")[-1])

paddlex/modules/ts_anomaly_detection/trainer.py

@@ -19,7 +19,7 @@ from pathlib import Path
 import tarfile
 import lazy_paddle as paddle
 
-from ..base import BaseTrainer, BaseTrainDeamon
+from ..base import BaseTrainer
 from ...utils.config import AttrDict
 from .model_list import MODELS
 
@@ -29,17 +29,6 @@ class TSADTrainer(BaseTrainer):
 
     entities = MODELS
 
-    def build_deamon(self, config: AttrDict) -> "TSADTrainDeamon":
-        """build deamon thread for saving training outputs timely
-
-        Args:
-            config (AttrDict): PaddleX pipeline config, which is loaded from pipeline yaml file.
-
-        Returns:
-            TSADTrainDeamon: the training deamon thread object for saving training outputs timely.
-        """
-        return TSADTrainDeamon(config)
-
     def train(self):
         """firstly, update and dump train config, then train model"""
         # XXX: using super().train() instead when the train_hook() is supported.
@@ -106,152 +95,3 @@ training!"
         if self.global_config.output is not None:
             train_args["save_dir"] = self.global_config.output
         return train_args
-
-
-class TSADTrainDeamon(BaseTrainDeamon):
-    """DetTrainResultDemon"""
-
-    def get_watched_model(self):
-        """get the models needed to be watched"""
-        watched_models = []
-        watched_models.append("best")
-        return watched_models
-
-    def update(self):
-        """update train result json"""
-        self.processing = True
-        for i, result in enumerate(self.results):
-            self.results[i] = self.update_result(result, self.train_outputs[i])
-        self.save_json()
-        self.processing = False
-
-    def update_train_log(self, train_output):
-        """update train log"""
-        train_log_path = train_output / "train_ct.log"
-        with open(train_log_path, "w") as f:
-            seconds = time.time()
-            f.write(
-                "current training time: "
-                + time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(seconds))
-            )
-        f.close()
-        return train_log_path
-
-    def update_result(self, result, train_output):
-        """update every result"""
-        train_output = Path(train_output).resolve()
-        config_path = Path(train_output).joinpath("config.yaml").resolve()
-        if not config_path.exists():
-            return result
-
-        model_name = result["model_name"]
-        if (
-            model_name in self.config_recorder
-            and self.config_recorder[model_name] != config_path
-        ):
-            result["models"] = self.init_model_pkg()
-        result["config"] = config_path
-        self.config_recorder[model_name] = config_path
-
-        result["config"] = config_path
-        result["train_log"] = self.update_train_log(train_output)
-        result["visualdl_log"] = self.update_vdl_log(train_output)
-        result["label_dict"] = self.update_label_dict(train_output)
-        model = self.get_model(result["model_name"], config_path)
-        self.update_models(result, model, train_output, "best")
-
-        return result
-
-    def update_models(self, result, model, train_output, model_key):
-        """update info of the models to be saved"""
-        pdparams = Path(train_output).joinpath("best_accuracy.pdparams.tar")
-        if pdparams.exists():
-
-            score = self.get_score(Path(train_output).joinpath("score.json"))
-
-            result["models"][model_key] = {
-                "score": "%.3f" % score,
-                "pdparams": pdparams,
-                "pdema": "",
-                "pdopt": "",
-                "pdstates": "",
-                "inference_config": "",
-                "pdmodel": "",
-                "pdiparams": pdparams,
-                "pdiparams.info": "",
-            }
-            self.update_inference_model(
-                model,
-                train_output,
-                train_output.joinpath(f"inference"),
-                result["models"][model_key],
-            )
-
-    def update_inference_model(
-        self, model, weight_path, export_save_dir, result_the_model
-    ):
-        """update inference model"""
-        export_save_dir.mkdir(parents=True, exist_ok=True)
-        export_result = model.export(weight_path=weight_path, save_dir=export_save_dir)
-
-        if export_result.returncode == 0:
-            inference_config = export_save_dir.joinpath("inference.yml")
-            if not inference_config.exists():
-                inference_config = ""
-            use_pir = (
-                hasattr(paddle.framework, "use_pir_api")
-                and paddle.framework.use_pir_api()
-            )
-            pdmodel = (
-                export_save_dir.joinpath("inference.json")
-                if use_pir
-                else export_save_dir.joinpath("inference.pdmodel")
-            )
-            pdiparams = export_save_dir.joinpath("inference.pdiparams")
-            pdiparams_info = (
-                "" if use_pir else export_save_dir.joinpath("inference.pdiparams.info")
-            )
-        else:
-            inference_config = ""
-            pdmodel = ""
-            pdiparams = ""
-            pdiparams_info = ""
-
-        result_the_model["inference_config"] = inference_config
-        result_the_model["pdmodel"] = pdmodel
-        result_the_model["pdiparams"] = pdiparams
-        result_the_model["pdiparams.info"] = pdiparams_info
-
-    def get_score(self, score_path):
-        """get the score by pdstates file"""
-        if not Path(score_path).exists():
-            return 0
-        return json.load(open(score_path, "r"))["metric"]
-
-    def get_best_ckp_prefix(self):
-        """get the prefix of the best checkpoint file"""
-        pass
-
-    def get_epoch_id_by_pdparams_prefix(self):
-        """get the epoch_id by pdparams file"""
-        pass
-
-    def get_ith_ckp_prefix(self):
-        """get the prefix of the epoch_id checkpoint file"""
-        pass
-
-    def get_the_pdema_suffix(self):
-        """get the suffix of pdema file"""
-        pass
-
-    def get_the_pdopt_suffix(self):
-        """get the suffix of pdopt file"""
-        pass
-
-    def get_the_pdparams_suffix(self):
-        """get the suffix of pdparams file"""
-        pass
-
-    def get_the_pdstates_suffix(self):
-        """get the suffix of pdstates file"""
-        pass

paddlex/modules/ts_classification/trainer.py

@@ -19,7 +19,7 @@ import tarfile
 from pathlib import Path
 import lazy_paddle as paddle
 
-from ..base import BaseTrainer, BaseTrainDeamon
+from ..base import BaseTrainer
 from ...utils.config import AttrDict
 from .model_list import MODELS
 
@@ -29,17 +29,6 @@ class TSCLSTrainer(BaseTrainer):
 
     entities = MODELS
 
-    def build_deamon(self, config: AttrDict) -> "TSCLSTrainDeamon":
-        """build deamon thread for saving training outputs timely
-
-        Args:
-            config (AttrDict): PaddleX pipeline config, which is loaded from pipeline yaml file.
-
-        Returns:
-            TSCLSTrainDeamon: the training deamon thread object for saving training outputs timely.
-        """
-        return TSCLSTrainDeamon(config)
-
     def train(self):
         """firstly, update and dump train config, then train model"""
         # XXX: using super().train() instead when the train_hook() is supported.
@@ -101,152 +90,3 @@ training!"
         if self.global_config.output is not None:
             train_args["save_dir"] = self.global_config.output
         return train_args
-
-
-class TSCLSTrainDeamon(BaseTrainDeamon):
-    """TSCLSTrainResultDemon"""
-
-    def get_watched_model(self):
-        """get the models needed to be watched"""
-        watched_models = []
-        watched_models.append("best")
-        return watched_models
-
-    def update(self):
-        """update train result json"""
-        self.processing = True
-        for i, result in enumerate(self.results):
-            self.results[i] = self.update_result(result, self.train_outputs[i])
-        self.save_json()
-        self.processing = False
-
-    def update_train_log(self, train_output):
-        """update train log"""
-        train_log_path = train_output / "train_ct.log"
-        with open(train_log_path, "w") as f:
-            seconds = time.time()
-            f.write(
-                "current training time: "
-                + time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(seconds))
-            )
-        f.close()
-        return train_log_path
-
-    def update_result(self, result, train_output):
-        """update every result"""
-        train_output = Path(train_output).resolve()
-        config_path = Path(train_output).joinpath("config.yaml").resolve()
-        if not config_path.exists():
-            return result
-
-        model_name = result["model_name"]
-        if (
-            model_name in self.config_recorder
-            and self.config_recorder[model_name] != config_path
-        ):
-            result["models"] = self.init_model_pkg()
-        result["config"] = config_path
-        self.config_recorder[model_name] = config_path
-
-        result["config"] = config_path
-        result["train_log"] = self.update_train_log(train_output)
-        result["visualdl_log"] = self.update_vdl_log(train_output)
-        result["label_dict"] = self.update_label_dict(train_output)
-        model = self.get_model(result["model_name"], config_path)
-        self.update_models(result, model, train_output, "best")
-
-        return result
-
-    def update_models(self, result, model, train_output, model_key):
-        """update info of the models to be saved"""
-        pdparams = Path(train_output).joinpath("best_accuracy.pdparams.tar")
-        if pdparams.exists():
-
-            score = self.get_score(Path(train_output).joinpath("score.json"))
-
-            result["models"][model_key] = {
-                "score": "%.3f" % score,
-                "pdparams": pdparams,
-                "pdema": "",
-                "pdopt": "",
-                "pdstates": "",
-                "inference_config": "",
-                "pdmodel": "",
-                "pdiparams": pdparams,
-                "pdiparams.info": "",
-            }
-            self.update_inference_model(
-                model,
-                train_output,
-                train_output.joinpath(f"inference"),
-                result["models"][model_key],
-            )
-
-    def update_inference_model(
-        self, model, weight_path, export_save_dir, result_the_model
-    ):
-        """update inference model"""
-        export_save_dir.mkdir(parents=True, exist_ok=True)
-        export_result = model.export(weight_path=weight_path, save_dir=export_save_dir)
-
-        if export_result.returncode == 0:
-            inference_config = export_save_dir.joinpath("inference.yml")
-            if not inference_config.exists():
-                inference_config = ""
-            use_pir = (
-                hasattr(paddle.framework, "use_pir_api")
-                and paddle.framework.use_pir_api()
-            )
-            pdmodel = (
-                export_save_dir.joinpath("inference.json")
-                if use_pir
-                else export_save_dir.joinpath("inference.pdmodel")
-            )
-            pdiparams = export_save_dir.joinpath("inference.pdiparams")
-            pdiparams_info = (
-                "" if use_pir else export_save_dir.joinpath("inference.pdiparams.info")
-            )
-        else:
-            inference_config = ""
-            pdmodel = ""
-            pdiparams = ""
-            pdiparams_info = ""
-
-        result_the_model["inference_config"] = inference_config
-        result_the_model["pdmodel"] = pdmodel
-        result_the_model["pdiparams"] = pdiparams
-        result_the_model["pdiparams.info"] = pdiparams_info
-
-    def get_score(self, score_path):
-        """get the score by pdstates file"""
-        if not Path(score_path).exists():
-            return 0
-        return json.load(open(score_path))["metric"]
-
-    def get_best_ckp_prefix(self):
-        """get the prefix of the best checkpoint file"""
-        pass
-
-    def get_epoch_id_by_pdparams_prefix(self):
-        """get the epoch_id by pdparams file"""
-        pass
-
-    def get_ith_ckp_prefix(self):
-        """get the prefix of the epoch_id checkpoint file"""
-        pass
-
-    def get_the_pdema_suffix(self):
-        """get the suffix of pdema file"""
-        pass
-
-    def get_the_pdopt_suffix(self):
-        """get the suffix of pdopt file"""
-        pass
-
-    def get_the_pdparams_suffix(self):
-        """get the suffix of pdparams file"""
-        pass
-
-    def get_the_pdstates_suffix(self):
-        """get the suffix of pdstates file"""
-        pass

paddlex/modules/ts_forecast/trainer.py

@@ -19,7 +19,7 @@ import tarfile
 from pathlib import Path
 import lazy_paddle as paddle
 
-from ..base import BaseTrainer, BaseTrainDeamon
+from ..base import BaseTrainer
 from ...utils.config import AttrDict
 from .model_list import MODELS
 
@@ -29,17 +29,6 @@ class TSFCTrainer(BaseTrainer):
 
     entities = MODELS
 
-    def build_deamon(self, config: AttrDict) -> "TSFCTrainDeamon":
-        """build deamon thread for saving training outputs timely
-
-        Args:
-            config (AttrDict): PaddleX pipeline config, which is loaded from pipeline yaml file.
-
-        Returns:
-            TSFCTrainDeamon: the training deamon thread object for saving training outputs timely.
-        """
-        return TSFCTrainDeamon(config)
-
     def train(self):
         """firstly, update and dump train config, then train model"""
         # XXX: using super().train() instead when the train_hook() is supported.
@@ -101,152 +90,3 @@ training!"
         if self.global_config.output is not None:
             train_args["save_dir"] = self.global_config.output
         return train_args
-
-
-class TSFCTrainDeamon(BaseTrainDeamon):
-    """TSFCTrainResultDemon"""
-
-    def get_watched_model(self):
-        """get the models needed to be watched"""
-        watched_models = []
-        watched_models.append("best")
-        return watched_models
-
-    def update(self):
-        """update train result json"""
-        self.processing = True
-        for i, result in enumerate(self.results):
-            self.results[i] = self.update_result(result, self.train_outputs[i])
-        self.save_json()
-        self.processing = False
-
-    def update_train_log(self, train_output):
-        """update train log"""
-        train_log_path = train_output / "train_ct.log"
-        with open(train_log_path, "w") as f:
-            seconds = time.time()
-            f.write(
-                "current training time: "
-                + time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(seconds))
-            )
-        f.close()
-        return train_log_path
-
-    def update_result(self, result, train_output):
-        """update every result"""
-        train_output = Path(train_output).resolve()
-        config_path = Path(train_output).joinpath("config.yaml").resolve()
-        if not config_path.exists():
-            return result
-
-        model_name = result["model_name"]
-        if (
-            model_name in self.config_recorder
-            and self.config_recorder[model_name] != config_path
-        ):
-            result["models"] = self.init_model_pkg()
-        result["config"] = config_path
-        self.config_recorder[model_name] = config_path
-
-        result["config"] = config_path
-        result["train_log"] = self.update_train_log(train_output)
-        result["visualdl_log"] = self.update_vdl_log(train_output)
-        result["label_dict"] = self.update_label_dict(train_output)
-        model = self.get_model(result["model_name"], config_path)
-        self.update_models(result, model, train_output, "best")
-
-        return result
-
-    def update_models(self, result, model, train_output, model_key):
-        """update info of the models to be saved"""
-        pdparams = Path(train_output).joinpath("best_accuracy.pdparams.tar")
-        if pdparams.exists():
-
-            score = self.get_score(Path(train_output).joinpath("score.json"))
-
-            result["models"][model_key] = {
-                "score": "%.3f" % score,
-                "pdparams": pdparams,
-                "pdema": "",
-                "pdopt": "",
-                "pdstates": "",
-                "inference_config": "",
-                "pdmodel": "",
-                "pdiparams": pdparams,
-                "pdiparams.info": "",
-            }
-            self.update_inference_model(
-                model,
-                train_output,
-                train_output.joinpath(f"inference"),
-                result["models"][model_key],
-            )
-
-    def update_inference_model(
-        self, model, weight_path, export_save_dir, result_the_model
-    ):
-        """update inference model"""
-        export_save_dir.mkdir(parents=True, exist_ok=True)
-        export_result = model.export(weight_path=weight_path, save_dir=export_save_dir)
-
-        if export_result.returncode == 0:
-            inference_config = export_save_dir.joinpath("inference.yml")
-            if not inference_config.exists():
-                inference_config = ""
-            use_pir = (
-                hasattr(paddle.framework, "use_pir_api")
-                and paddle.framework.use_pir_api()
-            )
-            pdmodel = (
-                export_save_dir.joinpath("inference.json")
-                if use_pir
-                else export_save_dir.joinpath("inference.pdmodel")
-            )
-            pdiparams = export_save_dir.joinpath("inference.pdiparams")
-            pdiparams_info = (
-                "" if use_pir else export_save_dir.joinpath("inference.pdiparams.info")
-            )
-        else:
-            inference_config = ""
-            pdmodel = ""
-            pdiparams = ""
-            pdiparams_info = ""
-
-        result_the_model["inference_config"] = inference_config
-        result_the_model["pdmodel"] = pdmodel
-        result_the_model["pdiparams"] = pdiparams
-        result_the_model["pdiparams.info"] = pdiparams_info
-
-    def get_score(self, score_path):
-        """get the score by pdstates file"""
-        if not Path(score_path).exists():
-            return 0
-        return json.load(open(score_path))["metric"]
-
-    def get_best_ckp_prefix(self):
-        """get the prefix of the best checkpoint file"""
-        pass
-
-    def get_epoch_id_by_pdparams_prefix(self):
-        """get the epoch_id by pdparams file"""
-        pass
-
-    def get_ith_ckp_prefix(self):
-        """get the prefix of the epoch_id checkpoint file"""
-        pass
-
-    def get_the_pdema_suffix(self):
-        """get the suffix of pdema file"""
-        pass
-
-    def get_the_pdopt_suffix(self):
-        """get the suffix of pdopt file"""
-        pass
-
-    def get_the_pdparams_suffix(self):
-        """get the suffix of pdparams file"""
-        pass
-
-    def get_the_pdstates_suffix(self):
-        """get the suffix of pdstates file"""
-        pass