From 0bfba555c4d72a9ba7e9c9ba0be4d45345a7e4fb Mon Sep 17 00:00:00 2001 From: tangwei Date: Thu, 9 Apr 2020 23:24:28 +0800 Subject: [PATCH] update to fleetrec --- __init__.py => fleetrec/__init__.py | 0 {examples => fleetrec/examples}/__init__.py | 0 .../examples}/ctr-dnn_train_cluster.yaml | 6 +++--- .../examples}/ctr-dnn_train_single.yaml | 6 +++--- {examples => fleetrec/examples}/train.py | 2 +- {metrics => fleetrec/metrics}/__init__.py | 0 {metrics => fleetrec/metrics}/auc_metrics.py | 0 {metrics => fleetrec/metrics}/base.py | 0 {models => fleetrec/models}/__init__.py | 0 {models => fleetrec/models}/base.py | 0 {models => fleetrec/models}/ctr_dnn/__init__.py | 0 .../models}/ctr_dnn/data/test/sample_test.txt | 0 .../models}/ctr_dnn/data/train/sample_train.txt | 0 .../models}/ctr_dnn/dataloader.py | 0 {models => fleetrec/models}/ctr_dnn/dataset.py | 0 .../models}/ctr_dnn/hyper_parameters.yaml | 0 {models => fleetrec/models}/ctr_dnn/model.py | 2 +- {models => fleetrec/models}/layers.py | 0 .../models}/word2vec/hyper_parameters.yaml | 0 {models => fleetrec/models}/word2vec/model.py | 0 {models => fleetrec/models}/word2vec/reader.py | 0 .../models}/word2vec/sample_test.txt | 0 .../models}/word2vec/sample_train.txt | 0 {reader => fleetrec/reader}/__init__.py | 0 {reader => fleetrec/reader}/data_loader.py | 0 {reader => fleetrec/reader}/dataset.py | 0 {trainer => fleetrec/trainer}/__init__.py | 0 {trainer => fleetrec/trainer}/cluster_trainer.py | 0 {trainer => fleetrec/trainer}/ctr_trainer.py | 10 +++++----- {trainer => fleetrec/trainer}/factory.py | 16 ++++++++-------- {trainer => fleetrec/trainer}/local_engine.py | 2 +- {trainer => fleetrec/trainer}/single_trainer.py | 0 {trainer => fleetrec/trainer}/trainer.py | 0 .../trainer}/transpiler_trainer.py | 0 {utils => fleetrec/utils}/__init__.py | 0 {utils => fleetrec/utils}/envs.py | 2 +- {utils => fleetrec/utils}/fs.py | 0 {utils => fleetrec/utils}/table.py | 0 {utils => fleetrec/utils}/util.py | 0 39 files changed, 23 insertions(+), 23 deletions(-) rename __init__.py => fleetrec/__init__.py (100%) rename {examples => fleetrec/examples}/__init__.py (100%) rename {examples => fleetrec/examples}/ctr-dnn_train_cluster.yaml (91%) rename {examples => fleetrec/examples}/ctr-dnn_train_single.yaml (91%) rename {examples => fleetrec/examples}/train.py (96%) rename {metrics => fleetrec/metrics}/__init__.py (100%) rename {metrics => fleetrec/metrics}/auc_metrics.py (100%) rename {metrics => fleetrec/metrics}/base.py (100%) rename {models => fleetrec/models}/__init__.py (100%) rename {models => fleetrec/models}/base.py (100%) rename {models => fleetrec/models}/ctr_dnn/__init__.py (100%) rename {models => fleetrec/models}/ctr_dnn/data/test/sample_test.txt (100%) rename {models => fleetrec/models}/ctr_dnn/data/train/sample_train.txt (100%) rename {models => fleetrec/models}/ctr_dnn/dataloader.py (100%) rename {models => fleetrec/models}/ctr_dnn/dataset.py (100%) rename {models => fleetrec/models}/ctr_dnn/hyper_parameters.yaml (100%) rename {models => fleetrec/models}/ctr_dnn/model.py (99%) rename {models => fleetrec/models}/layers.py (100%) rename {models => fleetrec/models}/word2vec/hyper_parameters.yaml (100%) rename {models => fleetrec/models}/word2vec/model.py (100%) rename {models => fleetrec/models}/word2vec/reader.py (100%) rename {models => fleetrec/models}/word2vec/sample_test.txt (100%) rename {models => fleetrec/models}/word2vec/sample_train.txt (100%) rename {reader => fleetrec/reader}/__init__.py (100%) rename {reader => fleetrec/reader}/data_loader.py (100%) rename {reader => fleetrec/reader}/dataset.py (100%) rename {trainer => fleetrec/trainer}/__init__.py (100%) rename {trainer => fleetrec/trainer}/cluster_trainer.py (100%) rename {trainer => fleetrec/trainer}/ctr_trainer.py (98%) rename {trainer => fleetrec/trainer}/factory.py (89%) rename {trainer => fleetrec/trainer}/local_engine.py (98%) rename {trainer => fleetrec/trainer}/single_trainer.py (100%) rename {trainer => fleetrec/trainer}/trainer.py (100%) rename {trainer => fleetrec/trainer}/transpiler_trainer.py (100%) rename {utils => fleetrec/utils}/__init__.py (100%) rename {utils => fleetrec/utils}/envs.py (97%) rename {utils => fleetrec/utils}/fs.py (100%) rename {utils => fleetrec/utils}/table.py (100%) rename {utils => fleetrec/utils}/util.py (100%) diff --git a/__init__.py b/fleetrec/__init__.py similarity index 100% rename from __init__.py rename to fleetrec/__init__.py diff --git a/examples/__init__.py b/fleetrec/examples/__init__.py similarity index 100% rename from examples/__init__.py rename to fleetrec/examples/__init__.py diff --git a/examples/ctr-dnn_train_cluster.yaml b/fleetrec/examples/ctr-dnn_train_cluster.yaml similarity index 91% rename from examples/ctr-dnn_train_cluster.yaml rename to fleetrec/examples/ctr-dnn_train_cluster.yaml index d24bf882..8eb9eb1e 100644 --- a/examples/ctr-dnn_train_cluster.yaml +++ b/fleetrec/examples/ctr-dnn_train_cluster.yaml @@ -41,11 +41,11 @@ train: reader: mode: "dataset" batch_size: 2 - pipe_command: "python /paddle/fleet_rec/models/ctr_dnn/dataset.py" - train_data_path: "/paddle/fleet_rec/models/ctr_dnn/data/train" + pipe_command: "python /paddle/fleetrec/models/ctr_dnn/dataset.py" + train_data_path: "/paddle/fleetrec/models/ctr_dnn/data/train" model: - models: "fleet_rec.models.ctr_dnn.model" + models: "fleetrec.models.ctr_dnn.model" hyper_parameters: sparse_inputs_slots: 27 sparse_feature_number: 1000001 diff --git a/examples/ctr-dnn_train_single.yaml b/fleetrec/examples/ctr-dnn_train_single.yaml similarity index 91% rename from examples/ctr-dnn_train_single.yaml rename to fleetrec/examples/ctr-dnn_train_single.yaml index 1dc61903..4091c78e 100644 --- a/examples/ctr-dnn_train_single.yaml +++ b/fleetrec/examples/ctr-dnn_train_single.yaml @@ -35,11 +35,11 @@ train: reader: mode: "dataset" batch_size: 2 - pipe_command: "python /paddle/fleet_rec/models/ctr_dnn/dataset.py" - train_data_path: "/paddle/fleet_rec/models/ctr_dnn/data/train" + pipe_command: "python /paddle/fleetrec/models/ctr_dnn/dataset.py" + train_data_path: "/paddle/fleetrec/models/ctr_dnn/data/train" model: - models: "fleet_rec.models.ctr_dnn.model" + models: "fleetrec.models.ctr_dnn.model" hyper_parameters: sparse_inputs_slots: 27 sparse_feature_number: 1000001 diff --git a/examples/train.py b/fleetrec/examples/train.py similarity index 96% rename from examples/train.py rename to fleetrec/examples/train.py index 4d84b871..fb2fd54b 100644 --- a/examples/train.py +++ b/fleetrec/examples/train.py @@ -26,7 +26,7 @@ import os -from fleet_rec.trainer.factory import TrainerFactory +from fleetrec.trainer.factory import TrainerFactory if __name__ == "__main__": diff --git a/metrics/__init__.py b/fleetrec/metrics/__init__.py similarity index 100% rename from metrics/__init__.py rename to fleetrec/metrics/__init__.py diff --git a/metrics/auc_metrics.py b/fleetrec/metrics/auc_metrics.py similarity index 100% rename from metrics/auc_metrics.py rename to fleetrec/metrics/auc_metrics.py diff --git a/metrics/base.py b/fleetrec/metrics/base.py similarity index 100% rename from metrics/base.py rename to fleetrec/metrics/base.py diff --git a/models/__init__.py b/fleetrec/models/__init__.py similarity index 100% rename from models/__init__.py rename to fleetrec/models/__init__.py diff --git a/models/base.py b/fleetrec/models/base.py similarity index 100% rename from models/base.py rename to fleetrec/models/base.py diff --git a/models/ctr_dnn/__init__.py b/fleetrec/models/ctr_dnn/__init__.py similarity index 100% rename from models/ctr_dnn/__init__.py rename to fleetrec/models/ctr_dnn/__init__.py diff --git a/models/ctr_dnn/data/test/sample_test.txt b/fleetrec/models/ctr_dnn/data/test/sample_test.txt similarity index 100% rename from models/ctr_dnn/data/test/sample_test.txt rename to fleetrec/models/ctr_dnn/data/test/sample_test.txt diff --git a/models/ctr_dnn/data/train/sample_train.txt b/fleetrec/models/ctr_dnn/data/train/sample_train.txt similarity index 100% rename from models/ctr_dnn/data/train/sample_train.txt rename to fleetrec/models/ctr_dnn/data/train/sample_train.txt diff --git a/models/ctr_dnn/dataloader.py b/fleetrec/models/ctr_dnn/dataloader.py similarity index 100% rename from models/ctr_dnn/dataloader.py rename to fleetrec/models/ctr_dnn/dataloader.py diff --git a/models/ctr_dnn/dataset.py b/fleetrec/models/ctr_dnn/dataset.py similarity index 100% rename from models/ctr_dnn/dataset.py rename to fleetrec/models/ctr_dnn/dataset.py diff --git a/models/ctr_dnn/hyper_parameters.yaml b/fleetrec/models/ctr_dnn/hyper_parameters.yaml similarity index 100% rename from models/ctr_dnn/hyper_parameters.yaml rename to fleetrec/models/ctr_dnn/hyper_parameters.yaml diff --git a/models/ctr_dnn/model.py b/fleetrec/models/ctr_dnn/model.py similarity index 99% rename from models/ctr_dnn/model.py rename to fleetrec/models/ctr_dnn/model.py index d144cc06..cd3c9df5 100644 --- a/models/ctr_dnn/model.py +++ b/fleetrec/models/ctr_dnn/model.py @@ -15,7 +15,7 @@ import math import paddle.fluid as fluid -from fleet_rec.utils import envs +from fleetrec.utils import envs class Train(object): diff --git a/models/layers.py b/fleetrec/models/layers.py similarity index 100% rename from models/layers.py rename to fleetrec/models/layers.py diff --git a/models/word2vec/hyper_parameters.yaml b/fleetrec/models/word2vec/hyper_parameters.yaml similarity index 100% rename from models/word2vec/hyper_parameters.yaml rename to fleetrec/models/word2vec/hyper_parameters.yaml diff --git a/models/word2vec/model.py b/fleetrec/models/word2vec/model.py similarity index 100% rename from models/word2vec/model.py rename to fleetrec/models/word2vec/model.py diff --git a/models/word2vec/reader.py b/fleetrec/models/word2vec/reader.py similarity index 100% rename from models/word2vec/reader.py rename to fleetrec/models/word2vec/reader.py diff --git a/models/word2vec/sample_test.txt b/fleetrec/models/word2vec/sample_test.txt similarity index 100% rename from models/word2vec/sample_test.txt rename to fleetrec/models/word2vec/sample_test.txt diff --git a/models/word2vec/sample_train.txt b/fleetrec/models/word2vec/sample_train.txt similarity index 100% rename from models/word2vec/sample_train.txt rename to fleetrec/models/word2vec/sample_train.txt diff --git a/reader/__init__.py b/fleetrec/reader/__init__.py similarity index 100% rename from reader/__init__.py rename to fleetrec/reader/__init__.py diff --git a/reader/data_loader.py b/fleetrec/reader/data_loader.py similarity index 100% rename from reader/data_loader.py rename to fleetrec/reader/data_loader.py diff --git a/reader/dataset.py b/fleetrec/reader/dataset.py similarity index 100% rename from reader/dataset.py rename to fleetrec/reader/dataset.py diff --git a/trainer/__init__.py b/fleetrec/trainer/__init__.py similarity index 100% rename from trainer/__init__.py rename to fleetrec/trainer/__init__.py diff --git a/trainer/cluster_trainer.py b/fleetrec/trainer/cluster_trainer.py similarity index 100% rename from trainer/cluster_trainer.py rename to fleetrec/trainer/cluster_trainer.py diff --git a/trainer/ctr_trainer.py b/fleetrec/trainer/ctr_trainer.py similarity index 98% rename from trainer/ctr_trainer.py rename to fleetrec/trainer/ctr_trainer.py index b58200c9..582d60f8 100755 --- a/trainer/ctr_trainer.py +++ b/fleetrec/trainer/ctr_trainer.py @@ -23,11 +23,11 @@ import paddle.fluid as fluid from paddle.fluid.incubate.fleet.parameter_server.pslib import fleet from paddle.fluid.incubate.fleet.base.role_maker import GeneralRoleMaker -from fleet_rec.utils import fs as fs -from fleet_rec.utils import util as util -from fleet_rec.metrics.auc_metrics import AUCMetric -from fleet_rec.models import base as model_basic -from fleet_rec.reader import dataset +from fleetrec.utils import fs as fs +from fleetrec.utils import util as util +from fleetrec.metrics.auc_metrics import AUCMetric +from fleetrec.models import base as model_basic +from fleetrec.reader import dataset from .trainer import Trainer diff --git a/trainer/factory.py b/fleetrec/trainer/factory.py similarity index 89% rename from trainer/factory.py rename to fleetrec/trainer/factory.py index 3301c13c..6434d7da 100644 --- a/trainer/factory.py +++ b/fleetrec/trainer/factory.py @@ -29,16 +29,16 @@ import sys import yaml -from fleet_rec.trainer.single_trainer import SingleTrainerWithDataloader -from fleet_rec.trainer.single_trainer import SingleTrainerWithDataset +from fleetrec.trainer.single_trainer import SingleTrainerWithDataloader +from fleetrec.trainer.single_trainer import SingleTrainerWithDataset -from fleet_rec.trainer.cluster_trainer import ClusterTrainerWithDataloader -from fleet_rec.trainer.cluster_trainer import ClusterTrainerWithDataset +from fleetrec.trainer.cluster_trainer import ClusterTrainerWithDataloader +from fleetrec.trainer.cluster_trainer import ClusterTrainerWithDataset -from fleet_rec.trainer.local_engine import Launch -from fleet_rec.trainer.ctr_trainer import CtrPaddleTrainer +from fleetrec.trainer.local_engine import Launch +from fleetrec.trainer.ctr_trainer import CtrPaddleTrainer -from fleet_rec.utils import envs +from fleetrec.utils import envs def str2bool(v): @@ -103,7 +103,7 @@ class TrainerFactory(object): with open(config, 'r') as rb: _config = yaml.load(rb.read(), Loader=yaml.FullLoader) else: - raise ValueError("fleet_rec's config only support yaml") + raise ValueError("fleetrec's config only support yaml") envs.set_global_envs(_config) mode = envs.get_global_env("train.trainer") diff --git a/trainer/local_engine.py b/fleetrec/trainer/local_engine.py similarity index 98% rename from trainer/local_engine.py rename to fleetrec/trainer/local_engine.py index fb883549..9a1c4ba0 100644 --- a/trainer/local_engine.py +++ b/fleetrec/trainer/local_engine.py @@ -38,7 +38,7 @@ def start_procs(args, yaml): user_endpoints_ips = [x.split(":")[0] for x in user_endpoints.split(",")] user_endpoints_port = [x.split(":")[1] for x in user_endpoints.split(",")] - factory = "fleet_rec.trainer.factory" + factory = "fleetrec.trainer.factory" cmd = [sys.executable, "-u", "-m", factory, yaml] for i in range(server_num): diff --git a/trainer/single_trainer.py b/fleetrec/trainer/single_trainer.py similarity index 100% rename from trainer/single_trainer.py rename to fleetrec/trainer/single_trainer.py diff --git a/trainer/trainer.py b/fleetrec/trainer/trainer.py similarity index 100% rename from trainer/trainer.py rename to fleetrec/trainer/trainer.py diff --git a/trainer/transpiler_trainer.py b/fleetrec/trainer/transpiler_trainer.py similarity index 100% rename from trainer/transpiler_trainer.py rename to fleetrec/trainer/transpiler_trainer.py diff --git a/utils/__init__.py b/fleetrec/utils/__init__.py similarity index 100% rename from utils/__init__.py rename to fleetrec/utils/__init__.py diff --git a/utils/envs.py b/fleetrec/utils/envs.py similarity index 97% rename from utils/envs.py rename to fleetrec/utils/envs.py index 0c467c28..95cf4447 100644 --- a/utils/envs.py +++ b/fleetrec/utils/envs.py @@ -70,7 +70,7 @@ def pretty_print_envs(envs, header=None): if header: draws += h_format.format(header[0], header[1]) else: - draws += h_format.format("fleet_rec Global Envs", "Value") + draws += h_format.format("fleetrec Global Envs", "Value") draws += line + "\n" diff --git a/utils/fs.py b/fleetrec/utils/fs.py similarity index 100% rename from utils/fs.py rename to fleetrec/utils/fs.py diff --git a/utils/table.py b/fleetrec/utils/table.py similarity index 100% rename from utils/table.py rename to fleetrec/utils/table.py diff --git a/utils/util.py b/fleetrec/utils/util.py similarity index 100% rename from utils/util.py rename to fleetrec/utils/util.py -- GitLab