Skip to content

H
hapi

PaddlePaddle / hapi

通知 11
Star 2
Fork 0
H
hapi

前往新版Gitcode，体验更适合开发者的 AI 搜索 >>

未验证 提交 07ede80c 编写于 作者: L LielinJiang 提交者： GitHub
浏览文件
操作

Merge pull request #66 from LielinJiang/add-examples 

Add comments and examples
上级 3c0f4c32 e677a23d
隐藏空白更改
内联 并排
Showing with 992 addition and 224 deletion
hapi/callbacks.py
浏览文件 @ 07ede80c
......@@ -33,7 +33,7 @@ def config_callbacks(callbacks=None,
cbks = callbacks or []
cbks = cbks if isinstance(cbks, (list, tuple)) else [cbks]
if not any(isinstance(k, ProgBarLogger) for k in cbks) and verbose:
cbks = cbks + [ProgBarLogger(log_freq, verbose=verbose)]
cbks = [ProgBarLogger(log_freq, verbose=verbose)] + cbks
if not any(isinstance(k, ModelCheckpoint) for k in cbks):
cbks = cbks + [ModelCheckpoint(save_freq, save_dir)]
......@@ -110,6 +110,9 @@ class CallbackList(object):
class Callback(object):
"""Base class used to build new callbacks.
"""
def __init__(self):
self.model = None
self.params = {}
......@@ -121,63 +124,101 @@ class Callback(object):
self.model = model
def on_train_begin(self, logs=None):
"""
"""Called at the start of training.
"""
def on_train_end(self, logs=None):
"""
"""Called at the end of training.
"""
def on_eval_begin(self, logs=None):
"""
"""Called at the start of evaluation.
"""
def on_eval_end(self, logs=None):
"""
"""Called at the end of evaluation.
"""
def on_test_begin(self, logs=None):
"""
"""Called at the beginning of predict.
"""
def on_test_end(self, logs=None):
"""
"""Called at the end of predict.
"""
def on_epoch_begin(self, epoch, logs=None):
"""
"""Called at the beginning of each epoch.
"""
def on_epoch_end(self, epoch, logs=None):
"""
"""Called at the end of each epoch.
"""
def on_train_batch_begin(self, step, logs=None):
"""
"""Called at the beginning of each batch in training.
"""
def on_train_batch_end(self, step, logs=None):
"""
"""Called at the end of each batch in training.
"""
def on_eval_batch_begin(self, step, logs=None):
"""
"""Called at the beginning of each batch in evaluation.
"""
def on_eval_batch_end(self, step, logs=None):
"""
"""Called at the end of each batch in evaluation.
"""
def on_eval_batch_begin(self, step, logs=None):
"""
def on_test_batch_begin(self, step, logs=None):
"""Called at the beginning of each batch in predict.
"""
def on_eval_batch_end(self, step, logs=None):
"""
def on_test_batch_end(self, step, logs=None):
"""Called at the end of each batch in predict.
"""
class ProgBarLogger(Callback):
"""Logger callback function
Args:
log_freq (int): The frequency, in number of steps, the logs such as `loss`,
`metrics` are printed. Default: 1.
verbose (int): The verbosity mode, should be 0, 1, or 2.
0 = silent, 1 = progress bar, 2 = one line per epoch. Default: 2.
Examples:
.. code-block:: python
import numpy as np
from paddle import fluid
from hapi.metrics import Accuracy
from hapi.loss import CrossEntropy
from hapi.datasets import MNIST
from hapi.vision.transforms import Compose, Resize
from hapi.vision.models import LeNet
from hapi.callbacks import ProgBarLogger
from hapi.model import Input, set_device
inputs = [Input([-1, 1, 28, 28], 'float32', name='image')]
labels = [Input([None, 1], 'int64', name='label')]
train_dataset = MNIST(mode='train')
model = LeNet()
optim = fluid.optimizer.Adam(0.001)
model.prepare(optimizer=optim,
loss_function=CrossEntropy(),
metrics=Accuracy(),
inputs=inputs,
labels=labels)
callback = ProgBarLogger(log_freq=10)
model.fit(train_dataset, batch_size=64, callbacks=callback)
"""
def __init__(self, log_freq=1, verbose=2):
self.epochs = None
self.steps = None
......@@ -207,9 +248,11 @@ class ProgBarLogger(Callback):
metrics = getattr(self, '%s_metrics' % (mode))
progbar = getattr(self, '%s_progbar' % (mode))
steps = getattr(self, '%s_step' % (mode))
for k in metrics:
if k in logs:
values.append((k, logs[k]))
progbar.update(steps, values)
def on_train_batch_end(self, step, logs=None):
......@@ -230,6 +273,7 @@ class ProgBarLogger(Callback):
self.eval_metrics = logs.get('metrics_name', [])
self.eval_step = 0
self.evaled_samples = 0
self.eval_progbar = ProgressBar(
num=self.eval_steps, verbose=self.verbose)
if self._is_print():
......@@ -245,14 +289,79 @@ class ProgBarLogger(Callback):
if self.eval_steps is None or self.eval_step < self.eval_steps:
self._updates(logs, 'eval')
def on_test_begin(self, logs=None):
self.test_steps = logs.get('steps', None)
self.test_metrics = logs.get('metrics_name', [])
self.test_step = 0
self.tested_samples = 0
self.test_progbar = ProgressBar(
num=self.test_steps, verbose=self.verbose)
if self._is_print():
print('Predict begin...')
def on_test_batch_end(self, step, logs=None):
logs = logs or {}
self.test_step += 1
samples = logs.get('batch_size', 1)
self.tested_samples += samples
if self.test_step % self.log_freq == 0 and self._is_print():
if self.test_steps is None or self.test_step < self.test_steps:
self._updates(logs, 'test')
def on_eval_end(self, logs=None):
logs = logs or {}
if self._is_print() and (self.steps is not None):
self._updates(logs, 'eval')
print('Eval samples: %d' % (self.evaled_samples))
def on_test_end(self, logs=None):
logs = logs or {}
if self._is_print():
if self.test_step % self.log_freq != 0 or self.verbose == 1:
self._updates(logs, 'test')
print('Predict samples: %d' % (self.tested_samples))
class ModelCheckpoint(Callback):
"""Model checkpoint callback function
Args:
save_freq(int): The frequency, in number of epochs, the model checkpoint
are saved. Default: 1.
save_dir(str|None): The directory to save checkpoint during training.
If None, will not save checkpoint. Default: None.
Examples:
.. code-block:: python
import numpy as np
from paddle import fluid
from hapi.metrics import Accuracy
from hapi.loss import CrossEntropy
from hapi.datasets import MNIST
from hapi.vision.transforms import Compose, Resize
from hapi.vision.models import LeNet
from hapi.callbacks import ModelCheckpoint
from hapi.model import Input, set_device
inputs = [Input([-1, 1, 28, 28], 'float32', name='image')]
labels = [Input([None, 1], 'int64', name='label')]
train_dataset = MNIST(mode='train')
model = LeNet()
optim = fluid.optimizer.Adam(0.001)
model.prepare(optimizer=optim,
loss_function=CrossEntropy(),
metrics=Accuracy(),
inputs=inputs,
labels=labels)
callback = ModelCheckpoint(save_dir='./temp')
model.fit(train_dataset, batch_size=64, callbacks=callback)
"""
def __init__(self, save_freq=1, save_dir=None):
self.save_freq = save_freq
self.save_dir = save_dir
......
hapi/datasets/folder.py
浏览文件 @ 07ede80c
......@@ -34,13 +34,10 @@ def has_valid_extension(filename, extensions):
return filename.lower().endswith(extensions)
def make_dataset(dir, class_to_idx, extensions=None, is_valid_file=None):
def make_dataset(dir, class_to_idx, extensions, is_valid_file=None):
images = []
dir = os.path.expanduser(dir)
if not ((extensions is None) ^ (is_valid_file is None)):
raise ValueError(
"Both extensions and is_valid_file cannot be None or not None at the same time"
)
if extensions is not None:
def is_valid_file(x):
......@@ -200,10 +197,7 @@ class ImageFolder(Dataset):
samples = []
path = os.path.expanduser(root)
if not ((extensions is None) ^ (is_valid_file is None)):
raise ValueError(
"Both extensions and is_valid_file cannot be None or not None at the same time"
)
if extensions is not None:
def is_valid_file(x):
......
hapi/datasets/utils.py
浏览文件 @ 07ede80c
......@@ -25,5 +25,5 @@ def _check_exists_and_download(path, url, md5, module_name, download=True):
if download:
return paddle.dataset.common.download(url, module_name, md5)
else:
raise FileNotFoundError(
'{} not exists and auto download disabled'.format(path))
raise ValueError('{} not exists and auto download disabled'.format(
path))
hapi/distributed.py
浏览文件 @ 07ede80c
......@@ -48,6 +48,35 @@ class DistributedBatchSampler(BatchSampler):
batch indices. Default False.
drop_last(bool): whether drop the last incomplete batch dataset size
is not divisible by the batch size. Default False
Examples:
.. code-block:: python
import numpy as np
from hapi.datasets import MNIST
from hapi.distributed import DistributedBatchSampler
class MnistDataset(MNIST):
def __init__(self, mode, return_label=True):
super(MnistDataset, self).__init__(mode=mode)
self.return_label = return_label
def __getitem__(self, idx):
img = np.reshape(self.images[idx], [1, 28, 28])
if self.return_label:
return img, np.array(self.labels[idx]).astype('int64')
return img,
def __len__(self):
return len(self.images)
train_dataset = MnistDataset(mode='train')
dist_train_dataloader = DistributedBatchSampler(train_dataset, batch_size=64)
for data in dist_train_dataloader:
# do something
break
"""
def __init__(self, dataset, batch_size, shuffle=False, drop_last=False):
......
hapi/download.py
浏览文件 @ 07ede80c
......@@ -17,16 +17,40 @@ from __future__ import division
from __future__ import print_function
import os
import sys
import os.path as osp
import shutil
import requests
import tqdm
import hashlib
import time
from collections import OrderedDict
from paddle.fluid.dygraph.parallel import ParallelEnv
try:
from tqdm import tqdm
except:
class tqdm(object):
def __init__(self, total=None):
self.total = total
self.n = 0
def update(self, n):
self.n += n
if self.total is None:
sys.stderr.write("\r{0:.1f} bytes".format(self.n))
else:
sys.stderr.write("\r{0:.1f}%".format(100 * self.n / float(
self.total)))
sys.stderr.flush()
def __enter__(self):
return self
def __exit__(self, exc_type, exc_val, exc_tb):
sys.stderr.write('\n')
import logging
logger = logging.getLogger(__name__)
......@@ -36,25 +60,43 @@ WEIGHTS_HOME = osp.expanduser("~/.cache/paddle/hapi/weights")
DOWNLOAD_RETRY_LIMIT = 3
nlp_models = OrderedDict(
(('RoBERTa-zh-base', 'https://bert-models.bj.bcebos.com/chinese_roberta_wwm_ext_L-12_H-768_A-12.tar.gz'),
('RoBERTa-zh-large', 'https://bert-models.bj.bcebos.com/chinese_roberta_wwm_large_ext_L-24_H-1024_A-16.tar.gz'),
('ERNIE-v2-en-base', 'https://ernie.bj.bcebos.com/ERNIE_Base_en_stable-2.0.0.tar.gz'),
('ERNIE-v2-en-large', 'https://ernie.bj.bcebos.com/ERNIE_Large_en_stable-2.0.0.tar.gz'),
('XLNet-cased-base', 'https://xlnet.bj.bcebos.com/xlnet_cased_L-12_H-768_A-12.tgz'),
('XLNet-cased-large', 'https://xlnet.bj.bcebos.com/xlnet_cased_L-24_H-1024_A-16.tgz'),
('ERNIE-v1-zh-base', 'https://baidu-nlp.bj.bcebos.com/ERNIE_stable-1.0.1.tar.gz'),
('ERNIE-v1-zh-base-max-len-512', 'https://ernie.bj.bcebos.com/ERNIE_1.0_max-len-512.tar.gz'),
('BERT-en-uncased-large-whole-word-masking', 'https://bert-models.bj.bcebos.com/wwm_uncased_L-24_H-1024_A-16.tar.gz'),
('BERT-en-cased-large-whole-word-masking', 'https://bert-models.bj.bcebos.com/wwm_cased_L-24_H-1024_A-16.tar.gz'),
('BERT-en-uncased-base', 'https://bert-models.bj.bcebos.com/uncased_L-12_H-768_A-12.tar.gz'),
('BERT-en-uncased-large', 'https://bert-models.bj.bcebos.com/uncased_L-24_H-1024_A-16.tar.gz'),
('BERT-en-cased-base', 'https://bert-models.bj.bcebos.com/cased_L-12_H-768_A-12.tar.gz'),
('BERT-en-cased-large','https://bert-models.bj.bcebos.com/cased_L-24_H-1024_A-16.tar.gz'),
('BERT-multilingual-uncased-base', 'https://bert-models.bj.bcebos.com/multilingual_L-12_H-768_A-12.tar.gz'),
('BERT-multilingual-cased-base', 'https://bert-models.bj.bcebos.com/multi_cased_L-12_H-768_A-12.tar.gz'),
('BERT-zh-base', 'https://bert-models.bj.bcebos.com/chinese_L-12_H-768_A-12.tar.gz'),)
)
nlp_models = OrderedDict((
('RoBERTa-zh-base',
'https://bert-models.bj.bcebos.com/chinese_roberta_wwm_ext_L-12_H-768_A-12.tar.gz'
),
('RoBERTa-zh-large',
'https://bert-models.bj.bcebos.com/chinese_roberta_wwm_large_ext_L-24_H-1024_A-16.tar.gz'
),
('ERNIE-v2-en-base',
'https://ernie.bj.bcebos.com/ERNIE_Base_en_stable-2.0.0.tar.gz'),
('ERNIE-v2-en-large',
'https://ernie.bj.bcebos.com/ERNIE_Large_en_stable-2.0.0.tar.gz'),
('XLNet-cased-base',
'https://xlnet.bj.bcebos.com/xlnet_cased_L-12_H-768_A-12.tgz'),
('XLNet-cased-large',
'https://xlnet.bj.bcebos.com/xlnet_cased_L-24_H-1024_A-16.tgz'),
('ERNIE-v1-zh-base',
'https://baidu-nlp.bj.bcebos.com/ERNIE_stable-1.0.1.tar.gz'),
('ERNIE-v1-zh-base-max-len-512',
'https://ernie.bj.bcebos.com/ERNIE_1.0_max-len-512.tar.gz'),
('BERT-en-uncased-large-whole-word-masking',
'https://bert-models.bj.bcebos.com/wwm_uncased_L-24_H-1024_A-16.tar.gz'),
('BERT-en-cased-large-whole-word-masking',
'https://bert-models.bj.bcebos.com/wwm_cased_L-24_H-1024_A-16.tar.gz'),
('BERT-en-uncased-base',
'https://bert-models.bj.bcebos.com/uncased_L-12_H-768_A-12.tar.gz'),
('BERT-en-uncased-large',
'https://bert-models.bj.bcebos.com/uncased_L-24_H-1024_A-16.tar.gz'),
('BERT-en-cased-base',
'https://bert-models.bj.bcebos.com/cased_L-12_H-768_A-12.tar.gz'),
('BERT-en-cased-large',
'https://bert-models.bj.bcebos.com/cased_L-24_H-1024_A-16.tar.gz'),
('BERT-multilingual-uncased-base',
'https://bert-models.bj.bcebos.com/multilingual_L-12_H-768_A-12.tar.gz'),
('BERT-multilingual-cased-base',
'https://bert-models.bj.bcebos.com/multi_cased_L-12_H-768_A-12.tar.gz'),
('BERT-zh-base',
'https://bert-models.bj.bcebos.com/chinese_L-12_H-768_A-12.tar.gz'), ))
def is_url(path):
......@@ -76,6 +118,15 @@ def get_weights_path_from_url(url, md5sum=None):
Returns:
str: a local path to save downloaded weights.
Examples:
.. code-block:: python
from hapi.download import get_weights_path_from_url
resnet18_pretrained_weight_url = 'https://paddle-hapi.bj.bcebos.com/models/resnet18.pdparams'
local_weight_path = get_weights_path_from_url(resnet18_pretrained_weight_url)
"""
path = get_path_from_url(url, WEIGHTS_HOME, md5sum)
return path
......@@ -153,11 +204,10 @@ def _download(url, path, md5sum=None):
total_size = req.headers.get('content-length')
with open(tmp_fullname, 'wb') as f:
if total_size:
for chunk in tqdm.tqdm(
req.iter_content(chunk_size=1024),
total=(int(total_size) + 1023) // 1024,
unit='KB'):
f.write(chunk)
with tqdm(total=(int(total_size) + 1023) // 1024) as pbar:
for chunk in req.iter_content(chunk_size=1024):
f.write(chunk)
pbar.update(1)
else:
for chunk in req.iter_content(chunk_size=1024):
if chunk:
......
hapi/logger.py
浏览文件 @ 07ede80c
......@@ -36,13 +36,13 @@ def setup_logger(output=None, name="hapi", log_level=logging.INFO):
logger.propagate = False
logger.setLevel(log_level)
format_str = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
# stdout logging: only local rank==0
local_rank = ParallelEnv().local_rank
if local_rank == 0:
if local_rank == 0 and len(logger.handlers) == 0:
ch = logging.StreamHandler(stream=sys.stdout)
ch.setLevel(log_level)
format_str = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
ch.setFormatter(logging.Formatter(format_str))
logger.addHandler(ch)
......@@ -52,6 +52,7 @@ def setup_logger(output=None, name="hapi", log_level=logging.INFO):
filename = output
else:
filename = os.path.join(output, "log.txt")
if local_rank > 0:
filename = filename + ".rank{}".format(local_rank)
......
hapi/loss.py
浏览文件 @ 07ede80c
......@@ -30,8 +30,8 @@ class Loss(object):
Base class for loss, encapsulates loss logic and APIs
Usage:
custom_loss = CustomLoss()
loss = custom_loss(inputs, labels)
custom_loss = CustomLoss()
loss = custom_loss(inputs, labels)
"""
def __init__(self, average=True):
......@@ -63,6 +63,21 @@ class CrossEntropy(Loss):
average (bool, optional): Indicate whether to average the loss, Default: True.
Returns:
list[Variable]: The tensor variable storing the cross_entropy_loss of inputs and labels.
Examples:
.. code-block:: python
from hapi.model import Input
from hapi.vision.models import LeNet
from hapi.loss import CrossEntropy
inputs = [Input([-1, 1, 28, 28], 'float32', name='image')]
labels = [Input([None, 1], 'int64', name='label')]
model = LeNet()
loss = CrossEntropy()
model.prepare(loss_function=loss, inputs=inputs, labels=labels)
"""
def __init__(self, average=True):
......@@ -85,6 +100,20 @@ class SoftmaxWithCrossEntropy(Loss):
average (bool, optional): Indicate whether to average the loss, Default: True.
Returns:
list[Variable]: The tensor variable storing the cross_entropy_loss of inputs and labels.
Examples:
.. code-block:: python
from hapi.model import Input
from hapi.vision.models import LeNet
from hapi.loss import SoftmaxWithCrossEntropy
inputs = [Input([-1, 1, 28, 28], 'float32', name='image')]
labels = [Input([None, 1], 'int64', name='label')]
model = LeNet(classifier_activation=None)
loss = SoftmaxWithCrossEntropy()
model.prepare(loss_function=loss, inputs=inputs, labels=labels)
"""
def __init__(self, average=True):
......
hapi/model.py
浏览文件 @ 07ede80c
......@@ -20,7 +20,6 @@ import pickle
import numpy as np
import six
import warnings
import tqdm
from collections import Iterable
from paddle import fluid
......@@ -1257,11 +1256,6 @@ class Model(fluid.dygraph.Layer):
assert train_data is not None, \
"train_data must be given!"
if fluid.in_dygraph_mode():
feed_list = None
else:
feed_list = [x.forward() for x in self._inputs + self._labels]
if isinstance(train_data, Dataset):
train_sampler = DistributedBatchSampler(
train_data,
......@@ -1272,7 +1266,6 @@ class Model(fluid.dygraph.Layer):
train_data,
batch_sampler=train_sampler,
places=self._place,
feed_list=feed_list,
num_workers=num_workers,
return_list=True)
else:
......@@ -1285,7 +1278,6 @@ class Model(fluid.dygraph.Layer):
eval_data,
batch_sampler=eval_sampler,
places=self._place,
feed_list=feed_list,
num_workers=num_workers,
return_list=True)
elif eval_data is not None:
......@@ -1295,7 +1287,7 @@ class Model(fluid.dygraph.Layer):
do_eval = eval_loader is not None
self._test_dataloader = eval_loader
metrics_name = self._metrics_name()
steps = self._len_data_loader(train_loader)
cbks = config_callbacks(
callbacks,
......@@ -1311,26 +1303,19 @@ class Model(fluid.dygraph.Layer):
cbks.on_begin('train')
for epoch in range(epochs):
# FIXME: adapt to DataLoader
loader = train_loader
if not isinstance(train_loader, Iterable):
loader = train_loader()
logs = self._run_one_epoch(
loader, cbks, 'train', metrics_name, epoch=epoch)
cbks.on_epoch_begin(epoch)
logs = self._run_one_epoch(train_loader, cbks, 'train')
cbks.on_epoch_end(epoch, logs)
if do_eval and epoch % eval_freq == 0:
# FIXME: adapt to DataLoader
loader = eval_loader
if not isinstance(eval_loader, Iterable):
loader = eval_loader()
eval_steps = self._len_data_loader(loader)
eval_steps = self._len_data_loader(eval_loader)
cbks.on_begin('eval', {
'steps': eval_steps,
'metrics_name': metrics_name
'metrics_name': self._metrics_name()
})
logs = self._run_one_epoch(loader, cbks, 'eval', metrics_name)
logs = self._run_one_epoch(eval_loader, cbks, 'eval')
cbks.on_end('eval', logs)
......@@ -1369,12 +1354,41 @@ class Model(fluid.dygraph.Layer):
Returns:
dict: Result of metric. The key is the names of Metric,
value is a scalar or numpy.array.
"""
if fluid.in_dygraph_mode():
feed_list = None
else:
feed_list = [x.forward() for x in self._inputs + self._labels]
Examples:
.. code-block:: python
# declarative mode
import numpy as np
from hapi.metrics import Accuracy
from hapi.datasets import MNIST
from hapi.vision.transforms import Compose,Resize
from hapi.vision.models import LeNet
from hapi.model import Input, set_device
inputs = [Input([-1, 1, 28, 28], 'float32', name='image')]
labels = [Input([None, 1], 'int64', name='label')]
val_dataset = MNIST(mode='test')
model = LeNet()
model.prepare(metrics=Accuracy(), inputs=inputs, labels=labels)
result = model.evaluate(val_dataset, batch_size=64)
print(result)
# imperative mode
import paddle.fluid.dygraph as dg
place = set_device('cpu')
with dg.guard(place) as g:
model = LeNet()
model.prepare(metrics=Accuracy(), inputs=inputs, labels=labels)
result = model.evaluate(val_dataset, batch_size=64)
print(result)
"""
if eval_data is not None and isinstance(eval_data, Dataset):
eval_sampler = DistributedBatchSampler(
......@@ -1383,14 +1397,12 @@ class Model(fluid.dygraph.Layer):
eval_data,
batch_sampler=eval_sampler,
places=self._place,
feed_list=feed_list,
num_workers=num_workers,
return_list=True)
else:
eval_loader = eval_data
self._test_dataloader = eval_loader
metrics_name = self._metrics_name()
cbks = config_callbacks(
callbacks,
......@@ -1399,16 +1411,13 @@ class Model(fluid.dygraph.Layer):
verbose=verbose,
metrics=self._metrics_name(), )
loader = eval_loader
if not isinstance(eval_loader, Iterable):
loader = eval_loader()
eval_steps = self._len_data_loader(eval_loader)
cbks.on_begin('eval', {
'steps': eval_steps,
'metrics_name': self._metrics_name()
})
eval_steps = self._len_data_loader(loader)
cbks.on_begin('eval',
{'steps': eval_steps,
'metrics_name': metrics_name})
logs = self._run_one_epoch(loader, cbks, 'eval', metrics_name)
logs = self._run_one_epoch(eval_loader, cbks, 'eval')
cbks.on_end('eval', logs)
......@@ -1424,7 +1433,8 @@ class Model(fluid.dygraph.Layer):
test_data,
batch_size=1,
num_workers=0,
stack_outputs=False):
stack_outputs=False,
callbacks=None):
"""
Compute the output predictions on testing data.
......@@ -1446,12 +1456,52 @@ class Model(fluid.dygraph.Layer):
it is recommended set as True if outputs contains no LoDTensor. Default: False.
Returns:
list: output of models.
"""
if fluid.in_dygraph_mode():
feed_list = None
else:
feed_list = [x.forward() for x in self._inputs]
Examples:
.. code-block:: python
# declarative mode
import numpy as np
from hapi.metrics import Accuracy
from hapi.datasets import MNIST
from hapi.vision.transforms import Compose,Resize
from hapi.vision.models import LeNet
from hapi.model import Input, set_device
class MnistDataset(MNIST):
def __init__(self, mode, return_label=True):
super(MnistDataset, self).__init__(mode=mode)
self.return_label = return_label
def __getitem__(self, idx):
img = np.reshape(self.images[idx], [1, 28, 28])
if self.return_label:
return img, np.array(self.labels[idx]).astype('int64')
return img,
def __len__(self):
return len(self.images)
inputs = [Input([-1, 1, 28, 28], 'float32', name='image')]
test_dataset = MnistDataset(mode='test', return_label=False)
model = LeNet()
model.prepare(inputs=inputs)
result = model.predict(test_dataset, batch_size=64)
print(result)
# imperative mode
import paddle.fluid.dygraph as dg
place = set_device('cpu')
with dg.guard(place) as g:
model = LeNet()
model.prepare(inputs=inputs)
result = model.predict(test_dataset, batch_size=64)
print(result)
"""
if test_data is not None and isinstance(test_data, Dataset):
test_sampler = DistributedBatchSampler(
......@@ -1460,7 +1510,6 @@ class Model(fluid.dygraph.Layer):
test_data,
batch_sampler=test_sampler,
places=self._place,
feed_list=feed_list,
num_workers=num_workers,
return_list=True)
else:
......@@ -1468,34 +1517,27 @@ class Model(fluid.dygraph.Layer):
self._test_dataloader = test_loader
loader = test_loader
if not isinstance(test_loader, Iterable):
loader = test_loader()
cbks = config_callbacks(callbacks, model=self, verbose=1)
outputs = []
count = 0
for data in tqdm.tqdm(loader):
data = flatten(data)
out = to_list(self.test_batch(data[:len(self._inputs)]))
outputs.append(out)
count += out[0].shape[0]
test_steps = self._len_data_loader(test_loader)
logs = {'steps': test_steps}
if test_loader is not None and self._adapter._nranks > 1 \
and isinstance(test_loader, DataLoader) \
and count > len(test_loader.dataset):
size = outputs[-1][0].shape[0] - (count - len(test_loader.dataset))
outputs[-1] = [o[:size] for o in outputs[-1]]
cbks.on_begin('test', logs)
# NOTE: for lod tensor output, we should not stack outputs
# for stacking may loss its detail info
outputs = []
logs, outputs = self._run_one_epoch(test_loader, cbks, 'test')
outputs = list(zip(*outputs))
# NOTE: for lod tensor output, we should not stack outputs
# for stacking may loss its detail info
if stack_outputs:
outputs = [np.vstack(outs) for outs in outputs]
self._test_dataloader = None
cbks.on_end('test', logs)
return outputs
def save_inference_model(self,
......@@ -1542,22 +1584,8 @@ class Model(fluid.dygraph.Layer):
params_filename=params_filename,
program_only=model_only)
def _run_one_epoch(self,
data_loader,
callbacks,
mode,
metrics_name,
epoch=None):
size = self._len_data_loader(data_loader)
logs = {
'steps': size,
'metrics_name': metrics_name,
}
if mode == 'train':
assert epoch is not None, 'when mode is train, epoch must be given'
callbacks.on_epoch_begin(epoch)
def _run_one_epoch(self, data_loader, callbacks, mode, logs={}):
outputs = []
for step, data in enumerate(data_loader):
# data might come from different types of data_loader and have
# different format, as following:
......@@ -1577,25 +1605,25 @@ class Model(fluid.dygraph.Layer):
0].shape) else data[0].shape[0]
callbacks.on_batch_begin(mode, step, logs)
if mode == 'train':
outs = self.train_batch(data[:len(self._inputs)],
data[len(self._inputs):])
else:
outs = self.eval_batch(data[:len(self._inputs)],
data[len(self._inputs):])
# losses
loss = outs[0] if self._metrics else outs
metrics = [[l[0] for l in loss]]
# metrics
for metric in self._metrics:
res = metric.accumulate()
metrics.extend(to_list(res))
assert len(metrics_name) == len(metrics)
for k, v in zip(metrics_name, metrics):
logs[k] = v
if mode != 'test':
outs = getattr(self, mode + '_batch')(data[:len(self._inputs)],
data[len(self._inputs):])
# losses
loss = outs[0] if self._metrics else outs
metrics = [[l[0] for l in loss]]
# metrics
for metric in self._metrics:
res = metric.accumulate()
metrics.extend(to_list(res))
assert len(self._metrics_name()) == len(metrics)
for k, v in zip(self._metrics_name(), metrics):
logs[k] = v
else:
outs = getattr(self, mode + '_batch')(data)
outputs.append(outs)
logs['step'] = step
if mode == 'train' or self._adapter._merge_count.get(
......@@ -1608,10 +1636,8 @@ class Model(fluid.dygraph.Layer):
callbacks.on_batch_end(mode, step, logs)
self._reset_metrics()
if mode == 'train':
assert epoch is not None, 'when mode is train, epoch must be given'
callbacks.on_epoch_end(epoch, logs)
if mode == 'test':
return logs, outputs
return logs
def _reset_metrics(self):
......
hapi/tests/dist_mnist.py
浏览文件 @ 07ede80c
......@@ -48,7 +48,7 @@ class MnistDataset(MNIST):
return len(self.images)
def get_predict_accuracy(pred, gt):
def compute_accuracy(pred, gt):
pred = np.argmax(pred, -1)
gt = np.array(gt)
......@@ -58,7 +58,7 @@ def get_predict_accuracy(pred, gt):
class TestModel(unittest.TestCase):
def fit(self, dynamic):
def run(self, dynamic):
device = set_device('gpu')
fluid.enable_dygraph(device) if dynamic else None
......@@ -74,7 +74,9 @@ class TestModel(unittest.TestCase):
model = LeNet()
optim = fluid.optimizer.Momentum(
learning_rate=0.01, momentum=.9, parameter_list=model.parameters())
learning_rate=0.001,
momentum=.9,
parameter_list=model.parameters())
loss = CrossEntropy()
model.prepare(optim, loss, Accuracy(), inputs, labels, device=device)
cbk = ProgBarLogger(50)
......@@ -92,15 +94,15 @@ class TestModel(unittest.TestCase):
np.testing.assert_equal(output[0].shape[0], len(test_dataset))
acc = get_predict_accuracy(output[0], val_dataset.labels)
acc = compute_accuracy(output[0], val_dataset.labels)
np.testing.assert_allclose(acc, eval_result['acc'])
def test_multiple_gpus_static(self):
self.fit(False)
self.run(False)
def test_multiple_gpus_dygraph(self):
self.fit(True)
self.run(True)
if __name__ == '__main__':
......
hapi/tests/test_callbacks.py
浏览文件 @ 07ede80c
......@@ -12,27 +12,43 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# when test, you should add hapi root path to the PYTHONPATH,
# export PYTHONPATH=PATH_TO_HAPI:$PYTHONPATH
import unittest
import time
import random
import tempfile
import shutil
from hapi.model import Input
from hapi.vision.models import LeNet
from hapi.callbacks import config_callbacks
class TestCallbacks(unittest.TestCase):
def test_callback(self):
def setUp(self):
self.save_dir = tempfile.mkdtemp()
def tearDown(self):
shutil.rmtree(self.save_dir)
def run_callback(self):
epochs = 2
steps = 50
freq = 1
freq = 2
eval_steps = 20
lenet = LeNet()
inputs = [Input([None, 1, 28, 28], 'float32', name='image')]
lenet.prepare(inputs=inputs)
cbks = config_callbacks(
model=lenet,
batch_size=128,
epochs=epochs,
steps=steps,
verbose=2,
metrics=['loss', 'acc'], )
log_freq=freq,
verbose=self.verbose,
metrics=['loss', 'acc'],
save_dir=self.save_dir)
cbks.on_begin('train')
logs = {'loss': 50.341673, 'acc': 0.00256}
......@@ -48,13 +64,12 @@ class TestCallbacks(unittest.TestCase):
eval_logs = {'eval_loss': 20.341673, 'eval_acc': 0.256}
params = {
'eval_steps': eval_steps,
'eval_metrics': ['eval_loss', 'eval_acc'],
'log_freq': 10,
'steps': eval_steps,
'metrics_name': ['eval_loss', 'eval_acc'],
}
cbks.on_begin('eval', params)
for step in range(eval_steps):
cbks.on_batch_begin('eval', step, logs)
cbks.on_batch_begin('eval', step, eval_logs)
eval_logs['eval_loss'] -= random.random() * 0.1
eval_logs['eval_acc'] += random.random() * 0.1
eval_logs['batch_size'] = 2
......@@ -62,8 +77,30 @@ class TestCallbacks(unittest.TestCase):
cbks.on_batch_end('eval', step, eval_logs)
cbks.on_end('eval', eval_logs)
test_logs = {}
params = {'steps': eval_steps}
cbks.on_begin('test', params)
for step in range(eval_steps):
cbks.on_batch_begin('test', step, test_logs)
test_logs['batch_size'] = 2
time.sleep(0.005)
cbks.on_batch_end('test', step, test_logs)
cbks.on_end('test', test_logs)
cbks.on_end('train')
def test_callback_verbose_0(self):
self.verbose = 0
self.run_callback()
def test_callback_verbose_1(self):
self.verbose = 1
self.run_callback()
def test_callback_verbose_2(self):
self.verbose = 2
self.run_callback()
if __name__ == '__main__':
unittest.main()
hapi/tests/test_datasets.py
浏览文件 @ 07ede80c
......@@ -20,11 +20,14 @@ import shutil
import cv2
from hapi.datasets import *
from hapi.datasets.utils import _check_exists_and_download
from hapi.vision.transforms import Compose
class TestFolderDatasets(unittest.TestCase):
def makedata(self):
def setUp(self):
self.data_dir = tempfile.mkdtemp()
self.empty_dir = tempfile.mkdtemp()
for i in range(2):
sub_dir = os.path.join(self.data_dir, 'class_' + str(i))
if not os.path.exists(sub_dir):
......@@ -34,8 +37,10 @@ class TestFolderDatasets(unittest.TestCase):
(32, 32, 3)) * 255).astype('uint8')
cv2.imwrite(os.path.join(sub_dir, str(j) + '.jpg'), fake_img)
def tearDown(self):
shutil.rmtree(self.data_dir)
def test_dataset(self):
self.makedata()
dataset_folder = DatasetFolder(self.data_dir)
for _ in dataset_folder:
......@@ -44,7 +49,30 @@ class TestFolderDatasets(unittest.TestCase):
assert len(dataset_folder) == 4
assert len(dataset_folder.classes) == 2
shutil.rmtree(self.data_dir)
transform = Compose([])
dataset_folder = DatasetFolder(self.data_dir, transform=transform)
for _ in dataset_folder:
pass
def test_folder(self):
loader = ImageFolder(self.data_dir)
for _ in loader:
pass
transform = Compose([])
loader = ImageFolder(self.data_dir, transform=transform)
for _ in loader:
pass
def test_errors(self):
with self.assertRaises(RuntimeError):
ImageFolder(self.empty_dir)
with self.assertRaises(RuntimeError):
DatasetFolder(self.empty_dir)
with self.assertRaises(ValueError):
_check_exists_and_download('temp_paddle', None, None, None, False)
class TestMNISTTest(unittest.TestCase):
......
hapi/tests/test_distributed.py
浏览文件 @ 07ede80c
......@@ -30,40 +30,21 @@ import paddle.distributed.cloud_utils as cloud_utils
def get_cluster_from_args(selected_gpus):
cluster_node_ips = '127.0.0.1'
node_ip = '127.0.0.1'
use_paddlecloud = False
started_port = None
node_ips = [x.strip() for x in cluster_node_ips.split(',')]
node_rank = node_ips.index(node_ip)
node_ips.index(node_ip)
free_ports = None
if not use_paddlecloud and len(node_ips) <= 1 and started_port is None:
free_ports = find_free_ports(len(selected_gpus))
if free_ports is not None:
free_ports = list(free_ports)
else:
started_port = 6070
free_ports = [
x for x in range(started_port, started_port + len(selected_gpus))
]
free_ports = find_free_ports(len(selected_gpus))
if free_ports is not None:
free_ports = list(free_ports)
return get_cluster(node_ips, node_ip, free_ports, selected_gpus)
def get_gpus(selected_gpus):
cuda_visible_devices = os.getenv("CUDA_VISIBLE_DEVICES")
if cuda_visible_devices is None or cuda_visible_devices == "":
selected_gpus = [x.strip() for x in selected_gpus.split(',')]
else:
cuda_visible_devices_list = cuda_visible_devices.split(',')
for x in selected_gpus.split(','):
assert x in cuda_visible_devices_list, "Can't find "\
"your selected_gpus %s in CUDA_VISIBLE_DEVICES[%s]."\
% (x, cuda_visible_devices)
selected_gpus = [
cuda_visible_devices_list.index(x.strip())
for x in selected_gpus.split(',')
]
selected_gpus = [x.strip() for x in selected_gpus.split(',')]
return selected_gpus
......@@ -94,7 +75,7 @@ def start_local_trainers(cluster,
print("trainer proc env:{}".format(current_env))
cmd = "python -m coverage run --branch -p " + training_script
cmd = "python -u " + training_script
print("start trainer proc:{} env:{}".format(cmd, proc_env))
......
hapi/vision/models/darknet.py
浏览文件 @ 07ede80c
......@@ -144,6 +144,13 @@ class DarkNet(Model):
will not be defined. Default: 1000.
with_pool (bool): use pool before the last fc layer or not. Default: True.
classifier_activation (str): activation for the last fc layer. Default: 'softmax'.
Examples:
.. code-block:: python
from hapi.vision.models import DarkNet
model = DarkNet()
"""
def __init__(self,
......@@ -233,5 +240,17 @@ def darknet53(pretrained=False, **kwargs):
input_channels (bool): channel number of input data, default 3.
pretrained (bool): If True, returns a model pre-trained on ImageNet,
default True.
Examples:
.. code-block:: python
from hapi.vision.models import darknet53
# build model
model = darknet53()
#build model and load imagenet pretrained weight
model = darknet53(pretrained=True)
"""
return _darknet('darknet53', 53, pretrained, **kwargs)
hapi/vision/models/lenet.py
浏览文件 @ 07ede80c
......@@ -29,6 +29,13 @@ class LeNet(Model):
num_classes (int): output dim of last fc layer. If num_classes <=0, last fc layer
will not be defined. Default: 10.
classifier_activation (str): activation for the last fc layer. Default: 'softmax'.
Examples:
.. code-block:: python
from hapi.vision.models import LeNet
model = LeNet()
"""
def __init__(self, num_classes=10, classifier_activation='softmax'):
......
hapi/vision/models/mobilenetv1.py
浏览文件 @ 07ede80c
......@@ -115,6 +115,13 @@ class MobileNetV1(Model):
will not be defined. Default: 1000.
with_pool (bool): use pool before the last fc layer or not. Default: True.
classifier_activation (str): activation for the last fc layer. Default: 'softmax'.
Examples:
.. code-block:: python
from hapi.vision.models import MobileNetV1
model = MobileNetV1()
"""
def __init__(self,
......@@ -282,6 +289,20 @@ def mobilenet_v1(pretrained=False, scale=1.0, **kwargs):
Args:
pretrained (bool): If True, returns a model pre-trained on ImageNet. Default: False.
scale: (float): scale of channels in each layer. Default: 1.0.
Examples:
.. code-block:: python
from hapi.vision.models import mobilenet_v1
# build model
model = mobilenet_v1()
#build model and load imagenet pretrained weight
model = mobilenet_v1(pretrained=True)
#build mobilenet v1 with scale=0.5
model = mobilenet_v1(scale=0.5)
"""
model = _mobilenet(
'mobilenetv1_' + str(scale), pretrained, scale=scale, **kwargs)
......
hapi/vision/models/mobilenetv2.py
浏览文件 @ 07ede80c
......@@ -160,6 +160,13 @@ class MobileNetV2(Model):
will not be defined. Default: 1000.
with_pool (bool): use pool before the last fc layer or not. Default: True.
classifier_activation (str): activation for the last fc layer. Default: 'softmax'.
Examples:
.. code-block:: python
from hapi.vision.models import MobileNetV2
model = MobileNetV2()
"""
def __init__(self,
......@@ -256,6 +263,20 @@ def mobilenet_v2(pretrained=False, scale=1.0, **kwargs):
Args:
pretrained (bool): If True, returns a model pre-trained on ImageNet. Default: False.
scale: (float): scale of channels in each layer. Default: 1.0.
Examples:
.. code-block:: python
from hapi.vision.models import mobilenet_v2
# build model
model = mobilenet_v2()
#build model and load imagenet pretrained weight
model = mobilenet_v2(pretrained=True)
#build mobilenet v2 with scale=0.5
model = mobilenet_v2(scale=0.5)
"""
model = _mobilenet(
'mobilenetv2_' + str(scale), pretrained, scale=scale, **kwargs)
......
hapi/vision/models/resnet.py
浏览文件 @ 07ede80c
......@@ -26,7 +26,8 @@ from hapi.model import Model
from hapi.download import get_weights_path_from_url
__all__ = [
'ResNet', 'resnet18', 'resnet34', 'resnet50', 'resnet101', 'resnet152'
'ResNet', 'resnet18', 'resnet34', 'resnet50', 'resnet101', 'resnet152',
'BottleneckBlock', 'BasicBlock'
]
model_urls = {
......@@ -35,7 +36,7 @@ model_urls = {
'resnet34': ('https://paddle-hapi.bj.bcebos.com/models/resnet34.pdparams',
'46bc9f7c3dd2e55b7866285bee91eff3'),
'resnet50': ('https://paddle-hapi.bj.bcebos.com/models/resnet50.pdparams',
'0884c9087266496c41c60d14a96f8530'),
'5ce890a9ad386df17cf7fe2313dca0a1'),
'resnet101':
('https://paddle-hapi.bj.bcebos.com/models/resnet101.pdparams',
'fb07a451df331e4b0bb861ed97c3a9b9'),
......@@ -75,7 +76,8 @@ class ConvBNLayer(fluid.dygraph.Layer):
class BasicBlock(fluid.dygraph.Layer):
"""residual block of resnet18 and resnet34
"""
expansion = 1
def __init__(self, num_channels, num_filters, stride, shortcut=True):
......@@ -117,6 +119,8 @@ class BasicBlock(fluid.dygraph.Layer):
class BottleneckBlock(fluid.dygraph.Layer):
"""residual block of resnet50, resnet101 amd resnet152
"""
expansion = 4
......@@ -177,6 +181,16 @@ class ResNet(Model):
will not be defined. Default: 1000.
with_pool (bool): use pool before the last fc layer or not. Default: True.
classifier_activation (str): activation for the last fc layer. Default: 'softmax'.
Examples:
.. code-block:: python
from hapi.vision.models import ResNet, BottleneckBlock, BasicBlock
resnet50 = ResNet(BottleneckBlock, 50)
resnet18 = ResNet(BasicBlock, 18)
"""
def __init__(self,
......@@ -280,6 +294,17 @@ def resnet18(pretrained=False, **kwargs):
Args:
pretrained (bool): If True, returns a model pre-trained on ImageNet
Examples:
.. code-block:: python
from hapi.vision.models import resnet18
# build model
model = resnet18()
#build model and load imagenet pretrained weight
model = resnet18(pretrained=True)
"""
return _resnet('resnet18', BasicBlock, 18, pretrained, **kwargs)
......@@ -289,6 +314,17 @@ def resnet34(pretrained=False, **kwargs):
Args:
pretrained (bool): If True, returns a model pre-trained on ImageNet
Examples:
.. code-block:: python
from hapi.vision.models import resnet34
# build model
model = resnet34()
#build model and load imagenet pretrained weight
model = resnet34(pretrained=True)
"""
return _resnet('resnet34', BasicBlock, 34, pretrained, **kwargs)
......@@ -298,6 +334,17 @@ def resnet50(pretrained=False, **kwargs):
Args:
pretrained (bool): If True, returns a model pre-trained on ImageNet
Examples:
.. code-block:: python
from hapi.vision.models import resnet50
# build model
model = resnet50()
#build model and load imagenet pretrained weight
model = resnet50(pretrained=True)
"""
return _resnet('resnet50', BottleneckBlock, 50, pretrained, **kwargs)
......@@ -307,6 +354,17 @@ def resnet101(pretrained=False, **kwargs):
Args:
pretrained (bool): If True, returns a model pre-trained on ImageNet
Examples:
.. code-block:: python
from hapi.vision.models import resnet101
# build model
model = resnet101()
#build model and load imagenet pretrained weight
model = resnet101(pretrained=True)
"""
return _resnet('resnet101', BottleneckBlock, 101, pretrained, **kwargs)
......@@ -316,5 +374,16 @@ def resnet152(pretrained=False, **kwargs):
Args:
pretrained (bool): If True, returns a model pre-trained on ImageNet
Examples:
.. code-block:: python
from hapi.vision.models import resnet152
# build model
model = resnet152()
#build model and load imagenet pretrained weight
model = resnet152(pretrained=True)
"""
return _resnet('resnet152', BottleneckBlock, 152, pretrained, **kwargs)
hapi/vision/models/vgg.py
浏览文件 @ 07ede80c
......@@ -143,6 +143,17 @@ def vgg11(pretrained=False, batch_norm=False, **kwargs):
Args:
pretrained (bool): If True, returns a model pre-trained on ImageNet. Default: False.
batch_norm (bool): If True, returns a model with batch_norm layer. Default: False.
Examples:
.. code-block:: python
from hapi.vision.models import vgg11
# build model
model = vgg11()
#build vgg11 model with batch_norm
model = vgg11(batch_norm=True)
"""
model_name = 'vgg11'
if batch_norm:
......@@ -156,6 +167,17 @@ def vgg13(pretrained=False, batch_norm=False, **kwargs):
Args:
pretrained (bool): If True, returns a model pre-trained on ImageNet. Default: False.
batch_norm (bool): If True, returns a model with batch_norm layer. Default: False.
Examples:
.. code-block:: python
from hapi.vision.models import vgg13
# build model
model = vgg13()
#build vgg13 model with batch_norm
model = vgg13(batch_norm=True)
"""
model_name = 'vgg13'
if batch_norm:
......@@ -169,6 +191,17 @@ def vgg16(pretrained=False, batch_norm=False, **kwargs):
Args:
pretrained (bool): If True, returns a model pre-trained on ImageNet. Default: False.
batch_norm (bool): If True, returns a model with batch_norm layer. Default: False.
Examples:
.. code-block:: python
from hapi.vision.models import vgg16
# build model
model = vgg16()
#build vgg16 model with batch_norm
model = vgg16(batch_norm=True)
"""
model_name = 'vgg16'
if batch_norm:
......@@ -182,6 +215,17 @@ def vgg19(pretrained=False, batch_norm=False, **kwargs):
Args:
pretrained (bool): If True, returns a model pre-trained on ImageNet. Default: False.
batch_norm (bool): If True, returns a model with batch_norm layer. Default: False.
Examples:
.. code-block:: python
from hapi.vision.models import vgg19
# build model
model = vgg19()
#build vgg19 model with batch_norm
model = vgg19(batch_norm=True)
"""
model_name = 'vgg19'
if batch_norm:
......
hapi/vision/transforms/functional.py
浏览文件 @ 07ede80c
......@@ -35,10 +35,27 @@ def flip(image, code):
Args:
image: Input image, with (H, W, C) shape
code: code that indicates the type of flip.
code: Code that indicates the type of flip.
-1 : Flip horizontally and vertically
0 : Flip vertically
1 : Flip horizontally
Examples:
.. code-block:: python
import numpy as np
from hapi.vision.transforms import functional as F
fake_img = np.random.rand(224, 224, 3)
# flip horizontally and vertically
F.flip(fake_img, -1)
# flip vertically
F.flip(fake_img, 0)
# flip horizontally
F.flip(fake_img, 1)
"""
return cv2.flip(image, flipCode=code)
......@@ -51,6 +68,18 @@ def resize(img, size, interpolation=cv2.INTER_LINEAR):
input: Input data, could be image or masks, with (H, W, C) shape
size: Target size of input data, with (height, width) shape.
interpolation: Interpolation method.
Examples:
.. code-block:: python
import numpy as np
from hapi.vision.transforms import functional as F
fake_img = np.random.rand(256, 256, 3)
F.resize(fake_img, 224)
F.resize(fake_img, (200, 150))
"""
if isinstance(interpolation, Sequence):
......
hapi/vision/transforms/transforms.py
浏览文件 @ 07ede80c
......@@ -61,7 +61,7 @@ class Compose(object):
together for a dataset transform.
Args:
transforms (list of ``Transform`` objects): list of transforms to compose.
transforms (list): List of transforms to compose.
Returns:
A compose object which is callable, __call__ for this Compose
......@@ -115,9 +115,70 @@ class BatchCompose(object):
"""Composes several batch transforms together
Args:
transforms (list of ``Transform`` objects): list of transforms to compose.
these transforms perform on batch data.
transforms (list): List of transforms to compose.
these transforms perform on batch data.
Examples:
.. code-block:: python
import numpy as np
from paddle.io import DataLoader
from hapi.model import set_device
from hapi.datasets import Flowers
from hapi.vision.transforms import Compose, BatchCompose, Resize
class NormalizeBatch(object):
def __init__(self,
mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225],
scale=True,
channel_first=True):
self.mean = mean
self.std = std
self.scale = scale
self.channel_first = channel_first
if not (isinstance(self.mean, list) and isinstance(self.std, list) and
isinstance(self.scale, bool)):
raise TypeError("{}: input type is invalid.".format(self))
from functools import reduce
if reduce(lambda x, y: x * y, self.std) == 0:
raise ValueError('{}: std is invalid!'.format(self))
def __call__(self, samples):
for i in range(len(samples)):
samples[i] = list(samples[i])
im = samples[i][0]
im = im.astype(np.float32, copy=False)
mean = np.array(self.mean)[np.newaxis, np.newaxis, :]
std = np.array(self.std)[np.newaxis, np.newaxis, :]
if self.scale:
im = im / 255.0
im -= mean
im /= std
if self.channel_first:
im = im.transpose((2, 0, 1))
samples[i][0] = im
return samples
transform = Compose([Resize((500, 500))])
flowers_dataset = Flowers(mode='test', transform=transform)
device = set_device('cpu')
collate_fn = BatchCompose([NormalizeBatch()])
loader = DataLoader(
flowers_dataset,
batch_size=4,
places=device,
return_list=True,
collate_fn=collate_fn)
for data in loader:
# do something
break
"""
def __init__(self, transforms=[]):
......@@ -148,7 +209,22 @@ class Resize(object):
smaller edge of the image will be matched to this number.
i.e, if height > width, then image will be rescaled to
(size * height / width, size)
interpolation (int): interpolation mode of resize. Default: cv2.INTER_LINEAR.
interpolation (int): Interpolation mode of resize. Default: cv2.INTER_LINEAR.
Examples:
.. code-block:: python
import numpy as np
from hapi.vision.transforms import Resize
transform = Resize(size=224)
fake_img = np.random.rand(500, 500, 3).astype('float32')
fake_img = transform(fake_img)
print(fake_img.shape)
"""
def __init__(self, size, interpolation=cv2.INTER_LINEAR):
......@@ -171,6 +247,21 @@ class RandomResizedCrop(object):
output_size (int|list|tuple): Target size of output image, with (height, width) shape.
scale (list|tuple): Range of size of the origin size cropped. Default: (0.08, 1.0)
ratio (list|tuple): Range of aspect ratio of the origin aspect ratio cropped. Default: (0.75, 1.33)
Examples:
.. code-block:: python
import numpy as np
from hapi.vision.transforms import RandomResizedCrop
transform = RandomResizedCrop(224)
fake_img = np.random.rand(500, 500, 3).astype('float32')
fake_img = transform(fake_img)
print(fake_img.shape)
"""
def __init__(self,
......@@ -231,8 +322,23 @@ class CenterCropResize(object):
Args:
size (int|list|tuple): Target size of output image, with (height, width) shape.
crop_padding (int): center crop with the padding. Default: 32.
interpolation (int): interpolation mode of resize. Default: cv2.INTER_LINEAR.
crop_padding (int): Center crop with the padding. Default: 32.
interpolation (int): Interpolation mode of resize. Default: cv2.INTER_LINEAR.
Examples:
.. code-block:: python
import numpy as np
from hapi.vision.transforms import CenterCropResize
transform = CenterCropResize(224)
fake_img = np.random.rand(500, 500, 3).astype('float32')
fake_img = transform(fake_img)
print(fake_img.shape)
"""
def __init__(self, size, crop_padding=32, interpolation=cv2.INTER_LINEAR):
......@@ -262,6 +368,21 @@ class CenterCrop(object):
Args:
output_size: Target size of output image, with (height, width) shape.
Examples:
.. code-block:: python
import numpy as np
from hapi.vision.transforms import CenterCrop
transform = CenterCrop(224)
fake_img = np.random.rand(500, 500, 3).astype('float32')
fake_img = transform(fake_img)
print(fake_img.shape)
"""
def __init__(self, output_size):
......@@ -288,7 +409,22 @@ class RandomHorizontalFlip(object):
"""Horizontally flip the input data randomly with a given probability.
Args:
prob (float): probability of the input data being flipped. Default: 0.5
prob (float): Probability of the input data being flipped. Default: 0.5
Examples:
.. code-block:: python
import numpy as np
from hapi.vision.transforms import RandomHorizontalFlip
transform = RandomHorizontalFlip(224)
fake_img = np.random.rand(500, 500, 3).astype('float32')
fake_img = transform(fake_img)
print(fake_img.shape)
"""
def __init__(self, prob=0.5):
......@@ -304,7 +440,22 @@ class RandomVerticalFlip(object):
"""Vertically flip the input data randomly with a given probability.
Args:
prob (float): probability of the input data being flipped. Default: 0.5
prob (float): Probability of the input data being flipped. Default: 0.5
Examples:
.. code-block:: python
import numpy as np
from hapi.vision.transforms import RandomVerticalFlip
transform = RandomVerticalFlip(224)
fake_img = np.random.rand(500, 500, 3).astype('float32')
fake_img = transform(fake_img)
print(fake_img.shape)
"""
def __init__(self, prob=0.5):
......@@ -325,6 +476,22 @@ class Normalize(object):
Args:
mean (int|float|list): Sequence of means for each channel.
std (int|float|list): Sequence of standard deviations for each channel.
Examples:
.. code-block:: python
import numpy as np
from hapi.vision.transforms import Normalize
normalize = Normalize(mean=[0.5, 0.5, 0.5],
std=[0.5, 0.5, 0.5])
fake_img = np.random.rand(3, 500, 500).astype('float32')
fake_img = normalize(fake_img)
print(fake_img.shape)
"""
......@@ -349,8 +516,23 @@ class Permute(object):
Input image should be HWC mode and an instance of numpy.ndarray.
Args:
mode: Output mode of input. Default: "CHW".
to_rgb: convert 'bgr' image to 'rgb'. Default: True.
mode (str): Output mode of input. Default: "CHW".
to_rgb (bool): Convert 'bgr' image to 'rgb'. Default: True.
Examples:
.. code-block:: python
import numpy as np
from hapi.vision.transforms import Permute
transform = Permute()
fake_img = np.random.rand(500, 500, 3).astype('float32')
fake_img = transform(fake_img)
print(fake_img.shape)
"""
def __init__(self, mode="CHW", to_rgb=True):
......@@ -373,8 +555,23 @@ class GaussianNoise(object):
Gaussian noise is generated with given mean and std.
Args:
mean: Gaussian mean used to generate noise.
std: Gaussian standard deviation used to generate noise.
mean (float): Gaussian mean used to generate noise.
std (float): Gaussian standard deviation used to generate noise.
Examples:
.. code-block:: python
import numpy as np
from hapi.vision.transforms import GaussianNoise
transform = GaussianNoise()
fake_img = np.random.rand(500, 500, 3).astype('float32')
fake_img = transform(fake_img)
print(fake_img.shape)
"""
def __init__(self, mean=0.0, std=1.0):
......@@ -392,8 +589,23 @@ class BrightnessTransform(object):
"""Adjust brightness of the image.
Args:
value: How much to adjust the brightness. Can be any
value (float): How much to adjust the brightness. Can be any
non negative number. 0 gives the original image
Examples:
.. code-block:: python
import numpy as np
from hapi.vision.transforms import BrightnessTransform
transform = BrightnessTransform(0.4)
fake_img = np.random.rand(500, 500, 3).astype('float32')
fake_img = transform(fake_img)
print(fake_img.shape)
"""
def __init__(self, value):
......@@ -416,8 +628,23 @@ class ContrastTransform(object):
"""Adjust contrast of the image.
Args:
value: How much to adjust the contrast. Can be any
value (float): How much to adjust the contrast. Can be any
non negative number. 0 gives the original image
Examples:
.. code-block:: python
import numpy as np
from hapi.vision.transforms import ContrastTransform
transform = ContrastTransform(0.4)
fake_img = np.random.rand(500, 500, 3).astype('float32')
fake_img = transform(fake_img)
print(fake_img.shape)
"""
def __init__(self, value):
......@@ -441,8 +668,23 @@ class SaturationTransform(object):
"""Adjust saturation of the image.
Args:
value: How much to adjust the saturation. Can be any
value (float): How much to adjust the saturation. Can be any
non negative number. 0 gives the original image
Examples:
.. code-block:: python
import numpy as np
from hapi.vision.transforms import SaturationTransform
transform = SaturationTransform(0.4)
fake_img = np.random.rand(500, 500, 3).astype('float32')
fake_img = transform(fake_img)
print(fake_img.shape)
"""
def __init__(self, value):
......@@ -467,8 +709,23 @@ class HueTransform(object):
"""Adjust hue of the image.
Args:
value: How much to adjust the hue. Can be any number
value (float): How much to adjust the hue. Can be any number
between 0 and 0.5, 0 gives the original image
Examples:
.. code-block:: python
import numpy as np
from hapi.vision.transforms import HueTransform
transform = HueTransform(0.4)
fake_img = np.random.rand(500, 500, 3).astype('float32')
fake_img = transform(fake_img)
print(fake_img.shape)
"""
def __init__(self, value):
......@@ -510,6 +767,21 @@ class ColorJitter(object):
hue: How much to jitter hue.
Chosen uniformly from [-hue, hue] or the given [min, max].
Should have 0<= hue <= 0.5 or -0.5 <= min <= max <= 0.5.
Examples:
.. code-block:: python
import numpy as np
from hapi.vision.transforms import ColorJitter
transform = ColorJitter(0.4)
fake_img = np.random.rand(500, 500, 3).astype('float32')
fake_img = transform(fake_img)
print(fake_img.shape)
"""
def __init__(self, brightness=0, contrast=0, saturation=0, hue=0):
......
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或