add code 9.6.0: prepare pikachu data

code/chapter09_computer-vision/9.6.0_prepare_pikachu.ipynb

+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# 9.6.0 准备皮卡丘数据集"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "import json\n",
+    "from tqdm import tqdm\n",
+    "import numpy as np\n",
+    "import matplotlib.pyplot as plt\n",
+    "from mxnet.gluon import utils as gutils # pip install mxnet\n",
+    "from mxnet import image\n",
+    "\n",
+    "data_dir = '../../data/pikachu'\n",
+    "os.makedirs(data_dir, exist_ok=True)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## 1. 下载原始数据集\n",
+    "见http://zh.d2l.ai/chapter_computer-vision/object-detection-dataset.html"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "def _download_pikachu(data_dir):\n",
+    "    root_url = ('https://apache-mxnet.s3-accelerate.amazonaws.com/'\n",
+    "                'gluon/dataset/pikachu/')\n",
+    "    dataset = {'train.rec': 'e6bcb6ffba1ac04ff8a9b1115e650af56ee969c8',\n",
+    "               'train.idx': 'dcf7318b2602c06428b9988470c731621716c393',\n",
+    "               'val.rec': 'd6c33f799b4d058e82f2cb5bd9a976f69d72d520'}\n",
+    "    for k, v in dataset.items():\n",
+    "        gutils.download(root_url + k, os.path.join(data_dir, k), sha1_hash=v)\n",
+    "\n",
+    "if not os.path.exists(os.path.join(data_dir, \"train.rec\")):\n",
+    "    print(\"下载原始数据集到%s...\" % data_dir)\n",
+    "    _download_pikachu(data_dir)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## 2. MXNet数据迭代器"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "def load_data_pikachu(batch_size, edge_size=256):  # edge_size：输出图像的宽和高\n",
+    "    train_iter = image.ImageDetIter(\n",
+    "        path_imgrec=os.path.join(data_dir, 'train.rec'),\n",
+    "        path_imgidx=os.path.join(data_dir, 'train.idx'),\n",
+    "        batch_size=batch_size,\n",
+    "        data_shape=(3, edge_size, edge_size),  # 输出图像的形状\n",
+    "#         shuffle=False,  # 以随机顺序读取数据集\n",
+    "#         rand_crop=1,  # 随机裁剪的概率为1\n",
+    "        min_object_covered=0.95, max_attempts=200)\n",
+    "    val_iter = image.ImageDetIter(\n",
+    "        path_imgrec=os.path.join(data_dir, 'val.rec'), batch_size=batch_size,\n",
+    "        data_shape=(3, edge_size, edge_size), shuffle=False)\n",
+    "    return train_iter, val_iter"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "((3, 256, 256), (1, 5))"
+      ]
+     },
+     "execution_count": 4,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "batch_size, edge_size = 1, 256\n",
+    "train_iter, val_iter = load_data_pikachu(batch_size, edge_size)\n",
+    "batch = train_iter.next()\n",
+    "batch.data[0][0].shape, batch.label[0][0].shape"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## 3. 转换成PNG图片并保存"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def process(data_iter, save_dir):\n",
+    "    \"\"\"batch size == 1\"\"\"\n",
+    "    data_iter.reset() # 从头开始\n",
+    "    all_label = dict()\n",
+    "    id = 1\n",
+    "    os.makedirs(os.path.join(save_dir, 'images'), exist_ok=True)\n",
+    "    for sample in tqdm(data_iter):\n",
+    "        x = sample.data[0][0].asnumpy().transpose((1,2,0))\n",
+    "        plt.imsave(os.path.join(save_dir, 'images', str(id) + '.png'), x / 255.0)\n",
+    "\n",
+    "        y = sample.label[0][0][0].asnumpy()\n",
+    "\n",
+    "        label = {}\n",
+    "        label[\"class\"] = int(y[0])\n",
+    "        label[\"loc\"] = y[1:].tolist()\n",
+    "\n",
+    "        all_label[str(id) + '.png'] = label.copy()\n",
+    "\n",
+    "        id += 1\n",
+    "\n",
+    "    with open(os.path.join(save_dir, 'label.json'), 'w') as f:\n",
+    "        json.dump(all_label, f, indent=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "900it [00:40, 22.03it/s]\n"
+     ]
+    }
+   ],
+   "source": [
+    "process(data_iter = train_iter, save_dir = os.path.join(data_dir, \"train\"))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "100it [00:04, 22.86it/s]\n"
+     ]
+    }
+   ],
+   "source": [
+    "process(data_iter = val_iter, save_dir = os.path.join(data_dir, \"val\"))"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.6.2"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}