end0tknr's kipple - 新web写経開発

http://d.hatena.ne.jp/end0tknr/ から移転しました

2層ニューラルネットワークに対する誤差 逆伝播法 - python

先日のエントリの続きであり、o'reilly「ゼロから作る Deep Learning」5章の写経.

end0tknr.hateblo.jp

github.com

誤差 逆伝播法 による 2層ニューラルネットワーク モデル

pythonコードだけでは分かりづらいので、図示してみました

f:id:end0tknr:20171123173523p:plain

ニューラルネットワークに対する誤差 逆伝播法 - python

#!/usr/local/python3/bin/python3
# coding: utf-8

try:
    import urllib.request
except ImportError:
    raise ImportError('You should use Python 3.x')
import os.path
import gzip
import pickle
import sys, os
sys.path.append(os.pardir)  # 親dirのfileをimportする為
import numpy as np
from collections import OrderedDict
import matplotlib.pyplot as plt


## https://github.com/oreilly-japan/deep-learning-from-scratch/tree/master/ch05

MNIST_DATASET_DIR = os.path.dirname(os.path.abspath(__file__))
MNIST_SAVE_FILE = MNIST_DATASET_DIR + "/mnist.pkl"
MNIST_URL_BASE = 'http://yann.lecun.com/exdb/mnist/'
MNIST_GZ_FILES = {
    'train_img':'train-images-idx3-ubyte.gz',
    'train_label':'train-labels-idx1-ubyte.gz',
    'test_img':'t10k-images-idx3-ubyte.gz',
    'test_label':'t10k-labels-idx1-ubyte.gz'
}
MNIST_IMG_SIZE = 784

######################################################

def main():
    # mnist読込み. 初回は yann.lecun.com よりdownload
    # x_train:教師 data(画像), t_train:教師 data(label)
    # x_test :test data(画像), t_test :test data(label)
    (x_train, t_train), (x_test, t_test) = \
      load_mnist(normalize=True, one_hot_label=True)


    # 784=28*28 , 10=0~9
    network = TwoLayerNet(input_size=784, hidden_size=50, output_size=10)

    iters_num = 10000  # 繰返し回数
    train_size = x_train.shape[0]
    batch_size = 100
    learning_rate = 0.1

    train_loss_list = []
    train_acc_list = []
    test_acc_list = []


    iter_per_epoch = max(train_size / batch_size, 1)


    for i in range(iters_num):
        batch_mask = np.random.choice(train_size, batch_size)
        x_batch = x_train[batch_mask]
        t_batch = t_train[batch_mask]

        # 勾配算出
        grad = network.gradient(x_batch, t_batch)

        # 更新
        for key in ('W1', 'b1', 'W2', 'b2'):
            network.params[key] -= learning_rate * grad[key]

        loss = network.loss(x_batch, t_batch)
        train_loss_list.append(loss)

        if i % iter_per_epoch == 0:
            train_acc = network.accuracy(x_train, t_train)
            test_acc = network.accuracy(x_test, t_test)
            train_acc_list.append(train_acc)
            test_acc_list.append(test_acc)

            print("W1:", network.params['W1'])
            print("b1:", network.params['b1'])
            print("W2:", network.params['W2'])
            print("b2:", network.params['b2'])

            print("ACCURACY OF TRAIN/TEST:", train_acc, "/", test_acc)


    # グラフの描画
    markers = {'train': 'o', 'test': 's'}
    x = np.arange(len(train_acc_list))
    plt.plot(x, train_acc_list, label='train acc')
    plt.plot(x, test_acc_list, label='test acc', linestyle='--')
    plt.xlabel("epochs")
    plt.ylabel("accuracy")
    plt.ylim(0, 1.0)
    plt.legend(loc='lower right')
    plt.savefig( 'train_neuralnet.png' )


######################################################

class TwoLayerNet:

    def __init__(self, input_size, hidden_size, output_size, weight_init_std = 0.01):
        # 重みの初期化
        self.params = {}
        self.params['W1'] = weight_init_std * np.random.randn(input_size, hidden_size)
        self.params['b1'] = np.zeros(hidden_size)
        self.params['W2'] = weight_init_std * np.random.randn(hidden_size, output_size) 
        self.params['b2'] = np.zeros(output_size)

        # レイヤの生成
        self.layers = OrderedDict()
        self.layers['Affine1'] = Affine(self.params['W1'], self.params['b1'])
        self.layers['Relu1'] = Relu()
        self.layers['Affine2'] = Affine(self.params['W2'], self.params['b2'])

        self.lastLayer = SoftmaxWithLoss()
        
    def predict(self, x):
        for layer in self.layers.values():
            x = layer.forward(x)
        
        return x
        
    # x:入力データ, t:教師データ
    def loss(self, x, t):
        y = self.predict(x)
        return self.lastLayer.forward(y, t)
    
    def accuracy(self, x, t):
        y = self.predict(x)
        y = np.argmax(y, axis=1)
        if t.ndim != 1 : t = np.argmax(t, axis=1)
        
        accuracy = np.sum(y == t) / float(x.shape[0])
        return accuracy
        
    def gradient(self, x, t):
        # forward
        self.loss(x, t)

        # backward
        dout = 1
        dout = self.lastLayer.backward(dout)
        
        layers = list(self.layers.values())
        layers.reverse()
        for layer in layers:
            dout = layer.backward(dout)

        # 設定
        grads = {}
        grads['W1'], grads['b1'] = self.layers['Affine1'].dW, self.layers['Affine1'].db
        grads['W2'], grads['b2'] = self.layers['Affine2'].dW, self.layers['Affine2'].db

        return grads

######################################################

class SoftmaxWithLoss:
    def __init__(self):
        self.loss = None
        self.y = None # softmaxの出力
        self.t = None # 教師データ

    def forward(self, x, t):
        self.t = t
        self.y = softmax(x)
        self.loss = cross_entropy_error(self.y, self.t)
        
        return self.loss

    def backward(self, dout=1):
        batch_size = self.t.shape[0]
        if self.t.size == self.y.size: # 教師データがone-hot-vectorの場合
            dx = (self.y - self.t) / batch_size
        else:
            dx = self.y.copy()
            dx[np.arange(batch_size), self.t] -= 1
            dx = dx / batch_size
        
        return dx

######################################################

class Affine:
    def __init__(self, W, b):
        self.W =W
        self.b = b
        
        self.x = None
        self.original_x_shape = None
        # 重み・バイアスパラメータの微分
        self.dW = None
        self.db = None

    def forward(self, x):
        # テンソル対応
        self.original_x_shape = x.shape
        x = x.reshape(x.shape[0], -1)
        self.x = x

        out = np.dot(self.x, self.W) + self.b

        return out

    def backward(self, dout):
        dx = np.dot(dout, self.W.T)
        self.dW = np.dot(self.x.T, dout)
        self.db = np.sum(dout, axis=0)
        
        dx = dx.reshape(*self.original_x_shape)  # 入力データの形状に戻す(テンソル対応)
        return dx


######################################################

class Relu:
    def __init__(self):
        self.mask = None

    def forward(self, x):
        self.mask = (x <= 0)
        out = x.copy()
        out[self.mask] = 0

        return out

    def backward(self, dout):
        dout[self.mask] = 0
        dx = dout

        return dx


######################################################

def softmax(x):
    if x.ndim == 2:
        x = x.T
        x = x - np.max(x, axis=0)
        y = np.exp(x) / np.sum(np.exp(x), axis=0)
        return y.T 

    x = x - np.max(x) # オーバーフロー対策
    return np.exp(x) / np.sum(np.exp(x))


def cross_entropy_error(y, t):
    if y.ndim == 1:
        t = t.reshape(1, t.size)
        y = y.reshape(1, y.size)
        
    # 教師データがone-hot-vectorの場合、正解ラベルのインデックスに変換
    if t.size == y.size:
        t = t.argmax(axis=1)
             
    batch_size = y.shape[0]
    return -np.sum(np.log(y[np.arange(batch_size), t])) / batch_size


######################################################

def load_mnist(normalize=True, flatten=True, one_hot_label=False):
    """MNISTの読み込み
    params
      normalize : 画像のピクセル値を0.0~1.0に正規化
      one_hot_label :
        one_hot_labelがTrueの場合、ラベルはone-hot配列として返す
      flatten : 画像を一次元配列に平にするかどうか
    
    returns
      (訓練画像, 訓練ラベル), (テスト画像, テストラベル)
    """
    if not os.path.exists(MNIST_SAVE_FILE):
        init_mnist()
        
    with open(MNIST_SAVE_FILE, 'rb') as f:
        dataset = pickle.load(f)
    
    if normalize:
        for key in ('train_img', 'test_img'):
            dataset[key] = dataset[key].astype(np.float32)
            dataset[key] /= 255.0
            
    if one_hot_label:
        dataset['train_label'] = _change_one_hot_label(dataset['train_label'])
        dataset['test_label'] =  _change_one_hot_label(dataset['test_label'])
    
    if not flatten:
         for key in ('train_img', 'test_img'):
            dataset[key] = dataset[key].reshape(-1, 1, 28, 28)

    return (dataset['train_img'], dataset['train_label']), \
           (dataset['test_img'], dataset['test_label'])


def init_mnist():
    download_mnist()

    dataset = {}
    dataset['train_img'] =   load_mnist_img(  MNIST_GZ_FILES['train_img'])
    dataset['test_img'] =    load_mnist_img(  MNIST_GZ_FILES['test_img'])
    dataset['train_label'] = load_mnist_label(MNIST_GZ_FILES['train_label'])
    dataset['test_label'] =  load_mnist_label(MNIST_GZ_FILES['test_label'])

    with open(MNIST_SAVE_FILE, 'wb') as f:
        pickle.dump(dataset, f, -1)


def download_mnist():
    
    for file_name in MNIST_GZ_FILES.values():
        file_path = MNIST_DATASET_DIR + "/" + file_name

        if os.path.exists(file_path):
            continue
        print("download",
              MNIST_URL_BASE + file_name,
              "to", MNIST_DATASET_DIR)
        
        urllib.request.urlretrieve(MNIST_URL_BASE + file_name, file_path)


def load_mnist_label(file_name):
    file_path = MNIST_DATASET_DIR + "/" + file_name

    # rb = バイナリの読込み
    with gzip.open(file_path, 'rb') as f:
        labels = np.frombuffer(f.read(), np.uint8, offset=8)
        # 上記の「offset」の必要性は理解していません
    return labels

def load_mnist_img(file_name):
    file_path = MNIST_DATASET_DIR + "/" + file_name
    
    # rb = バイナリの読込み
    with gzip.open(file_path, 'rb') as f:
        data = np.frombuffer(f.read(), np.uint8, offset=16)
        # 上記の「offset」の必要性は理解していません

    # numpy.reshape(-1, ...)で、一次元配列化
    data = data.reshape(-1, MNIST_IMG_SIZE)
    return data


def _change_one_hot_label(X):
    T = np.zeros((X.size, 10))
    for idx, row in enumerate(T):
        row[X[idx]] = 1
        
    return T


if __name__ == '__main__':
    main()

↑こう書くと↓こう表示されます

f:id:end0tknr:20171123152329p:plain