end0tknr's kipple - 新web写経開発

http://d.hatena.ne.jp/end0tknr/ から移転しました

2層ニューラルネットワークに対する確率的勾配降下法 - python

github.com

またも、 o'reilly「ゼロから作る Deep Learning」4章にある勾配降下法の写経.

今回は、2層ニューラルネットワークに対して実行します。

mnist, ミニバッチ, 数値微分, 勾配降下, 損失関数 等、 ディープラーニングに関する基本が揃っています。

まだまだ、全くゼロから、自分でsrc書けませんが

#!/usr/local/python3/bin/python3
# coding: utf-8

try:
    import urllib.request
except ImportError:
    raise ImportError('You should use Python 3.x')
import os.path
import gzip
import pickle
import sys, os
sys.path.append(os.pardir)  # 親dirのfileをimportする為
import numpy as np
import matplotlib.pyplot as plt
#from dataset.mnist import load_mnist
#from two_layer_net import TwoLayerNet

## https://github.com/oreilly-japan/deep-learning-from-scratch/tree/master/ch04

MNIST_DATASET_DIR = os.path.dirname(os.path.abspath(__file__))
MNIST_SAVE_FILE = MNIST_DATASET_DIR + "/mnist.pkl"
MNIST_URL_BASE = 'http://yann.lecun.com/exdb/mnist/'
MNIST_GZ_FILES = {
    'train_img':'train-images-idx3-ubyte.gz',
    'train_label':'train-labels-idx1-ubyte.gz',
    'test_img':'t10k-images-idx3-ubyte.gz',
    'test_label':'t10k-labels-idx1-ubyte.gz'
}
MNIST_IMG_SIZE = 784


def main():
    # mnist読込み. 初回は yann.lecun.com よりdownload
    # x_train:教師 data(画像), t_train:教師 data(label)
    # x_test :test data(画像), t_test :test data(label)
    (x_train, t_train), (x_test, t_test) = \
      load_mnist(normalize=True, one_hot_label=True)

    # 784=28*28 , 10=0~9
    network = TwoLayerNet(input_size=784, hidden_size=50, output_size=10)

    iters_num = 10000  # 繰返し回数
    train_size = x_train.shape[0]
    batch_size = 100
    learning_rate = 0.1

    train_loss_list = []
    train_acc_list = []
    test_acc_list = []

    iter_per_epoch = max(train_size / batch_size, 1)

    for i in range(iters_num):
        batch_mask = np.random.choice(train_size, batch_size)
        x_batch = x_train[batch_mask]
        t_batch = t_train[batch_mask]

        # 勾配の計算
        #grad = network.numerical_gradient(x_batch, t_batch)
        grad = network.gradient(x_batch, t_batch)

        # パラメータの更新
        for key in ('W1', 'b1', 'W2', 'b2'):
            network.params[key] -= learning_rate * grad[key]

        loss = network.loss(x_batch, t_batch)
        train_loss_list.append(loss)

        if i % iter_per_epoch == 0:
            train_acc = network.accuracy(x_train, t_train)
            test_acc = network.accuracy(x_test, t_test)
            train_acc_list.append(train_acc)
            test_acc_list.append(test_acc)
            print("train acc, test acc | " + str(train_acc) + ", " + str(test_acc))


    # グラフの描画
    markers = {'train': 'o', 'test': 's'}
    x = np.arange(len(train_acc_list))
    plt.plot(x, train_acc_list, label='train acc')
    plt.plot(x, test_acc_list, label='test acc', linestyle='--')
    plt.xlabel("epochs")
    plt.ylabel("accuracy")
    plt.ylim(0, 1.0)
    plt.legend(loc='lower right')
    plt.savefig( 'train_neuralnet.png' )



def load_mnist(normalize=True, flatten=True, one_hot_label=False):
    """MNISTの読み込み
    params
      normalize : 画像のピクセル値を0.0~1.0に正規化
      one_hot_label :
        one_hot_labelがTrueの場合、ラベルはone-hot配列として返す
      flatten : 画像を一次元配列に平にするかどうか
    
    returns
      (訓練画像, 訓練ラベル), (テスト画像, テストラベル)
    """
    if not os.path.exists(MNIST_SAVE_FILE):
        init_mnist()
        
    with open(MNIST_SAVE_FILE, 'rb') as f:
        dataset = pickle.load(f)
    
    if normalize:
        for key in ('train_img', 'test_img'):
            dataset[key] = dataset[key].astype(np.float32)
            dataset[key] /= 255.0
            
    if one_hot_label:
        dataset['train_label'] = _change_one_hot_label(dataset['train_label'])
        dataset['test_label'] =  _change_one_hot_label(dataset['test_label'])
    
    if not flatten:
         for key in ('train_img', 'test_img'):
            dataset[key] = dataset[key].reshape(-1, 1, 28, 28)

    return (dataset['train_img'], dataset['train_label']), \
           (dataset['test_img'], dataset['test_label'])


def init_mnist():
    download_mnist()

    dataset = {}
    dataset['train_img'] =   load_mnist_img(  MNIST_GZ_FILES['train_img'])
    dataset['test_img'] =    load_mnist_img(  MNIST_GZ_FILES['test_img'])
    dataset['train_label'] = load_mnist_label(MNIST_GZ_FILES['train_label'])
    dataset['test_label'] =  load_mnist_label(MNIST_GZ_FILES['test_label'])

    with open(MNIST_SAVE_FILE, 'wb') as f:
        pickle.dump(dataset, f, -1)


def download_mnist():
    
    for file_name in MNIST_GZ_FILES.values():
        file_path = MNIST_DATASET_DIR + "/" + file_name

        if os.path.exists(file_path):
            continue
        print("download",
              MNIST_URL_BASE + file_name,
              "to", MNIST_DATASET_DIR)
        
        urllib.request.urlretrieve(MNIST_URL_BASE + file_name, file_path)


def load_mnist_label(file_name):
    file_path = MNIST_DATASET_DIR + "/" + file_name

    # rb = バイナリの読込み
    with gzip.open(file_path, 'rb') as f:
        labels = np.frombuffer(f.read(), np.uint8, offset=8)
        # 上記の「offset」の必要性は理解していません
    return labels

def load_mnist_img(file_name):
    file_path = MNIST_DATASET_DIR + "/" + file_name
    
    # rb = バイナリの読込み
    with gzip.open(file_path, 'rb') as f:
        data = np.frombuffer(f.read(), np.uint8, offset=16)
        # 上記の「offset」の必要性は理解していません

    # numpy.reshape(-1, ...)で、一次元配列化
    data = data.reshape(-1, MNIST_IMG_SIZE)
    return data


def _change_one_hot_label(X):
    T = np.zeros((X.size, 10))
    for idx, row in enumerate(T):
        row[X[idx]] = 1
        
    return T


class TwoLayerNet:

    def __init__(self, input_size, hidden_size, output_size, weight_init_std=0.01):
        # 重みの初期化
        self.params = {}
        self.params['W1'] = weight_init_std * np.random.randn(input_size, hidden_size)
        self.params['b1'] = np.zeros(hidden_size)
        self.params['W2'] = weight_init_std * np.random.randn(hidden_size, output_size)
        self.params['b2'] = np.zeros(output_size)

    def predict(self, x):
        W1, W2 = self.params['W1'], self.params['W2']
        b1, b2 = self.params['b1'], self.params['b2']
    
        a1 = np.dot(x, W1) + b1
        z1 = sigmoid(a1)
        a2 = np.dot(z1, W2) + b2
        y = softmax(a2)
        
        return y
        
    # x:入力データ, t:教師データ
    def loss(self, x, t):
        y = self.predict(x)
        
        return cross_entropy_error(y, t)
    
    def accuracy(self, x, t):
        y = self.predict(x)
        y = np.argmax(y, axis=1)
        t = np.argmax(t, axis=1)
        
        accuracy = np.sum(y == t) / float(x.shape[0])
        return accuracy
        
    # x:入力データ, t:教師データ
    def numerical_gradient(self, x, t):
        loss_W = lambda W: self.loss(x, t)
        
        grads = {}
        grads['W1'] = numerical_gradient(loss_W, self.params['W1'])
        grads['b1'] = numerical_gradient(loss_W, self.params['b1'])
        grads['W2'] = numerical_gradient(loss_W, self.params['W2'])
        grads['b2'] = numerical_gradient(loss_W, self.params['b2'])
        
        return grads
        
    def gradient(self, x, t):
        W1, W2 = self.params['W1'], self.params['W2']
        b1, b2 = self.params['b1'], self.params['b2']
        grads = {}
        
        batch_num = x.shape[0]
        
        # forward
        a1 = np.dot(x, W1) + b1
        z1 = sigmoid(a1)
        a2 = np.dot(z1, W2) + b2
        y = softmax(a2)
        
        # backward
        dy = (y - t) / batch_num
        grads['W2'] = np.dot(z1.T, dy)
        grads['b2'] = np.sum(dy, axis=0)
        
        da1 = np.dot(dy, W2.T)
        dz1 = sigmoid_grad(a1) * da1
        grads['W1'] = np.dot(x.T, dz1)
        grads['b1'] = np.sum(dz1, axis=0)

        return grads


def numerical_gradient(f, x):
    h = 1e-4 # 0.0001
    grad = np.zeros_like(x)
    
    it = np.nditer(x, flags=['multi_index'], op_flags=['readwrite'])
    while not it.finished:
        idx = it.multi_index
        tmp_val = x[idx]
        x[idx] = float(tmp_val) + h
        fxh1 = f(x) # f(x+h)
        
        x[idx] = tmp_val - h 
        fxh2 = f(x) # f(x-h)
        grad[idx] = (fxh1 - fxh2) / (2*h)
        
        x[idx] = tmp_val # 値を元に戻す
        it.iternext()   
        
    return grad


def sigmoid(x):
    return 1 / (1 + np.exp(-x))    


def sigmoid_grad(x):
    return (1.0 - sigmoid(x)) * sigmoid(x)
    

def relu(x):
    return np.maximum(0, x)


def relu_grad(x):
    grad = np.zeros(x)
    grad[x>=0] = 1
    return grad
    

def softmax(x):
    if x.ndim == 2:
        x = x.T
        x = x - np.max(x, axis=0)
        y = np.exp(x) / np.sum(np.exp(x), axis=0)
        return y.T 

    x = x - np.max(x) # オーバーフロー対策
    return np.exp(x) / np.sum(np.exp(x))


def cross_entropy_error(y, t):
    if y.ndim == 1:
        t = t.reshape(1, t.size)
        y = y.reshape(1, y.size)
        
    # 教師データがone-hot-vectorの場合、正解ラベルのインデックスに変換
    if t.size == y.size:
        t = t.argmax(axis=1)
             
    batch_size = y.shape[0]
    return -np.sum(np.log(y[np.arange(batch_size), t])) / batch_size


def softmax_loss(X, t):
    y = softmax(X)
    return cross_entropy_error(y, t)



if __name__ == '__main__':
    main()

↑こう書くと↓こう表示されます

f:id:end0tknr:20171120221330p:plain