Artificial Neural Networks

Introduction to artificial neural networks

: 인공지능의 기본 전제 중 하나는 사람의 지능이 필요한 작업을 기계가 처리하게 만드는 것이다. 그렇다면 사람의 두뇌를 모델 삼아 기계로 만들면 된다고 생각할 수 있는데 이러한 발상의 결과물이 바로 인공 신경망이다.
인공 신경망은 데이터에 존재한느 패턴을 찾아서 의미 있는 정보를 학습하도록 설계한 것이다. 이는 분류, 회귀, 분석, 분할을 비롯한 다양한 작업에 활용할 수 있다.

Building a Perceptron based classifier

: 퍼셉트론은 인공 신경망의 기본 구성 요소로 입력을 받아서 연산을 수행한 후 결과를 출력하는 하나의 뉴런으로 구성된다. 의사 결정은 간단한 선형 함수로 내린다. N차원의 입력 데이터 포인트를 처리하는 경우, 하나의 퍼셉트론은 N개의 숫자에 대한 가중치 합을 구한 후 여기에 상수, 바이어스 bias를 더해 결과를 출력한다.

import numpy as np 
import matplotlib.pyplot as plt 
import neurolab as nl

# Load input data  
text = np.loadtxt('data_perceptron.txt') 

# Separate datapoints and labels 
data = text[:, :2] 
labels = text[:, 2].reshape((text.shape[0], 1)) 

# Plot input data 
plt.figure() 
plt.scatter(data[:,0], data[:,1]) 
plt.xlabel('Dimension 1') 
plt.ylabel('Dimension 2') 
plt.title('Input data') 

# Define minimum and maximum values for each dimension 
dim1_min, dim1_max, dim2_min, dim2_max = 0, 1, 0, 1 

# Number of neurons in the output layer 
num_output = labels.shape[1] 

# Define a perceptron with 2 input neurons (because we  
# have 2 dimensions in the input data) 
dim1 = [dim1_min, dim1_max] 
dim2 = [dim2_min, dim2_max] 
perceptron = nl.net.newp([dim1, dim2], num_output) 

# Train the perceptron using the data 
error_progress = perceptron.train(data, labels, epochs=100, show=20, lr=0.03) 

# Plot the training progress 
plt.figure() 
plt.plot(error_progress) 
plt.xlabel('Number of epochs') 
plt.ylabel('Training error') 
plt.title('Training error progress') 
plt.grid() 
 
plt.show() 

Constructing a single layer neural network

: 퍼셉트론은 신경망을 처음 배우기에는 좋지만 활용 범위가 넓지 않다. 입력 데이터를 서로 독립적인 여러 개의 뉴런으로 처리해서 결과를 출력하는 단층 신경망에 대해 살펴보자.

import numpy as np 
import matplotlib.pyplot as plt 
import neurolab as nl

# Load input data 
text = np.loadtxt('data_simple_nn.txt') 

# Separate it into datapoints and labels 
data = text[:, 0:2] 
labels = text[:, 2:] 

# Plot input data 
plt.figure() 
plt.scatter(data[:,0], data[:,1]) 
plt.xlabel('Dimension 1') 
plt.ylabel('Dimension 2') 
plt.title('Input data') 

# Minimum and maximum values for each dimension 
dim1_min, dim1_max = data[:,0].min(), data[:,0].max() 
dim2_min, dim2_max = data[:,1].min(), data[:,1].max() 

# Define the number of neurons in the output layer 
num_output = labels.shape[1] 

# Define a single-layer neural network  
dim1 = [dim1_min, dim1_max] 
dim2 = [dim2_min, dim2_max] 
nn = nl.net.newp([dim1, dim2], num_output) 

# Train the neural network 
error_progress = nn.train(data, labels, epochs=100, show=20, lr=0.03) 

# Plot the training progress 
plt.figure() 
plt.plot(error_progress) 
plt.figure() 
plt.plot(error_progress) 
plt.xlabel('Number of epochs') 
plt.ylabel('Training error') 
plt.title('Training error progress') 
plt.grid() 
 
plt.show() 

# Run the classifier on test datapoints 
print('\nTest results:') 
data_test = [[0.4, 4.3], [4.4, 0.6], [4.7, 8.1]] 
for item in data_test: 
    print(item, '-->', nn.sim([item])[0]) 

Constructing a multilayer neural network

:신경망의 자유도를 높이면 정확도를 좀 더 높일 수 있다. 다시 말해 은닉 계층이 여러개인 다층 신경망을 활용한다는 것이다.

import numpy as np 
import matplotlib.pyplot as plt 
import neurolab as nl 

# Generate some training data 
min_val = -15 
max_val = 15 
num_points = 130 
x = np.linspace(min_val, max_val, num_points) 
y = 3 * np.square(x) + 5 
y /= np.linalg.norm(y) 

# Create data and labels 
data = x.reshape(num_points, 1) 
labels = y.reshape(num_points, 1) 

# Plot input data 
plt.figure() 
plt.scatter(data, labels) 
plt.xlabel('Dimension 1') 
plt.ylabel('Dimension 2') 
plt.title('Input data') 


# Define a multilayer neural network with 2 hidden layers; 
# First hidden layer consists of 10 neurons 
# Second hidden layer consists of 6 neurons 
# Output layer consists of 1 neuron 
nn = nl.net.newff([[min_val, max_val]], [10, 6, 1]) 

# Set the training algorithm to gradient descent 
nn.trainf = nl.train.train_gd 

# Train the neural network 
error_progress = nn.train(data, labels, epochs=2000, show=100, goal=0.01) 

# Run the neural network on training datapoints 
output = nn.sim(data) 
y_pred = output.reshape(num_points) 

# Plot training error 
plt.figure() 
plt.plot(error_progress) 
plt.xlabel('Number of epochs') 
plt.ylabel('Error') 
plt.title('Training error progress') 

# Plot the output  
x_dense = np.linspace(min_val, max_val, num_points * 2) 
y_dense_pred = nn.sim(x_dense.reshape(x_dense.size,1)).reshape(x_dense.size) 
 
plt.figure() 
plt.plot(x_dense, y_dense_pred, '-', x, y, '.', x, y_pred, 'p') 
plt.title('Actual vs predicted') 
 
plt.show()

Building a vector quantizer

: 벡터 양자화란 입력 데이터를 고정된 수의 대표점에 매핑하는 것을 말한다.

import numpy as np 
import matplotlib.pyplot as plt 
import neurolab as nl 

# Load input data 
text = np.loadtxt('data_vector_quantization.txt') 

# Separate it into data and labels 
data = text[:, 0:2] 
labels = text[:, 2:] 

# Define a neural network with 2 layers: 
# 10 neurons in input layer and 4 neurons in output layer 
num_input_neurons = 10 
num_output_neurons = 4 
weights = [1/num_output_neurons] * num_output_neurons 
nn = nl.net.newlvq(nl.tool.minmax(data), num_input_neurons, weights) 

# Train the neural network 
_ = nn.train(data, labels, epochs=500, goal=-1) 

# Create the input grid 
xx, yy = np.meshgrid(np.arange(0, 10, 0.2), np.arange(0, 10, 0.2)) 
xx.shape = xx.size, 1 
yy.shape = yy.size, 1 
grid_xy = np.concatenate((xx, yy), axis=1) 

# Evaluate the input grid of points 
grid_eval = nn.sim(grid_xy) 
Extract the four classes:# Define the 4 classes 
class_1 = data[labels[:,0] == 1] 
class_2 = data[labels[:,1] == 1] 
class_3 = data[labels[:,2] == 1] 
class_4 = data[labels[:,3] == 1] 

# Plot the outputs  
plt.plot(class_1[:,0], class_1[:,1], 'ko',  
        class_2[:,0], class_2[:,1], 'ko',  
        class_3[:,0], class_3[:,1], 'ko',  
        class_4[:,0], class_4[:,1], 'ko') 
plt.plot(grid_1[:,0], grid_1[:,1], 'm.', 
        grid_2[:,0], grid_2[:,1], 'bx', 
        grid_3[:,0], grid_3[:,1], 'c^',  
        grid_4[:,0], grid_4[:,1], 'y+') 
plt.axis([0, 10, 0, 10]) 
plt.xlabel('Dimension 1') 
plt.ylabel('Dimension 2') 
plt.title('Vector quantization') 
 
plt.show() 

Analyzing sequential data using recurrent neural networks

: 인공지능은 순차적인 데이터에 대한 모델을 만드는데도 뛰어난 성능을 발위하는데 그중에서도 재귀 신경망을 모델링하는데 특히 뛰어나다. 이는 시간적인 종속성/ 의존성이 특징인 시계열 데이터를 주로 다루는 모델이다.

import numpy as np 
import matplotlib.pyplot as plt 
import neurolab as nl

def get_data(num_points): 
    # Create sine waveforms 
    wave_1 = 0.5 * np.sin(np.arange(0, num_points)) 
    wave_2 = 3.6 * np.sin(np.arange(0, num_points)) 
    wave_3 = 1.1 * np.sin(np.arange(0, num_points)) 
    wave_4 = 4.7 * np.sin(np.arange(0, num_points)) 

    # Create varying amplitudes 
    amp_1 = np.ones(num_points) 
    amp_2 = 2.1 + np.zeros(num_points)  
    amp_3 = 3.2 * np.ones(num_points)  
    amp_4 = 0.8 + np.zeros(num_points)  
    
    wave = np.array([wave_1, wave_2, wave_3, wave_4]).reshape(num_points * 4, 1) 
    amp = np.array([[amp_1, amp_2, amp_3, amp_4]]).reshape(num_points * 4, 1) 
 
    return wave, amp  

# Visualize the output  
def visualize_output(nn, num_points_test): 
    wave, amp = get_data(num_points_test) 
    output = nn.sim(wave) 
    plt.plot(amp.reshape(num_points_test * 4)) 
    plt.plot(output.reshape(num_points_test * 4))

if __name__=='__main__': 
    # Create some sample data 
    num_points = 40 
    wave, amp = get_data(num_points) 
    
    # Create a recurrent neural network with 2 layers 
    nn = nl.net.newelm([[-2, 2]], [10, 1], [nl.trans.TanSig(), nl.trans.PureLin()]) 
    
    # Set the init functions for each layer  
    nn.layers[0].initf = nl.init.InitRand([-0.1, 0.1], 'wb') 
    nn.layers[1].initf = nl.init.InitRand([-0.1, 0.1], 'wb') 
    nn.init() 
    
    # Train the recurrent neural network 
    error_progress = nn.train(wave, amp, epochs=1200, show=100, goal=0.01) 
    
    # Run the training data through the network 
    output = nn.sim(wave) 
    
    # Plot the results 
    plt.subplot(211) 
    plt.plot(error_progress) 
    plt.xlabel('Number of epochs') 
    plt.ylabel('Error (MSE)') 
 
    plt.subplot(212) 
    plt.plot(amp.reshape(num_points * 4)) 
    plt.plot(output.reshape(num_points * 4)) 
    plt.legend(['Original', 'Predicted']) 
    
    # Testing the network performance on unknown data  
    plt.figure() 
 
    plt.subplot(211) 
    visualize_output(nn, 82) 
    plt.xlim([0, 300]) 
 
    plt.subplot(212) 
    visualize_output(nn, 49) 
    plt.xlim([0, 300]) 
 
    plt.show() 

Visualizing characters in an Optical Character Recognition (OCR) database

: 광학문자인식 역시 인공 신경망의 활용사례 중 하나이다.

import os 
import sys 
 
import cv2 
import numpy as np 

# Define the input file  
input_file = 'letter.data'  

# Define the visualization parameters  
img_resize_factor = 12 
start = 6
end = -1 
height, width = 16, 8 

# Iterate until the user presses the Esc key 
with open(input_file, 'r') as f: 
    for line in f.readlines(): 
        # Read the data 
        data = np.array([255 * float(x) for x in line.split('\t')[start:end]]) 
        # Reshape the data into a 2D image 
        img = np.reshape(data, (height, width)) 
        # Scale the image 
        img_scaled = cv2.resize(img, None, fx=img_resize_factor, fy=img_resize_factor) 
        # Display the image 
        cv2.imshow('Image', img_scaled) 
        
        # Check if the user pressed the Esc key 
        c = cv2.waitKey() 
        if c == 27: 
            break 

Building an Optical Character Recognition (OCR) engine

import numpy as np 
import neurolab as nl 

# Define the input file 
input_file = 'letter.data' 

# Define the number of datapoints to  
# be loaded from the input file 
num_datapoints = 50 

# String containing all the distinct characters 
orig_labels = 'omandig' 

# Compute the number of distinct characters 
num_orig_labels = len(orig_labels) 

# Define the training and testing parameters 
num_train = int(0.9 * num_datapoints) 
num_test = num_datapoints - num_train 

# Define the dataset extraction parameters  
start = 6 
end = -1 

# Creating the dataset 
data = [] 
labels = [] 
with open(input_file, 'r') as f: 
    for line in f.readlines(): 
        # Split the current line tabwise 
        list_vals = line.split('\t') 
        # Check if the label is in our ground truth  
        # labels. If not, we should skip it. 
        if list_vals[1] not in orig_labels: 
            continue 
        # Extract the current label and append it  
        # to the main list 
        label = np.zeros((num_orig_labels, 1)) 
        label[orig_labels.index(list_vals[1])] = 1 
        labels.append(label)         
        # Extract the character vector and append it to the main list 
        cur_char = np.array([float(x) for x in list_vals[start:end]]) 
        data.append(cur_char) 
        # Exit the loop once the required dataset has been created  
        if len(data) >= num_datapoints: 
            break 

# Convert the data and labels to numpy arrays 
data = np.asfarray(data) 
labels = np.array(labels).reshape(num_datapoints, num_orig_labels) 

# Extract the number of dimensions 
num_dims = len(data[0]) 

# Create a feedforward neural network 
nn = nl.net.newff([[0, 1] for _ in range(len(data[0]))],  
        [128, 16, num_orig_labels]) 
 
# Set the training algorithm to gradient descent 
nn.trainf = nl.train.train_gd

# Train the network 
error_progress = nn.train(data[:num_train,:], labels[:num_train,:],  
        epochs=10000, show=100, goal=0.01) 

# Predict the output for test inputs  
print('\nTesting on unknown data:') 
predicted_test = nn.sim(data[num_train:, :]) 
for i in range(num_test): 
    print('\nOriginal:', orig_labels[np.argmax(labels[i])]) 
    print('Predicted:', orig_labels[np.argmax(predicted_test[i])])