depth first search

#!/usr/bin/python
"""
Author: Jeremy M. Stober
Program: MULTILAYER.PY
Description: A simple implementation of a feed-forward multilayer network with back-propagation.
"""
 
import os, sys, getopt, pdb
from numpy import *
from numpy.random import *
import random as prandom
 
def logistic(vector):
    return 1 / (1 + exp(-vector))
 
def dlogistic(vector):
    return vector * (1 - vector)
 
def sigmoid(vector):
    return tanh(vector)
 
def dsigmoid(vector):
    return 1 - vector**2
 
class Multilayer(object):
 
    def __init__(self, alpha, eta, ninput, nhidden, noutput):
 
        # Create the weight sets. 
        self.input = standard_normal((ninput, nhidden))
        self.output = standard_normal((nhidden, noutput))
        self.storage = {} # Stored activations.
        self.alpha = alpha # Momentum!
        self.eta = eta #Learning rate!
        self.momentum = {'input' : zeros((ninput, nhidden)),
                         'output' : zeros((nhidden, noutput)) }
 
    def eval(self, input):
        # Use array masks if network is not fully connected.
        # Sum and squash the weights of the input activations.
        hidden = sigmoid(dot(input, self.input))
 
        # Sum and squash the weights of the hidden activations.
        output = sigmoid(dot(hidden, self.output))
 
        self.storage = {'input' : input, 'hidden' : hidden, 'output' : output}
 
        # Output elements are in the range (0,1).
        return output
 
    def supervise(self, input, response):
 
        # Will store the activations in self.storage.
        error = response - self.eval(input)
 
        # Backpropagate the error.        
        # Note: * is elementwise multiplication.
        doutput = error * dsigmoid(self.storage['output'])
        dhidden = dot(self.output,doutput) * dsigmoid(self.storage['hidden'])
 
        # We need to store the last change for momentum.
        deltas = { 'output' : self.eta * outer(self.storage['hidden'],doutput) + self.alpha * self.momentum['output'],
                   'input' :  self.eta * outer(self.storage['input'], dhidden) + self.alpha * self.momentum['input'] }
 
        self.output = self.output + deltas['output']
        self.input = self.input + deltas['input']
        self.momentum = deltas
 
        return sum(error ** 2) # Return sum squared error.
 
def test():
 
    patterns = [(array([0,1]), array([1])),
                (array([1,0]), array([1])),
                (array([1,1]), array([0])),
                (array([0,0]), array([0]))]
 
    multi = Multilayer(0.1, 0.5, 2, 2, 1)
 
    for i in range(10000):
        (input,response) = prandom.choice(patterns)
        multi.supervise(input, response)
 
    for i in range(10):
        (input,response) = prandom.choice(patterns)
        print input, multi.eval(input), response
 
 
def main():
 
    def usage():
	print sys.argv[0] + "[-h] [-d]"
 
    try:
        (options, args) = getopt.getopt(sys.argv[1:], 'dh', ['help','debug'])
    except getopt.GetoptError:
        # print help information and exit:
        usage()
        sys.exit(2)
 
    for o, a in options:
        if o in ('-h', '--help'):
            usage()
            sys.exit()
	elif o in ('-d', '--debug'):
	    pdb.set_trace()
 
    test()
 
 
if __name__ == "__main__":
    main()