1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
|
from __future__ import division
import time
import numpy as np
import cudamat as cm
import util
# initialize CUDA
cm.cublas_init()
cm.CUDAMatrix.init_random(1)
# load data
util.load('mnist.dat', globals())
dev_dat = cm.CUDAMatrix(cm.reformat(dat/255.))
# training parameters
epsilon = 0.1
momentum = 0.9
num_epochs = 30
batch_size = 128
num_batches = dat.shape[1]//batch_size
# model parameters
num_vis = dat.shape[0]
num_hid = 4096
# initialize weights
w_vh = cm.CUDAMatrix(0.1 * np.random.randn(num_vis, num_hid))
w_v = cm.CUDAMatrix(np.zeros((num_vis, 1)))
w_h = cm.CUDAMatrix(-4.*np.ones((num_hid, 1)))
# initialize weight updates
wu_vh = cm.CUDAMatrix(np.zeros((num_vis, num_hid)))
wu_v = cm.CUDAMatrix(np.zeros((num_vis, 1)))
wu_h = cm.CUDAMatrix(np.zeros((num_hid, 1)))
# initialize temporary storage
v = cm.empty((num_vis, batch_size))
h = cm.empty((num_hid, batch_size))
r = cm.empty((num_hid, batch_size))
start_time = time.time()
for epoch in range(num_epochs):
print("Epoch %i" % (epoch + 1))
err = []
for batch in range(num_batches):
# get current minibatch
v_true = dev_dat.slice(batch*batch_size,(batch + 1)*batch_size)
v.assign(v_true)
# apply momentum
wu_vh.mult(momentum)
wu_v.mult(momentum)
wu_h.mult(momentum)
# positive phase
cm.dot(w_vh.T, v, target = h)
h.add_col_vec(w_h)
h.apply_sigmoid()
wu_vh.add_dot(v, h.T)
wu_v.add_sums(v, axis = 1)
wu_h.add_sums(h, axis = 1)
# sample hiddens
r.fill_with_rand()
r.less_than(h, target = h)
# negative phase
cm.dot(w_vh, h, target = v)
v.add_col_vec(w_v)
v.apply_sigmoid()
cm.dot(w_vh.T, v, target = h)
h.add_col_vec(w_h)
h.apply_sigmoid()
wu_vh.subtract_dot(v, h.T)
wu_v.add_sums(v, axis = 1, mult = -1.)
wu_h.add_sums(h, axis = 1, mult = -1.)
# update weights
w_vh.add_mult(wu_vh, epsilon/batch_size)
w_v.add_mult(wu_v, epsilon/batch_size)
w_h.add_mult(wu_h, epsilon/batch_size)
# calculate reconstruction error
v.subtract(v_true)
err.append(v.euclid_norm()**2/(num_vis*batch_size))
print("Mean squared error: %f" % np.mean(err))
print("Time: %f" % (time.time() - start_time))
w_vh.copy_to_host()
util.save('weights.dat', 'w_vh', {'w_vh': w_vh.numpy_array})
cm.cublas_shutdown()
|
