doc update
[jacinto-ai/caffe-jacinto.git] / common_plot.py
1 import re
2 import os
3 import matplotlib.pyplot as plt
4 import matplotlib.cm as cm
5 import numpy as np
6 import scipy.spatial as spatial
9 def get_test_accuracy(log, top_k):
10     iteration = re.findall(r'Iteration (\d*), Testing net \(#0\)', log)
11     accuracy = re.findall(r'Test net output #\d: accuracy/top-{top_k} = (\d*.\d*)'.format(top_k=top_k), log)
12     if len(accuracy)==0:
13         accuracy = re.findall(r'Test net output #\d: top-{top_k} = (\d*.\d*)'.format(top_k=top_k), log)
14     if len(accuracy)==0:
15         accuracy = re.findall(r'Test net output #\d: loss/top-{top_k} = (\d*.\d*)'.format(top_k=top_k), log)
16     if len(accuracy)==0:
17         accuracy = re.findall(r'Test net output #\d: accuracy/top{top_k} = (\d*.\d*)'.format(top_k=top_k), log)
18     if len(accuracy)==0:
19         accuracy = re.findall(r'Test net output #\d: accuracy = (\d*.\d*)', log)
20     iteration = [int(i) for i in iteration]
21     accuracy = [float(i) for i in accuracy]
22     return iteration, accuracy
25 def get_test_loss(log):
26     iteration = re.findall(r'Iteration (\d*), Testing net ', log)
27     loss = re.findall(r'Test net output #\d: loss = (\d*.\d*)', log)
28     if len(loss)==0:
29         loss = re.findall(r'Test net output #\d: loss/loss = (\d*.\d*)', log)
30     iteration = [int(i) for i in iteration]
31     loss = [float(i) for i in loss]
32     return iteration, loss
34 def get_train_loss(log):
35     iteration = re.findall(r'Iteration (\d*), lr = ', log)
36     loss = re.findall(r'Train net output #\d: loss = (\d*.\d*)', log)
37     iteration = [int(i) for i in iteration]
38     loss = [float(i) for i in loss]
39     return iteration, loss
41 def get_epochs(log):
42     gpus = re.findall(r' GPU (\d*):', log)
43     num_gpus = len(gpus)
44     #print num_gpus
45     max_iter = re.findall(r'max_iter: (\d*)', log)
46     iter_size = re.findall(r'iter_size: (\d*)', log)
47     batch_size = re.findall(r'batch_size: (\d*)',log)
48     max_iter = int(max_iter[0])
49     if len(iter_size) >0:
50         iter_size=int(iter_size[0])
51     else:
52         iter_size=1
54     batch_size = int(batch_size[0])
55    # print max_iter, iter_size, batch_size
56     num_epochs = int(round( (max_iter * iter_size * batch_size*num_gpus) /  1281167. +0.5))
57     return max_iter, num_epochs
59 def get_net_name(log):
60     return re.findall(r"Solving (.*)\n", log)[0]
63 def parse_files(files, top_k=1, separate=False):
64     data = {}
65     for file in files:
66         with open(file, 'r') as fp:
67             log = fp.read()
68             net_name = os.path.basename(file) if separate else get_net_name(log)
69             if net_name not in data.keys():
70                 data[net_name] = {}
71                 data[net_name]["accuracy"] = {}
72                 data[net_name]["accuracy"]["accuracy"] = []
73                 data[net_name]["accuracy"]["iteration"] = []
74                 data[net_name]["loss"] = {}
75                 data[net_name]["loss"]["loss"] = []
76                 data[net_name]["loss"]["iteration"] = []
77                 data[net_name]["train_loss"] = {}
78                 data[net_name]["train_loss"]["loss"] = []
79                 data[net_name]["train_loss"]["iteration"] = []
81             max_iter, epochs = get_epochs(log)
82             #print epochs
83             scale = float(epochs) / max_iter
85             iteration, accuracy = get_test_accuracy(log, top_k)
86             iteration = [k*scale for k in iteration]
87             data[net_name]["accuracy"]["iteration"].extend(iteration)
88             data[net_name]["accuracy"]["accuracy"].extend(accuracy)
90             iteration, loss = get_test_loss(log)
91             iteration = [k*scale for k in iteration]
92             data[net_name]["loss"]["iteration"].extend(iteration)
93             data[net_name]["loss"]["loss"].extend(loss)
96             iteration, loss = get_train_loss(log)
97             iteration = [k*scale for k in iteration]
98             data[net_name]["train_loss"]["iteration"].extend(iteration)
99             data[net_name]["train_loss"]["loss"].extend(loss)
101     return data
104 def fmt(x, y):
105     return 'x: {x:0.2f}\ny: {y:0.2f}'.format(x=x, y=y)
108 class FollowDotCursor(object):
109     """Display the x,y location of the nearest data point.
110     http://stackoverflow.com/a/4674445/190597 (Joe Kington)
111     http://stackoverflow.com/a/20637433/190597 (unutbu)
112     """
113     def __init__(self, ax, x, y, formatter=fmt, offsets=(-20, 20)):
114         try:
115             x = np.asarray(x, dtype='float')
116         except (TypeError, ValueError):
117             x = np.asarray(mdates.date2num(x), dtype='float')
118         y = np.asarray(y, dtype='float')
119         mask = ~(np.isnan(x) | np.isnan(y))
120         x = x[mask]
121         y = y[mask]
122         self._points = np.column_stack((x, y))
123         self.offsets = offsets
124         y = y[np.abs(y - y.mean()) <= 3 * y.std()]
125         self.scale = x.ptp()
126         self.scale = y.ptp() / self.scale if self.scale else 1
127         self.tree = spatial.cKDTree(self.scaled(self._points))
128         self.formatter = formatter
129         self.ax = ax
130         self.fig = ax.figure
131         self.ax.xaxis.set_label_position('top')
132         self.dot = ax.scatter(
133             [x.min()], [y.min()], s=130, color='green', alpha=0.7)
134         self.annotation = self.setup_annotation()
135         plt.connect('motion_notify_event', self)
137     def scaled(self, points):
138         points = np.asarray(points)
139         return points * (self.scale, 1)
141     def __call__(self, event):
142         ax = self.ax
143         # event.inaxes is always the current axis. If you use twinx, ax could be
144         # a different axis.
145         if event.inaxes == ax:
146             x, y = event.xdata, event.ydata
147         elif event.inaxes is None:
148             return
149         else:
150             inv = ax.transData.inverted()
151             x, y = inv.transform([(event.x, event.y)]).ravel()
152         annotation = self.annotation
153         x, y = self.snap(x, y)
154         annotation.xy = x, y
155         annotation.set_text(self.formatter(x, y))
156         self.dot.set_offsets((x, y))
157         event.canvas.draw()
159     def setup_annotation(self):
160         """Draw and hide the annotation box."""
161         annotation = self.ax.annotate(
162             '', xy=(0, 0), ha = 'right',
163             xytext = self.offsets, textcoords = 'offset points', va = 'bottom',
164             bbox = dict(
165                 boxstyle='round,pad=0.5', fc='yellow', alpha=0.75),
166             arrowprops = dict(
167                 arrowstyle='->', connectionstyle='arc3,rad=0'))
168         return annotation
170     def snap(self, x, y):
171         """Return the value in self.tree closest to x, y."""
172         dist, idx = self.tree.query(self.scaled((x, y)), k=1, p=1)
173         try:
174             return self._points[idx]
175         except IndexError:
176             # IndexError: index out of bounds
177             return self._points[0]
180 def plot_accuracy(top_k, data, value_at_hover=False):
181     nets =  data.keys()
182     colors = iter(cm.rainbow(np.linspace(0, 1, len(nets))))
183     fig = plt.figure()
184     ax = fig.add_subplot(111)
185     for net in nets:
186         iteration = data[net]["accuracy"]["iteration"]
187         accuracy = data[net]["accuracy"]["accuracy"]
188         iteration, accuracy = (np.array(t) for t in zip(*sorted(zip(iteration, accuracy))))
189         ax.plot(iteration, accuracy*100, color=next(colors), linestyle='-')
190         if value_at_hover:
191             cursor = FollowDotCursor(ax, iteration, accuracy*100)
193     plt.legend(nets, loc='lower right')
194     plt.title("Top {}".format(top_k))
195     plt.xlabel("Epochs")
196     plt.ylabel("Accuracy [%]")
197     plt.ylim(0,100)
198     plt.grid()
199     return plt
202 def plot_loss(data, value_at_hover=False):
203     nets =  data.keys()
204     colors = iter(cm.rainbow(np.linspace(0, 1, len(nets))))
205     fig = plt.figure()
206     ax = fig.add_subplot(111)
207     for net in nets:
208         iteration = data[net]["loss"]["iteration"]
209         loss = data[net]["loss"]["loss"]
210         iteration, loss = (list(t) for t in zip(*sorted(zip(iteration, loss))))
211         ax.scatter(iteration, loss, color=next(colors))
212         if value_at_hover:
213             cursor = FollowDotCursor(ax, iteration, loss)
215     plt.legend(nets, loc='upper right')
216     plt.title("Log Loss")
217     plt.xlabel("Iteration")
218     plt.ylabel("Log Loss")
219     plt.xlim(0)
220     plt.grid()
221     return plt
223 def plot_train_loss(data, value_at_hover=False):
224     nets =  data.keys()
225     colors = iter(cm.rainbow(np.linspace(0, 1, len(nets))))
226     fig = plt.figure()
227     ax = fig.add_subplot(111)
228     for net in nets:
229         iteration = data[net]["train_loss"]["iteration"]
230         loss = data[net]["train_loss"]["loss"]
231         iteration, loss = (list(t) for t in zip(*sorted(zip(iteration, loss))))
232         ax.scatter(iteration, loss, color=next(colors))
233         if value_at_hover:
234             cursor = FollowDotCursor(ax, iteration, loss)
236     plt.legend(nets, loc='upper right')
237     plt.title("Log Loss")
238     plt.xlabel("Iteration")
239     plt.ylabel("Log Loss")
240     plt.xlim(0)
241     plt.grid()
242     return plt