1 /******************************************************************************
2 * Copyright (c) 2018, Texas Instruments Incorporated - http://www.ti.com/
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are met:
7 * * Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * * Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 * * Neither the name of Texas Instruments Incorporated nor the
13 * names of its contributors may be used to endorse or promote products
14 * derived from this software without specific prior written permission.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
17 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
20 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
21 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
22 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
23 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
24 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
25 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
26 * THE POSSIBILITY OF SUCH DAMAGE.
27 *****************************************************************************/
28 #include <signal.h>
29 #include <getopt.h>
30 #include <iostream>
31 #include <iomanip>
32 #include <fstream>
33 #include <cassert>
34 #include <string>
35 #include <functional>
36 #include <algorithm>
37 #include <time.h>
38 #include <unistd.h>
40 #include <queue>
41 #include <vector>
43 #include "executor.h"
44 #include "execution_object.h"
45 #include "configuration.h"
46 #include "imagenet_classes.h"
47 #include "imgutil.h"
49 #include "opencv2/core.hpp"
50 #include "opencv2/imgproc.hpp"
51 #include "opencv2/highgui.hpp"
52 #include "opencv2/videoio.hpp"
55 bool __TI_show_debug_ = false;
57 using namespace tinn;
58 using namespace tinn::imgutil;
59 using namespace cv;
61 #define NUM_VIDEO_FRAMES 100
63 bool RunConfiguration(const std::string& config_file, int num_devices,
64 DeviceType device_type, std::string& input_file);
65 bool RunAllConfigurations(int32_t num_devices, DeviceType device_type);
67 bool ReadFrame(ExecutionObject& eo, int frame_idx,
68 const Configuration& configuration, int num_frames,
69 std::string& image_file, VideoCapture &cap);
71 bool WriteFrameOutput(const ExecutionObject &eo);
73 static void ProcessArgs(int argc, char *argv[],
74 std::string& config,
75 int& num_devices,
76 DeviceType& device_type,
77 std::string& input_file);
79 static void DisplayHelp();
81 static double ms_diff(struct timespec &t0, struct timespec &t1)
82 { return (t1.tv_sec - t0.tv_sec) * 1e3 + (t1.tv_nsec - t0.tv_nsec) / 1e6; }
85 int main(int argc, char *argv[])
86 {
87 // Catch ctrl-c to ensure a clean exit
88 signal(SIGABRT, exit);
89 signal(SIGTERM, exit);
91 // If there are no devices capable of offloading TIDL on the SoC, exit
92 uint32_t num_dla = Executor::GetNumDevices(DeviceType::DLA);
93 uint32_t num_dsp = Executor::GetNumDevices(DeviceType::DSP);
94 if (num_dla == 0 && num_dsp == 0)
95 {
96 std::cout << "TI DL not supported on this SoC." << std::endl;
97 return EXIT_SUCCESS;
98 }
100 // Process arguments
101 chdir("../test");
102 std::string config = "j11_v2";
103 std::string input_file = "../test/testvecs/input/airshow.jpg";
104 int num_devices = 1;
105 DeviceType device_type = DeviceType::DLA;
106 ProcessArgs(argc, argv, config, num_devices, device_type, input_file);
108 std::cout << "Input: " << input_file << std::endl;
109 std::string config_file = "../test/testvecs/config/infer/tidl_config_"
110 + config + ".txt";
111 bool status = RunConfiguration(config_file, num_devices, device_type,
112 input_file);
114 if (!status)
115 {
116 std::cout << "imagenet FAILED" << std::endl;
117 return EXIT_FAILURE;
118 }
120 std::cout << "imagenet PASSED" << std::endl;
121 return EXIT_SUCCESS;
122 }
124 bool RunConfiguration(const std::string& config_file, int num_devices,
125 DeviceType device_type, std::string& input_file)
126 {
127 DeviceIds ids;
128 for (int i = 0; i < num_devices; i++)
129 ids.insert(static_cast<DeviceId>(i));
131 // Read the TI DL configuration file
132 Configuration configuration;
133 bool status = configuration.ReadFromFile(config_file);
134 if (!status)
135 {
136 std::cerr << "Error in configuration file: " << config_file
137 << std::endl;
138 return false;
139 }
141 // setup input
142 int num_frames = 1;
143 VideoCapture cap;
144 std::string image_file;
145 if (input_file == "camera")
146 {
147 cap = VideoCapture(1); // cap = VideoCapture("test.mp4");
148 if (! cap.isOpened())
149 {
150 std::cerr << "Cannot open camera input." << std::endl;
151 return false;
152 }
153 num_frames = NUM_VIDEO_FRAMES;
154 namedWindow("ImageNet", WINDOW_AUTOSIZE | CV_GUI_NORMAL);
155 }
156 else
157 {
158 image_file = input_file;
159 }
161 // Determine input frame size from configuration
162 size_t frame_sz = configuration.inWidth * configuration.inHeight *
163 configuration.inNumChannels;
165 try
166 {
167 // Create a executor with the approriate core type, number of cores
168 // and configuration specified
169 Executor executor(device_type, ids, configuration);
171 // Query Executor for set of ExecutionObjects created
172 const ExecutionObjects& execution_objects =
173 executor.GetExecutionObjects();
174 int num_eos = execution_objects.size();
176 // Allocate input and output buffers for each execution object
177 std::vector<void *> buffers;
178 for (auto &eo : execution_objects)
179 {
180 ArgInfo in = { ArgInfo(malloc_ddr<char>(frame_sz), frame_sz)};
181 ArgInfo out = { ArgInfo(malloc_ddr<char>(frame_sz), frame_sz)};
182 eo->SetInputOutputBuffer(in, out);
184 buffers.push_back(in.ptr());
185 buffers.push_back(out.ptr());
186 }
188 #define MAX_NUM_EOS 4
189 struct timespec t0[MAX_NUM_EOS], t1;
191 // Process frames with available execution objects in a pipelined manner
192 // additional num_eos iterations to flush the pipeline (epilogue)
193 for (int frame_idx = 0;
194 frame_idx < num_frames + num_eos; frame_idx++)
195 {
196 ExecutionObject* eo = execution_objects[frame_idx % num_eos].get();
198 // Wait for previous frame on the same eo to finish processing
199 if (eo->ProcessFrameWait())
200 {
201 clock_gettime(CLOCK_MONOTONIC, &t1);
202 double elapsed_host =
203 ms_diff(t0[eo->GetFrameIndex() % num_eos], t1);
204 double elapsed_device = eo->GetProcessTimeInMilliSeconds();
205 double overhead = 100 - (elapsed_device/elapsed_host*100);
207 std::cout << "frame[" << eo->GetFrameIndex() << "]: "
208 << "Time on device: "
209 << std::setw(6) << std::setprecision(4)
210 << elapsed_device << "ms, "
211 << "host: "
212 << std::setw(6) << std::setprecision(4)
213 << elapsed_host << "ms ";
214 std::cout << "API overhead: "
215 << std::setw(6) << std::setprecision(3)
216 << overhead << " %" << std::endl;
218 WriteFrameOutput(*eo);
219 }
221 // Read a frame and start processing it with current eo
222 if (ReadFrame(*eo, frame_idx, configuration, num_frames,
223 image_file, cap))
224 {
225 clock_gettime(CLOCK_MONOTONIC, &t0[frame_idx % num_eos]);
226 eo->ProcessFrameStartAsync();
227 }
228 }
230 for (auto b : buffers)
231 __free_ddr(b);
233 }
234 catch (tinn::Exception &e)
235 {
236 std::cerr << e.what() << std::endl;
237 status = false;
238 }
240 return status;
241 }
244 bool ReadFrame(ExecutionObject &eo, int frame_idx,
245 const Configuration& configuration, int num_frames,
246 std::string& image_file, VideoCapture &cap)
247 {
248 if (frame_idx >= num_frames)
249 return false;
250 eo.SetFrameIndex(frame_idx);
252 char* frame_buffer = eo.GetInputBufferPtr();
253 assert (frame_buffer != nullptr);
255 Mat image;
256 if (! image_file.empty())
257 {
258 image = cv::imread(image_file, CV_LOAD_IMAGE_COLOR);
259 if (image.empty())
260 {
261 std::cerr << "Unable to read from: " << image_file << std::endl;
262 return false;
263 }
264 }
265 else
266 {
267 Mat v_image;
268 if (! cap.grab()) return false;
269 if (! cap.retrieve(v_image)) return false;
270 // Crop 640x480 camera input to center 256x256 input
271 image = Mat(v_image, Rect(192, 112, 256, 256));
272 cv::imshow("ImageNet", image);
273 waitKey(2);
274 }
276 // TI DL image preprocessing, into frame_buffer
277 return PreProcImage(image, frame_buffer, 1, 3,
278 configuration.inWidth, configuration.inHeight,
279 configuration.inWidth,
280 configuration.inWidth * configuration.inHeight,
281 1, configuration.preProcType);
282 }
284 // Display top 5 classified imagenet classes with probabilities
285 bool WriteFrameOutput(const ExecutionObject &eo)
286 {
287 const int k = 5;
288 unsigned char *out = (unsigned char *) eo.GetOutputBufferPtr();
289 int out_size = eo.GetOutputBufferSizeInBytes();
291 // sort and get k largest values and corresponding indices
292 typedef std::pair<unsigned char, int> val_index;
293 auto constexpr cmp = [](val_index &left, val_index &right)
294 { return left.first > right.first; };
295 std::priority_queue<val_index, std::vector<val_index>, decltype(cmp)>
296 queue(cmp);
297 // initialize priority queue with smallest value on top
298 for (int i = 0; i < k; i++)
299 queue.push(val_index(out[i], i));
301 // for rest output, if larger than current min, pop min, push new val
302 for (int i = k; i < out_size; i++)
303 {
304 if (out[i] > queue.top().first)
305 {
306 queue.pop();
307 queue.push(val_index(out[i], i));
308 }
309 }
311 // output top k values in reverse order: largest val first
312 std::vector<val_index> sorted;
313 while (! queue.empty())
314 {
315 sorted.push_back(queue.top());
316 queue.pop();
317 }
319 for (int i = k - 1; i >= 0; i--)
320 {
321 std::cout << k-i << ": " << imagenet_classes[sorted[i].second]
322 << ", prob = " << (float) sorted[i].first / 256 << std::endl;
323 }
325 return true;
326 }
329 void ProcessArgs(int argc, char *argv[], std::string& config,
330 int& num_devices, DeviceType& device_type,
331 std::string& input_file)
332 {
333 const struct option long_options[] =
334 {
335 {"config", required_argument, 0, 'c'},
336 {"num_devices", required_argument, 0, 'n'},
337 {"device_type", required_argument, 0, 't'},
338 {"image_file", required_argument, 0, 'i'},
339 {"help", no_argument, 0, 'h'},
340 {"verbose", no_argument, 0, 'v'},
341 {0, 0, 0, 0}
342 };
344 int option_index = 0;
346 while (true)
347 {
348 int c = getopt_long(argc, argv, "c:n:t:i:hv", long_options, &option_index);
350 if (c == -1)
351 break;
353 switch (c)
354 {
355 case 'c': config = optarg;
356 break;
358 case 'n': num_devices = atoi(optarg);
359 assert (num_devices > 0 && num_devices <= 4);
360 break;
362 case 't': if (*optarg == 'e')
363 device_type = DeviceType::DLA;
364 else if (*optarg == 'd')
365 device_type = DeviceType::DSP;
366 else
367 {
368 std::cerr << "Invalid argument to -t, only e or d"
369 " allowed" << std::endl;
370 exit(EXIT_FAILURE);
371 }
372 break;
374 case 'i': input_file = optarg;
375 break;
377 case 'v': __TI_show_debug_ = true;
378 break;
380 case 'h': DisplayHelp();
381 exit(EXIT_SUCCESS);
382 break;
384 case '?': // Error in getopt_long
385 exit(EXIT_FAILURE);
386 break;
388 default:
389 std::cerr << "Unsupported option: " << c << std::endl;
390 break;
391 }
392 }
393 }
395 void DisplayHelp()
396 {
397 std::cout << "Usage: imagenet\n"
398 " Will run imagenet network to predict top 5 object"
399 " classes for the input.\n Use -c to run a"
400 " different imagenet network. Default is j11_v2.\n"
401 "Optional arguments:\n"
402 " -c <config> Valid configs: j11_bn, j11_prelu, j11_v2\n"
403 " -n <number of cores> Number of cores to use (1 - 4)\n"
404 " -t <d|e> Type of core. d -> DSP, e -> DLA\n"
405 " -i <image> Path to the image file\n"
406 " -i camera Use camera as input\n"
407 " -v Verbose output during execution\n"
408 " -h Help\n";
409 }