1 /******************************************************************************
2 * Copyright (c) 2018, Texas Instruments Incorporated - http://www.ti.com/
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are met:
7 * * Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * * Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 * * Neither the name of Texas Instruments Incorporated nor the
13 * names of its contributors may be used to endorse or promote products
14 * derived from this software without specific prior written permission.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
17 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
20 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
21 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
22 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
23 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
24 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
25 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
26 * THE POSSIBILITY OF SUCH DAMAGE.
27 *****************************************************************************/
28 #include <signal.h>
29 #include <getopt.h>
30 #include <iostream>
31 #include <iomanip>
32 #include <fstream>
33 #include <cassert>
34 #include <string>
35 #include <functional>
36 #include <algorithm>
37 #include <time.h>
38 #include <unistd.h>
40 #include <queue>
41 #include <vector>
43 #include "executor.h"
44 #include "execution_object.h"
45 #include "configuration.h"
46 #include "imagenet_classes.h"
47 #include "imgutil.h"
49 #include "opencv2/core.hpp"
50 #include "opencv2/imgproc.hpp"
51 #include "opencv2/highgui.hpp"
52 #include "opencv2/videoio.hpp"
55 bool __TI_show_debug_ = false;
57 using namespace tidl;
58 using namespace tidl::imgutil;
59 using namespace cv;
61 #define NUM_VIDEO_FRAMES 100
63 bool RunConfiguration(const std::string& config_file, int num_devices,
64 DeviceType device_type, std::string& input_file);
65 bool RunAllConfigurations(int32_t num_devices, DeviceType device_type);
67 bool ReadFrame(ExecutionObject& eo, int frame_idx,
68 const Configuration& configuration, int num_frames,
69 std::string& image_file, VideoCapture &cap);
71 bool WriteFrameOutput(const ExecutionObject &eo);
73 static void ProcessArgs(int argc, char *argv[],
74 std::string& config,
75 int& num_devices,
76 DeviceType& device_type,
77 std::string& input_file);
79 static void DisplayHelp();
81 static double ms_diff(struct timespec &t0, struct timespec &t1)
82 { return (t1.tv_sec - t0.tv_sec) * 1e3 + (t1.tv_nsec - t0.tv_nsec) / 1e6; }
85 int main(int argc, char *argv[])
86 {
87 // Catch ctrl-c to ensure a clean exit
88 signal(SIGABRT, exit);
89 signal(SIGTERM, exit);
91 // If there are no devices capable of offloading TIDL on the SoC, exit
92 uint32_t num_eve = Executor::GetNumDevices(DeviceType::EVE);
93 uint32_t num_dsp = Executor::GetNumDevices(DeviceType::DSP);
94 if (num_eve == 0 && num_dsp == 0)
95 {
96 std::cout << "TI DL not supported on this SoC." << std::endl;
97 return EXIT_SUCCESS;
98 }
100 // Process arguments
101 std::string config = "j11_v2";
102 std::string input_file = "../test/testvecs/input/objects/cat-pet-animal-domestic-104827.jpeg";
103 int num_devices = 1;
104 DeviceType device_type = (num_eve > 0 ? DeviceType::EVE:DeviceType::DSP);
105 ProcessArgs(argc, argv, config, num_devices, device_type, input_file);
107 std::cout << "Input: " << input_file << std::endl;
108 std::string config_file = "../test/testvecs/config/infer/tidl_config_"
109 + config + ".txt";
110 bool status = RunConfiguration(config_file, num_devices, device_type,
111 input_file);
113 if (!status)
114 {
115 std::cout << "imagenet FAILED" << std::endl;
116 return EXIT_FAILURE;
117 }
119 std::cout << "imagenet PASSED" << std::endl;
120 return EXIT_SUCCESS;
121 }
123 bool RunConfiguration(const std::string& config_file, int num_devices,
124 DeviceType device_type, std::string& input_file)
125 {
126 DeviceIds ids;
127 for (int i = 0; i < num_devices; i++)
128 ids.insert(static_cast<DeviceId>(i));
130 // Read the TI DL configuration file
131 Configuration configuration;
132 bool status = configuration.ReadFromFile(config_file);
133 if (!status)
134 {
135 std::cerr << "Error in configuration file: " << config_file
136 << std::endl;
137 return false;
138 }
140 // setup input
141 int num_frames = 1;
142 VideoCapture cap;
143 std::string image_file;
144 if (input_file == "camera")
145 {
146 cap = VideoCapture(1); // cap = VideoCapture("test.mp4");
147 if (! cap.isOpened())
148 {
149 std::cerr << "Cannot open camera input." << std::endl;
150 return false;
151 }
152 num_frames = NUM_VIDEO_FRAMES;
153 namedWindow("ImageNet", WINDOW_AUTOSIZE | CV_GUI_NORMAL);
154 }
155 else
156 {
157 image_file = input_file;
158 }
160 try
161 {
162 // Create a executor with the approriate core type, number of cores
163 // and configuration specified
164 Executor executor(device_type, ids, configuration);
166 // Query Executor for set of ExecutionObjects created
167 const ExecutionObjects& execution_objects =
168 executor.GetExecutionObjects();
169 int num_eos = execution_objects.size();
171 // Allocate input and output buffers for each execution object
172 std::vector<void *> buffers;
173 for (auto &eo : execution_objects)
174 {
175 size_t in_size = eo->GetInputBufferSizeInBytes();
176 size_t out_size = eo->GetOutputBufferSizeInBytes();
177 ArgInfo in = { ArgInfo(malloc(in_size), in_size)};
178 ArgInfo out = { ArgInfo(malloc(out_size), out_size)};
179 eo->SetInputOutputBuffer(in, out);
181 buffers.push_back(in.ptr());
182 buffers.push_back(out.ptr());
183 }
185 #define MAX_NUM_EOS 4
186 struct timespec t0[MAX_NUM_EOS], t1;
188 // Process frames with available execution objects in a pipelined manner
189 // additional num_eos iterations to flush the pipeline (epilogue)
190 for (int frame_idx = 0;
191 frame_idx < num_frames + num_eos; frame_idx++)
192 {
193 ExecutionObject* eo = execution_objects[frame_idx % num_eos].get();
195 // Wait for previous frame on the same eo to finish processing
196 if (eo->ProcessFrameWait())
197 {
198 clock_gettime(CLOCK_MONOTONIC, &t1);
199 double elapsed_host =
200 ms_diff(t0[eo->GetFrameIndex() % num_eos], t1);
201 double elapsed_device = eo->GetProcessTimeInMilliSeconds();
202 double overhead = 100 - (elapsed_device/elapsed_host*100);
204 std::cout << "frame[" << eo->GetFrameIndex() << "]: "
205 << "Time on device: "
206 << std::setw(6) << std::setprecision(4)
207 << elapsed_device << "ms, "
208 << "host: "
209 << std::setw(6) << std::setprecision(4)
210 << elapsed_host << "ms ";
211 std::cout << "API overhead: "
212 << std::setw(6) << std::setprecision(3)
213 << overhead << " %" << std::endl;
215 WriteFrameOutput(*eo);
216 }
218 // Read a frame and start processing it with current eo
219 if (ReadFrame(*eo, frame_idx, configuration, num_frames,
220 image_file, cap))
221 {
222 clock_gettime(CLOCK_MONOTONIC, &t0[frame_idx % num_eos]);
223 eo->ProcessFrameStartAsync();
224 }
225 }
227 for (auto b : buffers)
228 free(b);
230 }
231 catch (tidl::Exception &e)
232 {
233 std::cerr << e.what() << std::endl;
234 status = false;
235 }
237 return status;
238 }
241 bool ReadFrame(ExecutionObject &eo, int frame_idx,
242 const Configuration& configuration, int num_frames,
243 std::string& image_file, VideoCapture &cap)
244 {
245 if (frame_idx >= num_frames)
246 return false;
247 eo.SetFrameIndex(frame_idx);
249 char* frame_buffer = eo.GetInputBufferPtr();
250 assert (frame_buffer != nullptr);
252 Mat image;
253 if (! image_file.empty())
254 {
255 image = cv::imread(image_file, CV_LOAD_IMAGE_COLOR);
256 if (image.empty())
257 {
258 std::cerr << "Unable to read from: " << image_file << std::endl;
259 return false;
260 }
261 }
262 else
263 {
264 Mat v_image;
265 if (! cap.grab()) return false;
266 if (! cap.retrieve(v_image)) return false;
267 // Crop 640x480 camera input to center 256x256 input
268 image = Mat(v_image, Rect(192, 112, 256, 256));
269 cv::imshow("ImageNet", image);
270 waitKey(2);
271 }
273 // TI DL image preprocessing, into frame_buffer
274 return PreProcImage(image, frame_buffer, 1, 3,
275 configuration.inWidth, configuration.inHeight,
276 configuration.inWidth,
277 configuration.inWidth * configuration.inHeight,
278 1, configuration.preProcType);
279 }
281 // Display top 5 classified imagenet classes with probabilities
282 bool WriteFrameOutput(const ExecutionObject &eo)
283 {
284 const int k = 5;
285 unsigned char *out = (unsigned char *) eo.GetOutputBufferPtr();
286 int out_size = eo.GetOutputBufferSizeInBytes();
288 // sort and get k largest values and corresponding indices
289 typedef std::pair<unsigned char, int> val_index;
290 auto constexpr cmp = [](val_index &left, val_index &right)
291 { return left.first > right.first; };
292 std::priority_queue<val_index, std::vector<val_index>, decltype(cmp)>
293 queue(cmp);
294 // initialize priority queue with smallest value on top
295 for (int i = 0; i < k; i++)
296 queue.push(val_index(out[i], i));
298 // for rest output, if larger than current min, pop min, push new val
299 for (int i = k; i < out_size; i++)
300 {
301 if (out[i] > queue.top().first)
302 {
303 queue.pop();
304 queue.push(val_index(out[i], i));
305 }
306 }
308 // output top k values in reverse order: largest val first
309 std::vector<val_index> sorted;
310 while (! queue.empty())
311 {
312 sorted.push_back(queue.top());
313 queue.pop();
314 }
316 for (int i = k - 1; i >= 0; i--)
317 {
318 std::cout << k-i << ": " << imagenet_classes[sorted[i].second]
319 << ", prob = " << (float) sorted[i].first / 256 << std::endl;
320 }
322 return true;
323 }
326 void ProcessArgs(int argc, char *argv[], std::string& config,
327 int& num_devices, DeviceType& device_type,
328 std::string& input_file)
329 {
330 const struct option long_options[] =
331 {
332 {"config", required_argument, 0, 'c'},
333 {"num_devices", required_argument, 0, 'n'},
334 {"device_type", required_argument, 0, 't'},
335 {"image_file", required_argument, 0, 'i'},
336 {"help", no_argument, 0, 'h'},
337 {"verbose", no_argument, 0, 'v'},
338 {0, 0, 0, 0}
339 };
341 int option_index = 0;
343 while (true)
344 {
345 int c = getopt_long(argc, argv, "c:n:t:i:hv", long_options, &option_index);
347 if (c == -1)
348 break;
350 switch (c)
351 {
352 case 'c': config = optarg;
353 break;
355 case 'n': num_devices = atoi(optarg);
356 assert (num_devices > 0 && num_devices <= 4);
357 break;
359 case 't': if (*optarg == 'e')
360 device_type = DeviceType::EVE;
361 else if (*optarg == 'd')
362 device_type = DeviceType::DSP;
363 else
364 {
365 std::cerr << "Invalid argument to -t, only e or d"
366 " allowed" << std::endl;
367 exit(EXIT_FAILURE);
368 }
369 break;
371 case 'i': input_file = optarg;
372 break;
374 case 'v': __TI_show_debug_ = true;
375 break;
377 case 'h': DisplayHelp();
378 exit(EXIT_SUCCESS);
379 break;
381 case '?': // Error in getopt_long
382 exit(EXIT_FAILURE);
383 break;
385 default:
386 std::cerr << "Unsupported option: " << c << std::endl;
387 break;
388 }
389 }
390 }
392 void DisplayHelp()
393 {
394 std::cout << "Usage: imagenet\n"
395 " Will run imagenet network to predict top 5 object"
396 " classes for the input.\n Use -c to run a"
397 " different imagenet network. Default is j11_v2.\n"
398 "Optional arguments:\n"
399 " -c <config> Valid configs: j11_bn, j11_prelu, j11_v2\n"
400 " -n <number of cores> Number of cores to use (1 - 4)\n"
401 " -t <d|e> Type of core. d -> DSP, e -> EVE\n"
402 " -i <image> Path to the image file\n"
403 " -i camera Use camera as input\n"
404 " -v Verbose output during execution\n"
405 " -h Help\n";
406 }