1 /******************************************************************************
2 * Copyright (c) 2018, Texas Instruments Incorporated - http://www.ti.com/
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are met:
7 * * Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * * Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 * * Neither the name of Texas Instruments Incorporated nor the
13 * names of its contributors may be used to endorse or promote products
14 * derived from this software without specific prior written permission.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
17 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
20 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
21 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
22 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
23 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
24 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
25 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
26 * THE POSSIBILITY OF SUCH DAMAGE.
27 *****************************************************************************/
28 #include <signal.h>
29 #include <getopt.h>
30 #include <iostream>
31 #include <iomanip>
32 #include <fstream>
33 #include <cassert>
34 #include <string>
35 #include <functional>
36 #include <algorithm>
37 #include <time.h>
38 #include <unistd.h>
40 #include <queue>
41 #include <vector>
43 #include "executor.h"
44 #include "execution_object.h"
45 #include "configuration.h"
46 #include "imagenet_classes.h"
47 #include "imgutil.h"
49 #include "opencv2/core.hpp"
50 #include "opencv2/imgproc.hpp"
51 #include "opencv2/highgui.hpp"
52 #include "opencv2/videoio.hpp"
55 bool __TI_show_debug_ = false;
57 using namespace tinn;
58 using namespace tinn::imgutil;
59 using namespace cv;
61 #define NUM_VIDEO_FRAMES 100
63 bool RunConfiguration(const std::string& config_file, int num_devices,
64 DeviceType device_type, std::string& input_file);
65 bool RunAllConfigurations(int32_t num_devices, DeviceType device_type);
67 bool ReadFrame(ExecutionObject& eo, int frame_idx,
68 const Configuration& configuration, int num_frames,
69 std::string& image_file, VideoCapture &cap);
71 bool WriteFrameOutput(const ExecutionObject &eo);
73 static void ProcessArgs(int argc, char *argv[],
74 std::string& config,
75 int& num_devices,
76 DeviceType& device_type,
77 std::string& input_file);
79 static void DisplayHelp();
81 static double ms_diff(struct timespec &t0, struct timespec &t1)
82 { return (t1.tv_sec - t0.tv_sec) * 1e3 + (t1.tv_nsec - t0.tv_nsec) / 1e6; }
85 int main(int argc, char *argv[])
86 {
87 // Catch ctrl-c to ensure a clean exit
88 signal(SIGABRT, exit);
89 signal(SIGTERM, exit);
91 // If there are no devices capable of offloading TIDL on the SoC, exit
92 uint32_t num_dla = Executor::GetNumDevices(DeviceType::DLA);
93 uint32_t num_dsp = Executor::GetNumDevices(DeviceType::DSP);
94 if (num_dla == 0 && num_dsp == 0)
95 {
96 std::cout << "TI DL not supported on this SoC." << std::endl;
97 return EXIT_SUCCESS;
98 }
100 // Process arguments
101 std::string config = "j11_v2";
102 std::string input_file = "../test/testvecs/input/objects/cat-pet-animal-domestic-104827.jpeg";
103 int num_devices = 1;
104 DeviceType device_type = DeviceType::DLA;
105 ProcessArgs(argc, argv, config, num_devices, device_type, input_file);
107 std::cout << "Input: " << input_file << std::endl;
108 std::string config_file = "../test/testvecs/config/infer/tidl_config_"
109 + config + ".txt";
110 bool status = RunConfiguration(config_file, num_devices, device_type,
111 input_file);
113 if (!status)
114 {
115 std::cout << "imagenet FAILED" << std::endl;
116 return EXIT_FAILURE;
117 }
119 std::cout << "imagenet PASSED" << std::endl;
120 return EXIT_SUCCESS;
121 }
123 bool RunConfiguration(const std::string& config_file, int num_devices,
124 DeviceType device_type, std::string& input_file)
125 {
126 DeviceIds ids;
127 for (int i = 0; i < num_devices; i++)
128 ids.insert(static_cast<DeviceId>(i));
130 // Read the TI DL configuration file
131 Configuration configuration;
132 bool status = configuration.ReadFromFile(config_file);
133 if (!status)
134 {
135 std::cerr << "Error in configuration file: " << config_file
136 << std::endl;
137 return false;
138 }
140 // setup input
141 int num_frames = 1;
142 VideoCapture cap;
143 std::string image_file;
144 if (input_file == "camera")
145 {
146 cap = VideoCapture(1); // cap = VideoCapture("test.mp4");
147 if (! cap.isOpened())
148 {
149 std::cerr << "Cannot open camera input." << std::endl;
150 return false;
151 }
152 num_frames = NUM_VIDEO_FRAMES;
153 namedWindow("ImageNet", WINDOW_AUTOSIZE | CV_GUI_NORMAL);
154 }
155 else
156 {
157 image_file = input_file;
158 }
160 // Determine input frame size from configuration
161 size_t frame_sz = configuration.inWidth * configuration.inHeight *
162 configuration.inNumChannels;
164 try
165 {
166 // Create a executor with the approriate core type, number of cores
167 // and configuration specified
168 Executor executor(device_type, ids, configuration);
170 // Query Executor for set of ExecutionObjects created
171 const ExecutionObjects& execution_objects =
172 executor.GetExecutionObjects();
173 int num_eos = execution_objects.size();
175 // Allocate input and output buffers for each execution object
176 std::vector<void *> buffers;
177 for (auto &eo : execution_objects)
178 {
179 ArgInfo in = { ArgInfo(malloc(frame_sz), frame_sz)};
180 ArgInfo out = { ArgInfo(malloc(frame_sz), frame_sz)};
181 eo->SetInputOutputBuffer(in, out);
183 buffers.push_back(in.ptr());
184 buffers.push_back(out.ptr());
185 }
187 #define MAX_NUM_EOS 4
188 struct timespec t0[MAX_NUM_EOS], t1;
190 // Process frames with available execution objects in a pipelined manner
191 // additional num_eos iterations to flush the pipeline (epilogue)
192 for (int frame_idx = 0;
193 frame_idx < num_frames + num_eos; frame_idx++)
194 {
195 ExecutionObject* eo = execution_objects[frame_idx % num_eos].get();
197 // Wait for previous frame on the same eo to finish processing
198 if (eo->ProcessFrameWait())
199 {
200 clock_gettime(CLOCK_MONOTONIC, &t1);
201 double elapsed_host =
202 ms_diff(t0[eo->GetFrameIndex() % num_eos], t1);
203 double elapsed_device = eo->GetProcessTimeInMilliSeconds();
204 double overhead = 100 - (elapsed_device/elapsed_host*100);
206 std::cout << "frame[" << eo->GetFrameIndex() << "]: "
207 << "Time on device: "
208 << std::setw(6) << std::setprecision(4)
209 << elapsed_device << "ms, "
210 << "host: "
211 << std::setw(6) << std::setprecision(4)
212 << elapsed_host << "ms ";
213 std::cout << "API overhead: "
214 << std::setw(6) << std::setprecision(3)
215 << overhead << " %" << std::endl;
217 WriteFrameOutput(*eo);
218 }
220 // Read a frame and start processing it with current eo
221 if (ReadFrame(*eo, frame_idx, configuration, num_frames,
222 image_file, cap))
223 {
224 clock_gettime(CLOCK_MONOTONIC, &t0[frame_idx % num_eos]);
225 eo->ProcessFrameStartAsync();
226 }
227 }
229 for (auto b : buffers)
230 free(b);
232 }
233 catch (tinn::Exception &e)
234 {
235 std::cerr << e.what() << std::endl;
236 status = false;
237 }
239 return status;
240 }
243 bool ReadFrame(ExecutionObject &eo, int frame_idx,
244 const Configuration& configuration, int num_frames,
245 std::string& image_file, VideoCapture &cap)
246 {
247 if (frame_idx >= num_frames)
248 return false;
249 eo.SetFrameIndex(frame_idx);
251 char* frame_buffer = eo.GetInputBufferPtr();
252 assert (frame_buffer != nullptr);
254 Mat image;
255 if (! image_file.empty())
256 {
257 image = cv::imread(image_file, CV_LOAD_IMAGE_COLOR);
258 if (image.empty())
259 {
260 std::cerr << "Unable to read from: " << image_file << std::endl;
261 return false;
262 }
263 }
264 else
265 {
266 Mat v_image;
267 if (! cap.grab()) return false;
268 if (! cap.retrieve(v_image)) return false;
269 // Crop 640x480 camera input to center 256x256 input
270 image = Mat(v_image, Rect(192, 112, 256, 256));
271 cv::imshow("ImageNet", image);
272 waitKey(2);
273 }
275 // TI DL image preprocessing, into frame_buffer
276 return PreProcImage(image, frame_buffer, 1, 3,
277 configuration.inWidth, configuration.inHeight,
278 configuration.inWidth,
279 configuration.inWidth * configuration.inHeight,
280 1, configuration.preProcType);
281 }
283 // Display top 5 classified imagenet classes with probabilities
284 bool WriteFrameOutput(const ExecutionObject &eo)
285 {
286 const int k = 5;
287 unsigned char *out = (unsigned char *) eo.GetOutputBufferPtr();
288 int out_size = eo.GetOutputBufferSizeInBytes();
290 // sort and get k largest values and corresponding indices
291 typedef std::pair<unsigned char, int> val_index;
292 auto constexpr cmp = [](val_index &left, val_index &right)
293 { return left.first > right.first; };
294 std::priority_queue<val_index, std::vector<val_index>, decltype(cmp)>
295 queue(cmp);
296 // initialize priority queue with smallest value on top
297 for (int i = 0; i < k; i++)
298 queue.push(val_index(out[i], i));
300 // for rest output, if larger than current min, pop min, push new val
301 for (int i = k; i < out_size; i++)
302 {
303 if (out[i] > queue.top().first)
304 {
305 queue.pop();
306 queue.push(val_index(out[i], i));
307 }
308 }
310 // output top k values in reverse order: largest val first
311 std::vector<val_index> sorted;
312 while (! queue.empty())
313 {
314 sorted.push_back(queue.top());
315 queue.pop();
316 }
318 for (int i = k - 1; i >= 0; i--)
319 {
320 std::cout << k-i << ": " << imagenet_classes[sorted[i].second]
321 << ", prob = " << (float) sorted[i].first / 256 << std::endl;
322 }
324 return true;
325 }
328 void ProcessArgs(int argc, char *argv[], std::string& config,
329 int& num_devices, DeviceType& device_type,
330 std::string& input_file)
331 {
332 const struct option long_options[] =
333 {
334 {"config", required_argument, 0, 'c'},
335 {"num_devices", required_argument, 0, 'n'},
336 {"device_type", required_argument, 0, 't'},
337 {"image_file", required_argument, 0, 'i'},
338 {"help", no_argument, 0, 'h'},
339 {"verbose", no_argument, 0, 'v'},
340 {0, 0, 0, 0}
341 };
343 int option_index = 0;
345 while (true)
346 {
347 int c = getopt_long(argc, argv, "c:n:t:i:hv", long_options, &option_index);
349 if (c == -1)
350 break;
352 switch (c)
353 {
354 case 'c': config = optarg;
355 break;
357 case 'n': num_devices = atoi(optarg);
358 assert (num_devices > 0 && num_devices <= 4);
359 break;
361 case 't': if (*optarg == 'e')
362 device_type = DeviceType::DLA;
363 else if (*optarg == 'd')
364 device_type = DeviceType::DSP;
365 else
366 {
367 std::cerr << "Invalid argument to -t, only e or d"
368 " allowed" << std::endl;
369 exit(EXIT_FAILURE);
370 }
371 break;
373 case 'i': input_file = optarg;
374 break;
376 case 'v': __TI_show_debug_ = true;
377 break;
379 case 'h': DisplayHelp();
380 exit(EXIT_SUCCESS);
381 break;
383 case '?': // Error in getopt_long
384 exit(EXIT_FAILURE);
385 break;
387 default:
388 std::cerr << "Unsupported option: " << c << std::endl;
389 break;
390 }
391 }
392 }
394 void DisplayHelp()
395 {
396 std::cout << "Usage: imagenet\n"
397 " Will run imagenet network to predict top 5 object"
398 " classes for the input.\n Use -c to run a"
399 " different imagenet network. Default is j11_v2.\n"
400 "Optional arguments:\n"
401 " -c <config> Valid configs: j11_bn, j11_prelu, j11_v2\n"
402 " -n <number of cores> Number of cores to use (1 - 4)\n"
403 " -t <d|e> Type of core. d -> DSP, e -> DLA\n"
404 " -i <image> Path to the image file\n"
405 " -i camera Use camera as input\n"
406 " -v Verbose output during execution\n"
407 " -h Help\n";
408 }