1 /******************************************************************************
2 * Copyright (c) 2018, Texas Instruments Incorporated - http://www.ti.com/
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are met:
7 * * Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * * Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 * * Neither the name of Texas Instruments Incorporated nor the
13 * names of its contributors may be used to endorse or promote products
14 * derived from this software without specific prior written permission.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
17 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
20 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
21 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
22 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
23 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
24 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
25 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
26 * THE POSSIBILITY OF SUCH DAMAGE.
27 *****************************************************************************/
28 #include <signal.h>
29 #include <iostream>
30 #include <iomanip>
31 #include <fstream>
32 #include <cassert>
33 #include <string>
34 #include <functional>
35 #include <algorithm>
36 #include <time.h>
37 #include <unistd.h>
39 #include <queue>
40 #include <vector>
41 #include <chrono>
43 #include "executor.h"
44 #include "execution_object.h"
45 #include "configuration.h"
46 #include "imagenet_classes.h"
47 #include "imgutil.h"
48 #include "../common/video_utils.h"
50 #include "opencv2/core.hpp"
51 #include "opencv2/imgproc.hpp"
52 #include "opencv2/highgui.hpp"
53 #include "opencv2/videoio.hpp"
55 using namespace std;
56 using namespace tidl;
57 using namespace tidl::imgutil;
58 using namespace cv;
60 #define NUM_VIDEO_FRAMES 300
61 #define DEFAULT_CONFIG "j11_v2"
62 #define NUM_DEFAULT_INPUTS 1
63 const char *default_inputs[NUM_DEFAULT_INPUTS] =
64 {
65 "../test/testvecs/input/objects/cat-pet-animal-domestic-104827.jpeg"
66 };
68 Executor* CreateExecutor(DeviceType dt, uint32_t num, const Configuration& c);
69 bool RunConfiguration(cmdline_opts_t& opts);
70 bool ReadFrame(ExecutionObject& eo, uint32_t frame_idx, const Configuration& c,
71 const cmdline_opts_t& opts, VideoCapture &cap);
72 bool WriteFrameOutput(const ExecutionObject &eo);
73 void DisplayHelp();
76 int main(int argc, char *argv[])
77 {
78 // Catch ctrl-c to ensure a clean exit
79 signal(SIGABRT, exit);
80 signal(SIGTERM, exit);
82 // If there are no devices capable of offloading TIDL on the SoC, exit
83 uint32_t num_eves = Executor::GetNumDevices(DeviceType::EVE);
84 uint32_t num_dsps = Executor::GetNumDevices(DeviceType::DSP);
85 if (num_eves == 0 && num_dsps == 0)
86 {
87 cout << "TI DL not supported on this SoC." << endl;
88 return EXIT_SUCCESS;
89 }
91 // Process arguments
92 cmdline_opts_t opts;
93 opts.config = DEFAULT_CONFIG;
94 if (num_eves != 0) { opts.num_eves = 1; opts.num_dsps = 0; }
95 else { opts.num_eves = 0; opts.num_dsps = 1; }
96 if (! ProcessArgs(argc, argv, opts))
97 {
98 DisplayHelp();
99 exit(EXIT_SUCCESS);
100 }
101 assert(opts.num_dsps != 0 || opts.num_eves != 0);
102 if (opts.num_frames == 0)
103 opts.num_frames = (opts.is_camera_input || opts.is_video_input) ?
104 NUM_VIDEO_FRAMES : 1;
105 if (opts.input_file.empty())
106 cout << "Input: " << default_inputs[0] << endl;
107 else
108 cout << "Input: " << opts.input_file << endl;
110 // Run network
111 bool status = RunConfiguration(opts);
112 if (!status)
113 {
114 cout << "imagenet FAILED" << endl;
115 return EXIT_FAILURE;
116 }
118 cout << "imagenet PASSED" << endl;
119 return EXIT_SUCCESS;
120 }
122 bool RunConfiguration(cmdline_opts_t& opts)
123 {
124 // Read the TI DL configuration file
125 Configuration c;
126 string config_file = "../test/testvecs/config/infer/tidl_config_"
127 + opts.config + ".txt";
128 bool status = c.ReadFromFile(config_file);
129 if (!status)
130 {
131 cerr << "Error in configuration file: " << config_file << endl;
132 return false;
133 }
134 c.enableApiTrace = opts.verbose;
136 // setup camera/video input/output
137 VideoCapture cap;
138 if (! SetVideoInputOutput(cap, opts, "ImageNet")) return false;
140 try
141 {
142 // Create Executors with the approriate core type, number of cores
143 // and configuration specified
144 Executor* e_eve = CreateExecutor(DeviceType::EVE, opts.num_eves, c);
145 Executor* e_dsp = CreateExecutor(DeviceType::DSP, opts.num_dsps, c);
147 // Get ExecutionObjects from Executors
148 vector<ExecutionObject*> eos;
149 for (uint32_t i = 0; i < opts.num_eves; i++) eos.push_back((*e_eve)[i]);
150 for (uint32_t i = 0; i < opts.num_dsps; i++) eos.push_back((*e_dsp)[i]);
151 uint32_t num_eos = eos.size();
153 // Allocate input and output buffers for each ExecutionObject
154 AllocateMemory(eos);
156 chrono::time_point<chrono::steady_clock> tloop0, tloop1;
157 tloop0 = chrono::steady_clock::now();
159 // Process frames with available eos in a pipelined manner
160 // additional num_eos iterations to flush the pipeline (epilogue)
161 for (uint32_t frame_idx = 0;
162 frame_idx < opts.num_frames + num_eos; frame_idx++)
163 {
164 ExecutionObject* eo = eos[frame_idx % num_eos];
166 // Wait for previous frame on the same eo to finish processing
167 if (eo->ProcessFrameWait())
168 {
169 ReportTime(eo);
170 WriteFrameOutput(*eo);
171 }
173 // Read a frame and start processing it with current eo
174 if (ReadFrame(*eo, frame_idx, c, opts, cap))
175 eo->ProcessFrameStartAsync();
176 }
178 tloop1 = chrono::steady_clock::now();
179 chrono::duration<float> elapsed = tloop1 - tloop0;
180 cout << "Loop total time (including read/write/opencv/print/etc): "
181 << setw(6) << setprecision(4)
182 << (elapsed.count() * 1000) << "ms" << endl;
184 FreeMemory(eos);
185 delete e_eve;
186 delete e_dsp;
187 }
188 catch (tidl::Exception &e)
189 {
190 cerr << e.what() << endl;
191 status = false;
192 }
194 return status;
195 }
197 // Create an Executor with the specified type and number of EOs
198 Executor* CreateExecutor(DeviceType dt, uint32_t num, const Configuration& c)
199 {
200 if (num == 0) return nullptr;
202 DeviceIds ids;
203 for (uint32_t i = 0; i < num; i++)
204 ids.insert(static_cast<DeviceId>(i));
206 return new Executor(dt, ids, c);
207 }
209 bool ReadFrame(ExecutionObject &eo, uint32_t frame_idx, const Configuration& c,
210 const cmdline_opts_t& opts, VideoCapture &cap)
211 {
212 if (frame_idx >= opts.num_frames)
213 return false;
214 eo.SetFrameIndex(frame_idx);
216 char* frame_buffer = eo.GetInputBufferPtr();
217 assert (frame_buffer != nullptr);
219 Mat image;
220 if (! opts.is_camera_input && ! opts.is_video_input)
221 {
222 if (opts.input_file.empty())
223 image = cv::imread(default_inputs[frame_idx % NUM_DEFAULT_INPUTS],
224 CV_LOAD_IMAGE_COLOR);
225 else
226 image = cv::imread(opts.input_file, CV_LOAD_IMAGE_COLOR);
227 if (image.empty())
228 {
229 cerr << "Unable to read input image" << endl;
230 return false;
231 }
232 }
233 else
234 {
235 Mat v_image;
236 if (! cap.grab()) return false;
237 if (! cap.retrieve(v_image)) return false;
238 int orig_width = v_image.cols;
239 int orig_height = v_image.rows;
240 // Crop camera/video input to center 256x256 input
241 if (orig_width > 256 && orig_height > 256)
242 {
243 image = Mat(v_image, Rect((orig_width-256)/2, (orig_height-256)/2,
244 256, 256));
245 }
246 else
247 image = v_image;
248 cv::imshow("ImageNet", image);
249 waitKey(2);
250 }
252 // TI DL image preprocessing, into frame_buffer
253 return PreProcImage(image, frame_buffer, 1, 3, c.inWidth, c.inHeight,
254 c.inWidth, c.inWidth * c.inHeight, 1, c.preProcType);
255 }
257 // Display top 5 classified imagenet classes with probabilities
258 bool WriteFrameOutput(const ExecutionObject &eo)
259 {
260 const int k = 5;
261 unsigned char *out = (unsigned char *) eo.GetOutputBufferPtr();
262 int out_size = eo.GetOutputBufferSizeInBytes();
264 // sort and get k largest values and corresponding indices
265 typedef pair<unsigned char, int> val_index;
266 auto constexpr cmp = [](val_index &left, val_index &right)
267 { return left.first > right.first; };
268 priority_queue<val_index, vector<val_index>, decltype(cmp)> queue(cmp);
269 // initialize priority queue with smallest value on top
270 for (int i = 0; i < k; i++)
271 queue.push(val_index(out[i], i));
273 // for rest output, if larger than current min, pop min, push new val
274 for (int i = k; i < out_size; i++)
275 {
276 if (out[i] > queue.top().first)
277 {
278 queue.pop();
279 queue.push(val_index(out[i], i));
280 }
281 }
283 // output top k values in reverse order: largest val first
284 vector<val_index> sorted;
285 while (! queue.empty())
286 {
287 sorted.push_back(queue.top());
288 queue.pop();
289 }
291 for (int i = k - 1; i >= 0; i--)
292 cout << k-i << ": " << imagenet_classes[sorted[i].second] << endl;
294 return true;
295 }
297 void DisplayHelp()
298 {
299 cout <<
300 "Usage: imagenet\n"
301 " Will run imagenet network to predict top 5 object"
302 " classes for the input.\n Use -c to run a"
303 " different imagenet network. Default is j11_v2.\n"
304 "Optional arguments:\n"
305 " -c <config> Valid configs: j11_bn, j11_prelu, j11_v2\n"
306 " -d <number> Number of dsp cores to use\n"
307 " -e <number> Number of eve cores to use\n"
308 " -i <image> Path to the image file as input\n"
309 " -i camera<number> Use camera as input\n"
310 " video input port: /dev/video<number>\n"
311 " -i <name>.{mp4,mov,avi} Use video file as input\n"
312 " -f <number> Number of frames to process\n"
313 " -v Verbose output during execution\n"
314 " -h Help\n";
315 }