1 /******************************************************************************
2 * Copyright (c) 2018, Texas Instruments Incorporated - http://www.ti.com/
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are met:
7 * * Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * * Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 * * Neither the name of Texas Instruments Incorporated nor the
13 * names of its contributors may be used to endorse or promote products
14 * derived from this software without specific prior written permission.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
17 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
20 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
21 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
22 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
23 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
24 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
25 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
26 * THE POSSIBILITY OF SUCH DAMAGE.
27 *****************************************************************************/
28 #include <signal.h>
29 #include <getopt.h>
30 #include <iostream>
31 #include <iomanip>
32 #include <fstream>
33 #include <cassert>
34 #include <string>
35 #include <functional>
36 #include <queue>
37 #include <algorithm>
38 #include <time.h>
39 #include <memory.h>
40 #include <string.h>
42 #include "executor.h"
43 #include "execution_object.h"
44 #include "execution_object_pipeline.h"
45 #include "configuration.h"
46 #include "avg_fps_window.h"
48 #include "opencv2/core.hpp"
49 #include "opencv2/imgproc.hpp"
50 #include "opencv2/highgui.hpp"
51 #include "opencv2/videoio.hpp"
54 //#define TWO_ROIs
55 #define LIVE_DISPLAY
56 #define PERF_VERBOSE
57 //#define RMT_GST_STREAMER
59 #define MAX_NUM_ROI 4
61 int live_input = 1;
62 char video_clip[320];
64 #ifdef TWO_ROIs
65 #define RES_X 400
66 #define RES_Y 300
67 #define NUM_ROI_X 2
68 #define NUM_ROI_Y 1
69 #define X_OFFSET 0
70 #define X_STEP 176
71 #define Y_OFFSET 52
72 #define Y_STEP 224
73 #else
74 #define RES_X 480
75 #define RES_Y 480
76 #define NUM_ROI_X 1
77 #define NUM_ROI_Y 1
78 #define X_OFFSET 10
79 #define X_STEP 460
80 #define Y_OFFSET 10
81 #define Y_STEP 460
82 #endif
84 #define NUM_ROI (NUM_ROI_X * NUM_ROI_Y)
86 //Temporal averaging
87 int TOP_CANDIDATES = 3;
89 using namespace tidl;
90 using namespace cv;
92 #ifdef LIVE_DISPLAY
93 char imagenet_win[160];
94 char tmp_classwindow_string[160];
95 Mat classlist_image;
97 void imagenetCallBackFunc(int event, int x, int y, int flags, void* userdata)
98 {
99 if ( event == EVENT_RBUTTONDOWN )
100 {
101 std::cout << "Right button of the mouse is clicked - position (" << x << ", " << y << ")" << " ... prepare to exit!" << std::endl;
102 exit(0);
103 }
104 }
105 #endif
107 Mat in_image, image, r_image, cnn_image, show_image, bgr_frames[3];
108 Mat to_stream;
109 Rect rectCrop[NUM_ROI];
110 // Report average FPS across a sliding window of 16 frames
111 AvgFPSWindow fps_window(16);
113 static int tf_postprocess(uchar *in, int size, int roi_idx, int frame_idx, int f_id);
114 static void tf_preprocess(uchar *out, uchar *in, int size);
115 static int ShowRegion(int roi_history[]);
116 // from most recent to oldest at top indices
117 static int selclass_history[MAX_NUM_ROI][3];
119 bool __TI_show_debug_ = false;
121 bool RunConfiguration(const std::string& config_file, int num_layers_groups,
122 uint32_t num_dsps, uint32_t num_eves);
123 bool CreateExecutionObjectPipelines(uint32_t num_eves, uint32_t num_dsps,
124 Configuration& configuration,
125 uint32_t num_layers_groups,
126 Executor*& e_eve, Executor*& e_dsp,
127 std::vector<ExecutionObjectPipeline*>& eops);
128 void AllocateMemory(const std::vector<ExecutionObjectPipeline*>& eops);
129 void SetupLiveDisplay(uint32_t num_eves, uint32_t num_dsps);
130 bool SetupInput(VideoCapture& cap, VideoWriter& writer);
131 bool ReadFrame(ExecutionObjectPipeline* eop,
132 uint32_t frame_idx, uint32_t num_frames,
133 VideoCapture &cap, VideoWriter& writer);
134 void DisplayFrame(const ExecutionObjectPipeline* eop, VideoWriter& writer,
135 uint32_t frame_idx, uint32_t num_eops,
136 uint32_t num_eves, uint32_t num_dsps);
137 static void ProcessArgs(int argc, char *argv[],
138 std::string& config_file,
139 uint32_t & num_dsps, uint32_t &num_eves,
140 int & num_layers_groups);
141 void ReportTime(const ExecutionObjectPipeline* eop);
143 static void DisplayHelp();
144 extern std::string labels_classes[];
145 extern int IMAGE_CLASSES_NUM;
146 extern int selected_items_size;
147 extern int selected_items[];
148 extern int populate_selected_items (char *filename);
149 extern void populate_labels (char *filename);
152 int main(int argc, char *argv[])
153 {
154 // Catch ctrl-c to ensure a clean exit
155 signal(SIGABRT, exit);
156 signal(SIGTERM, exit);
158 // If there are no devices capable of offloading TIDL on the SoC, exit
159 uint32_t num_eves = Executor::GetNumDevices(DeviceType::EVE);
160 uint32_t num_dsps = Executor::GetNumDevices(DeviceType::DSP);
161 int num_layers_groups = 1;
163 if (num_eves == 0 && num_dsps == 0)
164 {
165 std::cout << "TI DL not supported on this SoC." << std::endl;
166 return EXIT_SUCCESS;
167 }
169 // Process arguments
170 std::string config_file;
171 ProcessArgs(argc, argv, config_file, num_dsps, num_eves, num_layers_groups);
173 bool status = false;
174 if (!config_file.empty()) {
175 std::cout << "Run single configuration: " << config_file << std::endl;
176 status = RunConfiguration(config_file, num_layers_groups, num_dsps, num_eves);
177 }
179 if (!status)
180 {
181 std::cout << "tidl FAILED" << std::endl;
182 return EXIT_FAILURE;
183 }
185 std::cout << "tidl PASSED" << std::endl;
186 return EXIT_SUCCESS;
187 }
189 bool RunConfiguration(const std::string& config_file, int num_layers_groups, uint32_t num_dsps, uint32_t num_eves)
190 {
192 // Read the TI DL configuration file
193 Configuration configuration;
194 bool status = configuration.ReadFromFile(config_file);
195 if (!status)
196 {
197 std::cerr << "Error in configuration file: " << config_file
198 << std::endl;
199 return false;
200 }
202 std::ifstream input_data_file(configuration.inData, std::ios::binary);
203 std::ofstream output_data_file(configuration.outData, std::ios::binary);
204 assert (input_data_file.good());
205 assert (output_data_file.good());
208 try
209 {
210 // Create ExecutionObjectPipelines
211 Executor *e_eve = NULL;
212 Executor *e_dsp = NULL;
213 std::vector<ExecutionObjectPipeline *> eops;
214 if (! CreateExecutionObjectPipelines(num_eves, num_dsps, configuration,
215 num_layers_groups, e_eve, e_dsp, eops))
216 return false;
217 uint32_t num_eops = eops.size();
219 // Allocate input/output memory for each EOP
220 AllocateMemory(eops);
222 // Setup Live Display
223 SetupLiveDisplay(num_eves, num_dsps);
225 // Setup Input
226 VideoCapture cap;
227 VideoWriter writer; // gstreamer
228 if (! SetupInput(cap, writer)) return false;
231 // More initialization
232 for (int k = 0; k < NUM_ROI; k++)
233 for(int i = 0; i < 3; i ++)
234 selclass_history[k][i] = -1;
235 int num_frames = configuration.numFrames;
236 std::cout << "About to start ProcessFrame loop!!" << std::endl;
238 // Process frames with available EOPs in a pipelined manner
239 // additional num_eops iterations to flush the pipeline (epilogue)
240 for (uint32_t frame_idx = 0;
241 frame_idx < configuration.numFrames + num_eops; frame_idx++)
242 {
243 ExecutionObjectPipeline* eop = eops[frame_idx % num_eops];
245 // Wait for previous frame on the same eo to finish processing
246 if (eop->ProcessFrameWait())
247 {
248 #ifdef PERF_VERBOSE
249 ReportTime(eop);
250 #endif
251 DisplayFrame(eop, writer, frame_idx, num_eops,
252 num_eves, num_dsps);
253 }
254 fps_window.Tick();
256 if (ReadFrame(eop, frame_idx, num_frames, cap, writer))
257 eop->ProcessFrameStartAsync();
258 }
260 // Cleanup
261 for (auto eop : eops)
262 {
263 free(eop->GetInputBufferPtr());
264 free(eop->GetOutputBufferPtr());
265 delete eop;
266 }
267 if(num_dsps) delete e_dsp;
268 if(num_eves) delete e_eve;
269 }
270 catch (tidl::Exception &e)
271 {
272 std::cerr << e.what() << std::endl;
273 status = false;
274 }
277 input_data_file.close();
278 output_data_file.close();
280 return status;
281 }
284 bool CreateExecutionObjectPipelines(uint32_t num_eves, uint32_t num_dsps,
285 Configuration& configuration,
286 uint32_t num_layers_groups,
287 Executor*& e_eve, Executor*& e_dsp,
288 std::vector<ExecutionObjectPipeline*>& eops)
289 {
290 DeviceIds ids_eve, ids_dsp;
291 for (uint32_t i = 0; i < num_eves; i++)
292 ids_eve.insert(static_cast<DeviceId>(i));
293 for (uint32_t i = 0; i < num_dsps; i++)
294 ids_dsp.insert(static_cast<DeviceId>(i));
295 const uint32_t buffer_factor = 2;
297 switch(num_layers_groups)
298 {
299 case 1: // Single layers group
300 e_eve = num_eves == 0 ? nullptr :
301 new Executor(DeviceType::EVE, ids_eve, configuration);
302 e_dsp = num_dsps == 0 ? nullptr :
303 new Executor(DeviceType::DSP, ids_dsp, configuration);
305 // Construct ExecutionObjectPipeline with single Execution Object to
306 // process each frame. This is parallel processing of frames with
307 // as many DSP and EVE cores that we have on hand.
308 // If buffer_factor == 2, duplicating EOPs for double buffering
309 // and overlapping host pre/post-processing with device processing
310 for (uint32_t j = 0; j < buffer_factor; j++)
311 {
312 for (uint32_t i = 0; i < num_eves; i++)
313 eops.push_back(new ExecutionObjectPipeline({(*e_eve)[i]}));
314 for (uint32_t i = 0; i < num_dsps; i++)
315 eops.push_back(new ExecutionObjectPipeline({(*e_dsp)[i]}));
316 }
317 break;
319 case 2: // Two layers group
320 // JacintoNet11 specific : specify only layers that will be in
321 // layers group 2 ... by default all other layers are in group 1.
322 configuration.layerIndex2LayerGroupId = { {12, 2}, {13, 2}, {14, 2} };
324 // Create Executors with the approriate core type, number of cores
325 // and configuration specified
326 // EVE will run layersGroupId 1 in the network, while
327 // DSP will run layersGroupId 2 in the network
328 e_eve = num_eves == 0 ? nullptr :
329 new Executor(DeviceType::EVE, ids_eve, configuration, 1);
330 e_dsp = num_dsps == 0 ? nullptr :
331 new Executor(DeviceType::DSP, ids_dsp, configuration, 2);
333 // Construct ExecutionObjectPipeline that utilizes multiple
334 // ExecutionObjects to process a single frame, each ExecutionObject
335 // processes one layerGroup of the network
336 // If buffer_factor == 2, duplicating EOPs for pipelining at
337 // EO level rather than at EOP level, in addition to double buffering
338 // and overlapping host pre/post-processing with device processing
339 for (uint32_t j = 0; j < buffer_factor; j++)
340 {
341 for (uint32_t i = 0; i < std::max(num_eves, num_dsps); i++)
342 eops.push_back(new ExecutionObjectPipeline(
343 {(*e_eve)[i%num_eves], (*e_dsp)[i%num_dsps]}));
344 }
345 break;
347 default:
348 std::cout << "Layers groups can be either 1 or 2!" << std::endl;
349 return false;
350 break;
351 }
353 return true;
354 }
356 void AllocateMemory(const std::vector<ExecutionObjectPipeline*>& eops)
357 {
358 for (auto eop : eops)
359 {
360 size_t in_size = eop->GetInputBufferSizeInBytes();
361 size_t out_size = eop->GetOutputBufferSizeInBytes();
362 void* in_ptr = malloc(in_size);
363 void* out_ptr = malloc(out_size);
364 assert(in_ptr != nullptr && out_ptr != nullptr);
366 ArgInfo in(in_ptr, in_size);
367 ArgInfo out(out_ptr, out_size);
368 eop->SetInputOutputBuffer(in, out);
369 }
370 }
372 void SetupLiveDisplay(uint32_t num_eves, uint32_t num_dsps)
373 {
374 #ifdef LIVE_DISPLAY
375 sprintf(imagenet_win, "Imagenet_EVEx%d_DSPx%d", num_eves, num_dsps);
377 if(NUM_ROI > 1)
378 {
379 for(int i = 0; i < NUM_ROI; i ++) {
380 char tmp_string[80];
381 sprintf(tmp_string, "ROI[%02d]", i);
382 namedWindow(tmp_string, WINDOW_AUTOSIZE | CV_GUI_NORMAL);
383 }
384 }
385 Mat sw_stack_image = imread(
386 "/usr/share/ti/tidl/examples/classification/tidl-sw-stack-small.png",
387 IMREAD_COLOR); // Read the file
388 if( sw_stack_image.empty() ) // Check for invalid input
389 {
390 std::cout << "Could not open or find the tidl-sw-stack-small image"
391 << std::endl ;
392 } else {
393 // Create a window for display.
394 namedWindow( "TIDL SW Stack", WINDOW_AUTOSIZE | CV_GUI_NORMAL );
395 // Show our image inside it.
396 cv::imshow( "TIDL SW Stack", sw_stack_image );
397 }
399 namedWindow("ClassList", WINDOW_AUTOSIZE | CV_GUI_NORMAL);
400 namedWindow(imagenet_win, WINDOW_AUTOSIZE | CV_GUI_NORMAL);
401 //set the callback function for any mouse event
402 setMouseCallback(imagenet_win, imagenetCallBackFunc, NULL);
404 classlist_image = cv::Mat::zeros(40 + selected_items_size * 20, 220,
405 CV_8UC3);
406 //Erase window
407 classlist_image.setTo(Scalar::all(0));
409 for (int i = 0; i < selected_items_size; i ++)
410 {
411 sprintf(tmp_classwindow_string, "%2d) %12s", 1+i,
412 labels_classes[selected_items[i]].c_str());
413 cv::putText(classlist_image, tmp_classwindow_string,
414 cv::Point(5, 40 + i * 20),
415 cv::FONT_HERSHEY_COMPLEX_SMALL,
416 0.75,
417 cv::Scalar(255,255,255), 1, 8);
418 }
419 cv::imshow("ClassList", classlist_image);
420 #endif
421 }
423 bool SetupInput(VideoCapture& cap, VideoWriter& writer)
424 {
425 if(live_input >= 0)
426 {
427 cap.open(live_input);
429 const double fps = cap.get(CAP_PROP_FPS);
430 const int width = cap.get(CAP_PROP_FRAME_WIDTH);
431 const int height = cap.get(CAP_PROP_FRAME_HEIGHT);
432 std::cout << "Capture camera with " << fps << " fps, " << width << "x"
433 << height << " px" << std::endl;
435 #ifdef RMT_GST_STREAMER
436 writer.open(" appsrc ! videoconvert ! video/x-raw, format=(string)NV12, width=(int)640, height=(int)480, framerate=(fraction)30/1 ! \
437 ducatih264enc bitrate=2000 ! queue ! h264parse config-interval=1 ! \
438 mpegtsmux ! udpsink host=192.168.1.2 sync=false port=5000",
439 0,fps,Size(640,480),true);
441 if (!writer.isOpened()) {
442 cap.release();
443 std::cerr << "Can't create gstreamer writer. "
444 << "Do you have the correct version installed?" << std::endl;
445 std::cerr << "Print out OpenCV build information" << std::endl;
446 std::cout << getBuildInformation() << std::endl;
447 return false;
448 }
449 #endif
450 } else {
451 std::cout << "Video input clip: " << video_clip << std::endl;
452 cap.open(std::string(video_clip));
453 const double fps = cap.get(CAP_PROP_FPS);
454 const int width = cap.get(CAP_PROP_FRAME_WIDTH);
455 const int height = cap.get(CAP_PROP_FRAME_HEIGHT);
456 std::cout << "Clip with " << fps << " fps, " << width << "x"
457 << height << " px" << std::endl;
458 }
460 if (!cap.isOpened()) {
461 std::cout << "Video input not opened!" << std::endl;
462 return false;
463 }
465 for (int y = 0; y < NUM_ROI_Y; y ++) {
466 for (int x = 0; x < NUM_ROI_X; x ++) {
467 rectCrop[y * NUM_ROI_X + x] = Rect(X_OFFSET + x * X_STEP,
468 Y_OFFSET + y * Y_STEP, X_STEP, Y_STEP);
469 std::cout << "Rect[" << X_OFFSET + x * X_STEP << ", "
470 << Y_OFFSET + y * Y_STEP << "]" << std::endl;
471 }
472 }
474 return true;
475 }
477 bool ReadFrame(ExecutionObjectPipeline* eop,
478 uint32_t frame_idx, uint32_t num_frames,
479 VideoCapture &cap, VideoWriter& writer)
480 {
481 if (cap.grab() && frame_idx < num_frames)
482 {
483 if (cap.retrieve(in_image))
484 {
485 if(live_input >= 0)
486 { //Crop central square portion
487 int loc_xmin = (in_image.size().width - in_image.size().height) / 2; //Central position
488 int loc_ymin = 0;
489 int loc_w = in_image.size().height;
490 int loc_h = in_image.size().height;
492 cv::resize(in_image(Rect(loc_xmin, loc_ymin, loc_w, loc_h)), image, Size(RES_X, RES_Y));
493 } else {
494 if((in_image.size().width != RES_X) || (in_image.size().height != RES_Y))
495 {
496 cv::resize(in_image, image, Size(RES_X,RES_Y));
497 }
498 }
500 r_image = Mat(image, rectCrop[frame_idx % NUM_ROI]);
502 #ifdef LIVE_DISPLAY
503 if(NUM_ROI > 1)
504 {
505 char tmp_string[80];
506 sprintf(tmp_string, "ROI[%02d]", frame_idx % NUM_ROI);
507 cv::imshow(tmp_string, r_image);
508 }
509 #endif
510 //Convert from BGR pixel interleaved to BGR plane interleaved!
511 cv::resize(r_image, cnn_image, Size(224,224));
512 cv::split(cnn_image, bgr_frames);
513 tf_preprocess((uchar*) eop->GetInputBufferPtr(),
514 bgr_frames[0].ptr(), 224*224);
515 tf_preprocess((uchar*) eop->GetInputBufferPtr()+224*224,
516 bgr_frames[1].ptr(), 224*224);
517 tf_preprocess((uchar*) eop->GetInputBufferPtr()+2*224*224,
518 bgr_frames[2].ptr(), 224*224);
519 eop->SetFrameIndex(frame_idx);
521 #ifdef RMT_GST_STREAMER
522 cv::resize(Mat(image, Rect(0,32,640,448)), to_stream,
523 Size(640,480));
524 writer << to_stream;
525 #endif
527 #ifdef LIVE_DISPLAY
528 //waitKey(2);
529 image.copyTo(show_image);
530 #endif
531 return true;
532 }
533 } else {
534 if(live_input == -1) {
535 //Rewind!
536 cap.release();
537 cap.open(std::string(video_clip));
538 }
539 }
541 return false;
542 }
544 void ReportTime(const ExecutionObjectPipeline* eop)
545 {
546 uint32_t frame_index = eop->GetFrameIndex();
547 std::string device_name = eop->GetDeviceName();
548 float elapsed_host = eop->GetHostProcessTimeInMilliSeconds();
549 float elapsed_device = eop->GetProcessTimeInMilliSeconds();
550 double overhead = 100 - (elapsed_device/elapsed_host*100);
551 std::cout << "frame[" << frame_index << "]: "
552 << "Time on " << device_name << ": "
553 << std::setw(6) << std::setprecision(4)
554 << elapsed_device << "ms, "
555 << "host: "
556 << std::setw(6) << std::setprecision(4)
557 << elapsed_host << "ms ";
558 std::cout << "API overhead: "
559 << std::setw(6) << std::setprecision(3)
560 << overhead << " %" << std::endl;
561 }
563 void DisplayFrame(const ExecutionObjectPipeline* eop, VideoWriter& writer,
564 uint32_t frame_idx, uint32_t num_eops,
565 uint32_t num_eves, uint32_t num_dsps)
566 {
567 int f_id = eop->GetFrameIndex();
568 int curr_roi = f_id % NUM_ROI;
569 int is_object = tf_postprocess((uchar*) eop->GetOutputBufferPtr(),
570 IMAGE_CLASSES_NUM, curr_roi, frame_idx, f_id);
571 selclass_history[curr_roi][2] = selclass_history[curr_roi][1];
572 selclass_history[curr_roi][1] = selclass_history[curr_roi][0];
573 selclass_history[curr_roi][0] = is_object;
574 for (int r = 0; r < NUM_ROI; r ++)
575 {
576 int rpt_id = ShowRegion(selclass_history[r]);
577 if(rpt_id >= 0)
578 {
579 // overlay the display window, if ball seen during last two times
580 cv::putText(show_image, labels_classes[rpt_id].c_str(),
581 cv::Point(rectCrop[r].x + 5,rectCrop[r].y + 20), // Coordinates
582 cv::FONT_HERSHEY_COMPLEX_SMALL, // Font
583 1.0, // Scale. 2.0 = 2x bigger
584 cv::Scalar(0,0,255), // Color
585 1, // Thickness
586 8); // Line type
587 cv::rectangle(show_image, rectCrop[r], Scalar(255,0,0), 3);
588 std::cout << "ROI(" << r << ")(" << rpt_id << ")="
589 << labels_classes[rpt_id].c_str() << std::endl;
591 classlist_image.setTo(Scalar::all(0));
592 for (int k = 0; k < selected_items_size; k ++)
593 {
594 sprintf(tmp_classwindow_string, "%2d) %12s", 1+k,
595 labels_classes[selected_items[k]].c_str());
596 cv::putText(classlist_image, tmp_classwindow_string,
597 cv::Point(5, 40 + k * 20),
598 cv::FONT_HERSHEY_COMPLEX_SMALL,
599 0.75,
600 selected_items[k] == rpt_id ? cv::Scalar(0,0,255) :
601 cv::Scalar(255,255,255), 1, 8);
602 }
604 double avg_fps = fps_window.UpdateAvgFPS();
605 sprintf(tmp_classwindow_string, "FPS:%5.2lf", avg_fps );
607 #ifdef PERF_VERBOSE
608 std::cout << "Device:" << eop->GetDeviceName() << " eops("
609 << num_eops << "), EVES(" << num_eves << ") DSPS("
610 << num_dsps << ") FPS:" << avg_fps << std::endl;
611 #endif
612 cv::putText(classlist_image, tmp_classwindow_string,
613 cv::Point(5, 20),
614 cv::FONT_HERSHEY_COMPLEX_SMALL,
615 0.75,
616 cv::Scalar(0,255,0), 1, 8);
617 cv::imshow("ClassList", classlist_image);
618 }
619 }
621 #ifdef LIVE_DISPLAY
622 cv::imshow(imagenet_win, show_image);
623 #endif
625 #ifdef RMT_GST_STREAMER
626 cv::resize(show_image, to_stream, cv::Size(640,480));
627 writer << to_stream;
628 #endif
630 #ifdef LIVE_DISPLAY
631 waitKey(2);
632 #endif
633 }
635 // Function to process all command line arguments
636 void ProcessArgs(int argc, char *argv[], std::string& config_file,
637 uint32_t & num_dsps, uint32_t & num_eves, int & num_layers_groups )
638 {
639 const struct option long_options[] =
640 {
641 {"labels_classes_file", required_argument, 0, 'l'},
642 {"selected_classes_file", required_argument, 0, 's'},
643 {"config_file", required_argument, 0, 'c'},
644 {"num_dsps", required_argument, 0, 'd'},
645 {"num_eves", required_argument, 0, 'e'},
646 {"num_layers_groups", required_argument, 0, 'g'},
647 {"help", no_argument, 0, 'h'},
648 {"verbose", no_argument, 0, 'v'},
649 {0, 0, 0, 0}
650 };
652 int option_index = 0;
654 while (true)
655 {
656 int c = getopt_long(argc, argv, "l:c:s:i:d:e:g:hv", long_options, &option_index);
658 if (c == -1)
659 break;
661 switch (c)
662 {
663 case 'l': populate_labels(optarg);
664 break;
666 case 's': populate_selected_items(optarg);
667 break;
669 case 'i': if(strlen(optarg) == 1)
670 {
671 live_input = atoi(optarg);
672 } else {
673 live_input = -1;
674 strcpy(video_clip, optarg);
675 }
676 break;
678 case 'c': config_file = optarg;
679 break;
681 case 'g': num_layers_groups = atoi(optarg);
682 assert(num_layers_groups >= 1 && num_layers_groups <= 2);
683 break;
685 case 'd': num_dsps = atoi(optarg);
686 assert (num_dsps >= 0 && num_dsps <= 2);
687 break;
689 case 'e': num_eves = atoi(optarg);
690 assert (num_eves >= 0 && num_eves <= 2);
691 break;
693 case 'v': __TI_show_debug_ = true;
694 break;
696 case 'h': DisplayHelp();
697 exit(EXIT_SUCCESS);
698 break;
700 case '?': // Error in getopt_long
701 exit(EXIT_FAILURE);
702 break;
704 default:
705 std::cerr << "Unsupported option: " << c << std::endl;
706 break;
707 }
708 }
710 // if no eves available, we can only run full net as one layer group
711 if (num_eves == 0) num_layers_groups = 1;
712 }
714 void DisplayHelp()
715 {
716 std::cout << "Usage: tidl_classification\n"
717 " Will run all available networks if tidl is invoked without"
718 " any arguments.\n Use -c to run a single network.\n"
719 "Optional arguments:\n"
720 " -c Path to the configuration file\n"
721 " -d <number of DSP cores> Number of DSP cores to use (0 - 2)\n"
722 " -e <number of EVE cores> Number of EVE cores to use (0 - 2)\n"
723 " -g <1|2> Number of layer groups\n"
724 " -l List of label strings (of all classes in model)\n"
725 " -s List of strings with selected classes\n"
726 " -i Video input (for camera:0,1 or video clip)\n"
727 " -v Verbose output during execution\n"
728 " -h Help\n";
729 }
731 // Function to filter all the reported decisions
732 bool tf_expected_id(int id)
733 {
734 // Filter out unexpected IDs
735 for (int i = 0; i < selected_items_size; i ++)
736 {
737 if(id == selected_items[i]) return true;
738 }
739 return false;
740 }
742 int tf_postprocess(uchar *in, int size, int roi_idx, int frame_idx, int f_id)
743 {
744 //prob_i = exp(TIDL_Lib_output_i) / sum(exp(TIDL_Lib_output))
745 // sort and get k largest values and corresponding indices
746 const int k = TOP_CANDIDATES;
747 int rpt_id = -1;
749 typedef std::pair<uchar, int> val_index;
750 auto constexpr cmp = [](val_index &left, val_index &right) { return left.first > right.first; };
751 std::priority_queue<val_index, std::vector<val_index>, decltype(cmp)> queue(cmp);
752 // initialize priority queue with smallest value on top
753 for (int i = 0; i < k; i++) {
754 queue.push(val_index(in[i], i));
755 }
756 // for rest input, if larger than current minimum, pop mininum, push new val
757 for (int i = k; i < size; i++)
758 {
759 if (in[i] > queue.top().first)
760 {
761 queue.pop();
762 queue.push(val_index(in[i], i));
763 }
764 }
766 // output top k values in reverse order: largest val first
767 std::vector<val_index> sorted;
768 while (! queue.empty())
769 {
770 sorted.push_back(queue.top());
771 queue.pop();
772 }
774 for (int i = k-1; i >= 0; i--)
775 {
776 int id = sorted[i].second;
778 if (tf_expected_id(id))
779 {
780 std::cout << "Frame:" << frame_idx << "," << f_id << " ROI[" << roi_idx << "]: rank="
781 << k-i << ", outval=" << (float)sorted[i].first / 255 << ", "
782 << labels_classes[sorted[i].second] << std::endl;
783 rpt_id = id;
784 }
785 }
786 return rpt_id;
787 }
789 void tf_preprocess(uchar *out, uchar *in, int size)
790 {
791 for (int i = 0; i < size; i++)
792 {
793 out[i] = (uchar) (in[i] /*- 128*/);
794 }
795 }
797 int ShowRegion(int roi_history[])
798 {
799 if((roi_history[0] >= 0) && (roi_history[0] == roi_history[1])) return roi_history[0];
800 if((roi_history[0] >= 0) && (roi_history[0] == roi_history[2])) return roi_history[0];
801 if((roi_history[1] >= 0) && (roi_history[1] == roi_history[2])) return roi_history[1];
802 return -1;
803 }