Optimize examples with EOP double buffering

- Improve overall loop performance for imagenet and segmentation
- Update documentation on performance
- MCT-1039

diff --git a/docs/source/example.rst b/docs/source/example.rst
index 2c37404d7adfa2848bfe333b605b5244fbade759..21721c18d7bbec6699cb5c6325c1a09992e3bd12 100644 (file)
--- a/docs/source/example.rst
+++ b/docs/source/example.rst
@@ -96,6 +96,33 @@ C66x      117.9 ms               118.7 ms             0.93 %
 The :term:`network<Network>` used in the example is jacintonet11v2. It has
 14 layers. Input to the network is RGB image of 224x224. Users can specify whether to run the network on EVE or C66x.
 
+The example code sets ``buffer_factor`` to 2 to create duplicated
+ExecutionObjectPipelines with identical ExecutionObjects to
+perform double buffering, so that host pre/post-processing can be overlapped
+with device processing (see comments in the code for details).
+The following table shows the loop overall time over 10 frames
+with single buffering and double buffering,
+``./imagenet -f 10 -d <num> -e <num>``.
+
+.. list-table:: Loop overall time over 10 frames
+   :header-rows: 1
+
+   * - Device(s)
+     - Single Buffering (buffer_factor=1)
+     - Double Buffering (buffer_factor=2)
+   * - 1 EVE
+     - 1744 ms
+     - 1167 ms
+   * - 2 EVEs
+     - 966 ms
+     - 795 ms
+   * - 1 C66x
+     - 1879 ms
+     - 1281 ms
+   * - 2 C66xs
+     - 1021 ms
+     - 814 ms
+
 .. note::
     The predicitions reported here are based on the output of the softmax
     layer in the network, which are not normalized to the real probabilities.
@@ -131,6 +158,33 @@ EVE         251.8 ms               254.2 ms             0.96 %
 C66x        812.7 ms               815.0 ms             0.27 %
 =======     ====================== ==================== ============
 
+The example code sets ``buffer_factor`` to 2 to create duplicated
+ExecutionObjectPipelines with identical ExecutionObjects to
+perform double buffering, so that host pre/post-processing can be overlapped
+with device processing (see comments in the code for details).
+The following table shows the loop overall time over 10 frames
+with single buffering and double buffering,
+``./segmentation -f 10 -d <num> -e <num>``.
+
+.. list-table:: Loop overall time over 10 frames
+   :header-rows: 1
+
+   * - Device(s)
+     - Single Buffering (buffer_factor=1)
+     - Double Buffering (buffer_factor=2)
+   * - 1 EVE
+     - 5233 ms
+     - 3017 ms
+   * - 2 EVEs
+     - 3032 ms
+     - 3015 ms
+   * - 1 C66x
+     - 10890 ms
+     - 8416 ms
+   * - 2 C66xs
+     - 5742 ms
+     - 4638 ms
+
 .. _ssd-example:
 
 SSD
@@ -168,6 +222,29 @@ Device        Device Processing Time Host Processing Time API Overhead
 EVE+C66x      169.5ms                172.0ms              1.68 %
 ========      ====================== ==================== ============
 
+The example code sets ``pipeline_depth`` to 2 to create duplicated
+ExecutionObjectPipelines with identical ExecutionObjects to
+perform pipelined execution at the ExecutionObject level.
+The side effect is that it also overlaps host pre/post-processing
+with device processing (see comments in the code for details).
+The following table shows the loop overall time over 10 frames
+with pipelining at ExecutionObjectPipeline level
+versus ExecutionObject level.
+``./ssd_multibox -f 10 -d <num> -e <num>``.
+
+.. list-table:: Loop overall time over 10 frames
+   :header-rows: 1
+
+   * - Device(s)
+     - pipeline_depth=1
+     - pipeline_depth=2
+   * - 1 EVE + 1 C66x
+     - 2900 ms
+     - 1735 ms
+   * - 2 EVEs + 2 C66xs
+     - 1630 ms
+     - 1408 ms
+
 Running Examples
 ----------------
 

diff --git a/examples/imagenet/main.cpp b/examples/imagenet/main.cpp
index f11e050f5d197641c37f706f3da9e01de368b1af..9039271de28b4a85f5a552cb94b90da9bc0c48ca 100644 (file)
--- a/examples/imagenet/main.cpp
+++ b/examples/imagenet/main.cpp
@@ -42,6 +42,7 @@
 
 #include "executor.h"
 #include "execution_object.h"
+#include "execution_object_pipeline.h"
 #include "configuration.h"
 #include "imagenet_classes.h"
 #include "imgutil.h"
@@ -67,9 +68,10 @@ const char *default_inputs[NUM_DEFAULT_INPUTS] =
 
 Executor* CreateExecutor(DeviceType dt, uint32_t num, const Configuration& c);
 bool RunConfiguration(cmdline_opts_t& opts);
-bool ReadFrame(ExecutionObject& eo, uint32_t frame_idx, const Configuration& c,
+bool ReadFrame(ExecutionObjectPipeline& eop,
+               uint32_t frame_idx, const Configuration& c,
                const cmdline_opts_t& opts, VideoCapture &cap);
-bool WriteFrameOutput(const ExecutionObject &eo);
+bool WriteFrameOutput(const ExecutionObjectPipeline &eop);
 void DisplayHelp();
 
 
@@ -150,29 +152,56 @@ bool RunConfiguration(cmdline_opts_t& opts)
         for (uint32_t i = 0; i < opts.num_dsps; i++) eos.push_back((*e_dsp)[i]);
         uint32_t num_eos = eos.size();
 
-        // Allocate input and output buffers for each ExecutionObject
-        AllocateMemory(eos);
+        // Use duplicate EOPs to do double buffering on frame input/output
+        //    because each EOP has its own set of input/output buffers,
+        //    so that host ReadFrame() can be overlapped with device processing
+        // Use one EO as an example, with different buffer_factor,
+        //    we have different execution behavior:
+        // If buffer_factor is set to 1 -> single buffering
+        //    we create one EOP: eop0 (eo0)
+        //    pipeline execution of multiple frames over time is as follows:
+        //    --------------------- time ------------------->
+        //    eop0: [RF][eo0.....][WF]
+        //    eop0:                   [RF][eo0.....][WF]
+        //    eop0:                                     [RF][eo0.....][WF]
+        // If buffer_factor is set to 2 -> double buffering
+        //    we create two EOPs: eop0 (eo0), eop1(eo0)
+        //    pipeline execution of multiple frames over time is as follows:
+        //    --------------------- time ------------------->
+        //    eop0: [RF][eo0.....][WF]
+        //    eop1:     [RF]      [eo0.....][WF]
+        //    eop0:                   [RF]  [eo0.....][WF]
+        //    eop1:                             [RF]  [eo0.....][WF]
+        vector<ExecutionObjectPipeline *> eops;
+        uint32_t buffer_factor = 2;  // set to 1 for single buffering
+        for (uint32_t j = 0; j < buffer_factor; j++)
+            for (uint32_t i = 0; i < num_eos; i++)
+                eops.push_back(new ExecutionObjectPipeline({eos[i]}));
+        uint32_t num_eops = eops.size();
+
+        // Allocate input and output buffers for each EOP
+        AllocateMemory(eops);
 
         chrono::time_point<chrono::steady_clock> tloop0, tloop1;
         tloop0 = chrono::steady_clock::now();
 
-        // Process frames with available eos in a pipelined manner
+        // Process frames with available eops in a pipelined manner
         // additional num_eos iterations to flush the pipeline (epilogue)
         for (uint32_t frame_idx = 0;
-             frame_idx < opts.num_frames + num_eos; frame_idx++)
+             frame_idx < opts.num_frames + num_eops; frame_idx++)
         {
-            ExecutionObject* eo = eos[frame_idx % num_eos];
+            ExecutionObjectPipeline* eop = eops[frame_idx % num_eops];
 
-            // Wait for previous frame on the same eo to finish processing
-            if (eo->ProcessFrameWait())
+            // Wait for previous frame on the same eop to finish processing
+            if (eop->ProcessFrameWait())
             {
-                ReportTime(eo);
-                WriteFrameOutput(*eo);
+                ReportTime(eop);
+                WriteFrameOutput(*eop);
             }
 
-            // Read a frame and start processing it with current eo
-            if (ReadFrame(*eo, frame_idx, c, opts, cap))
-                eo->ProcessFrameStartAsync();
+            // Read a frame and start processing it with current eop
+            if (ReadFrame(*eop, frame_idx, c, opts, cap))
+                eop->ProcessFrameStartAsync();
         }
 
         tloop1 = chrono::steady_clock::now();
@@ -181,7 +210,8 @@ bool RunConfiguration(cmdline_opts_t& opts)
                   << setw(6) << setprecision(4)
                   << (elapsed.count() * 1000) << "ms" << endl;
 
-        FreeMemory(eos);
+        FreeMemory(eops);
+        for (auto eop : eops)  delete eop;
         delete e_eve;
         delete e_dsp;
     }
@@ -206,15 +236,16 @@ Executor* CreateExecutor(DeviceType dt, uint32_t num, const Configuration& c)
     return new Executor(dt, ids, c);
 }
 
-bool ReadFrame(ExecutionObject &eo, uint32_t frame_idx, const Configuration& c,
+bool ReadFrame(ExecutionObjectPipeline &eop,
+               uint32_t frame_idx, const Configuration& c,
                const cmdline_opts_t& opts, VideoCapture &cap)
 {
     if (frame_idx >= opts.num_frames)
         return false;
 
-    eo.SetFrameIndex(frame_idx);
+    eop.SetFrameIndex(frame_idx);
 
-    char*  frame_buffer = eo.GetInputBufferPtr();
+    char*  frame_buffer = eop.GetInputBufferPtr();
     assert (frame_buffer != nullptr);
 
     Mat image;
@@ -256,11 +287,11 @@ bool ReadFrame(ExecutionObject &eo, uint32_t frame_idx, const Configuration& c,
 }
 
 // Display top 5 classified imagenet classes with probabilities
-bool WriteFrameOutput(const ExecutionObject &eo)
+bool WriteFrameOutput(const ExecutionObjectPipeline &eop)
 {
     const int k = 5;
-    unsigned char *out = (unsigned char *) eo.GetOutputBufferPtr();
-    int out_size = eo.GetOutputBufferSizeInBytes();
+    unsigned char *out = (unsigned char *) eop.GetOutputBufferPtr();
+    int out_size = eop.GetOutputBufferSizeInBytes();
 
     // sort and get k largest values and corresponding indices
     typedef pair<unsigned char, int> val_index;

diff --git a/examples/segmentation/main.cpp b/examples/segmentation/main.cpp
index b0aef2f6e4558c79d663644f97d1c8d701b39ea6..35294c4d444a302dbf203369317d7adde1313a1b 100644 (file)
--- a/examples/segmentation/main.cpp
+++ b/examples/segmentation/main.cpp
@@ -65,10 +65,11 @@ uint32_t orig_height;
 
 bool RunConfiguration(const cmdline_opts_t& opts);
 Executor* CreateExecutor(DeviceType dt, uint32_t num, const Configuration& c);
-bool ReadFrame(ExecutionObject& eo, uint32_t frame_idx, const Configuration& c,
+bool ReadFrame(ExecutionObjectPipeline& eop,
+               uint32_t frame_idx, const Configuration& c,
                const cmdline_opts_t& opts, VideoCapture &cap);
-bool WriteFrameOutput(const ExecutionObject &eo, const Configuration& c,
-                      const cmdline_opts_t& opts);
+bool WriteFrameOutput(const ExecutionObjectPipeline &eop,
+                      const Configuration& c, const cmdline_opts_t& opts);
 void DisplayHelp();
 
 
@@ -157,29 +158,56 @@ bool RunConfiguration(const cmdline_opts_t& opts)
         for (uint32_t i = 0; i < opts.num_dsps; i++) eos.push_back((*e_dsp)[i]);
         uint32_t num_eos = eos.size();
 
-        // Allocate input and output buffers for each execution object
-        AllocateMemory(eos);
+        // Use duplicate EOPs to do double buffering on frame input/output
+        //    because each EOP has its own set of input/output buffers,
+        //    so that host ReadFrame() can be overlapped with device processing
+        // Use one EO as an example, with different buffer_factor,
+        //    we have different execution behavior:
+        // If buffer_factor is set to 1 -> single buffering
+        //    we create one EOP: eop0 (eo0)
+        //    pipeline execution of multiple frames over time is as follows:
+        //    --------------------- time ------------------->
+        //    eop0: [RF][eo0.....][WF]
+        //    eop0:                   [RF][eo0.....][WF]
+        //    eop0:                                     [RF][eo0.....][WF]
+        // If buffer_factor is set to 2 -> double buffering
+        //    we create two EOPs: eop0 (eo0), eop1(eo0)
+        //    pipeline execution of multiple frames over time is as follows:
+        //    --------------------- time ------------------->
+        //    eop0: [RF][eo0.....][WF]
+        //    eop1:     [RF]      [eo0.....][WF]
+        //    eop0:                   [RF]  [eo0.....][WF]
+        //    eop1:                             [RF]  [eo0.....][WF]
+        vector<ExecutionObjectPipeline *> eops;
+        uint32_t buffer_factor = 2;  // set to 1 for single buffering
+        for (uint32_t j = 0; j < buffer_factor; j++)
+            for (uint32_t i = 0; i < num_eos; i++)
+                eops.push_back(new ExecutionObjectPipeline({eos[i]}));
+        uint32_t num_eops = eops.size();
+
+        // Allocate input and output buffers for each EOP
+        AllocateMemory(eops);
 
         chrono::time_point<chrono::steady_clock> tloop0, tloop1;
         tloop0 = chrono::steady_clock::now();
 
-        // Process frames with available eos in a pipelined manner
+        // Process frames with available eops in a pipelined manner
         // additional num_eos iterations to flush the pipeline (epilogue)
         for (uint32_t frame_idx = 0;
-             frame_idx < opts.num_frames + num_eos; frame_idx++)
+             frame_idx < opts.num_frames + num_eops; frame_idx++)
         {
-            ExecutionObject* eo = eos[frame_idx % num_eos];
+            ExecutionObjectPipeline* eop = eops[frame_idx % num_eops];
 
-            // Wait for previous frame on the same eo to finish processing
-            if (eo->ProcessFrameWait())
+            // Wait for previous frame on the same eop to finish processing
+            if (eop->ProcessFrameWait())
             {
-                ReportTime(eo);
-                WriteFrameOutput(*eo, c, opts);
+                ReportTime(eop);
+                WriteFrameOutput(*eop, c, opts);
             }
 
-            // Read a frame and start processing it with current eo
-            if (ReadFrame(*eo, frame_idx, c, opts, cap))
-                eo->ProcessFrameStartAsync();
+            // Read a frame and start processing it with current eop
+            if (ReadFrame(*eop, frame_idx, c, opts, cap))
+                eop->ProcessFrameStartAsync();
         }
 
         tloop1 = chrono::steady_clock::now();
@@ -188,7 +216,8 @@ bool RunConfiguration(const cmdline_opts_t& opts)
                   << setw(6) << setprecision(4)
                   << (elapsed.count() * 1000) << "ms" << endl;
 
-        FreeMemory(eos);
+        FreeMemory(eops);
+        for (auto eop : eops)  delete eop;
         delete e_eve;
         delete e_dsp;
     }
@@ -213,14 +242,15 @@ Executor* CreateExecutor(DeviceType dt, uint32_t num, const Configuration& c)
     return new Executor(dt, ids, c);
 }
 
-bool ReadFrame(ExecutionObject &eo, uint32_t frame_idx, const Configuration& c,
+bool ReadFrame(ExecutionObjectPipeline &eop,
+               uint32_t frame_idx, const Configuration& c,
                const cmdline_opts_t& opts, VideoCapture &cap)
 {
     if (frame_idx >= opts.num_frames)
         return false;
-    eo.SetFrameIndex(frame_idx);
+    eop.SetFrameIndex(frame_idx);
 
-    char*  frame_buffer = eo.GetInputBufferPtr();
+    char*  frame_buffer = eop.GetInputBufferPtr();
     assert (frame_buffer != nullptr);
     int channel_size = c.inWidth * c.inHeight;
 
@@ -288,10 +318,11 @@ void CreateMask(uchar *classes, uchar *mb, uchar *mg, uchar* mr,
 }
 
 // Create frame overlayed with pixel-level segmentation
-bool WriteFrameOutput(const ExecutionObject &eo, const Configuration& c,
+bool WriteFrameOutput(const ExecutionObjectPipeline &eop,
+                      const Configuration& c,
                       const cmdline_opts_t& opts)
 {
-    unsigned char *out = (unsigned char *) eo.GetOutputBufferPtr();
+    unsigned char *out = (unsigned char *) eop.GetOutputBufferPtr();
     int width          = c.inWidth;
     int height         = c.inHeight;
     int channel_size   = width * height;
@@ -305,7 +336,7 @@ bool WriteFrameOutput(const ExecutionObject &eo, const Configuration& c,
     cv::merge(bgr, 3, mask);
 
     // Asseembly original frame
-    unsigned char *in = (unsigned char *) eo.GetInputBufferPtr();
+    unsigned char *in = (unsigned char *) eop.GetInputBufferPtr();
     bgr[0] = Mat(height, width, CV_8UC(1), in);
     bgr[1] = Mat(height, width, CV_8UC(1), in + channel_size);
     bgr[2] = Mat(height, width, CV_8UC(1), in + channel_size*2);
@@ -327,7 +358,7 @@ bool WriteFrameOutput(const ExecutionObject &eo, const Configuration& c,
     }
     else
     {
-        int frame_index = eo.GetFrameIndex();
+        int frame_index = eop.GetFrameIndex();
         char outfile_name[64];
         if (opts.input_file.empty())
         {