tidl_api/inc/subgraph_runtime.h

   1 /******************************************************************************
   2  * Copyright (c) 2019 Texas Instruments Incorporated - http://www.ti.com/
   3  *  All rights reserved.
   4  *
   5  *  Redistribution and use in source and binary forms, with or without
   6  *  modification, are permitted provided that the following conditions are met:
   7  *      * Redistributions of source code must retain the above copyright
   8  *        notice, this list of conditions and the following disclaimer.
   9  *      * Redistributions in binary form must reproduce the above copyright
  10  *        notice, this list of conditions and the following disclaimer in the
  11  *        documentation and/or other materials provided with the distribution.
  12  *      * Neither the name of Texas Instruments Incorporated nor the
  13  *        names of its contributors may be used to endorse or promote products
  14  *        derived from this software without specific prior written permission.
  15  *
  16  *  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  17  *  AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  18  *  IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  19  *  ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  20  *  LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  21  *  CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  22  *  SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  23  *  INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  24  *  CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  25  *  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
  26  *  THE POSSIBILITY OF SUCH DAMAGE.
  27  *****************************************************************************/
  28
  29 //! @file subgraph_runtime.h
  30
  31 #pragma once
  32
  33 extern "C" {
  34
  35 //! @brief Top level API to get preferred batch_size for a subgraph
  36 //!        Best performance comes with preferred batch_size processing
  37 //!        plus multi-threaded (num_threads = 2) processing
  38 //! @param total_subgraphs  total number of TIDL subgraphs in whole inference
  39 //! @return preferred batch size
  40 extern int TidlGetPreferredBatchSize(int total_subgraphs);
  41
  42 //! @brief Top level API to initialize a TIDL subgraph on device
  43 //!        If not invoked ahead of time, TidlRunSubgraph() will call this
  44 //!        function before any inference
  45 //! @param total_subgraphs  total number of TIDL subgraphs in whole inference
  46 //! @param subgraph_id  index of current TIDL subgraph
  47 extern void TidlInitSubgraph(int total_subgraphs,
  48                              int subgraph_id
  49                             );
  50
  51 //! @brief Top level inference to run a TIDL subgraph
  52 //! @param total_subgraphs  total number of TIDL subgraphs in whole inference
  53 //! @param subgraph_id  index of current TIDL subgraph
  54 //! @param batch_size  number of samples/inferences in this batch
  55 //! @param num_inputs_per_inference  number of inputs to TIDL subgraph
  56 //!        for every sample/inference
  57 //! @param num_outputs_per_inference  number of outputs from TIDL subgraph
  58 //!        for every sample/inference
  59 //! @param input_tensors  input data to TIDL subgraph, layout as
  60 //!        batch1_input1, batch1_input2, ..., batch1_inputM,
  61 //!        ... ... ...
  62 //!        batchN_input1, batchN_input2, ..., batchN_inputM
  63 //! @param output_tensors  output data from TIDL subgraph, layout as
  64 //!        batch1_output1, batch1_output2, ..., batch1_outputK,
  65 //!        ... ... ...
  66 //!        batchN_output1, batchN_output2, ..., batchN_outputK
  67 extern void TidlRunSubgraph(int total_subgraphs,
  68                             int subgraph_id,
  69                             int batch_size,
  70                             int num_inputs_per_inference,
  71                             int num_outputs_per_inference,
  72                             float **input_tensors,
  73                             float **output_tensors
  74                            );
  75
  76 }  // extern "C"
  77
  78
  79 #if 0
  80 // Auto-generated code from Relay/TVM compilation step after
  81 // partitioning and lowering to backend implementation
  82
  83 void TVM_TidlFunction(int total_subgraphs, int subgraph_id,
  84                      int num_input_tensors, int num_output_tensors,
  85                      PackedArgs args)
  86 {
  87   float** in_data  = new float*[num_inputs_per_inference * batch_size];
  88   float** out_data = new float*[num_outputs_per_inference * batch_size];
  89
  90   for (in j = 0; j < batch_size; j++)
  91   {
  92     for (int i = 0; i < num_inputs_per_inference + num_outputs_per_inference;
  93          i++)
  94       if (i < num_inputs_per_inference)
  95         in_data[j * num_inputs_per_inference + i] = args.data[i][j];
  96       else
  97         out_data[j * num_outpus_per_inference + i - num_inputs_per_inference]
  98                                                   = args.data[i][j];
  99   }
 100
 101   // call into this function in libtidl.so
 102   // dlopen("libtidl_api.so")
 103   // TidlFunc = dlsym("TidlRunSubgraph");
 104   (*TidlFunc)(total_subgraphs, subgraph_id, batch_size
 105               num_inputs_per_inference, num_outputs_per_inference,
 106               in_data, out_data);
 107
 108   delete [] in_data;
 109   delete [] out_data;
 110 }
 111 #endif
 112