65db5b5dc2e1c14dc77a039080b172d0bb66a1d8
[tidl/tidl-api.git] / tidl_api / inc / subgraph_runtime.h
1 /******************************************************************************
2  * Copyright (c) 2019 Texas Instruments Incorporated - http://www.ti.com/
3  *  All rights reserved.
4  *
5  *  Redistribution and use in source and binary forms, with or without
6  *  modification, are permitted provided that the following conditions are met:
7  *      * Redistributions of source code must retain the above copyright
8  *        notice, this list of conditions and the following disclaimer.
9  *      * Redistributions in binary form must reproduce the above copyright
10  *        notice, this list of conditions and the following disclaimer in the
11  *        documentation and/or other materials provided with the distribution.
12  *      * Neither the name of Texas Instruments Incorporated nor the
13  *        names of its contributors may be used to endorse or promote products
14  *        derived from this software without specific prior written permission.
15  *
16  *  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
17  *  AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18  *  IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19  *  ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
20  *  LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
21  *  CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
22  *  SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
23  *  INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
24  *  CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
25  *  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
26  *  THE POSSIBILITY OF SUCH DAMAGE.
27  *****************************************************************************/
29 //! @file subgraph_runtime.h
31 #pragma once
33 extern "C" {
35 //! @brief Top level API to get preferred batch_size for a subgraph
36 //!        Best performance comes with preferred batch_size processing
37 //!        plus multi-threaded (num_threads = 2) processing
38 //! @param total_subgraphs  total number of TIDL subgraphs in whole inference
39 //! @return preferred batch size
40 extern int TidlGetPreferredBatchSize(int total_subgraphs);
42 //! @brief Top level API to initialize a TIDL subgraph on device
43 //!        If not invoked ahead of time, TidlRunSubgraph() will call this
44 //!        function before any inference
45 //! @param total_subgraphs  total number of TIDL subgraphs in whole inference
46 //! @param subgraph_id  index of current TIDL subgraph
47 extern void TidlInitSubgraph(int total_subgraphs,
48                              int subgraph_id
49                             );
51 //! @brief Top level inference to run a TIDL subgraph
52 //! @param total_subgraphs  total number of TIDL subgraphs in whole inference
53 //! @param subgraph_id  index of current TIDL subgraph
54 //! @param batch_size  number of samples/inferences in this batch
55 //! @param num_inputs_per_inference  number of inputs to TIDL subgraph
56 //!        for every sample/inference
57 //! @param num_outputs_per_inference  number of outputs from TIDL subgraph
58 //!        for every sample/inference
59 //! @param input_tensors  input data to TIDL subgraph, layout as
60 //!        batch1_input1, batch1_input2, ..., batch1_inputM,
61 //!        ... ... ...
62 //!        batchN_input1, batchN_input2, ..., batchN_inputM
63 //! @param output_tensors  output data from TIDL subgraph, layout as
64 //!        batch1_output1, batch1_output2, ..., batch1_outputK,
65 //!        ... ... ...
66 //!        batchN_output1, batchN_output2, ..., batchN_outputK
67 extern void TidlRunSubgraph(int total_subgraphs,
68                             int subgraph_id,
69                             int batch_size,
70                             int num_inputs_per_inference,
71                             int num_outputs_per_inference,
72                             float **input_tensors,
73                             float **output_tensors
74                            );
76 }  // extern "C"
79 #if 0
80 // Auto-generated code from Relay/TVM compilation step after
81 // partitioning and lowering to backend implementation
83 void TVM_TidlFunction(int total_subgraphs, int subgraph_id,
84                      int num_input_tensors, int num_output_tensors,
85                      PackedArgs args)
86 {
87   float** in_data  = new float*[num_inputs_per_inference * batch_size];
88   float** out_data = new float*[num_outputs_per_inference * batch_size];
90   for (in j = 0; j < batch_size; j++)
91   {
92     for (int i = 0; i < num_inputs_per_inference + num_outputs_per_inference;
93          i++)
94       if (i < num_inputs_per_inference)
95         in_data[j * num_inputs_per_inference + i] = args.data[i][j];
96       else
97         out_data[j * num_outpus_per_inference + i - num_inputs_per_inference]
98                                                   = args.data[i][j];
99   }
101   // call into this function in libtidl.so
102   // dlopen("libtidl_api.so")
103   // TidlFunc = dlsym("TidlRunSubgraph");
104   (*TidlFunc)(total_subgraphs, subgraph_id, batch_size
105               num_inputs_per_inference, num_outputs_per_inference,
106               in_data, out_data);
108   delete [] in_data;
109   delete [] out_data;
111 #endif