Subgraph: use Layer2Group map in config file
[tidl/tidl-api.git] / tidl_api / inc / subgraph_runtime.h
1 /******************************************************************************
2  * Copyright (c) 2019 Texas Instruments Incorporated - http://www.ti.com/
3  *  All rights reserved.
4  *
5  *  Redistribution and use in source and binary forms, with or without
6  *  modification, are permitted provided that the following conditions are met:
7  *      * Redistributions of source code must retain the above copyright
8  *        notice, this list of conditions and the following disclaimer.
9  *      * Redistributions in binary form must reproduce the above copyright
10  *        notice, this list of conditions and the following disclaimer in the
11  *        documentation and/or other materials provided with the distribution.
12  *      * Neither the name of Texas Instruments Incorporated nor the
13  *        names of its contributors may be used to endorse or promote products
14  *        derived from this software without specific prior written permission.
15  *
16  *  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
17  *  AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18  *  IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19  *  ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
20  *  LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
21  *  CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
22  *  SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
23  *  INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
24  *  CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
25  *  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
26  *  THE POSSIBILITY OF SUCH DAMAGE.
27  *****************************************************************************/
29 //! @file subgraph_runtime.h
31 #pragma once
33 extern "C" {
35 //! @brief Top level API to get preferred batch_size for a subgraph
36 //!        Best performance comes with preferred batch_size processing
37 //!        plus multi-threaded (num_threads = 2) processing
38 //! @param total_subgraphs  total number of TIDL subgraphs in whole inference
39 //! @return preferred batch size
40 extern int TidlGetPreferredBatchSize(int total_subgraphs);
42 //! @brief Top level API to initialize a TIDL subgraph on device
43 //!        If not invoked ahead of time, TidlRunSubgraph() will call this
44 //!        function before any inference
45 //! @param total_subgraphs  total number of TIDL subgraphs in whole inference
46 //! @param subgraph_id  index of current TIDL subgraph
47 extern void TidlInitSubgraph(int total_subgraphs,
48                              int subgraph_id
49                             );
51 //! @brief Top level API to free a TIDL subgraph on device
52 //! @param total_subgraphs  total number of TIDL subgraphs in whole inference
53 //! @param subgraph_id  index of current TIDL subgraph
54 extern void TidlFreeSubgraph(int total_subgraphs,
55                              int subgraph_id
56                             );
58 //! @brief Top level inference to run a TIDL subgraph
59 //! @param total_subgraphs  total number of TIDL subgraphs in whole inference
60 //! @param subgraph_id  index of current TIDL subgraph
61 //! @param batch_size  number of samples/inferences in this batch
62 //! @param num_inputs_per_inference  number of inputs to TIDL subgraph
63 //!        for every sample/inference
64 //! @param num_outputs_per_inference  number of outputs from TIDL subgraph
65 //!        for every sample/inference
66 //! @param input_tensors  input data to TIDL subgraph, layout as
67 //!        batch1_input1, batch1_input2, ..., batch1_inputM,
68 //!        ... ... ...
69 //!        batchN_input1, batchN_input2, ..., batchN_inputM
70 //! @param output_tensors  output data from TIDL subgraph, layout as
71 //!        batch1_output1, batch1_output2, ..., batch1_outputK,
72 //!        ... ... ...
73 //!        batchN_output1, batchN_output2, ..., batchN_outputK
74 extern void TidlRunSubgraph(int total_subgraphs,
75                             int subgraph_id,
76                             int batch_size,
77                             int num_inputs_per_inference,
78                             int num_outputs_per_inference,
79                             float **input_tensors,
80                             float **output_tensors
81                            );
83 }  // extern "C"
86 #if 0
87 // Auto-generated code from Relay/TVM compilation step after
88 // partitioning and lowering to backend implementation
90 void TVM_TidlFunction(int total_subgraphs, int subgraph_id,
91                      int num_input_tensors, int num_output_tensors,
92                      PackedArgs args)
93 {
94   float** in_data  = new float*[num_inputs_per_inference * batch_size];
95   float** out_data = new float*[num_outputs_per_inference * batch_size];
97   for (in j = 0; j < batch_size; j++)
98   {
99     for (int i = 0; i < num_inputs_per_inference + num_outputs_per_inference;
100          i++)
101       if (i < num_inputs_per_inference)
102         in_data[j * num_inputs_per_inference + i] = args.data[i][j];
103       else
104         out_data[j * num_outpus_per_inference + i - num_inputs_per_inference]
105                                                   = args.data[i][j];
106   }
108   // call into this function in libtidl.so
109   // dlopen("libtidl_api.so")
110   // TidlFunc = dlsym("TidlRunSubgraph");
111   (*TidlFunc)(total_subgraphs, subgraph_id, batch_size
112               num_inputs_per_inference, num_outputs_per_inference,
113               in_data, out_data);
115   delete [] in_data;
116   delete [] out_data;
118 #endif