Update TIDL network data structure
[tidl/tidl-api.git] / tidl_api / src / ocl_device.cpp
1 /******************************************************************************
2  * Copyright (c) 2017-2018  Texas Instruments Incorporated - http://www.ti.com/
3  *   All rights reserved.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions are met:
7  *     * Redistributions of source code must retain the above copyright
8  *       notice, this list of conditions and the following disclaimer.
9  *     * Redistributions in binary form must reproduce the above copyright
10  *       notice, this list of conditions and the following disclaimer in the
11  *       documentation and/or other materials provided with the distribution.
12  *     * Neither the name of Texas Instruments Incorporated nor the
13  *       names of its contributors may be used to endorse or promote products
14  *       derived from this software without specific prior written permission.
15  *
16  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
17  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
20  * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
21  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
22  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
23  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
24  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
25  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
26  * THE POSSIBILITY OF SUCH DAMAGE.
27  *****************************************************************************/
30 #include <cstdlib>
31 #include <cassert>
32 using std::size_t;
34 #include <iostream>
36 #include "ocl_device.h"
37 #include "ocl_util.h"
38 #include "trace.h"
39 #include "../dsp/ocl_wrapper.dsp_h"
41 using namespace tidl;
43 static const char* error2string(cl_int err);
44 static void        errorCheck(cl_int ret, int line);
46 Device::Device(cl_device_type t, const DeviceIds& ids):
47                 device_type_m(t), device_ids_m(ids)
48 {
49     TRACE::print("\tOCL Device: %s created\n",
50               device_type_m == CL_DEVICE_TYPE_ACCELERATOR ? "DSP" :
51               device_type_m == CL_DEVICE_TYPE_CUSTOM ? "EVE" : "Unknown");
53     for (int i = 0; i < MAX_DEVICES; i++)
54         queue_m[i] = nullptr;
56 }
58 DspDevice::DspDevice(const DeviceIds& ids, const std::string &binary_filename):
59               Device(CL_DEVICE_TYPE_ACCELERATOR, ids)
60 {
61     cl_uint num_devices_found;
62     cl_device_id device_ids[MAX_DEVICES];
64     cl_int errcode = clGetDeviceIDs(0,               // platform
65                              device_type_m,          // device_type
66                              MAX_DEVICES,            // num_entries
67                              device_ids,             // devices
68                              &num_devices_found);    // num_devices
69     errorCheck(errcode, __LINE__);
71     if (num_devices_found != 1)
72         throw Exception("OpenCL DSP device not found",
73                         __FILE__, __FUNCTION__, __LINE__);
75     cl_int num_compute_units;
76     errcode = clGetDeviceInfo(device_ids[0],
77                               CL_DEVICE_MAX_COMPUTE_UNITS,
78                               sizeof(num_compute_units),
79                               &num_compute_units,
80                               nullptr);
82     if (num_compute_units == 1)
83     {
84         context_m = clCreateContextFromType(0,              // properties
85                                             device_type_m,  // device_type
86                                             0,              // pfn_notify
87                                             0,              // user_data
88                                             &errcode);
89         errorCheck(errcode, __LINE__);
91         // Queue 0 on device 0
92         queue_m[0] = clCreateCommandQueue(context_m,
93                                           device_ids[0],
94                                           CL_QUEUE_PROFILING_ENABLE|
95                                          CL_QUEUE_OUT_OF_ORDER_EXEC_MODE_ENABLE,
96                                           &errcode);
97         errorCheck(errcode, __LINE__);
98         BuildProgramFromBinary(binary_filename, device_ids, 1);
99     }
100     else
101     {
102         const cl_uint NUM_SUB_DEVICES = 2;
104         // Create 2 sub-device's, each consisting of a C66x DSP
105         cl_device_partition_property properties[3] =
106                                         { CL_DEVICE_PARTITION_EQUALLY, 1, 0 };
108         // Query the number of sub-devices that can be created
109         cl_uint n_sub_devices = 0;
110         errcode = clCreateSubDevices(device_ids[0],      // in_device
111                                      properties,         // properties
112                                      0,                  // num_devices
113                                      NULL,               // out_devices
114                                      &n_sub_devices);    // num_devices_ret
115         errorCheck(errcode, __LINE__);
117         assert(n_sub_devices == NUM_SUB_DEVICES);
119         // Create the sub-devices
120         cl_device_id sub_devices[NUM_SUB_DEVICES] = {0, 0};
121         errcode = clCreateSubDevices(device_ids[0],        // in_device
122                                      properties,           // properties
123                                      n_sub_devices,        // num_devices
124                                      sub_devices,          // out_devices
125                                      nullptr);             // num_devices_ret
126         errorCheck(errcode, __LINE__);
128         // Create a context containing the sub-devices
129         context_m = clCreateContext(NULL,               // properties
130                                     NUM_SUB_DEVICES,    // num_devices
131                                     sub_devices,        // devices
132                                     NULL,               // pfn_notify
133                                     NULL,               // user_data
134                                     &errcode);          // errcode_ret
135         errorCheck(errcode, __LINE__);
137         // Create queues to each sub-device
138         for (auto id : device_ids_m)
139         {
140             int index = static_cast<int>(id);
141             queue_m[index] = clCreateCommandQueue(context_m,
142                                           sub_devices[index],
143                                           CL_QUEUE_PROFILING_ENABLE|
144                                          CL_QUEUE_OUT_OF_ORDER_EXEC_MODE_ENABLE,
145                                           &errcode);
146             errorCheck(errcode, __LINE__);
147         }
149         BuildProgramFromBinary(binary_filename, sub_devices, NUM_SUB_DEVICES);
150     }
152     errcode = clGetDeviceInfo(device_ids[0],
153                                 CL_DEVICE_MAX_CLOCK_FREQUENCY,
154                                 sizeof(freq_in_mhz_m),
155                                 &freq_in_mhz_m,
156                                 nullptr);
157     errorCheck(errcode, __LINE__);
161 EveDevice::EveDevice(const DeviceIds& ids, const std::string &kernel_names):
162             Device(CL_DEVICE_TYPE_CUSTOM, ids)
164     cl_uint num_devices_found;
165     cl_device_id all_device_ids[MAX_DEVICES];
167     // Find all the OpenCL devices available of the given type
168     cl_int errcode = clGetDeviceIDs(0,              // platform
169                              device_type_m,         // device_type
170                              MAX_DEVICES,           // num_entries
171                              all_device_ids,        // devices
172                              &num_devices_found);   // num_devices
173     errorCheck(errcode, __LINE__);
175     assert (num_devices_found >= device_ids_m.size());
177     context_m = clCreateContextFromType(0,              // properties
178                                         device_type_m,  // device_type
179                                         0,              // pfn_notify
180                                         0,              // user_data
181                                         &errcode);
182     errorCheck(errcode, __LINE__);
185     // Create command queues to OpenCL devices specified by the
186     // device_ids_m set.
187     for (auto id : device_ids_m)
188     {
189         int index = static_cast<int>(id);
190         queue_m[index] = clCreateCommandQueue(context_m,
191                                       all_device_ids[index],
192                                       CL_QUEUE_PROFILING_ENABLE|
193                                       CL_QUEUE_OUT_OF_ORDER_EXEC_MODE_ENABLE,
194                                       &errcode);
195         errorCheck(errcode, __LINE__);
196     }
198     BuildProgramFromBinary(kernel_names, all_device_ids, device_ids_m.size());
200     errcode = clGetDeviceInfo(all_device_ids[0],
201                                 CL_DEVICE_MAX_CLOCK_FREQUENCY,
202                                 sizeof(freq_in_mhz_m),
203                                 &freq_in_mhz_m,
204                                 nullptr);
205     errorCheck(errcode, __LINE__);
209 bool DspDevice::BuildProgramFromBinary(const std::string &BFN,
210                                        cl_device_id device_ids[],
211                                        int num_devices)
213     size_t bin_len = ocl_wrapper_dsp_bin_len;
215     assert (bin_len != 0);
217     // Casting to make ocl_read_binary work with clCreateProgramWithBinary
218     const unsigned char *bin_arrc = reinterpret_cast <const unsigned char *>
219                                     (ocl_wrapper_dsp_bin);
221     size_t lengths[num_devices];
222     for (int i=0; i < num_devices; i++) lengths[i] = bin_len;
224     const unsigned char* binaries[num_devices];
225     for (int i=0; i < num_devices; i++) binaries[i] = bin_arrc;
227     cl_int err;
228     program_m = clCreateProgramWithBinary(context_m,
229                                           num_devices,
230                                           device_ids,          // device_list
231                                           lengths,
232                                           binaries,
233                                           0,                   // binary_status
234                                           &err);
235     errorCheck(err, __LINE__);
237     const char *options = "";
238     err = clBuildProgram(program_m, num_devices, device_ids, options, 0, 0);
239     errorCheck(err, __LINE__);
241     return true;
244 bool EveDevice::BuildProgramFromBinary(const std::string& kernel_names,
245                                        cl_device_id device_ids[],
246                                        int num_devices)
248     cl_int err;
249     cl_device_id executor_device_ids[MAX_DEVICES];
251     int i = 0;
252     for (auto id : device_ids_m)
253         executor_device_ids[i++] = device_ids[static_cast<int>(id)];
255     program_m = clCreateProgramWithBuiltInKernels(context_m,
256                                           num_devices,
257                                           executor_device_ids,  // device_list
258                                           kernel_names.c_str(),
259                                           &err);
260     errorCheck(err, __LINE__);
262     return true;
265 Kernel::Kernel(Device* device, const std::string& name,
266                const KernelArgs& args, uint8_t device_index):
267            name_m(name), device_m(device), device_index_m(device_index)
269     TRACE::print("Creating kernel %s\n", name.c_str());
270     cl_int err;
271     kernel_m = clCreateKernel(device_m->program_m, name_m.c_str(), &err);
272     errorCheck(err, __LINE__);
274     for (int i=0; i < tidl::internal::NUM_CONTEXTS; i++)
275         event_m[i] = nullptr;
277     int arg_index = 0;
278     for (const auto& arg : args)
279     {
280         if (!arg.isLocal())
281         {
282             if (arg.kind() == DeviceArgInfo::Kind::BUFFER)
283             {
284                 cl_mem buffer = device_m->CreateBuffer(arg);
286                 clSetKernelArg(kernel_m, arg_index, sizeof(cl_mem), &buffer);
287                 TRACE::print("  Arg[%d]: %p\n", arg_index, buffer);
289                 if (buffer)
290                     buffers_m.push_back(buffer);
291             }
292             else if (arg.kind() == DeviceArgInfo::Kind::SCALAR)
293             {
294                 clSetKernelArg(kernel_m, arg_index, arg.size(), arg.ptr());
295                 TRACE::print("  Arg[%d]: %p\n", arg_index, arg.ptr());
296             }
297             else
298             {
299                 assert ("DeviceArgInfo kind not supported");
300             }
301         }
302         else
303         {
304             clSetKernelArg(kernel_m, arg_index, arg.size(), NULL);
305             TRACE::print("  Arg[%d]: local, %d\n", arg_index, arg.size());
306         }
307         arg_index++;
309     }
312 bool Kernel::UpdateScalarArg(uint32_t index, size_t size, const void *value)
314     cl_int ret = clSetKernelArg(kernel_m, index, size, value);
315     return ret == CL_SUCCESS;
318 Kernel& Kernel::RunAsync(uint32_t context_idx)
320     // Execute kernel
321     TRACE::print("\tKernel: device %d executing %s, context %d\n",
322                  device_index_m, name_m.c_str(), context_idx);
323     cl_int ret = clEnqueueTask(device_m->queue_m[device_index_m],
324                                kernel_m, 0, 0, &event_m[context_idx]);
325     errorCheck(ret, __LINE__);
327     return *this;
330 bool Kernel::Wait(uint32_t context_idx)
332     // Wait called without a corresponding RunAsync
333     if (event_m[context_idx] == nullptr)
334         return false;
336     TRACE::print("\tKernel: waiting context %d...\n", context_idx);
337     cl_int ret = clWaitForEvents(1, &event_m[context_idx]);
338     errorCheck(ret, __LINE__);
340     ret = clReleaseEvent(event_m[context_idx]);
341     errorCheck(ret, __LINE__);
342     event_m[context_idx] = nullptr;
344     TRACE::print("\tKernel: finished execution\n");
346     return true;
349 extern void CallbackWrapper(void *user_data) __attribute__((weak));
351 static
352 void EventCallback(cl_event event, cl_int exec_status, void *user_data)
354     if (exec_status != CL_SUCCESS || user_data == nullptr)  return;
355     if (CallbackWrapper)  CallbackWrapper(user_data);
358 bool Kernel::AddCallback(void *user_data, uint32_t context_idx)
360     if (event_m[context_idx] == nullptr)
361         return false;
363     return clSetEventCallback(event_m[context_idx], CL_COMPLETE, EventCallback,
364                               user_data) == CL_SUCCESS;
367 Kernel::~Kernel()
369     for (auto b : buffers_m)
370         device_m->ReleaseBuffer(b);
372     clReleaseKernel(kernel_m);
375 cl_mem Device::CreateBuffer(const DeviceArgInfo &Arg)
377     size_t  size     = Arg.size();
378     void   *host_ptr = Arg.ptr();
380     if (host_ptr == nullptr)
381     {
382         TRACE::print("\tOCL Create B:%p\n", nullptr);
383         return nullptr;
384     }
386     bool hostPtrInCMEM = __is_in_malloced_region(host_ptr);
388     // Conservative till we have sufficient information.
389     cl_mem_flags flag = CL_MEM_READ_WRITE;
391     if (hostPtrInCMEM) flag |= (cl_mem_flags)CL_MEM_USE_HOST_PTR;
392     else               flag |= (cl_mem_flags)CL_MEM_COPY_HOST_PTR;
394     cl_int       errcode;
395     cl_mem buffer = clCreateBuffer(context_m,
396                                    flag,
397                                    size,
398                                    host_ptr,
399                                    &errcode);
400     errorCheck(errcode, __LINE__);
402     TRACE::print("\tOCL Create B:%p\n", buffer);
404     return buffer;
407 void Device::ReleaseBuffer(cl_mem M)
409     TRACE::print("\tOCL Release B:%p\n", M);
410     clReleaseMemObject(M);
413 /// Release resources associated with an OpenCL device
414 Device::~Device()
416     TRACE::print("\tOCL Device: deleted\n");
417     for (unsigned int i = 0; i < device_ids_m.size(); i++)
418     {
419         clFinish(queue_m[i]);
420         clReleaseCommandQueue (queue_m[i]);
421     }
423     clReleaseProgram      (program_m);
424     clReleaseContext      (context_m);
427 void errorCheck(cl_int ret, int line)
429     if (ret != CL_SUCCESS)
430     {
431         std::cerr << "ERROR: [ Line: " << line << "] " << error2string(ret) << std::endl;
432         exit(ret);
433     }
436 /// Convert OpenCL error codes to a string
437 const char* error2string(cl_int err)
439     switch(err)
440     {
441          case   0: return "CL_SUCCESS";
442          case  -1: return "CL_DEVICE_NOT_FOUND";
443          case  -2: return "CL_DEVICE_NOT_AVAILABLE";
444          case  -3: return "CL_COMPILER_NOT_AVAILABLE";
445          case  -4: return "CL_MEM_OBJECT_ALLOCATION_FAILURE";
446          case  -5: return "CL_OUT_OF_RESOURCES";
447          case  -6: return "CL_OUT_OF_HOST_MEMORY";
448          case  -7: return "CL_PROFILING_INFO_NOT_AVAILABLE";
449          case  -8: return "CL_MEM_COPY_OVERLAP";
450          case  -9: return "CL_IMAGE_FORMAT_MISMATCH";
451          case -10: return "CL_IMAGE_FORMAT_NOT_SUPPORTED";
452          case -11: return "CL_BUILD_PROGRAM_FAILURE";
453          case -12: return "CL_MAP_FAILURE";
455          case -30: return "CL_INVALID_VALUE";
456          case -31: return "CL_INVALID_DEVICE_TYPE";
457          case -32: return "CL_INVALID_PLATFORM";
458          case -33: return "CL_INVALID_DEVICE";
459          case -34: return "CL_INVALID_CONTEXT";
460          case -35: return "CL_INVALID_QUEUE_PROPERTIES";
461          case -36: return "CL_INVALID_COMMAND_QUEUE";
462          case -37: return "CL_INVALID_HOST_PTR";
463          case -38: return "CL_INVALID_MEM_OBJECT";
464          case -39: return "CL_INVALID_IMAGE_FORMAT_DESCRIPTOR";
465          case -40: return "CL_INVALID_IMAGE_SIZE";
466          case -41: return "CL_INVALID_SAMPLER";
467          case -42: return "CL_INVALID_BINARY";
468          case -43: return "CL_INVALID_BUILD_OPTIONS";
469          case -44: return "CL_INVALID_PROGRAM";
470          case -45: return "CL_INVALID_PROGRAM_EXECUTABLE";
471          case -46: return "CL_INVALID_KERNEL_NAME";
472          case -47: return "CL_INVALID_KERNEL_DEFINITION";
473          case -48: return "CL_INVALID_KERNEL";
474          case -49: return "CL_INVALID_ARG_INDEX";
475          case -50: return "CL_INVALID_ARG_VALUE";
476          case -51: return "CL_INVALID_ARG_SIZE";
477          case -52: return "CL_INVALID_KERNEL_ARGS";
478          case -53: return "CL_INVALID_WORK_DIMENSION";
479          case -54: return "CL_INVALID_WORK_GROUP_SIZE";
480          case -55: return "CL_INVALID_WORK_ITEM_SIZE";
481          case -56: return "CL_INVALID_GLOBAL_OFFSET";
482          case -57: return "CL_INVALID_EVENT_WAIT_LIST";
483          case -58: return "CL_INVALID_EVENT";
484          case -59: return "CL_INVALID_OPERATION";
485          case -60: return "CL_INVALID_GL_OBJECT";
486          case -61: return "CL_INVALID_BUFFER_SIZE";
487          case -62: return "CL_INVALID_MIP_LEVEL";
488          case -63: return "CL_INVALID_GLOBAL_WORK_SIZE";
489          default: return "Unknown OpenCL error";
490     }
493 Device::Ptr Device::Create(DeviceType core_type, const DeviceIds& ids,
494                            const std::string& name)
496     Device::Ptr p(nullptr);
497     if (core_type == DeviceType::DSP)
498         p.reset(new DspDevice(ids, name));
499     else if (core_type == DeviceType::EVE)
500         p.reset(new EveDevice(ids, name));
502     return p;
505 // Minimum version of OpenCL required for this version of TIDL API
506 #define MIN_OCL_VERSION "01.01.18.00"
507 static bool CheckOpenCLVersion(cl_platform_id id)
509     cl_int err;
510     size_t length;
511     err = clGetPlatformInfo(id, CL_PLATFORM_VERSION, 0, nullptr, &length);
512     if (err != CL_SUCCESS) return false;
514     std::unique_ptr<char> version(new char[length]);
515     err = clGetPlatformInfo(id, CL_PLATFORM_VERSION, length, version.get(),
516                             nullptr);
517     if (err != CL_SUCCESS) return false;
519     std::string v(version.get());
521     if (v.substr(v.find("01."), sizeof(MIN_OCL_VERSION)) >= MIN_OCL_VERSION)
522         return true;
524     std::cerr << "TIDL API Error: OpenCL " << MIN_OCL_VERSION
525               << " or higher required." << std::endl;
527     return false;
530 static bool PlatformIsAM57()
532     cl_platform_id id;
533     cl_int err;
535     err = clGetPlatformIDs(1, &id, nullptr);
536     if (err != CL_SUCCESS) return false;
538     if (!CheckOpenCLVersion(id))
539        return false;
541     // Check if the device name is AM57
542     size_t length;
543     err = clGetPlatformInfo(id, CL_PLATFORM_NAME, 0, nullptr, &length);
544     if (err != CL_SUCCESS) return false;
546     std::unique_ptr<char> name(new char[length]);
548     err = clGetPlatformInfo(id, CL_PLATFORM_NAME, length, name.get(), nullptr);
549     if (err != CL_SUCCESS) return false;
551     std::string platform_name(name.get());
553     if (platform_name.find("AM57") == std::string::npos)
554         return false;
556     return true;
559 // TI DL is supported on AM57x - EVE or C66x devices
560 uint32_t Device::GetNumDevices(DeviceType device_type)
562     if (!PlatformIsAM57()) return 0;
564     // Convert DeviceType to OpenCL device type
565     cl_device_type t = (device_type == DeviceType::EVE) ?
566                                     CL_DEVICE_TYPE_CUSTOM :
567                                     CL_DEVICE_TYPE_ACCELERATOR;
569     // Find all the OpenCL devices available
570     cl_uint num_devices_found;
571     cl_device_id all_device_ids[MAX_DEVICES];
573     cl_int errcode = clGetDeviceIDs(0,                   // platform
574                                     t,                   // device_type
575                                     MAX_DEVICES,         // num_entries
576                                     all_device_ids,      // devices
577                                     &num_devices_found); // num_devices
580     if (errcode != CL_SUCCESS)            return 0;
581     if (num_devices_found == 0)           return 0;
583     // DSP, return the number of compute units since we maintain a
584     // queue to each compute unit (i.e. C66x DSP)
585     if (t == CL_DEVICE_TYPE_ACCELERATOR)
586     {
587         cl_int num_compute_units;
588         errcode = clGetDeviceInfo(all_device_ids[0],
589                                 CL_DEVICE_MAX_COMPUTE_UNITS,
590                                 sizeof(num_compute_units),
591                                 &num_compute_units,
592                                 nullptr);
593         if (errcode != CL_SUCCESS)
594             return 0;
596         return num_compute_units;
597     }
599     // EVE, return the number of devices since each EVE is a device
600     return num_devices_found;