1 /******************************************************************************
2 * Copyright (c) 2017-2018 Texas Instruments Incorporated - http://www.ti.com/
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are met:
7 * * Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * * Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 * * Neither the name of Texas Instruments Incorporated nor the
13 * names of its contributors may be used to endorse or promote products
14 * derived from this software without specific prior written permission.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
17 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
20 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
21 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
22 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
23 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
24 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
25 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
26 * THE POSSIBILITY OF SUCH DAMAGE.
27 *****************************************************************************/
30 #include <cstdlib>
31 #include <cassert>
32 using std::size_t;
34 #include <iostream>
36 #include "ocl_device.h"
37 #include "ocl_util.h"
38 #include "trace.h"
39 #include "../dsp/ocl_wrapper.dsp_h"
41 using namespace tidl;
43 static const char* error2string(cl_int err);
44 static void errorCheck(cl_int ret, int line);
46 Device::Device(cl_device_type t, const DeviceIds& ids):
47 device_type_m(t), device_ids_m(ids)
48 {
49 TRACE::print("\tOCL Device: %s created\n",
50 device_type_m == CL_DEVICE_TYPE_ACCELERATOR ? "DSP" :
51 device_type_m == CL_DEVICE_TYPE_CUSTOM ? "EVE" : "Unknown");
53 for (int i = 0; i < MAX_DEVICES; i++)
54 queue_m[i] = nullptr;
56 }
58 DspDevice::DspDevice(const DeviceIds& ids, const std::string &binary_filename):
59 Device(CL_DEVICE_TYPE_ACCELERATOR, ids)
60 {
61 cl_uint num_devices_found;
62 cl_device_id device_ids[MAX_DEVICES];
64 cl_int errcode = clGetDeviceIDs(0, // platform
65 device_type_m, // device_type
66 MAX_DEVICES, // num_entries
67 device_ids, // devices
68 &num_devices_found); // num_devices
69 errorCheck(errcode, __LINE__);
71 if (num_devices_found != 1)
72 throw Exception("OpenCL DSP device not found",
73 __FILE__, __FUNCTION__, __LINE__);
75 cl_int num_compute_units;
76 errcode = clGetDeviceInfo(device_ids[0],
77 CL_DEVICE_MAX_COMPUTE_UNITS,
78 sizeof(num_compute_units),
79 &num_compute_units,
80 nullptr);
82 if (num_compute_units == 1)
83 {
84 context_m = clCreateContextFromType(0, // properties
85 device_type_m, // device_type
86 0, // pfn_notify
87 0, // user_data
88 &errcode);
89 errorCheck(errcode, __LINE__);
91 // Queue 0 on device 0
92 queue_m[0] = clCreateCommandQueue(context_m,
93 device_ids[0],
94 CL_QUEUE_PROFILING_ENABLE,
95 &errcode);
96 errorCheck(errcode, __LINE__);
97 BuildProgramFromBinary(binary_filename, device_ids, 1);
98 }
99 else
100 {
101 const cl_uint NUM_SUB_DEVICES = 2;
103 // Create 2 sub-device's, each consisting of a C66x DSP
104 cl_device_partition_property properties[3] =
105 { CL_DEVICE_PARTITION_EQUALLY, 1, 0 };
107 // Query the number of sub-devices that can be created
108 cl_uint n_sub_devices = 0;
109 errcode = clCreateSubDevices(device_ids[0], // in_device
110 properties, // properties
111 0, // num_devices
112 NULL, // out_devices
113 &n_sub_devices); // num_devices_ret
114 errorCheck(errcode, __LINE__);
116 assert(n_sub_devices == NUM_SUB_DEVICES);
118 // Create the sub-devices
119 cl_device_id sub_devices[NUM_SUB_DEVICES] = {0, 0};
120 errcode = clCreateSubDevices(device_ids[0], // in_device
121 properties, // properties
122 n_sub_devices, // num_devices
123 sub_devices, // out_devices
124 nullptr); // num_devices_ret
125 errorCheck(errcode, __LINE__);
127 // Create a context containing the sub-devices
128 context_m = clCreateContext(NULL, // properties
129 NUM_SUB_DEVICES, // num_devices
130 sub_devices, // devices
131 NULL, // pfn_notify
132 NULL, // user_data
133 &errcode); // errcode_ret
134 errorCheck(errcode, __LINE__);
136 // Create queues to each sub-device
137 for (auto id : device_ids_m)
138 {
139 int index = static_cast<int>(id);
140 queue_m[index] = clCreateCommandQueue(context_m,
141 sub_devices[index],
142 CL_QUEUE_PROFILING_ENABLE,
143 &errcode);
144 errorCheck(errcode, __LINE__);
145 }
147 BuildProgramFromBinary(binary_filename, sub_devices, NUM_SUB_DEVICES);
148 }
150 errcode = clGetDeviceInfo(device_ids[0],
151 CL_DEVICE_MAX_CLOCK_FREQUENCY,
152 sizeof(freq_in_mhz_m),
153 &freq_in_mhz_m,
154 nullptr);
155 errorCheck(errcode, __LINE__);
156 }
159 EveDevice::EveDevice(const DeviceIds& ids, const std::string &kernel_names):
160 Device(CL_DEVICE_TYPE_CUSTOM, ids)
161 {
162 cl_uint num_devices_found;
163 cl_device_id all_device_ids[MAX_DEVICES];
165 // Find all the OpenCL devices available of the given type
166 cl_int errcode = clGetDeviceIDs(0, // platform
167 device_type_m, // device_type
168 MAX_DEVICES, // num_entries
169 all_device_ids, // devices
170 &num_devices_found); // num_devices
171 errorCheck(errcode, __LINE__);
173 assert (num_devices_found >= device_ids_m.size());
175 context_m = clCreateContextFromType(0, // properties
176 device_type_m, // device_type
177 0, // pfn_notify
178 0, // user_data
179 &errcode);
180 errorCheck(errcode, __LINE__);
183 // Create command queues to OpenCL devices specified by the
184 // device_ids_m set.
185 for (auto id : device_ids_m)
186 {
187 int index = static_cast<int>(id);
188 queue_m[index] = clCreateCommandQueue(context_m,
189 all_device_ids[index],
190 CL_QUEUE_PROFILING_ENABLE|
191 CL_QUEUE_OUT_OF_ORDER_EXEC_MODE_ENABLE,
192 &errcode);
193 errorCheck(errcode, __LINE__);
194 }
196 BuildProgramFromBinary(kernel_names, all_device_ids, device_ids_m.size());
198 errcode = clGetDeviceInfo(all_device_ids[0],
199 CL_DEVICE_MAX_CLOCK_FREQUENCY,
200 sizeof(freq_in_mhz_m),
201 &freq_in_mhz_m,
202 nullptr);
203 errorCheck(errcode, __LINE__);
204 }
207 bool DspDevice::BuildProgramFromBinary(const std::string &BFN,
208 cl_device_id device_ids[],
209 int num_devices)
210 {
211 size_t bin_len = ocl_wrapper_dsp_bin_len;
213 assert (bin_len != 0);
215 // Casting to make ocl_read_binary work with clCreateProgramWithBinary
216 const unsigned char *bin_arrc = reinterpret_cast <const unsigned char *>
217 (ocl_wrapper_dsp_bin);
219 size_t lengths[num_devices];
220 for (int i=0; i < num_devices; i++) lengths[i] = bin_len;
222 const unsigned char* binaries[num_devices];
223 for (int i=0; i < num_devices; i++) binaries[i] = bin_arrc;
225 cl_int err;
226 program_m = clCreateProgramWithBinary(context_m,
227 num_devices,
228 device_ids, // device_list
229 lengths,
230 binaries,
231 0, // binary_status
232 &err);
233 errorCheck(err, __LINE__);
235 const char *options = "";
236 err = clBuildProgram(program_m, num_devices, device_ids, options, 0, 0);
237 errorCheck(err, __LINE__);
239 return true;
240 }
242 bool EveDevice::BuildProgramFromBinary(const std::string& kernel_names,
243 cl_device_id device_ids[],
244 int num_devices)
245 {
246 cl_int err;
247 cl_device_id executor_device_ids[MAX_DEVICES];
249 int i = 0;
250 for (auto id : device_ids_m)
251 executor_device_ids[i++] = device_ids[static_cast<int>(id)];
253 program_m = clCreateProgramWithBuiltInKernels(context_m,
254 num_devices,
255 executor_device_ids, // device_list
256 kernel_names.c_str(),
257 &err);
258 errorCheck(err, __LINE__);
260 return true;
261 }
263 Kernel::Kernel(Device* device, const std::string& name,
264 const KernelArgs& args, uint8_t device_index):
265 name_m(name), device_m(device), device_index_m(device_index)
266 {
267 TRACE::print("Creating kernel %s\n", name.c_str());
268 cl_int err;
269 kernel_m = clCreateKernel(device_m->program_m, name_m.c_str(), &err);
270 errorCheck(err, __LINE__);
272 for (int i=0; i < tidl::internal::NUM_CONTEXTS; i++)
273 event_m[i] = nullptr;
275 int arg_index = 0;
276 for (const auto& arg : args)
277 {
278 if (!arg.isLocal())
279 {
280 if (arg.kind() == DeviceArgInfo::Kind::BUFFER)
281 {
282 cl_mem buffer = device_m->CreateBuffer(arg);
284 clSetKernelArg(kernel_m, arg_index, sizeof(cl_mem), &buffer);
285 TRACE::print(" Arg[%d]: %p\n", arg_index, buffer);
287 if (buffer)
288 buffers_m.push_back(buffer);
289 }
290 else if (arg.kind() == DeviceArgInfo::Kind::SCALAR)
291 {
292 clSetKernelArg(kernel_m, arg_index, arg.size(), arg.ptr());
293 TRACE::print(" Arg[%d]: %p\n", arg_index, arg.ptr());
294 }
295 else
296 {
297 assert ("DeviceArgInfo kind not supported");
298 }
299 }
300 else
301 {
302 clSetKernelArg(kernel_m, arg_index, arg.size(), NULL);
303 TRACE::print(" Arg[%d]: local, %d\n", arg_index, arg.size());
304 }
305 arg_index++;
307 }
308 }
310 bool Kernel::UpdateScalarArg(uint32_t index, size_t size, const void *value)
311 {
312 cl_int ret = clSetKernelArg(kernel_m, index, size, value);
313 return ret == CL_SUCCESS;
314 }
316 Kernel& Kernel::RunAsync(uint32_t context_idx)
317 {
318 // Execute kernel
319 TRACE::print("\tKernel: device %d executing %s, context %d\n",
320 device_index_m, name_m.c_str(), context_idx);
321 cl_int ret = clEnqueueTask(device_m->queue_m[device_index_m],
322 kernel_m, 0, 0, &event_m[context_idx]);
323 errorCheck(ret, __LINE__);
325 return *this;
326 }
328 bool Kernel::Wait(uint32_t context_idx)
329 {
330 // Wait called without a corresponding RunAsync
331 if (event_m[context_idx] == nullptr)
332 return false;
334 TRACE::print("\tKernel: waiting context %d...\n", context_idx);
335 cl_int ret = clWaitForEvents(1, &event_m[context_idx]);
336 errorCheck(ret, __LINE__);
338 ret = clReleaseEvent(event_m[context_idx]);
339 errorCheck(ret, __LINE__);
340 event_m[context_idx] = nullptr;
342 TRACE::print("\tKernel: finished execution\n");
344 return true;
345 }
347 extern void CallbackWrapper(void *user_data) __attribute__((weak));
349 static
350 void EventCallback(cl_event event, cl_int exec_status, void *user_data)
351 {
352 if (exec_status != CL_SUCCESS || user_data == nullptr) return;
353 if (CallbackWrapper) CallbackWrapper(user_data);
354 }
356 bool Kernel::AddCallback(void *user_data, uint32_t context_idx)
357 {
358 if (event_m[context_idx] == nullptr)
359 return false;
361 return clSetEventCallback(event_m[context_idx], CL_COMPLETE, EventCallback,
362 user_data) == CL_SUCCESS;
363 }
365 Kernel::~Kernel()
366 {
367 for (auto b : buffers_m)
368 device_m->ReleaseBuffer(b);
370 clReleaseKernel(kernel_m);
371 }
373 cl_mem Device::CreateBuffer(const DeviceArgInfo &Arg)
374 {
375 size_t size = Arg.size();
376 void *host_ptr = Arg.ptr();
378 if (host_ptr == nullptr)
379 {
380 TRACE::print("\tOCL Create B:%p\n", nullptr);
381 return nullptr;
382 }
384 bool hostPtrInCMEM = __is_in_malloced_region(host_ptr);
386 // Conservative till we have sufficient information.
387 cl_mem_flags flag = CL_MEM_READ_WRITE;
389 if (hostPtrInCMEM) flag |= (cl_mem_flags)CL_MEM_USE_HOST_PTR;
390 else flag |= (cl_mem_flags)CL_MEM_COPY_HOST_PTR;
392 cl_int errcode;
393 cl_mem buffer = clCreateBuffer(context_m,
394 flag,
395 size,
396 host_ptr,
397 &errcode);
398 errorCheck(errcode, __LINE__);
400 TRACE::print("\tOCL Create B:%p\n", buffer);
402 return buffer;
403 }
405 void Device::ReleaseBuffer(cl_mem M)
406 {
407 TRACE::print("\tOCL Release B:%p\n", M);
408 clReleaseMemObject(M);
409 }
411 /// Release resources associated with an OpenCL device
412 Device::~Device()
413 {
414 TRACE::print("\tOCL Device: deleted\n");
415 for (unsigned int i = 0; i < device_ids_m.size(); i++)
416 {
417 clFinish(queue_m[i]);
418 clReleaseCommandQueue (queue_m[i]);
419 }
421 clReleaseProgram (program_m);
422 clReleaseContext (context_m);
423 }
425 void errorCheck(cl_int ret, int line)
426 {
427 if (ret != CL_SUCCESS)
428 {
429 std::cerr << "ERROR: [ Line: " << line << "] " << error2string(ret) << std::endl;
430 exit(ret);
431 }
432 }
434 /// Convert OpenCL error codes to a string
435 const char* error2string(cl_int err)
436 {
437 switch(err)
438 {
439 case 0: return "CL_SUCCESS";
440 case -1: return "CL_DEVICE_NOT_FOUND";
441 case -2: return "CL_DEVICE_NOT_AVAILABLE";
442 case -3: return "CL_COMPILER_NOT_AVAILABLE";
443 case -4: return "CL_MEM_OBJECT_ALLOCATION_FAILURE";
444 case -5: return "CL_OUT_OF_RESOURCES";
445 case -6: return "CL_OUT_OF_HOST_MEMORY";
446 case -7: return "CL_PROFILING_INFO_NOT_AVAILABLE";
447 case -8: return "CL_MEM_COPY_OVERLAP";
448 case -9: return "CL_IMAGE_FORMAT_MISMATCH";
449 case -10: return "CL_IMAGE_FORMAT_NOT_SUPPORTED";
450 case -11: return "CL_BUILD_PROGRAM_FAILURE";
451 case -12: return "CL_MAP_FAILURE";
453 case -30: return "CL_INVALID_VALUE";
454 case -31: return "CL_INVALID_DEVICE_TYPE";
455 case -32: return "CL_INVALID_PLATFORM";
456 case -33: return "CL_INVALID_DEVICE";
457 case -34: return "CL_INVALID_CONTEXT";
458 case -35: return "CL_INVALID_QUEUE_PROPERTIES";
459 case -36: return "CL_INVALID_COMMAND_QUEUE";
460 case -37: return "CL_INVALID_HOST_PTR";
461 case -38: return "CL_INVALID_MEM_OBJECT";
462 case -39: return "CL_INVALID_IMAGE_FORMAT_DESCRIPTOR";
463 case -40: return "CL_INVALID_IMAGE_SIZE";
464 case -41: return "CL_INVALID_SAMPLER";
465 case -42: return "CL_INVALID_BINARY";
466 case -43: return "CL_INVALID_BUILD_OPTIONS";
467 case -44: return "CL_INVALID_PROGRAM";
468 case -45: return "CL_INVALID_PROGRAM_EXECUTABLE";
469 case -46: return "CL_INVALID_KERNEL_NAME";
470 case -47: return "CL_INVALID_KERNEL_DEFINITION";
471 case -48: return "CL_INVALID_KERNEL";
472 case -49: return "CL_INVALID_ARG_INDEX";
473 case -50: return "CL_INVALID_ARG_VALUE";
474 case -51: return "CL_INVALID_ARG_SIZE";
475 case -52: return "CL_INVALID_KERNEL_ARGS";
476 case -53: return "CL_INVALID_WORK_DIMENSION";
477 case -54: return "CL_INVALID_WORK_GROUP_SIZE";
478 case -55: return "CL_INVALID_WORK_ITEM_SIZE";
479 case -56: return "CL_INVALID_GLOBAL_OFFSET";
480 case -57: return "CL_INVALID_EVENT_WAIT_LIST";
481 case -58: return "CL_INVALID_EVENT";
482 case -59: return "CL_INVALID_OPERATION";
483 case -60: return "CL_INVALID_GL_OBJECT";
484 case -61: return "CL_INVALID_BUFFER_SIZE";
485 case -62: return "CL_INVALID_MIP_LEVEL";
486 case -63: return "CL_INVALID_GLOBAL_WORK_SIZE";
487 default: return "Unknown OpenCL error";
488 }
489 }
491 Device::Ptr Device::Create(DeviceType core_type, const DeviceIds& ids,
492 const std::string& name)
493 {
494 Device::Ptr p(nullptr);
495 if (core_type == DeviceType::DSP)
496 p.reset(new DspDevice(ids, name));
497 else if (core_type == DeviceType::EVE)
498 p.reset(new EveDevice(ids, name));
500 return p;
501 }
503 // Minimum version of OpenCL required for this version of TIDL API
504 #define MIN_OCL_VERSION "01.01.17.00"
505 static bool CheckOpenCLVersion(cl_platform_id id)
506 {
507 cl_int err;
508 size_t length;
509 err = clGetPlatformInfo(id, CL_PLATFORM_VERSION, 0, nullptr, &length);
510 if (err != CL_SUCCESS) return false;
512 std::unique_ptr<char> version(new char[length]);
513 err = clGetPlatformInfo(id, CL_PLATFORM_VERSION, length, version.get(),
514 nullptr);
515 if (err != CL_SUCCESS) return false;
517 std::string v(version.get());
519 if (v.substr(v.find("01."), sizeof(MIN_OCL_VERSION)) >= MIN_OCL_VERSION)
520 return true;
522 std::cerr << "TIDL API Error: OpenCL " << MIN_OCL_VERSION
523 << " or higher required." << std::endl;
525 return false;
526 }
528 static bool PlatformIsAM57()
529 {
530 cl_platform_id id;
531 cl_int err;
533 err = clGetPlatformIDs(1, &id, nullptr);
534 if (err != CL_SUCCESS) return false;
536 if (!CheckOpenCLVersion(id))
537 return false;
539 // Check if the device name is AM57
540 size_t length;
541 err = clGetPlatformInfo(id, CL_PLATFORM_NAME, 0, nullptr, &length);
542 if (err != CL_SUCCESS) return false;
544 std::unique_ptr<char> name(new char[length]);
546 err = clGetPlatformInfo(id, CL_PLATFORM_NAME, length, name.get(), nullptr);
547 if (err != CL_SUCCESS) return false;
549 std::string platform_name(name.get());
551 if (platform_name.find("AM57") == std::string::npos)
552 return false;
554 return true;
555 }
557 // TI DL is supported on AM57x - EVE or C66x devices
558 uint32_t Device::GetNumDevices(DeviceType device_type)
559 {
560 if (!PlatformIsAM57()) return 0;
562 // Convert DeviceType to OpenCL device type
563 cl_device_type t = (device_type == DeviceType::EVE) ?
564 CL_DEVICE_TYPE_CUSTOM :
565 CL_DEVICE_TYPE_ACCELERATOR;
567 // Find all the OpenCL devices available
568 cl_uint num_devices_found;
569 cl_device_id all_device_ids[MAX_DEVICES];
571 cl_int errcode = clGetDeviceIDs(0, // platform
572 t, // device_type
573 MAX_DEVICES, // num_entries
574 all_device_ids, // devices
575 &num_devices_found); // num_devices
578 if (errcode != CL_SUCCESS) return 0;
579 if (num_devices_found == 0) return 0;
581 // DSP, return the number of compute units since we maintain a
582 // queue to each compute unit (i.e. C66x DSP)
583 if (t == CL_DEVICE_TYPE_ACCELERATOR)
584 {
585 cl_int num_compute_units;
586 errcode = clGetDeviceInfo(all_device_ids[0],
587 CL_DEVICE_MAX_COMPUTE_UNITS,
588 sizeof(num_compute_units),
589 &num_compute_units,
590 nullptr);
591 if (errcode != CL_SUCCESS)
592 return 0;
594 return num_compute_units;
595 }
597 // EVE, return the number of devices since each EVE is a device
598 return num_devices_found;
599 }