| /* |
| Copyright (c) 2014-2016 Intel Corporation. All Rights Reserved. |
| |
| Redistribution and use in source and binary forms, with or without |
| modification, are permitted provided that the following conditions |
| are met: |
| |
| * Redistributions of source code must retain the above copyright |
| notice, this list of conditions and the following disclaimer. |
| * Redistributions in binary form must reproduce the above copyright |
| notice, this list of conditions and the following disclaimer in the |
| documentation and/or other materials provided with the distribution. |
| * Neither the name of Intel Corporation nor the names of its |
| contributors may be used to endorse or promote products derived |
| from this software without specific prior written permission. |
| |
| THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
| "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
| LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
| A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
| HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
| SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
| LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
| DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
| THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
| OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| */ |
| |
| |
| #include <omp.h> |
| //#include <stdlib.h> |
| //#include "offload.h" |
| #include "compiler_if_host.h" |
| |
| |
| // OpenMP API |
| |
| void omp_set_default_device(int num) __GOMP_NOTHROW |
| { |
| if (num >= 0) { |
| __omp_device_num = num; |
| } |
| } |
| |
| int omp_get_default_device(void) __GOMP_NOTHROW |
| { |
| return __omp_device_num; |
| } |
| |
| int omp_get_num_devices() __GOMP_NOTHROW |
| { |
| __offload_init_library(); |
| return mic_engines_total; |
| } |
| |
| // OpenMP 4.5 APIs |
| |
| // COI supports 3-dim multiD transfers |
| #define MAX_ARRAY_RANK 3 |
| |
| int omp_get_initial_device( |
| void |
| ) __GOMP_NOTHROW |
| { |
| return -1; |
| } |
| |
| void* omp_target_alloc( |
| size_t size, |
| int device_num |
| ) __GOMP_NOTHROW |
| { |
| __offload_init_library(); |
| |
| OFFLOAD_TRACE(2, "omp_target_alloc(%lld, %d)\n", size, device_num); |
| |
| if (device_num < -1) { |
| LIBOFFLOAD_ERROR(c_invalid_device_number); |
| exit(1); |
| } |
| |
| void* result = 0; |
| |
| // malloc on CPU |
| if (device_num == -1) { |
| // We do not check for malloc returning NULL because the |
| // specification of this API includes the possibility of failure. |
| // The user will check the returned result |
| result = malloc(size); |
| return result; |
| } |
| |
| OFFLOAD ofld = OFFLOAD_TARGET_ACQUIRE( |
| TARGET_MIC, device_num, 0, NULL, __func__, 0); |
| if (ofld != 0) { |
| VarDesc vars[2] = {0}; |
| |
| vars[0].type.src = c_data; |
| vars[0].type.dst = c_data; |
| vars[0].direction.bits = c_parameter_in; |
| vars[0].size = sizeof(size); |
| vars[0].count = 1; |
| vars[0].ptr = &size; |
| |
| vars[1].type.src = c_data; |
| vars[1].type.dst = c_data; |
| vars[1].direction.bits = c_parameter_out; |
| vars[1].size = sizeof(result); |
| vars[1].count = 1; |
| vars[1].ptr = &result; |
| |
| OFFLOAD_OFFLOAD(ofld, "omp_target_alloc_target", |
| 0, 2, vars, NULL, 0, 0, 0); |
| } |
| return result; |
| } |
| |
| void omp_target_free( |
| void *device_ptr, |
| int device_num |
| ) __GOMP_NOTHROW |
| { |
| __offload_init_library(); |
| |
| OFFLOAD_TRACE(2, "omp_target_free(%p, %d)\n", device_ptr, device_num); |
| |
| if (device_num < -1) { |
| LIBOFFLOAD_ERROR(c_invalid_device_number); |
| exit(1); |
| } |
| |
| // free on CPU |
| if (device_num == -1) { |
| free(device_ptr); |
| return; |
| } |
| |
| OFFLOAD ofld = OFFLOAD_TARGET_ACQUIRE( |
| TARGET_MIC, device_num, 0, NULL, __func__, 0); |
| if (ofld) { |
| VarDesc vars[1] = {0}; |
| |
| vars[0].type.src = c_data; |
| vars[0].type.dst = c_data; |
| vars[0].direction.bits = c_parameter_in; |
| vars[0].size = sizeof(device_ptr); |
| vars[0].count = 1; |
| vars[0].ptr = &device_ptr; |
| |
| OFFLOAD_OFFLOAD(ofld, "omp_target_free_target", |
| 0, 1, vars, NULL, 0, 0, 0); |
| } |
| } |
| |
| int omp_target_is_present( |
| void *ptr, |
| int device_num |
| ) __GOMP_NOTHROW |
| { |
| __offload_init_library(); |
| |
| OFFLOAD_TRACE(2, "omp_target_is_present(%p, %d)\n", ptr, device_num); |
| |
| if (device_num < -1) { |
| LIBOFFLOAD_ERROR(c_invalid_device_number); |
| exit(1); |
| } |
| |
| if (device_num == -1) { |
| return false; |
| } |
| |
| // If OpenMP allows wrap-around for device numbers, enable next line |
| //device_num %= mic_engines_total; |
| |
| // lookup existing association in pointer table |
| PtrData* ptr_data = mic_engines[device_num].find_ptr_data(ptr); |
| if (ptr_data == 0) { |
| OFFLOAD_TRACE(3, "Address %p is not mapped on device %d\n", |
| ptr, device_num); |
| return false; |
| } |
| |
| OFFLOAD_TRACE(3, "Address %p found mapped on device %d\n", |
| ptr, device_num); |
| return true; |
| } |
| |
| int omp_target_memcpy( |
| void *dst, |
| void *src, |
| size_t length, |
| size_t dst_offset, |
| size_t src_offset, |
| int dst_device, |
| int src_device |
| ) __GOMP_NOTHROW |
| { |
| __offload_init_library(); |
| |
| OFFLOAD_TRACE(2, "omp_target_memcpy(%p, %p, %lld, %lld, %lld, %d, %d)\n", |
| dst, src, length, dst_offset, src_offset, dst_device, src_device); |
| |
| if (dst_device < -1 || src_device < -1) { |
| LIBOFFLOAD_ERROR(c_invalid_device_number); |
| exit(1); |
| } |
| |
| char* srcp = (char *)src + src_offset; |
| char* dstp = (char *)dst + dst_offset; |
| |
| if (src_device == -1) { |
| // Source is CPU |
| if (dst_device == -1) { |
| // CPU -> CPU |
| memcpy(dstp, srcp, length); |
| return 0; |
| } else { |
| // CPU -> MIC |
| // COIBufferWrite |
| // If OpenMP allows wrap-around for device numbers, enable next line |
| //dst_device %= mic_engines_total; |
| |
| OFFLOAD_TRACE(3, "Creating buffer from sink memory %llx\n", dstp); |
| COIBUFFER mic_buf; |
| COIRESULT res = COI::BufferCreateFromMemory(length, |
| COI_BUFFER_NORMAL, COI_SINK_MEMORY, dstp, |
| 1, &mic_engines[dst_device].get_process(), |
| &mic_buf); |
| if (res != COI_SUCCESS) { |
| LIBOFFLOAD_ERROR(c_buf_create_from_mem, res); |
| return 1; |
| } |
| res = COI::BufferWrite(mic_buf, 0, srcp, length, |
| COI_COPY_UNSPECIFIED, 0, 0, 0); |
| if (res != COI_SUCCESS) { |
| LIBOFFLOAD_ERROR(c_buf_write, res); |
| return 1; |
| } |
| res = COI::BufferDestroy(mic_buf); |
| if (res != COI_SUCCESS) { |
| LIBOFFLOAD_ERROR(c_buf_destroy, res); |
| return 1; |
| } |
| return 0; |
| } |
| } else { |
| // Source is device |
| if (dst_device == -1) { |
| // MIC -> CPU |
| // COIBufferRead |
| |
| // If OpenMP allows wrap-around for device numbers, enable next line |
| //src_device %= mic_engines_total; |
| |
| OFFLOAD_TRACE(3, "Creating buffer from sink memory %llx\n", srcp); |
| COIBUFFER mic_buf; |
| COIRESULT res = COI::BufferCreateFromMemory(length, |
| COI_BUFFER_NORMAL, COI_SINK_MEMORY, srcp, |
| 1, &mic_engines[src_device].get_process(), |
| &mic_buf); |
| if (res != COI_SUCCESS) { |
| LIBOFFLOAD_ERROR(c_buf_create_from_mem, res); |
| return 1; |
| } |
| res = COI::BufferRead(mic_buf, 0, dstp, length, |
| COI_COPY_UNSPECIFIED, 0, 0, 0); |
| if (res != COI_SUCCESS) { |
| LIBOFFLOAD_ERROR(c_buf_read, res); |
| return 1; |
| } |
| res = COI::BufferDestroy(mic_buf); |
| if (res != COI_SUCCESS) { |
| LIBOFFLOAD_ERROR(c_buf_destroy, res); |
| return 1; |
| } |
| return 0; |
| } else { |
| // some MIC -> some MIC |
| if (src_device == dst_device) { |
| // MIC local copy will be done as remote memcpy |
| |
| OFFLOAD ofld = OFFLOAD_TARGET_ACQUIRE(TARGET_MIC, src_device, |
| 0, NULL, __func__, 0); |
| if (ofld) { |
| VarDesc vars[3] = {0}; |
| |
| vars[0].type.src = c_data; |
| vars[0].type.dst = c_data; |
| vars[0].direction.bits = c_parameter_in; |
| vars[0].size = sizeof(dstp); |
| vars[0].count = 1; |
| vars[0].ptr = &dstp; |
| |
| vars[1].type.src = c_data; |
| vars[1].type.dst = c_data; |
| vars[1].direction.bits = c_parameter_in; |
| vars[1].size = sizeof(srcp); |
| vars[1].count = 1; |
| vars[1].ptr = &srcp; |
| |
| vars[2].type.src = c_data; |
| vars[2].type.dst = c_data; |
| vars[2].direction.bits = c_parameter_in; |
| vars[2].size = sizeof(length); |
| vars[2].count = 1; |
| vars[2].ptr = &length; |
| |
| OFFLOAD_OFFLOAD(ofld, "omp_target_memcpy_target", |
| 0, 3, vars, NULL, 0, 0, 0); |
| return 0; |
| } else { |
| return 1; |
| } |
| } else { |
| // MICx -> MICy |
| // Allocate CPU buffer |
| char *cpu_mem = (char *)malloc(length); |
| if (cpu_mem == 0) { |
| LIBOFFLOAD_ERROR(c_malloc); |
| return 1; |
| } |
| int retval = 1; |
| if (omp_target_memcpy( |
| cpu_mem, srcp, length, 0, 0, -1, src_device) == 0) { |
| retval = omp_target_memcpy( |
| dstp, cpu_mem, length, 0, 0, dst_device, -1); |
| } |
| free(cpu_mem); |
| return retval; |
| } |
| } |
| } |
| } |
| |
| static size_t bytesize_at_this_dimension( |
| size_t element_size, |
| int num_dims, |
| const size_t* dimensions |
| ) |
| { |
| if (num_dims > 1) { |
| return dimensions[1] * |
| bytesize_at_this_dimension( |
| element_size, num_dims-1, dimensions+1); |
| } else { |
| return element_size; |
| } |
| } |
| |
| static void memcpy_rect( |
| char *dst, |
| char *src, |
| size_t element_size, |
| int num_dims, |
| const size_t *volume, |
| const size_t *dst_offsets, |
| const size_t *src_offsets, |
| const size_t *dst_dimensions, |
| const size_t *src_dimensions |
| ) |
| { |
| if (num_dims > 1) { |
| int count = volume[0]; |
| int dst_index = dst_offsets[0]; |
| int src_index = src_offsets[0]; |
| size_t dst_element_size = |
| bytesize_at_this_dimension(element_size, num_dims, dst_dimensions); |
| size_t src_element_size = |
| bytesize_at_this_dimension(element_size, num_dims, src_dimensions); |
| for (; count>0; dst_index++, src_index++, count--) { |
| memcpy_rect(dst+dst_element_size*dst_index, |
| src+src_element_size*src_index, |
| element_size, num_dims-1, volume+1, |
| dst_offsets+1, src_offsets+1, |
| dst_dimensions+1, src_dimensions+1); |
| } |
| } else { |
| memcpy(dst+dst_offsets[0]*element_size, |
| src+src_offsets[0]*element_size, |
| element_size * volume[0]); |
| } |
| } |
| |
| int omp_target_memcpy_rect( |
| void *dst_, |
| void *src_, |
| size_t element_size, |
| int num_dims, |
| const size_t *volume, |
| const size_t *dst_offsets, |
| const size_t *src_offsets, |
| const size_t *dst_dimensions, |
| const size_t *src_dimensions, |
| int dst_device, |
| int src_device |
| ) __GOMP_NOTHROW |
| { |
| char *dst = (char *)dst_; |
| char *src = (char *)src_; |
| |
| __offload_init_library(); |
| |
| OFFLOAD_TRACE(2, "omp_target_memcpy_rect(%p, %p, %lld, %d, " |
| "%p, %p, %p, %p, %p, %d, %d)\n", |
| dst, src, element_size, num_dims, |
| volume, dst_offsets, src_offsets, |
| dst_dimensions, src_dimensions, dst_device, src_device); |
| |
| // MAX_ARRAY_RANK dimensions are supported |
| if (dst == 0 && src == 0) { |
| return MAX_ARRAY_RANK; |
| } |
| |
| if (num_dims < 1 || num_dims > MAX_ARRAY_RANK || |
| element_size < 1 || |
| volume == 0 || dst_offsets == 0 || src_offsets == 0 || |
| dst_dimensions == 0 || src_dimensions == 0) { |
| return 1; |
| } |
| |
| if (dst_device < -1 || src_device < -1) { |
| LIBOFFLOAD_ERROR(c_invalid_device_number); |
| exit(1); |
| } |
| |
| if (src_device == -1) { |
| // Source is CPU |
| if (dst_device == -1) { |
| // CPU -> CPU |
| memcpy_rect((char*)dst, (char*)src, element_size, num_dims, volume, |
| dst_offsets, src_offsets, |
| dst_dimensions, src_dimensions); |
| return 0; |
| } else { |
| // CPU -> MIC |
| // COIBufferWriteMultiD |
| struct arr_desc dst_desc; |
| struct arr_desc src_desc; |
| |
| dst_desc.base = (int64_t)dst; |
| dst_desc.rank = num_dims; |
| |
| src_desc.base = (int64_t)src; |
| src_desc.rank = num_dims; |
| |
| for (int i=0; i<num_dims; i++) |
| { |
| dst_desc.dim[i].size = bytesize_at_this_dimension( |
| element_size, |
| num_dims - i, |
| dst_dimensions + i); |
| dst_desc.dim[i].lindex = 0; |
| dst_desc.dim[i].lower = dst_offsets[i]; |
| dst_desc.dim[i].upper = dst_offsets[i] + volume[i] - 1; |
| dst_desc.dim[i].stride = 1; |
| |
| src_desc.dim[i].size = bytesize_at_this_dimension( |
| element_size, |
| num_dims - i, |
| src_dimensions + i); |
| src_desc.dim[i].lindex = 0; |
| src_desc.dim[i].lower = src_offsets[i]; |
| src_desc.dim[i].upper = src_offsets[i] + volume[i] - 1; |
| src_desc.dim[i].stride = 1; |
| } |
| __arr_desc_dump("", "dst", (const Arr_Desc*)&dst_desc, false, false); |
| __arr_desc_dump("", "src", (const Arr_Desc*)&src_desc, false, false); |
| |
| // If OpenMP allows wrap-around for device numbers, enable next line |
| //dst_device %= mic_engines_total; |
| |
| // Compute MIC buffer size |
| size_t dst_length = dst_dimensions[0] * bytesize_at_this_dimension( |
| element_size, |
| num_dims, |
| dst_dimensions); |
| |
| OFFLOAD_TRACE(3, |
| "Creating buffer from sink memory %llx of size %lld\n", |
| dst, dst_length); |
| COIBUFFER mic_buf; |
| COIRESULT res = COI::BufferCreateFromMemory(dst_length, |
| COI_BUFFER_NORMAL, COI_SINK_MEMORY, dst, |
| 1, &mic_engines[dst_device].get_process(), |
| &mic_buf); |
| if (res != COI_SUCCESS) { |
| LIBOFFLOAD_ERROR(c_buf_create_from_mem, res); |
| return 1; |
| } |
| res = COI::BufferWriteMultiD(mic_buf, |
| mic_engines[dst_device].get_process(), |
| 0, &dst_desc, &src_desc, |
| COI_COPY_UNSPECIFIED, 0, 0, 0); |
| if (res != COI_SUCCESS) { |
| LIBOFFLOAD_ERROR(c_buf_write, res); |
| return 1; |
| } |
| res = COI::BufferDestroy(mic_buf); |
| if (res != COI_SUCCESS) { |
| LIBOFFLOAD_ERROR(c_buf_destroy, res); |
| return 1; |
| } |
| return 0; |
| } |
| } else { |
| // Source is device |
| if (dst_device == -1) { |
| // COIBufferReadMultiD |
| struct arr_desc dst_desc; |
| struct arr_desc src_desc; |
| |
| dst_desc.base = (int64_t)dst; |
| dst_desc.rank = num_dims; |
| |
| src_desc.base = (int64_t)src; |
| src_desc.rank = num_dims; |
| |
| for (int i=0; i<num_dims; i++) |
| { |
| dst_desc.dim[i].size = bytesize_at_this_dimension( |
| element_size, |
| num_dims - i, |
| dst_dimensions + i); |
| dst_desc.dim[i].lindex = 0; |
| dst_desc.dim[i].lower = dst_offsets[i]; |
| dst_desc.dim[i].upper = dst_offsets[i] + volume[i] - 1; |
| dst_desc.dim[i].stride = 1; |
| |
| src_desc.dim[i].size = bytesize_at_this_dimension( |
| element_size, |
| num_dims - i, |
| src_dimensions + i); |
| src_desc.dim[i].lindex = 0; |
| src_desc.dim[i].lower = src_offsets[i]; |
| src_desc.dim[i].upper = src_offsets[i] + volume[i] - 1; |
| src_desc.dim[i].stride = 1; |
| } |
| __arr_desc_dump("", "dst", (const Arr_Desc*)&dst_desc, false, false); |
| __arr_desc_dump("", "src", (const Arr_Desc*)&src_desc, false, false); |
| |
| // If OpenMP allows wrap-around for device numbers, enable next line |
| //src_device %= mic_engines_total; |
| |
| // Compute MIC buffer size |
| size_t src_length = src_dimensions[0] * bytesize_at_this_dimension( |
| element_size, |
| num_dims, |
| src_dimensions); |
| |
| OFFLOAD_TRACE(3, |
| "Creating buffer from sink memory %llx of size %lld\n", |
| src, src_length); |
| COIBUFFER mic_buf; |
| COIRESULT res = COI::BufferCreateFromMemory(src_length, |
| COI_BUFFER_NORMAL, COI_SINK_MEMORY, src, |
| 1, &mic_engines[src_device].get_process(), |
| &mic_buf); |
| if (res != COI_SUCCESS) { |
| LIBOFFLOAD_ERROR(c_buf_create_from_mem, res); |
| return 1; |
| } |
| res = COI::BufferReadMultiD(mic_buf, 0, |
| &dst_desc, &src_desc, |
| COI_COPY_UNSPECIFIED, 0, 0, 0); |
| if (res != COI_SUCCESS) { |
| LIBOFFLOAD_ERROR(c_buf_write, res); |
| return 1; |
| } |
| res = COI::BufferDestroy(mic_buf); |
| if (res != COI_SUCCESS) { |
| LIBOFFLOAD_ERROR(c_buf_destroy, res); |
| return 1; |
| } |
| return 0; |
| } else { |
| // some MIC -> some MIC |
| if (src_device == dst_device) { |
| // MIC local copy will be done as remote memcpy_rect |
| struct parameters { |
| void *dst; |
| void *src; |
| size_t element_size; |
| int num_dims; |
| size_t array_info[MAX_ARRAY_RANK*5]; |
| } parameters = {dst, src, element_size, num_dims}; |
| int result; |
| |
| for (int i=0; i<num_dims; i++) |
| { |
| parameters.array_info[i] = volume[i]; |
| parameters.array_info[i+num_dims] = dst_offsets[i]; |
| parameters.array_info[i+num_dims*2] = src_offsets[i]; |
| parameters.array_info[i+num_dims*3] = dst_dimensions[i]; |
| parameters.array_info[i+num_dims*4] = src_dimensions[i]; |
| } |
| |
| OFFLOAD ofld = OFFLOAD_TARGET_ACQUIRE(TARGET_MIC, src_device, |
| 0, NULL, __func__, 0); |
| if (ofld) { |
| VarDesc vars[1] = {0}; |
| |
| vars[0].type.src = c_data; |
| vars[0].type.dst = c_data; |
| vars[0].direction.bits = c_parameter_in; |
| vars[0].size = sizeof(parameters) - |
| (MAX_ARRAY_RANK - num_dims) * |
| 5 * sizeof(size_t); |
| vars[0].count = 1; |
| vars[0].ptr = ¶meters; |
| |
| OFFLOAD_OFFLOAD(ofld, "omp_target_memcpy_rect_target", |
| 0, 1, vars, NULL, 0, 0, 0); |
| return 0; |
| } else { |
| return 1; |
| } |
| } else { |
| // MICx -> MICy |
| |
| // Compute transfer byte-count |
| size_t dst_length = element_size; |
| for (int i=0; i<num_dims; i++) { |
| dst_length *= volume[i]; |
| } |
| |
| // Allocate CPU buffer |
| char *cpu_mem = (char *)malloc(dst_length); |
| if (cpu_mem == 0) { |
| LIBOFFLOAD_ERROR(c_malloc); |
| return 1; |
| } |
| |
| // Create CPU offset and dimension arrays |
| // The CPU array collects the data in a contiguous block |
| size_t cpu_offsets[MAX_ARRAY_RANK]; |
| size_t cpu_dimensions[MAX_ARRAY_RANK]; |
| for (int i=0; i<num_dims; i++) { |
| cpu_offsets[i] = 0; |
| cpu_dimensions[i] = volume[i]; |
| } |
| |
| int retval = 1; |
| if (omp_target_memcpy_rect( |
| cpu_mem, src, element_size, num_dims, volume, |
| cpu_offsets, src_offsets, |
| cpu_dimensions, src_dimensions, |
| -1, src_device) == 0) { |
| retval = omp_target_memcpy_rect( |
| dst, cpu_mem, element_size, num_dims, volume, |
| dst_offsets, cpu_offsets, |
| dst_dimensions, cpu_dimensions, |
| dst_device, -1); |
| } |
| free(cpu_mem); |
| return retval; |
| } |
| } |
| } |
| } |
| |
| // host_ptr is key in table that yields association on device |
| // A COIBUFFER of specified size is created from the memory at |
| // device_ptr+device_offset on device_num |
| int omp_target_associate_ptr( |
| void *host_ptr, |
| void *device_ptr, |
| size_t size, |
| size_t device_offset, |
| int device_num |
| ) __GOMP_NOTHROW |
| { |
| COIRESULT res; |
| |
| __offload_init_library(); |
| |
| OFFLOAD_TRACE(2, "omp_target_associate_ptr(%p, %p, %lld, %lld, %d)\n", |
| host_ptr, device_ptr, size, device_offset, device_num); |
| |
| if (device_num < -1) { |
| LIBOFFLOAD_ERROR(c_invalid_device_number); |
| exit(1); |
| } |
| |
| // Associating to CPU is treated as failure |
| if (device_num == -1) { |
| return 1; |
| } |
| |
| // An incorrect size is treated as failure |
| if (size < 0) { |
| return 1; |
| } |
| |
| // If OpenMP allows wrap-around for device numbers, enable next line |
| //Engine& device = mic_engines[device_num % mic_engines_total]; |
| Engine& device = mic_engines[device_num]; |
| |
| // Does host pointer have association already? |
| // lookup existing association in pointer table |
| PtrData* ptr_data = device.find_ptr_data(host_ptr); |
| if (ptr_data != 0) { |
| OFFLOAD_TRACE(3, "Address %p is already mapped on device %d\n", |
| host_ptr, device_num); |
| // Is current device pointer and offset same as existing? |
| if ((void*)ptr_data->mic_addr == device_ptr && |
| (size_t)ptr_data->alloc_disp == device_offset) { |
| return 0; |
| } else { |
| return 1; |
| } |
| } |
| |
| // Create association |
| OFFLOAD_TRACE(3, "Creating association for data: addr %p, length %lld\n", |
| host_ptr, size); |
| |
| bool is_new; |
| ptr_data = device.insert_ptr_data(host_ptr, size, is_new); |
| ptr_data->is_omp_associate = true; |
| |
| // create CPU buffer |
| OFFLOAD_TRACE(3, |
| "Creating buffer from source memory %p, length %lld\n", |
| host_ptr, size); |
| |
| // result is not checked because we can continue without cpu |
| // buffer. In this case we will use COIBufferRead/Write |
| // instead of COIBufferCopy. |
| |
| COI::BufferCreateFromMemory(size, |
| COI_BUFFER_OPENCL, |
| 0, |
| host_ptr, |
| 1, |
| &device.get_process(), |
| &ptr_data->cpu_buf); |
| |
| // create MIC buffer |
| OFFLOAD_TRACE(3, |
| "Creating buffer from sink memory: addr %p, size %lld\n", |
| (char *)device_ptr + device_offset, size); |
| res = COI::BufferCreateFromMemory(size, |
| COI_BUFFER_NORMAL, |
| COI_SINK_MEMORY, |
| device_ptr, |
| 1, |
| &device.get_process(), |
| &ptr_data->mic_buf); |
| if (res != COI_SUCCESS) { |
| ptr_data->alloc_ptr_data_lock.unlock(); |
| return 1; |
| } |
| |
| // make buffer valid on the device. |
| res = COI::BufferSetState(ptr_data->mic_buf, |
| device.get_process(), |
| COI_BUFFER_VALID, |
| COI_BUFFER_NO_MOVE, |
| 0, 0, 0); |
| if (res != COI_SUCCESS) { |
| ptr_data->alloc_ptr_data_lock.unlock(); |
| return 1; |
| } |
| |
| res = COI::BufferSetState(ptr_data->mic_buf, |
| COI_PROCESS_SOURCE, |
| COI_BUFFER_INVALID, |
| COI_BUFFER_NO_MOVE, |
| 0, 0, 0); |
| if (res != COI_SUCCESS) { |
| ptr_data->alloc_ptr_data_lock.unlock(); |
| return 1; |
| } |
| ptr_data->alloc_disp = device_offset; |
| ptr_data->alloc_ptr_data_lock.unlock(); |
| |
| return 0; |
| } |
| |
| int omp_target_disassociate_ptr( |
| void *host_ptr, |
| int device_num |
| ) __GOMP_NOTHROW |
| { |
| COIRESULT res; |
| |
| __offload_init_library(); |
| |
| OFFLOAD_TRACE(2, "omp_target_disassociate_ptr(%p, %d)\n", |
| host_ptr, device_num); |
| |
| if (device_num < -1) { |
| LIBOFFLOAD_ERROR(c_invalid_device_number); |
| exit(1); |
| } |
| |
| // Dissociating from CPU is treated as failure |
| if (device_num == -1) { |
| return 1; |
| } |
| |
| // If OpenMP allows wrap-around for device numbers, enable next line |
| //Engine& device = mic_engines[device_num % mic_engines_total]; |
| Engine& device = mic_engines[device_num]; |
| |
| // Lookup existing association in pointer table |
| PtrData* ptr_data = device.find_ptr_data(host_ptr); |
| |
| // Attempt to disassociate unassociated pointer is a failure |
| if (ptr_data == 0) { |
| return 1; |
| } |
| |
| // Destroy buffers |
| if (ptr_data->cpu_buf != 0) { |
| OFFLOAD_TRACE(3, "Destroying CPU buffer %p\n", ptr_data->cpu_buf); |
| COI::BufferDestroy(ptr_data->cpu_buf); |
| } |
| if (ptr_data->mic_buf != 0) { |
| OFFLOAD_TRACE(3, "Destroying MIC buffer %p\n", ptr_data->mic_buf); |
| COI::BufferDestroy(ptr_data->mic_buf); |
| } |
| |
| // Remove association from map |
| OFFLOAD_TRACE(3, "Removing association for addr %p\n", |
| ptr_data->cpu_addr.start()); |
| device.remove_ptr_data(ptr_data->cpu_addr.start()); |
| |
| return 0; |
| } |
| |
| // End of OpenMP 4.5 APIs |
| |
| |
| // OpenMP API wrappers |
| |
| static void omp_set_int_target( |
| TARGET_TYPE target_type, |
| int target_number, |
| int setting, |
| const char* f_name |
| ) |
| { |
| OFFLOAD ofld = OFFLOAD_TARGET_ACQUIRE(target_type, target_number, 0, NULL, |
| f_name, 0); |
| if (ofld) { |
| VarDesc vars[1] = {0}; |
| |
| vars[0].type.src = c_data; |
| vars[0].type.dst = c_data; |
| vars[0].direction.bits = c_parameter_in; |
| vars[0].size = sizeof(int); |
| vars[0].count = 1; |
| vars[0].ptr = &setting; |
| |
| OFFLOAD_OFFLOAD(ofld, f_name, 0, 1, vars, NULL, 0, 0, 0); |
| } |
| } |
| |
| static int omp_get_int_target( |
| TARGET_TYPE target_type, |
| int target_number, |
| const char * f_name |
| ) |
| { |
| int setting = 0; |
| |
| OFFLOAD ofld = OFFLOAD_TARGET_ACQUIRE(target_type, target_number, 0, NULL, |
| f_name, 0); |
| if (ofld) { |
| VarDesc vars[1] = {0}; |
| |
| vars[0].type.src = c_data; |
| vars[0].type.dst = c_data; |
| vars[0].direction.bits = c_parameter_out; |
| vars[0].size = sizeof(int); |
| vars[0].count = 1; |
| vars[0].ptr = &setting; |
| |
| OFFLOAD_OFFLOAD(ofld, f_name, 0, 1, vars, NULL, 0, 0, 0); |
| } |
| return setting; |
| } |
| |
| void omp_set_num_threads_target( |
| TARGET_TYPE target_type, |
| int target_number, |
| int num_threads |
| ) |
| { |
| omp_set_int_target(target_type, target_number, num_threads, |
| "omp_set_num_threads_target"); |
| } |
| |
| int omp_get_max_threads_target( |
| TARGET_TYPE target_type, |
| int target_number |
| ) |
| { |
| return omp_get_int_target(target_type, target_number, |
| "omp_get_max_threads_target"); |
| } |
| |
| int omp_get_num_procs_target( |
| TARGET_TYPE target_type, |
| int target_number |
| ) |
| { |
| return omp_get_int_target(target_type, target_number, |
| "omp_get_num_procs_target"); |
| } |
| |
| void omp_set_dynamic_target( |
| TARGET_TYPE target_type, |
| int target_number, |
| int num_threads |
| ) |
| { |
| omp_set_int_target(target_type, target_number, num_threads, |
| "omp_set_dynamic_target"); |
| } |
| |
| int omp_get_dynamic_target( |
| TARGET_TYPE target_type, |
| int target_number |
| ) |
| { |
| return omp_get_int_target(target_type, target_number, |
| "omp_get_dynamic_target"); |
| } |
| |
| void omp_set_nested_target( |
| TARGET_TYPE target_type, |
| int target_number, |
| int nested |
| ) |
| { |
| omp_set_int_target(target_type, target_number, nested, |
| "omp_set_nested_target"); |
| } |
| |
| int omp_get_nested_target( |
| TARGET_TYPE target_type, |
| int target_number |
| ) |
| { |
| return omp_get_int_target(target_type, target_number, |
| "omp_get_nested_target"); |
| } |
| |
| void omp_set_schedule_target( |
| TARGET_TYPE target_type, |
| int target_number, |
| omp_sched_t kind, |
| int modifier |
| ) |
| { |
| OFFLOAD ofld = OFFLOAD_TARGET_ACQUIRE(target_type, target_number, 0, NULL, |
| __func__, 0); |
| if (ofld != 0) { |
| VarDesc vars[2] = {0}; |
| |
| vars[0].type.src = c_data; |
| vars[0].type.dst = c_data; |
| vars[0].direction.bits = c_parameter_in; |
| vars[0].size = sizeof(omp_sched_t); |
| vars[0].count = 1; |
| vars[0].ptr = &kind; |
| |
| vars[1].type.src = c_data; |
| vars[1].type.dst = c_data; |
| vars[1].direction.bits = c_parameter_in; |
| vars[1].size = sizeof(int); |
| vars[1].count = 1; |
| vars[1].ptr = &modifier; |
| |
| OFFLOAD_OFFLOAD(ofld, "omp_set_schedule_target", |
| 0, 2, vars, NULL, 0, 0, 0); |
| } |
| } |
| |
| void omp_get_schedule_target( |
| TARGET_TYPE target_type, |
| int target_number, |
| omp_sched_t *kind, |
| int *modifier |
| ) |
| { |
| OFFLOAD ofld = OFFLOAD_TARGET_ACQUIRE(target_type, target_number, 0, NULL, |
| __func__, 0); |
| if (ofld != 0) { |
| VarDesc vars[2] = {0}; |
| |
| vars[0].type.src = c_data; |
| vars[0].type.dst = c_data; |
| vars[0].direction.bits = c_parameter_out; |
| vars[0].size = sizeof(omp_sched_t); |
| vars[0].count = 1; |
| vars[0].ptr = kind; |
| |
| vars[1].type.src = c_data; |
| vars[1].type.dst = c_data; |
| vars[1].direction.bits = c_parameter_out; |
| vars[1].size = sizeof(int); |
| vars[1].count = 1; |
| vars[1].ptr = modifier; |
| |
| OFFLOAD_OFFLOAD(ofld, "omp_get_schedule_target", |
| 0, 2, vars, NULL, 0, 0, 0); |
| } |
| } |
| |
| // lock API functions |
| |
| void omp_init_lock_target( |
| TARGET_TYPE target_type, |
| int target_number, |
| omp_lock_target_t *lock |
| ) |
| { |
| OFFLOAD ofld = OFFLOAD_TARGET_ACQUIRE(target_type, target_number, 0, NULL, |
| __func__, 0); |
| if (ofld != 0) { |
| VarDesc vars[1] = {0}; |
| |
| vars[0].type.src = c_data; |
| vars[0].type.dst = c_data; |
| vars[0].direction.bits = c_parameter_out; |
| vars[0].size = sizeof(omp_lock_target_t); |
| vars[0].count = 1; |
| vars[0].ptr = lock; |
| |
| OFFLOAD_OFFLOAD(ofld, "omp_init_lock_target", |
| 0, 1, vars, NULL, 0, 0, 0); |
| } |
| } |
| |
| void omp_destroy_lock_target( |
| TARGET_TYPE target_type, |
| int target_number, |
| omp_lock_target_t *lock |
| ) |
| { |
| OFFLOAD ofld = OFFLOAD_TARGET_ACQUIRE(target_type, target_number, 0, NULL, |
| __func__, 0); |
| if (ofld != 0) { |
| VarDesc vars[1] = {0}; |
| |
| vars[0].type.src = c_data; |
| vars[0].type.dst = c_data; |
| vars[0].direction.bits = c_parameter_in; |
| vars[0].size = sizeof(omp_lock_target_t); |
| vars[0].count = 1; |
| vars[0].ptr = lock; |
| |
| OFFLOAD_OFFLOAD(ofld, "omp_destroy_lock_target", |
| 0, 1, vars, NULL, 0, 0, 0); |
| } |
| } |
| |
| void omp_set_lock_target( |
| TARGET_TYPE target_type, |
| int target_number, |
| omp_lock_target_t *lock |
| ) |
| { |
| OFFLOAD ofld = OFFLOAD_TARGET_ACQUIRE(target_type, target_number, 0, NULL, |
| __func__, 0); |
| if (ofld != 0) { |
| VarDesc vars[1] = {0}; |
| |
| vars[0].type.src = c_data; |
| vars[0].type.dst = c_data; |
| vars[0].direction.bits = c_parameter_inout; |
| vars[0].size = sizeof(omp_lock_target_t); |
| vars[0].count = 1; |
| vars[0].ptr = lock; |
| |
| OFFLOAD_OFFLOAD(ofld, "omp_set_lock_target", |
| 0, 1, vars, NULL, 0, 0, 0); |
| } |
| } |
| |
| void omp_unset_lock_target( |
| TARGET_TYPE target_type, |
| int target_number, |
| omp_lock_target_t *lock |
| ) |
| { |
| OFFLOAD ofld = OFFLOAD_TARGET_ACQUIRE(target_type, target_number, 0, NULL, |
| __func__, 0); |
| if (ofld != 0) { |
| VarDesc vars[1] = {0}; |
| |
| vars[0].type.src = c_data; |
| vars[0].type.dst = c_data; |
| vars[0].direction.bits = c_parameter_inout; |
| vars[0].size = sizeof(omp_lock_target_t); |
| vars[0].count = 1; |
| vars[0].ptr = lock; |
| |
| OFFLOAD_OFFLOAD(ofld, "omp_unset_lock_target", |
| 0, 1, vars, NULL, 0, 0, 0); |
| } |
| } |
| |
| int omp_test_lock_target( |
| TARGET_TYPE target_type, |
| int target_number, |
| omp_lock_target_t *lock |
| ) |
| { |
| int result = 0; |
| |
| OFFLOAD ofld = OFFLOAD_TARGET_ACQUIRE(target_type, target_number, 0, NULL, |
| __func__, 0); |
| if (ofld != 0) { |
| VarDesc vars[2] = {0}; |
| |
| vars[0].type.src = c_data; |
| vars[0].type.dst = c_data; |
| vars[0].direction.bits = c_parameter_inout; |
| vars[0].size = sizeof(omp_lock_target_t); |
| vars[0].count = 1; |
| vars[0].ptr = lock; |
| |
| vars[1].type.src = c_data; |
| vars[1].type.dst = c_data; |
| vars[1].direction.bits = c_parameter_out; |
| vars[1].size = sizeof(int); |
| vars[1].count = 1; |
| vars[1].ptr = &result; |
| |
| OFFLOAD_OFFLOAD(ofld, "omp_test_lock_target", |
| 0, 2, vars, NULL, 0, 0, 0); |
| } |
| return result; |
| } |
| |
| // nested lock API functions |
| |
| void omp_init_nest_lock_target( |
| TARGET_TYPE target_type, |
| int target_number, |
| omp_nest_lock_target_t *lock |
| ) |
| { |
| OFFLOAD ofld = OFFLOAD_TARGET_ACQUIRE(target_type, target_number, 0, NULL, |
| __func__, 0); |
| if (ofld != 0) { |
| VarDesc vars[1] = {0}; |
| |
| vars[0].type.src = c_data; |
| vars[0].type.dst = c_data; |
| vars[0].direction.bits = c_parameter_out; |
| vars[0].size = sizeof(omp_nest_lock_target_t); |
| vars[0].count = 1; |
| vars[0].ptr = lock; |
| |
| OFFLOAD_OFFLOAD(ofld, "omp_init_nest_lock_target", |
| 0, 1, vars, NULL, 0, 0, 0); |
| } |
| } |
| |
| void omp_destroy_nest_lock_target( |
| TARGET_TYPE target_type, |
| int target_number, |
| omp_nest_lock_target_t *lock |
| ) |
| { |
| OFFLOAD ofld = OFFLOAD_TARGET_ACQUIRE(target_type, target_number, 0, NULL, |
| __func__, 0); |
| if (ofld != 0) { |
| VarDesc vars[1] = {0}; |
| |
| vars[0].type.src = c_data; |
| vars[0].type.dst = c_data; |
| vars[0].direction.bits = c_parameter_in; |
| vars[0].size = sizeof(omp_nest_lock_target_t); |
| vars[0].count = 1; |
| vars[0].ptr = lock; |
| |
| OFFLOAD_OFFLOAD(ofld, "omp_destroy_nest_lock_target", |
| 0, 1, vars, NULL, 0, 0, 0); |
| } |
| } |
| |
| void omp_set_nest_lock_target( |
| TARGET_TYPE target_type, |
| int target_number, |
| omp_nest_lock_target_t *lock |
| ) |
| { |
| OFFLOAD ofld = OFFLOAD_TARGET_ACQUIRE(target_type, target_number, 0, NULL, |
| __func__, 0); |
| if (ofld != 0) { |
| VarDesc vars[1] = {0}; |
| |
| vars[0].type.src = c_data; |
| vars[0].type.dst = c_data; |
| vars[0].direction.bits = c_parameter_inout; |
| vars[0].size = sizeof(omp_nest_lock_target_t); |
| vars[0].count = 1; |
| vars[0].ptr = lock; |
| |
| OFFLOAD_OFFLOAD(ofld, "omp_set_nest_lock_target", |
| 0, 1, vars, NULL, 0, 0, 0); |
| } |
| } |
| |
| void omp_unset_nest_lock_target( |
| TARGET_TYPE target_type, |
| int target_number, |
| omp_nest_lock_target_t *lock |
| ) |
| { |
| OFFLOAD ofld = OFFLOAD_TARGET_ACQUIRE(target_type, target_number, 0, NULL, |
| __func__, 0); |
| if (ofld != 0) { |
| VarDesc vars[1] = {0}; |
| |
| vars[0].type.src = c_data; |
| vars[0].type.dst = c_data; |
| vars[0].direction.bits = c_parameter_inout; |
| vars[0].size = sizeof(omp_nest_lock_target_t); |
| vars[0].count = 1; |
| vars[0].ptr = lock; |
| |
| OFFLOAD_OFFLOAD(ofld, "omp_unset_nest_lock_target", |
| 0, 1, vars, NULL, 0, 0, 0); |
| } |
| } |
| |
| int omp_test_nest_lock_target( |
| TARGET_TYPE target_type, |
| int target_number, |
| omp_nest_lock_target_t *lock |
| ) |
| { |
| int result = 0; |
| |
| OFFLOAD ofld = OFFLOAD_TARGET_ACQUIRE(target_type, target_number, 0, NULL, |
| __func__, 0); |
| if (ofld != 0) { |
| VarDesc vars[2] = {0}; |
| |
| vars[0].type.src = c_data; |
| vars[0].type.dst = c_data; |
| vars[0].direction.bits = c_parameter_inout; |
| vars[0].size = sizeof(omp_nest_lock_target_t); |
| vars[0].count = 1; |
| vars[0].ptr = lock; |
| |
| vars[1].type.src = c_data; |
| vars[1].type.dst = c_data; |
| vars[1].direction.bits = c_parameter_out; |
| vars[1].size = sizeof(int); |
| vars[1].count = 1; |
| vars[1].ptr = &result; |
| |
| OFFLOAD_OFFLOAD(ofld, "omp_test_nest_lock_target", |
| 0, 2, vars, NULL, 0, 0, 0); |
| } |
| return result; |
| } |
