/* * Copyright 2016 Google Inc. * * Use of this source code is governed by a BSD-style license that can * be found in the LICENSE file. * */ // // // #include <stdio.h> #include <stdlib.h> #include <string.h> #include <inttypes.h> // // // #include "common/cl/assert_cl.h" #include "common/macros.h" #include "common/util.h" // // // #include "hs_cl.h" // // // struct hs_cl { struct hs_cl_target_config config; uint32_t key_val_size; uint32_t slab_keys; uint32_t bs_slabs_log2_ru; uint32_t bc_slabs_log2_max; struct { uint32_t count; cl_kernel * bs; cl_kernel * bc; cl_kernel * fm[3]; cl_kernel * hm[3]; cl_kernel * transpose; cl_kernel all[]; } kernels; }; // // // struct hs_state { #ifndef NDEBUG cl_ulong t_total; // 0 #endif cl_command_queue cq; // key buffers cl_mem vin; cl_mem vout; // can be vin // enforces ordering on out-of-order queue cl_event wait_list[3]; // worst case uint32_t wait_list_size; // bx_ru is number of rounded up warps in vin uint32_t bx_ru; }; // // // static void hs_state_wait_list_release(struct hs_state * const state) { for (uint32_t ii=0; ii<state->wait_list_size; ii++) cl(ReleaseEvent(state->wait_list[ii])); state->wait_list_size = 0; } static void hs_state_wait_list_update(struct hs_state * const state, uint32_t const wait_list_size, cl_event const * const wait_list) { uint32_t const new_size = state->wait_list_size + wait_list_size; for (uint32_t ii=state->wait_list_size; ii<new_size; ii++) state->wait_list[ii] = wait_list[ii]; state->wait_list_size = new_size; } // // // #ifdef NDEBUG #define HS_STATE_WAIT_LIST_PROFILE(state) #define HS_STATE_WAIT_LIST_PROFILE_EX(state,wait_list_size,wait_list) #else #include <stdio.h> #define HS_STATE_WAIT_LIST_PROFILE(state) \ hs_state_wait_list_profile(state, \ state->wait_list_size, \ state->wait_list) #define HS_STATE_WAIT_LIST_PROFILE_EX(state,wait_list_size,wait_list) \ hs_state_wait_list_profile(state, \ wait_list_size, \ wait_list) static void hs_state_wait_list_profile(struct hs_state * const state, uint32_t const wait_list_size, cl_event const * const wait_list) { cl(Finish(state->cq)); cl_command_queue_properties props; cl(GetCommandQueueInfo(state->cq, CL_QUEUE_PROPERTIES, sizeof(props), &props, NULL)); for (uint32_t ii=0; ii<wait_list_size; ii++) { cl_event event = wait_list[ii]; // // profiling // cl_ulong t_start=0, t_end=0; if (props & CL_QUEUE_PROFILING_ENABLE) { // start cl(GetEventProfilingInfo(event, CL_PROFILING_COMMAND_START, sizeof(cl_ulong), &t_start, NULL)); // end cl(GetEventProfilingInfo(event, CL_PROFILING_COMMAND_END, sizeof(cl_ulong), &t_end, NULL)); state->t_total += t_end - t_start; } // // status // cl_int status; cl_command_type type; cl_get_event_info(event,&status,&type); fprintf(stdout,"%-13s, %-28s, %20"PRIu64", %20"PRIu64", %20"PRIu64", %20"PRIu64"\n", cl_get_event_command_status_string(status), cl_get_event_command_type_string(type), t_start,t_end,t_end-t_start,state->t_total); } } #endif // // // #ifdef NDEBUG #define HS_TRACE(k,g,l) #else #define HS_KERNEL_NAME_MAX 20 static void hs_trace(cl_kernel kernel, uint32_t const dim, size_t const * const global_work_size) { if (kernel == NULL) return; char name[HS_KERNEL_NAME_MAX]; cl(GetKernelInfo(kernel,CL_KERNEL_FUNCTION_NAME,HS_KERNEL_NAME_MAX,name,NULL)); fprintf(stderr,"%-19s ( %6zu, %6zu, %6zu )\n", name, global_work_size[0], dim < 2 ? 0 : global_work_size[1], dim < 3 ? 0 : global_work_size[2]); } #define HS_TRACE(k,d,g) hs_trace(k,d,g) #endif // // // static void hs_transpose(struct hs_cl const * const hs, struct hs_state * const state) { size_t const size[1] = { state->bx_ru << hs->config.slab.threads_log2 }; cl_kernel kernel = hs->kernels.transpose[0]; HS_TRACE(kernel,1,size); // // The transpose kernel operates on a single slab. For now, let's // rely on the driver to choose a workgroup size. // // size_t local_work_size[1] = { HS_SLAB_THREADS }; // cl(SetKernelArg(kernel,0,sizeof(state->vout),&state->vout)); cl_event wait_list_out[1]; cl(EnqueueNDRangeKernel(state->cq, kernel, 1, NULL, size, NULL, state->wait_list_size, state->wait_list, wait_list_out)); hs_state_wait_list_release(state); hs_state_wait_list_update(state,1,wait_list_out); HS_STATE_WAIT_LIST_PROFILE(state); } // // // static void hs_hm_enqueue(struct hs_cl const * const hs, struct hs_state * const state, uint32_t const scale_log2, uint32_t const spans, uint32_t const span_threads) { // // Note that some platforms might need to use .z on large grids // size_t const size[3] = { span_threads, spans, 1 }; cl_kernel kernel = hs->kernels.hm[scale_log2][0]; HS_TRACE(kernel,3,size); cl(SetKernelArg(kernel,0,sizeof(state->vout),&state->vout)); cl_event wait_list_out[1]; cl(EnqueueNDRangeKernel(state->cq, kernel, 3, NULL, size, NULL, state->wait_list_size, state->wait_list, wait_list_out)); hs_state_wait_list_release(state); hs_state_wait_list_update(state,1,wait_list_out); HS_STATE_WAIT_LIST_PROFILE(state); } // // // static uint32_t hs_hm(struct hs_cl const * const hs, struct hs_state * const state, uint32_t const down_slabs, uint32_t const clean_slabs_log2) { // how many scaled half-merge spans are there? uint32_t const frac_ru = (1 << clean_slabs_log2) - 1; uint32_t const spans = (down_slabs + frac_ru) >> clean_slabs_log2; // for now, just clamp to the max uint32_t const log2_rem = clean_slabs_log2 - hs->bc_slabs_log2_max; uint32_t const scale_log2 = MIN_MACRO(hs->config.merge.hm.scale_max,log2_rem); uint32_t const log2_out = log2_rem - scale_log2; // size the grid uint32_t const span_threads = hs->slab_keys << log2_out; // launch the hm kernel hs_hm_enqueue(hs, state, scale_log2, spans, span_threads); return log2_out; } // // // static void hs_bc_enqueue(struct hs_cl const * const hs, struct hs_state * const state, uint32_t const full, uint32_t const clean_slabs_log2) { size_t const size[1] = { full << hs->config.slab.threads_log2 }; cl_kernel kernel = hs->kernels.bc[clean_slabs_log2]; HS_TRACE(kernel,1,size); cl(SetKernelArg(kernel,0,sizeof(state->vout),&state->vout)); cl_event wait_list_out[1]; cl(EnqueueNDRangeKernel(state->cq, kernel, 1, NULL, size, NULL, state->wait_list_size, state->wait_list, wait_list_out)); hs_state_wait_list_release(state); hs_state_wait_list_update(state,1,wait_list_out); HS_STATE_WAIT_LIST_PROFILE(state); } // // // static void hs_bc(struct hs_cl const * const hs, struct hs_state * const state, uint32_t const down_slabs, uint32_t const clean_slabs_log2) { // block clean the minimal number of down_slabs_log2 spans uint32_t const frac_ru = (1u << clean_slabs_log2) - 1; uint32_t const full = (down_slabs + frac_ru) & ~frac_ru; // we better be capable of cleaning at least two warps !!! hs_bc_enqueue(hs,state,full,clean_slabs_log2); } // // FIXME -- some of this logic can be skipped if BS is a power-of-two // static void hs_fm_enqueue(struct hs_cl const * const hs, struct hs_state * const state, uint32_t const scale_log2, uint32_t const fm_full, uint32_t const fm_frac, uint32_t const span_threads) { // // Note that some platforms might need to use .z on large grids // uint32_t wait_list_out_size = 0; cl_event wait_list_out[2]; if (fm_full > 0) { size_t const size_full[3] = { span_threads, fm_full, 1 }; cl_kernel kernel_full = hs->kernels.fm[scale_log2][hs->bs_slabs_log2_ru-1+scale_log2]; HS_TRACE(kernel_full,3,size_full); cl(SetKernelArg(kernel_full,0,sizeof(state->vout),&state->vout)); cl(EnqueueNDRangeKernel(state->cq, kernel_full, 3, NULL, size_full, NULL, state->wait_list_size, state->wait_list, wait_list_out+wait_list_out_size++)); } if (fm_frac > 0) { size_t const offset_frac[3] = { 0, fm_full, 0 }; size_t const size_frac [3] = { span_threads, 1, 1 }; cl_kernel kernel_frac = hs->kernels.fm[scale_log2][msb_idx_u32(fm_frac)]; HS_TRACE(kernel_frac,3,size_frac); cl(SetKernelArg(kernel_frac,0,sizeof(state->vout),&state->vout)); cl(EnqueueNDRangeKernel(state->cq, kernel_frac, 3, offset_frac, size_frac, NULL, state->wait_list_size, state->wait_list, wait_list_out+wait_list_out_size++)); } hs_state_wait_list_release(state); hs_state_wait_list_update(state,wait_list_out_size,wait_list_out); HS_STATE_WAIT_LIST_PROFILE(state); } // // // static uint32_t hs_fm(struct hs_cl const * const hs, struct hs_state * const state, uint32_t * const down_slabs, uint32_t const up_scale_log2) { // // FIXME OPTIMIZATION: in previous HotSort launchers it's sometimes // a performance win to bias toward launching the smaller flip merge // kernel in order to get more warps in flight (increased // occupancy). This is useful when merging small numbers of slabs. // // Note that HS_FM_SCALE_MIN will always be 0 or 1. // // So, for now, just clamp to the max until there is a reason to // restore the fancier and probably low-impact approach. // uint32_t const scale_log2 = MIN_MACRO(hs->config.merge.fm.scale_max,up_scale_log2); uint32_t const clean_log2 = up_scale_log2 - scale_log2; // number of slabs in a full-sized scaled flip-merge span uint32_t const full_span_slabs = hs->config.block.slabs << up_scale_log2; // how many full-sized scaled flip-merge spans are there? uint32_t fm_full = state->bx_ru / full_span_slabs; uint32_t fm_frac = 0; // initialize down_slabs *down_slabs = fm_full * full_span_slabs; // how many half-size scaled + fractional scaled spans are there? uint32_t const span_rem = state->bx_ru - *down_slabs; uint32_t const half_span_slabs = full_span_slabs >> 1; // if we have over a half-span then fractionally merge it if (span_rem > half_span_slabs) { // the remaining slabs will be cleaned *down_slabs += span_rem; uint32_t const frac_rem = span_rem - half_span_slabs; uint32_t const frac_rem_pow2 = pow2_ru_u32(frac_rem); if (frac_rem_pow2 >= half_span_slabs) { // bump it up to a full span fm_full += 1; } else { // otherwise, add fractional fm_frac = MAX_MACRO(1,frac_rem_pow2 >> clean_log2); } } // size the grid uint32_t const span_threads = hs->slab_keys << clean_log2; // // launch the fm kernel // hs_fm_enqueue(hs, state, scale_log2, fm_full, fm_frac, span_threads); return clean_log2; } // // // static void hs_bs_enqueue(struct hs_cl const * const hs, struct hs_state * const state, uint32_t const full, uint32_t const frac, uint32_t const wait_list_size, cl_event * wait_list) { uint32_t wait_list_out_size = 0; cl_event wait_list_out[2]; if (full > 0) { size_t const size_full[1] = { full << hs->config.slab.threads_log2 }; cl_kernel kernel_full = hs->kernels.bs[hs->bs_slabs_log2_ru]; HS_TRACE(kernel_full,1,size_full); cl(SetKernelArg(kernel_full,0,sizeof(state->vin), &state->vin)); cl(SetKernelArg(kernel_full,1,sizeof(state->vout),&state->vout)); cl(EnqueueNDRangeKernel(state->cq, kernel_full, 1, NULL, size_full, NULL, wait_list_size, wait_list, wait_list_out+wait_list_out_size++)); } if (frac > 0) { size_t const offset_frac[1] = { full << hs->config.slab.threads_log2 }; size_t const size_frac [1] = { frac << hs->config.slab.threads_log2 }; cl_kernel kernel_frac = hs->kernels.bs[msb_idx_u32(frac)]; HS_TRACE(kernel_frac,1,size_frac); cl(SetKernelArg(kernel_frac,0,sizeof(state->vin), &state->vin)); cl(SetKernelArg(kernel_frac,1,sizeof(state->vout),&state->vout)); cl(EnqueueNDRangeKernel(state->cq, kernel_frac, 1, offset_frac, size_frac, NULL, wait_list_size, wait_list, wait_list_out+wait_list_out_size++)); } hs_state_wait_list_release(state); hs_state_wait_list_update(state,wait_list_out_size,wait_list_out); HS_STATE_WAIT_LIST_PROFILE(state); } // // // static void hs_bs(struct hs_cl const * const hs, struct hs_state * const state, uint32_t const count_padded_in, uint32_t const wait_list_size, cl_event * wait_list) { uint32_t const slabs_in = count_padded_in / hs->slab_keys; uint32_t const full = (slabs_in / hs->config.block.slabs) * hs->config.block.slabs; uint32_t const frac = slabs_in - full; hs_bs_enqueue(hs,state, full,frac, wait_list_size,wait_list); } // // // static void hs_keyset_pre_sort(struct hs_cl const * const hs, struct hs_state * const state, uint32_t const count, uint32_t const count_hi, uint32_t const wait_list_size, cl_event * wait_list, cl_event * event) { uint32_t const vin_span = count_hi - count; uint32_t const pattern = UINT32_MAX; cl(EnqueueFillBuffer(state->cq, state->vin, &pattern, sizeof(pattern), count * hs->key_val_size, vin_span * hs->key_val_size, wait_list_size, wait_list, event)); HS_STATE_WAIT_LIST_PROFILE_EX(state,1,event); } // // // static void hs_keyset_pre_merge(struct hs_cl const * const hs, struct hs_state * const state, uint32_t const count_lo, uint32_t const count_hi, uint32_t const wait_list_size, cl_event * wait_list) { uint32_t const vout_span = count_hi - count_lo; uint32_t const pattern = UINT32_MAX; // appends event to incoming wait list cl(EnqueueFillBuffer(state->cq, state->vout, &pattern, sizeof(pattern), count_lo * hs->key_val_size, vout_span * hs->key_val_size, wait_list_size, wait_list, state->wait_list+state->wait_list_size++)); HS_STATE_WAIT_LIST_PROFILE(state); } // // We want concurrent kernel execution to occur in a few places. // // The summary is: // // 1) If necessary, some max valued keys are written to the end of // the vin/vout buffers. // // 2) Blocks of slabs of keys are sorted. // // 3) If necesary, the blocks of slabs are merged until complete. // // 4) If requested, the slabs will be converted from slab ordering // to linear ordering. // // Below is the general "happens-before" relationship between HotSort // compute kernels. // // Note the diagram assumes vin and vout are different buffers. If // they're not, then the first merge doesn't include the pad_vout // event in the wait list. // // +----------+ +---------+ // | pad_vout | | pad_vin | // +----+-----+ +----+----+ // | | // | WAITFOR(pad_vin) // | | // | +-----v-----+ // | | | // | +----v----+ +----v----+ // | | bs_full | | bs_frac | // | +----+----+ +----+----+ // | | | // | +-----v-----+ // | | // | +------NO------JUST ONE BLOCK? // | / | // |/ YES // + | // | v // | END_WITH_EVENTS(bs_full,bs_frac) // | // | // WAITFOR(pad_vout,bs_full,bs_frac) >>> first iteration of loop <<< // | // | // +-----------<------------+ // | | // +-----v-----+ | // | | | // +----v----+ +----v----+ | // | fm_full | | fm_frac | | // +----+----+ +----+----+ | // | | ^ // +-----v-----+ | // | | // WAITFOR(fm_full,fm_frac) | // | | // v | // +--v--+ WAITFOR(bc) // | hm | | // +-----+ | // | | // WAITFOR(hm) | // | ^ // +--v--+ | // | bc | | // +-----+ | // | | // v | // MERGING COMPLETE?-------NO------+ // | // YES // | // v // END_WITH_EVENTS(bc) // void hs_cl_sort(struct hs_cl const * const hs, cl_command_queue cq, uint32_t const wait_list_size, cl_event * wait_list, cl_event * event, cl_mem vin, cl_mem vout, uint32_t const count, uint32_t const count_padded_in, uint32_t const count_padded_out, bool const linearize) { // is this sort in place? bool const is_in_place = (vout == NULL); // cq, buffers, wait list and slab count struct hs_state state = { #ifndef NDEBUG .t_total = 0, #endif .cq = cq, .vin = vin, .vout = is_in_place ? vin : vout, .wait_list_size = 0, .bx_ru = (count + hs->slab_keys - 1) / hs->slab_keys }; // initialize vin uint32_t const count_hi = is_in_place ? count_padded_out : count_padded_in; bool const is_pre_sort_keyset_reqd = count_hi > count; cl_event event_keyset_pre_sort[1]; // initialize any trailing keys in vin before sorting if (is_pre_sort_keyset_reqd) { hs_keyset_pre_sort(hs,&state, count,count_hi, wait_list_size,wait_list, event_keyset_pre_sort); } // initialize any trailing keys in vout before merging if (!is_in_place && (count_padded_out > count_padded_in)) { hs_keyset_pre_merge(hs,&state, count_padded_in,count_padded_out, wait_list_size,wait_list); } // // sort blocks of slabs // hs_bs(hs,&state, count_padded_in, is_pre_sort_keyset_reqd ? 1 : wait_list_size, is_pre_sort_keyset_reqd ? event_keyset_pre_sort : wait_list); // release the event if (is_pre_sort_keyset_reqd) cl(ReleaseEvent(event_keyset_pre_sort[0])); // // we're done if this was a single bs block... // // otherwise, merge sorted spans of slabs until done // if (state.bx_ru > hs->config.block.slabs) { int32_t up_scale_log2 = 1; while (true) { uint32_t down_slabs; // flip merge slabs -- return span of slabs that must be cleaned uint32_t clean_slabs_log2 = hs_fm(hs,&state, &down_slabs, up_scale_log2); // if span is gt largest slab block cleaner then half merge while (clean_slabs_log2 > hs->bc_slabs_log2_max) { clean_slabs_log2 = hs_hm(hs,&state, down_slabs, clean_slabs_log2); } // launch clean slab grid -- is it the final launch? hs_bc(hs,&state,down_slabs,clean_slabs_log2); // was this the final block clean? if (((uint32_t)hs->config.block.slabs << up_scale_log2) >= state.bx_ru) break; // otherwise, merge twice as many slabs up_scale_log2 += 1; } } // slabs or linear? if (linearize) { hs_transpose(hs,&state); } // does the caller want the final event? if (event != NULL) { *event = state.wait_list[0]; } else { cl(ReleaseEvent(state.wait_list[0])); } } // // all grids will be computed as a function of the minimum number of slabs // void hs_cl_pad(struct hs_cl const * const hs, uint32_t const count, uint32_t * const count_padded_in, uint32_t * const count_padded_out) { // // round up the count to slabs // uint32_t const slabs_ru = (count + hs->slab_keys - 1) / hs->slab_keys; uint32_t const blocks = slabs_ru / hs->config.block.slabs; uint32_t const block_slabs = blocks * hs->config.block.slabs; uint32_t const slabs_ru_rem = slabs_ru - block_slabs; uint32_t const slabs_ru_rem_ru = MIN_MACRO(pow2_ru_u32(slabs_ru_rem),hs->config.block.slabs); *count_padded_in = (block_slabs + slabs_ru_rem_ru) * hs->slab_keys; *count_padded_out = *count_padded_in; // // will merging be required? // if (slabs_ru > hs->config.block.slabs) { // more than one block uint32_t const blocks_lo = pow2_rd_u32(blocks); uint32_t const block_slabs_lo = blocks_lo * hs->config.block.slabs; uint32_t const block_slabs_rem = slabs_ru - block_slabs_lo; if (block_slabs_rem > 0) { uint32_t const block_slabs_rem_ru = pow2_ru_u32(block_slabs_rem); uint32_t const block_slabs_hi = MAX_MACRO(block_slabs_rem_ru, blocks_lo << (1 - hs->config.merge.fm.scale_min)); uint32_t const block_slabs_padded_out = MIN_MACRO(block_slabs_lo+block_slabs_hi, block_slabs_lo*2); // clamp non-pow2 blocks *count_padded_out = block_slabs_padded_out * hs->slab_keys; } } } // // // static void hs_create_kernel(cl_program program, cl_kernel * const kernel, char const * const name) { cl_int err; *kernel = clCreateKernel(program,name,&err); cl_ok(err); } static void hs_create_kernels(cl_program program, cl_kernel * kernels, char name_template[], size_t const name_template_size, uint32_t const count) { char const n_max = '0'+(char)count; for (char n = '0'; n<n_max; n++) { cl_int err; name_template[name_template_size-2] = n; *kernels++ = clCreateKernel(program,name_template,&err); cl_ok(err); } } // // // struct hs_cl * hs_cl_create(struct hs_cl_target const * const target, cl_context context, cl_device_id device_id) { // // immediately try to build the OpenCL program // bool const is_binary = (target->program[0] == 0); uint32_t const program_size = NPBTOHL_MACRO(target->program+1); cl_program program; if (is_binary) // program is a binary { cl_int status, err; size_t const bins_sizeof[] = { program_size }; unsigned char const * bins[] = { target->program+5 }; program = clCreateProgramWithBinary(context, 1, &device_id, bins_sizeof, bins, &status, &err); cl_ok(err); fprintf(stdout,"Building binary... "); fflush(stdout); cl(BuildProgram(program, 1, &device_id, NULL, NULL, NULL)); } else // program is source code { cl_int err; size_t const strings_sizeof[] = { program_size }; char const * strings[] = { (char*)target->program+5 }; program = clCreateProgramWithSource(context, 1, strings, strings_sizeof, &err); cl_ok(err); char const * const options = "-cl-std=CL1.2 -cl-fast-relaxed-math " "-cl-no-signed-zeros -cl-mad-enable " "-cl-denorms-are-zero " "-cl-kernel-arg-info"; fprintf(stdout,"Building source... "); fflush(stdout); cl(BuildProgram(program, 1, &device_id, options, NULL, NULL)); } // // we reference these values a lot // uint32_t const bs_slabs_log2_ru = msb_idx_u32(pow2_ru_u32(target->config.block.slabs)); uint32_t const bc_slabs_log2_max = msb_idx_u32(pow2_rd_u32(target->config.block.slabs)); // // how many kernels will be created? // uint32_t const count_bs = bs_slabs_log2_ru + 1; uint32_t const count_bc = bc_slabs_log2_max + 1; uint32_t count_fm[3] = { 0 }; uint32_t count_hm[3] = { 0 }; // guaranteed to be in range [0,2] for (uint32_t scale = target->config.merge.fm.scale_min; scale <= target->config.merge.fm.scale_max; scale++) { uint32_t fm_left = (target->config.block.slabs / 2) << scale; count_fm[scale] = msb_idx_u32(pow2_ru_u32(fm_left)) + 1; } // guaranteed to be in range [0,2] for (uint32_t scale = target->config.merge.hm.scale_min; scale <= target->config.merge.hm.scale_max; scale++) { count_hm[scale] = 1; } uint32_t const count_all = 1 + count_bs + count_bc + count_fm[0] + count_fm[1] + count_fm[2] + count_hm[0] + count_hm[1] + count_hm[2]; // // allocate hs_cl // struct hs_cl * hs = malloc(sizeof(*hs) + sizeof(cl_kernel) * count_all); memcpy(&hs->config,&target->config,sizeof(hs->config)); // save some frequently used calculated values hs->key_val_size = (target->config.words.key + target->config.words.val) * 4; hs->slab_keys = target->config.slab.height << target->config.slab.width_log2; hs->bs_slabs_log2_ru = bs_slabs_log2_ru; hs->bc_slabs_log2_max = bc_slabs_log2_max; // save kernel count hs->kernels.count = count_all; // // create all the kernels and release the program // cl_kernel * kernel_next = hs->kernels.all; // // BS // { hs->kernels.bs = kernel_next; char bs_name[] = { "hs_kernel_bs_X" }; hs_create_kernels(program, kernel_next, bs_name,sizeof(bs_name), count_bs); kernel_next += count_bs; } // // BC // { hs->kernels.bc = kernel_next; char bc_name[] = { "hs_kernel_bc_X" }; hs_create_kernels(program, kernel_next, bc_name,sizeof(bc_name), count_bc); kernel_next += count_bc; } // // FM // if (count_fm[0] > 0) { hs->kernels.fm[0] = kernel_next; char fm_0_name[] = { "hs_kernel_fm_0_X" }; hs_create_kernels(program, kernel_next, fm_0_name,sizeof(fm_0_name), count_fm[0]); kernel_next += count_fm[0]; } if (count_fm[1] > 0) { hs->kernels.fm[1] = kernel_next; char fm_1_name[] = { "hs_kernel_fm_1_X" }; hs_create_kernels(program, kernel_next, fm_1_name,sizeof(fm_1_name), count_fm[1]); kernel_next += count_fm[1]; } if (count_fm[2] > 0) { hs->kernels.fm[2] = kernel_next; char fm_2_name[] = { "hs_kernel_fm_2_X" }; hs_create_kernels(program, kernel_next, fm_2_name,sizeof(fm_2_name), count_fm[2]); kernel_next += count_fm[2]; } // // HM // if (count_hm[0] > 0) { hs->kernels.hm[0] = kernel_next; hs_create_kernel(program, kernel_next, "hs_kernel_hm_0"); kernel_next += count_hm[0]; } if (count_hm[1] > 0) { hs->kernels.hm[1] = kernel_next; hs_create_kernel(program, kernel_next, "hs_kernel_hm_1"); kernel_next += count_hm[1]; } if (count_hm[2] > 0) { hs->kernels.hm[2] = kernel_next; hs_create_kernel(program, kernel_next, "hs_kernel_hm_2"); kernel_next += count_hm[2]; // unnecessary } // // TRANSPOSE // { hs->kernels.transpose = kernel_next; hs_create_kernel(program, kernel_next, "hs_kernel_transpose"); kernel_next += 1; } return hs; } // // // void hs_cl_release(struct hs_cl * const hs) { for (uint32_t ii=0; ii<hs->kernels.count; ii++) cl(ReleaseKernel(hs->kernels.all[ii])); free(hs); } // // //