ccl_program_wrapper.c

Go to the documentation of this file.

/*
 * This file is part of cf4ocl (C Framework for OpenCL).
 *
 * cf4ocl is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Lesser General Public License as
 * published by the Free Software Foundation, either version 3 of the
 * License, or (at your option) any later version.
 *
 * cf4ocl is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with cf4ocl. If not, see
 * <http://www.gnu.org/licenses/>.
 * */

#include "ccl_program_wrapper.h"
#include "_ccl_abstract_dev_container_wrapper.h"
#include "_ccl_defs.h"

/* Valid file name characters. */
#define CCL_VALIDFILECHARS "abcdefghijklmnopqrstuvwxyzABCDEFGH" \
    "IJKLMNOPQRSTUVWXYZ0123456789_."

struct ccl_program {

    CCLDevContainer base;

    GHashTable* binaries;

    GHashTable* krnls;

    GHashTable* build_logs;

    gchar* build_logs_concat;

};

struct ccl_program_binary {

    unsigned char* data;

    size_t size;
};

static void ccl_program_clear_build_logs(CCLProgram* prg) {

    /* Make sure prg is not NULL. */
    g_return_if_fail(prg != NULL);

    /* If the build logs table was created... */
    if (prg->build_logs != NULL) {

        /*...free it and the included logs. */
        g_hash_table_destroy(prg->build_logs);

        /* Set to NULL to allow reuse. */
        prg->build_logs = NULL;

    }
}

static void ccl_program_release_fields(CCLProgram* prg) {

    /* Make sure prg wrapper object is not NULL. */
    g_return_if_fail(prg != NULL);

    /* Release devices. */
    ccl_dev_container_release_devices((CCLDevContainer*) prg);

    /* If the kernels table was created...*/
    if (prg->krnls != NULL) {

        /* ...free the kernel table and reduce reference count of
         * kernels in table (this is done automatically by the
         * ccl_kernel_destroy() function passed as a destructor
         * parameter during table creation). */
        g_hash_table_destroy(prg->krnls);

    }

    /* If the binaries table was created... */
    if (prg->binaries != NULL) {

        /*...free it and the included binaries. */
        g_hash_table_destroy(prg->binaries);

    }

    /* Destroy table of build logs. */
    ccl_program_clear_build_logs(prg);

    /* If there is a concatenated build log... */
    if (prg->build_logs_concat != NULL) {

        /*...free it. */
        g_free(prg->build_logs_concat);

    }

}

static CCLWrapperInfo* ccl_program_get_cldevices(
    CCLDevContainer* devcon, CCLErr** err) {

    return ccl_program_get_info(devcon, CL_PROGRAM_DEVICES, err);
}

static CCLProgramBinary* ccl_program_binary_new(
    unsigned char* data, size_t size) {

    CCLProgramBinary* pbin = g_slice_new(CCLProgramBinary);

    pbin->data = data;
    pbin->size = size;

    return pbin;

}

static void ccl_program_binary_destroy(CCLProgramBinary* pbin) {

    /* Make sure pbin is not NULL. */
    g_return_if_fail(pbin != NULL);

    if (pbin->size > 0)
        g_free(pbin->data);
    g_slice_free(CCLProgramBinary, pbin);

}

#define ccl_program_binary_new_empty() ccl_program_binary_new(NULL, 0)

CCL_EXPORT
CCLProgram* ccl_program_new_wrap(cl_program program) {

    return (CCLProgram*) ccl_wrapper_new(
        CCL_PROGRAM, (void*) program, sizeof(CCLProgram));

}

CCL_EXPORT
void ccl_program_destroy(CCLProgram* prg) {

    ccl_wrapper_unref((CCLWrapper*) prg, sizeof(CCLProgram),
        (ccl_wrapper_release_fields) ccl_program_release_fields,
        (ccl_wrapper_release_cl_object) clReleaseProgram, NULL);

}

CCL_EXPORT
CCLProgram* ccl_program_new_from_source_file(CCLContext* ctx,
    const char* filename, CCLErr** err) {

    return ccl_program_new_from_source_files(ctx, 1, &(filename), err);

}

CCL_EXPORT
CCLProgram* ccl_program_new_from_source_files(CCLContext* ctx,
    cl_uint count, const char** filenames, CCLErr** err) {

    /* Make sure ctx is not NULL. */
    g_return_val_if_fail(ctx != NULL, NULL);
    /* Make sure err is NULL or it is not set. */
    g_return_val_if_fail(err == NULL || *err == NULL, NULL);
    /* Make sure filenames is not NULL. */
    g_return_val_if_fail(filenames != NULL, NULL);
    /* Make sure count > 0. */
    g_return_val_if_fail(count > 0, NULL);

    /* Error reporting object. */
    CCLErr* err_internal = NULL;
    /* Program wrapper object to return. */
    CCLProgram* prg = NULL;
    /* Source files contents. */
    gchar** strings = NULL;

    /* Allocate space for the specified number of source file
     * strings. */
    strings = g_slice_alloc0(count * sizeof(gchar*));

    /* Read source files contents. */
    for (cl_uint i = 0; i < count; ++i) {

        g_file_get_contents(
            filenames[i], &strings[i], NULL, &err_internal);
        ccl_if_err_propagate_goto(err, err_internal, error_handler);

    }

    /* Create program from sources. */
    prg = ccl_program_new_from_sources(ctx, count,
        (const char**) strings, NULL, &err_internal);
    ccl_if_err_propagate_goto(err, err_internal, error_handler);

    /* If we got here, everything is OK. */
    g_assert(err == NULL || *err == NULL);
    goto finish;

error_handler:

    /* If we got here there was an error, verify that it is so. */
    g_assert(err == NULL || *err != NULL);

finish:

    /* Free stuff. */
    if (strings != NULL) {
        for (cl_uint i = 0; i < count; ++i) {
            if (strings[i] != NULL) {
                g_free(strings[i]);
            }
        }
        g_slice_free1(count * sizeof(gchar*), strings);
    }

    /* Return prg. */
    return prg;

}

CCL_EXPORT
CCLProgram* ccl_program_new_from_source(CCLContext* ctx,
    const char* string, CCLErr** err) {

    return ccl_program_new_from_sources(ctx, 1, &string, NULL, err);
}

CCL_EXPORT
CCLProgram* ccl_program_new_from_sources(CCLContext* ctx,
    cl_uint count, const char** strings, const size_t* lengths,
    CCLErr** err) {

    /* Make sure ctx is not NULL. */
    g_return_val_if_fail(ctx != NULL, NULL);
    /* Make sure err is NULL or it is not set. */
    g_return_val_if_fail(err == NULL || *err == NULL, NULL);

    cl_int ocl_status;
    cl_program program = NULL;
    CCLProgram* prg = NULL;

    /* Build program from sources. */
    program = clCreateProgramWithSource(
        ccl_context_unwrap(ctx), count, strings, lengths, &ocl_status);
    ccl_if_err_create_goto(*err, CCL_OCL_ERROR,
        CL_SUCCESS != ocl_status, ocl_status, error_handler,
        "%s: unable to create cl_program with source (OpenCL error %d: %s).",
        CCL_STRD, ocl_status, ccl_err(ocl_status));

    /* Wrap OpenCL program object. */
    prg = ccl_program_new_wrap(program);

    /* If we got here, everything is OK. */
    g_assert(err == NULL || *err == NULL);
    goto finish;

error_handler:

    /* If we got here there was an error, verify that it is so. */
    g_assert(err == NULL || *err != NULL);

finish:

    /* Return prg. */
    return prg;

}

CCL_EXPORT
CCLProgram* ccl_program_new_from_binary_file(CCLContext* ctx,
    CCLDevice* dev, const char* filename, cl_int *binary_status,
    CCLErr** err) {

    return ccl_program_new_from_binary_files(
        ctx, 1, &(dev), &(filename), binary_status, err);
}

CCL_EXPORT
CCLProgram* ccl_program_new_from_binary_files(CCLContext* ctx,
    cl_uint num_devices, CCLDevice* const* devs, const char** filenames,
    cl_int *binary_status, CCLErr** err) {

    /* Make sure err is NULL or it is not set. */
    g_return_val_if_fail(err == NULL || *err == NULL, NULL);
    /* Make sure ctx is not NULL. */
    g_return_val_if_fail(ctx != NULL, NULL);
    /* Make sure devs is not NULL. */
    g_return_val_if_fail(devs != NULL, NULL);
    /* Make sure filenames is not NULL. */
    g_return_val_if_fail(filenames != NULL, NULL);
    /* Make sure num_devices > 0. */
    g_return_val_if_fail(num_devices > 0, NULL);

    CCLErr* err_internal = NULL;
    CCLProgramBinary** bins = NULL;
    CCLProgram* prg = NULL;

    /* Open files and create binaries. */
    bins = g_slice_alloc0(num_devices * sizeof(CCLProgramBinary*));
    for (cl_uint i = 0; i < num_devices; ++i) {
        bins[i] = ccl_program_binary_new_empty();
        g_file_get_contents(filenames[i], (char**) &bins[i]->data,
            &bins[i]->size, &err_internal);
        ccl_if_err_propagate_goto(err, err_internal, error_handler);
    }

    /* Create program. */
    prg = ccl_program_new_from_binaries(
        ctx, num_devices, devs, bins, binary_status, &err_internal);
    ccl_if_err_propagate_goto(err, err_internal, error_handler);

    /* If we got here, everything is OK. */
    g_assert(err == NULL || *err == NULL);
    goto finish;

error_handler:

    /* If we got here there was an error, verify that it is so. */
    g_assert(err == NULL || *err != NULL);

finish:

    /* Free stuff if necessary. */
    if (bins != NULL) {
        for (cl_uint i = 0; i < num_devices; ++i) {
            if (bins[i] != NULL) {
                ccl_program_binary_destroy(bins[i]);
            }
        }
        g_slice_free1(num_devices * sizeof(CCLProgramBinary*), bins);
    }

    /* Return prg. */
    return prg;

}

CCL_EXPORT
CCLProgram* ccl_program_new_from_binary(CCLContext* ctx, CCLDevice* dev,
    CCLProgramBinary* binary, cl_int *binary_status, CCLErr** err) {

    return ccl_program_new_from_binaries(ctx, 1, &dev, &binary,
        binary_status, err);
}

CCL_EXPORT
CCLProgram* ccl_program_new_from_binaries(CCLContext* ctx,
    cl_uint num_devices, CCLDevice* const* devs, CCLProgramBinary** bins,
    cl_int *binary_status, CCLErr** err) {

    /* Make sure err is NULL or it is not set. */
    g_return_val_if_fail(err == NULL || *err == NULL, NULL);
    /* Make sure ctx is not NULL. */
    g_return_val_if_fail(ctx != NULL, NULL);
    /* Make sure devs is not NULL. */
    g_return_val_if_fail(devs != NULL, NULL);
    /* Make sure num_devices > 0. */
    g_return_val_if_fail(num_devices > 0, NULL);

    cl_int ocl_status;
    CCLProgram* prg = NULL;
    cl_program program = NULL;
    cl_device_id* device_list = NULL;
    size_t* lengths = NULL;
    unsigned char** bins_raw = NULL;

    /* Unwrap devices, binaries and lengths. */
    device_list = g_slice_alloc(num_devices * sizeof(cl_device_id));
    lengths = g_slice_alloc(num_devices * sizeof(size_t));
    bins_raw = g_slice_alloc(num_devices * sizeof(unsigned char*));
    for (guint i = 0; i < num_devices; ++i) {
        device_list[i] = ccl_device_unwrap(devs[i]);
        lengths[i] = bins[i]->size;
        bins_raw[i] = (unsigned char*) bins[i]->data;
    }

    /* Create program. */
    program = clCreateProgramWithBinary(ccl_context_unwrap(ctx),
        num_devices, device_list, lengths,
        (const unsigned char**) bins_raw,
        binary_status, &ocl_status);
    ccl_if_err_create_goto(*err, CCL_OCL_ERROR,
        CL_SUCCESS != ocl_status, ocl_status, error_handler,
        "%s: unable to create cl_program from binaries (OpenCL error %d: %s).",
        CCL_STRD, ocl_status, ccl_err(ocl_status));

    /* Wrap OpenCL program object. */
    prg = ccl_program_new_wrap(program);

    /* If we got here, everything is OK. */
    g_assert(err == NULL || *err == NULL);
    goto finish;

error_handler:

    /* If we got here there was an error, verify that it is so. */
    g_assert(err == NULL || *err != NULL);

finish:

    /* Free stuff if necessary. */
    if (device_list != NULL)
        g_slice_free1(num_devices * sizeof(cl_device_id), device_list);
    if (lengths != NULL)
        g_slice_free1(num_devices * sizeof(size_t), lengths);
    if (bins_raw != NULL)
        g_slice_free1(num_devices * sizeof(unsigned char*), bins_raw);

    /* Return prg. */
    return prg;

}

CCL_EXPORT
CCLProgram* ccl_program_new_from_built_in_kernels(CCLContext* ctx,
    cl_uint num_devices, CCLDevice* const* devs, const char *kernel_names,
    CCLErr** err) {

    /* Make sure ctx is not NULL. */
    g_return_val_if_fail(ctx != NULL, NULL);
    /* Make sure err is NULL or it is not set. */
    g_return_val_if_fail(err == NULL || *err == NULL, NULL);
    /* Make sure num_devices > 0. */
    g_return_val_if_fail(num_devices > 0, NULL);

    /* OpenCL function return status. */
    cl_int ocl_status;
    /* OpenCL program object. */
    cl_program program = NULL;
    /* Program wrapper object. */
    CCLProgram* prg = NULL;
    /* List of cl_device_id objects. */
    cl_device_id* device_list = NULL;
    /* OpenCL version of underlying platform. */
    cl_uint ocl_ver;
    /* Internal error handling object. */
    CCLErr* err_internal = NULL;

#ifndef CL_VERSION_1_2

    /* Mark unused variables to avoid compiler warnings. */
    CCL_UNUSED(program);
    CCL_UNUSED(devs);
    CCL_UNUSED(kernel_names);
    CCL_UNUSED(ocl_status);
    CCL_UNUSED(ocl_ver);
    CCL_UNUSED(err_internal);

    /* If cf4ocl was not compiled with support for OpenCL >= 1.2, always throw
     * error. */
    ccl_if_err_create_goto(*err, CCL_ERROR, TRUE,
        CCL_ERROR_UNSUPPORTED_OCL, error_handler,
        "%s: Program creation from built-in kernels requires cf4ocl to be "
        "deployed with support for OpenCL version 1.2 or newer.",
        CCL_STRD);

#else

    /* Check that context platform is >= OpenCL 1.2 */
    ocl_ver = ccl_context_get_opencl_version(ctx, &err_internal);
    ccl_if_err_propagate_goto(err, err_internal, error_handler);

    /* If OpenCL version is not >= 1.2, throw error. */
    ccl_if_err_create_goto(*err, CCL_ERROR, ocl_ver < 120,
        CCL_ERROR_UNSUPPORTED_OCL, error_handler,
        "%s: Program creation with built-in kernels requires OpenCL " \
        "version 1.2 or newer.", CCL_STRD);

    /* Unwrap devices. */
    device_list = g_slice_alloc(num_devices * sizeof(cl_device_id));
    for (guint i = 0; i < num_devices; ++i) {
        device_list[i] = ccl_device_unwrap(devs[i]);
    }

    /* Create program. */
    program = clCreateProgramWithBuiltInKernels(
        ccl_context_unwrap(ctx), num_devices, device_list, kernel_names,
        &ocl_status);

    /* Create kernel from built-in kernels. */
    ccl_if_err_create_goto(*err, CCL_OCL_ERROR,
        CL_SUCCESS != ocl_status, ocl_status, error_handler,
        "%s: unable to create cl_program from built-in kernels "
        "(OpenCL error %d: %s).",
        CCL_STRD, ocl_status, ccl_err(ocl_status));

    /* Wrap OpenCL program object. */
    prg = ccl_program_new_wrap(program);

#endif

    /* If we got here, everything is OK. */
    g_assert(err == NULL || *err == NULL);
    goto finish;

error_handler:

    /* If we got here there was an error, verify that it is so. */
    g_assert(err == NULL || *err != NULL);

finish:

    /* Free stuff if necessary. */
    if (device_list != NULL)
        g_slice_free1(num_devices * sizeof(cl_device_id), device_list);

    /* Return prg. */
    return prg;

}

CCL_EXPORT
cl_bool ccl_program_build(
    CCLProgram* prg, const char* options, CCLErr** err) {

    return ccl_program_build_full(
        prg, 0, NULL, options, NULL, NULL, err);
}

CCL_EXPORT
cl_bool ccl_program_build_full(CCLProgram* prg,
    cl_uint num_devices, CCLDevice* const* devs, const char *options,
    ccl_program_callback pfn_notify, void *user_data, CCLErr** err) {

    /* Make sure prg is not NULL. */
    g_return_val_if_fail(prg != NULL, CL_FALSE);
    /* Make sure err is NULL or it is not set. */
    g_return_val_if_fail(err == NULL || *err == NULL, CL_FALSE);

    /* Array of unwrapped devices. */
    cl_device_id* cl_devices = NULL;
    /* Status of OpenCL function call. */
    cl_int ocl_status;
    /* Result of function call. */
    cl_bool result;

    /* Clear build logs cache. */
    ccl_program_clear_build_logs(prg);

    /* Check if its necessary to unwrap devices. */
    if ((devs != NULL) && (num_devices > 0)) {
        cl_devices = g_slice_alloc0(sizeof(cl_device_id) * num_devices);
        /* Unwrap devices. */
        for (guint i = 0; i < num_devices; ++i)
            cl_devices[i] = ccl_device_unwrap(devs[i]);
    }

    /* Build program. */
    ocl_status = clBuildProgram(ccl_program_unwrap(prg),
        num_devices, cl_devices, options, pfn_notify, user_data);

    /* Check for any other errors. */
    ccl_if_err_create_goto(*err, CCL_OCL_ERROR,
        CL_SUCCESS != ocl_status, ocl_status, error_handler,
        "%s: unable to build program (OpenCL error %d: %s).",
        CCL_STRD, ocl_status, ccl_err(ocl_status));

    /* If we got here, everything is OK. */
    g_assert(err == NULL || *err == NULL);
    result = CL_TRUE;
    goto finish;

error_handler:

    /* If we got here there was an error, verify that it is so. */
    g_assert(err == NULL || *err != NULL);

    /* Bad result. */
    result = CL_FALSE;

finish:

    /* Check if necessary to release array of unwrapped devices. */
    if (cl_devices != NULL) {
        g_slice_free1(sizeof(cl_device_id) * num_devices, cl_devices);
    }

    /* Return result of function call. */
    return result;

}

CCL_EXPORT
const char* ccl_program_get_build_log(CCLProgram* prg, CCLErr** err) {

    /* Make sure prg is not NULL. */
    g_return_val_if_fail(prg != NULL, NULL);
    /* Make sure err is NULL or it is not set. */
    g_return_val_if_fail((err) == NULL || *(err) == NULL, NULL);

    /* Counter. */
    cl_uint i = 0;
    /* Number of devices in program. */
    cl_uint num_devs = 0;
    /* Current device. */
    CCLDevice* dev = NULL;
    /* Name of current device. */
    char* dev_name;
    /* Build log for current device.*/
    const char* build_log;
    /* Internal error handling object. */
    CCLErr* err_internal = NULL;

    /* Complete build log string object. */
    GString* build_log_obj = g_string_new("");

    /* Clear any previously obtained concatenated build log. */
    if (prg->build_logs_concat) {
        g_free(prg->build_logs_concat);
        prg->build_logs_concat = NULL;
    }

    /* How many devices in program? */
    num_devs = ccl_program_get_num_devices(prg, &err_internal);
    ccl_if_err_propagate_goto(err, err_internal, error_handler);

    /* For each device in program: */
    for (i = 0; i < num_devs; i++) {

        /* Get the respective device wrapper. */
        dev = ccl_program_get_device(prg, i, &err_internal);
        ccl_if_err_propagate_goto(err, err_internal, error_handler);

        /* Is it a valid device? */
        if (dev) {

            /* If so, get its name. */
            dev_name = ccl_device_get_info_array(
                dev, CL_DEVICE_NAME, char*, &err_internal);
            ccl_if_err_propagate_goto(err, err_internal, error_handler);

            /* Get the respective build log. */
            build_log = ccl_program_get_device_build_log(
                prg, dev, &err_internal);
            ccl_if_err_propagate_goto(err, err_internal, error_handler);

            /* Append build log to string of concatenated build logs. */
            g_string_append_printf(build_log_obj, "\n### Build log "
                "for device '%s'\n\n%s\n\n", dev_name,
                build_log != NULL ? build_log : "Not available");
        }
    }

    /* Add the newly concatenated build log. */
    prg->build_logs_concat = build_log_obj->str;

    /* Release the string object (but not the underlying character array). */
    g_string_free(build_log_obj, FALSE);

    /* If we got here, everything is OK. */
    g_assert(err == NULL || *err == NULL);
    goto finish;

error_handler:

    /* If we got here there was an error, verify that it is so. */
    g_assert(err == NULL || *err != NULL);

finish:

    /* Return concatenated build log. */
    return (const char*) prg->build_logs_concat;
}

CCL_EXPORT
const char* ccl_program_get_device_build_log(
    CCLProgram* prg, CCLDevice* dev, CCLErr** err) {

    /* Make sure prg is not NULL. */
    g_return_val_if_fail(prg != NULL, NULL);
    /* Make sure dev is not NULL. */
    g_return_val_if_fail(dev != NULL, NULL);
    /* Make sure err is NULL or it is not set. */
    g_return_val_if_fail(err == NULL || *err == NULL, NULL);

    /* Build log for current device.*/
    char* build_log_dev = NULL;
    /* Error reporting object. */
    CCLErr* err_internal = NULL;

    /* Is build log for this device already in cache? */
    if ((prg->build_logs == NULL) || ((build_log_dev = (char*)
        g_hash_table_lookup(prg->build_logs, (gconstpointer) dev)) == NULL)) {

        /* Build log for the specified device is not in cache, get it. */
        build_log_dev = ccl_program_get_build_info_array(
            prg, dev, CL_PROGRAM_BUILD_LOG, char*, &err_internal);
        ccl_if_err_propagate_goto(err, err_internal, error_handler);

        /* If build log is not empty, keep it in build logs cache. */
        if ((build_log_dev != NULL) && (strlen(build_log_dev) > 0)) {

            /* Check if build logs cache is initialized, if not, initialize
             * it. */
            if (!prg->build_logs) {
                prg->build_logs = g_hash_table_new(
                    g_direct_hash, g_direct_equal);
            }

            /* Keep build log for current device in build logs cache. Previous
             * logs are freed automatically. */
            g_hash_table_insert(prg->build_logs,
                (gpointer) dev,
                (gpointer) build_log_dev);

        }
    }

    /* If we got here, everything is OK. */
    g_assert(err == NULL || *err == NULL);
    goto finish;

error_handler:

    /* If we got here there was an error, verify that it is so. */
    g_assert(err == NULL || *err != NULL);

finish:

    /* Return build log for the specified device, if it exists. */
    return build_log_dev;

}

CCL_EXPORT
cl_bool ccl_program_compile(CCLProgram* prg, cl_uint num_devices,
    CCLDevice* const* devs, const char* options, cl_uint num_input_headers,
    CCLProgram** prg_input_headers, const char** header_include_names,
    ccl_program_callback pfn_notify, void* user_data, CCLErr** err) {

    /* Make sure prg is not NULL. */
    g_return_val_if_fail(prg != NULL, CL_FALSE);
    /* Make sure err is NULL or it is not set. */
    g_return_val_if_fail(err == NULL || *err == NULL, CL_FALSE);

    /* Array of unwrapped devices. */
    cl_device_id* cl_devices = NULL;
    /* Array of unwrapped programs (input headers). */
    cl_program* input_headers = NULL;
    /* Status of OpenCL function call. */
    cl_int ocl_status;
    /* Result of function call. */
    cl_bool result;
    /* OpenCL version of underlying platform. */
    cl_uint ocl_ver;
    /* Internal error handling object. */
    CCLErr* err_internal = NULL;

#ifndef CL_VERSION_1_2

    /* Mark unused variables to avoid compiler warnings. */
    CCL_UNUSED(err_internal);
    CCL_UNUSED(ocl_ver);
    CCL_UNUSED(ocl_status);
    CCL_UNUSED(devs);
    CCL_UNUSED(options);
    CCL_UNUSED(prg_input_headers);
    CCL_UNUSED(header_include_names);
    CCL_UNUSED(pfn_notify);
    CCL_UNUSED(user_data);

    /* If cf4ocl was not compiled with support for OpenCL >= 1.2, always throw
     * error. */
    ccl_if_err_create_goto(*err, CCL_ERROR, TRUE,
        CCL_ERROR_UNSUPPORTED_OCL, error_handler,
        "%s: Program compilation requires cf4ocl to be deployed with support "
        "for OpenCL version 1.2 or newer.",
        CCL_STRD);

#else

    /* Check that context platform is >= OpenCL 1.2 */
    ocl_ver = ccl_program_get_opencl_version(prg, &err_internal);
    ccl_if_err_propagate_goto(err, err_internal, error_handler);

    /* If OpenCL version is not >= 1.2, throw error. */
    ccl_if_err_create_goto(*err, CCL_ERROR, ocl_ver < 120,
        CCL_ERROR_UNSUPPORTED_OCL, error_handler,
        "%s: Program compilation requires OpenCL version 1.2 or newer.",
        CCL_STRD);

    /* Clear build logs cache. */
    ccl_program_clear_build_logs(prg);

    /* Check if its necessary to unwrap devices. */
    if ((devs != NULL) && (num_devices > 0)) {
        cl_devices = g_slice_alloc0(sizeof(cl_device_id) * num_devices);
        /* Unwrap devices. */
        for (guint i = 0; i < num_devices; ++i)
            cl_devices[i] = ccl_device_unwrap(devs[i]);
    }

    /* Check if its necessary to unwrap programs (input headers). */
    if ((prg_input_headers != NULL) && (num_input_headers > 0)) {
        input_headers =
            g_slice_alloc0(sizeof(cl_program) * num_input_headers);
        /* Unwrap program objects. */
        for (guint i = 0; i < num_input_headers; ++i)
            input_headers[i] = ccl_program_unwrap(prg_input_headers[i]);
    }

    /* Compile program. */
    ocl_status = clCompileProgram(ccl_program_unwrap(prg), num_devices,
        (const cl_device_id*) cl_devices, options, num_input_headers,
        (const cl_program*) input_headers, header_include_names,
        pfn_notify, user_data);

    /* Check for errors. */
    ccl_if_err_create_goto(*err, CCL_OCL_ERROR,
        CL_SUCCESS != ocl_status, ocl_status, error_handler,
        "%s: unable to compile program (OpenCL error %d: %s).",
        CCL_STRD, ocl_status, ccl_err(ocl_status));

#endif

    /* If we got here, everything is OK. */
    g_assert(err == NULL || *err == NULL);
    result = CL_TRUE;
    goto finish;

error_handler:

    /* If we got here there was an error, verify that it is so. */
    g_assert(err == NULL || *err != NULL);

    /* Bad result. */
    result = CL_FALSE;

finish:

    /* Check if necessary to release array of unwrapped devices. */
    if (cl_devices != NULL) {
        g_slice_free1(sizeof(cl_device_id) * num_devices, cl_devices);
    }
    /* Check if necessary to release array of input headers (programs). */
    if (input_headers != NULL) {
        g_slice_free1(
            sizeof(cl_program) * num_input_headers, input_headers);
    }

    /* Return result of function call. */
    return result;

}

CCL_EXPORT
CCLProgram* ccl_program_link(CCLContext* ctx, cl_uint num_devices,
    CCLDevice* const* devs, const char* options, cl_uint num_input_programs,
    CCLProgram** input_prgs, ccl_program_callback pfn_notify,
    void* user_data, CCLErr** err) {

    /* Make sure ctx is not NULL. */
    g_return_val_if_fail(ctx != NULL, CL_FALSE);
    /* Make sure err is NULL or it is not set. */
    g_return_val_if_fail(err == NULL || *err == NULL, CL_FALSE);

    /* Array of unwrapped devices. */
    cl_device_id* cl_devices = NULL;
    /* Array of unwrapped input programs. */
    cl_program* input_programs = NULL;
    /* Status of OpenCL function call. */
    cl_int ocl_status;
    /* Program wrapper object to create. */
    CCLProgram* prg = NULL;
    /* OpenCL program object to create and wrap. */
    cl_program program = NULL;
    /* OpenCL version of underlying platform. */
    cl_uint ocl_ver;
    /* Internal error handling object. */
    CCLErr* err_internal = NULL;

#ifndef CL_VERSION_1_2

    /* Mark unused variables to avoid compiler warnings. */
    CCL_UNUSED(err_internal);
    CCL_UNUSED(ocl_ver);
    CCL_UNUSED(program);
    CCL_UNUSED(ocl_status);
    CCL_UNUSED(devs);
    CCL_UNUSED(options);
    CCL_UNUSED(input_prgs);
    CCL_UNUSED(pfn_notify);
    CCL_UNUSED(user_data);

    /* If cf4ocl was not compiled with support for OpenCL >= 1.2, always throw
     * error. */
    ccl_if_err_create_goto(*err, CCL_ERROR, TRUE,
        CCL_ERROR_UNSUPPORTED_OCL, error_handler,
        "%s: Program linking requires cf4ocl to be deployed with support "
        "for OpenCL version 1.2 or newer.",
        CCL_STRD);

#else

    /* Check that context platform is >= OpenCL 1.2 */
    ocl_ver = ccl_context_get_opencl_version(ctx, &err_internal);
    ccl_if_err_propagate_goto(err, err_internal, error_handler);

    /* If OpenCL version is not >= 1.2, throw error. */
    ccl_if_err_create_goto(*err, CCL_ERROR, ocl_ver < 120,
        CCL_ERROR_UNSUPPORTED_OCL, error_handler,
        "%s: Program linking requires OpenCL version 1.2 or newer.",
        CCL_STRD);

    /* Check if its necessary to unwrap devices. */
    if ((devs != NULL) && (num_devices > 0)) {
        cl_devices = g_slice_alloc0(sizeof(cl_device_id) * num_devices);
        /* Unwrap devices. */
        for (guint i = 0; i < num_devices; ++i)
            cl_devices[i] = ccl_device_unwrap(devs[i]);
    }

    /* Check if its necessary to unwrap input programs. */
    if ((input_prgs != NULL) && (num_input_programs > 0)) {
        input_programs =
            g_slice_alloc0(sizeof(cl_program) * num_input_programs);
        /* Unwrap program objects. */
        for (guint i = 0; i < num_input_programs; ++i)
            input_programs[i] = ccl_program_unwrap(input_prgs[i]);
    }

    /* Link program. */
    program = clLinkProgram(ccl_context_unwrap(ctx), num_devices,
        (const cl_device_id*) cl_devices, options, num_input_programs,
        (const cl_program*) input_programs, pfn_notify, user_data,
        &ocl_status);

    /* Check for errors. */
    ccl_if_err_create_goto(*err, CCL_OCL_ERROR,
        CL_SUCCESS != ocl_status, ocl_status, error_handler,
        "%s: unable to link program (OpenCL error %d: %s).",
        CCL_STRD, ocl_status, ccl_err(ocl_status));

    /* Wrap OpenCL program object. */
    prg = ccl_program_new_wrap(program);

#endif

    /* If we got here, everything is OK. */
    g_assert(err == NULL || *err == NULL);
    goto finish;

error_handler:

    /* If we got here there was an error, verify that it is so. */
    g_assert(err == NULL || *err != NULL);

finish:

    /* Check if necessary to release array of unwrapped devices. */
    if (cl_devices != NULL) {
        g_slice_free1(sizeof(cl_device_id) * num_devices, cl_devices);
    }
    /* Check if necessary to release array of input programs. */
    if (input_programs != NULL) {
        g_slice_free1(
            sizeof(cl_program) * num_input_programs, input_programs);
    }

    /* Return linked program wrapper. */
    return prg;

}


CCL_EXPORT
cl_uint ccl_program_get_opencl_version(CCLProgram* prg, CCLErr** err) {

    /* Make sure number prg is not NULL. */
    g_return_val_if_fail(prg != NULL, 0);
    /* Make sure err is NULL or it is not set. */
    g_return_val_if_fail(err == NULL || *err == NULL, 0);

    cl_context context;
    CCLContext* ctx;
    CCLErr* err_internal = NULL;
    cl_uint ocl_ver;

    /* Get cl_context object for this program. */
    context = ccl_program_get_info_scalar(
        prg, CL_PROGRAM_CONTEXT, cl_context, &err_internal);
    ccl_if_err_propagate_goto(err, err_internal, error_handler);

    /* Get context wrapper. */
    ctx = ccl_context_new_wrap(context);

    /* Get OpenCL version. */
    ocl_ver = ccl_context_get_opencl_version(ctx, &err_internal);
    ccl_if_err_propagate_goto(err, err_internal, error_handler);

    /* Unref. the context wrapper. */
    ccl_context_unref(ctx);

    /* If we got here, everything is OK. */
    g_assert(err == NULL || *err == NULL);
    goto finish;

error_handler:

    /* If we got here there was an error, verify that it is so. */
    g_assert(err == NULL || *err != NULL);
    ocl_ver = 0;

finish:

    /* Return event wrapper. */
    return ocl_ver;

}

CCL_EXPORT
CCLKernel* ccl_program_get_kernel(
    CCLProgram* prg, const char* kernel_name, CCLErr** err) {

    /* Make sure err is NULL or it is not set. */
    g_return_val_if_fail((err) == NULL || *(err) == NULL, NULL);
    /* Make sure prg is not NULL. */
    g_return_val_if_fail(prg != NULL, NULL);
    /* Make sure kernel_name is not NULL. */
    g_return_val_if_fail(kernel_name != NULL, NULL);

    /* Internal error reporting object. */
    CCLErr* err_internal = NULL;
    /* Kernel wrapper object. */
    CCLKernel* krnl = NULL;

    /* If kernels table is not yet initialized, then
     * initialize it. */
    if (prg->krnls == NULL) {
        prg->krnls = g_hash_table_new_full(g_str_hash, g_str_equal,
            NULL, (GDestroyNotify) ccl_kernel_destroy);
    }

    /* Check if requested kernel is already present in the kernels
     * table. */
    if (g_hash_table_contains(
        prg->krnls, kernel_name)) {

        /* If so, retrieve it from there. */
        krnl = g_hash_table_lookup(
            prg->krnls, kernel_name);

    } else {

        /* Otherwise, get it from OpenCL program object.*/
        krnl = ccl_kernel_new(prg, kernel_name, &err_internal);
        ccl_if_err_propagate_goto(err, err_internal, error_handler);

        /* Keep new kernel wrapper in table. */
        g_hash_table_insert(prg->krnls, (gpointer) kernel_name, krnl);

    }

    /* If we got here, everything is OK. */
    g_assert(err == NULL || *err == NULL);
    goto finish;

error_handler:

    /* If we got here there was an error, verify that it is so. */
    g_assert(err == NULL || *err != NULL);

finish:

    /* Return kernel wrapper. */
    return krnl;

}

CCL_EXPORT
CCLEvent* ccl_program_enqueue_kernel(CCLProgram* prg,
    const char* kernel_name, CCLQueue* cq, cl_uint work_dim,
    const size_t* global_work_offset, const size_t* global_work_size,
    const size_t* local_work_size, CCLEventWaitList* evt_wait_lst,
    CCLErr** err, ...) {

    /* Event wrapper. */
    CCLEvent* evt;
    /* The va_list, which represents the variable argument list. */
    va_list args_va;
    /* Array of arguments, to be created from the va_list. */
    void** args_array = NULL;
    /* Number of arguments. */
    guint num_args = 0;
    /* Aux. arg. when cycling through the va_list. */
    void* aux_arg;

    /* Initialize the va_list. */
    va_start(args_va, err);

    /* Get first argument. */
    aux_arg = va_arg(args_va, void*);

    /* Check if any arguments are given, and if so, populate array
     * of arguments. */
    if (aux_arg != NULL) {

        /* 1. Determine number of arguments. */

        while (aux_arg != NULL) {
            num_args++;
            aux_arg = va_arg(args_va, void*);
        }
        va_end(args_va);

        /* 2. Populate array of arguments. */

        args_array = g_slice_alloc((num_args + 1) * sizeof(void*));
        va_start(args_va, err);

        for (guint i = 0; i < num_args; ++i) {
            aux_arg = va_arg(args_va, void*);
            args_array[i] = aux_arg;
        }
        va_end(args_va);
        args_array[num_args] = NULL;

    }

    /* Enqueue kernel using the array of arguments version of this
     * function. */
    evt = ccl_program_enqueue_kernel_v(prg, kernel_name, cq, work_dim,
        global_work_offset, global_work_size, local_work_size,
        evt_wait_lst, args_array, err);

    /* If any arguments are given... */
    if (num_args > 0) {

        /* Free the array of arguments. */
        g_slice_free1((num_args + 1) * sizeof(void*), args_array);

    }

    /* Return the event. */
    return evt;

}

CCL_EXPORT
CCLEvent* ccl_program_enqueue_kernel_v(CCLProgram* prg,
    const char* kernel_name, CCLQueue* cq, cl_uint work_dim,
    const size_t* global_work_offset, const size_t* global_work_size,
    const size_t* local_work_size, CCLEventWaitList* evt_wait_lst,
    void** args, CCLErr** err) {

    /* Make sure err is NULL or it is not set. */
    g_return_val_if_fail((err) == NULL || *(err) == NULL, NULL);

    CCLEvent* evt;
    CCLKernel* krnl;

    krnl = ccl_program_get_kernel(prg, kernel_name, err);
    if (krnl == NULL) return NULL;

    evt = ccl_kernel_set_args_and_enqueue_ndrange_v(krnl, cq, work_dim,
        global_work_offset, global_work_size, local_work_size,
        evt_wait_lst, args, err);
    return evt;
}

static void ccl_program_load_binaries(CCLProgram* prg, CCLErr** err) {

    /* Make sure err is NULL or it is not set. */
    g_return_if_fail((err) == NULL || *(err) == NULL);

    /* Make sure prg is not NULL. */
    g_return_if_fail(prg != NULL);

    /* Make sure binaries table is initialized. */
    g_return_if_fail(prg->binaries != NULL);

    cl_uint num_devices;
    cl_device_id* devices;
    size_t* binary_sizes;
    CCLWrapperInfo* info;
    unsigned char** bins_raw;
    CCLErr* err_internal = NULL;
    cl_int ocl_status;

    /* Get number of program devices. */
    info = ccl_program_get_info(prg, CL_PROGRAM_NUM_DEVICES, &err_internal);
    ccl_if_err_propagate_goto(err, err_internal, error_handler);
    num_devices = *((cl_uint*) info->value);

    /* Get program devices. */
    info = ccl_program_get_info(prg, CL_PROGRAM_DEVICES, &err_internal);
    ccl_if_err_propagate_goto(err, err_internal, error_handler);
    devices = (cl_device_id*) info->value;

    /* Get binary sizes. */
    info = ccl_program_get_info(prg, CL_PROGRAM_BINARY_SIZES, &err_internal);
    ccl_if_err_propagate_goto(err, err_internal, error_handler);
    binary_sizes = (size_t*) info->value;

    /* Allocate memory for binaries. */
    bins_raw = g_slice_alloc0(num_devices * sizeof(unsigned char*));
    for (guint i = 0; i < num_devices; i++) {
        if (binary_sizes[i] > 0) {
            bins_raw[i] = g_malloc(binary_sizes[i]);
        }
    }

    /* Get binaries. */
    ocl_status = clGetProgramInfo(ccl_program_unwrap(prg),
        CL_PROGRAM_BINARIES, num_devices * sizeof(unsigned char*),
        bins_raw, NULL);
    ccl_if_err_create_goto(*err, CCL_OCL_ERROR,
        CL_SUCCESS != ocl_status, ocl_status, error_handler,
        "%s: unable to get binaries from program (OpenCL error %d: %s).",
        CCL_STRD, ocl_status, ccl_err(ocl_status));

    /* Fill binaries table, associating each device with a
     * CCLProgramBinary* object containing the binary and its size. */
    for (guint i = 0; i < num_devices; ++i) {

        CCLProgramBinary* bin = ccl_program_binary_new_empty();
        bin->size = binary_sizes[i];
        bin->data = bins_raw[i];

        g_hash_table_replace(prg->binaries, devices[i], bin);
    }

    /* Free memory allocated for binary array. */
    g_slice_free1(num_devices * sizeof(unsigned char*), bins_raw);

    /* If we got here, everything is OK. */
    g_assert(err == NULL || *err == NULL);
    return;

error_handler:

    /* If we got here there was an error, verify that it is so. */
    g_assert(err == NULL || *err != NULL);
    return;

}

CCL_EXPORT
CCLProgramBinary* ccl_program_get_binary(
    CCLProgram* prg, CCLDevice* dev, CCLErr** err) {

    /* Make sure err is NULL or it is not set. */
    g_return_val_if_fail((err) == NULL || *(err) == NULL, NULL);
    /* Make sure prg is not NULL. */
    g_return_val_if_fail(prg != NULL, NULL);
    /* Make sure dev is not NULL. */
    g_return_val_if_fail(dev != NULL, NULL);

    /* Internal error handling object. */
    CCLErr* err_internal = NULL;
    /* Binary code object. */
    CCLProgramBinary* binary = NULL;

    /* Check if binaries table is initialized. */
    if (prg->binaries == NULL) {

        /* Initialize binaries table. */
        prg->binaries = g_hash_table_new_full(
            g_direct_hash, g_direct_equal, NULL,
            (GDestroyNotify) ccl_program_binary_destroy);

        /* Load binaries. */
        ccl_program_load_binaries(prg, &err_internal);
        ccl_if_err_propagate_goto(err, err_internal, error_handler);
    }

    /* Check if given device exists in the list of program devices. */
    if (g_hash_table_contains(prg->binaries, ccl_device_unwrap(dev))) {

        /* It exists, get it. */
        binary = g_hash_table_lookup(
            prg->binaries, ccl_device_unwrap(dev));

        /* If NULL, then perform a new binary fetch on the CL program
         * object... */
        ccl_program_load_binaries(prg, &err_internal);
        ccl_if_err_propagate_goto(err, err_internal, error_handler);

        /* ...and get it again. If it's NULL it's because binary isn't
         * compiled for given device. */
        binary = g_hash_table_lookup(
            prg->binaries, ccl_device_unwrap(dev));

    } else {

        /* Device does not exist in list of program devices. */
        ccl_if_err_create_goto(*err, CCL_ERROR, TRUE,
            CCL_ERROR_DEVICE_NOT_FOUND,
            error_handler, "%s: device is not part of program devices.",
            CCL_STRD);
    }

    /* If we got here, everything is OK. */
    g_assert(err == NULL || *err == NULL);
    goto finish;

error_handler:

    /* If we got here there was an error, verify that it is so. */
    g_assert(err == NULL || *err != NULL);

finish:

    /* Return kernel wrapper. */
    return binary;
}

CCL_EXPORT
cl_bool ccl_program_save_binary(CCLProgram* prg, CCLDevice* dev,
    const char* filename, CCLErr** err) {

    /* Make sure err is NULL or it is not set. */
    g_return_val_if_fail((err) == NULL || *(err) == NULL, CL_FALSE);
    /* Make sure prg is not NULL. */
    g_return_val_if_fail(prg != NULL, CL_FALSE);
    /* Make sure filename is not NULL. */
    g_return_val_if_fail(filename != NULL, CL_FALSE);

    /* OpenCL function status. */
    cl_bool status;
    /* Internal error handling object. */
    CCLErr* err_internal = NULL;
    /* The binary code object. */
    CCLProgramBinary* binary = NULL;

    /* Get the binary code object for the specified device and check
     * for associated errors. */
    binary = ccl_program_get_binary(prg, dev, &err_internal);
    ccl_if_err_propagate_goto(err, err_internal, error_handler);

    ccl_if_err_create_goto(*err, CCL_ERROR, binary->size == 0,
        CCL_ERROR_INVALID_DATA, error_handler,
        "%s: binary for given device has size 0.", CCL_STRD);

    /* Save binary code to specified file. */
    g_file_set_contents(filename, (const gchar*) binary->data,
        binary->size, &err_internal);
    ccl_if_err_propagate_goto(err, err_internal, error_handler);

    /* If we got here, everything is OK. */
    g_assert(err == NULL || *err == NULL);
    status = CL_TRUE;
    goto finish;

error_handler:

    /* If we got here there was an error, verify that it is so. */
    g_assert(err == NULL || *err != NULL);

    status = CL_FALSE;

finish:

    /* Return function status. */
    return status;
}

CCL_EXPORT
cl_bool ccl_program_save_all_binaries(CCLProgram* prg,
    const char* file_prefix, const char* file_suffix, char*** filenames,
    CCLErr** err) {

    /* Make sure err is NULL or it is not set. */
    g_return_val_if_fail((err) == NULL || *(err) == NULL, CL_FALSE);
    /* Make sure prg is not NULL. */
    g_return_val_if_fail(prg != NULL, CL_FALSE);
    /* Make sure file prefix and suffix are not NULL. */
    g_return_val_if_fail(
        (file_prefix != NULL) && (file_suffix != NULL), CL_FALSE);

    /* Internal error handling object. */
    CCLErr* err_internal = NULL;
    /* Number of devices. */
    guint num_devices;
    /* OpenCL function return status. */
    cl_bool status;

    /* Get number of devices. */
    num_devices = ccl_program_get_num_devices(prg, &err_internal);
    ccl_if_err_propagate_goto(err, err_internal, error_handler);

    /* Allocate space for filenames. */
    if (filenames != NULL)
        *filenames = g_malloc((num_devices + 1) * sizeof(char**));

    /* Allocate space for*/
    /* Cycle through each device. */
    for (guint i = 0; i < num_devices; ++i) {

        /* Save binaries, one per device. */

        /* Device wrapper object. */
        CCLDevice* dev = NULL;
        /* Variable part of filename. */
        gchar* file_middle = NULL;
        /* Complete filename. */
        gchar* filename;

        /* Get next device associated with program. */
        dev = ccl_program_get_device(prg, i, &err_internal);
        ccl_if_err_propagate_goto(err, err_internal, error_handler);

        /* Determine the variable part of current filename. */
        file_middle = g_strdup(
            ccl_device_get_info_array(
                dev, CL_DEVICE_NAME, char*, &err_internal));
        ccl_if_err_propagate_goto(err, err_internal, error_handler);

        g_strcanon(file_middle, CCL_VALIDFILECHARS, '_');

        /* Determine complete filename of current file. */
        filename = g_strdup_printf("%s%s_%02d%s",
            file_prefix, file_middle, i, file_suffix);

        /* Save current binary to file. */
        ccl_program_save_binary(prg, dev, filename, &err_internal);
        ccl_if_err_propagate_goto(err, err_internal, error_handler);

        /* Keep filename or free it, depending if the filenames
         * variable is NULL or not. */
        if (filenames != NULL) {
            (*filenames)[i] = filename;
        } else {
            g_free(filename);
        }

        /* Free middle filename variable. */
        g_free(file_middle);
    }

    /* Signal the end of the array with NULL, so it can be freed with GLib's
     * g_strfreev() function or with its cf4ocl wrapper, ccl_strv_clear(). */
    if (filenames != NULL)
        (*filenames)[num_devices] = NULL;

    /* If we got here, everything is OK. */
    g_assert(err == NULL || *err == NULL);
    status = CL_TRUE;
    goto finish;

error_handler:

    /* If we got here there was an error, verify that it is so. */
    g_assert(err == NULL || *err != NULL);

    status = CL_FALSE;

finish:

    /* Return function status. */
    return status;

}

CCL_EXPORT
CCLDevice* ccl_program_get_device(
    CCLProgram* prg, cl_uint index, CCLErr** err) {

    return ccl_dev_container_get_device(
        (CCLDevContainer*) prg, ccl_program_get_cldevices, index, err);
}

CCL_EXPORT
cl_uint ccl_program_get_num_devices(CCLProgram* prg, CCLErr** err) {

    return ccl_dev_container_get_num_devices(
        (CCLDevContainer*) prg, ccl_program_get_cldevices, err);

}

CCL_EXPORT
CCLDevice* const* ccl_program_get_all_devices(CCLProgram* prg,
    CCLErr** err) {

    return ccl_dev_container_get_all_devices((CCLDevContainer*) prg,
        ccl_program_get_cldevices, err);

}