/*
* Copyright (C) 2016-2017 ARM Limited. All rights reserved.
*
* Copyright (C) 2008 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <cstdlib>
#include <string.h>
#include <errno.h>
#include <inttypes.h>
#include <pthread.h>
#include <log/log.h>
#include <cutils/atomic.h>
#include <ion/ion.h>
#include <sys/ioctl.h>
#include <hardware/hardware.h>
#if GRALLOC_USE_GRALLOC1_API == 1
#include <hardware/gralloc1.h>
#else
#include <hardware/gralloc.h>
#endif
#include "mali_gralloc_module.h"
#include "mali_gralloc_private_interface_types.h"
#include "mali_gralloc_buffer.h"
#include "gralloc_helper.h"
#include "framebuffer_device.h"
#include "mali_gralloc_formats.h"
#include "mali_gralloc_usages.h"
#include "mali_gralloc_bufferdescriptor.h"
#include "ion_4.12.h"
#define ION_SYSTEM (char*)"ion_system_heap"
#define ION_CMA (char*)"linux,cma"
static bool gralloc_legacy_ion;
static int system_heap_id;
static int cma_heap_id;
static void mali_gralloc_ion_free_internal(buffer_handle_t *pHandle, uint32_t num_hnds);
static void init_afbc(uint8_t *buf, uint64_t internal_format, int w, int h)
{
uint32_t n_headers = (w * h) / 256;
uint32_t body_offset = n_headers * 16;
uint32_t headers[][4] = {
{ body_offset, 0x1, 0x10000, 0x0 }, /* Layouts 0, 3, 4 */
{ (body_offset + (1 << 28)), 0x80200040, 0x1004000, 0x20080 } /* Layouts 1, 5 */
};
uint32_t i, layout;
/* For AFBC 1.2, header buffer can be initilized to 0 for Layouts 0, 3, 4 */
if (internal_format & MALI_GRALLOC_INTFMT_AFBC_TILED_HEADERS)
{
memset(headers[0], 0, sizeof(uint32_t) * 4);
}
/* map format if necessary (also removes internal extension bits) */
uint64_t base_format = internal_format & MALI_GRALLOC_INTFMT_FMT_MASK;
switch (base_format)
{
case MALI_GRALLOC_FORMAT_INTERNAL_RGBA_8888:
case MALI_GRALLOC_FORMAT_INTERNAL_RGBX_8888:
case MALI_GRALLOC_FORMAT_INTERNAL_RGB_888:
case MALI_GRALLOC_FORMAT_INTERNAL_RGB_565:
case MALI_GRALLOC_FORMAT_INTERNAL_BGRA_8888:
layout = 0;
break;
case MALI_GRALLOC_FORMAT_INTERNAL_YV12:
case MALI_GRALLOC_FORMAT_INTERNAL_NV12:
case MALI_GRALLOC_FORMAT_INTERNAL_NV21:
layout = 1;
break;
default:
layout = 0;
}
ALOGV("Writing AFBC header layout %d for format %" PRIu64, layout, base_format);
for (i = 0; i < n_headers; i++)
{
memcpy(buf, headers[layout], sizeof(headers[layout]));
buf += sizeof(headers[layout]);
}
}
static int find_heap_id(int ion_client, char *name)
{
int i, ret, cnt, heap_id = -1;
struct ion_heap_data *data;
ret = ion_query_heap_cnt(ion_client, &cnt);
if (ret)
{
AERR("ion count query failed with %s", strerror(errno));
return -1;
}
data = (struct ion_heap_data *)malloc(cnt * sizeof(*data));
if (!data)
{
AERR("Error allocating data %s\n", strerror(errno));
return -1;
}
ret = ion_query_get_heaps(ion_client, cnt, data);
if (ret)
{
AERR("Error querying heaps from ion %s", strerror(errno));
}
else
{
for (i = 0; i < cnt; i++) {
if (strcmp(data[i].name, name) == 0) {
heap_id = data[i].heap_id;
break;
}
}
if (i == cnt)
{
AERR("No %s Heap Found amongst %d heaps\n", name, cnt);
heap_id = -1;
}
}
free(data);
return heap_id;
}
static int alloc_from_ion_heap(int ion_fd, size_t size, unsigned int heap_mask, unsigned int flags, int *min_pgsz)
{
ion_user_handle_t ion_hnd = -1;
int shared_fd, ret;
if ((ion_fd < 0) || (size <= 0) || (heap_mask == 0) || (min_pgsz == NULL))
{
return -1;
}
/**
* step 1: ion_alloc new ion_hnd
* step 2: ion_share from ion_hnd and get shared_fd
* step 3: ion free the given ion_hnd
* step 4: when we need to free this ion buffer, just close the shared_fd,
* kernel will count the reference of file struct, so it's safe to
* be transfered between processes.
*/
ret = ion_alloc(ion_fd, size, 0, heap_mask, flags, &ion_hnd);
if (ret < 0)
{
#if defined(ION_HEAP_SECURE_MASK)
if (heap_mask == ION_HEAP_SECURE_MASK)
{
return -1;
}
else
#endif
{
/* If everything else failed try system heap */
flags = 0; /* Fallback option flags are not longer valid */
heap_mask = ION_HEAP_SYSTEM_MASK;
ret = ion_alloc(ion_fd, size, 0, heap_mask, flags, &ion_hnd);
}
}
ret = ion_share(ion_fd, ion_hnd, &shared_fd);
if (ret != 0)
{
AERR("ion_share( %d ) failed", ion_fd);
shared_fd = -1;
}
ret = ion_free(ion_fd, ion_hnd);
if (0 != ret)
{
AERR("ion_free( %d ) failed", ion_fd);
close(shared_fd);
shared_fd = -1;
}
if (ret >= 0)
{
switch (heap_mask)
{
case ION_HEAP_SYSTEM_MASK:
*min_pgsz = SZ_4K;
break;
case ION_HEAP_SYSTEM_CONTIG_MASK:
case ION_HEAP_CARVEOUT_MASK:
#ifdef ION_HEAP_TYPE_DMA_MASK
case ION_HEAP_TYPE_DMA_MASK:
#endif
*min_pgsz = size;
break;
#ifdef ION_HEAP_CHUNK_MASK
/* NOTE: if have this heap make sure your ION chunk size is 2M*/
case ION_HEAP_CHUNK_MASK:
*min_pgsz = SZ_2M;
break;
#endif
#ifdef ION_HEAP_COMPOUND_PAGE_MASK
case ION_HEAP_COMPOUND_PAGE_MASK:
*min_pgsz = SZ_2M;
break;
#endif
/* If have customized heap please set the suitable pg type according to
* the customized ION implementation
*/
#ifdef ION_HEAP_CUSTOM_MASK
case ION_HEAP_CUSTOM_MASK:
*min_pgsz = SZ_4K;
break;
#endif
default:
*min_pgsz = SZ_4K;
break;
}
}
return shared_fd;
}
unsigned int pick_ion_heap(uint64_t usage)
{
unsigned int heap_mask;
if (usage & GRALLOC_USAGE_PROTECTED)
{
#if defined(ION_HEAP_SECURE_MASK)
heap_mask = ION_HEAP_SECURE_MASK;
#else
AERR("Protected ION memory is not supported on this platform.");
return 0;
#endif
}
#if defined(ION_HEAP_TYPE_COMPOUND_PAGE_MASK) && GRALLOC_USE_ION_COMPOUND_PAGE_HEAP
else if (!(usage & GRALLOC_USAGE_HW_VIDEO_ENCODER) && (usage & (GRALLOC_USAGE_HW_FB | GRALLOC_USAGE_HW_COMPOSER)))
{
heap_mask = ION_HEAP_TYPE_COMPOUND_PAGE_MASK;
}
#elif defined(ION_HEAP_TYPE_DMA_MASK) && GRALLOC_USE_ION_DMA_HEAP
else if (!(usage & GRALLOC_USAGE_HW_VIDEO_ENCODER) && (usage & (GRALLOC_USAGE_HW_FB)))
{
heap_mask = ION_HEAP_TYPE_DMA_MASK;
}
#endif
else
{
heap_mask = ION_HEAP_SYSTEM_MASK;
}
return heap_mask;
}
void set_ion_flags(unsigned int heap_mask, uint64_t usage, unsigned int *priv_heap_flag, int *ion_flags)
{
#if !GRALLOC_USE_ION_DMA_HEAP
GRALLOC_UNUSED(heap_mask);
#endif
if (priv_heap_flag)
{
#if defined(ION_HEAP_TYPE_DMA_MASK) && GRALLOC_USE_ION_DMA_HEAP
if (heap_mask == ION_HEAP_TYPE_DMA_MASK)
{
*priv_heap_flag = private_handle_t::PRIV_FLAGS_USES_ION_DMA_HEAP;
}
#endif
}
if (ion_flags)
{
#if defined(ION_HEAP_TYPE_DMA_MASK) && GRALLOC_USE_ION_DMA_HEAP
if (heap_mask != ION_HEAP_TYPE_DMA_MASK)
{
#endif
if ((usage & GRALLOC_USAGE_SW_READ_MASK) == GRALLOC_USAGE_SW_READ_OFTEN)
{
*ion_flags = ION_FLAG_CACHED | ION_FLAG_CACHED_NEEDS_SYNC;
}
#if defined(ION_HEAP_TYPE_DMA_MASK) && GRALLOC_USE_ION_DMA_HEAP
}
#endif
}
}
static bool check_buffers_sharable(const gralloc_buffer_descriptor_t *descriptors, uint32_t numDescriptors)
{
unsigned int shared_backend_heap_mask = 0;
int shared_ion_flags = 0;
uint64_t usage;
uint32_t i;
if (numDescriptors <= 1)
{
return false;
}
for (i = 0; i < numDescriptors; i++)
{
unsigned int heap_mask;
int ion_flags;
buffer_descriptor_t *bufDescriptor = (buffer_descriptor_t *)descriptors[i];
usage = bufDescriptor->consumer_usage | bufDescriptor->producer_usage;
heap_mask = pick_ion_heap(usage);
if (0 == heap_mask)
{
return false;
}
set_ion_flags(heap_mask, usage, NULL, &ion_flags);
if (0 != shared_backend_heap_mask)
{
if (shared_backend_heap_mask != heap_mask || shared_ion_flags != ion_flags)
{
return false;
}
}
else
{
shared_backend_heap_mask = heap_mask;
shared_ion_flags = ion_flags;
}
}
return true;
}
static int get_max_buffer_descriptor_index(const gralloc_buffer_descriptor_t *descriptors, uint32_t numDescriptors)
{
uint32_t i, max_buffer_index = 0;
size_t max_buffer_size = 0;
for (i = 0; i < numDescriptors; i++)
{
buffer_descriptor_t *bufDescriptor = (buffer_descriptor_t *)descriptors[i];
if (max_buffer_size < bufDescriptor->size)
{
max_buffer_index = i;
max_buffer_size = bufDescriptor->size;
}
}
return max_buffer_index;
}
int mali_gralloc_ion_allocate(mali_gralloc_module *m, const gralloc_buffer_descriptor_t *descriptors,
uint32_t numDescriptors, buffer_handle_t *pHandle, bool *shared_backend)
{
static int support_protected = 1; /* initially, assume we support protected memory */
unsigned int heap_mask, priv_heap_flag = 0;
unsigned char *cpu_ptr = NULL;
uint64_t usage;
uint32_t i, max_buffer_index = 0;
int shared_fd, ret, ion_flags = 0;
int min_pgsz = 0;
if (m->ion_client < 0)
{
m->ion_client = ion_open();
if (m->ion_client < 0)
{
AERR("ion_open failed with %s", strerror(errno));
return -1;
}
gralloc_legacy_ion = ion_is_legacy(m->ion_client);
if (!gralloc_legacy_ion)
{
system_heap_id = find_heap_id(m->ion_client, ION_SYSTEM);
cma_heap_id = find_heap_id(m->ion_client, ION_CMA);
if (system_heap_id < 0)
{
ion_close(m->ion_client);
m->ion_client = -1;
AERR( "ion_open failed: no system heap found" );
return -1;
}
if (cma_heap_id < 0) {
AERR("No cma heap found, falling back to system");
cma_heap_id = system_heap_id;
}
}
}
*shared_backend = check_buffers_sharable(descriptors, numDescriptors);
if (*shared_backend)
{
buffer_descriptor_t *max_bufDescriptor;
max_buffer_index = get_max_buffer_descriptor_index(descriptors, numDescriptors);
max_bufDescriptor = (buffer_descriptor_t *)(descriptors[max_buffer_index]);
usage = max_bufDescriptor->consumer_usage | max_bufDescriptor->producer_usage;
heap_mask = pick_ion_heap(usage);
if (heap_mask == 0)
{
AERR("Failed to find an appropriate ion heap");
return -1;
}
set_ion_flags(heap_mask, usage, &priv_heap_flag, &ion_flags);
if (gralloc_legacy_ion)
{
shared_fd = alloc_from_ion_heap(m->ion_client, max_bufDescriptor->size, heap_mask, ion_flags, &min_pgsz);
}
else
{
int heap = 1 << system_heap_id;
if (heap_mask == ION_HEAP_TYPE_DMA_MASK)
heap = 1 << cma_heap_id;
ret = ion_alloc_fd(m->ion_client, max_bufDescriptor->size, 0, heap, 0, &(shared_fd));
if (ret != 0)
{
AERR("Failed to ion_alloc_fd from ion_client:%d", m->ion_client);
return -1;
}
min_pgsz = SZ_4K;
}
if (shared_fd < 0)
{
AERR("ion_alloc failed form client: ( %d )", m->ion_client);
return -1;
}
for (i = 0; i < numDescriptors; i++)
{
buffer_descriptor_t *bufDescriptor = (buffer_descriptor_t *)(descriptors[i]);
int tmp_fd;
if (i != max_buffer_index)
{
tmp_fd = dup(shared_fd);
if (tmp_fd < 0)
{
/* need to free already allocated memory. */
mali_gralloc_ion_free_internal(pHandle, numDescriptors);
return -1;
}
}
else
{
tmp_fd = shared_fd;
}
private_handle_t *hnd = new private_handle_t(
private_handle_t::PRIV_FLAGS_USES_ION | priv_heap_flag, bufDescriptor->size, min_pgsz,
bufDescriptor->consumer_usage, bufDescriptor->producer_usage, tmp_fd, bufDescriptor->hal_format,
bufDescriptor->internal_format, bufDescriptor->byte_stride, bufDescriptor->width, bufDescriptor->height,
bufDescriptor->pixel_stride, bufDescriptor->internalWidth, bufDescriptor->internalHeight,
max_bufDescriptor->size);
if (NULL == hnd)
{
mali_gralloc_ion_free_internal(pHandle, numDescriptors);
return -1;
}
pHandle[i] = hnd;
}
}
else
{
for (i = 0; i < numDescriptors; i++)
{
buffer_descriptor_t *bufDescriptor = (buffer_descriptor_t *)(descriptors[i]);
usage = bufDescriptor->consumer_usage | bufDescriptor->producer_usage;
heap_mask = pick_ion_heap(usage);
if (heap_mask == 0)
{
AERR("Failed to find an appropriate ion heap");
mali_gralloc_ion_free_internal(pHandle, numDescriptors);
return -1;
}
set_ion_flags(heap_mask, usage, &priv_heap_flag, &ion_flags);
if (gralloc_legacy_ion)
{
shared_fd = alloc_from_ion_heap(m->ion_client, bufDescriptor->size, heap_mask, ion_flags, &min_pgsz);
}
else
{
int heap = 1 << system_heap_id;
if (heap_mask == ION_HEAP_TYPE_DMA_MASK)
heap = 1 << cma_heap_id;
ret = ion_alloc_fd(m->ion_client, bufDescriptor->size, 0, heap, 0, &(shared_fd));
if (ret != 0)
{
AERR("Failed to ion_alloc_fd from ion_client:%d", m->ion_client);
mali_gralloc_ion_free_internal(pHandle, numDescriptors);
return -1;
}
min_pgsz = SZ_4K;
}
if (shared_fd < 0)
{
AERR("ion_alloc failed from client ( %d )", m->ion_client);
/* need to free already allocated memory. not just this one */
mali_gralloc_ion_free_internal(pHandle, numDescriptors);
return -1;
}
private_handle_t *hnd = new private_handle_t(
private_handle_t::PRIV_FLAGS_USES_ION | priv_heap_flag, bufDescriptor->size, min_pgsz,
bufDescriptor->consumer_usage, bufDescriptor->producer_usage, shared_fd, bufDescriptor->hal_format,
bufDescriptor->internal_format, bufDescriptor->byte_stride, bufDescriptor->width, bufDescriptor->height,
bufDescriptor->pixel_stride, bufDescriptor->internalWidth, bufDescriptor->internalHeight,
bufDescriptor->size);
if (NULL == hnd)
{
mali_gralloc_ion_free_internal(pHandle, numDescriptors);
return -1;
}
pHandle[i] = hnd;
}
}
for (i = 0; i < numDescriptors; i++)
{
buffer_descriptor_t *bufDescriptor = (buffer_descriptor_t *)(descriptors[i]);
private_handle_t *hnd = (private_handle_t *)(pHandle[i]);
usage = bufDescriptor->consumer_usage | bufDescriptor->producer_usage;
if (!(usage & GRALLOC_USAGE_PROTECTED))
{
cpu_ptr =
(unsigned char *)mmap(NULL, bufDescriptor->size, PROT_READ | PROT_WRITE, MAP_SHARED, hnd->share_fd, 0);
if (MAP_FAILED == cpu_ptr)
{
AERR("mmap failed from client ( %d ), fd ( %d )", m->ion_client, hnd->share_fd);
mali_gralloc_ion_free_internal(pHandle, numDescriptors);
return -1;
}
#if GRALLOC_INIT_AFBC == 1
if ((bufDescriptor->internal_format & MALI_GRALLOC_INTFMT_AFBCENABLE_MASK) && (!(*shared_backend)))
{
init_afbc(cpu_ptr, bufDescriptor->internal_format, bufDescriptor->width, bufDescriptor->height);
}
#endif
hnd->base = cpu_ptr;
}
}
return 0;
}
void mali_gralloc_ion_free(private_handle_t const *hnd)
{
if (hnd->flags & private_handle_t::PRIV_FLAGS_FRAMEBUFFER)
{
return;
}
else if (hnd->flags & private_handle_t::PRIV_FLAGS_USES_ION)
{
/* Buffer might be unregistered already so we need to assure we have a valid handle*/
if (0 != hnd->base)
{
if (0 != munmap((void *)hnd->base, hnd->size))
{
AERR("Failed to munmap handle %p", hnd);
}
}
close(hnd->share_fd);
memset((void *)hnd, 0, sizeof(*hnd));
}
}
static void mali_gralloc_ion_free_internal(buffer_handle_t *pHandle, uint32_t num_hnds)
{
uint32_t i = 0;
for (i = 0; i < num_hnds; i++)
{
if (NULL != pHandle[i])
{
mali_gralloc_ion_free((private_handle_t *)(pHandle[i]));
}
}
return;
}
void mali_gralloc_ion_sync(const mali_gralloc_module *m, private_handle_t *hnd)
{
if (!gralloc_legacy_ion)
return;
if (m != NULL && hnd != NULL)
{
switch (hnd->flags & private_handle_t::PRIV_FLAGS_USES_ION)
{
case private_handle_t::PRIV_FLAGS_USES_ION:
if (!(hnd->flags & private_handle_t::PRIV_FLAGS_USES_ION_DMA_HEAP))
{
ion_sync_fd(m->ion_client, hnd->share_fd);
}
break;
}
}
}
int mali_gralloc_ion_map(private_handle_t *hnd)
{
int retval = -EINVAL;
switch (hnd->flags & private_handle_t::PRIV_FLAGS_USES_ION)
{
case private_handle_t::PRIV_FLAGS_USES_ION:
unsigned char *mappedAddress;
size_t size = hnd->size;
hw_module_t *pmodule = NULL;
private_module_t *m = NULL;
if (hw_get_module(GRALLOC_HARDWARE_MODULE_ID, (const hw_module_t **)&pmodule) == 0)
{
m = reinterpret_cast<private_module_t *>(pmodule);
}
else
{
AERR("Could not get gralloc module for handle: %p", hnd);
retval = -errno;
break;
}
/* the test condition is set to m->ion_client <= 0 here, because:
* 1) module structure are initialized to 0 if no initial value is applied
* 2) a second user process should get a ion fd greater than 0.
*/
if (m->ion_client <= 0)
{
/* a second user process must obtain a client handle first via ion_open before it can obtain the shared ion buffer*/
m->ion_client = ion_open();
if (m->ion_client < 0)
{
AERR("Could not open ion device for handle: %p", hnd);
retval = -errno;
break;
}
}
mappedAddress = (unsigned char *)mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, hnd->share_fd, 0);
if (MAP_FAILED == mappedAddress)
{
AERR("mmap( share_fd:%d ) failed with %s", hnd->share_fd, strerror(errno));
retval = -errno;
break;
}
hnd->base = (void *)(uintptr_t(mappedAddress) + hnd->offset);
retval = 0;
break;
}
return retval;
}
void mali_gralloc_ion_unmap(private_handle_t *hnd)
{
switch (hnd->flags & private_handle_t::PRIV_FLAGS_USES_ION)
{
case private_handle_t::PRIV_FLAGS_USES_ION:
void *base = (void *)hnd->base;
size_t size = hnd->size;
if (munmap(base, size) < 0)
{
AERR("Could not munmap base:%p size:%zd '%s'", base, size, strerror(errno));
}
break;
}
}
int mali_gralloc_ion_device_close(struct hw_device_t *device)
{
#if GRALLOC_USE_GRALLOC1_API == 1
gralloc1_device_t *dev = reinterpret_cast<gralloc1_device_t *>(device);
#else
alloc_device_t *dev = reinterpret_cast<alloc_device_t *>(device);
#endif
if (dev)
{
private_module_t *m = reinterpret_cast<private_module_t *>(dev->common.module);
if (m->ion_client != -1)
{
if (0 != ion_close(m->ion_client))
{
AERR("Failed to close ion_client: %d err=%s", m->ion_client, strerror(errno));
}
m->ion_client = -1;
}
delete dev;
}
return 0;
}