Surface <-> ANativeWindow

Ref. frameworks/base/native/android/native_window_jni.cpp

#include <android/native_window_jni.h>
#include <android/native_window.h>
#include <system/window.h>

#include <gui/Surface.h>
#include <utils/StrongPointer.h>

#include <android_runtime/android_view_Surface.h>

using namespace android;

ANativeWindow* ANativeWindow_fromSurface(JNIEnv* env, jobject surface) {
    sp<ANativeWindow> win = android_view_Surface_getNativeWindow(env, surface);
    if (win != NULL) {
        ANativeWindow_acquire(win.get());
    }
    return win.get();
}

jobject ANativeWindow_toSurface(JNIEnv* env, ANativeWindow* window) {
    if (window == NULL) {
        return NULL;
    }
    sp<Surface> surface = static_cast<Surface*>(window);
    return android_view_Surface_createFromSurface(env, surface);
}

NativeHandle -> HGraphicBufferProducer -> ANativeWindow

Ref. frameworks/hardware/interfaces/automotive/display/aidl/vts/functional/VtsHalCarDisplayTargetTest.cpp

#include <aidl/Gtest.h>
#include <aidl/Vintf.h>
#include <aidl/android/frameworks/automotive/display/DisplayDesc.h>
#include <aidl/android/frameworks/automotive/display/ICarDisplayProxy.h>
#include <aidlcommonsupport/NativeHandle.h>
#include <android-base/logging.h>
#include <android/binder_ibinder.h>
#include <android/binder_manager.h>
#include <android/binder_process.h>
#include <android/binder_status.h>
#include <bufferqueueconverter/BufferQueueConverter.h>

namespace {

using ::aidl::android::frameworks::automotive::display::DisplayDesc;
using ::aidl::android::frameworks::automotive::display::ICarDisplayProxy;
using ::aidl::android::hardware::common::NativeHandle;

android::sp<HGraphicBufferProducer> convertNativeHandleToHGBP(const NativeHandle& aidlHandle) {
    native_handle_t* handle = ::android::dupFromAidl(aidlHandle);
    if (handle == nullptr || handle->numFds != 0 ||
        handle->numInts < std::ceil(sizeof(size_t) / sizeof(int))) {
        LOG(ERROR) << "Invalid native handle";
        return nullptr;
    }
    android::hardware::hidl_vec<uint8_t> halToken;
    halToken.setToExternal(reinterpret_cast<uint8_t*>(const_cast<int*>(&(handle->data[1]))),
                           handle->data[0]);
    android::sp<HGraphicBufferProducer> hgbp =
        HGraphicBufferProducer::castFrom(::android::retrieveHalInterface(halToken));
    return std::move(hgbp);
}

}  // namespace

TEST_P(CarDisplayAidlTest, getIGBPObject) {
    LOG(INFO) << "Test getHGraphicBufferProducer method";

    for (const auto& id : mDisplayIds) {
        // Get a display info.
        DisplayDesc desc;
        ASSERT_TRUE(mDisplayProxy->getDisplayInfo(id, &desc).isOk());

        // Get a HGBP object as a native handle object.
        NativeHandle handle;
        ASSERT_TRUE(mDisplayProxy->getHGraphicBufferProducer(id, &handle).isOk());

        // Convert a native handle object into a HGBP object.
        android::sp<android::hardware::graphics::bufferqueue::V2_0::IGraphicBufferProducer>
            gfxBufferProducer = convertNativeHandleToHGBP(handle);
        ASSERT_NE(gfxBufferProducer, nullptr);

        // Create a Surface object.
        android::SurfaceHolderUniquePtr surfaceHolder = getSurfaceFromHGBP(gfxBufferProducer);
        ASSERT_NE(surfaceHolder, nullptr);

        // Verify the size.
        ANativeWindow* nativeWindow = getNativeWindow(surfaceHolder.get());
        ASSERT_EQ(desc.width, ANativeWindow_getWidth(nativeWindow));
        ASSERT_EQ(desc.height, ANativeWindow_getHeight(nativeWindow));
    }
}

HGraphicBufferProducer -> android::IGraphicBufferProducer

Ref. frameworks/native/libs/bufferqueueconverter/BufferQueueConverter.cpp

#include <gui/Surface.h>
#include <gui/bufferqueue/2.0/H2BGraphicBufferProducer.h>

#include "include/bufferqueueconverter/BufferQueueConverter.h"


using ::android::Surface;
using ::android::IGraphicBufferProducer;
using ::android::hardware::graphics::bufferqueue::V2_0::utils::H2BGraphicBufferProducer;


namespace android {

struct SurfaceHolder {
    sp<Surface> surface;
    SurfaceHolder(const sp<Surface>& s) : surface(s) {}
};

/**
 * Custom deleter for SurfaceHolder unique pointer
 */
void destroySurfaceHolder(SurfaceHolder* surfaceHolder) {
    delete surfaceHolder;
}


SurfaceHolderUniquePtr getSurfaceFromHGBP(const sp<HGraphicBufferProducer>& token) {
    if (token == nullptr) {
        ALOGE("Passed IGraphicBufferProducer handle is invalid.");
        return SurfaceHolderUniquePtr(nullptr, nullptr);
    }

    sp<IGraphicBufferProducer> bufferProducer = new H2BGraphicBufferProducer(token);
    if (bufferProducer == nullptr) {
        ALOGE("Failed to get IGraphicBufferProducer.");
        return SurfaceHolderUniquePtr(nullptr, nullptr);
    }

    sp<Surface> newSurface(new Surface(bufferProducer, true));
    if (newSurface == nullptr) {
        ALOGE("Failed to create Surface from HGBP.");
        return SurfaceHolderUniquePtr(nullptr, nullptr);
    }

    return SurfaceHolderUniquePtr(new SurfaceHolder(newSurface), destroySurfaceHolder);
}


ANativeWindow* getNativeWindow(SurfaceHolder* handle) {
    if (handle == nullptr) {
        ALOGE("SurfaceHolder is invalid.");
        return nullptr;
    }

    return static_cast<ANativeWindow*>(handle->surface.get());
}

} // namespace android

native_handle_t <-> NativeHandle

Ref. frameworks/base/services/core/jni/com_android_server_biometrics_SurfaceToNativeHandleConverter.cpp

#define LOG_TAG "SurfaceToNativeHandleConverter"

#include <android_os_NativeHandle.h>
#include <android_runtime/AndroidRuntime.h>
#include <android_runtime/android_view_Surface.h>
#include <gui/IGraphicBufferProducer.h>
#include <gui/Surface.h>
#include <gui/bufferqueue/1.0/WGraphicBufferProducer.h>
#include <utils/Log.h>

#include "jni.h"

namespace android {
namespace {

constexpr int WINDOW_HAL_TOKEN_SIZE_MAX = 256;

native_handle_t* convertHalTokenToNativeHandle(const HalToken& halToken) {
    // We attempt to store halToken in the ints of the native_handle_t after its
    // size. The first int stores the size of the token. We store this in an int
    // to avoid alignment issues where size_t and int do not have the same
    // alignment.
    size_t nhDataByteSize = halToken.size();
    if (nhDataByteSize > WINDOW_HAL_TOKEN_SIZE_MAX) {
        // The size of the token isn't reasonable..
        return nullptr;
    }
    size_t numInts = ceil(nhDataByteSize / sizeof(int)) + 1;

    // We don't check for overflow, whether numInts can fit in an int, since we
    // expect WINDOW_HAL_TOKEN_SIZE_MAX to be a reasonable limit.
    // create a native_handle_t with 0 numFds and numInts number of ints.
    native_handle_t* nh = native_handle_create(0, numInts);
    if (!nh) {
        return nullptr;
    }
    // Store the size of the token in the first int.
    nh->data[0] = nhDataByteSize;
    memcpy(&(nh->data[1]), halToken.data(), nhDataByteSize);
    return nh;
}

HalToken convertNativeHandleToHalToken(native_handle_t* handle) {
    int size = handle->data[0];
    auto data = reinterpret_cast<uint8_t*>(&handle->data[1]);
    HalToken halToken;
    halToken.setToExternal(data, size);
    return halToken;
}

jobject acquireSurfaceHandle(JNIEnv* env, jobject /* clazz */, jobject jSurface) {
    ALOGD("%s", __func__);
    if (jSurface == nullptr) {
        ALOGE("%s: jSurface is null", __func__);
        return nullptr;
    }

    sp<Surface> surface = android_view_Surface_getSurface(env, jSurface);
    if (surface == nullptr) {
        ALOGE("%s: surface is null", __func__);
        return nullptr;
    }

    sp<IGraphicBufferProducer> igbp = surface->getIGraphicBufferProducer();
    sp<HGraphicBufferProducer> hgbp = new TWGraphicBufferProducer<HGraphicBufferProducer>(igbp);
    // The HAL token will be closed in releaseSurfaceHandle.
    HalToken halToken;
    createHalToken(hgbp, &halToken);

    native_handle_t* native_handle = convertHalTokenToNativeHandle(halToken);
    if (native_handle == nullptr) {
        ALOGE("%s: native_handle is null", __func__);
        return nullptr;
    }
    jobject jHandle = JNativeHandle::MakeJavaNativeHandleObj(env, native_handle);
    native_handle_delete(native_handle);

    return jHandle;
}

void releaseSurfaceHandle(JNIEnv* env, jobject /* clazz */, jobject jHandle) {
    ALOGD("%s", __func__);
    // Creates a native handle from a Java handle. We must call native_handle_delete when we're done
    // with it because we created it, but we shouldn't call native_handle_close because we don't own
    // the underlying FDs.
    native_handle_t* handle =
            JNativeHandle::MakeCppNativeHandle(env, jHandle, nullptr /* storage */);
    if (handle == nullptr) {
        ALOGE("%s: handle is null", __func__);
        return;
    }

    HalToken token = convertNativeHandleToHalToken(handle);
    ALOGD("%s: deleteHalToken, success: %d", __func__, deleteHalToken(token));
    ALOGD("%s: native_handle_delete, success: %d", __func__, !native_handle_delete(handle));
}

const JNINativeMethod method_table[] = {
        {"acquireSurfaceHandle", "(Landroid/view/Surface;)Landroid/os/NativeHandle;",
         reinterpret_cast<void*>(acquireSurfaceHandle)},
        {"releaseSurfaceHandle", "(Landroid/os/NativeHandle;)V",
         reinterpret_cast<void*>(releaseSurfaceHandle)},
};
} // namespace

int register_android_server_FaceService(JNIEnv* env) {
    return AndroidRuntime::
            registerNativeMethods(env, "com/android/server/biometrics/sensors/face/FaceService",
                                  method_table, NELEM(method_table));
}
} // namespace android

native_handle_t -> Java NativeHandle

Ref. frameworks/base/core/jni/android_os_NativeHandle.h

#ifndef ANDROID_OS_NATIVE_HANDLE_H
#define ANDROID_OS_NATIVE_HANDLE_H

#include "hwbinder/EphemeralStorage.h"

#include <cutils/native_handle.h>
#include <jni.h>

namespace android {

struct JNativeHandle {

    /**
     * Returns a Java NativeHandle object representing the parameterized
     * native_handle_t instance.
     */
    static jobject MakeJavaNativeHandleObj(JNIEnv *env, const native_handle_t *handle);

    /**
     * Returns a heap-allocated native_handle_t instance representing the
     * parameterized Java object. Note that if no valid EphemeralStorage*
     * parameter is supplied (storage is nullptr), the return value must
     * be explicitly deallocated (using native_handle_delete).
     */
    static native_handle_t* MakeCppNativeHandle(JNIEnv *env, jobject jHandle,
            EphemeralStorage *storage);

    /**
     * Returns an (uninitialized) array of Java NativeHandle objects.
     */
    static jobjectArray AllocJavaNativeHandleObjArray(JNIEnv *env, jsize length);
};

}

#endif  // ANDROID_OS_NATIVE_HANDLE_H

AImageReader::getWindowNativeHandle

media_status_t AImageReader::getWindowNativeHandle(native_handle **handle) {
    if (mWindowHandle != nullptr) {
        *handle = mWindowHandle;
        return AMEDIA_OK;
    }
    sp<HGraphicBufferProducer> hgbp =
        new TWGraphicBufferProducer<HGraphicBufferProducer>(mProducer);
    HalToken halToken;
    if (!createHalToken(hgbp, &halToken)) {
        return AMEDIA_ERROR_UNKNOWN;
    }
    mWindowHandle = convertHalTokenToNativeHandle(halToken);
    if (!mWindowHandle) {
        return AMEDIA_ERROR_UNKNOWN;
    }
    *handle = mWindowHandle;
    return AMEDIA_OK;
}

JAVA AIDL NativeHandle -> android.os.NativeHandle

Ref. frameworks/base/services/core/java/com/android/server/biometrics/sensors/face/aidl/AidlNativeHandleUtils.java

package com.android.server.biometrics.sensors.face.aidl;

import android.annotation.Nullable;
import android.os.ParcelFileDescriptor;

import java.io.FileDescriptor;
import java.io.IOException;

/**
 * A utility class for the AIDL implementation of NativeHandle - {@link
 * android.hardware.common.NativeHandle}.
 */
public final class AidlNativeHandleUtils {

    /**
     * Converts a {@link android.os.NativeHandle} to a {@link android.hardware.common.NativeHandle}
     * by duplicating the underlying file descriptors.
     *
     * Both the original and new handle must be closed after use.
     *
     * @param handle {@link android.os.NativeHandle}. Can be null.
     * @return a {@link android.hardware.common.NativeHandle} representation of {@code handle}.
     * Returns null if {@code handle} is null.
     * @throws IOException if any of the underlying calls to {@code dup} fail.
     */
    public static @Nullable android.hardware.common.NativeHandle dup(
            @Nullable android.os.NativeHandle handle) throws IOException {
        if (handle == null) {
            return null;
        }
        android.hardware.common.NativeHandle res = new android.hardware.common.NativeHandle();
        final FileDescriptor[] fds = handle.getFileDescriptors();
        res.ints = handle.getInts().clone();
        res.fds = new ParcelFileDescriptor[fds.length];
        for (int i = 0; i < fds.length; ++i) {
            res.fds[i] = ParcelFileDescriptor.dup(fds[i]);
        }
        return res;
    }

    /**
     * Closes the file descriptors contained within a {@link android.hardware.common.NativeHandle}.
     * This is a no-op if the handle is null.
     *
     * This should only be used for handles that own their file descriptors, for example handles
     * obtained using {@link #dup(android.os.NativeHandle)}.
     *
     * @param handle {@link android.hardware.common.NativeHandle}. Can be null.
     * @throws IOException if any of the underlying calls to {@code close} fail.
     */
    public static void close(@Nullable android.hardware.common.NativeHandle handle)
            throws IOException {
        if (handle != null) {
            for (ParcelFileDescriptor fd : handle.fds) {
                if (fd != null) {
                    fd.close();
                }
            }
        }
    }
}

native_handle_t <-> AIDL NativeHandle

Ref. hardware/interfaces/common/support/NativeHandle.cpp

#include <aidlcommonsupport/NativeHandle.h>

#include <fcntl.h>

namespace android {

using aidl::android::hardware::common::NativeHandle;

/**
 * Checks if a NativeHandle is null
 */
bool isAidlNativeHandleEmpty(const NativeHandle& handle) {
    return handle.fds.empty() && handle.ints.empty();
}

static native_handle_t* fromAidl(const NativeHandle& handle, bool doDup) {
    native_handle_t* to = native_handle_create(handle.fds.size(), handle.ints.size());
    if (!to) return nullptr;

    for (size_t i = 0; i < handle.fds.size(); i++) {
        int fd = handle.fds[i].get();
        to->data[i] = doDup ? fcntl(fd, F_DUPFD_CLOEXEC, 0) : fd;
    }
    memcpy(to->data + handle.fds.size(), handle.ints.data(), handle.ints.size() * sizeof(int));
    return to;
}

native_handle_t* makeFromAidl(const NativeHandle& handle) {
    return fromAidl(handle, false /* doDup */);
}
native_handle_t* dupFromAidl(const NativeHandle& handle) {
    return fromAidl(handle, true /* doDup */);
}

static NativeHandle toAidl(const native_handle_t* handle, bool doDup) {
    NativeHandle to;

    to.fds = std::vector<ndk::ScopedFileDescriptor>(handle->numFds);
    for (size_t i = 0; i < handle->numFds; i++) {
        int fd = handle->data[i];
        to.fds.at(i).set(doDup ? fcntl(fd, F_DUPFD_CLOEXEC, 0) : fd);
    }

    to.ints = std::vector<int32_t>(handle->data + handle->numFds,
                                   handle->data + handle->numFds + handle->numInts);
    return to;
}

NativeHandle makeToAidl(const native_handle_t* handle) {
    return toAidl(handle, false /* doDup */);
}

NativeHandle dupToAidl(const native_handle_t* handle) {
    return toAidl(handle, true /* doDup */);
}

}  // namespace android

`android.hardware.HardwareBuffer` to `android.media.Image`

Ref. frameworks/base/media/jni/android_media_MediaCodec.cpp

static jobject android_media_MediaCodec_mapHardwareBuffer(JNIEnv *env, jclass, jobject bufferObj) {
    ALOGV("android_media_MediaCodec_mapHardwareBuffer");
    AHardwareBuffer *hardwareBuffer = android_hardware_HardwareBuffer_getNativeHardwareBuffer(
            env, bufferObj);
    AHardwareBuffer_Desc desc;
    AHardwareBuffer_describe(hardwareBuffer, &desc);
    if (desc.format != AHARDWAREBUFFER_FORMAT_Y8Cb8Cr8_420) {
        ALOGI("mapHardwareBuffer: unmappable format: %d", desc.format);
        return nullptr;
    }
    if ((desc.usage & AHARDWAREBUFFER_USAGE_CPU_READ_MASK) == 0) {
        ALOGI("mapHardwareBuffer: buffer not CPU readable");
        return nullptr;
    }
    bool readOnly = ((desc.usage & AHARDWAREBUFFER_USAGE_CPU_WRITE_MASK) == 0);

    uint64_t cpuUsage = 0;
    cpuUsage |= (desc.usage & AHARDWAREBUFFER_USAGE_CPU_READ_MASK);
    cpuUsage |= (desc.usage & AHARDWAREBUFFER_USAGE_CPU_WRITE_MASK);

    AHardwareBuffer_Planes planes;
    int err = AHardwareBuffer_lockPlanes(
            hardwareBuffer, cpuUsage, -1 /* fence */, nullptr /* rect */, &planes);
    if (err != 0) {
        ALOGI("mapHardwareBuffer: Failed to lock planes (err=%d)", err);
        return nullptr;
    }

    if (planes.planeCount != 3) {
        ALOGI("mapHardwareBuffer: planeCount expected 3, actual %u", planes.planeCount);
        return nullptr;
    }

    ScopedLocalRef<jobjectArray> buffersArray{
            env, env->NewObjectArray(3, gByteBufferInfo.clazz, NULL)};
    ScopedLocalRef<jintArray> rowStridesArray{env, env->NewIntArray(3)};
    ScopedLocalRef<jintArray> pixelStridesArray{env, env->NewIntArray(3)};

    jboolean isCopy = JNI_FALSE;
    jint *rowStrides = env->GetIntArrayElements(rowStridesArray.get(), &isCopy);
    jint *pixelStrides = env->GetIntArrayElements(rowStridesArray.get(), &isCopy);

    // For Y plane
    int rowSampling = 1;
    int colSampling = 1;
    // plane indices are Y-U-V.
    for (uint32_t i = 0; i < 3; ++i) {
        const AHardwareBuffer_Plane &plane = planes.planes[i];
        int maxRowOffset = plane.rowStride * (desc.height / rowSampling - 1);
        int maxColOffset = plane.pixelStride * (desc.width / colSampling - 1);
        int maxOffset = maxRowOffset + maxColOffset;
        ScopedLocalRef<jobject> byteBuffer{env, CreateByteBuffer(
                env,
                plane.data,
                maxOffset + 1,
                0,
                maxOffset + 1,
                readOnly,
                true)};

        env->SetObjectArrayElement(buffersArray.get(), i, byteBuffer.get());
        rowStrides[i] = plane.rowStride;
        pixelStrides[i] = plane.pixelStride;
        // For U-V planes
        rowSampling = 2;
        colSampling = 2;
    }

    env->ReleaseIntArrayElements(rowStridesArray.get(), rowStrides, 0);
    env->ReleaseIntArrayElements(pixelStridesArray.get(), pixelStrides, 0);
    rowStrides = pixelStrides = nullptr;

    ScopedLocalRef<jclass> imageClazz(
            env, env->FindClass("android/media/MediaCodec$MediaImage"));
    CHECK(imageClazz.get() != NULL);

    jmethodID imageConstructID = env->GetMethodID(imageClazz.get(), "<init>",
            "([Ljava/nio/ByteBuffer;[I[IIIIZJIILandroid/graphics/Rect;J)V");

    jobject img = env->NewObject(imageClazz.get(), imageConstructID,
            buffersArray.get(),
            rowStridesArray.get(),
            pixelStridesArray.get(),
            desc.width,
            desc.height,
            desc.format, // ???
            (jboolean)readOnly /* readOnly */,
            (jlong)0 /* timestamp */,
            (jint)0 /* xOffset */, (jint)0 /* yOffset */, nullptr /* cropRect */,
            (jlong)hardwareBuffer);

    // if MediaImage creation fails, return null
    if (env->ExceptionCheck()) {
        env->ExceptionDescribe();
        env->ExceptionClear();
        return nullptr;
    }

    AHardwareBuffer_acquire(hardwareBuffer);

    return img;
}

static void android_media_MediaCodec_closeMediaImage(JNIEnv *, jclass, jlong context) {
    ALOGV("android_media_MediaCodec_closeMediaImage");
    if (context == 0) {
        return;
    }
    AHardwareBuffer *hardwareBuffer = (AHardwareBuffer *)context;

    int err = AHardwareBuffer_unlock(hardwareBuffer, nullptr);
    if (err != 0) {
        ALOGI("closeMediaImage: failed to unlock (err=%d)", err);
        // Continue to release the hardwareBuffer
    }

    AHardwareBuffer_release(hardwareBuffer);
}

template <typename T>
static jobject CreateByteBuffer(
        JNIEnv *env, T *base, size_t capacity, size_t offset, size_t size,
        bool readOnly, bool clearBuffer) {
    jobject byteBuffer =
        env->NewDirectByteBuffer(
                const_cast<typename std::remove_const<T>::type *>(base),
                capacity);
    if (readOnly && byteBuffer != NULL) {
        jobject readOnlyBuffer = env->CallObjectMethod(
                byteBuffer, gByteBufferInfo.asReadOnlyBufferId);
        env->DeleteLocalRef(byteBuffer);
        byteBuffer = readOnlyBuffer;
    }
    if (byteBuffer == NULL) {
        return nullptr;
    }
    jobject me = env->CallObjectMethod(
            byteBuffer, gByteBufferInfo.orderId, gByteBufferInfo.nativeByteOrder);
    env->DeleteLocalRef(me);
    me = env->CallObjectMethod(
            byteBuffer, gByteBufferInfo.limitId,
            clearBuffer ? capacity : offset + size);
    env->DeleteLocalRef(me);
    me = env->CallObjectMethod(
            byteBuffer, gByteBufferInfo.positionId,
            clearBuffer ? 0 : offset);
    env->DeleteLocalRef(me);
    me = NULL;
    return byteBuffer;
}

Surface ‐ ANativeWindow ‐ IGraphicBufferProducer ‐ NativeHandle - tonykwok/made-mistakes-again GitHub Wiki

Surface <-> ANativeWindow

NativeHandle -> HGraphicBufferProducer -> ANativeWindow

HGraphicBufferProducer -> android::IGraphicBufferProducer

native_handle_t <-> NativeHandle

native_handle_t -> Java NativeHandle

AImageReader::getWindowNativeHandle

JAVA AIDL NativeHandle -> android.os.NativeHandle

native_handle_t <-> AIDL NativeHandle

`android.hardware.HardwareBuffer` to `android.media.Image`

⚠️ GitHub.com Fallback ⚠️

Surface ‐ ANativeWindow ‐ IGraphicBufferProducer ‐ NativeHandle - tonykwok/made-mistakes-again GitHub Wiki

Surface <-> ANativeWindow

NativeHandle -> HGraphicBufferProducer -> ANativeWindow

HGraphicBufferProducer -> android::IGraphicBufferProducer

native_handle_t <-> NativeHandle

native_handle_t -> Java NativeHandle

AImageReader::getWindowNativeHandle

JAVA AIDL NativeHandle -> android.os.NativeHandle

native_handle_t <-> AIDL NativeHandle

android.hardware.HardwareBuffer to android.media.Image

⚠️ **GitHub.com Fallback** ⚠️

`android.hardware.HardwareBuffer` to `android.media.Image`

⚠️ GitHub.com Fallback ⚠️