GLES Recoring on Android - tonykwok/made-mistakes-again GitHub Wiki
When records camera preview stream uses GLES and MediaCodec and saves as an mp4 file on Android, during playback the color may be incorrect if you did not set the COLOR_STANDARD correctly, for example:
1. Camera preview stream is BT.601
-
MediaCodecwithout settingCOLOR_STANDARDexplicitly (same as preview)
Color range : Limited Color primaries : BT.709 colour_primaries_Original : BT.601 PAL Transfer characteristics : BT.709 transfer_characteristics_Original : BT.601 Matrix coefficients : BT.709 matrix_coefficients_Original : BT.470 System B/G (equivalent to BT.601) mdhd_Duration : 1932 Codec configuration box : avcC
-
MediaCodecwithCOLOR_STANDARDbeing set to BT.709 explicitly (RED becomes ORANGE)
Color range : Limited
Color primaries : BT.709
Transfer characteristics : BT.709
Matrix coefficients : BT.709
Codec configuration box : avcC
-
MediaCodecwithCOLOR_STANDARDbeing set to BT.601 PAL explicitly （same as preview）
Color range : Limited Color primaries : BT.601 PAL Transfer characteristics : BT.709 transfer_characteristics_Original : BT.601 Matrix coefficients : BT.601 matrix_coefficients_Original : BT.470 System B/G (equivalent to BT.601) mdhd_Duration : 1900 Codec configuration box : avcC
-
MediaCodecwithCOLOR_STANDARDbeing to BT.601 NTSC explicitly （same as preview）
Color range : Limited
Color primaries : BT.601 NTSC
Transfer characteristics : BT.709
transfer_characteristics_Original : BT.601
Matrix coefficients : BT.601
mdhd_Duration : 1878
Codec configuration box : avcC
2. Camera preview stream is BT.709
-
MediaCodecwithout settingCOLOR_STANDARDexplicitly (RED becomes ORANGE)
Color range : Limited
Color primaries : BT.709
Transfer characteristics : BT.709
transfer_characteristics_Original : BT.601
Matrix coefficients : BT.709
matrix_coefficients_Original : Identity
Codec configuration box : avcC
-
MediaCodecwithCOLOR_STANDARDbeing set to BT.709 explicitly (RED becomes ORANGE)
Color range : Limited
Color primaries : BT.709
Transfer characteristics : BT.709
Matrix coefficients : BT.709
mdhd_Duration : 4650
Codec configuration box : avcC
-
MediaCodecwithCOLOR_STANDARDbeing set to BT.601 APL explicitly (same as preview)
Color range : Limited
Color primaries : BT.601 PAL
Transfer characteristics : BT.709
transfer_characteristics_Original : BT.601
Matrix coefficients : BT.601
Codec configuration box : avcC
-
MediaCodecwithCOLOR_STANDARDbeing set to BT.601 NTSC explicitly (same as preview)
Color range : Limited
Color primaries : BT.601 NTSC
Transfer characteristics : BT.709
transfer_characteristics_Original : BT.601
Matrix coefficients : BT.601
Codec configuration box : avcC
3. Solution
// Kr = 0.2126, Kg = 0.7152, Kb = 0.0722
private static final float[] MATRIX_COLOR_TRANSFORM_BT709_FULL = {
1f, 1f, 1f,
0f, -0.187321f, 1.8556f,
1.5748f, -0.468124f, 0.00000f,
};
// Kr = 0.299, Kg = 0.587, Kb = 0.114
private static final float[] MATRIX_COLOR_TRANSFORM_BT601_FULL = {
1f, 1f, 1f,
0f, -0.344136f, 1.772f,
1.402f, -0.714136f, 0.00000f,
};
"#version 300 es\n" +
"#extension GL_OES_EGL_image_external : require\n" +
"#extension GL_EXT_YUV_target : require\n" +
"precision mediump float;\n" +
"uniform mat3 uColorTransform;\n" +
"uniform mat3 uColorTransform2;\n" +
"uniform float uDisplayP3ToSrgb;\n" +
"uniform float uBT2020Smpte170M;\n" +
"uniform float uAlpha;\n" +
"uniform float uMixAlpha;\n" +
// "uniform samplerExternalOES sTexture;\n" +
"uniform __samplerExternal2DY2YEXT sTexture;\n" +
// "varying vec2 vTexCoord; \n" +
"uniform sampler3D uLookup;\n" +
"in vec2 vTexCoord;\n" +
"out vec4 outColor;\n" +
"const mat3 CCM_XYZ_TO_SRGB = mat3(\n" +
" 3.240969941904521, -0.9692436362808798, 0.05563007969699361,\n" +
" -1.537383177570093, 1.87596750150772, -0.2039769588889765,\n" +
" -0.4986107602930033, 0.04155505740717561, 1.056971514242878\n" +
");\n" +
"const mat3 CCM_DISPLAY_P3_TO_XYZ = mat3(\n" +
" 0.4865709486482162, 0.2289745640697488, 0.0,\n" +
" 0.2656676931690929, 0.6917385218365062, 0.04511338185890257,\n" +
" 0.1982172852343625, 0.079286914093745, 1.04394436890097500\n" +
");\n" +
"const mat3 CCM_XYZ_TO_BT2020 = mat3(\n" +
" 1.716651187971267, -0.666684351832489, 0.01763985744531091,\n" +
" -0.3556707837763924, 1.616481236634939, -0.04277061325780865,\n" +
" -0.2533662813736598, 0.01576854581391113, 0.942103121235474\n" +
");\n" +
"float oetf_inverse_SMPTE_170M(float rgb) {\n" +
" return rgb < 0.081 ? rgb / 4.5 : pow((rgb + 0.099) / 1.099, 1.0 / 0.45);\n" +
"}\n" +
"float oetf_SMPTE_170M(float linear) {\n" +
" return linear < 0.018 ? 4.5 * linear : 1.099 * pow(linear, 0.45) - 0.099;\n" +
"}\n" +
"float oetf_inverse_BT470_BG(float rgb) {\n" +
" return pow(rgb, 2.8);\n" +
"}\n" +
"float oetf_BT470_BG(float linear) {\n" +
" return pow(linear, 1.0 / 2.8);\n" +
"}\n" +
"float oetf_inverse_sRGB(float rgb) {\n" +
" return rgb <= 0.04045 ? rgb / 12.92 : pow((rgb + 0.055) / 1.055, 2.4);\n" +
"}\n" +
"float oetf_sRGB(float linear) {\n" +
" return linear <= 0.0031308 ? 12.92 * linear : 1.055 * pow(linear, 1.0 / 2.4) - 0.055;\n" +
"}\n" +
// "float det(mat2 matrix) {\n" +
// " return matrix[0].x * matrix[1].y - matrix[0].y * matrix[1].x;\n" +
// "}\n" +
// "\n" +
// "mat3 inverse(mat3 matrix) {\n" +
// " vec3 row0 = matrix[0];\n" +
// " vec3 row1 = matrix[1];\n" +
// " vec3 row2 = matrix[2];\n" +
// "\n" +
// " vec3 minors0 = vec3(\n" +
// " det(mat2(row1.y, row1.z, row2.y, row2.z)),\n" +
// " det(mat2(row1.z, row1.x, row2.z, row2.x)),\n" +
// " det(mat2(row1.x, row1.y, row2.x, row2.y))\n" +
// " );\n" +
// " vec3 minors1 = vec3(\n" +
// " det(mat2(row2.y, row2.z, row0.y, row0.z)),\n" +
// " det(mat2(row2.z, row2.x, row0.z, row0.x)),\n" +
// " det(mat2(row2.x, row2.y, row0.x, row0.y))\n" +
// " );\n" +
// " vec3 minors2 = vec3(\n" +
// " det(mat2(row0.y, row0.z, row1.y, row1.z)),\n" +
// " det(mat2(row0.z, row0.x, row1.z, row1.x)),\n" +
// " det(mat2(row0.x, row0.y, row1.x, row1.y))\n" +
// " );\n" +
// "\n" +
// " mat3 adj = transpose(mat3(minors0, minors1, minors2));\n" +
// "\n" +
// " return (1.0 / dot(row0, minors0)) * adj;\n" +
// "}\n" +
"void main() {\n" +
// " vec4 color = texture2D(sTexture, vTexCoord);\n" +
" vec3 srcYuv = texture(sTexture, vTexCoord).xyz;\n" +
" vec3 yuvOffset;\n" +
// " yuvOffset.x = srcYuv.r - 0.0625;\n" + // y
// " yuvOffset.y = srcYuv.g - 0.5;\n" + // u
// " yuvOffset.z = srcYuv.b - 0.5;\n" + // v
" yuvOffset.x = srcYuv.r - 0.0;\n" + // y
" yuvOffset.y = srcYuv.g - 0.5;\n" + // u
" yuvOffset.z = srcYuv.b - 0.5;\n" + // v
" vec4 color = vec4(uColorTransform * yuvOffset, 1.0);\n" + // 601
// "color.r = clamp(color.r, 0.0, 1.0);\n" +
// "color.g = clamp(color.g, 0.0, 1.0);\n" +
// "color.b = clamp(color.b, 0.0, 1.0);\n" +
// " color.r = oetf_inverse_sRGB(color.r);\n" +
// " color.g = oetf_inverse_sRGB(color.g);\n" +
// " color.b = oetf_inverse_sRGB(color.b);\n" +
//
// "vec3 myuv;\n" +
// "myuv.x = clamp(0.299 * color.r + 0.587 * color.g + 0.114 * color.b, 0.0, 1.0);\n" +
// "myuv.y = clamp(-0.16874 * color.r - 0.33126 * color.g + 0.5 * color.b + 0.5, 0.0, 1.0);\n" +
// "myuv.z = clamp(0.5 * color.r - 0.41869 * color.g - 0.08131 * color.b + 0.5, 0.0, 1.0);\n" +
//
// "myuv.x = clamp(myuv.x, 0.0, 1.0);\n" +
// "myuv.y = clamp(myuv.y - 0.5, 0.0, 1.0);\n" +
// "myuv.z = clamp(myuv.z - 0.5, 0.0, 1.0);\n" +
//
// "color = vec4(uColorTransform2 * myuv, 1.0);\n" +
//
// "color.r = clamp(color.r, 0.0, 1.0);\n" +
// "color.g = clamp(color.g, 0.0, 1.0);\n" +
// "color.b = clamp(color.b, 0.0, 1.0);\n" +
//
// " color.r = oetf_sRGB(color.r);\n" +
// " color.g = oetf_sRGB(color.g);\n" +
// " color.b = oetf_sRGB(color.b);\n" +
// yuvCscStandardEXT conv1 = itu_601;
// yuvCscStandardEXT conv2 = itu_601_full_range;
// yuvCscStandardEXT conv3 = itu_709;
// " vec4 color = vec4(yuv_2_rgb(srcYuv, itu_709), 1.0);\n" +
// " if (uDisplayP3ToSrgb > 0.5) {\n" +
" color.r = oetf_inverse_sRGB(color.r);\n" +
" color.g = oetf_inverse_sRGB(color.g);\n" +
" color.b = oetf_inverse_sRGB(color.b);\n" +
" vec3 xyz = CCM_DISPLAY_P3_TO_XYZ * color.rgb;\n" +
" vec3 rgb = CCM_XYZ_TO_SRGB * xyz;\n" +
// " color.rgb = texture(uLookup, rgb).rgb;\n" +
// " color.rgb = uColorTransform * color.rgb;\n" +
" color.r = oetf_sRGB(rgb.r);\n" +
" color.g = oetf_sRGB(rgb.g);\n" +
" color.b = oetf_sRGB(rgb.b);\n" +
// " }\n" +
"color.rgb = vec3(uColorTransform * inverse(uColorTransform2) * color.rgb);\n" + // 709
// " if (uBT2020Smpte170M > 0.5) {\n" +
// " color.r = oetf_inverse_SMPTE_170M(color.r);\n" +
// " color.g = oetf_inverse_SMPTE_170M(color.g);\n" +
// " color.b = oetf_inverse_SMPTE_170M(color.b);\n" +
// " }\n" +
//
// "color.r = clamp(color.r, 0.0, 1.0);\n" +
// "color.g = clamp(color.g, 0.0, 1.0);\n" +
// "color.b = clamp(color.b, 0.0, 1.0);\n" +
" outColor = color*uAlpha; \n" +
" if (uMixAlpha >= 0.0) {\n" +
" outColor.a = uMixAlpha;\n" +
" }\n" +
"}";