From c19c1912399280c081c05b537408108a2dede9ff Mon Sep 17 00:00:00 2001 From: VaL Doroshchuk Date: Mon, 4 May 2020 16:50:26 +0200 Subject: [PATCH 1/1] Vivante: Remap video buffer each time Due to some stuttering issues on some platforms, designed following flow: 1. Some amount of buffers are created to decode the video by a driver. 2. We receive a buffer and generate a texture for it. 3. If a texture is already generated for this memory, just use this texutre. It was supposed to solve some performance issues on some platforms. But suddenly found that the video buffers are handled/sent not continuously, also processed asynchronous, which seems leads to the texture/data is updated while we are using it. Sounds like the data should be remapped. Suggesting to keep 1 texture available and remove remembering textures by a buffer. Call glTexDirectVIVMap each time when the video buffer is mapped. NOTE: Impacts to CPU, the patch increases usage of CPU. Pick-to: 5.15 Fixes: QTBUG-76270 Change-Id: I15b17f0402a9dccb4414121a1835aee225e4480b --- .../videonode/imx6/qsgvivantevideomaterial.cpp | 144 +++++++++------------ .../videonode/imx6/qsgvivantevideomaterial.h | 5 +- 2 files changed, 64 insertions(+), 85 deletions(-) diff --git a/src/plugins/videonode/imx6/qsgvivantevideomaterial.cpp b/src/plugins/videonode/imx6/qsgvivantevideomaterial.cpp index e200e8d16..13963d183 100644 --- a/src/plugins/videonode/imx6/qsgvivantevideomaterial.cpp +++ b/src/plugins/videonode/imx6/qsgvivantevideomaterial.cpp @@ -60,6 +60,7 @@ QSGVivanteVideoMaterial::QSGVivanteVideoMaterial() : mWidth(0), mHeight(0), mFormat(QVideoFrame::Format_Invalid), + mTexture(0), mCurrentTexture(0), mMappable(true), mTexDirectTexture(0) @@ -71,6 +72,7 @@ QSGVivanteVideoMaterial::QSGVivanteVideoMaterial() : setFlag(Blending, false); mShader = new QSGVivanteVideoMaterialShader; + glGenTextures(1, &mTexture); } QSGVivanteVideoMaterial::~QSGVivanteVideoMaterial() @@ -90,7 +92,7 @@ QSGMaterialShader *QSGVivanteVideoMaterial::createShader() const { int QSGVivanteVideoMaterial::compare(const QSGMaterial *other) const { if (this->type() == other->type()) { const QSGVivanteVideoMaterial *m = static_cast(other); - if (this->mBitsToTextureMap == m->mBitsToTextureMap) + if (this->mTexture == m->mTexture) return 0; else return 1; @@ -130,15 +132,7 @@ void QSGVivanteVideoMaterial::bind() void QSGVivanteVideoMaterial::clearTextures() { - for (auto it = mBitsToTextureMap.cbegin(), end = mBitsToTextureMap.cend(); it != end; ++it) { - GLuint id = it.value(); -#ifdef QT_VIVANTE_VIDEO_DEBUG - qDebug() << "delete texture: " << id; -#endif - glDeleteTextures(1, &id); - } - mBitsToTextureMap.clear(); - + glDeleteTextures(1, &mTexture); if (mTexDirectTexture) { glDeleteTextures(1, &mTexDirectTexture); mTexDirectTexture = 0; @@ -178,83 +172,71 @@ GLuint QSGVivanteVideoMaterial::vivanteMapping(QVideoFrame vF) if (vF.map(QAbstractVideoBuffer::ReadOnly)) { if (mMappable) { - if (!mBitsToTextureMap.contains(vF.bits())) { - // Haven't yet seen this logical address: map to texture. - GLuint tmpTexId; - glGenTextures(1, &tmpTexId); - mBitsToTextureMap.insert(vF.bits(), tmpTexId); - - // Determine the full width & height. Full means: actual width/height plus extra padding pixels. - // The full width can be deduced from the bytesPerLine value. The full height is calculated - // by calculating the distance between the start of the first and second planes, and dividing - // it by the stride (= the bytesPerLine). If there is only one plane, we don't worry about - // extra padding rows, since there are no adjacent extra planes. - // XXX: This assumes the distance between bits(1) and bits(0) is exactly the size of the first - // plane (the Y plane in the case of YUV data). A better way would be to have a dedicated - // planeSize() or planeOffset() getter. - // Also, this assumes that planes are tightly packed, that is, there is no space between them. - // It is okay to assume this here though, because the Vivante direct textures also assume that. - // In other words, if the planes aren't tightly packed, then the direct textures won't be able - // to render the frame correctly anyway. - int fullWidth = vF.bytesPerLine() / QSGVivanteVideoNode::getBytesForPixelFormat(vF.pixelFormat()); - int fullHeight = (vF.planeCount() > 1) ? ((vF.bits(1) - vF.bits(0)) / vF.bytesPerLine()) : vF.height(); - - // The uscale is the ratio of actual width to the full width (same for vscale and height). - // Since the vivante direct textures do not offer a way to explicitly specify the amount of padding - // columns and rows, we use a trick. We show the full frame - including the padding pixels - in the - // texture, but render only a subset of that texture. This subset goes from (0,0) to (uScale, vScale). - // In the shader, the texture coordinates (which go from (0.0, 0.0) to (1.0, 1.0)) are multiplied by - // the u/v scale values. Since 1.0 * x = x, this effectively limits the texture coordinates from - // (0.0, 0.0) - (1.0, 1.0) to (0.0, 0.0) - (uScale, vScale). - float uScale = float(vF.width()) / float(fullWidth); - float vScale = float(vF.height()) / float(fullHeight); - mShader->setUVScale(uScale, vScale); - - const uchar *constBits = vF.bits(); - void *bits = (void*)constBits; + // Determine the full width & height. Full means: actual width/height plus extra padding pixels. + // The full width can be deduced from the bytesPerLine value. The full height is calculated + // by calculating the distance between the start of the first and second planes, and dividing + // it by the stride (= the bytesPerLine). If there is only one plane, we don't worry about + // extra padding rows, since there are no adjacent extra planes. + // XXX: This assumes the distance between bits(1) and bits(0) is exactly the size of the first + // plane (the Y plane in the case of YUV data). A better way would be to have a dedicated + // planeSize() or planeOffset() getter. + // Also, this assumes that planes are tightly packed, that is, there is no space between them. + // It is okay to assume this here though, because the Vivante direct textures also assume that. + // In other words, if the planes aren't tightly packed, then the direct textures won't be able + // to render the frame correctly anyway. + int fullWidth = vF.bytesPerLine() / QSGVivanteVideoNode::getBytesForPixelFormat(vF.pixelFormat()); + int fullHeight = (vF.planeCount() > 1) ? ((vF.bits(1) - vF.bits(0)) / vF.bytesPerLine()) : vF.height(); + + // The uscale is the ratio of actual width to the full width (same for vscale and height). + // Since the vivante direct textures do not offer a way to explicitly specify the amount of padding + // columns and rows, we use a trick. We show the full frame - including the padding pixels - in the + // texture, but render only a subset of that texture. This subset goes from (0,0) to (uScale, vScale). + // In the shader, the texture coordinates (which go from (0.0, 0.0) to (1.0, 1.0)) are multiplied by + // the u/v scale values. Since 1.0 * x = x, this effectively limits the texture coordinates from + // (0.0, 0.0) - (1.0, 1.0) to (0.0, 0.0) - (uScale, vScale). + float uScale = float(vF.width()) / float(fullWidth); + float vScale = float(vF.height()) / float(fullHeight); + mShader->setUVScale(uScale, vScale); + + const uchar *constBits = vF.bits(); + void *bits = (void*)constBits; #ifdef QT_VIVANTE_VIDEO_DEBUG - qDebug() << Q_FUNC_INFO - << "new texture, texId: " << tmpTexId - << "; constBits: " << constBits - << "; actual/full width: " << vF.width() << "/" << fullWidth - << "; actual/full height: " << vF.height() << "/" << fullHeight - << "; UV scale: U " << uScale << " V " << vScale; + qDebug() << Q_FUNC_INFO + << "new texture, texId: " << mTexture + << "; constBits: " << constBits + << "; actual/full width: " << vF.width() << "/" << fullWidth + << "; actual/full height: " << vF.height() << "/" << fullHeight + << "; UV scale: U " << uScale << " V " << vScale; #endif - GLuint physical = ~0U; + GLuint physical = ~0U; #if GST_CHECK_VERSION(1,14,0) - auto buffer = reinterpret_cast(vF.buffer()); - auto mem = gst_buffer_peek_memory(buffer->buffer(), 0); - auto phys_addr = gst_is_phys_memory(mem) ? gst_phys_memory_get_phys_addr(mem) : 0; - if (phys_addr) - physical = phys_addr; + auto buffer = reinterpret_cast(vF.buffer()); + auto mem = gst_buffer_peek_memory(buffer->buffer(), 0); + auto phys_addr = gst_is_phys_memory(mem) ? gst_phys_memory_get_phys_addr(mem) : 0; + if (phys_addr) + physical = phys_addr; #endif - glBindTexture(GL_TEXTURE_2D, tmpTexId); - glTexDirectVIVMap_LOCAL(GL_TEXTURE_2D, - fullWidth, fullHeight, - QSGVivanteVideoNode::getVideoFormat2GLFormatMap().value(vF.pixelFormat()), - &bits, &physical); - - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); - glTexDirectInvalidateVIV_LOCAL(GL_TEXTURE_2D); - - mMapError = glGetError(); - if (mMapError == GL_NO_ERROR) - return tmpTexId; - - // Error occurred. - // Fallback to copying data. - } else { - // Fastest path: already seen this logical address. Just - // indicate that the data belonging to the texture has changed. - glBindTexture(GL_TEXTURE_2D, mBitsToTextureMap.value(vF.bits())); - glTexDirectInvalidateVIV_LOCAL(GL_TEXTURE_2D); - return mBitsToTextureMap.value(vF.bits()); - } + glBindTexture(GL_TEXTURE_2D, mTexture); + glTexDirectVIVMap_LOCAL(GL_TEXTURE_2D, + fullWidth, fullHeight, + QSGVivanteVideoNode::getVideoFormat2GLFormatMap().value(vF.pixelFormat()), + &bits, &physical); + + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); + + glTexDirectInvalidateVIV_LOCAL(GL_TEXTURE_2D); + + mMapError = glGetError(); + if (mMapError == GL_NO_ERROR) + return mTexture; + + // Error occurred. + // Fallback to copying data. } // Cannot map. So copy. diff --git a/src/plugins/videonode/imx6/qsgvivantevideomaterial.h b/src/plugins/videonode/imx6/qsgvivantevideomaterial.h index db59e8fc7..bdf3b5b35 100644 --- a/src/plugins/videonode/imx6/qsgvivantevideomaterial.h +++ b/src/plugins/videonode/imx6/qsgvivantevideomaterial.h @@ -40,9 +40,6 @@ #ifndef QSGVIDEOMATERIAL_VIVMAP_H #define QSGVIDEOMATERIAL_VIVMAP_H -#include -#include - #include #include #include @@ -77,7 +74,7 @@ private: int mHeight; QVideoFrame::PixelFormat mFormat; - QMap mBitsToTextureMap; + GLuint mTexture; QVideoFrame mCurrentFrame; GLuint mCurrentTexture; bool mMappable; -- 2.16.3