Detect in-flight buffers caused by asynchronous frame rendering

--HG--
branch : opengl3

code/nel/src/3d/driver/opengl3/driver_opengl.cpp

@@ -39,7 +39,7 @@ using namespace std;
 using namespace NLMISC;
 
 
-
+#define NL3D_GL3_FRAME_IN_FLIGHT_DEBUG 0
 
 
 // ***************************************************************************
@@ -313,6 +313,9 @@ CDriverGL3::CDriverGL3()
 	//
 	_CurrentOcclusionQuery = NULL;
 	_SwapBufferCounter = 0;
+	_SwapBufferInFlight = 0;
+	for (size_t i = 0; i < NL3D_GL3_BUFFER_QUEUE_MAX; ++i)
+		_SwapBufferSync[i] = 0;
 
 	_LightMapDynamicLightEnabled = false;
 	_LightMapDynamicLightDirty= false;
@@ -590,9 +593,38 @@ void CDriverGL3::setColorMask (bool bRed, bool bGreen, bool bBlue, bool bAlpha)
 // --------------------------------------------------
 bool CDriverGL3::swapBuffers()
 {
-	H_AUTO_OGL(CDriverGL3_swapBuffers)
+	H_AUTO_OGL(CDriverGL3_swapBuffers);
 
-	++ _SwapBufferCounter;
+	// Set fence
+	size_t syncI = _SwapBufferCounter % NL3D_GL3_BUFFER_QUEUE_MAX;
+	if (_SwapBufferSync[syncI]) // Wait for oldest fence, if this is still in flight
+	{
+#if NL3D_GL3_FRAME_IN_FLIGHT_DEBUG
+		nldebug("Wait for oldest fence");
+#endif
+		GLenum syncR = nglClientWaitSync(_SwapBufferSync[syncI], 0, 1000000000ULL);
+		switch (syncR)
+		{
+		case GL_ALREADY_SIGNALED:
+		case GL_CONDITION_SATISFIED:
+			break;
+		case GL_TIMEOUT_EXPIRED:
+			nlwarning("Timeout expired for glClientWaitSync");
+			break;
+		case GL_WAIT_FAILED:
+			nlwarning("Wait failed for glClientWaitSync");
+			break;
+		default:
+			nlwarning("Unknown glClientWaitSync result");
+		}
+		nglDeleteSync(_SwapBufferSync[syncI]);
+		++_SwapBufferInFlight;
+	}
+	// Fence occurs before the frame swap, because it only is relevant for user supplied buffers. The framebuffer is not our concern
+	_SwapBufferSync[syncI] = nglFenceSync(GL_SYNC_GPU_COMMANDS_COMPLETE, 0);
+	nlassert(_SwapBufferSync[syncI]);
+
+	++_SwapBufferCounter;
 
 	if (!_WndActive)
 	{
@@ -658,6 +690,38 @@ bool CDriverGL3::swapBuffers()
 
 	// Check all vertex buffer to see which one are lost
 	updateLostBuffers();
+	
+	// Check in flight buffers, also checks the current one
+	for (size_t i = 0; i < NL3D_GL3_BUFFER_QUEUE_MAX; ++i)
+	{
+		size_t syncJ = (syncI + 1 + i) % NL3D_GL3_BUFFER_QUEUE_MAX;
+		if (_SwapBufferSync[syncJ]) // If there's a frame in flight
+		{
+			GLint status = 0;
+			nglGetSynciv(_SwapBufferSync[syncJ], GL_SYNC_STATUS, 1, NULL, &status);
+			if (status == GL_SIGNALED)
+			{
+				// Frame is no longer in flight
+				nglDeleteSync(_SwapBufferSync[syncJ]);
+				_SwapBufferSync[syncJ] = 0;
+#if NL3D_GL3_FRAME_IN_FLIGHT_DEBUG
+				nldebug("Frame %u no longer in flight", (unsigned int)_SwapBufferInFlight);
+#endif
+				++_SwapBufferInFlight;
+				nlassert(_SwapBufferInFlight <= _SwapBufferCounter);
+			}
+			else
+			{
+				// Following frames are still in flight
+				break;
+			}
+		}
+	}
+
+	nlassert(_SwapBufferInFlight <= _SwapBufferCounter);
+#if NL3D_GL3_FRAME_IN_FLIGHT_DEBUG
+	nldebug("Swap buffer: %u, in flight: %u", (unsigned int)_SwapBufferCounter, (unsigned int)_SwapBufferInFlight);
+#endif
 
 	return true;
 }
@@ -695,6 +759,15 @@ bool CDriverGL3::release()
 	nlassert(_DepthStencilFBOs.empty());
 
 	_SwapBufferCounter = 0;
+	_SwapBufferInFlight = 0;
+	for (size_t i = 0; i < NL3D_GL3_BUFFER_QUEUE_MAX; ++i)
+	{
+		if (_SwapBufferSync[i])
+		{
+			nglDeleteSync(_SwapBufferSync[i]);
+			_SwapBufferSync[i] = 0;
+		}
+	}
 
 	// delete querries
 	while (!_OcclusionQueryList.empty())

code/nel/src/3d/driver/opengl3/driver_opengl.h

@@ -81,6 +81,9 @@
 using NLMISC::CMatrix;
 using NLMISC::CVector;
 
+#define NL3D_GL3_BUFFER_NOT_IN_FLIGHT (std::numeric_limits<uint64>::max())
+#define NL3D_GL3_BUFFER_QUEUE_MAX (3) // Maximum is three frames behind
+
 namespace NL3D {
 namespace NLDRIVERGL3 {
 
@@ -678,6 +681,7 @@ public:
 	virtual bool			supportPackedDepthStencil() const;
 
 	virtual uint64			getSwapBufferCounter() const { return _SwapBufferCounter; }
+	inline uint64			getSwapBufferInFlight() const { return _SwapBufferInFlight; }
 
 	virtual void			setCullMode(TCullMode cullMode);
 	virtual	TCullMode       getCullMode() const;
@@ -1342,6 +1346,9 @@ protected:
 	// is the window active ,
 	bool					_WndActive;
 	uint64					_SwapBufferCounter;
+private:
+	uint64					_SwapBufferInFlight;
+	GLsync					_SwapBufferSync[NL3D_GL3_BUFFER_QUEUE_MAX];
 public:
 	void incrementResetCounter() { ++_ResetCounter; }
 	bool isWndActive() const { return _WndActive; }

code/nel/src/3d/driver/opengl3/driver_opengl_extension.cpp

@@ -632,7 +632,7 @@ bool	registerGlExtensions(CGlExtensions &ext)
 	}
 
 	// Check GL_AMD_pinned_memory
-	ext.AMDPinnedMemory = setupAMDPinnedMemory(glext);
+	ext.AMDPinnedMemory = false; // setupAMDPinnedMemory(glext); // TODO: Proper frame sync check
 
 	// Get the maximum fragment texture unites
 	glGetIntegerv(GL_MAX_TEXTURE_IMAGE_UNITS, &ext.MaxFragmentTextureImageUnits);

code/nel/src/3d/driver/opengl3/driver_opengl_rendering.cpp

@@ -67,6 +67,9 @@ bool CDriverGL3::renderLines(CMaterial& mat, uint32 firstIndex, uint32 nlines)
 	_PrimitiveProfileIn.NLines+= nlines;
 	_PrimitiveProfileOut.NLines+= nlines;
 
+	if (_CurrentVertexBufferGL)
+		_CurrentVertexBufferGL->setFrameInFlight(_SwapBufferCounter);
+
 	return true;
 }
 
@@ -118,6 +121,9 @@ bool CDriverGL3::renderTriangles(CMaterial& mat, uint32 firstIndex, uint32 ntris
 	_PrimitiveProfileIn.NTriangles+= ntris;
 	_PrimitiveProfileOut.NTriangles+= ntris * nPass;
 
+	if (_CurrentVertexBufferGL)
+		_CurrentVertexBufferGL->setFrameInFlight(_SwapBufferCounter);
+
 	return true;
 }
 
@@ -151,6 +157,9 @@ bool CDriverGL3::renderSimpleTriangles(uint32 firstTri, uint32 ntris)
 	_PrimitiveProfileIn.NTriangles+= ntris;
 	_PrimitiveProfileOut.NTriangles+= ntris;
 
+	if (_CurrentVertexBufferGL)
+		_CurrentVertexBufferGL->setFrameInFlight(_SwapBufferCounter);
+
 	return true;
 }
 
@@ -189,6 +198,9 @@ bool CDriverGL3::renderRawPoints(CMaterial& mat, uint32 startIndex, uint32 numPo
 	_PrimitiveProfileIn.NPoints+= numPoints;
 	_PrimitiveProfileOut.NPoints+= numPoints * nPass;
 
+	if (_CurrentVertexBufferGL)
+		_CurrentVertexBufferGL->setFrameInFlight(_SwapBufferCounter);
+
 	return true;
 }
 
@@ -227,6 +239,9 @@ bool CDriverGL3::renderRawLines(CMaterial& mat, uint32 startIndex, uint32 numLin
 	_PrimitiveProfileIn.NLines  += numLines ;
 	_PrimitiveProfileOut.NLines += numLines  * nPass;
 
+	if (_CurrentVertexBufferGL)
+		_CurrentVertexBufferGL->setFrameInFlight(_SwapBufferCounter);
+
 	return true;
 }
 
@@ -267,6 +282,9 @@ bool CDriverGL3::renderRawTriangles(CMaterial& mat, uint32 startIndex, uint32 nu
 	_PrimitiveProfileIn.NTriangles  += numTris ;
 	_PrimitiveProfileOut.NTriangles += numTris  * nPass;
 
+	if (_CurrentVertexBufferGL)
+		_CurrentVertexBufferGL->setFrameInFlight(_SwapBufferCounter);
+
 	return true;
 }
 
@@ -387,6 +405,9 @@ bool CDriverGL3::renderRawQuads(CMaterial& mat, uint32 startIndex, uint32 numQua
 	_PrimitiveProfileIn.NQuads  += numQuads ;
 	_PrimitiveProfileOut.NQuads += numQuads  * nPass;
 
+	if (_CurrentVertexBufferGL)
+		_CurrentVertexBufferGL->setFrameInFlight(_SwapBufferCounter);
+
 	return true;
 }
 

code/nel/src/3d/driver/opengl3/driver_opengl_vertex_buffer.cpp

@@ -51,7 +51,8 @@ IVertexBufferGL3::~IVertexBufferGL3()
 CVertexBufferGL3::CVertexBufferGL3(CDriverGL3 *drv, uint size, uint numVertices, CVertexBuffer::TPreferredMemory preferred, CVertexBuffer *vb) 
 	: IVertexBufferGL3(drv, vb, IVertexBufferGL3::GL3),
 	m_VertexPtr(NULL),
-	m_VertexObjectId(0)
+	m_VertexObjectId(0), 
+	m_FrameInFlight(NL3D_GL3_BUFFER_NOT_IN_FLIGHT)
 {
 	H_AUTO_OGL(CVertexBufferGLARB_CVertexBufferGLARB)
 
@@ -295,6 +296,15 @@ void CVertexBufferGL3::setupVBInfos(CVertexBufferInfo &vb)
 
 // ***************************************************************************
 
+void CVertexBufferGL3::setFrameInFlight(uint64 swapBufferCounter)
+{
+	H_AUTO_OGL(CVertexBufferGL3_setFrameInFlight);
+
+	m_FrameInFlight = swapBufferCounter;
+}
+
+// ***************************************************************************
+
 void CVertexBufferGL3::invalidate()
 {
 	H_AUTO_OGL(CVertexBufferGLARB_invalidate)
@@ -317,7 +327,8 @@ CVertexBufferAMDPinned::CVertexBufferAMDPinned(CDriverGL3 *drv, uint size, uint
 	: IVertexBufferGL3(drv, vb, IVertexBufferGL3::AMDPinned),
 	m_MemType(preferred),
 	m_VertexPtr(NULL),
-	m_VertexObjectId(0)
+	m_VertexObjectId(0),
+	m_FrameInFlight(NL3D_GL3_BUFFER_NOT_IN_FLIGHT)
 {
 	H_AUTO_OGL(CVertexBufferAMDPinned_CVertexBufferAMDPinned)
 
@@ -498,11 +509,20 @@ void CVertexBufferAMDPinned::disable()
 
 void CVertexBufferAMDPinned::setupVBInfos(CVertexBufferInfo &vb)
 {
-	H_AUTO_OGL(CVertexBufferAMDPinned_setupVBInfos)
+	H_AUTO_OGL(CVertexBufferAMDPinned_setupVBInfos);
 
 	vb.VertexObjectId = m_VertexObjectId;
 }
 
+// ***************************************************************************
+
+void CVertexBufferAMDPinned::setFrameInFlight(uint64 swapBufferCounter)
+{
+	H_AUTO_OGL(CVertexBufferAMDPinned_setFrameInFlight);
+
+	m_FrameInFlight = swapBufferCounter;
+}
+
 // ***************************************************************************
 // ***************************************************************************
 // ***************************************************************************

code/nel/src/3d/driver/opengl3/driver_opengl_vertex_buffer.h

@@ -42,6 +42,7 @@ public:
 	virtual	void enable() = 0;
 	virtual	void disable() = 0;
 	virtual void setupVBInfos(CVertexBufferInfo &vb) = 0;
+	virtual void setFrameInFlight(uint64 swapBufferCounter) = 0;
 
 	// test if buffer content is invalid. If so, no rendering should occurs (rendering should silently fail)
 	inline bool isInvalid() { return m_Invalid; }
@@ -71,6 +72,7 @@ public:
 	virtual	void enable();
 	virtual	void disable();
 	virtual void setupVBInfos(CVertexBufferInfo &vb);
+	virtual void setFrameInFlight(uint64 swapBufferCounter);
 	// @}
 
 	/// Invalidate the buffer (when it is lost, or when a lock fails)
@@ -88,6 +90,8 @@ private:
 	std::list<CVertexBufferGL3*>::iterator m_IteratorInLostVBList;
 
 	uint m_VertexObjectId;
+
+	uint64 m_FrameInFlight; // TODO: Array of sz NL3D_GL3_BUFFER_QUEUE_MAX
 };
 
 class CVertexBufferAMDPinned : public IVertexBufferGL3
@@ -105,6 +109,7 @@ public:
 	virtual	void enable();
 	virtual	void disable();
 	virtual void setupVBInfos(CVertexBufferInfo &vb);
+	virtual void setFrameInFlight(uint64 swapBufferCounter);
 	// @}
 
 private:
@@ -113,6 +118,8 @@ private:
 	void *m_VertexPtrAligned;
 	void *m_VertexPtr;
 	uint m_VertexObjectId;
+
+	uint64 m_FrameInFlight;
 };
 
 } // NLDRIVERGL3