Reset active builtin shader when using user shader in D3D driver, fix ryzom/ryzomcore#603

nel/src/3d/driver/direct3d/driver_direct3d.cpp

@@ -3830,9 +3830,15 @@ void CDriverD3D::CLightState::apply(CDriverD3D *driver)
 void CDriverD3D::CRenderTargetState::apply(CDriverD3D *driver)
 {
 	H_AUTO_D3D(CDriverD3D_CRenderTargetState);
-	driver->_DeviceInterface->SetRenderTarget (0, Target);
+	nlassert(TargetOwned); // Can only apply once!
+	driver->_DeviceInterface->SetRenderTarget(0, Target);
 	driver->setupViewport(driver->_Viewport);
 	driver->setupScissor(driver->_Scissor);
+	if (TargetOwned)
+	{
+		Target->Release();
+		TargetOwned = false;
+	}
 }
 
 // ***************************************************************************

nel/src/3d/driver/direct3d/driver_direct3d.h

@@ -1546,11 +1546,13 @@ public:
 			Texture = NULL;
 			Level = 0;
 			CubeFace = 0;
+			TargetOwned = false;
 		}
 		IDirect3DSurface9	*Target;
 		ITexture			*Texture;
 		uint8				Level;
 		uint8				CubeFace;
+		bool				TargetOwned;
 		virtual void apply(CDriverD3D *driver);
 	};
 
@@ -2076,10 +2078,17 @@ public:
 		NL_D3D_CACHE_TEST(CacheTest_RenderTarget, _RenderTarget.Target != target)
 #endif // NL_D3D_USE_RENDER_STATE_CACHE
 		{
+			if (_RenderTarget.TargetOwned)
+			{
+				nlassert(_RenderTarget.Target);
+				_RenderTarget.Target->Release();
+			}
 			_RenderTarget.Target = target;
 			_RenderTarget.Texture = texture;
 			_RenderTarget.Level = level;
 			_RenderTarget.CubeFace = cubeFace;
+			_RenderTarget.TargetOwned = target;
+			target->AddRef();
 
 			touchRenderVariable (&_RenderTarget);
 

nel/src/3d/driver/direct3d/driver_direct3d_material.cpp

@@ -706,9 +706,13 @@ bool CDriverD3D::setupMaterial(CMaterial &mat)
 				// Set the texture states
 				if (text || (stage == 0))
 				{
-					// Doesn't use a pixel shader ? Set the textures stages
-					if (pShader->PixelShader == NULL)
+					if (matShader == CMaterial::Program)
 					{
+						// Do nothing for user pixel shader
+					}
+					else if (!pShader->PixelShader)
+					{
+						// Doesn't use a pixel shader ? Set the textures stages
 						if (pShader->RGBPipe[stage])
 						{
 							setTextureState (stage, D3DTSS_COLOROP, pShader->ColorOp[stage]);
@@ -1145,7 +1149,7 @@ bool CDriverD3D::setupMaterial(CMaterial &mat)
 
 			}
 			break;
-			case CMaterial::Cloud:
+		case CMaterial::Cloud:
 			{
 				H_AUTO_D3D(CDriverD3D_setupMaterial_setupCloudShader)
 				activeShader (&_ShaderCloud);
@@ -1167,7 +1171,7 @@ bool CDriverD3D::setupMaterial(CMaterial &mat)
 					return false;
 			}
 			break;
-			case CMaterial::Water:
+		case CMaterial::Water:
 			{
 				H_AUTO_D3D(CDriverD3D_setupMaterial_setupWaterShader)
 				activeShader(mat.getTexture(3) ? &_ShaderWaterDiffuse : &_ShaderWaterNoDiffuse);
@@ -1296,7 +1300,14 @@ bool CDriverD3D::setupMaterial(CMaterial &mat)
 					}
 				}
 			}
-			// CMaterial::Water
+			break; // CMaterial::Water
+		case CMaterial::Program:
+			{
+				H_AUTO_D3D(CDriverD3D_setupMaterial_setupProgramshader)
+					// No material shader
+					activeShader(NULL);
+			}
+			break;
 		}
 
 		// New material setuped

nel/src/3d/driver/direct3d/driver_direct3d_texture.cpp

@@ -1084,6 +1084,7 @@ void CDriverD3D::swapTextureHandle(ITexture &tex0, ITexture &tex1)
 	swap(t0->Height, t1->Height);
 	swap(t0->SrcCompressed, t1->SrcCompressed);
 	swap(t0->IsCube, t1->IsCube);
+	swap(t0->RenderTarget, t1->RenderTarget);
 	swap(t0->Levels, t1->Levels);
 	swap(t0->FirstMipMap, t1->FirstMipMap);
 	swap(t0->TextureMemory, t1->TextureMemory);

nel/src/3d/fxaa.cpp

@@ -246,6 +246,7 @@ void CFXAA::applyEffect()
 
 	// create render target
 	CTextureUser *otherRenderTarget = m_Driver->getRenderTargetManager().getRenderTarget(width, height, mode2D);
+	nlassert(otherRenderTarget);
 
 	// swap render target
 	CTextureUser texNull;