Added: #1440 Write directory dependencies from the tool logger. Write depend and error log in tga2dds tool

--HG--
branch : build_pipeline_v3

code/nel/include/nel/misc/tool_logger.h

@@ -56,6 +56,7 @@ enum TError
 enum TDepend
 {
 	BUILD, 
+	DIRECTORY, 
 	RUNTIME, 
 };
 
@@ -144,6 +145,9 @@ public:
 			case BUILD:
 				fwrite("BUILD", 1, 5, m_DependLog);
 				break;
+			case DIRECTORY:
+				fwrite("DIRECTORY", 1, 9, m_DependLog);
+				break;
 			case RUNTIME:
 				fwrite("RUNTIME", 1, 7, m_DependLog);
 				break;

code/nel/tools/3d/build_interface/main.cpp

@@ -256,6 +256,8 @@ int main(int nNbArg, char **ppArgs)
 			ToolLogger.release();
 			return -1;
 		}
+		ToolLogger.writeDepend(PIPELINE::DIRECTORY, tgaName, sDir);
+		ToolLogger.writeDepend(PIPELINE::DIRECTORY, uvName, sDir);
 		CPath::getPathContent(sDir, false, false, true, AllMapNames);
 	}
 

code/nel/tools/3d/tga_2_dds/tga2dds.cpp

@@ -25,10 +25,13 @@
 
 #include "../s3tc_compressor_lib/s3tc_compressor.h"
 
+#include <sstream>
+#include "nel/misc/tool_logger.h"
 
 using namespace NLMISC;
 using namespace std;
 
+PIPELINE::CToolLogger ToolLogger;
 
 #define	TGA8	8
 #define	TGA16	16
@@ -204,7 +207,7 @@ void writeInstructions()
 	cout<<"extension \"_usercolor\""<<endl;
 	cout<<"ex : pic.tga, the associated user color file must be : pic_usercolor.tga"<<endl;
 	cout<<endl;
-	cout<<"syntax : tga2dds <input> [-o <output.dds>] [-a <algo>] [-m]"<<endl;
+	cout<<"syntax : tga2dds <input> [-o <output.dds>] [-a <algo>] [-m] [-d dependLog] [-e errorLog]"<<endl;
 	cout<<endl;
 	cout<<"with"<<endl;
 	cout<<"algo      : 1  for DXTC1 (no alpha)"<<endl;
@@ -246,6 +249,8 @@ string		OptOutputFileName;
 uint8		OptAlgo = NOT_DEFINED;
 bool		OptMipMap = false;
 uint		Reduce = 0;
+bool		ForceSkipCheck = false;
+bool		CheckUserColor = true;
 bool	parseOptions(int argc, char **argv)
 {
 	for(sint i=2;i<argc;i++)
@@ -309,6 +314,20 @@ bool	parseOptions(int argc, char **argv)
 			Reduce = 7;
 		else if(!strcmp(argv[i], "-r8"))
 			Reduce = 8;
+		else if(!strcmp(argv[i], "-d"))
+		{
+			++i;
+			ToolLogger.initDepend(argv[i]);
+		}
+		else if(!strcmp(argv[i], "-e"))
+		{
+			++i;
+			ToolLogger.initDepend(argv[i]);
+		}
+		else if(!strcmp(argv[i], "-f"))
+			ForceSkipCheck = true;
+		else if(!strcmp(argv[i], "-nousercolor"))
+			CheckUserColor = false;
 		// What is this option?
 		else
 		{
@@ -382,6 +401,8 @@ int main(int argc, char **argv)
 	if(!parseOptions(argc, argv))
 	{
 		writeInstructions();
+		ToolLogger.writeError(PIPELINE::ERROR, "", "Invalid settings");
+		ToolLogger.release();
 		return 0;
 	}
 
@@ -394,23 +415,33 @@ int main(int argc, char **argv)
 	std::string inputFileName(argv[1]);
 	if(inputFileName.find("_usercolor")<inputFileName.length())
 	{
+		ToolLogger.writeError(PIPELINE::ERROR, inputFileName, "User color cannot be converted directly, it is the source file for a channel of another dds file");
+		ToolLogger.release();
 		return 0;
 	}
 	NLMISC::CIFile input;
 	if(!input.open(inputFileName))
 	{
 		cerr<<"Can't open input file "<<inputFileName<<endl;
+		ToolLogger.writeError(PIPELINE::ERROR, inputFileName, "Can't open input file");
+		ToolLogger.release();
 		return 1;
 	}
 	uint8 imageDepth = picTga.load(input);
 	if(imageDepth==0)
 	{
 		cerr<<"Can't load file : "<<inputFileName<<endl;
+		ToolLogger.writeError(PIPELINE::ERROR, inputFileName, "Can't load file");
+		ToolLogger.release();
 		return 1;
 	}
 	if(imageDepth!=16 && imageDepth!=24 && imageDepth!=32 && imageDepth!=8)
 	{
-		cerr<<"Image not supported : "<<imageDepth<<endl;
+		stringstream ss;
+		ss << "Image depth not supported: " << imageDepth;
+		cerr << ss.str() << endl;
+		ToolLogger.writeError(PIPELINE::ERROR, inputFileName, ss.str());
+		ToolLogger.release();
 		return 1;
 	}
 	input.close();
@@ -444,11 +475,15 @@ int main(int argc, char **argv)
 			algo = DXT5;
 	}
 
+	ToolLogger.writeDepend(PIPELINE::BUILD, outputFileName, inputFileName);
+
 	// Data check
 	//===========
-	if(dataCheck(inputFileName,outputFileName, OptAlgo, OptMipMap))
+	if(!ForceSkipCheck && dataCheck(inputFileName,outputFileName, OptAlgo, OptMipMap))
 	{
 		cout<<outputFileName<<" : a recent dds file already exists"<<endl;
+		ToolLogger.writeError(PIPELINE::WARNING, outputFileName, "A more recent dds file already exists, this file may not have been built correctly");
+		ToolLogger.release();
 		return 0;
 	}
 
@@ -499,8 +534,10 @@ int main(int argc, char **argv)
 
 	// Reading second Tga for user color, don't complain if _usercolor is missing
 	NLMISC::CIFile input2;
-	if (CPath::exists(userColorFileName) && input2.open(userColorFileName))
+	if (CheckUserColor && CPath::exists(userColorFileName) && input2.open(userColorFileName))
 	{
+		ToolLogger.writeDepend(PIPELINE::BUILD, outputFileName, userColorFileName);
+
 		picTga2.load(input2);
 		uint32 height2 = picTga2.getHeight();
 		uint32 width2 = picTga2.getWidth();
@@ -606,6 +643,8 @@ int main(int argc, char **argv)
 		if(!output.open(outputFileName))
 		{
 			cerr<<"Can't open output file "<<outputFileName<<endl;
+			ToolLogger.writeError(PIPELINE::ERROR, outputFileName, "Can't open output file");
+			ToolLogger.release();
 			return 1;
 		}
 		try
@@ -632,6 +671,8 @@ int main(int argc, char **argv)
 		catch(const NLMISC::EWriteError &e)
 		{
 			cerr<<e.what()<<endl;
+			ToolLogger.writeError(PIPELINE::ERROR, outputFileName, std::string("Write error: ") + e.what());
+			ToolLogger.release();
 			return 1;
 		}
 
@@ -668,6 +709,8 @@ int main(int argc, char **argv)
 		if(!output.open(outputFileName))
 		{
 			cerr<<"Can't open output file "<<outputFileName<<endl;
+			ToolLogger.writeError(PIPELINE::ERROR, outputFileName, "Can't open output file");
+			ToolLogger.release();
 			return 1;
 		}
 		try
@@ -678,11 +721,14 @@ int main(int argc, char **argv)
 		catch(const NLMISC::EWriteError &e)
 		{
 			cerr<<e.what()<<endl;
+			ToolLogger.writeError(PIPELINE::ERROR, outputFileName, std::string("Write error: ") + e.what());
+			ToolLogger.release();
 			return 1;
 		}
 
 		output.close();
 	}
 
+	ToolLogger.release();
 	return 0;
 }	

code/nel/tools/pipeline/plugin_library/pipeline_process.cpp

@@ -48,6 +48,50 @@ IPipelineProcess *IPipelineProcess::getInstance()
 	return static_cast<IPipelineProcess *>(NLMISC::INelContext::getInstance().getSingletonPointer("IPipelineProcess"));
 }
 
+bool IPipelineProcess::getValue(bool &result, const std::string &name)
+{
+	// It's true.
+	std::string resultString;
+	if (!getValue(resultString, name))
+		return false;
+	if (resultString == "true")
+		result = true;
+	if (resultString == "false")
+		result = false;
+	else
+	{
+		nlwarning("Value '%s' with result '%s' is not a boolean", name.c_str(), resultString.c_str());
+		return false;
+	}
+	return true;
+}
+
+bool IPipelineProcess::getValue(uint &result, const std::string &name)
+{
+	std::string resultString;
+	if (!getValue(resultString, name))
+		return false;
+	if (!NLMISC::fromString(resultString, result))
+	{
+		nlwarning("Value '%s' with result '%s' is not an integer", name.c_str(), resultString.c_str());
+		return false;
+	}
+	return true;
+}
+
+bool IPipelineProcess::getValue(sint &result, const std::string &name)
+{
+	std::string resultString;
+	if (!getValue(resultString, name))
+		return false;
+	if (!NLMISC::fromString(resultString, result))
+	{
+		nlwarning("Value '%s' with result '%s' is not an integer", name.c_str(), resultString.c_str());
+		return false;
+	}
+	return true;
+}
+
 } /* namespace PIPELINE */
 
 /* end of file */

code/nel/tools/pipeline/plugin_library/pipeline_process.h

@@ -90,6 +90,9 @@ public:
 	
 	/// Get a value from the currently active project configuration. If false, don't use, no need to write warnings to service log, already written, set exit state and exit if necessary
 	virtual bool getValue(std::string &result, const std::string &name) = 0;
+	bool getValue(bool &result, const std::string &name);
+	bool getValue(uint &result, const std::string &name);
+	bool getValue(sint &result, const std::string &name);
 	virtual bool getValues(std::vector<std::string> &resultAppend, const std::string &name) = 0;
 	virtual bool getValueNb(uint &result, const std::string &name) = 0;
 

code/nel/tools/pipeline/service/metadata_storage.cpp

@@ -80,11 +80,12 @@ void CFileDepend::CDependency::serial(NLMISC::IStream &stream) throw (NLMISC::ES
 
 void CFileDepend::serial(NLMISC::IStream &stream) throw (NLMISC::EStream)
 {
-	uint version = stream.serialVersion(2);
+	uint version = stream.serialVersion(3);
 	if (version >= 2) stream.serial(BuildStart); else BuildStart = 0;
 	stream.serial(CRC32);
 	stream.serialCont(Dependencies);
-	stream.serialCont(RuntimeDependencies);
+	if (version >= 3) stream.serialCont(DirectoryDependencies); else DirectoryDependencies.clear();
+	if (version >= 3) stream.serialCont(RuntimeDependencies); else { std::vector<CDependency> dummy; stream.serialCont(dummy); }
 }
 
 void CProcessResult::CFileResult::serial(NLMISC::IStream &stream) throw (NLMISC::EStream)

code/nel/tools/pipeline/service/metadata_storage.h

@@ -114,7 +114,8 @@ public:
 		void serial(NLMISC::IStream &stream) throw (NLMISC::EStream);
 	};
 	std::vector<CDependency> Dependencies;
-	std::vector<CDependency> RuntimeDependencies; // informational (TODO)
+	std::vector<std::string> DirectoryDependencies; // In case the file needs to be informed when files are added to a directory.
+	std::vector<std::string> RuntimeDependencies; // informational (TODO) User can use this to check if any files are missing.
 
 	void serial(NLMISC::IStream &stream) throw (NLMISC::EStream);
 };

code/nel/tools/pipeline/service/pipeline_process_impl_buildinfo.cpp

@@ -540,6 +540,7 @@ bool CPipelineProcessImpl::needsToBeRebuiltSub(const std::vector<std::string> &i
 	// Find out if any files were added in dependency directories since last build start
 	if (inputModified)
 	{
+		// TODO: ONLY CHECK INPUT PATHS GIVEN BY DEPEND META FILES
 		for (std::vector<std::string>::const_iterator it = inputPaths.begin(), end = inputPaths.end(); it != end; ++it)
 		{
 			const std::string &path = *it;

code/nel/tools/pipeline/service/pipeline_process_impl_toollog.cpp

@@ -153,6 +153,8 @@ void CPipelineProcessImpl::parseToolLog(const std::string &dependLogFile, const
 				TDepend type;
 				if (tabbedLine[0] == "BUILD")
 					type = BUILD;
+				else if (tabbedLine[0] == "DIRECTORY")
+					type = DIRECTORY;
 				else if (tabbedLine[0] == "RUNTIME")
 					type = RUNTIME;
 				else
@@ -166,7 +168,7 @@ void CPipelineProcessImpl::parseToolLog(const std::string &dependLogFile, const
 				}
 				std::string outputFile = standardizePath(tabbedLine[1], false);
 				// std::string outputFileMacro = macroPath(outputFile);
-				std::string inputFile = standardizePath(tabbedLine[2], false);
+				std::string inputFile = standardizePath(tabbedLine[2], type == DIRECTORY);
 				std::string inputFileMacro = macroPath(inputFile);
 				std::map<std::string, CFileDepend>::iterator metaDependIt = metaDepends.find(outputFile);
 				if (metaDependIt == metaDepends.end())
@@ -187,46 +189,58 @@ void CPipelineProcessImpl::parseToolLog(const std::string &dependLogFile, const
 
 					m_FileStatusOutputCache[outputFile] = status;
 				}
-				CFileDepend::CDependency dependency;
+				switch (type)
-				dependency.MacroPath = inputFileMacro;
-				std::map<std::string, CFileStatus>::iterator statusIt = statusCache.find(inputFile);
-				if (statusIt == statusCache.end())
 				{
-					if (!isFileDependency(inputFile))
+				case BUILD:
 					{
-						m_SubTaskErrorMessage = std::string("Invalid dependency '") + inputFile + "'";
+						CFileDepend::CDependency dependency;
-						m_SubTaskResult = FINISH_ERROR;
+						dependency.MacroPath = inputFileMacro;
-						file.close();
+						std::map<std::string, CFileStatus>::iterator statusIt = statusCache.find(inputFile);
-						return;
+						if (statusIt == statusCache.end())
+						{
+							if (!isFileDependency(inputFile))
+							{
+								m_SubTaskErrorMessage = std::string("Invalid dependency '") + inputFile + "'";
+								m_SubTaskResult = FINISH_ERROR;
+								file.close();
+								return;
+							}
+							CFileStatus statusOriginal;
+							if (!getDependencyFileStatusCached(statusOriginal, inputFile))
+							{
+								m_SubTaskErrorMessage = std::string("Cached status for '") + inputFile + "' does not exist, this may be a programming error";
+								m_SubTaskResult = FINISH_ERROR;
+								file.close();
+								return;
+							}
+							CFileStatus status;
+							if (!getDependencyFileStatusLatest(status, inputFile))
+							{
+								m_SubTaskErrorMessage = std::string("Invalid status for '") + inputFile + "', file may have changed during build";
+								m_SubTaskResult = FINISH_ERROR;
+								file.close();
+								return;
+							}
+							if (statusOriginal.CRC32 != status.CRC32)
+							{
+								m_SubTaskErrorMessage = std::string("Status checksums changed for '") + inputFile + "', file has changed during build";
+								m_SubTaskResult = FINISH_ERROR;
+								file.close();
+								return;
+							}
+							statusCache[inputFile] = status;
+							statusIt = statusCache.find(inputFile);
+						}
+						dependency.CRC32 = statusIt->second.CRC32;
+						metaDependIt->second.Dependencies.push_back(dependency);
 					}
-					CFileStatus statusOriginal;
+				case DIRECTORY:
-					if (!getDependencyFileStatusCached(statusOriginal, inputFile))
+					metaDependIt->second.DirectoryDependencies.push_back(inputFileMacro);
-					{
+					break;
-						m_SubTaskErrorMessage = std::string("Cached status for '") + inputFile + "' does not exist, this may be a programming error";
+				case RUNTIME:
-						m_SubTaskResult = FINISH_ERROR;
+					metaDependIt->second.RuntimeDependencies.push_back(inputFileMacro);
-						file.close();
+					break;
-						return;
-					}
-					CFileStatus status;
-					if (!getDependencyFileStatusLatest(status, inputFile))
-					{
-						m_SubTaskErrorMessage = std::string("Invalid status for '") + inputFile + "', file may have changed during build";
-						m_SubTaskResult = FINISH_ERROR;
-						file.close();
-						return;
-					}
-					if (statusOriginal.CRC32 != status.CRC32)
-					{
-						m_SubTaskErrorMessage = std::string("Status checksums changed for '") + inputFile + "', file has changed during build";
-						m_SubTaskResult = FINISH_ERROR;
-						file.close();
-						return;
-					}
-					statusCache[inputFile] = status;
-					statusIt = statusCache.find(inputFile);
 				}
-				dependency.CRC32 = statusIt->second.CRC32;
-				metaDependIt->second.Dependencies.push_back(dependency);
 			}
 		}
 

code/ryzom/common/data_leveldesign/leveldesign/DFN/pipeline/pipeline_process_texture_dds.dfn

@@ -3,7 +3,10 @@
   <ELEMENT Name="SrcDirectories" Type="Type" Filename="pipeline_path_endslash.typ" Array="true"/>
   <ELEMENT Name="DstDirectory" Type="Type" Filename="pipeline_path_endslash.typ"/>
   <ELEMENT Name="Algorithm" Type="Type" Filename="string.typ"/>
-  <ELEMENT Name="CreateMipMap" Type="Type" Filename="boolean.typ"/>
+  <ELEMENT Name="CreateMipMap" Type="Type" Filename="boolean.typ" Default="false"/>
   <ELEMENT Name="ReduceFactor" Type="Type" Filename="uint8.typ" Default="0"/>
-  <LOG>Sat Aug 04 15:22:53 2012 (kaetemi) Dfn Structure = </LOG>
+  <ELEMENT Name="CheckUserColor" Type="Type" Filename="boolean.typ" Default="false"/>
+  <LOG>Sat Aug 04 15:22:53 2012 (kaetemi) Dfn Structure = 
+Sat Aug 04 16:36:51 2012 (kaetemi) Dfn Structure = 
+Sat Aug 04 16:38:11 2012 (kaetemi) Dfn Structure = </LOG>
 </DFN>