Merged: build_pipeline_v3

--HG--
branch : build_pipeline_v3

code/nel/tools/3d/build_interface/main.cpp

@@ -280,6 +280,9 @@ int main(int nNbArg, char **ppArgs)
 			inFile.open( AllMapNames[i] );
 			pBtmp->load(inFile);
 			AllMaps.push_back(pBtmp);
+
+			if (pBtmp->getWidth() == 0 || pBtmp->getHeight() == 0)
+				ToolLogger.writeError(PIPELINE::WARNING, AllMapNames[i], "Bitmap of width or height 0, corrupt data");
 		}
 		catch (const NLMISC::Exception &e)
 		{

code/nel/tools/pipeline/plugin_library/pipeline_interface.h

@@ -60,6 +60,7 @@ public:
 
 	// ***************** PLUGIN UTILITY FUNCTIONS *****************
 	// ************* DO NOT USE FROM INSIDE A PROCESS *************
+	// ************ USE ONLY FOR SPECIAL TOOL COMMANDS ************
 
 	/// Get the configuration file of the pipeline service. Must only be used for configuration values that may be different on different services, such as tool paths.
 	virtual NLMISC::CConfigFile &getConfigFile() = 0;
@@ -78,6 +79,9 @@ public:
 
 	/// Call when a runnable task has ended to reset to STATE_IDLE.
 	virtual void endedDirectCode() = 0;
+
+	/// Check if service is exiting
+	virtual bool isExiting() = 0;
 	
 }; /* class IPipelineInterface */
 

code/nel/tools/pipeline/plugin_max/pipeline_plugin_max.cpp

@@ -36,6 +36,9 @@
 // NeL includes
 #include "nel/misc/dynloadlib.h"
 #include "nel/misc/debug.h"
+#include "nel/misc/path.h"
+#include "nel/misc/algo.h"
+#include "nel/misc/file.h"
 
 // Project includes
 #include "../plugin_library/pipeline_interface.h"
@@ -83,6 +86,463 @@ CPipelinePluginMax::~CPipelinePluginMax()
 	
 }*/
 
+namespace {
+
+std::set<std::string> MissingFiles;
+std::map<std::string, std::string> KnownFileCache;
+
+// COPY FROM PIPELINE_SERVICE.CPP WITH BACKSLASHES INSTEAD OF FORWARD SLASHES
+std::string standardizePath(const std::string &path, bool addFinalSlash)
+{
+	// check empty path
+	if (path.empty())
+		return "";
+
+	std::string newPath;
+	newPath.resize(path.size() + 1);
+
+	std::string::size_type j = 0;
+	for (std::string::size_type i = 0; i < path.size(); ++i)
+	{
+		if (path[i] == '\\' || path[i] == '/')
+		{
+			if (j <= 1 && path[i] == '\\')
+			{
+				// for windows network
+				newPath[j] = '\\';
+				++j;
+			}
+			else if (j == 0 || newPath[j - 1] != '\\')
+			{
+				newPath[j] = '\\';
+				++j;
+			}
+		}
+		else
+		{
+			newPath[j] = path[i];
+			++j;
+		}
+	}
+	newPath[j] = 0;
+	newPath.resize(j);
+
+	// add terminal slash
+	if (addFinalSlash && newPath[newPath.size()-1] != '\\')
+		newPath += '\\';
+
+	return newPath;
+}
+
+inline bool isCharacter(char c)
+{
+	return (32 <= c /*&& c <= 127) || (161 <= c*/ && c <= 255);
+}
+
+inline char stripFrenchLocale(char c)
+{
+	if (192 <= c && c <= 197) return 'a';
+	if (200 <= c && c <= 203) return 'e';
+	if (204 <= c && c <= 207) return 'i';
+	if (210 <= c && c <= 214) return 'o';
+	if (217 <= c && c <= 220) return 'u';
+	if (c == 221) return 'y';
+	if (224 <= c && c <= 229) return 'a';
+	if (232 <= c && c <= 235) return 'e';
+	if (236 <= c && c <= 239) return 'i';
+	if (242 <= c && c <= 246) return 'o';
+	if (249 <= c && c <= 252) return 'u';
+	if (c == 253 || c == 255) return 'y';
+	return c;
+}
+
+// maxRewritePaths W:/database/interfaces/anims_max
+
+std::string rewritePath(const std::string &path, const std::string &databaseDirectory)
+{
+	static std::set<std::string> fileNameCache;
+
+	std::string stdPath = standardizePath(path, false);
+	for (std::string::size_type i = 0; i < stdPath.size(); ++i)
+		stdPath[i] = stripFrenchLocale(stdPath[i]);
+	stdPath = NLMISC::toLower(stdPath);
+
+	// TODO: remove ./stuff/caravan/agents/_textures/actors/trame.png
+
+	NLMISC::strFindReplace(stdPath, "w:\\database\\", databaseDirectory);
+	NLMISC::strFindReplace(stdPath, "\\\\amiga\\3d\\database\\", databaseDirectory);
+	NLMISC::strFindReplace(stdPath, "ma_hom_armor_01", "ma_hom_armor01");
+	NLMISC::strFindReplace(stdPath, "ma_hof_armor_01", "ma_hof_armor01");
+	NLMISC::strFindReplace(stdPath, "ma_hom_armor_00", "ma_hom_armor00");
+	NLMISC::strFindReplace(stdPath, "ma_hof_armor_00", "ma_hof_armor00");
+	NLMISC::strFindReplace(stdPath, "zo_hom_armor_00", "zo_hom_armor00");
+	NLMISC::strFindReplace(stdPath, "zo_hof_armor_00", "zo_hof_armor00");
+	NLMISC::strFindReplace(stdPath, "fy_hof_cheveux_shave01", "fy_hof_cheveux_shave");
+
+	NLMISC::strFindReplace(stdPath, "matis\\agents\\_textures\\actors\\ma_hom_underwear_avtbras", "tryker\\agents\\_textures\\actors\\tr_hom_underwear_avtbras");
+	NLMISC::strFindReplace(stdPath, "matis\\agents\\_textures\\actors\\ma_hom_underwear_epaule", "tryker\\agents\\_textures\\actors\\tr_hom_underwear_epaule");
+	NLMISC::strFindReplace(stdPath, "matis\\agents\\_textures\\actors\\ma_hom_underwear_hand-downside", "tryker\\agents\\_textures\\actors\\tr_hom_underwear_hand-downside");
+	NLMISC::strFindReplace(stdPath, "matis\\agents\\_textures\\actors\\ma_hom_underwear_hand-upside", "tryker\\agents\\_textures\\actors\\tr_hom_underwear_hand-upside");
+	NLMISC::strFindReplace(stdPath, "matis\\agents\\_textures\\actors\\ma_hom_underwear_molet", "tryker\\agents\\_textures\\actors\\tr_hom_underwear_molet");
+	NLMISC::strFindReplace(stdPath, "matis\\agents\\_textures\\actors\\ma_hof_underwear_hand-downside", "tryker\\agents\\_textures\\actors\\tr_hof_underwear_hand-downside");
+	NLMISC::strFindReplace(stdPath, "matis\\agents\\_textures\\actors\\ma_hof_underwear_hand-upside", "tryker\\agents\\_textures\\actors\\tr_hof_underwear_hand-upside");
+
+	NLMISC::strFindReplace(stdPath, "tr_hof_underwear_torso", "tr_hof_underwear_torse");
+	NLMISC::strFindReplace(stdPath, "tr_hof_underwear_hand-", "tr_hof_underwear_hand_");
+
+	NLMISC::strFindReplace(stdPath, "fy_hom_armor00_avbras.", "fy_hom_armor00_avbras_c1.");
+	NLMISC::strFindReplace(stdPath, "fy_hom_armor00_bottes.", "fy_hom_armor00_bottes_c1.");
+	NLMISC::strFindReplace(stdPath, "fy_hom_armor00_bras.", "fy_hom_armor00_bras_c1.");
+	NLMISC::strFindReplace(stdPath, "fy_hom_armor00_cuissear.", "fy_hom_armor00_cuissear_c1.");
+	NLMISC::strFindReplace(stdPath, "fy_hom_armor00_cuisseav.", "fy_hom_armor00_cuisseav_c1.");
+	NLMISC::strFindReplace(stdPath, "fy_hom_armor00_dos.", "fy_hom_armor00_dos_c1.");
+	NLMISC::strFindReplace(stdPath, "fy_hom_armor00_hand-downside.", "fy_hom_armor00_hand-downside_c1.");
+	NLMISC::strFindReplace(stdPath, "fy_hom_armor00_hand-upside.", "fy_hom_armor00_hand-upside_c1.");
+	NLMISC::strFindReplace(stdPath, "fy_hom_armor00_pieds.", "fy_hom_armor00_pieds_c1.");
+	NLMISC::strFindReplace(stdPath, "fy_hom_armor00_torse.", "fy_hom_armor00_torse_c1.");
+
+	NLMISC::strFindReplace(stdPath, "interfaces_visage", "visage_interface");
+
+	if (stdPath.find("\\trame.") != std::string::npos)
+		stdPath = standardizePath(databaseDirectory + "/stuff/lod_actors/texture_lod/trame.png", false);
+	if (stdPath.find("\\tr_hof_visage_c1.") != std::string::npos)
+		stdPath = standardizePath(databaseDirectory + "/stuff/tryker/agents/_textures/actors/tr_hof_visage_c1.png", false);
+	if (stdPath.find("\\tr_hof_visage_c2.") != std::string::npos)
+		stdPath = standardizePath(databaseDirectory + "/stuff/tryker/agents/_textures/actors/tr_hof_visage_c2.png", false);
+	if (stdPath.find("\\tr_hof_visage_c3.") != std::string::npos)
+		stdPath = standardizePath(databaseDirectory + "/stuff/tryker/agents/_textures/actors/tr_hof_visage_c3.png", false);
+	if (stdPath.find("\\ma_hof_cheveux_medium02.") != std::string::npos)
+		stdPath = standardizePath(databaseDirectory + "/stuff/matis/agents/_textures/actors/ma_hof_cheveux_medium02.png", false);
+
+	/*if (stdPath.size() > path.size())
+	{
+		nlwarning("Path size becomes too large: '%s' -> '%s'", path.c_str(), stdPath.c_str());
+		return path;
+	}*/
+
+	if (NLMISC::CFile::getFilename(stdPath) == stdPath)
+	{
+		breakable
+		{
+			if (fileNameCache.find(stdPath) == fileNameCache.end() || KnownFileCache.find(stdPath) == KnownFileCache.end())
+			{
+				if (stdPath[stdPath.size() - 3] == 't' && stdPath[stdPath.size() - 2] == 'g' && stdPath[stdPath.size() - 1] == 'a')
+				{
+					stdPath[stdPath.size() - 3] = 'p';
+					stdPath[stdPath.size() - 2] = 'n';
+					stdPath[stdPath.size() - 1] = 'g';
+					if (fileNameCache.find(stdPath) != fileNameCache.end())
+						break;
+				}
+				nlwarning("File name not known: '%s' ('%s')", path.c_str(), stdPath.c_str());
+				return stdPath;
+			}
+		}
+		return stdPath;
+	}
+	else
+	{
+		if (stdPath.find(databaseDirectory) == std::string::npos)
+		{
+			if (stdPath[1] == ':' || (stdPath[0] == '\\' && stdPath[1] == '\\'))
+			{
+				if (KnownFileCache.find(NLMISC::CFile::getFilename(stdPath)) != KnownFileCache.end())
+					return KnownFileCache[NLMISC::CFile::getFilename(stdPath)];
+
+				if (stdPath[stdPath.size() - 3] == 't' && stdPath[stdPath.size() - 2] == 'g' && stdPath[stdPath.size() - 1] == 'a')
+				{ stdPath[stdPath.size() - 3] = 'p'; stdPath[stdPath.size() - 2] = 'n'; stdPath[stdPath.size() - 1] = 'g'; }
+				if (KnownFileCache.find(NLMISC::CFile::getFilename(stdPath)) != KnownFileCache.end())
+					return KnownFileCache[NLMISC::CFile::getFilename(stdPath)];
+
+				if (stdPath[stdPath.size() - 3] == 'p' && stdPath[stdPath.size() - 2] == 'n' && stdPath[stdPath.size() - 1] == 'g')
+				{ stdPath[stdPath.size() - 3] = 't'; stdPath[stdPath.size() - 2] = 'g'; stdPath[stdPath.size() - 1] = 'a'; }
+				if (KnownFileCache.find(NLMISC::CFile::getFilename(stdPath)) != KnownFileCache.end())
+					return KnownFileCache[NLMISC::CFile::getFilename(stdPath)];
+
+				nlwarning("Path not in database: '%s' ('%s')", path.c_str(), stdPath.c_str());
+				MissingFiles.insert(path);
+				return stdPath;
+			}
+			else
+			{
+				// invalid path, don't care too much
+				return stdPath;
+			}
+		}
+
+		breakable
+		{
+			if (!NLMISC::CFile::fileExists(stdPath))
+			{
+				if (stdPath[stdPath.size() - 3] == 't' && stdPath[stdPath.size() - 2] == 'g' && stdPath[stdPath.size() - 1] == 'a')
+				{
+					stdPath[stdPath.size() - 3] = 'p';
+					stdPath[stdPath.size() - 2] = 'n';
+					stdPath[stdPath.size() - 1] = 'g';
+					if (NLMISC::CFile::fileExists(stdPath))
+						break;
+				}
+				{
+					std::string stdPathVv2 = standardizePath(NLMISC::CFile::getPath(stdPath) + "/vv2/" + NLMISC::CFile::getFilename(stdPath), false);
+					bool vv2works = false;
+					if (NLMISC::CFile::fileExists(stdPathVv2))
+					{
+						vv2works = true;
+					}
+					else
+					{
+						if (stdPathVv2[stdPathVv2.size() - 3] == 'p' && stdPathVv2[stdPathVv2.size() - 2] == 'n' && stdPathVv2[stdPathVv2.size() - 1] == 'g')
+						{
+							stdPathVv2[stdPathVv2.size() - 3] = 't';
+							stdPathVv2[stdPathVv2.size() - 2] = 'g';
+							stdPathVv2[stdPathVv2.size() - 1] = 'a';
+							if (NLMISC::CFile::fileExists(stdPathVv2))
+							{
+								vv2works = true;
+							}
+						}
+					}
+					if (vv2works)
+					{
+						// try with vv2
+						/*if (stdPathVv2.size() > path.size())
+						{
+							nlwarning("Path with vv2 size becomes too large: '%s' -> '%s'", path.c_str(), stdPathVv2.c_str());
+							return stdPath;
+						}
+						else
+						{*/
+							stdPath = stdPathVv2;
+							break;
+						//}
+					}
+				}
+				// try find
+				if (KnownFileCache.find(NLMISC::CFile::getFilename(stdPath)) != KnownFileCache.end())
+					return KnownFileCache[NLMISC::CFile::getFilename(stdPath)];
+
+				if (stdPath[stdPath.size() - 3] == 't' && stdPath[stdPath.size() - 2] == 'g' && stdPath[stdPath.size() - 1] == 'a')
+				{ stdPath[stdPath.size() - 3] = 'p'; stdPath[stdPath.size() - 2] = 'n'; stdPath[stdPath.size() - 1] = 'g'; }
+				if (KnownFileCache.find(NLMISC::CFile::getFilename(stdPath)) != KnownFileCache.end())
+					return KnownFileCache[NLMISC::CFile::getFilename(stdPath)];
+
+				if (stdPath[stdPath.size() - 3] == 'p' && stdPath[stdPath.size() - 2] == 'n' && stdPath[stdPath.size() - 1] == 'g')
+				{ stdPath[stdPath.size() - 3] = 't'; stdPath[stdPath.size() - 2] = 'g'; stdPath[stdPath.size() - 1] = 'a'; }
+				if (KnownFileCache.find(NLMISC::CFile::getFilename(stdPath)) != KnownFileCache.end())
+					return KnownFileCache[NLMISC::CFile::getFilename(stdPath)];
+
+				nlwarning("Path file does not exist: '%s' ('%s')", path.c_str(), stdPath.c_str());
+				MissingFiles.insert(path);
+				return stdPath;
+			}
+		}
+	}
+
+	fileNameCache.insert(NLMISC::CFile::getFilename(stdPath));
+	return stdPath;
+}
+
+class CMaxRewritePathsCommand : public NLMISC::IRunnable
+{
+public:
+	NLMISC::CLog *Log;
+	std::string SrcDirectoryRecursive;
+	std::string DatabaseDirectory;
+
+	virtual void getName(std::string &result) const 
+	{ result = "CMaxRewritePathsCommand"; }
+
+	void doFile(const std::string &filePath)
+	{
+		if (filePath[filePath.size() - 3] == 'm' && filePath[filePath.size() - 2] == 'a' && filePath[filePath.size() - 1] == 'x')
+		{
+			nldebug("File: '%s'", filePath.c_str());
+
+			std::vector<char> buffer;
+			buffer.resize(NLMISC::CFile::getFileSize(filePath));
+
+			// read
+			{
+				NLMISC::CIFile ifile;
+				ifile.open(filePath, false);
+				ifile.serialBuffer((uint8 *)(&buffer[0]), buffer.size());
+				ifile.close();
+			}
+
+			// find paths
+			for (std::vector<char>::size_type i = 256; i < buffer.size(); ++i) // skip the first 256 lol :)
+			{
+				if (((NLMISC::toLower(buffer[i - 1]) == 'x' && NLMISC::toLower(buffer [i - 2]) == 'a' && NLMISC::toLower(buffer[i - 3]) == 'm')
+					|| (NLMISC::toLower(buffer[i - 1]) == 'a' && NLMISC::toLower(buffer [i - 2]) == 'g' && NLMISC::toLower(buffer[i - 3]) == 't')
+					|| (NLMISC::toLower(buffer[i - 1]) == 'g' && NLMISC::toLower(buffer [i - 2]) == 'n' && NLMISC::toLower(buffer[i - 3]) == 'p'))
+					&& (NLMISC::toLower(buffer[i - 4]) == '.'))
+				{
+					// buffer[i] is the character after the path! :)
+					std::vector<char>::size_type beginPath = 0;
+					for (std::vector<char>::size_type j = i - 4; j > 0; --j)
+					{
+						if (!isCharacter(buffer[j]))
+						{
+							if (buffer[j + 1] == '\\' && buffer[j + 2] == '\\' || buffer[j] == 0)
+							{
+								beginPath = j + 1;
+								break;
+							}
+							// nlwarning("Invalid characters '%i' in path at %i, len %i!", (uint32)buffer[j], (uint32)j, (uint32)(i - j));
+							// beginPath = j + 1; // test
+							break;
+						}
+						if (buffer[j] == ':')
+						{
+							beginPath = j - 1;
+							break;
+						}
+					}
+					if (beginPath != 0)
+					{
+						std::vector<char>::size_type sizePath = i - beginPath;
+						std::string foundPath = std::string(&buffer[beginPath], sizePath); //std::string(buffer.at(beginPath), buffer.at(i));
+						//nldebug("Found path: '%s' from %i to %i", foundPath.c_str(), (uint32)beginPath, (uint32)i);
+						std::string fixedPath = rewritePath(foundPath, DatabaseDirectory);
+						//nldebug("Rewrite to: '%s'", fixedPath.c_str());
+					}
+				}
+			}
+
+			//NLMISC::CFile::re
+
+			// ...
+		}
+	}
+
+	// maxRewritePaths W:/database/interfaces/anims_max
+
+	void doDirectory(const std::string &directoryPath)
+	{
+		nldebug("Directory: '%s'", directoryPath.c_str());
+
+		std::string dirPath = standardizePath(directoryPath, true);
+		std::vector<std::string> dirContents;
+
+		NLMISC::CPath::getPathContent(dirPath, false, true, true, dirContents);
+		
+		for (std::vector<std::string>::iterator it = dirContents.begin(), end = dirContents.end(); it != end; ++it)
+		{
+			const std::string &subPath = standardizePath(*it, false);
+			
+			if (NLMISC::CFile::isDirectory(subPath))
+			{
+				if (subPath.find("\\.") == std::string.npos)
+					doDirectory(subPath);
+			}
+			else
+				doFile(subPath);
+			
+			if (PIPELINE::IPipelineInterface::getInstance()->isExiting())
+				return;
+		}
+	}
+
+	virtual void run()
+	{
+		std::string tempDirectory = PIPELINE::IPipelineProcess::getInstance()->getTempDirectory();
+		DatabaseDirectory = standardizePath(PIPELINE::IPipelineInterface::getInstance()->getConfigFile().getVar("DatabaseDirectory").asString(0), true);
+		nlinfo("DatabaseDirectory: '%s'", DatabaseDirectory.c_str());
+
+		doDirectory(SrcDirectoryRecursive);
+
+		for (std::set<std::string>::iterator it = MissingFiles.begin(), end = MissingFiles.end(); it != end; ++it)
+			nlinfo("Missing: '%s'", (*it).c_str());
+
+		PIPELINE::IPipelineInterface::getInstance()->endedRunnableTask();
+	}
+};
+CMaxRewritePathsCommand s_MaxRewritePathsCommand;
+
+class CMaxRewriteInitCacheCommand : public NLMISC::IRunnable
+{
+public:
+	NLMISC::CLog *Log;
+	std::string SrcDirectoryRecursive;
+	std::string DatabaseDirectory;
+
+	virtual void getName(std::string &result) const 
+	{ result = "CMaxRewriteInitCacheCommand"; }
+
+	void doFile(const std::string &filePath)
+	{
+		KnownFileCache[NLMISC::CFile::getFilename(filePath)] = standardizePath(filePath, true);
+	}
+
+	// maxRewritePaths W:/database/interfaces/anims_max
+
+	void doDirectory(const std::string &directoryPath)
+	{
+		nldebug("Directory: '%s'", directoryPath.c_str());
+
+		std::string dirPath = standardizePath(directoryPath, true);
+		std::vector<std::string> dirContents;
+
+		NLMISC::CPath::getPathContent(dirPath, false, true, true, dirContents);
+		
+		for (std::vector<std::string>::iterator it = dirContents.begin(), end = dirContents.end(); it != end; ++it)
+		{
+			const std::string &subPath = standardizePath(*it, false);
+			
+			if (NLMISC::CFile::isDirectory(subPath))
+			{
+				if (subPath.find("\\.") == std::string.npos)
+					doDirectory(subPath);
+			}
+			else
+				doFile(subPath);
+			
+			if (PIPELINE::IPipelineInterface::getInstance()->isExiting())
+				return;
+		}
+	}
+
+	virtual void run()
+	{
+		std::string tempDirectory = PIPELINE::IPipelineProcess::getInstance()->getTempDirectory();
+		DatabaseDirectory = standardizePath(PIPELINE::IPipelineInterface::getInstance()->getConfigFile().getVar("DatabaseDirectory").asString(0), true);
+		nlinfo("DatabaseDirectory: '%s'", DatabaseDirectory.c_str());
+
+		doDirectory(SrcDirectoryRecursive);
+
+		PIPELINE::IPipelineInterface::getInstance()->endedRunnableTask();
+	}
+};
+CMaxRewriteInitCacheCommand s_MaxRewriteInitCacheCommand;
+
+} /* anonymous namespace */
+
 } /* namespace PIPELINE */
 
+NLMISC_CATEGORISED_COMMAND(max, maxRewriteInitCache, "Find all .tga, .png and .max files", "<srcDirectoryRecursive>")
+{
+	if(args.size() != 1) return false;
+	PIPELINE::s_MaxRewriteInitCacheCommand.Log = &log;
+	PIPELINE::s_MaxRewriteInitCacheCommand.SrcDirectoryRecursive = args[0];
+	if (!PIPELINE::IPipelineInterface::getInstance()->tryRunnableTask("COMMAND_MAX_REWRITE_INIT_CACHE", &PIPELINE::s_MaxRewriteInitCacheCommand))
+	{
+		log.displayNL("Busy.");
+		return false;
+	}
+	return true;
+}
+
+NLMISC_CATEGORISED_COMMAND(max, maxRewritePaths, "Rewrite all paths to .tga, .png and .max files found in a max file", "<srcDirectoryRecursive>")
+{
+	if(args.size() != 1) return false;
+	PIPELINE::s_MaxRewritePathsCommand.Log = &log;
+	PIPELINE::s_MaxRewritePathsCommand.SrcDirectoryRecursive = args[0];
+	if (!PIPELINE::IPipelineInterface::getInstance()->tryRunnableTask("COMMAND_MAX_REWRITE_PATHS", &PIPELINE::s_MaxRewritePathsCommand))
+	{
+		log.displayNL("Busy.");
+		return false;
+	}
+	return true;
+}
+
 /* end of file */

code/nel/tools/pipeline/plugin_max/pipeline_plugin_max.h

@@ -32,6 +32,8 @@
 // STL includes
 
 // NeL includes
+#include <nel/misc/command.h>
+#include <nel/misc/task_manager.h>
 
 // Project includes
 

code/nel/tools/pipeline/service/example_cfg/pipeline_service_default.cfg

@@ -54,6 +54,9 @@ ToolBnpMake = "bnp_make";
 // MasterIgnoreProcessPlugins = { }; // Only used by the master service. NOT USED, USE WORKSPACE INTEAD!
 // MasterTerminalPassword = "MASTERCLIENTPASSWORD"; // Only used by the master service.
 
+// Log containing the stuff that would be sent from the master to the user terminal
+MasterLog = SharedWork + "/master.log";
+
 // MasterSlavePassword = "MASTERSLAVEPASSWORD";
 MasterAddress = "localhost";
 MasterPort = 50123;

code/nel/tools/pipeline/service/metadata_storage.cpp

@@ -50,8 +50,10 @@ namespace PIPELINE {
 void CFileError::serial(NLMISC::IStream &stream) throw (NLMISC::EStream)
 {
 	uint version = stream.serialVersion(1);
+	stream.serial((uint8 &)MasterTime); // does this work?
 	stream.serial(Project);
-	stream.serial(Process);
+	stream.serial(Plugin);
+	stream.serial(Time);
 	stream.serial(Message);
 }
 

code/nel/tools/pipeline/service/metadata_storage.h

@@ -33,6 +33,7 @@
 
 // NeL includes
 #include <vector>
+#include <nel/misc/tool_logger.h>
 
 // Project includes
 #include "workspace_storage.h"
@@ -76,11 +77,11 @@ public:
 struct CFileError
 {
 public:
-	uint32 Time; // The time when this error occured.
-	// TFileState Level; // Success, Warning, Error, Removal
+	uint32 MasterTime; // The time when this error occured.
+	TError Level;
 	std::string Project;
-	std::string Process;
-	// std::string Plugin;
+	std::string Plugin;
+	std::string Time;
 	std::string Message;
 
 	void serial(NLMISC::IStream &stream) throw (NLMISC::EStream);
@@ -180,6 +181,9 @@ public:
 	// static void createRemove(const CFileRemove &remove, const std::string &path); // Remove cannot be modified after creation, only erased.
 	// static void eraseRemove(const std::string &path);
 
+	static std::string getErrorPath(const std::string &file) { return "[TODO]"; }
+	static void appendError(const CFileError &error, const std::string &metaPath) { /* TODO */ }
+
 	static std::string getDependPath(const std::string &file);
 	static bool readDepend(CFileDepend &depend, const std::string &metaPath);
 	static void writeDepend(const CFileDepend &depend, const std::string &metaPath);

code/nel/tools/pipeline/service/module_pipeline_master.cpp

@@ -36,6 +36,7 @@
 #include <nel/misc/debug.h>
 #include <nel/net/service.h>
 #include <nel/misc/task_manager.h>
+#include <nel/misc/tool_logger.h>
 
 // Project includes
 #include "info_flags.h"
@@ -157,7 +158,7 @@ protected:
 	bool m_VerifyOnly;
 
 public:
-	CModulePipelineMaster() : m_BuildWorking(false), m_AbortRequested(false), m_TaskManager(NULL), m_WaitingCallbacks("WaitingCallbacks"), m_UpdateTasks("UpdateTasks")
+	CModulePipelineMaster() : m_BuildWorking(false), m_AbortRequested(false), m_TaskManager(NULL), m_WaitingCallbacks("WaitingCallbacks"), m_UpdateTasks("UpdateTasks"), m_MasterLog(NULL)
 	{
 		g_IsMaster = true;
 		m_TaskManager = new NLMISC::CTaskManager();
@@ -397,6 +398,7 @@ public:
 								CProcessPluginInfo pluginInfo;
 								g_PipelineWorkspace->getProcessPlugin(pluginInfo, taskInfo->ProcessPluginId);
 								nlinfo("Dispatching task '%i' ('%s': '%s') to slave '%s'", taskInfo->Id.Global, taskInfo->ProjectName.c_str(), pluginInfo.Handler.c_str(), it->second->Proxy.getModuleProxy()->getModuleName().c_str());
+								notifyTerminalTaskBegin(taskInfo->Id.Global);
 							}
 						}
 					}
@@ -431,6 +433,8 @@ public:
 
 				PIPELINE::endedBuildReadyMaster();
 
+				notifyTerminalBuildEnd();
+
 				nlinfo("#####################################");
 				nlinfo("#####################################");
 				nlinfo("! DONE !");
@@ -503,6 +507,31 @@ public:
 		CInfoFlags::getInstance()->addFlag(PIPELINE_INFO_SLAVE_REJECTED);
 	}
 
+	virtual void slaveLoggedToolError(NLNET::IModuleProxy *sender, uint8 type, const std::string &macroPath, const std::string &time, const std::string &error)
+	{
+		TSlaveMap::iterator slaveIt = m_Slaves.find(sender);
+		if (slaveIt == m_Slaves.end()) { nlerror("Received 'slaveLoggedToolError' from unknown slave at '%s'", sender->getModuleName().c_str()); m_Slaves.erase(sender); /*m_SlavesMutex.unlock();*/ return; }
+		CSlave *slave = slaveIt->second;
+		
+		if (!macroPath.empty())
+		{
+			CBuildTaskInfo *task = m_BuildTaskQueue.getTaskInfo(slave->ActiveTaskId);
+			CProcessPluginInfo pluginInfo;
+			g_PipelineWorkspace->getProcessPlugin(pluginInfo, task->ProcessPluginId);
+			
+			CFileError fe;
+			fe.MasterTime = CTime::getSecondsSince1970();
+			fe.Level = (TError)type;
+			fe.Message = error;
+			fe.Time = time;
+			fe.Project = task->ProjectName;;
+			fe.Plugin = pluginInfo.Handler;
+			CMetadataStorage::appendError(fe, CMetadataStorage::getErrorPath(unMacroPath(macroPath)));
+		}
+
+		notifyTerminalTaskMessage(slave->ActiveTaskId, (TError)type, macroPath, time, error);
+	}
+
 	virtual void slaveReloadedSheets(NLNET::IModuleProxy *sender)
 	{
 		//m_SlavesMutex.lock();
@@ -550,11 +579,69 @@ public:
 	///////////////////////////////////////////////////////////////////
 	///////////////////////////////////////////////////////////////////
 	///////////////////////////////////////////////////////////////////
+	
+	FILE *m_MasterLog;
+
+	void notifyTerminalBuildStart()
+	{
+		nlinfo("");
+		// TODO NOTIFY TERMINAL
+		if (m_MasterLog)
+		{
+			fflush(m_MasterLog);
+			fclose(m_MasterLog);
+			m_MasterLog = NULL;
+		}
+		m_MasterLog = fopen(IService::getInstance()->ConfigFile.getVar("MasterLog").asString(0).c_str(), "at");
+		fprintf(m_MasterLog, "[BUILD_START]\n");
+		fflush(m_MasterLog);
+	}
+
+	// task info
+	void notifyTerminalTaskInfo(uint32 taskId, const std::string &projectName, const std::string &processHandler)
+	{
+		nlinfo("taskId: %i, projectName: %s, processHandler: %s", taskId, projectName.c_str(), processHandler.c_str());
+		// TODO NOTIFY TERMINAL
+		fprintf(m_MasterLog, "[TASK_INFO] taskId: %i, projectName: %s, processHandler: %s\n", taskId, projectName.c_str(), processHandler.c_str());
+		fflush(m_MasterLog);
+	}
+
+	// task begin
+	void notifyTerminalTaskBegin(uint32 taskId)
+	{
+		nlinfo("taskId: %i", taskId);
+		// TODO NOTIFY TERMINAL
+		fprintf(m_MasterLog, "[TASK_BEGIN] taskId: %i\n", taskId);
+		fflush(m_MasterLog);
+	}
 
+	// task issues
+	void notifyTerminalTaskMessage(uint32 taskId, TError type, const std::string &macroPath, const std::string &time, const std::string &error)
+	{
+		nlinfo("taskId: %i, type: %i, macroPath: %s, time: %s, error: %s", taskId, (uint32)type, macroPath.c_str(), time.c_str(), error.c_str());
+		// TODO NOTIFY TERMINAL (send type as uint8 as usual)
+		fprintf(m_MasterLog, "[TASK_MESSAGE] taskId: %i, type: %i, macroPath: %s, time: %s, error: %s\n", taskId, (uint32)type, macroPath.c_str(), time.c_str(), error.c_str());
+		fflush(m_MasterLog);
+	}
+
+	// task end
 	void notifyTerminalTaskState(uint32 taskId, TProcessResult errorLevel, const std::string &errorMessage)
 	{
 		nlinfo("taskId: %i, errorLevel: %i, errorMessage: %s", taskId, (uint32)errorLevel, errorMessage.c_str());
 		// TODO NOTIFY TERMINAL (send errorlevel as uint8 as usual)
+		fprintf(m_MasterLog, "[TASK_STATE] taskId: %i, errorLevel: %i, errorMessage: %s\n", taskId, (uint32)errorLevel, errorMessage.c_str());
+		fflush(m_MasterLog);
+	}
+
+	// build end (any remaining tasks are considered aborted)
+	void notifyTerminalBuildEnd()
+	{
+		nlinfo("");
+		// TODO NOTIFY TERMINAL
+		fprintf(m_MasterLog, "[BUILD_END]\n");
+		fflush(m_MasterLog);
+		fclose(m_MasterLog);
+		m_MasterLog = NULL;
 	}
 
 	///////////////////////////////////////////////////////////////////
@@ -711,11 +798,26 @@ public:
 	{
 		if (PIPELINE::tryBuildReadyMaster())
 		{
+			notifyTerminalBuildStart();
 			m_BuildWorking = true;
 			m_BypassErrors = bypassEros;
 			m_VerifyOnly = verifyOnly;
 			m_BuildTaskQueue.resetQueue();
 			m_BuildTaskQueue.loadQueue(g_PipelineWorkspace, bypassEros);
+
+			// notify terminal task info
+			{
+				std::vector<PIPELINE::CBuildTaskInfo *> tasks;
+				m_BuildTaskQueue.listTaskQueueByMostDependents(tasks);
+				for (std::vector<PIPELINE::CBuildTaskInfo *>::iterator it = tasks.begin(), end = tasks.end(); it != end; ++it)
+				{
+					PIPELINE::CBuildTaskInfo *task = *it;
+					PIPELINE::CProcessPluginInfo pluginInfo;
+					PIPELINE::g_PipelineWorkspace->getProcessPlugin(pluginInfo, task->ProcessPluginId);
+					notifyTerminalTaskInfo(task->Id.Global, task->ProjectName, pluginInfo.Handler);
+				}
+			}
+
 			return true;
 		}
 		return false;

code/nel/tools/pipeline/service/module_pipeline_master_itf.cpp

@@ -34,6 +34,10 @@ namespace PIPELINE
 			// if this assert, you have a doubly message name in your interface definition !
 			nlassert(res.second);
 			
+			res = handlers.insert(std::make_pair(std::string("TL_ERR_LOG"), &CModulePipelineMasterSkel::slaveLoggedToolError_skel));
+			// if this assert, you have a doubly message name in your interface definition !
+			nlassert(res.second);
+			
 			res = handlers.insert(std::make_pair(std::string("RE_SHEETS_OK"), &CModulePipelineMasterSkel::slaveReloadedSheets_skel));
 			// if this assert, you have a doubly message name in your interface definition !
 			nlassert(res.second);
@@ -103,6 +107,20 @@ namespace PIPELINE
 		slaveRefusedBuildTask(sender);
 	}
 
+	void CModulePipelineMasterSkel::slaveLoggedToolError_skel(NLNET::IModuleProxy *sender, const NLNET::CMessage &__message)
+	{
+		H_AUTO(CModulePipelineMasterSkel_slaveLoggedToolError_TL_ERR_LOG);
+		uint8	type;
+			nlRead(__message, serial, type);
+		std::string	macroPath;
+			nlRead(__message, serial, macroPath);
+		std::string	time;
+			nlRead(__message, serial, time);
+		std::string	error;
+			nlRead(__message, serial, error);
+		slaveLoggedToolError(sender, type, macroPath, time, error);
+	}
+
 	void CModulePipelineMasterSkel::slaveReloadedSheets_skel(NLNET::IModuleProxy *sender, const NLNET::CMessage &__message)
 	{
 		H_AUTO(CModulePipelineMasterSkel_slaveReloadedSheets_RE_SHEETS_OK);
@@ -197,6 +215,24 @@ namespace PIPELINE
 		}
 	}
 		// 
+	void CModulePipelineMasterProxy::slaveLoggedToolError(NLNET::IModule *sender, uint8 type, const std::string &macroPath, const std::string &time, const std::string &error)
+	{
+		if (_LocalModuleSkel && _LocalModule->isImmediateDispatchingSupported())
+		{
+			// immediate local synchronous dispatching
+			_LocalModuleSkel->slaveLoggedToolError(_ModuleProxy->getModuleGateway()->getPluggedModuleProxy(sender), type, macroPath, time, error);
+		}
+		else
+		{
+			// send the message for remote dispatching and execution or local queing
+			NLNET::CMessage __message;
+
+			buildMessageFor_slaveLoggedToolError(__message, type, macroPath, time, error);
+
+			_ModuleProxy->sendModuleMessage(sender, __message);
+		}
+	}
+		// 
 	void CModulePipelineMasterProxy::slaveReloadedSheets(NLNET::IModule *sender)
 	{
 		if (_LocalModuleSkel && _LocalModule->isImmediateDispatchingSupported())
@@ -334,6 +370,19 @@ namespace PIPELINE
 		return __message;
 	}
 
+	// Message serializer. Return the message received in reference for easier integration
+	const NLNET::CMessage &CModulePipelineMasterProxy::buildMessageFor_slaveLoggedToolError(NLNET::CMessage &__message, uint8 type, const std::string &macroPath, const std::string &time, const std::string &error)
+	{
+		__message.setType("TL_ERR_LOG");
+			nlWrite(__message, serial, type);
+			nlWrite(__message, serial, const_cast < std::string& > (macroPath));
+			nlWrite(__message, serial, const_cast < std::string& > (time));
+			nlWrite(__message, serial, const_cast < std::string& > (error));
+
+
+		return __message;
+	}
+
 	// Message serializer. Return the message received in reference for easier integration
 	const NLNET::CMessage &CModulePipelineMasterProxy::buildMessageFor_slaveReloadedSheets(NLNET::CMessage &__message)
 	{

code/nel/tools/pipeline/service/module_pipeline_master_itf.h

@@ -66,6 +66,8 @@ namespace PIPELINE
 
 		void slaveRefusedBuildTask_skel(NLNET::IModuleProxy *sender, const NLNET::CMessage &__message);
 
+		void slaveLoggedToolError_skel(NLNET::IModuleProxy *sender, const NLNET::CMessage &__message);
+
 		void slaveReloadedSheets_skel(NLNET::IModuleProxy *sender, const NLNET::CMessage &__message);
 
 		void slaveBuildReadySuccess_skel(NLNET::IModuleProxy *sender, const NLNET::CMessage &__message);
@@ -95,6 +97,8 @@ namespace PIPELINE
 		// 
 		virtual void slaveRefusedBuildTask(NLNET::IModuleProxy *sender) =0;
 		// 
+		virtual void slaveLoggedToolError(NLNET::IModuleProxy *sender, uint8 type, const std::string &macroPath, const std::string &time, const std::string &error) =0;
+		// 
 		virtual void slaveReloadedSheets(NLNET::IModuleProxy *sender) =0;
 		// 
 		virtual void slaveBuildReadySuccess(NLNET::IModuleProxy *sender) =0;
@@ -162,6 +166,8 @@ namespace PIPELINE
 		// 
 		void slaveRefusedBuildTask(NLNET::IModule *sender);
 		// 
+		void slaveLoggedToolError(NLNET::IModule *sender, uint8 type, const std::string &macroPath, const std::string &time, const std::string &error);
+		// 
 		void slaveReloadedSheets(NLNET::IModule *sender);
 		// 
 		void slaveBuildReadySuccess(NLNET::IModule *sender);
@@ -183,6 +189,9 @@ namespace PIPELINE
 		// Message serializer. Return the message received in reference for easier integration
 		static const NLNET::CMessage &buildMessageFor_slaveRefusedBuildTask(NLNET::CMessage &__message);
 	
+		// Message serializer. Return the message received in reference for easier integration
+		static const NLNET::CMessage &buildMessageFor_slaveLoggedToolError(NLNET::CMessage &__message, uint8 type, const std::string &macroPath, const std::string &time, const std::string &error);
+	
 		// Message serializer. Return the message received in reference for easier integration
 		static const NLNET::CMessage &buildMessageFor_slaveReloadedSheets(NLNET::CMessage &__message);
 	

code/nel/tools/pipeline/service/module_pipeline_master_itf.xml

@@ -19,6 +19,15 @@
 				<doc line=""/>
 			</method>
 
+			<method name="slaveLoggedToolError" msg="TL_ERR_LOG">
+				<doc line=""/>
+
+				<param type="uint8" name="type" />
+				<param type="std::string" name="macroPath" byref="true" />
+				<param type="std::string" name="time" byref="true" />
+				<param type="std::string" name="error" byref="true" />
+			</method>
+
 			<method name="slaveReloadedSheets" msg="RE_SHEETS_OK">
 				<doc line=""/>
 			</method>

code/nel/tools/pipeline/service/module_pipeline_slave.cpp

@@ -114,8 +114,18 @@ public:
 	
 	bool m_PluginBuildDone;
 
+	struct CErrorLogData
+	{
+		TError Type;
+		std::string Path;
+		std::string Time;
+		std::string Error;
+	};
+
+	NLMISC::CSynchronized<std::vector<CErrorLogData> > m_ErrorLogWaiting;
+
 public:
-	CModulePipelineSlave() : m_Master(NULL), m_TestCommand(false), m_ReloadSheetsState(REQUEST_NONE), m_BuildReadyState(false), m_SlaveTaskState(IDLE_WAIT_MASTER), m_TaskManager(NULL), m_StatusUpdateMasterDone("StatusUpdateMasterDone"), m_StatusUpdateSlaveDone("StatusUpdateSlaveDone"), m_ActiveProject(NULL), m_ActiveProcess(NULL), m_AbortRequested(false), m_PluginBuildDone(false)
+	CModulePipelineSlave() : m_Master(NULL), m_TestCommand(false), m_ReloadSheetsState(REQUEST_NONE), m_BuildReadyState(false), m_SlaveTaskState(IDLE_WAIT_MASTER), m_TaskManager(NULL), m_StatusUpdateMasterDone("StatusUpdateMasterDone"), m_StatusUpdateSlaveDone("StatusUpdateSlaveDone"), m_ActiveProject(NULL), m_ActiveProcess(NULL), m_AbortRequested(false), m_PluginBuildDone(false), m_ErrorLogWaiting("ErrorLogWaiting")
 	{
 		NLMISC::CSynchronized<bool>::CAccessor(&m_StatusUpdateMasterDone).value() = false;
 		NLMISC::CSynchronized<bool>::CAccessor(&m_StatusUpdateSlaveDone).value() = false;
@@ -215,13 +225,35 @@ public:
 			abortBuildTask(NULL);
 			leaveBuildReadyState(NULL); // leave state if building
 
+			clearErrorLogQueue();
+
 			delete m_Master;
 			m_Master = NULL;
 		}
 	}
 
+	void clearErrorLogQueue()
+	{
+		NLMISC::CSynchronized<std::vector<CErrorLogData> >::CAccessor errorLogQueue(&m_ErrorLogWaiting);
+		errorLogQueue.value().clear();
+	}
+
+	void sendErrorLogQueue()
+	{
+		NLMISC::CSynchronized<std::vector<CErrorLogData> >::CAccessor errorLogQueue(&m_ErrorLogWaiting);
+		for (std::vector<CErrorLogData>::iterator it = errorLogQueue.value().begin(), end = errorLogQueue.value().end(); it != end; ++it)
+		{
+			const CErrorLogData &erdt = *it;
+			nlassert(m_Master);
+			m_Master->slaveLoggedToolError(this, (uint8)erdt.Type, macroPath(erdt.Path), erdt.Time, erdt.Error);
+		}
+		errorLogQueue.value().clear();
+	}
+
 	virtual void onModuleUpdate()
 	{
+		sendErrorLogQueue();
+
 		if (m_ReloadSheetsState == REQUEST_MADE)
 		{
 			if (PIPELINE::reloadSheets())
@@ -313,6 +345,7 @@ public:
 			{
 				m_SlaveTaskState = SOMEWHERE_INBETWEEN;
 				CInfoFlags::getInstance()->removeFlag(PIPELINE_INFO_PLUGIN_WORKING);
+				sendErrorLogQueue();
 				if (m_AbortRequested)
 				{
 					nlinfo("Aborted slave task while plugin was working");
@@ -525,6 +558,18 @@ public:
 	///////////////////////////////////////////////////////////////////
 	///////////////////////////////////////////////////////////////////
 
+	void notifyErrorLog(TError type, const std::string &path, const std::string &time, const std::string &error)
+	{
+		// Received an error log from a plugin tool
+		NLMISC::CSynchronized<std::vector<CErrorLogData> >::CAccessor errorLogQueue(&m_ErrorLogWaiting);
+		CErrorLogData erdt;
+		erdt.Type = type;
+		erdt.Path = path;
+		erdt.Time = time;
+		erdt.Error = error;
+		errorLogQueue.value().push_back(erdt);
+	}
+
 	class CPluginBuildTask : public IRunnable
 	{
 	public:
@@ -537,6 +582,7 @@ public:
 			case PIPELINE::PLUGIN_REGISTERED_CLASS:
 				{
 					PIPELINE::IProcessHandler *processHandler = static_cast<PIPELINE::IProcessHandler *>(NLMISC::CClassRegistry::create(m_Slave->m_ActiveProcess->m_ActivePlugin.Handler));
+					m_Slave->m_ActiveProcess->m_ErrorLogCallback = TErrorLogCallback(m_Slave, &CModulePipelineSlave::notifyErrorLog);
 					processHandler->setPipelineProcess(m_Slave->m_ActiveProcess);
 					m_Slave->m_ActiveProcess->m_SubTaskResult = FINISH_SUCCESS;
 					processHandler->build();
@@ -639,6 +685,14 @@ public:
 	void finishedTask(TProcessResult errorLevel, std::string errorMessage)
 	{
 		nlinfo("errorLevel: %i, errorMessage: %s", (uint32)errorLevel, errorMessage.c_str());
+		stringstream ss;
+		ss << "Build: " << m_ActiveProcess->m_StatsBuild << ", Skip: " << m_ActiveProcess->m_StatsSkip << ", Invalid: " << m_ActiveProcess->m_StatsInvalid << ", Total: "
+			<< (m_ActiveProcess->m_StatsBuild + m_ActiveProcess->m_StatsSkip + m_ActiveProcess->m_StatsInvalid);
+		nlinfo("%s", ss.str().c_str());
+		stringstream ss2;
+		ss2 << NLMISC::CTime::getSecondsSince1970();
+		if (m_Master) // TODO: Maybe send this as part of the finished build task message, would be cleaner ;)
+			m_Master->slaveLoggedToolError(this, (uint8)MESSAGE, "", ss2.str(), ss.str());
 		clearActiveProcess();
 		m_SlaveTaskState = IDLE_WAIT_MASTER;
 		if (m_Master) // else was disconnect

code/nel/tools/pipeline/service/pipeline_interface_impl.cpp

@@ -88,6 +88,11 @@ void CPipelineInterfaceImpl::endedDirectCode()
 	PIPELINE::endedDirectTask();
 }
 
+bool CPipelineInterfaceImpl::isExiting()
+{
+	return g_IsExiting;
+}
+
 } /* namespace PIPELINE */
 
 /* end of file */

code/nel/tools/pipeline/service/pipeline_interface_impl.h

@@ -61,6 +61,7 @@ public:
 	virtual void endedRunnableTask();
 	virtual bool tryDirectCode(const std::string &stateName);
 	virtual void endedDirectCode();
+	virtual bool isExiting();
 }; /* class CPipelineInterfaceImpl */
 
 } /* namespace PIPELINE */

code/nel/tools/pipeline/service/pipeline_process_impl.cpp

@@ -50,7 +50,7 @@ using namespace std;
 
 namespace PIPELINE {
 
-CPipelineProcessImpl::CPipelineProcessImpl(CPipelineProject *activeProject) : m_ActiveProject(activeProject), m_SubTaskResult(FINISH_NOT), m_Aborting(false)
+CPipelineProcessImpl::CPipelineProcessImpl(CPipelineProject *activeProject) : m_ActiveProject(activeProject), m_SubTaskResult(FINISH_NOT), m_Aborting(false), m_StatsBuild(0), m_StatsSkip(0), m_StatsInvalid(0)
 {
 	if (activeProject == NULL)
 	{

code/nel/tools/pipeline/service/pipeline_process_impl.h

@@ -32,15 +32,19 @@
 // STL includes
 
 // NeL includes
+#include <nel/misc/tool_logger.h>
 
 // Project includes
 #include "../plugin_library/pipeline_process.h"
 #include "pipeline_workspace.h"
 #include "metadata_storage.h"
+#include "callback.h"
 
 namespace PIPELINE {
 	class CPipelineProject;
 
+typedef CCallback<void, TError /* type */, const std::string & /* path */, const std::string & /* time */ , const std::string & /* error */ > TErrorLogCallback;
+
 /**
  * \brief CPipelineProcessImpl
  * \date 2012-03-03 09:33GMT
@@ -108,6 +112,11 @@ private:
 	std::set<std::string> m_ListDependentFiles;
 	
 	bool m_Aborting;
+	TErrorLogCallback m_ErrorLogCallback;
+
+	uint32 m_StatsBuild;
+	uint32 m_StatsSkip;
+	uint32 m_StatsInvalid;
 
 private:
 	bool getDependencyFileStatusCached(CFileStatus &fileStatus, const std::string &filePath);

code/nel/tools/pipeline/service/pipeline_process_impl_buildinfo.cpp

@@ -183,7 +183,10 @@ bool CPipelineProcessImpl::hasFileBeenAddedSince(const std::string &inputFile, u
 bool CPipelineProcessImpl::needsToBeRebuilt(const std::vector<std::string> &inputPaths, bool inputDepends)
 {
 	if (m_SubTaskResult != FINISH_SUCCESS)
+	{
+		++m_StatsInvalid;
 		return false; // Cannot continue on previous failure.
+	}
 
 	m_SubTaskResult = FINISH_NOT;
 
@@ -195,12 +198,14 @@ bool CPipelineProcessImpl::needsToBeRebuilt(const std::vector<std::string> &inpu
 		{
 			m_SubTaskResult = FINISH_ERROR;
 			m_SubTaskErrorMessage = std::string("Input file '") + path + "' cannot be a directory";
+			++m_StatsInvalid;
 			return false; // Error, cannot rebuild.
 		}
 		if (!isFileDependency(path))
 		{
 			m_SubTaskResult = FINISH_ERROR;
 			m_SubTaskErrorMessage = std::string("File '") + path + "' is not part of the dependencies";
+			++m_StatsInvalid;
 			return false; // Error, cannot rebuild.
 		}
 	}
@@ -233,6 +238,7 @@ bool CPipelineProcessImpl::needsToBeRebuilt(const std::vector<std::string> &inpu
 			{
 				nldebug("No output files were tampered with since last successful build, rebuild not needed");
 				m_SubTaskResult = FINISH_SUCCESS;
+				++m_StatsSkip;
 				return false; // No rebuild required.
 			}
 		}
@@ -262,6 +268,7 @@ bool CPipelineProcessImpl::needsToBeRebuilt(const std::vector<std::string> &inpu
 		{
 			nldebug("Not all input files have an .output files, rebuild");
 			m_SubTaskResult = FINISH_SUCCESS;
+			++m_StatsBuild;
 			return true; // Rebuild
 		}
 		else
@@ -277,7 +284,10 @@ bool CPipelineProcessImpl::needsToBeRebuilt(const std::vector<std::string> &inpu
 bool CPipelineProcessImpl::needsToBeRebuildSubByOutput(const std::vector<std::string> &inputPaths, bool inputChanged, bool inputDepends)
 {
 	if (m_SubTaskResult != FINISH_SUCCESS)
+	{
+		++m_StatsInvalid;
 		return false; // Cannot continue on previous failure.
+	}
 	
 	m_SubTaskResult = FINISH_NOT;
 
@@ -311,6 +321,7 @@ bool CPipelineProcessImpl::needsToBeRebuildSubByOutput(const std::vector<std::st
 				// Require rebuild because we don't know if there's new output
 				nldebug("Input file '%s' has output file with no output files, state not known, so rebuild", path.c_str());
 				m_SubTaskResult = FINISH_SUCCESS;
+				++m_StatsBuild;
 				return true; // Rebuild.
 			}
 		}
@@ -331,7 +342,10 @@ bool CPipelineProcessImpl::needsToBeRebuildSubByOutput(const std::vector<std::st
 bool CPipelineProcessImpl::needsToBeRebuilt(const std::vector<std::string> &inputPaths, const std::vector<std::string> &outputPaths, bool inputDepends)
 {
 	if (m_SubTaskResult != FINISH_SUCCESS)
+	{
+		++m_StatsInvalid;
 		return false; // Cannot continue on previous failure.
+	}
 	
 	m_SubTaskResult = FINISH_NOT;
 
@@ -348,6 +362,7 @@ bool CPipelineProcessImpl::needsToBeRebuilt(const std::vector<std::string> &inpu
 			{
 				m_SubTaskResult = FINISH_ERROR;
 				m_SubTaskErrorMessage = std::string("Directory '") + path + "' is not part of the dependencies";
+				++m_StatsInvalid;
 				return false; // Error, cannot rebuild.
 			}
 		}
@@ -358,6 +373,7 @@ bool CPipelineProcessImpl::needsToBeRebuilt(const std::vector<std::string> &inpu
 			{
 				m_SubTaskResult = FINISH_ERROR;
 				m_SubTaskErrorMessage = std::string("File '") + path + "' is not part of the dependencies";
+				++m_StatsInvalid;
 				return false; // Error, cannot rebuild.
 			}
 		}
@@ -407,6 +423,7 @@ bool CPipelineProcessImpl::needsToBeRebuilt(const std::vector<std::string> &inpu
 		{
 			nldebug("No output files were tampered with since last successful build, rebuild not needed");
 			m_SubTaskResult = FINISH_SUCCESS;
+			++m_StatsSkip;
 			return false; // No rebuild required.
 		}
 		else
@@ -421,7 +438,10 @@ bool CPipelineProcessImpl::needsToBeRebuilt(const std::vector<std::string> &inpu
 bool CPipelineProcessImpl::needsToBeRebuiltSub(const std::vector<std::string> &inputPaths, const std::vector<std::string> &outputPaths, bool inputModified, bool inputDepends)
 {
 	if (m_SubTaskResult != FINISH_SUCCESS)
+	{
+		++m_StatsInvalid;
 		return false; // Cannot continue on previous failure.
+	}
 	
 	m_SubTaskResult = FINISH_NOT;
 
@@ -434,6 +454,7 @@ bool CPipelineProcessImpl::needsToBeRebuiltSub(const std::vector<std::string> &i
 		{
 			m_SubTaskResult = FINISH_ERROR;
 			m_SubTaskErrorMessage = std::string("Output file '") + path + "' cannot be a directory";
+			++m_StatsInvalid;
 			return false; // Error, cannot rebuild.
 		}
 	}
@@ -447,6 +468,7 @@ bool CPipelineProcessImpl::needsToBeRebuiltSub(const std::vector<std::string> &i
 			// If so, rebuild
 			nldebug("Output file '%s' has been removed, rebuild", path.c_str());
 			m_SubTaskResult = FINISH_SUCCESS;
+			++m_StatsBuild;
 			return true; // Rebuild.
 		}
 	}
@@ -461,6 +483,7 @@ bool CPipelineProcessImpl::needsToBeRebuiltSub(const std::vector<std::string> &i
 			// If so, rebuild
 			nldebug("Output file '%s' does not exist, rebuild", path.c_str());
 			m_SubTaskResult = FINISH_SUCCESS;
+			++m_StatsBuild;
 			return true; // Rebuild.
 		}
 	}
@@ -498,6 +521,7 @@ bool CPipelineProcessImpl::needsToBeRebuiltSub(const std::vector<std::string> &i
 		{
 			nlwarning("Depend file for existing output '%s' does not exist, this should not happen, rebuild", path.c_str());
 			m_SubTaskResult = FINISH_SUCCESS;
+			++m_StatsBuild;
 			return true; // Rebuild.
 		}
 		else
@@ -513,6 +537,7 @@ bool CPipelineProcessImpl::needsToBeRebuiltSub(const std::vector<std::string> &i
 					// FIXME: Is it really necessary to recalculate the crc32 if the status file is broken for an output file? Useful though for some rare cases.
 					m_SubTaskResult = FINISH_ERROR;
 					m_SubTaskErrorMessage = std::string("Could not get status for output file '") + path + "', this should never happen at all, coding error";
+					++m_StatsInvalid;
 					return false; // Error, cannot rebuild.
 				}
 				else
@@ -521,6 +546,7 @@ bool CPipelineProcessImpl::needsToBeRebuiltSub(const std::vector<std::string> &i
 					{
 						nlwarning("Status checksum %i for output file '%s' does match depend checksum %i, output file was modified, this should not happen, rebuild", metaStatus.CRC32, path.c_str(), metaDepend.CRC32);
 						m_SubTaskResult = FINISH_SUCCESS;
+						++m_StatsBuild;
 						return true; // Rebuild.
 					}
 				}
@@ -537,6 +563,7 @@ bool CPipelineProcessImpl::needsToBeRebuiltSub(const std::vector<std::string> &i
 					{
 						nlwarning("Output file '%s' depends on unknown file '%s', rebuild", path.c_str(), dependencyFile.c_str());
 						m_SubTaskResult = FINISH_SUCCESS;
+						++m_StatsBuild;
 						return true; // Rebuild.
 					}
 					else
@@ -545,6 +572,7 @@ bool CPipelineProcessImpl::needsToBeRebuiltSub(const std::vector<std::string> &i
 						{
 							nldebug("Status checksum %i for input file '%s' does match depend checksum %i, input file was modified, rebuild", metaStatus.CRC32, dependencyFile.c_str(), dependency.CRC32);
 							m_SubTaskResult = FINISH_SUCCESS;
+							++m_StatsBuild;
 							return true; // Rebuild.
 						}
 					}
@@ -574,6 +602,7 @@ bool CPipelineProcessImpl::needsToBeRebuiltSub(const std::vector<std::string> &i
 				{
 					nldebug("Found a file added after last build start in a dependency directory that is not known by the depend files, rebuild");
 					m_SubTaskResult = FINISH_SUCCESS;
+					++m_StatsBuild;
 					return true; // Rebuild.
 				}
 			}
@@ -585,6 +614,7 @@ bool CPipelineProcessImpl::needsToBeRebuiltSub(const std::vector<std::string> &i
 					{
 						nldebug("Found a dependency file added after last build start that is not known by the depend files, rebuild");
 						m_SubTaskResult = FINISH_SUCCESS;
+						++m_StatsBuild;
 						return true; // Rebuild.
 					}
 				}
@@ -595,6 +625,7 @@ bool CPipelineProcessImpl::needsToBeRebuiltSub(const std::vector<std::string> &i
 	// (if any checksum was different, require rebuild, if no checksums were different, no rebuild is needed)
 	nldebug("No differences found, no rebuild needed");
 	m_SubTaskResult = FINISH_SUCCESS;
+	++m_StatsSkip;
 	return false; // Rebuild not necessary.
 }
 

code/nel/tools/pipeline/service/pipeline_process_impl_toollog.cpp

@@ -160,7 +160,8 @@ void CPipelineProcessImpl::parseToolLog(const std::string &dependLogFile, const
 				}
 				std::string path = standardizePath(tabbedLine[1], false);
 				nlwarning("Read error log line: %s, %s, %s, %s", tabbedLine[0].c_str(), path.c_str(), tabbedLine[2].c_str(), tabbedLine[3].c_str());
-				// TODO: Notify the master to write to the .errors meta file and update any connected terminals
+				// Notify the master through the slave to write to the .errors meta file and update any connected terminals
+				m_ErrorLogCallback(type, path, tabbedLine[2], tabbedLine[3]);
 				if (type == ERROR)
 					++nbErrors;
 			}

code/ryzom/common/data_leveldesign/leveldesign/pipeline/ryzom_core.pipeline_workspace

@@ -3,17 +3,7 @@
   <STRUCT>
     <ATOM Name="Description" Value="Ryzom Core"/>
     <ARRAY Name="Projects">
-      <ATOM Value="gamedev.pipeline_package"/>
-      <ATOM Value="data_common.pipeline_package"/>
       <ATOM Value="common_interface.pipeline_project"/>
-      <ATOM Value="interfaces.pipeline_package"/>
-      <ATOM Value="common_objects_generique.pipeline_project"/>
-      <ATOM Value="common_objects_fyros.pipeline_project"/>
-      <ATOM Value="common_objects_tryker.pipeline_project"/>
-      <ATOM Value="common_objects_matis.pipeline_project"/>
-      <ATOM Value="common_objects_zorai.pipeline_project"/>
-      <ATOM Value="common_objects_caravan.pipeline_project"/>
-      <ATOM Value="objects.pipeline_package"/>
     </ARRAY>
     <ARRAY Name="Plugins">
       <ATOM Value="plugin_nel.pipeline_plugin"/>
@@ -24,17 +14,16 @@
   <STRUCT/>
   <STRUCT/>
   <STRUCT/>
-  <LOG>Sat Feb 18 13:48:48 2012 (Kaetemi) .Name = Ryzom Core
-Sat Feb 18 13:48:48 2012 (Kaetemi) formName Resized = 1
-Sat Feb 18 14:44:25 2012 (Kaetemi) .Description = Ryzom Core
-Sat Feb 18 14:44:37 2012 (Kaetemi) .Projects[0] = common_interface.pipeline_project
-Sat Feb 18 23:23:17 2012 (Kaetemi) .Description = Ryzom Core Test
-Sat Feb 18 23:23:26 2012 (Kaetemi) .Description = Ryzom Core
-Sat Mar 03 10:58:02 2012 (Kaetemi) .Plugins[0] = plugin_nel.pipeline_plugin
-Sat Mar 03 10:58:02 2012 (Kaetemi) .Plugins[1] = plugin_max.pipeline_plugin
-Sat Mar 03 10:58:02 2012 (Kaetemi) formName Resized = 2
-Sat Aug 04 21:04:25 2012 (kaetemi) .Projects[1] = interfaces.pipeline_package
-Sat Aug 04 21:04:25 2012 (kaetemi) formName Resized = 2
-Sat Aug 04 21:09:02 2012 (kaetemi) .Projects[0] = interfaces.pipeline_package
-Sat Aug 04 21:09:02 2012 (kaetemi) .Projects[1] = common_interface.pipeline_project</LOG>
+  <COMMENTS>      &lt;ATOM Value="gamedev.pipeline_package"/&gt;
+      &lt;ATOM Value="data_common.pipeline_package"/&gt;
+      &lt;ATOM Value="common_interface.pipeline_project"/&gt;
+      &lt;ATOM Value="interfaces.pipeline_package"/&gt;
+      &lt;ATOM Value="common_objects_generique.pipeline_project"/&gt;
+      &lt;ATOM Value="common_objects_fyros.pipeline_project"/&gt;
+      &lt;ATOM Value="common_objects_tryker.pipeline_project"/&gt;
+      &lt;ATOM Value="common_objects_matis.pipeline_project"/&gt;
+      &lt;ATOM Value="common_objects_zorai.pipeline_project"/&gt;
+      &lt;ATOM Value="common_objects_caravan.pipeline_project"/&gt;
+      &lt;ATOM Value="objects.pipeline_package"/&gt;</COMMENTS>
+  <LOG></LOG>
 </FORM>