Added: #1440 Send tool errors from slave to master

--HG--
branch : build_pipeline_v3

code/nel/tools/pipeline/service/example_cfg/pipeline_service_default.cfg

@@ -54,6 +54,9 @@ ToolBnpMake = "bnp_make";
 // MasterIgnoreProcessPlugins = { }; // Only used by the master service. NOT USED, USE WORKSPACE INTEAD!
 // MasterTerminalPassword = "MASTERCLIENTPASSWORD"; // Only used by the master service.
 
+// Log containing the stuff that would be sent from the master to the user terminal
+MasterLog = SharedWork + "/master.log";
+
 // MasterSlavePassword = "MASTERSLAVEPASSWORD";
 MasterAddress = "localhost";
 MasterPort = 50123;

code/nel/tools/pipeline/service/module_pipeline_master.cpp

@@ -36,6 +36,7 @@
 #include <nel/misc/debug.h>
 #include <nel/net/service.h>
 #include <nel/misc/task_manager.h>
+#include <nel/misc/tool_logger.h>
 
 // Project includes
 #include "info_flags.h"
@@ -157,7 +158,7 @@ protected:
 	bool m_VerifyOnly;
 
 public:
-	CModulePipelineMaster() : m_BuildWorking(false), m_AbortRequested(false), m_TaskManager(NULL), m_WaitingCallbacks("WaitingCallbacks"), m_UpdateTasks("UpdateTasks")
+	CModulePipelineMaster() : m_BuildWorking(false), m_AbortRequested(false), m_TaskManager(NULL), m_WaitingCallbacks("WaitingCallbacks"), m_UpdateTasks("UpdateTasks"), m_MasterLog(NULL)
 	{
 		g_IsMaster = true;
 		m_TaskManager = new NLMISC::CTaskManager();
@@ -397,6 +398,7 @@ public:
 								CProcessPluginInfo pluginInfo;
 								g_PipelineWorkspace->getProcessPlugin(pluginInfo, taskInfo->ProcessPluginId);
 								nlinfo("Dispatching task '%i' ('%s': '%s') to slave '%s'", taskInfo->Id.Global, taskInfo->ProjectName.c_str(), pluginInfo.Handler.c_str(), it->second->Proxy.getModuleProxy()->getModuleName().c_str());
+								notifyTerminalTaskBegin(taskInfo->Id.Global);
 							}
 						}
 					}
@@ -431,6 +433,8 @@ public:
 
 				PIPELINE::endedBuildReadyMaster();
 
+				notifyTerminalBuildEnd();
+
 				nlinfo("#####################################");
 				nlinfo("#####################################");
 				nlinfo("! DONE !");
@@ -503,6 +507,15 @@ public:
 		CInfoFlags::getInstance()->addFlag(PIPELINE_INFO_SLAVE_REJECTED);
 	}
 
+	virtual void slaveLoggedToolError(NLNET::IModuleProxy *sender, uint8 type, const std::string &macroPath, const std::string &time, const std::string &error)
+	{
+		TSlaveMap::iterator slaveIt = m_Slaves.find(sender);
+		if (slaveIt == m_Slaves.end()) { nlerror("Received 'slaveLoggedToolError' from unknown slave at '%s'", sender->getModuleName().c_str()); m_Slaves.erase(sender); /*m_SlavesMutex.unlock();*/ return; }
+		CSlave *slave = slaveIt->second;
+
+		notifyTerminalTaskMessage(slave->ActiveTaskId, (TError)type, macroPath, time, error);
+	}
+
 	virtual void slaveReloadedSheets(NLNET::IModuleProxy *sender)
 	{
 		//m_SlavesMutex.lock();
@@ -550,11 +563,69 @@ public:
 	///////////////////////////////////////////////////////////////////
 	///////////////////////////////////////////////////////////////////
 	///////////////////////////////////////////////////////////////////
+	
+	FILE *m_MasterLog;
+
+	void notifyTerminalBuildStart()
+	{
+		nlinfo("");
+		// TODO NOTIFY TERMINAL
+		if (m_MasterLog)
+		{
+			fflush(m_MasterLog);
+			fclose(m_MasterLog);
+			m_MasterLog = NULL;
+		}
+		m_MasterLog = fopen(IService::getInstance()->ConfigFile.getVar("MasterLog").asString(0).c_str(), "at");
+		fprintf(m_MasterLog, "[BUILD_START]\n");
+		fflush(m_MasterLog);
+	}
+
+	// task info
+	void notifyTerminalTaskInfo(uint32 taskId, const std::string &projectName, const std::string &processHandler)
+	{
+		nlinfo("taskId: %i, projectName: %s, processHandler: %s", taskId, projectName.c_str(), processHandler.c_str());
+		// TODO NOTIFY TERMINAL
+		fprintf(m_MasterLog, "[TASK_INFO] taskId: %i, projectName: %s, processHandler: %s\n", taskId, projectName.c_str(), processHandler.c_str());
+		fflush(m_MasterLog);
+	}
 
+	// task begin
+	void notifyTerminalTaskBegin(uint32 taskId)
+	{
+		nlinfo("taskId: %i", taskId);
+		// TODO NOTIFY TERMINAL
+		fprintf(m_MasterLog, "[TASK_BEGIN] taskId: %i\n", taskId);
+		fflush(m_MasterLog);
+	}
+
+	// task issues
+	void notifyTerminalTaskMessage(uint32 taskId, TError type, const std::string &macroPath, const std::string &time, const std::string &error)
+	{
+		nlinfo("taskId: %i, type: %i, macroPath: %s, time: %s, error: %s", taskId, (uint32)type, macroPath.c_str(), time.c_str(), error.c_str());
+		// TODO NOTIFY TERMINAL (send type as uint8 as usual)
+		fprintf(m_MasterLog, "[TASK_MESSAGE] taskId: %i, type: %i, macroPath: %s, time: %s, error: %s\n", taskId, (uint32)type, macroPath.c_str(), time.c_str(), error.c_str());
+		fflush(m_MasterLog);
+	}
+
+	// task end
 	void notifyTerminalTaskState(uint32 taskId, TProcessResult errorLevel, const std::string &errorMessage)
 	{
 		nlinfo("taskId: %i, errorLevel: %i, errorMessage: %s", taskId, (uint32)errorLevel, errorMessage.c_str());
 		// TODO NOTIFY TERMINAL (send errorlevel as uint8 as usual)
+		fprintf(m_MasterLog, "[TASK_STATE] taskId: %i, errorLevel: %i, errorMessage: %s\n", taskId, (uint32)errorLevel, errorMessage.c_str());
+		fflush(m_MasterLog);
+	}
+
+	// build end (any remaining tasks are considered aborted)
+	void notifyTerminalBuildEnd()
+	{
+		nlinfo("");
+		// TODO NOTIFY TERMINAL
+		fprintf(m_MasterLog, "[BUILD_END]\n");
+		fflush(m_MasterLog);
+		fclose(m_MasterLog);
+		m_MasterLog = NULL;
 	}
 
 	///////////////////////////////////////////////////////////////////
@@ -711,11 +782,26 @@ public:
 	{
 		if (PIPELINE::tryBuildReadyMaster())
 		{
+			notifyTerminalBuildStart();
 			m_BuildWorking = true;
 			m_BypassErrors = bypassEros;
 			m_VerifyOnly = verifyOnly;
 			m_BuildTaskQueue.resetQueue();
 			m_BuildTaskQueue.loadQueue(g_PipelineWorkspace, bypassEros);
+
+			// notify terminal task info
+			{
+				std::vector<PIPELINE::CBuildTaskInfo *> tasks;
+				m_BuildTaskQueue.listTaskQueueByMostDependents(tasks);
+				for (std::vector<PIPELINE::CBuildTaskInfo *>::iterator it = tasks.begin(), end = tasks.end(); it != end; ++it)
+				{
+					PIPELINE::CBuildTaskInfo *task = *it;
+					PIPELINE::CProcessPluginInfo pluginInfo;
+					PIPELINE::g_PipelineWorkspace->getProcessPlugin(pluginInfo, task->ProcessPluginId);
+					notifyTerminalTaskInfo(task->Id.Global, task->ProjectName, pluginInfo.Handler);
+				}
+			}
+
 			return true;
 		}
 		return false;

code/nel/tools/pipeline/service/module_pipeline_master_itf.cpp

@@ -34,6 +34,10 @@ namespace PIPELINE
 			// if this assert, you have a doubly message name in your interface definition !
 			nlassert(res.second);
 			
+			res = handlers.insert(std::make_pair(std::string("TL_ERR_LOG"), &CModulePipelineMasterSkel::slaveLoggedToolError_skel));
+			// if this assert, you have a doubly message name in your interface definition !
+			nlassert(res.second);
+			
 			res = handlers.insert(std::make_pair(std::string("RE_SHEETS_OK"), &CModulePipelineMasterSkel::slaveReloadedSheets_skel));
 			// if this assert, you have a doubly message name in your interface definition !
 			nlassert(res.second);
@@ -103,6 +107,20 @@ namespace PIPELINE
 		slaveRefusedBuildTask(sender);
 	}
 
+	void CModulePipelineMasterSkel::slaveLoggedToolError_skel(NLNET::IModuleProxy *sender, const NLNET::CMessage &__message)
+	{
+		H_AUTO(CModulePipelineMasterSkel_slaveLoggedToolError_TL_ERR_LOG);
+		uint8	type;
+			nlRead(__message, serial, type);
+		std::string	macroPath;
+			nlRead(__message, serial, macroPath);
+		std::string	time;
+			nlRead(__message, serial, time);
+		std::string	error;
+			nlRead(__message, serial, error);
+		slaveLoggedToolError(sender, type, macroPath, time, error);
+	}
+
 	void CModulePipelineMasterSkel::slaveReloadedSheets_skel(NLNET::IModuleProxy *sender, const NLNET::CMessage &__message)
 	{
 		H_AUTO(CModulePipelineMasterSkel_slaveReloadedSheets_RE_SHEETS_OK);
@@ -197,6 +215,24 @@ namespace PIPELINE
 		}
 	}
 		// 
+	void CModulePipelineMasterProxy::slaveLoggedToolError(NLNET::IModule *sender, uint8 type, const std::string &macroPath, const std::string &time, const std::string &error)
+	{
+		if (_LocalModuleSkel && _LocalModule->isImmediateDispatchingSupported())
+		{
+			// immediate local synchronous dispatching
+			_LocalModuleSkel->slaveLoggedToolError(_ModuleProxy->getModuleGateway()->getPluggedModuleProxy(sender), type, macroPath, time, error);
+		}
+		else
+		{
+			// send the message for remote dispatching and execution or local queing
+			NLNET::CMessage __message;
+
+			buildMessageFor_slaveLoggedToolError(__message, type, macroPath, time, error);
+
+			_ModuleProxy->sendModuleMessage(sender, __message);
+		}
+	}
+		// 
 	void CModulePipelineMasterProxy::slaveReloadedSheets(NLNET::IModule *sender)
 	{
 		if (_LocalModuleSkel && _LocalModule->isImmediateDispatchingSupported())
@@ -334,6 +370,19 @@ namespace PIPELINE
 		return __message;
 	}
 
+	// Message serializer. Return the message received in reference for easier integration
+	const NLNET::CMessage &CModulePipelineMasterProxy::buildMessageFor_slaveLoggedToolError(NLNET::CMessage &__message, uint8 type, const std::string &macroPath, const std::string &time, const std::string &error)
+	{
+		__message.setType("TL_ERR_LOG");
+			nlWrite(__message, serial, type);
+			nlWrite(__message, serial, const_cast < std::string& > (macroPath));
+			nlWrite(__message, serial, const_cast < std::string& > (time));
+			nlWrite(__message, serial, const_cast < std::string& > (error));
+
+
+		return __message;
+	}
+
 	// Message serializer. Return the message received in reference for easier integration
 	const NLNET::CMessage &CModulePipelineMasterProxy::buildMessageFor_slaveReloadedSheets(NLNET::CMessage &__message)
 	{

code/nel/tools/pipeline/service/module_pipeline_master_itf.h

@@ -66,6 +66,8 @@ namespace PIPELINE
 
 		void slaveRefusedBuildTask_skel(NLNET::IModuleProxy *sender, const NLNET::CMessage &__message);
 
+		void slaveLoggedToolError_skel(NLNET::IModuleProxy *sender, const NLNET::CMessage &__message);
+
 		void slaveReloadedSheets_skel(NLNET::IModuleProxy *sender, const NLNET::CMessage &__message);
 
 		void slaveBuildReadySuccess_skel(NLNET::IModuleProxy *sender, const NLNET::CMessage &__message);
@@ -95,6 +97,8 @@ namespace PIPELINE
 		// 
 		virtual void slaveRefusedBuildTask(NLNET::IModuleProxy *sender) =0;
 		// 
+		virtual void slaveLoggedToolError(NLNET::IModuleProxy *sender, uint8 type, const std::string &macroPath, const std::string &time, const std::string &error) =0;
+		// 
 		virtual void slaveReloadedSheets(NLNET::IModuleProxy *sender) =0;
 		// 
 		virtual void slaveBuildReadySuccess(NLNET::IModuleProxy *sender) =0;
@@ -162,6 +166,8 @@ namespace PIPELINE
 		// 
 		void slaveRefusedBuildTask(NLNET::IModule *sender);
 		// 
+		void slaveLoggedToolError(NLNET::IModule *sender, uint8 type, const std::string &macroPath, const std::string &time, const std::string &error);
+		// 
 		void slaveReloadedSheets(NLNET::IModule *sender);
 		// 
 		void slaveBuildReadySuccess(NLNET::IModule *sender);
@@ -183,6 +189,9 @@ namespace PIPELINE
 		// Message serializer. Return the message received in reference for easier integration
 		static const NLNET::CMessage &buildMessageFor_slaveRefusedBuildTask(NLNET::CMessage &__message);
 	
+		// Message serializer. Return the message received in reference for easier integration
+		static const NLNET::CMessage &buildMessageFor_slaveLoggedToolError(NLNET::CMessage &__message, uint8 type, const std::string &macroPath, const std::string &time, const std::string &error);
+	
 		// Message serializer. Return the message received in reference for easier integration
 		static const NLNET::CMessage &buildMessageFor_slaveReloadedSheets(NLNET::CMessage &__message);
 	

code/nel/tools/pipeline/service/module_pipeline_master_itf.xml

@@ -19,6 +19,15 @@
 				<doc line=""/>
 			</method>
 
+			<method name="slaveLoggedToolError" msg="TL_ERR_LOG">
+				<doc line=""/>
+
+				<param type="uint8" name="type" />
+				<param type="std::string" name="macroPath" byref="true" />
+				<param type="std::string" name="time" byref="true" />
+				<param type="std::string" name="error" byref="true" />
+			</method>
+
 			<method name="slaveReloadedSheets" msg="RE_SHEETS_OK">
 				<doc line=""/>
 			</method>

code/nel/tools/pipeline/service/module_pipeline_slave.cpp

@@ -114,8 +114,18 @@ public:
 	
 	bool m_PluginBuildDone;
 
+	struct CErrorLogData
+	{
+		TError Type;
+		std::string Path;
+		std::string Time;
+		std::string Error;
+	};
+
+	NLMISC::CSynchronized<std::vector<CErrorLogData> > m_ErrorLogWaiting;
+
 public:
-	CModulePipelineSlave() : m_Master(NULL), m_TestCommand(false), m_ReloadSheetsState(REQUEST_NONE), m_BuildReadyState(false), m_SlaveTaskState(IDLE_WAIT_MASTER), m_TaskManager(NULL), m_StatusUpdateMasterDone("StatusUpdateMasterDone"), m_StatusUpdateSlaveDone("StatusUpdateSlaveDone"), m_ActiveProject(NULL), m_ActiveProcess(NULL), m_AbortRequested(false), m_PluginBuildDone(false)
+	CModulePipelineSlave() : m_Master(NULL), m_TestCommand(false), m_ReloadSheetsState(REQUEST_NONE), m_BuildReadyState(false), m_SlaveTaskState(IDLE_WAIT_MASTER), m_TaskManager(NULL), m_StatusUpdateMasterDone("StatusUpdateMasterDone"), m_StatusUpdateSlaveDone("StatusUpdateSlaveDone"), m_ActiveProject(NULL), m_ActiveProcess(NULL), m_AbortRequested(false), m_PluginBuildDone(false), m_ErrorLogWaiting("ErrorLogWaiting")
 	{
 		NLMISC::CSynchronized<bool>::CAccessor(&m_StatusUpdateMasterDone).value() = false;
 		NLMISC::CSynchronized<bool>::CAccessor(&m_StatusUpdateSlaveDone).value() = false;
@@ -215,13 +225,35 @@ public:
 			abortBuildTask(NULL);
 			leaveBuildReadyState(NULL); // leave state if building
 
+			clearErrorLogQueue();
+
 			delete m_Master;
 			m_Master = NULL;
 		}
 	}
 
+	void clearErrorLogQueue()
+	{
+		NLMISC::CSynchronized<std::vector<CErrorLogData> >::CAccessor errorLogQueue(&m_ErrorLogWaiting);
+		errorLogQueue.value().clear();
+	}
+
+	void sendErrorLogQueue()
+	{
+		NLMISC::CSynchronized<std::vector<CErrorLogData> >::CAccessor errorLogQueue(&m_ErrorLogWaiting);
+		for (std::vector<CErrorLogData>::iterator it = errorLogQueue.value().begin(), end = errorLogQueue.value().end(); it != end; ++it)
+		{
+			const CErrorLogData &erdt = *it;
+			nlassert(m_Master);
+			m_Master->slaveLoggedToolError(this, (uint8)erdt.Type, macroPath(erdt.Path), erdt.Time, erdt.Error);
+		}
+		errorLogQueue.value().clear();
+	}
+
 	virtual void onModuleUpdate()
 	{
+		sendErrorLogQueue();
+
 		if (m_ReloadSheetsState == REQUEST_MADE)
 		{
 			if (PIPELINE::reloadSheets())
@@ -313,6 +345,7 @@ public:
 			{
 				m_SlaveTaskState = SOMEWHERE_INBETWEEN;
 				CInfoFlags::getInstance()->removeFlag(PIPELINE_INFO_PLUGIN_WORKING);
+				sendErrorLogQueue();
 				if (m_AbortRequested)
 				{
 					nlinfo("Aborted slave task while plugin was working");
@@ -525,6 +558,18 @@ public:
 	///////////////////////////////////////////////////////////////////
 	///////////////////////////////////////////////////////////////////
 
+	void notifyErrorLog(TError type, const std::string &path, const std::string &time, const std::string &error)
+	{
+		// Received an error log from a plugin tool
+		NLMISC::CSynchronized<std::vector<CErrorLogData> >::CAccessor errorLogQueue(&m_ErrorLogWaiting);
+		CErrorLogData erdt;
+		erdt.Type = type;
+		erdt.Path = path;
+		erdt.Time = time;
+		erdt.Error = error;
+		errorLogQueue.value().push_back(erdt);
+	}
+
 	class CPluginBuildTask : public IRunnable
 	{
 	public:
@@ -537,6 +582,7 @@ public:
 			case PIPELINE::PLUGIN_REGISTERED_CLASS:
 				{
 					PIPELINE::IProcessHandler *processHandler = static_cast<PIPELINE::IProcessHandler *>(NLMISC::CClassRegistry::create(m_Slave->m_ActiveProcess->m_ActivePlugin.Handler));
+					m_Slave->m_ActiveProcess->m_ErrorLogCallback = TErrorLogCallback(m_Slave, &CModulePipelineSlave::notifyErrorLog);
 					processHandler->setPipelineProcess(m_Slave->m_ActiveProcess);
 					m_Slave->m_ActiveProcess->m_SubTaskResult = FINISH_SUCCESS;
 					processHandler->build();

code/nel/tools/pipeline/service/pipeline_process_impl.h

@@ -32,15 +32,19 @@
 // STL includes
 
 // NeL includes
+#include <nel/misc/tool_logger.h>
 
 // Project includes
 #include "../plugin_library/pipeline_process.h"
 #include "pipeline_workspace.h"
 #include "metadata_storage.h"
+#include "callback.h"
 
 namespace PIPELINE {
 	class CPipelineProject;
 
+typedef CCallback<void, TError /* type */, const std::string & /* path */, const std::string & /* time */ , const std::string & /* error */ > TErrorLogCallback;
+
 /**
  * \brief CPipelineProcessImpl
  * \date 2012-03-03 09:33GMT
@@ -108,6 +112,7 @@ private:
 	std::set<std::string> m_ListDependentFiles;
 	
 	bool m_Aborting;
+	TErrorLogCallback m_ErrorLogCallback;
 
 private:
 	bool getDependencyFileStatusCached(CFileStatus &fileStatus, const std::string &filePath);

code/ryzom/common/data_leveldesign/leveldesign/pipeline/ryzom_core.pipeline_workspace

@@ -3,17 +3,7 @@
   <STRUCT>
     <ATOM Name="Description" Value="Ryzom Core"/>
     <ARRAY Name="Projects">
-      <ATOM Value="gamedev.pipeline_package"/>
-      <ATOM Value="data_common.pipeline_package"/>
       <ATOM Value="common_interface.pipeline_project"/>
-      <ATOM Value="interfaces.pipeline_package"/>
-      <ATOM Value="common_objects_generique.pipeline_project"/>
-      <ATOM Value="common_objects_fyros.pipeline_project"/>
-      <ATOM Value="common_objects_tryker.pipeline_project"/>
-      <ATOM Value="common_objects_matis.pipeline_project"/>
-      <ATOM Value="common_objects_zorai.pipeline_project"/>
-      <ATOM Value="common_objects_caravan.pipeline_project"/>
-      <ATOM Value="objects.pipeline_package"/>
     </ARRAY>
     <ARRAY Name="Plugins">
       <ATOM Value="plugin_nel.pipeline_plugin"/>
@@ -24,17 +14,16 @@
   <STRUCT/>
   <STRUCT/>
   <STRUCT/>
-  <LOG>Sat Feb 18 13:48:48 2012 (Kaetemi) .Name = Ryzom Core
+  <COMMENTS>      &lt;ATOM Value="gamedev.pipeline_package"/&gt;
-Sat Feb 18 13:48:48 2012 (Kaetemi) formName Resized = 1
+      &lt;ATOM Value="data_common.pipeline_package"/&gt;
-Sat Feb 18 14:44:25 2012 (Kaetemi) .Description = Ryzom Core
+      &lt;ATOM Value="common_interface.pipeline_project"/&gt;
-Sat Feb 18 14:44:37 2012 (Kaetemi) .Projects[0] = common_interface.pipeline_project
+      &lt;ATOM Value="interfaces.pipeline_package"/&gt;
-Sat Feb 18 23:23:17 2012 (Kaetemi) .Description = Ryzom Core Test
+      &lt;ATOM Value="common_objects_generique.pipeline_project"/&gt;
-Sat Feb 18 23:23:26 2012 (Kaetemi) .Description = Ryzom Core
+      &lt;ATOM Value="common_objects_fyros.pipeline_project"/&gt;
-Sat Mar 03 10:58:02 2012 (Kaetemi) .Plugins[0] = plugin_nel.pipeline_plugin
+      &lt;ATOM Value="common_objects_tryker.pipeline_project"/&gt;
-Sat Mar 03 10:58:02 2012 (Kaetemi) .Plugins[1] = plugin_max.pipeline_plugin
+      &lt;ATOM Value="common_objects_matis.pipeline_project"/&gt;
-Sat Mar 03 10:58:02 2012 (Kaetemi) formName Resized = 2
+      &lt;ATOM Value="common_objects_zorai.pipeline_project"/&gt;
-Sat Aug 04 21:04:25 2012 (kaetemi) .Projects[1] = interfaces.pipeline_package
+      &lt;ATOM Value="common_objects_caravan.pipeline_project"/&gt;
-Sat Aug 04 21:04:25 2012 (kaetemi) formName Resized = 2
+      &lt;ATOM Value="objects.pipeline_package"/&gt;</COMMENTS>
-Sat Aug 04 21:09:02 2012 (kaetemi) .Projects[0] = interfaces.pipeline_package
+  <LOG></LOG>
-Sat Aug 04 21:09:02 2012 (kaetemi) .Projects[1] = common_interface.pipeline_project</LOG>
 </FORM>