Added: #1440 Informational flags and other debug tools

--HG--
branch : build_pipeline_v3

code/nel/tools/pipeline/service/database_status.cpp

@@ -157,6 +157,15 @@ bool CDatabaseStatus::getFileStatus(CFileStatus &fileStatus, const std::string &
 	return seemsValid;
 }
 
+bool CDatabaseStatus::getFileStatus(std::map<std::string, CFileStatus> &fileStatusMap, const std::vector<std::string> &paths)
+{
+	for (std::vector<std::string>::const_iterator it = paths.begin(), end = paths.end(); it != end; ++it)
+	{
+		
+	}
+	return false;
+}
+
 namespace {
 
 class CUpdateFileStatus : public IRunnable

code/nel/tools/pipeline/service/database_status.h

@@ -111,6 +111,8 @@ public:
 	void updateDatabaseStatus(const CCallback<void> &callback);
 	/// Runs an update of the file status of given paths asynchronously. Warning: If g_IsExiting during callback then update is incomplete. Callback is always called when done (or failed). Do NOT use the wait parameter. Do NOT use recurse, please. Recurse directories beforehand. Paths may contain db and pl macros.
 	void updateDatabaseStatus(const CCallback<void> &callback, const std::vector<std::string> &paths, bool wait = false, bool recurse = false);
+	/// Gets the last file statuses of given paths in a map. Directories are scanned for files, non recursively. Returns false if one of the statuses is bad (not updated; file changed inbetween). Considered as build error.
+	bool getFileStatus(std::map<std::string, CFileStatus> &fileStatusMap, const std::vector<std::string> &paths);
 	
 	void getFileErrors(CFileErrors &fileErrors, const std::string &filePath, uint32 newerThan = 0) const;
 	void addFileError(const std::string &filePath, const CFileError &fileError);

code/nel/tools/pipeline/service/example_cfg/pipeline_service_default.cfg

@@ -30,6 +30,11 @@ DontUseNS = 1;
 NSHost = "localhost";
 
 WindowStyle = "WIN";
-DisplayedVariables = { "Status|pipelineServiceState", "FileQueue|asyncFileQueueCount" };
+DisplayedVariables = 
+{ 
+	"Status|pipelineServiceState", "FileQueue|asyncFileQueueCount", 
+	"", "InfoFlags|infoFlags", 
+	"", "@Reload Sheets|reloadSheets", "@Busy Test|busyTestState" ,
+};
 
 DontUseAES = 1;

code/nel/tools/pipeline/service/example_cfg/pipeline_service_master.cfg

@@ -20,3 +20,8 @@ StartCommands +=
 	"moduleManager.createModule ModulePipelineSlave slave",
 	"slave.plug gw",
 };
+
+DisplayedVariables +=
+{
+	"", "@Master Reload Sheets|master.reloadSheets", "@Status Update All|updateDatabaseStatus", 
+};

code/nel/tools/pipeline/service/info_flags.cpp

@@ -0,0 +1,108 @@
+/**
+ * \file info_flags.cpp
+ * \brief CInfoFlags
+ * \date 2012-03-04 10:46GMT
+ * \author Jan Boon (Kaetemi)
+ * CInfoFlags
+ */
+
+/* 
+ * Copyright (C) 2012  by authors
+ * 
+ * This file is part of RYZOM CORE PIPELINE.
+ * RYZOM CORE PIPELINE is free software: you can redistribute it
+ * and/or modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation, either version 2 of
+ * the License, or (at your option) any later version.
+ * 
+ * RYZOM CORE PIPELINE is distributed in the hope that it will be
+ * useful, but WITHOUT ANY WARRANTY; without even the implied warranty
+ * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ * 
+ * You should have received a copy of the GNU General Public License
+ * along with RYZOM CORE PIPELINE; see the file COPYING.  If not, see
+ * <http://www.gnu.org/licenses/>.
+ */
+
+#include <nel/misc/types_nl.h>
+#include "info_flags.h"
+
+// STL includes
+#include <sstream>
+
+// NeL includes
+// #include <nel/misc/debug.h>
+
+// Project includes
+#include "info_flags.h"
+
+using namespace std;
+// using namespace NLMISC;
+
+NLMISC::CVariable<std::string> InfoFlags("pipeline", "infoFlags", "READ ONLY", "<DISABLED>");
+
+namespace PIPELINE {
+
+CInfoFlags::CInfoFlags()
+{
+	updateInfoFlags();
+}
+
+CInfoFlags::~CInfoFlags()
+{
+	InfoFlags = "<DISABLED>";
+}
+
+void CInfoFlags::addFlag(const std::string &flagName)
+{
+	nldebug("addFlag: %s", flagName.c_str());
+	if (m_FlagMap.find(flagName) != m_FlagMap.end())
+	{
+		++m_FlagMap[flagName];
+	}
+	else
+	{
+		m_FlagMap[flagName] = 1;
+	}
+	updateInfoFlags();
+}
+
+void CInfoFlags::removeFlag(const std::string &flagName)
+{
+	nldebug("removeFlag: %s", flagName.c_str());
+	std::map<std::string, uint>::iterator it = m_FlagMap.find(flagName);
+	if (it != m_FlagMap.end())
+	{
+		if (it->second == 1)
+			m_FlagMap.erase(it);
+		else
+			--it->second;
+	}
+	updateInfoFlags();
+}
+
+void CInfoFlags::updateInfoFlags()
+{
+	if (m_FlagMap.empty())
+	{
+		InfoFlags = "<NONE>";
+	}
+	else
+	{
+		std::stringstream ss;
+		for (std::map<std::string, uint>::iterator it = m_FlagMap.begin(), end = m_FlagMap.end(); it != end; ++it)
+		{
+			ss << it->first;
+			if (it ->second > 1)
+				ss << " (" << it->second << ")";
+			ss << ", ";
+		}
+		std::string s = ss.str();
+		InfoFlags = s.substr(0, s.size() - 2);
+	}
+}
+
+} /* namespace PIPELINE */
+
+/* end of file */

code/nel/tools/pipeline/service/info_flags.h

@@ -0,0 +1,68 @@
+/**
+ * \file info_flags.h
+ * \brief CInfoFlags
+ * \date 2012-03-04 10:46GMT
+ * \author Jan Boon (Kaetemi)
+ * CInfoFlags
+ */
+
+/* 
+ * Copyright (C) 2012  by authors
+ * 
+ * This file is part of RYZOM CORE PIPELINE.
+ * RYZOM CORE PIPELINE is free software: you can redistribute it
+ * and/or modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation, either version 2 of
+ * the License, or (at your option) any later version.
+ * 
+ * RYZOM CORE PIPELINE is distributed in the hope that it will be
+ * useful, but WITHOUT ANY WARRANTY; without even the implied warranty
+ * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ * 
+ * You should have received a copy of the GNU General Public License
+ * along with RYZOM CORE PIPELINE; see the file COPYING.  If not, see
+ * <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef PIPELINE_INFO_FLAGS_H
+#define PIPELINE_INFO_FLAGS_H
+#include <nel/misc/types_nl.h>
+
+// STL includes
+
+// NeL includes
+#include <nel/misc/singleton.h>
+#include <nel/misc/variable.h>
+
+// Project includes
+
+namespace PIPELINE {
+
+/**
+ * \brief CInfoFlags
+ * \date 2012-03-04 10:46GMT
+ * \author Jan Boon (Kaetemi)
+ * CInfoFlags
+ */
+	class CInfoFlags : public NLMISC::CManualSingleton<CInfoFlags>
+{
+protected:
+	std::map<std::string, uint> m_FlagMap;
+public:
+	CInfoFlags();
+	virtual ~CInfoFlags();
+
+	void addFlag(const std::string &flagName);
+	void removeFlag(const std::string &flagName);
+
+private:
+	void updateInfoFlags();
+
+}; /* class CInfoFlags */
+
+} /* namespace PIPELINE */
+
+#endif /* #ifndef PIPELINE_INFO_FLAGS_H */
+
+/* end of file */

code/nel/tools/pipeline/service/module_pipeline_master.cpp

@@ -36,6 +36,7 @@
 #include <nel/misc/debug.h>
 
 // Project includes
+#include "info_flags.h"
 #include "module_pipeline_slave_itf.h"
 #include "pipeline_service.h"
 #include "database_status.h"
@@ -46,6 +47,8 @@ using namespace NLNET;
 
 namespace PIPELINE {
 
+#define PIPELINE_INFO_MASTER_RELOAD_SHEETS "MASTER_RELOAD_SHEETS"
+
 /**
  * \brief CModulePipelineMaster
  * \date 2012-03-03 16:26GMT
@@ -70,6 +73,14 @@ class CModulePipelineMaster :
 		uint32 ActiveTaskId;
 		bool SheetsOk;
 
+		~CSlave()
+		{
+			if (!SheetsOk)
+			{
+				CInfoFlags::getInstance()->removeFlag(PIPELINE_INFO_MASTER_RELOAD_SHEETS);
+			}
+		}
+
 		void cbUpdateDatabaseStatus()
 		{
 			Proxy.masterUpdatedDatabaseStatus(Master);
@@ -159,11 +170,11 @@ public:
 		// if state build, iterate trough all slaves to see if any is free, and check if there's any waiting tasks
 	}
 
-	virtual void slaveFinishedBuildTask(NLNET::IModuleProxy *sender, uint32 taskId)
+	virtual void slaveFinishedBuildTask(NLNET::IModuleProxy *sender, uint32 taskId, uint8 errorLevel)
 	{
 		// TODO
 	}
-
+	
 	virtual void slaveRefusedBuildTask(NLNET::IModuleProxy *sender, uint32 taskId)
 	{
 		// TODO
@@ -173,6 +184,7 @@ public:
 	{
 		CSlave *slave = m_Slaves[sender];
 		slave->SheetsOk = true;
+		CInfoFlags::getInstance()->removeFlag(PIPELINE_INFO_MASTER_RELOAD_SHEETS);
 	}
 
 	virtual void vectorPushString(NLNET::IModuleProxy *sender, const std::string &str)
@@ -180,7 +192,7 @@ public:
 		CSlave *slave = m_Slaves[sender];
 		slave->Vector.push_back(str);
 	}
-	 
+	
 	virtual void updateDatabaseStatusByVector(NLNET::IModuleProxy *sender)
 	{
 		CSlave *slave = m_Slaves[sender];
@@ -197,18 +209,29 @@ protected:
 	{
 		if (args.size() != 0) return false;
 
-		m_SlavesMutex.lock();
+		if (PIPELINE::tryDirectTask("MASTER_RELOAD_SHEETS"))
-		
-		for (TSlaveMap::iterator it = m_Slaves.begin(), end = m_Slaves.end(); it != end; ++it)
 		{
-			CSlave *slave = it->second;
+			m_SlavesMutex.lock();
-			slave->SheetsOk = false;
+			
-			slave->Proxy.reloadSheets(this);
+			for (TSlaveMap::iterator it = m_Slaves.begin(), end = m_Slaves.end(); it != end; ++it)
-		}
+			{
-		
+				CSlave *slave = it->second;
-		m_SlavesMutex.unlock();
+				slave->SheetsOk = false;
+				slave->Proxy.reloadSheets(this);
+				CInfoFlags::getInstance()->addFlag(PIPELINE_INFO_MASTER_RELOAD_SHEETS);
+			}
+			
+			m_SlavesMutex.unlock();
 
-		return true;
+			PIPELINE::endedDirectTask();
+
+			return true;
+		}
+		else
+		{
+			log.displayNL("Busy");
+			return false;
+		}
 	}
 
 }; /* class CModulePipelineMaster */

code/nel/tools/pipeline/service/module_pipeline_master_itf.xml

@@ -4,13 +4,14 @@
 
 		<module_interface name="CModulePipelineMaster">
 
-			<method name="slaveFinishedBuildTask" msg="RE_BT_OK">
+			<method name="slaveFinishedBuildTask" msg="RE_BT_DONE">
 				<doc line=""/>
 
 				<param type="uint32"		name="taskId"	/>
+				<param type="uint8"			name="errorLevel"	/>
 			</method>
 
-			<method name="slaveRefusedBuildTask" msg="RE_BT_FAIL">
+			<method name="slaveRefusedBuildTask" msg="RE_BT_REFUSED">
 				<doc line=""/>
 
 				<param type="uint32"		name="taskId"	/>

code/nel/tools/pipeline/service/module_pipeline_slave.cpp

@@ -34,6 +34,7 @@
 #include <nel/misc/debug.h>
 
 // Project includes
+#include "info_flags.h"
 #include "module_pipeline_master_itf.h"
 #include "pipeline_service.h"
 #include "../plugin_library/process_info.h"
@@ -46,6 +47,15 @@ using namespace NLNET;
 
 namespace PIPELINE {
 
+#define PIPELINE_INFO_SLAVE_RELOAD_SHEETS "SLAVE_RELOAD_SHEETS"
+
+enum TRequestState
+{
+	REQUEST_NONE, 
+	REQUEST_MADE, 
+	REQUEST_WORKING, 
+};
+
 /**
  * \brief CModulePipelineSlave
  * \date 2012-03-03 16:26GMT
@@ -59,10 +69,10 @@ class CModulePipelineSlave :
 public:
 	CModulePipelineMasterProxy *m_Master;
 	bool m_TestCommand;
-	bool m_RequestedReloadSheets;
+	TRequestState m_ReloadSheetsState;
 	
 public:
-	CModulePipelineSlave() : m_Master(NULL), m_TestCommand(false), m_RequestedReloadSheets(false)
+	CModulePipelineSlave() : m_Master(NULL), m_TestCommand(false), m_ReloadSheetsState(REQUEST_NONE)
 	{
 		
 	}
@@ -106,8 +116,22 @@ public:
 
 	virtual void onModuleUpdate()
 	{
-		if (m_RequestedReloadSheets)
+		if (m_ReloadSheetsState == REQUEST_MADE)
-			m_RequestedReloadSheets = !PIPELINE::reloadSheets();
+		{
+			if (PIPELINE::reloadSheets())
+			{
+				m_ReloadSheetsState = REQUEST_WORKING;
+			}
+		}
+		else if (m_ReloadSheetsState == REQUEST_WORKING)
+		{
+			if (PIPELINE::isServiceStateIdle())
+			{
+				m_ReloadSheetsState = REQUEST_NONE;
+				m_Master->slaveReloadedSheets(this);
+				CInfoFlags::getInstance()->removeFlag(PIPELINE_INFO_SLAVE_RELOAD_SHEETS);
+			}
+		}
 	}
 
 	virtual void startBuildTask(NLNET::IModuleProxy *sender, uint32 taskId, const std::string &projectName, const std::string &processHandler)
@@ -131,8 +155,9 @@ public:
 
 	virtual void reloadSheets(NLNET::IModuleProxy *sender)
 	{
-		if (!PIPELINE::reloadSheets())
+		CInfoFlags::getInstance()->addFlag(PIPELINE_INFO_SLAVE_RELOAD_SHEETS);
-			m_RequestedReloadSheets = true;
+		if (PIPELINE::reloadSheets()) m_ReloadSheetsState = REQUEST_WORKING;
+		else m_ReloadSheetsState = REQUEST_MADE;
 	}
 	
 protected:

code/nel/tools/pipeline/service/pipeline_service.cpp

@@ -46,8 +46,10 @@
 #include <nel/misc/async_file_manager.h>
 #include <nel/misc/algo.h>
 #include <nel/misc/dynloadlib.h>
+#include <nel/net/module_manager.h>
 
 // Project includes
+#include "info_flags.h"
 #include "pipeline_workspace.h"
 #include "pipeline_project.h"
 #include "database_status.h"
@@ -110,9 +112,12 @@ enum EState
 	STATE_RELOAD_SHEETS, 
 	STATE_DATABASE_STATUS, 
 	STATE_RUNNABLE_TASK, 
+	STATE_BUSY_TEST, 
+	STATE_DIRECT_CODE, 
 };
 
 /// Data
+CInfoFlags *s_InfoFlags = NULL;
 CTaskManager *s_TaskManager = NULL;
 CPipelineInterfaceImpl *s_PipelineInterfaceImpl = NULL;
 CPipelineProcessImpl *s_PipelineProcessImpl = NULL;
@@ -158,6 +163,11 @@ bool tryStateTask(EState state, IRunnable *task)
 
 } /* anonymous namespace */
 
+bool isServiceStateIdle()
+{
+	return (s_State == STATE_IDLE);
+}
+
 bool tryRunnableTask(std::string stateName, IRunnable *task)
 {
 	// copy paste from above.
@@ -173,22 +183,54 @@ bool tryRunnableTask(std::string stateName, IRunnable *task)
 	s_StateMutex.leave();
 	if (!result) return false;
 
-	nlassert(s_State != STATE_IDLE);
+	nlassert(s_State == STATE_RUNNABLE_TASK);
 	
 	s_TaskManager->addTask(task);
 	
 	return true;
 }
 
-void endedRunnableTask()
+namespace {
+
+void endedRunnableTask(EState state)
 {
-	nlassert(s_State != STATE_IDLE);
+	nlassert(s_State == state);
 
 	s_StateMutex.enter();
 	s_State = STATE_IDLE;
 	s_StateMutex.leave();
 }
 
+} /* anonymous namespace */
+
+void endedRunnableTask()
+{
+	endedRunnableTask(STATE_RUNNABLE_TASK);
+}
+
+bool tryDirectTask(const std::string &stateName)
+{
+	bool result = false;
+	s_StateMutex.enter();
+	result = (s_State == STATE_IDLE);
+	if (result)
+	{
+		s_State = STATE_DIRECT_CODE;
+		s_StateRunnableTaskName = stateName;
+	}
+	s_StateMutex.leave();
+	if (!result) return false;
+
+	nlassert(s_State == STATE_DIRECT_CODE);
+	
+	return true;
+}
+
+void endedDirectTask()
+{
+	endedRunnableTask(STATE_DIRECT_CODE);
+}
+
 // ******************************************************************
 
 namespace {
@@ -230,7 +272,7 @@ class CReloadSheets : public IRunnable
 		releaseSheets();
 		initSheets();
 		
-		endedRunnableTask();
+		endedRunnableTask(STATE_RELOAD_SHEETS);
 	}
 };
 CReloadSheets s_ReloadSheets;
@@ -253,7 +295,7 @@ class CUpdateDatabaseStatus : public IRunnable
 	
 	void databaseStatusUpdated()
 	{
-		endedRunnableTask();
+		endedRunnableTask(STATE_DATABASE_STATUS);
 	}
 
 	virtual void run()
@@ -270,6 +312,28 @@ bool updateDatabaseStatus()
 
 // ******************************************************************
 
+// ******************************************************************
+
+class CBusyTestStatus : public IRunnable
+{
+	virtual void getName(std::string &result) const 
+	{ result = "CBusyTestStatus"; }
+
+	virtual void run()
+	{
+		nlSleep(20000);
+		endedRunnableTask(STATE_BUSY_TEST);
+	}
+};
+CBusyTestStatus s_BusyTestStatus;
+
+bool busyTestStatus()
+{
+	return tryStateTask(STATE_BUSY_TEST, &s_BusyTestStatus);
+}
+
+// ******************************************************************
+
 /**
  * \brief CPipelineService
  * \date 2012-02-18 17:25GMT
@@ -309,6 +373,10 @@ public:
 	/// Initializes the service (must be called before the first call to update())
 	virtual void init()
 	{
+		s_InfoFlags = new CInfoFlags();
+
+		s_InfoFlags->addFlag("INIT");
+
 		g_DatabaseDirectory = CPath::standardizePath(ConfigFile.getVar("DatabaseDirectory").asString(), true);
 		if (!CFile::isDirectory(g_DatabaseDirectory)) nlwarning("'DatabaseDirectory' does not exist! (%s)", g_DatabaseDirectory.c_str());
 		ConfigFile.getVar("DatabaseDirectory").setAsString(g_DatabaseDirectory);
@@ -337,6 +405,8 @@ public:
 			}
 			else delete library;
 		}
+
+		s_InfoFlags->removeFlag("INIT");
 	}
 	
 	/// This function is called every "frame" (you must call init() before). It returns false if the service is stopped.
@@ -350,6 +420,10 @@ public:
 	{
 		g_IsExiting = true;
 
+		s_InfoFlags->addFlag("RELEASE");
+
+		NLNET::IModuleManager::releaseInstance();
+
 		while (NLMISC::CAsyncFileManager::getInstance().getNumWaitingTasks() > 0)
 		{
 			nlSleep(10);
@@ -363,6 +437,8 @@ public:
 		}
 		s_LoadedLibraries.clear();
 
+		CClassRegistry::release();
+
 		delete s_PipelineProcessImpl;
 		s_PipelineProcessImpl = NULL;
 
@@ -376,6 +452,11 @@ public:
 
 		delete s_TaskManager;
 		s_TaskManager = NULL;
+
+		s_InfoFlags->removeFlag("RELEASE");
+
+		delete s_InfoFlags;
+		s_InfoFlags = NULL;
 	}
 	
 }; /* class CPipelineService */
@@ -401,7 +482,13 @@ NLMISC_DYNVARIABLE(std::string, pipelineServiceState, "State of the pipeline ser
 				*pointer = "DATABASE_STATUS";
 				break;
 			case PIPELINE::STATE_RUNNABLE_TASK:
-				*pointer = PIPELINE::s_StateRunnableTaskName;
+				*pointer = "RT: " + PIPELINE::s_StateRunnableTaskName;
+				break;
+			case PIPELINE::STATE_DIRECT_CODE:
+				*pointer = "DC: " + PIPELINE::s_StateRunnableTaskName;
+				break;
+			case PIPELINE::STATE_BUSY_TEST:
+				*pointer = "BUSY_TEST";
 				break;
 		}
 	}
@@ -438,6 +525,17 @@ NLMISC_COMMAND(updateDatabaseStatus, "Updates the entire database status. This a
 	return true;
 }
 
+NLMISC_COMMAND(busyTestState, "Keeps the service busy for twenty seconds.", "")
+{
+	if(args.size() != 0) return false;
+	if (!PIPELINE::busyTestStatus())
+	{
+		log.displayNL("Already busy");
+		return false;
+	}
+	return true;
+}
+
 NLMISC_COMMAND(dumpTaskManager, "Dumps the task manager.", "")
 {
 	if(args.size() != 0) return false;

code/nel/tools/pipeline/service/pipeline_service.h

@@ -70,6 +70,12 @@ std::string macroPath(const std::string &path);
 bool tryRunnableTask(std::string stateName, NLMISC::IRunnable *task);
 void endedRunnableTask();
 
+bool tryDirectTask(const std::string &stateName);
+void endedDirectTask();
+
+/// Only use for informational purposes, service may not be idle after calling this. Not for thread safe usage.
+bool isServiceStateIdle();
+
 bool reloadSheets();
 
 extern NLGEORGES::UFormLoader *g_FormLoader;