Added: #1440 Initial CRC32 verification for workspace sheets

--HG--
branch : build_pipeline_v3

code/nel/tools/pipeline/service/database_status.cpp

@@ -63,6 +63,23 @@ bool isInRootDirectoryFast(std::string &rootDirectoryName, std::string &rootDire
 	return false;
 }
 
+bool isInSheetsDirectoryFast(std::string &sheetDirectoryName, std::string &sheetDirectoryPath, const std::string &path)
+{
+	{
+		sheetDirectoryName = "WorkspaceDfnDirectory";
+		CConfigFile::CVar &dir = NLNET::IService::getInstance()->ConfigFile.getVar(sheetDirectoryName);
+		sheetDirectoryPath = CPath::standardizePath(dir.asString(), true);
+		if (path.find(sheetDirectoryPath) == 0) return true;
+	}
+	{
+		sheetDirectoryName = "WorkspaceSheetDirectory";
+		CConfigFile::CVar &dir = NLNET::IService::getInstance()->ConfigFile.getVar(sheetDirectoryName);
+		sheetDirectoryPath = CPath::standardizePath(dir.asString(), true);
+		if (path.find(sheetDirectoryPath) == 0) return true;
+	}
+	return false;
+}
+
 /// Input must be normalized path
 bool isInWorkspaceDirectoryFast(const std::string &path)
 {
@@ -75,6 +92,12 @@ std::string dropRootDirectoryFast(const std::string &path, const std::string &ro
 	return path.substr(rootDirectoryPath.length());
 }
 
+/// Input must be normalized path in sheets directory
+std::string dropSheetDirectoryFast(const std::string &path, const std::string &sheetDirectoryPath)
+{
+	return path.substr(sheetDirectoryPath.length());
+}
+
 /// Input must be normalized path in pipeline directory
 std::string dropWorkspaceDirectoryFast(const std::string &path)
 {
@@ -99,15 +122,23 @@ std::string getMetaFilePath(const std::string &path, const std::string &dotSuffi
 	{
 		std::string rootDirectoryName;
 		std::string rootDirectoryPath;
-		if (isInRootDirectoryFast(rootDirectoryName, rootDirectoryPath, stdPath))
+		if (isInSheetsDirectoryFast(rootDirectoryName, rootDirectoryPath, stdPath))
 		{
-			std::string relPath = dropRootDirectoryFast(stdPath, rootDirectoryPath);
+			std::string relPath = dropSheetDirectoryFast(stdPath, rootDirectoryPath);
-			return g_WorkDir + PIPELINE_DIRECTORY_PREFIX_ROOT + NLMISC::toLower(rootDirectoryName) + PIPELINE_DATABASE_META_SUFFIX + "/" + relPath + dotSuffix;
+			return g_WorkDir + PIPELINE_DIRECTORY_PREFIX_SHEET + NLMISC::toLower(rootDirectoryName) + PIPELINE_DATABASE_META_SUFFIX + "/" + relPath + dotSuffix;
 		}
 		else
 		{
-			nlerror("Path is not in database or pipeline (%s)", path.c_str());
+			if (isInRootDirectoryFast(rootDirectoryName, rootDirectoryPath, stdPath))
-			return path + dotSuffix;
+			{
+				std::string relPath = dropRootDirectoryFast(stdPath, rootDirectoryPath);
+				return g_WorkDir + PIPELINE_DIRECTORY_PREFIX_ROOT + NLMISC::toLower(rootDirectoryName) + PIPELINE_DATABASE_META_SUFFIX + "/" + relPath + dotSuffix;
+			}
+			else
+			{
+				nlerror("Path is not in database or pipeline (%s)", path.c_str());
+				return path + dotSuffix;
+			}
 		}
 	}
 }

code/nel/tools/pipeline/service/module_pipeline_master.cpp

@@ -34,6 +34,7 @@
 
 // NeL includes
 #include <nel/misc/debug.h>
+#include <nel/net/service.h>
 
 // Project includes
 #include "info_flags.h"
@@ -148,9 +149,30 @@ public:
 	{
 		CModuleBase::initModule(initInfo);
 		CModulePipelineMasterSkel::init(this);
+		if (PIPELINE::tryDirectTask("MASTER_INIT_SHEETS"))
+		{
+			updateSheetsDatabaseStatus(CCallback<void>(this, &CModulePipelineMaster::cbMasterInitSheets));
+		}
+		else
+		{
+			nlerror("Cannot initialize master, the service is busy. This should never happen");
+		}
 		return true;
 	}
 
+	void updateSheetsDatabaseStatus(const CCallback<void> &callback)
+	{
+		std::vector<std::string> sheetPaths;
+		// sheetPaths.push_back(NLNET::IService::getInstance()->ConfigFile.getVar("WorkspaceDfnDirectory").asString()); // not really necessary to check the dfn's
+		sheetPaths.push_back(NLNET::IService::getInstance()->ConfigFile.getVar("WorkspaceSheetDirectory").asString());
+		g_DatabaseStatus->updateDatabaseStatus(callback, sheetPaths, false, true);
+	}
+
+	void cbMasterInitSheets()
+	{
+		PIPELINE::endedDirectTask();
+	}
+
 	virtual void onModuleUp(IModuleProxy *moduleProxy)
 	{
 		if (moduleProxy->getModuleClassName() == "ModulePipelineSlave")
@@ -418,20 +440,7 @@ protected:
 
 		if (PIPELINE::tryDirectTask("MASTER_RELOAD_SHEETS"))
 		{
-			m_SlavesMutex.lock();
+			updateSheetsDatabaseStatus(CCallback<void>(this, &CModulePipelineMaster::cbMasterSheetReloadUpdate));
-			
-			for (TSlaveMap::iterator it = m_Slaves.begin(), end = m_Slaves.end(); it != end; ++it)
-			{
-				CSlave *slave = it->second;
-				slave->SheetsOk = false;
-				slave->PluginsAvailable.clear();
-				slave->Proxy.reloadSheets(this);
-				CInfoFlags::getInstance()->addFlag(PIPELINE_INFO_MASTER_RELOAD_SHEETS);
-			}
-			
-			m_SlavesMutex.unlock();
-
-			PIPELINE::endedDirectTask();
 
 			return true;
 		}
@@ -442,6 +451,24 @@ protected:
 		}
 	}
 	
+	void cbMasterSheetReloadUpdate()
+	{
+		m_SlavesMutex.lock();
+			
+		for (TSlaveMap::iterator it = m_Slaves.begin(), end = m_Slaves.end(); it != end; ++it)
+		{
+			CSlave *slave = it->second;
+			slave->SheetsOk = false;
+			slave->PluginsAvailable.clear();
+			slave->Proxy.reloadSheets(this);
+			CInfoFlags::getInstance()->addFlag(PIPELINE_INFO_MASTER_RELOAD_SHEETS);
+		}
+		
+		m_SlavesMutex.unlock();
+
+		PIPELINE::endedDirectTask();
+	}
+
 	NLMISC_CLASS_COMMAND_DECL(build)
 	{
 		bool bypassErrors = false;

code/nel/tools/pipeline/service/module_pipeline_slave.cpp

@@ -110,6 +110,8 @@ public:
 	{
 		// TODO: AUTHENTICATE OR GATEWAY SECURITY?
 		CModulePipelineMasterProxy master(sender);
+		if (!g_PipelineWorkspace->loadCRC32())
+			nlerror("Failed sheets CRC32. Sheets were modified inbetween launching services. This causes newly loaded services to be out of sync. Not allowed");
 		sendMasterAvailablePlugins(&master);
 	}
 	
@@ -144,6 +146,8 @@ public:
 			if (PIPELINE::isServiceStateIdle())
 			{
 				m_ReloadSheetsState = REQUEST_NONE;
+				if (!g_PipelineWorkspace->loadCRC32())
+					nlerror("Failed sheets CRC32. Sheets were modified inbetween launching services. This causes newly loaded services to be out of sync. Not allowed");
 				sendMasterAvailablePlugins(m_Master);
 				m_Master->slaveReloadedSheets(this);
 				CInfoFlags::getInstance()->removeFlag(PIPELINE_INFO_SLAVE_RELOAD_SHEETS);

code/nel/tools/pipeline/service/pipeline_project.cpp

@@ -47,7 +47,7 @@ using namespace std;
 
 namespace PIPELINE {
 
-CPipelineProject::CPipelineProject(CPipelineWorkspace *workspace, NLGEORGES::UForm *form) : m_Workspace(workspace), m_Form(form)
+CPipelineProject::CPipelineProject(CPipelineWorkspace *workspace, NLGEORGES::UForm *form) : m_Workspace(workspace), m_Form(form), m_ChangedReference(0), m_FileSizeReference(0), m_CRC32(0)
 {
 	
 }

code/nel/tools/pipeline/service/pipeline_project.h

@@ -48,10 +48,15 @@ namespace PIPELINE {
  */
 class CPipelineProject
 {
+	friend class PIPELINE::CPipelineWorkspace;
+
 protected:
 	CPipelineWorkspace *m_Workspace;
 	NLMISC::CRefPtr<NLGEORGES::UForm> m_Form;
 	std::string m_TempDirectory;
+	uint32 m_ChangedReference;
+	uint32 m_FileSizeReference;
+	uint32 m_CRC32;
 
 public:
 	CPipelineProject(CPipelineWorkspace *workspace, NLGEORGES::UForm *form);

code/nel/tools/pipeline/service/pipeline_service.cpp

@@ -354,8 +354,10 @@ namespace {
 
 void initSheets()
 {
-	std::string dfnDirectory = IService::getInstance()->ConfigFile.getVar("WorkspaceDfnDirectory").asString();
+	std::string dfnDirectory = CPath::standardizePath(IService::getInstance()->ConfigFile.getVar("WorkspaceDfnDirectory").asString(), true);
-	std::string sheetDirectory = IService::getInstance()->ConfigFile.getVar("WorkspaceSheetDirectory").asString();
+	IService::getInstance()->ConfigFile.getVar("WorkspaceDfnDirectory").setAsString(dfnDirectory);
+	std::string sheetDirectory = CPath::standardizePath(IService::getInstance()->ConfigFile.getVar("WorkspaceSheetDirectory").asString(), true);
+	IService::getInstance()->ConfigFile.getVar("WorkspaceSheetDirectory").setAsString(sheetDirectory);
 	
 	if (!CFile::isDirectory(dfnDirectory)) nlerror("'WorkspaceDfnDirectory' does not exist! (%s)", dfnDirectory.c_str());
 	nlinfo("Adding 'WorkspaceDfnDirectory' to search path (%s)", dfnDirectory.c_str());

code/nel/tools/pipeline/service/pipeline_service.h

@@ -59,6 +59,8 @@ extern std::string g_WorkDir;
 #define PIPELINE_DIRECTORY_UNKNOWN_PREFIX "unknown."
 #define PIPELINE_DIRECTORY_PREFIX_PROJECT "project."
 #define PIPELINE_DIRECTORY_PREFIX_ROOT "root."
+#define PIPELINE_DIRECTORY_PREFIX_SHEET "pipeline."
+
 #define PIPELINE_DIRECTORY_TEMP_SUFFIX ".temp"
 
 /// Unmacros a path, and standardizes it as well.

code/nel/tools/pipeline/service/pipeline_workspace.cpp

@@ -39,6 +39,7 @@
 #include "pipeline_project.h"
 #include "pipeline_service.h"
 #include "pipeline_interface_impl.h"
+#include "database_status.h"
 
 using namespace std;
 // using namespace NLMISC;
@@ -46,8 +47,14 @@ using namespace NLGEORGES;
 
 namespace PIPELINE {
 
-CPipelineWorkspace::CPipelineWorkspace(NLGEORGES::UFormLoader *formLoader, const std::string &sheetName) : m_FormLoader(formLoader)
+CPipelineWorkspace::CPipelineWorkspace(NLGEORGES::UFormLoader *formLoader, const std::string &sheetName) : m_FormLoader(formLoader), m_ChangedReference(0), m_FileSizeReference(0), m_CRC32(0)
 {
+	std::string fullSheetPath = NLMISC::CPath::lookup(sheetName);
+	{
+		m_ChangedReference = NLMISC::CFile::getFileModificationDate(fullSheetPath);
+		m_FileSizeReference = NLMISC::CFile::getFileSize(fullSheetPath);
+	}
+
 	m_Form = formLoader->loadForm(sheetName.c_str());
 	std::string description;
 	m_Form->getRootNode().getValueByName(description, "Description");
@@ -91,7 +98,17 @@ CPipelineWorkspace::CPipelineWorkspace(NLGEORGES::UFormLoader *formLoader, const
 				{
 					std::string projectName = NLMISC::CFile::getFilenameWithoutExtension(projectSheet);
 					if (m_Projects.find(projectName) == m_Projects.end())
-						m_Projects[projectName] = new CPipelineProject(this, formLoader->loadForm(projectSheet.c_str()));
+					{
+						std::string fullProjectSheetPath = NLMISC::CPath::lookup(projectSheet);
+						uint32 changedReference = NLMISC::CFile::getFileModificationDate(fullProjectSheetPath);
+						uint32 fileSizeReference = NLMISC::CFile::getFileSize(fullProjectSheetPath);
+						CPipelineProject *project = new CPipelineProject(this, formLoader->loadForm(projectSheet.c_str()));
+						m_Projects[projectName] = project;
+						nlassert(changedReference == NLMISC::CFile::getFileModificationDate(fullProjectSheetPath));
+						nlassert(fileSizeReference == NLMISC::CFile::getFileSize(fullProjectSheetPath));
+						project->m_ChangedReference = changedReference;
+						project->m_FileSizeReference = fileSizeReference;
+					}
 					else
 						nlwarning("Project '%s' in '%s' already", projectSheet.c_str(), m_Form->getFilename().c_str());
 				}
@@ -106,6 +123,11 @@ CPipelineWorkspace::CPipelineWorkspace(NLGEORGES::UFormLoader *formLoader, const
 			nlwarning("Missing 'Projects' in '%s'", m_Form->getFilename().c_str());
 		}
 	}
+
+	{
+		nlassert(m_ChangedReference == NLMISC::CFile::getFileModificationDate(fullSheetPath));
+		nlassert(m_FileSizeReference == NLMISC::CFile::getFileSize(fullSheetPath));
+	}
 }
 
 CPipelineWorkspace::~CPipelineWorkspace()
@@ -335,6 +357,54 @@ CPipelineProject *CPipelineWorkspace::getProject(const std::string &project)
 	return it->second;
 }
 
+bool CPipelineWorkspace::loadCRC32()
+{
+	bool ok = true;
+	{
+		std::string sheetPath = NLMISC::CPath::lookup(m_Form->getFilename());
+		CFileStatus fileStatus;
+		if (g_DatabaseStatus->getFileStatus(fileStatus, sheetPath)
+			&& fileStatus.LastChangedReference == m_ChangedReference
+			&& fileStatus.LastFileSizeReference == m_FileSizeReference)
+		{
+			m_CRC32 = fileStatus.CRC32;
+		}
+		else
+		{
+			nlwarning("Workspace sheet '%s' not synchronized", sheetPath.c_str());
+			ok = false;
+		}
+	}
+	for (std::map<std::string, CPipelineProject *>::iterator it = m_Projects.begin(), end = m_Projects.end(); it != end; ++it)
+	{
+		CPipelineProject *project = it->second;
+		{
+			std::string sheetPath = NLMISC::CPath::lookup(project->m_Form->getFilename());
+			CFileStatus fileStatus;
+			if (g_DatabaseStatus->getFileStatus(fileStatus, sheetPath)
+				&& fileStatus.LastChangedReference == project->m_ChangedReference
+				&& fileStatus.LastFileSizeReference == project->m_FileSizeReference)
+			{
+				project->m_CRC32 = fileStatus.CRC32;
+			}
+			else
+			{
+				nlwarning("Workspace sheet '%s' not synchronized", sheetPath.c_str());
+				ok = false;
+			}
+		}
+	}
+	if (!ok)
+	{
+		nlwarning("Workspace not synchronized");
+	}
+	else
+	{
+		nldebug("Workspace CRC32 synchronized");
+	}
+	return ok;
+}
+
 } /* namespace PIPELINE */
 
 /* end of file */

code/nel/tools/pipeline/service/pipeline_workspace.h

@@ -78,13 +78,16 @@ struct CProcessPluginInfo
  */
 class CPipelineWorkspace
 {
-	friend class CPipelineProject;
+	friend class PIPELINE::CPipelineProject;
 
 protected:
 	NLGEORGES::UFormLoader *m_FormLoader;
 	NLMISC::CRefPtr<NLGEORGES::UForm> m_Form;
 	std::vector<NLMISC::CRefPtr<NLGEORGES::UForm> > m_Plugins;
 	std::map<std::string, CPipelineProject *> m_Projects;
+	uint32 m_ChangedReference;
+	uint32 m_FileSizeReference;
+	uint32 m_CRC32;
 
 public:
 	CPipelineWorkspace(NLGEORGES::UFormLoader *formLoader, const std::string &sheetName);
@@ -100,6 +103,9 @@ public:
 	/// Makes a list of the global id of all available plugins to this service
 	void listAvailablePlugins(std::vector<uint32> &result);
 
+	/// Loads the CRC32 of the sheets, false if something out of sync
+	bool loadCRC32();
+
 }; /* class CPipelineWorkspace */
 
 } /* namespace PIPELINE */