From 085170e60767bfd6c6a31e6222ee0942d7db3074 Mon Sep 17 00:00:00 2001 From: kaetemi Date: Sat, 10 Mar 2012 11:08:43 +0100 Subject: [PATCH] Added: #1440 Initial CRC32 verification for workspace sheets --HG-- branch : build_pipeline_v3 --- .../pipeline/service/database_status.cpp | 41 ++++++++-- .../service/module_pipeline_master.cpp | 55 ++++++++++---- .../service/module_pipeline_slave.cpp | 4 + .../pipeline/service/pipeline_project.cpp | 2 +- .../tools/pipeline/service/pipeline_project.h | 5 ++ .../pipeline/service/pipeline_service.cpp | 6 +- .../tools/pipeline/service/pipeline_service.h | 2 + .../pipeline/service/pipeline_workspace.cpp | 74 ++++++++++++++++++- .../pipeline/service/pipeline_workspace.h | 8 +- 9 files changed, 172 insertions(+), 25 deletions(-) diff --git a/code/nel/tools/pipeline/service/database_status.cpp b/code/nel/tools/pipeline/service/database_status.cpp index c64023a81..0fbd87a79 100644 --- a/code/nel/tools/pipeline/service/database_status.cpp +++ b/code/nel/tools/pipeline/service/database_status.cpp @@ -63,6 +63,23 @@ bool isInRootDirectoryFast(std::string &rootDirectoryName, std::string &rootDire return false; } +bool isInSheetsDirectoryFast(std::string &sheetDirectoryName, std::string &sheetDirectoryPath, const std::string &path) +{ + { + sheetDirectoryName = "WorkspaceDfnDirectory"; + CConfigFile::CVar &dir = NLNET::IService::getInstance()->ConfigFile.getVar(sheetDirectoryName); + sheetDirectoryPath = CPath::standardizePath(dir.asString(), true); + if (path.find(sheetDirectoryPath) == 0) return true; + } + { + sheetDirectoryName = "WorkspaceSheetDirectory"; + CConfigFile::CVar &dir = NLNET::IService::getInstance()->ConfigFile.getVar(sheetDirectoryName); + sheetDirectoryPath = CPath::standardizePath(dir.asString(), true); + if (path.find(sheetDirectoryPath) == 0) return true; + } + return false; +} + /// Input must be normalized path bool isInWorkspaceDirectoryFast(const std::string &path) { @@ -75,6 +92,12 @@ std::string dropRootDirectoryFast(const std::string &path, const std::string &ro return path.substr(rootDirectoryPath.length()); } +/// Input must be normalized path in sheets directory +std::string dropSheetDirectoryFast(const std::string &path, const std::string &sheetDirectoryPath) +{ + return path.substr(sheetDirectoryPath.length()); +} + /// Input must be normalized path in pipeline directory std::string dropWorkspaceDirectoryFast(const std::string &path) { @@ -99,15 +122,23 @@ std::string getMetaFilePath(const std::string &path, const std::string &dotSuffi { std::string rootDirectoryName; std::string rootDirectoryPath; - if (isInRootDirectoryFast(rootDirectoryName, rootDirectoryPath, stdPath)) + if (isInSheetsDirectoryFast(rootDirectoryName, rootDirectoryPath, stdPath)) { - std::string relPath = dropRootDirectoryFast(stdPath, rootDirectoryPath); - return g_WorkDir + PIPELINE_DIRECTORY_PREFIX_ROOT + NLMISC::toLower(rootDirectoryName) + PIPELINE_DATABASE_META_SUFFIX + "/" + relPath + dotSuffix; + std::string relPath = dropSheetDirectoryFast(stdPath, rootDirectoryPath); + return g_WorkDir + PIPELINE_DIRECTORY_PREFIX_SHEET + NLMISC::toLower(rootDirectoryName) + PIPELINE_DATABASE_META_SUFFIX + "/" + relPath + dotSuffix; } else { - nlerror("Path is not in database or pipeline (%s)", path.c_str()); - return path + dotSuffix; + if (isInRootDirectoryFast(rootDirectoryName, rootDirectoryPath, stdPath)) + { + std::string relPath = dropRootDirectoryFast(stdPath, rootDirectoryPath); + return g_WorkDir + PIPELINE_DIRECTORY_PREFIX_ROOT + NLMISC::toLower(rootDirectoryName) + PIPELINE_DATABASE_META_SUFFIX + "/" + relPath + dotSuffix; + } + else + { + nlerror("Path is not in database or pipeline (%s)", path.c_str()); + return path + dotSuffix; + } } } } diff --git a/code/nel/tools/pipeline/service/module_pipeline_master.cpp b/code/nel/tools/pipeline/service/module_pipeline_master.cpp index 211aa6443..02b8ef3bb 100644 --- a/code/nel/tools/pipeline/service/module_pipeline_master.cpp +++ b/code/nel/tools/pipeline/service/module_pipeline_master.cpp @@ -34,6 +34,7 @@ // NeL includes #include +#include // Project includes #include "info_flags.h" @@ -148,9 +149,30 @@ public: { CModuleBase::initModule(initInfo); CModulePipelineMasterSkel::init(this); + if (PIPELINE::tryDirectTask("MASTER_INIT_SHEETS")) + { + updateSheetsDatabaseStatus(CCallback(this, &CModulePipelineMaster::cbMasterInitSheets)); + } + else + { + nlerror("Cannot initialize master, the service is busy. This should never happen"); + } return true; } + void updateSheetsDatabaseStatus(const CCallback &callback) + { + std::vector sheetPaths; + // sheetPaths.push_back(NLNET::IService::getInstance()->ConfigFile.getVar("WorkspaceDfnDirectory").asString()); // not really necessary to check the dfn's + sheetPaths.push_back(NLNET::IService::getInstance()->ConfigFile.getVar("WorkspaceSheetDirectory").asString()); + g_DatabaseStatus->updateDatabaseStatus(callback, sheetPaths, false, true); + } + + void cbMasterInitSheets() + { + PIPELINE::endedDirectTask(); + } + virtual void onModuleUp(IModuleProxy *moduleProxy) { if (moduleProxy->getModuleClassName() == "ModulePipelineSlave") @@ -418,20 +440,7 @@ protected: if (PIPELINE::tryDirectTask("MASTER_RELOAD_SHEETS")) { - m_SlavesMutex.lock(); - - for (TSlaveMap::iterator it = m_Slaves.begin(), end = m_Slaves.end(); it != end; ++it) - { - CSlave *slave = it->second; - slave->SheetsOk = false; - slave->PluginsAvailable.clear(); - slave->Proxy.reloadSheets(this); - CInfoFlags::getInstance()->addFlag(PIPELINE_INFO_MASTER_RELOAD_SHEETS); - } - - m_SlavesMutex.unlock(); - - PIPELINE::endedDirectTask(); + updateSheetsDatabaseStatus(CCallback(this, &CModulePipelineMaster::cbMasterSheetReloadUpdate)); return true; } @@ -441,6 +450,24 @@ protected: return false; } } + + void cbMasterSheetReloadUpdate() + { + m_SlavesMutex.lock(); + + for (TSlaveMap::iterator it = m_Slaves.begin(), end = m_Slaves.end(); it != end; ++it) + { + CSlave *slave = it->second; + slave->SheetsOk = false; + slave->PluginsAvailable.clear(); + slave->Proxy.reloadSheets(this); + CInfoFlags::getInstance()->addFlag(PIPELINE_INFO_MASTER_RELOAD_SHEETS); + } + + m_SlavesMutex.unlock(); + + PIPELINE::endedDirectTask(); + } NLMISC_CLASS_COMMAND_DECL(build) { diff --git a/code/nel/tools/pipeline/service/module_pipeline_slave.cpp b/code/nel/tools/pipeline/service/module_pipeline_slave.cpp index 93713366e..eb82d51c6 100644 --- a/code/nel/tools/pipeline/service/module_pipeline_slave.cpp +++ b/code/nel/tools/pipeline/service/module_pipeline_slave.cpp @@ -110,6 +110,8 @@ public: { // TODO: AUTHENTICATE OR GATEWAY SECURITY? CModulePipelineMasterProxy master(sender); + if (!g_PipelineWorkspace->loadCRC32()) + nlerror("Failed sheets CRC32. Sheets were modified inbetween launching services. This causes newly loaded services to be out of sync. Not allowed"); sendMasterAvailablePlugins(&master); } @@ -144,6 +146,8 @@ public: if (PIPELINE::isServiceStateIdle()) { m_ReloadSheetsState = REQUEST_NONE; + if (!g_PipelineWorkspace->loadCRC32()) + nlerror("Failed sheets CRC32. Sheets were modified inbetween launching services. This causes newly loaded services to be out of sync. Not allowed"); sendMasterAvailablePlugins(m_Master); m_Master->slaveReloadedSheets(this); CInfoFlags::getInstance()->removeFlag(PIPELINE_INFO_SLAVE_RELOAD_SHEETS); diff --git a/code/nel/tools/pipeline/service/pipeline_project.cpp b/code/nel/tools/pipeline/service/pipeline_project.cpp index 5b34592e2..c072ce564 100644 --- a/code/nel/tools/pipeline/service/pipeline_project.cpp +++ b/code/nel/tools/pipeline/service/pipeline_project.cpp @@ -47,7 +47,7 @@ using namespace std; namespace PIPELINE { -CPipelineProject::CPipelineProject(CPipelineWorkspace *workspace, NLGEORGES::UForm *form) : m_Workspace(workspace), m_Form(form) +CPipelineProject::CPipelineProject(CPipelineWorkspace *workspace, NLGEORGES::UForm *form) : m_Workspace(workspace), m_Form(form), m_ChangedReference(0), m_FileSizeReference(0), m_CRC32(0) { } diff --git a/code/nel/tools/pipeline/service/pipeline_project.h b/code/nel/tools/pipeline/service/pipeline_project.h index 51666b627..d5aa3dda7 100644 --- a/code/nel/tools/pipeline/service/pipeline_project.h +++ b/code/nel/tools/pipeline/service/pipeline_project.h @@ -48,10 +48,15 @@ namespace PIPELINE { */ class CPipelineProject { + friend class PIPELINE::CPipelineWorkspace; + protected: CPipelineWorkspace *m_Workspace; NLMISC::CRefPtr m_Form; std::string m_TempDirectory; + uint32 m_ChangedReference; + uint32 m_FileSizeReference; + uint32 m_CRC32; public: CPipelineProject(CPipelineWorkspace *workspace, NLGEORGES::UForm *form); diff --git a/code/nel/tools/pipeline/service/pipeline_service.cpp b/code/nel/tools/pipeline/service/pipeline_service.cpp index 06deff3fc..510cc0c7e 100644 --- a/code/nel/tools/pipeline/service/pipeline_service.cpp +++ b/code/nel/tools/pipeline/service/pipeline_service.cpp @@ -354,8 +354,10 @@ namespace { void initSheets() { - std::string dfnDirectory = IService::getInstance()->ConfigFile.getVar("WorkspaceDfnDirectory").asString(); - std::string sheetDirectory = IService::getInstance()->ConfigFile.getVar("WorkspaceSheetDirectory").asString(); + std::string dfnDirectory = CPath::standardizePath(IService::getInstance()->ConfigFile.getVar("WorkspaceDfnDirectory").asString(), true); + IService::getInstance()->ConfigFile.getVar("WorkspaceDfnDirectory").setAsString(dfnDirectory); + std::string sheetDirectory = CPath::standardizePath(IService::getInstance()->ConfigFile.getVar("WorkspaceSheetDirectory").asString(), true); + IService::getInstance()->ConfigFile.getVar("WorkspaceSheetDirectory").setAsString(sheetDirectory); if (!CFile::isDirectory(dfnDirectory)) nlerror("'WorkspaceDfnDirectory' does not exist! (%s)", dfnDirectory.c_str()); nlinfo("Adding 'WorkspaceDfnDirectory' to search path (%s)", dfnDirectory.c_str()); diff --git a/code/nel/tools/pipeline/service/pipeline_service.h b/code/nel/tools/pipeline/service/pipeline_service.h index 7fc1cc237..a75896be3 100644 --- a/code/nel/tools/pipeline/service/pipeline_service.h +++ b/code/nel/tools/pipeline/service/pipeline_service.h @@ -59,6 +59,8 @@ extern std::string g_WorkDir; #define PIPELINE_DIRECTORY_UNKNOWN_PREFIX "unknown." #define PIPELINE_DIRECTORY_PREFIX_PROJECT "project." #define PIPELINE_DIRECTORY_PREFIX_ROOT "root." +#define PIPELINE_DIRECTORY_PREFIX_SHEET "pipeline." + #define PIPELINE_DIRECTORY_TEMP_SUFFIX ".temp" /// Unmacros a path, and standardizes it as well. diff --git a/code/nel/tools/pipeline/service/pipeline_workspace.cpp b/code/nel/tools/pipeline/service/pipeline_workspace.cpp index c35d493ab..3d7d3d0a4 100644 --- a/code/nel/tools/pipeline/service/pipeline_workspace.cpp +++ b/code/nel/tools/pipeline/service/pipeline_workspace.cpp @@ -39,6 +39,7 @@ #include "pipeline_project.h" #include "pipeline_service.h" #include "pipeline_interface_impl.h" +#include "database_status.h" using namespace std; // using namespace NLMISC; @@ -46,8 +47,14 @@ using namespace NLGEORGES; namespace PIPELINE { -CPipelineWorkspace::CPipelineWorkspace(NLGEORGES::UFormLoader *formLoader, const std::string &sheetName) : m_FormLoader(formLoader) +CPipelineWorkspace::CPipelineWorkspace(NLGEORGES::UFormLoader *formLoader, const std::string &sheetName) : m_FormLoader(formLoader), m_ChangedReference(0), m_FileSizeReference(0), m_CRC32(0) { + std::string fullSheetPath = NLMISC::CPath::lookup(sheetName); + { + m_ChangedReference = NLMISC::CFile::getFileModificationDate(fullSheetPath); + m_FileSizeReference = NLMISC::CFile::getFileSize(fullSheetPath); + } + m_Form = formLoader->loadForm(sheetName.c_str()); std::string description; m_Form->getRootNode().getValueByName(description, "Description"); @@ -91,7 +98,17 @@ CPipelineWorkspace::CPipelineWorkspace(NLGEORGES::UFormLoader *formLoader, const { std::string projectName = NLMISC::CFile::getFilenameWithoutExtension(projectSheet); if (m_Projects.find(projectName) == m_Projects.end()) - m_Projects[projectName] = new CPipelineProject(this, formLoader->loadForm(projectSheet.c_str())); + { + std::string fullProjectSheetPath = NLMISC::CPath::lookup(projectSheet); + uint32 changedReference = NLMISC::CFile::getFileModificationDate(fullProjectSheetPath); + uint32 fileSizeReference = NLMISC::CFile::getFileSize(fullProjectSheetPath); + CPipelineProject *project = new CPipelineProject(this, formLoader->loadForm(projectSheet.c_str())); + m_Projects[projectName] = project; + nlassert(changedReference == NLMISC::CFile::getFileModificationDate(fullProjectSheetPath)); + nlassert(fileSizeReference == NLMISC::CFile::getFileSize(fullProjectSheetPath)); + project->m_ChangedReference = changedReference; + project->m_FileSizeReference = fileSizeReference; + } else nlwarning("Project '%s' in '%s' already", projectSheet.c_str(), m_Form->getFilename().c_str()); } @@ -106,6 +123,11 @@ CPipelineWorkspace::CPipelineWorkspace(NLGEORGES::UFormLoader *formLoader, const nlwarning("Missing 'Projects' in '%s'", m_Form->getFilename().c_str()); } } + + { + nlassert(m_ChangedReference == NLMISC::CFile::getFileModificationDate(fullSheetPath)); + nlassert(m_FileSizeReference == NLMISC::CFile::getFileSize(fullSheetPath)); + } } CPipelineWorkspace::~CPipelineWorkspace() @@ -335,6 +357,54 @@ CPipelineProject *CPipelineWorkspace::getProject(const std::string &project) return it->second; } +bool CPipelineWorkspace::loadCRC32() +{ + bool ok = true; + { + std::string sheetPath = NLMISC::CPath::lookup(m_Form->getFilename()); + CFileStatus fileStatus; + if (g_DatabaseStatus->getFileStatus(fileStatus, sheetPath) + && fileStatus.LastChangedReference == m_ChangedReference + && fileStatus.LastFileSizeReference == m_FileSizeReference) + { + m_CRC32 = fileStatus.CRC32; + } + else + { + nlwarning("Workspace sheet '%s' not synchronized", sheetPath.c_str()); + ok = false; + } + } + for (std::map::iterator it = m_Projects.begin(), end = m_Projects.end(); it != end; ++it) + { + CPipelineProject *project = it->second; + { + std::string sheetPath = NLMISC::CPath::lookup(project->m_Form->getFilename()); + CFileStatus fileStatus; + if (g_DatabaseStatus->getFileStatus(fileStatus, sheetPath) + && fileStatus.LastChangedReference == project->m_ChangedReference + && fileStatus.LastFileSizeReference == project->m_FileSizeReference) + { + project->m_CRC32 = fileStatus.CRC32; + } + else + { + nlwarning("Workspace sheet '%s' not synchronized", sheetPath.c_str()); + ok = false; + } + } + } + if (!ok) + { + nlwarning("Workspace not synchronized"); + } + else + { + nldebug("Workspace CRC32 synchronized"); + } + return ok; +} + } /* namespace PIPELINE */ /* end of file */ diff --git a/code/nel/tools/pipeline/service/pipeline_workspace.h b/code/nel/tools/pipeline/service/pipeline_workspace.h index cb1069bcd..5d9da1be8 100644 --- a/code/nel/tools/pipeline/service/pipeline_workspace.h +++ b/code/nel/tools/pipeline/service/pipeline_workspace.h @@ -78,13 +78,16 @@ struct CProcessPluginInfo */ class CPipelineWorkspace { - friend class CPipelineProject; + friend class PIPELINE::CPipelineProject; protected: NLGEORGES::UFormLoader *m_FormLoader; NLMISC::CRefPtr m_Form; std::vector > m_Plugins; std::map m_Projects; + uint32 m_ChangedReference; + uint32 m_FileSizeReference; + uint32 m_CRC32; public: CPipelineWorkspace(NLGEORGES::UFormLoader *formLoader, const std::string &sheetName); @@ -100,6 +103,9 @@ public: /// Makes a list of the global id of all available plugins to this service void listAvailablePlugins(std::vector &result); + /// Loads the CRC32 of the sheets, false if something out of sync + bool loadCRC32(); + }; /* class CPipelineWorkspace */ } /* namespace PIPELINE */