From 0cd3a76a3ed1ff2f16a75898d06dd68b8ab6ad3a Mon Sep 17 00:00:00 2001 From: kaetemi Date: Sun, 4 Mar 2012 13:27:03 +0100 Subject: [PATCH] Added: #1440 Informational flags and other debug tools --HG-- branch : build_pipeline_v3 --- .../pipeline/service/database_status.cpp | 9 ++ .../tools/pipeline/service/database_status.h | 2 + .../example_cfg/pipeline_service_default.cfg | 7 +- .../example_cfg/pipeline_service_master.cfg | 5 + .../nel/tools/pipeline/service/info_flags.cpp | 108 +++++++++++++++++ code/nel/tools/pipeline/service/info_flags.h | 68 +++++++++++ .../service/module_pipeline_master.cpp | 49 +++++--- .../service/module_pipeline_master_itf.cpp | Bin 16762 -> 17152 bytes .../service/module_pipeline_master_itf.h | Bin 12316 -> 12424 bytes .../service/module_pipeline_master_itf.xml | 5 +- .../service/module_pipeline_slave.cpp | 37 +++++- .../pipeline/service/pipeline_service.cpp | 110 +++++++++++++++++- .../tools/pipeline/service/pipeline_service.h | 6 + 13 files changed, 378 insertions(+), 28 deletions(-) create mode 100644 code/nel/tools/pipeline/service/info_flags.cpp create mode 100644 code/nel/tools/pipeline/service/info_flags.h diff --git a/code/nel/tools/pipeline/service/database_status.cpp b/code/nel/tools/pipeline/service/database_status.cpp index f3ad5891a..6d4994667 100644 --- a/code/nel/tools/pipeline/service/database_status.cpp +++ b/code/nel/tools/pipeline/service/database_status.cpp @@ -157,6 +157,15 @@ bool CDatabaseStatus::getFileStatus(CFileStatus &fileStatus, const std::string & return seemsValid; } +bool CDatabaseStatus::getFileStatus(std::map &fileStatusMap, const std::vector &paths) +{ + for (std::vector::const_iterator it = paths.begin(), end = paths.end(); it != end; ++it) + { + + } + return false; +} + namespace { class CUpdateFileStatus : public IRunnable diff --git a/code/nel/tools/pipeline/service/database_status.h b/code/nel/tools/pipeline/service/database_status.h index 8d563214c..40b70ec9b 100644 --- a/code/nel/tools/pipeline/service/database_status.h +++ b/code/nel/tools/pipeline/service/database_status.h @@ -111,6 +111,8 @@ public: void updateDatabaseStatus(const CCallback &callback); /// Runs an update of the file status of given paths asynchronously. Warning: If g_IsExiting during callback then update is incomplete. Callback is always called when done (or failed). Do NOT use the wait parameter. Do NOT use recurse, please. Recurse directories beforehand. Paths may contain db and pl macros. void updateDatabaseStatus(const CCallback &callback, const std::vector &paths, bool wait = false, bool recurse = false); + /// Gets the last file statuses of given paths in a map. Directories are scanned for files, non recursively. Returns false if one of the statuses is bad (not updated; file changed inbetween). Considered as build error. + bool getFileStatus(std::map &fileStatusMap, const std::vector &paths); void getFileErrors(CFileErrors &fileErrors, const std::string &filePath, uint32 newerThan = 0) const; void addFileError(const std::string &filePath, const CFileError &fileError); diff --git a/code/nel/tools/pipeline/service/example_cfg/pipeline_service_default.cfg b/code/nel/tools/pipeline/service/example_cfg/pipeline_service_default.cfg index a0227733d..c36f1a190 100644 --- a/code/nel/tools/pipeline/service/example_cfg/pipeline_service_default.cfg +++ b/code/nel/tools/pipeline/service/example_cfg/pipeline_service_default.cfg @@ -30,6 +30,11 @@ DontUseNS = 1; NSHost = "localhost"; WindowStyle = "WIN"; -DisplayedVariables = { "Status|pipelineServiceState", "FileQueue|asyncFileQueueCount" }; +DisplayedVariables = +{ + "Status|pipelineServiceState", "FileQueue|asyncFileQueueCount", + "", "InfoFlags|infoFlags", + "", "@Reload Sheets|reloadSheets", "@Busy Test|busyTestState" , +}; DontUseAES = 1; diff --git a/code/nel/tools/pipeline/service/example_cfg/pipeline_service_master.cfg b/code/nel/tools/pipeline/service/example_cfg/pipeline_service_master.cfg index eadef870d..ff329f7d2 100644 --- a/code/nel/tools/pipeline/service/example_cfg/pipeline_service_master.cfg +++ b/code/nel/tools/pipeline/service/example_cfg/pipeline_service_master.cfg @@ -20,3 +20,8 @@ StartCommands += "moduleManager.createModule ModulePipelineSlave slave", "slave.plug gw", }; + +DisplayedVariables += +{ + "", "@Master Reload Sheets|master.reloadSheets", "@Status Update All|updateDatabaseStatus", +}; diff --git a/code/nel/tools/pipeline/service/info_flags.cpp b/code/nel/tools/pipeline/service/info_flags.cpp new file mode 100644 index 000000000..2fd6b668c --- /dev/null +++ b/code/nel/tools/pipeline/service/info_flags.cpp @@ -0,0 +1,108 @@ +/** + * \file info_flags.cpp + * \brief CInfoFlags + * \date 2012-03-04 10:46GMT + * \author Jan Boon (Kaetemi) + * CInfoFlags + */ + +/* + * Copyright (C) 2012 by authors + * + * This file is part of RYZOM CORE PIPELINE. + * RYZOM CORE PIPELINE is free software: you can redistribute it + * and/or modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, either version 2 of + * the License, or (at your option) any later version. + * + * RYZOM CORE PIPELINE is distributed in the hope that it will be + * useful, but WITHOUT ANY WARRANTY; without even the implied warranty + * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with RYZOM CORE PIPELINE; see the file COPYING. If not, see + * . + */ + +#include +#include "info_flags.h" + +// STL includes +#include + +// NeL includes +// #include + +// Project includes +#include "info_flags.h" + +using namespace std; +// using namespace NLMISC; + +NLMISC::CVariable InfoFlags("pipeline", "infoFlags", "READ ONLY", ""); + +namespace PIPELINE { + +CInfoFlags::CInfoFlags() +{ + updateInfoFlags(); +} + +CInfoFlags::~CInfoFlags() +{ + InfoFlags = ""; +} + +void CInfoFlags::addFlag(const std::string &flagName) +{ + nldebug("addFlag: %s", flagName.c_str()); + if (m_FlagMap.find(flagName) != m_FlagMap.end()) + { + ++m_FlagMap[flagName]; + } + else + { + m_FlagMap[flagName] = 1; + } + updateInfoFlags(); +} + +void CInfoFlags::removeFlag(const std::string &flagName) +{ + nldebug("removeFlag: %s", flagName.c_str()); + std::map::iterator it = m_FlagMap.find(flagName); + if (it != m_FlagMap.end()) + { + if (it->second == 1) + m_FlagMap.erase(it); + else + --it->second; + } + updateInfoFlags(); +} + +void CInfoFlags::updateInfoFlags() +{ + if (m_FlagMap.empty()) + { + InfoFlags = ""; + } + else + { + std::stringstream ss; + for (std::map::iterator it = m_FlagMap.begin(), end = m_FlagMap.end(); it != end; ++it) + { + ss << it->first; + if (it ->second > 1) + ss << " (" << it->second << ")"; + ss << ", "; + } + std::string s = ss.str(); + InfoFlags = s.substr(0, s.size() - 2); + } +} + +} /* namespace PIPELINE */ + +/* end of file */ diff --git a/code/nel/tools/pipeline/service/info_flags.h b/code/nel/tools/pipeline/service/info_flags.h new file mode 100644 index 000000000..0eac6390d --- /dev/null +++ b/code/nel/tools/pipeline/service/info_flags.h @@ -0,0 +1,68 @@ +/** + * \file info_flags.h + * \brief CInfoFlags + * \date 2012-03-04 10:46GMT + * \author Jan Boon (Kaetemi) + * CInfoFlags + */ + +/* + * Copyright (C) 2012 by authors + * + * This file is part of RYZOM CORE PIPELINE. + * RYZOM CORE PIPELINE is free software: you can redistribute it + * and/or modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, either version 2 of + * the License, or (at your option) any later version. + * + * RYZOM CORE PIPELINE is distributed in the hope that it will be + * useful, but WITHOUT ANY WARRANTY; without even the implied warranty + * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with RYZOM CORE PIPELINE; see the file COPYING. If not, see + * . + */ + +#ifndef PIPELINE_INFO_FLAGS_H +#define PIPELINE_INFO_FLAGS_H +#include + +// STL includes + +// NeL includes +#include +#include + +// Project includes + +namespace PIPELINE { + +/** + * \brief CInfoFlags + * \date 2012-03-04 10:46GMT + * \author Jan Boon (Kaetemi) + * CInfoFlags + */ + class CInfoFlags : public NLMISC::CManualSingleton +{ +protected: + std::map m_FlagMap; +public: + CInfoFlags(); + virtual ~CInfoFlags(); + + void addFlag(const std::string &flagName); + void removeFlag(const std::string &flagName); + +private: + void updateInfoFlags(); + +}; /* class CInfoFlags */ + +} /* namespace PIPELINE */ + +#endif /* #ifndef PIPELINE_INFO_FLAGS_H */ + +/* end of file */ diff --git a/code/nel/tools/pipeline/service/module_pipeline_master.cpp b/code/nel/tools/pipeline/service/module_pipeline_master.cpp index 4e9170b67..c5583d56e 100644 --- a/code/nel/tools/pipeline/service/module_pipeline_master.cpp +++ b/code/nel/tools/pipeline/service/module_pipeline_master.cpp @@ -36,6 +36,7 @@ #include // Project includes +#include "info_flags.h" #include "module_pipeline_slave_itf.h" #include "pipeline_service.h" #include "database_status.h" @@ -46,6 +47,8 @@ using namespace NLNET; namespace PIPELINE { +#define PIPELINE_INFO_MASTER_RELOAD_SHEETS "MASTER_RELOAD_SHEETS" + /** * \brief CModulePipelineMaster * \date 2012-03-03 16:26GMT @@ -70,6 +73,14 @@ class CModulePipelineMaster : uint32 ActiveTaskId; bool SheetsOk; + ~CSlave() + { + if (!SheetsOk) + { + CInfoFlags::getInstance()->removeFlag(PIPELINE_INFO_MASTER_RELOAD_SHEETS); + } + } + void cbUpdateDatabaseStatus() { Proxy.masterUpdatedDatabaseStatus(Master); @@ -159,11 +170,11 @@ public: // if state build, iterate trough all slaves to see if any is free, and check if there's any waiting tasks } - virtual void slaveFinishedBuildTask(NLNET::IModuleProxy *sender, uint32 taskId) + virtual void slaveFinishedBuildTask(NLNET::IModuleProxy *sender, uint32 taskId, uint8 errorLevel) { // TODO } - + virtual void slaveRefusedBuildTask(NLNET::IModuleProxy *sender, uint32 taskId) { // TODO @@ -173,6 +184,7 @@ public: { CSlave *slave = m_Slaves[sender]; slave->SheetsOk = true; + CInfoFlags::getInstance()->removeFlag(PIPELINE_INFO_MASTER_RELOAD_SHEETS); } virtual void vectorPushString(NLNET::IModuleProxy *sender, const std::string &str) @@ -180,7 +192,7 @@ public: CSlave *slave = m_Slaves[sender]; slave->Vector.push_back(str); } - + virtual void updateDatabaseStatusByVector(NLNET::IModuleProxy *sender) { CSlave *slave = m_Slaves[sender]; @@ -197,18 +209,29 @@ protected: { if (args.size() != 0) return false; - m_SlavesMutex.lock(); - - for (TSlaveMap::iterator it = m_Slaves.begin(), end = m_Slaves.end(); it != end; ++it) + if (PIPELINE::tryDirectTask("MASTER_RELOAD_SHEETS")) { - CSlave *slave = it->second; - slave->SheetsOk = false; - slave->Proxy.reloadSheets(this); - } - - m_SlavesMutex.unlock(); + m_SlavesMutex.lock(); + + for (TSlaveMap::iterator it = m_Slaves.begin(), end = m_Slaves.end(); it != end; ++it) + { + CSlave *slave = it->second; + slave->SheetsOk = false; + slave->Proxy.reloadSheets(this); + CInfoFlags::getInstance()->addFlag(PIPELINE_INFO_MASTER_RELOAD_SHEETS); + } + + m_SlavesMutex.unlock(); - return true; + PIPELINE::endedDirectTask(); + + return true; + } + else + { + log.displayNL("Busy"); + return false; + } } }; /* class CModulePipelineMaster */ diff --git a/code/nel/tools/pipeline/service/module_pipeline_master_itf.cpp b/code/nel/tools/pipeline/service/module_pipeline_master_itf.cpp index 3e59a5e446196d106976dcad6044528cbcee0f47..b0ebfc06b3983dd2d5d284b9e57217d756e849a9 100644 GIT binary patch delta 460 zcmey>#MscrxIv4J-G#xQ!H>ap@_lZ{&3SC|7D?HS) z?c66jn2Ss785}2n;pz&F;1>i*O;7OAu{=bgV*Li H)^)4^b=q_K delta 220 zcmZo@WBk>`xIv3;G832IWIkb?%`4b47}?zz92qivw)Z$T>EBk@!jl`Pgx0V<}*@b zocu?V3#fb(NaT;74%=sqYGnAWrL4sxS8#yUtU3@?V3Y$Ar iwS<78F3kQ6-jfwAqBoaWw=qJs27x5tj)q8fu>%0QB1EqM diff --git a/code/nel/tools/pipeline/service/module_pipeline_master_itf.h b/code/nel/tools/pipeline/service/module_pipeline_master_itf.h index ddb7a4a3897f72d920ab7072ef413c2e2c41083f..b3d159e5f64d7bd427533ebe30c75a08701b69a7 100644 GIT binary patch delta 166 zcmbP}(2=-dp186Og91Y-LncEWLkWWgke$j<1cdoO>;q(%0ddacjgr=rWyHBAR|qIf s*3%T2EFi42*-j#jmnhA)YQ~dm1Z2P(6?phI3#b(_6Qgl+qvmWj0G#M0$N&HU delta 51 zcmV-30L=f0VVq#Fo)?pR7)X;ICOVVA5FV4RB^tAQ7)%Y5jw=e2kR~Xz9ww{ - + + - + diff --git a/code/nel/tools/pipeline/service/module_pipeline_slave.cpp b/code/nel/tools/pipeline/service/module_pipeline_slave.cpp index 562960171..11543c544 100644 --- a/code/nel/tools/pipeline/service/module_pipeline_slave.cpp +++ b/code/nel/tools/pipeline/service/module_pipeline_slave.cpp @@ -34,6 +34,7 @@ #include // Project includes +#include "info_flags.h" #include "module_pipeline_master_itf.h" #include "pipeline_service.h" #include "../plugin_library/process_info.h" @@ -46,6 +47,15 @@ using namespace NLNET; namespace PIPELINE { +#define PIPELINE_INFO_SLAVE_RELOAD_SHEETS "SLAVE_RELOAD_SHEETS" + +enum TRequestState +{ + REQUEST_NONE, + REQUEST_MADE, + REQUEST_WORKING, +}; + /** * \brief CModulePipelineSlave * \date 2012-03-03 16:26GMT @@ -59,10 +69,10 @@ class CModulePipelineSlave : public: CModulePipelineMasterProxy *m_Master; bool m_TestCommand; - bool m_RequestedReloadSheets; + TRequestState m_ReloadSheetsState; public: - CModulePipelineSlave() : m_Master(NULL), m_TestCommand(false), m_RequestedReloadSheets(false) + CModulePipelineSlave() : m_Master(NULL), m_TestCommand(false), m_ReloadSheetsState(REQUEST_NONE) { } @@ -106,8 +116,22 @@ public: virtual void onModuleUpdate() { - if (m_RequestedReloadSheets) - m_RequestedReloadSheets = !PIPELINE::reloadSheets(); + if (m_ReloadSheetsState == REQUEST_MADE) + { + if (PIPELINE::reloadSheets()) + { + m_ReloadSheetsState = REQUEST_WORKING; + } + } + else if (m_ReloadSheetsState == REQUEST_WORKING) + { + if (PIPELINE::isServiceStateIdle()) + { + m_ReloadSheetsState = REQUEST_NONE; + m_Master->slaveReloadedSheets(this); + CInfoFlags::getInstance()->removeFlag(PIPELINE_INFO_SLAVE_RELOAD_SHEETS); + } + } } virtual void startBuildTask(NLNET::IModuleProxy *sender, uint32 taskId, const std::string &projectName, const std::string &processHandler) @@ -131,8 +155,9 @@ public: virtual void reloadSheets(NLNET::IModuleProxy *sender) { - if (!PIPELINE::reloadSheets()) - m_RequestedReloadSheets = true; + CInfoFlags::getInstance()->addFlag(PIPELINE_INFO_SLAVE_RELOAD_SHEETS); + if (PIPELINE::reloadSheets()) m_ReloadSheetsState = REQUEST_WORKING; + else m_ReloadSheetsState = REQUEST_MADE; } protected: diff --git a/code/nel/tools/pipeline/service/pipeline_service.cpp b/code/nel/tools/pipeline/service/pipeline_service.cpp index 144658241..15c8777fd 100644 --- a/code/nel/tools/pipeline/service/pipeline_service.cpp +++ b/code/nel/tools/pipeline/service/pipeline_service.cpp @@ -46,8 +46,10 @@ #include #include #include +#include // Project includes +#include "info_flags.h" #include "pipeline_workspace.h" #include "pipeline_project.h" #include "database_status.h" @@ -110,9 +112,12 @@ enum EState STATE_RELOAD_SHEETS, STATE_DATABASE_STATUS, STATE_RUNNABLE_TASK, + STATE_BUSY_TEST, + STATE_DIRECT_CODE, }; /// Data +CInfoFlags *s_InfoFlags = NULL; CTaskManager *s_TaskManager = NULL; CPipelineInterfaceImpl *s_PipelineInterfaceImpl = NULL; CPipelineProcessImpl *s_PipelineProcessImpl = NULL; @@ -158,6 +163,11 @@ bool tryStateTask(EState state, IRunnable *task) } /* anonymous namespace */ +bool isServiceStateIdle() +{ + return (s_State == STATE_IDLE); +} + bool tryRunnableTask(std::string stateName, IRunnable *task) { // copy paste from above. @@ -173,22 +183,54 @@ bool tryRunnableTask(std::string stateName, IRunnable *task) s_StateMutex.leave(); if (!result) return false; - nlassert(s_State != STATE_IDLE); + nlassert(s_State == STATE_RUNNABLE_TASK); s_TaskManager->addTask(task); return true; } -void endedRunnableTask() +namespace { + +void endedRunnableTask(EState state) { - nlassert(s_State != STATE_IDLE); + nlassert(s_State == state); s_StateMutex.enter(); s_State = STATE_IDLE; s_StateMutex.leave(); } +} /* anonymous namespace */ + +void endedRunnableTask() +{ + endedRunnableTask(STATE_RUNNABLE_TASK); +} + +bool tryDirectTask(const std::string &stateName) +{ + bool result = false; + s_StateMutex.enter(); + result = (s_State == STATE_IDLE); + if (result) + { + s_State = STATE_DIRECT_CODE; + s_StateRunnableTaskName = stateName; + } + s_StateMutex.leave(); + if (!result) return false; + + nlassert(s_State == STATE_DIRECT_CODE); + + return true; +} + +void endedDirectTask() +{ + endedRunnableTask(STATE_DIRECT_CODE); +} + // ****************************************************************** namespace { @@ -230,7 +272,7 @@ class CReloadSheets : public IRunnable releaseSheets(); initSheets(); - endedRunnableTask(); + endedRunnableTask(STATE_RELOAD_SHEETS); } }; CReloadSheets s_ReloadSheets; @@ -253,7 +295,7 @@ class CUpdateDatabaseStatus : public IRunnable void databaseStatusUpdated() { - endedRunnableTask(); + endedRunnableTask(STATE_DATABASE_STATUS); } virtual void run() @@ -270,6 +312,28 @@ bool updateDatabaseStatus() // ****************************************************************** +// ****************************************************************** + +class CBusyTestStatus : public IRunnable +{ + virtual void getName(std::string &result) const + { result = "CBusyTestStatus"; } + + virtual void run() + { + nlSleep(20000); + endedRunnableTask(STATE_BUSY_TEST); + } +}; +CBusyTestStatus s_BusyTestStatus; + +bool busyTestStatus() +{ + return tryStateTask(STATE_BUSY_TEST, &s_BusyTestStatus); +} + +// ****************************************************************** + /** * \brief CPipelineService * \date 2012-02-18 17:25GMT @@ -309,6 +373,10 @@ public: /// Initializes the service (must be called before the first call to update()) virtual void init() { + s_InfoFlags = new CInfoFlags(); + + s_InfoFlags->addFlag("INIT"); + g_DatabaseDirectory = CPath::standardizePath(ConfigFile.getVar("DatabaseDirectory").asString(), true); if (!CFile::isDirectory(g_DatabaseDirectory)) nlwarning("'DatabaseDirectory' does not exist! (%s)", g_DatabaseDirectory.c_str()); ConfigFile.getVar("DatabaseDirectory").setAsString(g_DatabaseDirectory); @@ -337,6 +405,8 @@ public: } else delete library; } + + s_InfoFlags->removeFlag("INIT"); } /// This function is called every "frame" (you must call init() before). It returns false if the service is stopped. @@ -350,6 +420,10 @@ public: { g_IsExiting = true; + s_InfoFlags->addFlag("RELEASE"); + + NLNET::IModuleManager::releaseInstance(); + while (NLMISC::CAsyncFileManager::getInstance().getNumWaitingTasks() > 0) { nlSleep(10); @@ -363,6 +437,8 @@ public: } s_LoadedLibraries.clear(); + CClassRegistry::release(); + delete s_PipelineProcessImpl; s_PipelineProcessImpl = NULL; @@ -376,6 +452,11 @@ public: delete s_TaskManager; s_TaskManager = NULL; + + s_InfoFlags->removeFlag("RELEASE"); + + delete s_InfoFlags; + s_InfoFlags = NULL; } }; /* class CPipelineService */ @@ -401,7 +482,13 @@ NLMISC_DYNVARIABLE(std::string, pipelineServiceState, "State of the pipeline ser *pointer = "DATABASE_STATUS"; break; case PIPELINE::STATE_RUNNABLE_TASK: - *pointer = PIPELINE::s_StateRunnableTaskName; + *pointer = "RT: " + PIPELINE::s_StateRunnableTaskName; + break; + case PIPELINE::STATE_DIRECT_CODE: + *pointer = "DC: " + PIPELINE::s_StateRunnableTaskName; + break; + case PIPELINE::STATE_BUSY_TEST: + *pointer = "BUSY_TEST"; break; } } @@ -438,6 +525,17 @@ NLMISC_COMMAND(updateDatabaseStatus, "Updates the entire database status. This a return true; } +NLMISC_COMMAND(busyTestState, "Keeps the service busy for twenty seconds.", "") +{ + if(args.size() != 0) return false; + if (!PIPELINE::busyTestStatus()) + { + log.displayNL("Already busy"); + return false; + } + return true; +} + NLMISC_COMMAND(dumpTaskManager, "Dumps the task manager.", "") { if(args.size() != 0) return false; diff --git a/code/nel/tools/pipeline/service/pipeline_service.h b/code/nel/tools/pipeline/service/pipeline_service.h index 3f3812c45..30207a610 100644 --- a/code/nel/tools/pipeline/service/pipeline_service.h +++ b/code/nel/tools/pipeline/service/pipeline_service.h @@ -70,6 +70,12 @@ std::string macroPath(const std::string &path); bool tryRunnableTask(std::string stateName, NLMISC::IRunnable *task); void endedRunnableTask(); +bool tryDirectTask(const std::string &stateName); +void endedDirectTask(); + +/// Only use for informational purposes, service may not be idle after calling this. Not for thread safe usage. +bool isServiceStateIdle(); + bool reloadSheets(); extern NLGEORGES::UFormLoader *g_FormLoader;