diff --git a/code/nel/tools/pipeline/service/metadata_storage.h b/code/nel/tools/pipeline/service/metadata_storage.h index e9fed90f2..1f243e0fc 100644 --- a/code/nel/tools/pipeline/service/metadata_storage.h +++ b/code/nel/tools/pipeline/service/metadata_storage.h @@ -113,7 +113,7 @@ public: void serial(NLMISC::IStream &stream) throw (NLMISC::EStream); }; std::vector Dependencies; - std::vector RuntimeDependencies; // informational + std::vector RuntimeDependencies; // informational (TODO) void serial(NLMISC::IStream &stream) throw (NLMISC::EStream); }; diff --git a/code/nel/tools/pipeline/service/module_pipeline_slave.cpp b/code/nel/tools/pipeline/service/module_pipeline_slave.cpp index 797bbc07c..9b027d568 100644 --- a/code/nel/tools/pipeline/service/module_pipeline_slave.cpp +++ b/code/nel/tools/pipeline/service/module_pipeline_slave.cpp @@ -102,51 +102,21 @@ public: NLMISC::CSynchronized m_StatusUpdateMasterDone; NLMISC::CSynchronized m_StatusUpdateSlaveDone; - CPipelineProject *m_ActiveProject; - CPipelineProcessImpl *m_ActiveProcess; // TODO: Maybe it would be easier to go directly to CPipelineProject from the plugin and provide an interface therefore. - CProcessPluginInfo m_ActivePlugin; - bool m_AbortRequested; + CPipelineProject *m_ActiveProject; + CPipelineProcessImpl *m_ActiveProcess; // TODO: Maybe it would be easier to go directly to CPipelineProject from the plugin and provide an interface therefore. + std::vector m_DependentDirectories; std::vector m_DependentFiles; - CProcessResult m_ResultPreviousSuccess; - std::map m_FileStatusInputCache; - std::map m_FileStatusOutputCache; - - /// List of dependencies (files) that were removed, or never existed (in which case remove time is 0). - std::map m_FileRemoveInputCache; - - /// Result of current process - CProcessResult m_ResultCurrent; - - /// Result of the last subtask (usually in the taskmanager), check this after the task has completed to make sure all went fine. - TProcessResult m_SubTaskResult; - std::string m_SubTaskErrorMessage; - - - std::set m_ListInputAdded; - std::set m_ListInputChanged; - std::set m_ListInputRemoved; - - std::set m_ListOutputChanged; - std::set m_ListOutputChangedNG; // after .depend check, found that dependencies changed, so not good - std::set m_ListOutputChangedOK; // idem but dependencies did not change, so ok - std::set m_ListOutputRemoved; // changed_ng and removed end up being the same, it needs to be rebuilt ;) - - std::set m_ListDependentDirectories; - std::set m_ListDependentFiles; - public: - CModulePipelineSlave() : m_Master(NULL), m_TestCommand(false), m_ReloadSheetsState(REQUEST_NONE), m_BuildReadyState(false), m_SlaveTaskState(IDLE_WAIT_MASTER), m_TaskManager(NULL), m_StatusUpdateMasterDone("StatusUpdateMasterDone"), m_StatusUpdateSlaveDone("StatusUpdateSlaveDone"), m_ActiveProject(NULL), m_ActiveProcess(NULL), m_AbortRequested(false), m_SubTaskResult(FINISH_NOT) + CModulePipelineSlave() : m_Master(NULL), m_TestCommand(false), m_ReloadSheetsState(REQUEST_NONE), m_BuildReadyState(false), m_SlaveTaskState(IDLE_WAIT_MASTER), m_TaskManager(NULL), m_StatusUpdateMasterDone("StatusUpdateMasterDone"), m_StatusUpdateSlaveDone("StatusUpdateSlaveDone"), m_ActiveProject(NULL), m_ActiveProcess(NULL), m_AbortRequested(false) { NLMISC::CSynchronized::CAccessor(&m_StatusUpdateMasterDone).value() = false; NLMISC::CSynchronized::CAccessor(&m_StatusUpdateSlaveDone).value() = false; m_TaskManager = new NLMISC::CTaskManager(); - m_ResultPreviousSuccess.clear(); - m_ResultCurrent.clear(); } virtual ~CModulePipelineSlave() @@ -310,10 +280,10 @@ public: m_SlaveTaskState = SOMEWHERE_INBETWEEN; CInfoFlags::getInstance()->removeFlag(PIPELINE_INFO_GET_REMOVE); - if (m_SubTaskResult != FINISH_SUCCESS) + if (m_ActiveProcess->m_SubTaskResult != FINISH_SUCCESS) { // Bye bye. - finishedTask(m_SubTaskResult, m_SubTaskErrorMessage); + finishedTask(m_ActiveProcess->m_SubTaskResult, m_ActiveProcess->m_SubTaskErrorMessage); break; } @@ -321,7 +291,7 @@ public: buildListsOfFiles(); // Set the actual start time of the actual build - m_ResultCurrent.BuildStart = NLMISC::CTime::getSecondsSince1970(); + m_ActiveProcess->m_ResultCurrent.BuildStart = NLMISC::CTime::getSecondsSince1970(); // TODO *************************************************************************** *************** **** BUILD beginTheBuildThing(); @@ -343,7 +313,7 @@ public: void cbFile(const std::string &filePath, const CFileStatus &fileStatus, bool success) { if (success) - m_Slave->m_FileStatusOutputCache[filePath] = fileStatus; + m_Slave->m_ActiveProcess->m_FileStatusOutputCache[filePath] = fileStatus; } void cbDone() @@ -358,7 +328,7 @@ public: g_DatabaseStatus->updateDatabaseStatus( CCallback(this, &CStatusUpdateSlaveTask::cbDone), TFileStatusCallback(this, &CStatusUpdateSlaveTask::cbFile), - m_Slave->m_ResultPreviousSuccess.MacroPaths, true, false); + m_Slave->m_ActiveProcess->m_ResultPreviousSuccess.MacroPaths, true, false); // Mark as done { @@ -384,14 +354,13 @@ public: nlassert(NLMISC::CSynchronized::CAccessor(&m_StatusUpdateMasterDone).value() == false); nlassert(NLMISC::CSynchronized::CAccessor(&m_StatusUpdateSlaveDone).value() == false); } - nlassert(m_ResultCurrent.BuildStart == 0); - nlassert(m_ResultPreviousSuccess.BuildStart == 0); + nlassert(m_ActiveProcess); // Set start time (not necessary, it will be set again later, when the actual build starts) - m_ResultCurrent.BuildStart = NLMISC::CTime::getSecondsSince1970(); + m_ActiveProcess->m_ResultCurrent.BuildStart = NLMISC::CTime::getSecondsSince1970(); // Read the previous process result - CMetadataStorage::readProcessResult(m_ResultPreviousSuccess, CMetadataStorage::getResultPath(m_ActiveProject->getName(), m_ActivePlugin.Handler)); + CMetadataStorage::readProcessResult(m_ActiveProcess->m_ResultPreviousSuccess, CMetadataStorage::getResultPath(m_ActiveProject->getName(), m_ActiveProcess->m_ActivePlugin.Handler)); // Start the client update CInfoFlags::getInstance()->addFlag(PIPELINE_INFO_STATUS_UPDATE_SLAVE); @@ -400,11 +369,11 @@ public: // Start the master update CInfoFlags::getInstance()->addFlag(PIPELINE_INFO_STATUS_UPDATE_MASTER); //std::vector result; - switch (m_ActivePlugin.InfoType) + switch (m_ActiveProcess->m_ActivePlugin.InfoType) { case PIPELINE::PLUGIN_REGISTERED_CLASS: { - PIPELINE::IProcessInfo *processInfo = static_cast(NLMISC::CClassRegistry::create(m_ActivePlugin.Info)); + PIPELINE::IProcessInfo *processInfo = static_cast(NLMISC::CClassRegistry::create(m_ActiveProcess->m_ActivePlugin.Info)); processInfo->setPipelineProcess(m_ActiveProcess); m_DependentDirectories.clear(); processInfo->getDependentDirectories(m_DependentDirectories); @@ -428,7 +397,7 @@ public: void beginGetRemove() { m_SlaveTaskState = GET_REMOVE; - m_SubTaskResult = FINISH_NOT; + m_ActiveProcess->m_SubTaskResult = FINISH_NOT; CInfoFlags::getInstance()->addFlag(PIPELINE_INFO_GET_REMOVE); m_RemovedTaskDone = false; m_TaskManager->addTask(new CGetRemovedTask(this)); @@ -443,19 +412,19 @@ public: { breakable { - if (!g_DatabaseStatus->getRemoved(m_Slave->m_FileRemoveInputCache, m_Slave->m_DependentDirectories)) + if (!g_DatabaseStatus->getRemoved(m_Slave->m_ActiveProcess->m_FileRemoveInputCache, m_Slave->m_DependentDirectories)) { - m_Slave->m_SubTaskResult = FINISH_ERROR; - m_Slave->m_SubTaskErrorMessage = "Metadata for removed files in directories not sane"; + m_Slave->m_ActiveProcess->m_SubTaskResult = FINISH_ERROR; + m_Slave->m_ActiveProcess->m_SubTaskErrorMessage = "Metadata for removed files in directories not sane"; break; } - if (!g_DatabaseStatus->getRemoved(m_Slave->m_FileRemoveInputCache, m_Slave->m_DependentFiles)) + if (!g_DatabaseStatus->getRemoved(m_Slave->m_ActiveProcess->m_FileRemoveInputCache, m_Slave->m_DependentFiles)) { - m_Slave->m_SubTaskResult = FINISH_ERROR; - m_Slave->m_SubTaskErrorMessage = "Metadata for removed files not sane"; + m_Slave->m_ActiveProcess->m_SubTaskResult = FINISH_ERROR; + m_Slave->m_ActiveProcess->m_SubTaskErrorMessage = "Metadata for removed files not sane"; break; } - m_Slave->m_SubTaskResult = FINISH_SUCCESS; + m_Slave->m_ActiveProcess->m_SubTaskResult = FINISH_SUCCESS; } m_Slave->m_RemovedTaskDone = true; } @@ -467,70 +436,63 @@ public: void buildListsOfFiles() { - for (std::map::const_iterator it = m_FileStatusInputCache.begin(), end = m_FileStatusInputCache.end(); it != end; ++it) + for (std::map::const_iterator it = m_ActiveProcess->m_FileStatusInputCache.begin(), end = m_ActiveProcess->m_FileStatusInputCache.end(); it != end; ++it) { - if (it->second.FirstSeen > m_ResultPreviousSuccess.BuildStart) + if (it->second.FirstSeen > m_ActiveProcess->m_ResultPreviousSuccess.BuildStart) { - m_ListInputAdded.insert(it->first); + m_ActiveProcess->m_ListInputAdded.insert(it->first); } - else if (it->second.LastUpdate > m_ResultPreviousSuccess.BuildStart) + else if (it->second.LastUpdate > m_ActiveProcess->m_ResultPreviousSuccess.BuildStart) { - m_ListInputChanged.insert(it->first); + m_ActiveProcess->m_ListInputChanged.insert(it->first); } } - for (std::map::const_iterator it = m_FileRemoveInputCache.begin(), end = m_FileRemoveInputCache.end(); it != end; ++it) + for (std::map::const_iterator it = m_ActiveProcess->m_FileRemoveInputCache.begin(), end = m_ActiveProcess->m_FileRemoveInputCache.end(); it != end; ++it) { - if (it->second.Lost > m_ResultPreviousSuccess.BuildStart) // or >= ? + if (it->second.Lost > m_ActiveProcess->m_ResultPreviousSuccess.BuildStart) // or >= ? { - m_ListInputRemoved.insert(it->first); + m_ActiveProcess->m_ListInputRemoved.insert(it->first); } } - for (std::map::const_iterator it = m_FileStatusOutputCache.begin(), end = m_FileStatusOutputCache.end(); it != end; ++it) + for (std::map::const_iterator it = m_ActiveProcess->m_FileStatusOutputCache.begin(), end = m_ActiveProcess->m_FileStatusOutputCache.end(); it != end; ++it) { - if (it->second.FirstSeen > m_ResultPreviousSuccess.BuildStart) + if (it->second.FirstSeen > m_ActiveProcess->m_ResultPreviousSuccess.BuildStart) { - m_ListOutputChanged.insert(it->first); + m_ActiveProcess->m_ListOutputChanged.insert(it->first); } } //for (std::vector::const_iterator it = m_ResultPreviousSuccess.MacroPaths.begin(), end = m_ResultPreviousSuccess.MacroPaths.end(); it != end; ++it) - for (uint i = 0; i < m_ResultPreviousSuccess.MacroPaths.size(); ++i) + for (uint i = 0; i < m_ActiveProcess->m_ResultPreviousSuccess.MacroPaths.size(); ++i) { - std::string filePath = unMacroPath(m_ResultPreviousSuccess.MacroPaths[i]); - std::map::const_iterator statusIt = m_FileStatusOutputCache.find(filePath); - if (statusIt == m_FileStatusOutputCache.end()) + std::string filePath = unMacroPath(m_ActiveProcess->m_ResultPreviousSuccess.MacroPaths[i]); + std::map::const_iterator statusIt = m_ActiveProcess->m_FileStatusOutputCache.find(filePath); + if (statusIt == m_ActiveProcess->m_FileStatusOutputCache.end()) { nlwarning("Spotted remove by not being in the cache"); - m_ListOutputRemoved.insert(filePath); + m_ActiveProcess->m_ListOutputRemoved.insert(filePath); } - else if (m_ResultPreviousSuccess.FileResults[i].CRC32 == 0) + else if (m_ActiveProcess->m_ResultPreviousSuccess.FileResults[i].CRC32 == 0) { nlwarning("Spotted remove by dummy CRC32 value"); // not sure which of these two .. dunno if non-existant files were in the cache // keep these codes anyway - m_ListOutputRemoved.insert(filePath); + m_ActiveProcess->m_ListOutputRemoved.insert(filePath); } else { const CFileStatus &status = statusIt->second; - if (status.CRC32 != m_ResultPreviousSuccess.FileResults[i].CRC32) + if (status.CRC32 != m_ActiveProcess->m_ResultPreviousSuccess.FileResults[i].CRC32) { nlwarning("Output file was illegally modified, probably by a previous build run that failed"); - m_ListOutputChanged.insert(filePath); - // !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - // (in the needbuild(something) function) - // TODO: IF THE OUTPUT IS ILLEGALLY MODIFIED (always the case in m_ListOutputChanged) - // USE THE .DEPEND FILE TO COMPARE THE CRC32 OF THE CURRENT STATUS WITH THE .DEPEND LOGGED CRC32 (this is the one from the failed build) - // AND ALSO COMPARE THE CRC32 OF ALL DEPENDENCY FILES WITH THE CACHED STATUS CRC32 OF THESE INPUT FILES - // IF ANY IS DIFFERENT, IT NEEDS A REBUILD - // !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + m_ActiveProcess->m_ListOutputChanged.insert(filePath); } } } for (std::vector::iterator it = m_DependentDirectories.begin(), end = m_DependentDirectories.end(); it != end; ++it) - m_ListDependentDirectories.insert(*it); + m_ActiveProcess->m_ListDependentDirectories.insert(*it); m_DependentDirectories.clear(); for (std::vector::iterator it = m_DependentFiles.begin(), end = m_DependentFiles.end(); it != end; ++it) - m_ListDependentFiles.insert(*it); + m_ActiveProcess->m_ListDependentFiles.insert(*it); m_DependentFiles.clear(); - m_SubTaskResult = FINISH_SUCCESS; + m_ActiveProcess->m_SubTaskResult = FINISH_SUCCESS; } /////////////////////////////////////////////////////////////////// @@ -546,463 +508,7 @@ public: /////////////////////////////////////////////////////////////////// /////////////////////////////////////////////////////////////////// - bool isFileDependency(const std::string &path) - { - if (m_ListDependentFiles.find(path) == m_ListDependentFiles.end()) // check if it's directly in the dependent files - { - // Not found! - std::string pathParent = path.substr(0, path.find_last_of('/') + 1); - if (m_ListDependentDirectories.find(pathParent) == m_ListDependentDirectories.end()) // check if it's inside one of the dependent directories - { - // Still not found! - return false; - } - } - return true; - } // ok - - bool isDirectoryDependency(const std::string &path) - { - return m_ListDependentDirectories.find(path) != m_ListDependentDirectories.end(); - } // ok - - bool hasInputDirectoryBeenModified(const std::string &inputDirectory) - { - // Check if any files added/changed/removed are part of this directory (slow). - for (std::set::const_iterator itr = m_ListInputAdded.begin(), endr = m_ListInputAdded.end(); itr != endr; ++itr) - { - const std::string &pathr = *itr; - if ((pathr.size() > inputDirectory.size()) - && (pathr.substr(0, inputDirectory.size()) == inputDirectory) // inside the path - && (pathr.substr(inputDirectory.size(), pathr.size() - inputDirectory.size())).find('/') == std::string::npos) // not in a further subdirectory - { - nldebug("Found added '%s' in dependency directory '%s'", pathr.c_str(), inputDirectory.c_str()); - return true; - } - } - for (std::set::const_iterator itr = m_ListInputChanged.begin(), endr = m_ListInputChanged.end(); itr != endr; ++itr) - { - const std::string &pathr = *itr; - if ((pathr.size() > inputDirectory.size()) - && (pathr.substr(0, inputDirectory.size()) == inputDirectory) // inside the path - && (pathr.substr(inputDirectory.size(), pathr.size() - inputDirectory.size())).find('/') == std::string::npos) // not in a further subdirectory - { - nldebug("Found changed '%s' in dependency directory '%s'", pathr.c_str(), inputDirectory.c_str()); - return true; - } - } - for (std::set::const_iterator itr = m_ListInputRemoved.begin(), endr = m_ListInputRemoved.end(); itr != endr; ++itr) - { - const std::string &pathr = *itr; - if ((pathr.size() > inputDirectory.size()) - && (pathr.substr(0, inputDirectory.size()) == inputDirectory) // inside the path - && (pathr.substr(inputDirectory.size(), pathr.size() - inputDirectory.size())).find('/') == std::string::npos) // not in a further subdirectory - { - nldebug("Found removed '%s' in dependency directory '%s'", pathr.c_str(), inputDirectory.c_str()); - return true; - } - } - // nldebug("Input directory '%s' not modified", inputDirectory.c_str()); - return false; - } // ok - - bool hasInputFileBeenModified(const std::string &inputFile) - { - return m_ListInputAdded.find(inputFile) != m_ListInputAdded.end() - || m_ListInputChanged.find(inputFile) != m_ListInputChanged.end() - || m_ListInputRemoved.find(inputFile) != m_ListInputRemoved.end(); - } // ok - - bool needsToBeRebuilt(const std::vector &inputPaths) - { - if (m_SubTaskResult != FINISH_SUCCESS) - return false; // Cannot continue on previous failure. - - m_SubTaskResult = FINISH_NOT; - - // Sanity check - for (std::vector::const_iterator it = inputPaths.begin(), end = inputPaths.end(); it != end; ++it) - { - const std::string &path = *it; - if (path[path.size() - 1] == '/') // isDirectory - { - m_SubTaskResult = FINISH_ERROR; - m_SubTaskErrorMessage = std::string("Input file '") + path + "' cannot be a directory"; - return false; // Error, cannot rebuild. - } - if (!isFileDependency(path)) - { - m_SubTaskResult = FINISH_ERROR; - m_SubTaskErrorMessage = std::string("File '") + path + "' is not part of the dependencies"; - return false; // Error, cannot rebuild. - } - } - - // Find out if anything happened to the input files. - bool inputFilesDifferent = false; - for (std::vector::const_iterator it = inputPaths.begin(), end = inputPaths.end(); it != end; ++it) - { - const std::string &path = *it; - if (hasInputFileBeenModified(path)) - { - // Found! - nldebug("Found added/changed/removed input file '%s'", path.c_str()); - inputFilesDifferent = true; - break; - } - } - if (!inputFilesDifferent) - { - nldebug("No added/changed/removed input files in this request"); - if (m_ListOutputChanged.size() == 0 && m_ListOutputRemoved.size() == 0) - { - nldebug("No output files were tampered with since last successful build, rebuild not needed"); - m_SubTaskResult = FINISH_SUCCESS; - return false; // No rebuild required. - } - else - { - nldebug("Output files may have changed, find out which output files are part of these input files"); - m_SubTaskResult = FINISH_SUCCESS; - return needsToBeRebuildSubByOutput(inputPaths, false); - } - } - else // input files have changed - { - // Find out if all the input files have a .output file - bool allInputFilesHaveOutputFile = true; - for (std::vector::const_iterator it = inputPaths.begin(), end = inputPaths.end(); it != end; ++it) - { - const std::string &path = *it; - std::string outputMetaPath = CMetadataStorage::getOutputPath(path, m_ActiveProject->getName(), m_ActivePlugin.Handler); - if (!CFile::fileExists(outputMetaPath)) - { - nldebug("Found an input file that has no .output meta file"); - allInputFilesHaveOutputFile = false; - break; - } - } - if (!allInputFilesHaveOutputFile) - { - nldebug("Not all input files have an .output files, rebuild"); - m_SubTaskResult = FINISH_SUCCESS; - return true; // Rebuild - } - else - { - nldebug("Output files may or may not be up to date, find out more, after the break"); - m_SubTaskResult = FINISH_SUCCESS; - return needsToBeRebuildSubByOutput(inputPaths, true); - } - } - // not reachable - } // ok - - bool needsToBeRebuildSubByOutput(const std::vector &inputPaths, bool inputChanged) - { - if (m_SubTaskResult != FINISH_SUCCESS) - return false; // Cannot continue on previous failure. - - m_SubTaskResult = FINISH_NOT; - - std::vector outputPaths; - - bool needsFurtherInfo = false; - - // For each inputPath - for (std::vector::const_iterator it = inputPaths.begin(), end = inputPaths.end(); it != end; ++it) - { - const std::string &path = *it; - // FIXME: What if an input file does not exist? Deleted long ago? Just print a warning (& notify terminal), keep calm, and carry on. - nlassert(!CFile::isDirectory(path)); // All input are files! Coding error otherwise, because should already be checked. - std::string metaOutputPath = CMetadataStorage::getOutputPath(path, m_ActiveProject->getName(), m_ActivePlugin.Handler); - nlassert(CFile::fileExists(metaOutputPath)); // Coding error otherwise, already must have checked beforehand that they all exist. - - // Read the .output file - CFileOutput metaOutput; - CMetadataStorage::readOutput(metaOutput, metaOutputPath); - std::vector localOutputPaths; - localOutputPaths.reserve(metaOutput.MacroPaths.size()); - for (std::vector::const_iterator it = metaOutput.MacroPaths.begin(), end = metaOutput.MacroPaths.end(); it != end; ++it) - localOutputPaths.push_back(unMacroPath(*it)); - - // If inputChanged & hasInputFileBeenModified(path) - if (inputChanged && hasInputFileBeenModified(path)) - { - // If .output file was empty - if (metaOutput.MacroPaths.empty()) - { - // Require rebuild because we don't know if there's new output - nldebug("Input file '%s' has output file with no output files, state not known, so rebuild", path.c_str()); - m_SubTaskResult = FINISH_SUCCESS; - return true; // Rebuild. - } - } - - // Copy the .output file into the output paths - for (std::vector::const_iterator it = localOutputPaths.begin(), end = localOutputPaths.end(); it != end; ++it) - outputPaths.push_back(*it); - // End - } - - // Output files may have already been built by a failed build, - // or they might have been tampered with after they were built. - nldebug("Need further information"); - m_SubTaskResult = FINISH_SUCCESS; - return needsToBeRebuiltSub(inputPaths, outputPaths, inputChanged); // to skip input changed checks because these are only input files and no input folders - } // ok ? ? - - bool needsToBeRebuilt(const std::vector &inputPaths, const std::vector &outputPaths) - { - if (m_SubTaskResult != FINISH_SUCCESS) - return false; // Cannot continue on previous failure. - - m_SubTaskResult = FINISH_NOT; - - // Sanity check of all the input paths, - // they must be either files that are dependency files or inside dependency directories - // or directories that are dependency directories. - for (std::vector::const_iterator it = inputPaths.begin(), end = inputPaths.end(); it != end; ++it) - { - const std::string &path = *it; - if (path[path.size() - 1] == '/') // isDirectory - { - // Check if this directory is in the dependencies. - if (!isDirectoryDependency(path)) - { - m_SubTaskResult = FINISH_ERROR; - m_SubTaskErrorMessage = std::string("Directory '") + path + "' is not part of the dependencies"; - return false; // Error, cannot rebuild. - } - } - else // isFile - { - // Check if this file is in the dependencies. - if (!isFileDependency(path)) - { - m_SubTaskResult = FINISH_ERROR; - m_SubTaskErrorMessage = std::string("File '") + path + "' is not part of the dependencies"; - return false; // Error, cannot rebuild. - } - } - } - - // Check if any of the input has changed - bool inputFilesDifferent = false; - for (std::vector::const_iterator it = inputPaths.begin(), end = inputPaths.end(); it != end; ++it) - { - const std::string &path = *it; - if (path[path.size() - 1] == '/') // isDirectory - { - if (hasInputDirectoryBeenModified(path)) - { - nldebug("Found modified input directory '%s'", path.c_str()); - inputFilesDifferent = true; - break; - } - } - else - { - if (hasInputFileBeenModified(path)) - { - nldebug("Found modified input file '%s'", path.c_str()); - inputFilesDifferent = true; - break; - } - } - } - - if (inputFilesDifferent) - { - nldebug("Input files were modified, check if output files were already built"); - m_SubTaskResult = FINISH_SUCCESS; - return needsToBeRebuiltSub(inputPaths, outputPaths, true); - } - else - { - nldebug("Input files were not modified"); - if (m_ListOutputChanged.size() == 0 && m_ListOutputRemoved.size() == 0) - { - nldebug("No output files were tampered with since last successful build, rebuild not needed"); - m_SubTaskResult = FINISH_SUCCESS; - return false; // No rebuild required. - } - else - { - nldebug("Output files may have changed, find out more"); - m_SubTaskResult = FINISH_SUCCESS; - return needsToBeRebuiltSub(inputPaths, outputPaths, false); - } - } - } - - bool needsToBeRebuiltSub(const std::vector &inputPaths, const std::vector &outputPaths, bool inputModified) - { - if (m_SubTaskResult != FINISH_SUCCESS) - return false; // Cannot continue on previous failure. - - m_SubTaskResult = FINISH_NOT; - - // Sanity check of all the output paths - // These must all be files, no directories allowed - for (std::vector::const_iterator it = outputPaths.begin(), end = outputPaths.end(); it != end; ++it) - { - const std::string &path = *it; - if (path[path.size() - 1] == '/') // isDirectory - { - m_SubTaskResult = FINISH_ERROR; - m_SubTaskErrorMessage = std::string("Output file '") + path + "' cannot be a directory"; - return false; // Error, cannot rebuild. - } - } - - // Check if any of the output paths are part of the removed output files - for (std::vector::const_iterator it = outputPaths.begin(), end = outputPaths.end(); it != end; ++it) - { - const std::string &path = *it; - if (m_ListOutputRemoved.find(*it) != m_ListOutputRemoved.end()) - { - // If so, rebuild - nldebug("Output file '%s' has been removed, rebuild", path.c_str()); - m_SubTaskResult = FINISH_SUCCESS; - return true; // Rebuild. - } - } - - // Check if any of the output files don't exist - for (std::vector::const_iterator it = outputPaths.begin(), end = outputPaths.end(); it != end; ++it) - { - // If so, rebuild - const std::string &path = *it; - if (!CFile::isExists(path)) - { - // If so, rebuild - nldebug("Output file '%s' does not exist, rebuild", path.c_str()); - m_SubTaskResult = FINISH_SUCCESS; - return true; // Rebuild. - } - } - - // Check if any of the output paths are part of the changed output files - bool outputChanged = false; - for (std::vector::const_iterator it = outputPaths.begin(), end = outputPaths.end(); it != end; ++it) - { - const std::string &path = *it; - if (m_ListOutputChanged.find(*it) != m_ListOutputChanged.end()) - { - nldebug("Output file '%s' has been changed", path.c_str()); - outputChanged = true; - break; - } - } - - if (!outputChanged && !inputModified) - { - nlerror("Should never reach this, this must have been cought earlier normally"); - } - - // Check the .depend files of all the output files // also check that they exist :) - for (std::vector::const_iterator it = outputPaths.begin(), end = outputPaths.end(); it != end; ++it) - { - const std::string &path = *it; - std::string metaDependPath = CMetadataStorage::getDependPath(path); - CFileDepend metaDepend; - if (!CMetadataStorage::readDepend(metaDepend, metaDependPath)) - { - nlwarning("Depend file for existing output '%s' does not exist, this should not happen, rebuild", path.c_str()); - m_SubTaskResult = FINISH_SUCCESS; - return true; // Rebuild. - } - else - { - if (outputChanged) - { - // Compare the output checksum with the status output checksum - CFileStatus metaStatus; - if (!g_DatabaseStatus->updateFileStatus(metaStatus, path)) - { - // FIXME: Is it really necessary to recalculate the crc32 if the status file is broken for an output file? Useful though for some rare cases. - m_SubTaskResult = FINISH_ERROR; - m_SubTaskErrorMessage = std::string("Could not get status for output file '") + path + "', this should never happen at all, coding error"; - return false; // Error, cannot rebuild. - } - else - { - if (metaStatus.CRC32 != metaDepend.CRC32) - { - nlwarning("Status checksum for '%s' does match depend checksum, output file was modified, this should not happen, rebuild", path.c_str()); - m_SubTaskResult = FINISH_SUCCESS; - return true; // Rebuild. - } - } - } - if (inputModified) - { - // Compare the input checksums with the cached input checksums - for (std::vector::const_iterator itd = metaDepend.Dependencies.begin(), endd = metaDepend.Dependencies.end(); itd != endd; ++itd) - { - const CFileDepend::CDependency &dependency = *itd; - std::string dependencyFile = unMacroPath(dependency.MacroPath); - CFileStatus metaStatus; - if (!getDependencyFileStatusCached(metaStatus, dependencyFile)) - { - nlwarning("Output file '%s' depends on unknown file '%s', rebuild", path.c_str(), dependencyFile.c_str()); - m_SubTaskResult = FINISH_SUCCESS; - return true; // Rebuild. - } - else - { - if (metaStatus.CRC32 != metaDepend.CRC32) - { - nldebug("Status checksum for '%s' does match depend checksum, input file was modified, rebuild", path.c_str()); - m_SubTaskResult = FINISH_SUCCESS; - return true; // Rebuild. - } - } - } - } - } - } - - // (if any checksum was different, require rebuild, if no checksums were different, no rebuild is needed) - nldebug("No differences found, no rebuild needed"); - m_SubTaskResult = FINISH_SUCCESS; - return false; // Rebuild not necessary. - } - - /// Set the exit message, exit the plugin immediately afterwards. - void setExit(const TProcessResult exitLevel, const std::string &exitMessage) - { - m_SubTaskResult = exitLevel; - m_SubTaskErrorMessage = exitMessage; - } - - /// Must verify this regularly to exit the plugin in case something went wrong. - bool needsExit() - { - // TODO: Bypass error feature. - if (m_SubTaskResult != FINISH_SUCCESS) - return true; - return false; - } - - // TODO: Make maps of the dependent files and directories after the vectors no longer needed - // Provide a function to check if a dependency is either in the dependent files or inside one of the directories to ensure the plugin is behaving sanely - // Slave should check this when it reads the depend log of a tool - - // TODO: Provide a function: bool needsToBeRebuilt(inputpaths, outputfiles) - // It should check if anything was added, changed or removed in the input paths. - // It should check if the output files were changed or removed. - // NOT NECESSARY AT ALL --- It should check if the dependencies in the metadata of the output in case the dependencies are not given in the inputpaths. - // NOT NECESSARY --- TODO: Provide a function for removing output files, it will also erase the status file and create a remove metadata file. - - // PROBABLY NOT NECESSARY -- Could check the .depend of all output files from the previous build if necessary instead of having output files and use those to flag additional removals. - - // TODO: (use this one for known input to unknown output) Also, instead of the .output file, we can check outputfileremoved and outputfilechanged's depend to flag input files that have lost output files!!! - - // TODO: If the state of an output file that is changed & ok or not changed is error, return error! - /////////////////////////////////////////////////////////////////// /////////////////////////////////////////////////////////////////// /////////////////////////////////////////////////////////////////// @@ -1023,7 +529,7 @@ public: // Set the active project and get the plugin information m_ActiveProject = g_PipelineWorkspace->getProject(projectName); m_ActiveProcess = new CPipelineProcessImpl(m_ActiveProject); - g_PipelineWorkspace->getProcessPlugin(m_ActivePlugin, pluginId); + g_PipelineWorkspace->getProcessPlugin(m_ActiveProcess->m_ActivePlugin, pluginId); // TODO: ERROR HANDLING !!! (see sanity check above) // master->slaveRefusedBuildTask(this); @@ -1035,15 +541,16 @@ public: void clearActiveProcess() { m_ActiveProject = NULL; + delete m_ActiveProcess; m_ActiveProcess = NULL; - m_ResultPreviousSuccess.clear(); + /*m_ResultPreviousSuccess.clear(); m_FileStatusInputCache.clear(); m_FileStatusOutputCache.clear(); m_FileRemoveInputCache.clear(); - m_ResultCurrent.clear(); + m_ResultCurrent.clear();*/ m_DependentDirectories.clear(); m_DependentFiles.clear(); - m_ListInputAdded.clear(); + /*m_ListInputAdded.clear(); m_ListInputChanged.clear(); m_ListInputRemoved.clear(); m_ListOutputChanged.clear(); @@ -1052,7 +559,7 @@ public: m_ListOutputChangedOK.clear(); m_ListDependentDirectories.clear(); m_ListDependentFiles.clear(); - m_SubTaskResult = FINISH_NOT; // ! // + m_SubTaskResult = FINISH_NOT;*/ // ! // } void finalizeAbort() @@ -1103,30 +610,7 @@ public: /////////////////////////////////////////////////////////////////// /////////////////////////////////////////////////////////////////// - // If this returns false, the status is not sane, and the build must error; or the file does not exist and must error or warn. - // Gets the file status as it was known at the beginning of the build or when this file was first known. - bool getDependencyFileStatusCached(CFileStatus &fileStatus, const std::string &filePath) // not by macro path :) - { - std::map::iterator statusIt = m_FileStatusInputCache.find(filePath); - if (statusIt == m_FileStatusInputCache.end()) - { - /*nldebug("File status '%s' not requested before, ensure this is not a dependency", filePath.c_str()); - if (g_DatabaseStatus->getFileStatus(fileStatus, filePath)) - { - m_FileStatusCache[filePath] = fileStatus; - return true; - } - else*/ return false; - } - fileStatus = statusIt->second; // copy - return true; - } - - // If this returns false, the status is not sane, and the build must error; or the file does not exist and must error or warn. - bool getDependencyFileStatusLatest(CFileStatus &fileStatus, const std::string &filePath) - { - return g_DatabaseStatus->getFileStatus(fileStatus, filePath); - } + virtual void addFileStatusToCache(NLNET::IModuleProxy *sender, const std::string ¯oPath, const CFileStatus &fileStatus) { @@ -1136,8 +620,8 @@ public: std::string filePath = unMacroPath(macroPath); // m_FileStatusInitializeMutex.enter(); - nlassert(m_FileStatusInputCache.find(filePath) == m_FileStatusInputCache.end()); // for now don't allow depending on own output within process :) - m_FileStatusInputCache[filePath] = fileStatus; + nlassert(m_ActiveProcess->m_FileStatusInputCache.find(filePath) == m_ActiveProcess->m_FileStatusInputCache.end()); // for now don't allow depending on own output within process :) + m_ActiveProcess->m_FileStatusInputCache[filePath] = fileStatus; // m_FileStatusInitializeMutex.leave(); } diff --git a/code/nel/tools/pipeline/service/pipeline_process_impl.cpp b/code/nel/tools/pipeline/service/pipeline_process_impl.cpp index 3bb52c3aa..245125e1b 100644 --- a/code/nel/tools/pipeline/service/pipeline_process_impl.cpp +++ b/code/nel/tools/pipeline/service/pipeline_process_impl.cpp @@ -45,13 +45,15 @@ using namespace std; namespace PIPELINE { -CPipelineProcessImpl::CPipelineProcessImpl(CPipelineProject *activeProject) : m_ActiveProject(activeProject) +CPipelineProcessImpl::CPipelineProcessImpl(CPipelineProject *activeProject) : m_ActiveProject(activeProject), m_SubTaskResult(FINISH_NOT) { if (activeProject == NULL) { nlassert(getInstance() == NULL); NLMISC::INelContext::getInstance().setSingletonPointer("IPipelineProcess", this); } + m_ResultPreviousSuccess.clear(); + m_ResultCurrent.clear(); } CPipelineProcessImpl::~CPipelineProcessImpl() @@ -135,31 +137,6 @@ bool CPipelineProcessImpl::getValueNb(uint &result, const std::string &name) } } -bool CPipelineProcessImpl::needsToBeRebuilt(const std::vector &inputPaths, const std::vector &outputPaths) -{ - return false; -} - -bool CPipelineProcessImpl::needsToBeRebuilt(const std::vector &inputPaths) -{ - return false; -} - -void CPipelineProcessImpl::parseToolLog(const std::string &dependLogFile, const std::string &errorLogFile, bool writeOutputMeta) -{ - -} - -bool CPipelineProcessImpl::needsExit() -{ - return true; -} - -void CPipelineProcessImpl::setExit(const TProcessResult exitLevel, const std::string &exitMessage) -{ - -} - } /* namespace PIPELINE */ /* end of file */ diff --git a/code/nel/tools/pipeline/service/pipeline_process_impl.h b/code/nel/tools/pipeline/service/pipeline_process_impl.h index 7990e088c..9a376daee 100644 --- a/code/nel/tools/pipeline/service/pipeline_process_impl.h +++ b/code/nel/tools/pipeline/service/pipeline_process_impl.h @@ -35,6 +35,8 @@ // Project includes #include "../plugin_library/pipeline_process.h" +#include "pipeline_workspace.h" +#include "metadata_storage.h" namespace PIPELINE { class CPipelineProject; @@ -46,7 +48,9 @@ namespace PIPELINE { * CPipelineProcessImpl */ class CPipelineProcessImpl : public IPipelineProcess -{ +{ + friend class CModulePipelineSlave; + private: CPipelineProject *m_ActiveProject; @@ -68,6 +72,45 @@ public: virtual bool needsExit(); virtual void setExit(const TProcessResult exitLevel, const std::string &exitMessage); +private: + CProcessPluginInfo m_ActivePlugin; + + CProcessResult m_ResultPreviousSuccess; + std::map m_FileStatusInputCache; + std::map m_FileStatusOutputCache; + + /// List of dependencies (files) that were removed, or never existed (in which case remove time is 0). + std::map m_FileRemoveInputCache; + + /// Result of current process + CProcessResult m_ResultCurrent; + + /// Result of the last subtask (usually in the taskmanager), check this after the task has completed to make sure all went fine. + TProcessResult m_SubTaskResult; + std::string m_SubTaskErrorMessage; + + std::set m_ListInputAdded; + std::set m_ListInputChanged; + std::set m_ListInputRemoved; + + std::set m_ListOutputChanged; + std::set m_ListOutputChangedNG; // after .depend check, found that dependencies changed, so not good + std::set m_ListOutputChangedOK; // idem but dependencies did not change, so ok + std::set m_ListOutputRemoved; // changed_ng and removed end up being the same, it needs to be rebuilt ;) + + std::set m_ListDependentDirectories; + std::set m_ListDependentFiles; + +private: + bool getDependencyFileStatusCached(CFileStatus &fileStatus, const std::string &filePath); + bool getDependencyFileStatusLatest(CFileStatus &fileStatus, const std::string &filePath); + bool isFileDependency(const std::string &path); + bool isDirectoryDependency(const std::string &path); + bool hasInputDirectoryBeenModified(const std::string &inputDirectory); + bool hasInputFileBeenModified(const std::string &inputFile); + bool needsToBeRebuildSubByOutput(const std::vector &inputPaths, bool inputChanged); + bool needsToBeRebuiltSub(const std::vector &inputPaths, const std::vector &outputPaths, bool inputModified); + }; /* class CPipelineProcessImpl */ } /* namespace PIPELINE */ diff --git a/code/nel/tools/pipeline/service/pipeline_process_impl_buildinfo.cpp b/code/nel/tools/pipeline/service/pipeline_process_impl_buildinfo.cpp new file mode 100644 index 000000000..6765379a3 --- /dev/null +++ b/code/nel/tools/pipeline/service/pipeline_process_impl_buildinfo.cpp @@ -0,0 +1,517 @@ +/** + * \file pipeline_process_impl.cpp + * \brief CPipelineProcessImpl + * \date 2012-08-03 18:22GMT + * \author Jan Boon (Kaetemi) + * CPipelineProcessImpl + */ + +/* + * Copyright (C) 2012 by authors + * + * This file is part of RYZOM CORE PIPELINE. + * RYZOM CORE PIPELINE is free software: you can redistribute it + * and/or modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, either version 2 of + * the License, or (at your option) any later version. + * + * RYZOM CORE PIPELINE is distributed in the hope that it will be + * useful, but WITHOUT ANY WARRANTY; without even the implied warranty + * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with RYZOM CORE PIPELINE; see the file COPYING. If not, see + * . + */ + +#include +#include "pipeline_process_impl.h" + +// STL includes +#include + +// NeL includes +#include +#include +#include +#include + +// Project includes +#include "pipeline_service.h" +#include "pipeline_project.h" +#include "database_status.h" + +using namespace std; +using namespace NLMISC; + +namespace PIPELINE { + +// If this returns false, the status is not sane, and the build must error; or the file does not exist and must error or warn. +// Gets the file status as it was known at the beginning of the build or when this file was first known. +bool CPipelineProcessImpl::getDependencyFileStatusCached(CFileStatus &fileStatus, const std::string &filePath) // not by macro path :) +{ + std::map::iterator statusIt = m_FileStatusInputCache.find(filePath); + if (statusIt == m_FileStatusInputCache.end()) + { + /*nldebug("File status '%s' not requested before, ensure this is not a dependency", filePath.c_str()); + if (g_DatabaseStatus->getFileStatus(fileStatus, filePath)) + { + m_FileStatusCache[filePath] = fileStatus; + return true; + } + else*/ return false; + } + fileStatus = statusIt->second; // copy + return true; +} + +// If this returns false, the status is not sane, and the build must error; or the file does not exist and must error or warn. +bool CPipelineProcessImpl::getDependencyFileStatusLatest(CFileStatus &fileStatus, const std::string &filePath) +{ + return g_DatabaseStatus->getFileStatus(fileStatus, filePath); +} + +bool CPipelineProcessImpl::isFileDependency(const std::string &path) +{ + if (m_ListDependentFiles.find(path) == m_ListDependentFiles.end()) // check if it's directly in the dependent files + { + // Not found! + std::string pathParent = path.substr(0, path.find_last_of('/') + 1); + if (m_ListDependentDirectories.find(pathParent) == m_ListDependentDirectories.end()) // check if it's inside one of the dependent directories + { + // Still not found! + return false; + } + } + return true; +} // ok + +bool CPipelineProcessImpl::isDirectoryDependency(const std::string &path) +{ + return m_ListDependentDirectories.find(path) != m_ListDependentDirectories.end(); +} // ok + +bool CPipelineProcessImpl::hasInputDirectoryBeenModified(const std::string &inputDirectory) +{ + // Check if any files added/changed/removed are part of this directory (slow). + for (std::set::const_iterator itr = m_ListInputAdded.begin(), endr = m_ListInputAdded.end(); itr != endr; ++itr) + { + const std::string &pathr = *itr; + if ((pathr.size() > inputDirectory.size()) + && (pathr.substr(0, inputDirectory.size()) == inputDirectory) // inside the path + && (pathr.substr(inputDirectory.size(), pathr.size() - inputDirectory.size())).find('/') == std::string::npos) // not in a further subdirectory + { + nldebug("Found added '%s' in dependency directory '%s'", pathr.c_str(), inputDirectory.c_str()); + return true; + } + } + for (std::set::const_iterator itr = m_ListInputChanged.begin(), endr = m_ListInputChanged.end(); itr != endr; ++itr) + { + const std::string &pathr = *itr; + if ((pathr.size() > inputDirectory.size()) + && (pathr.substr(0, inputDirectory.size()) == inputDirectory) // inside the path + && (pathr.substr(inputDirectory.size(), pathr.size() - inputDirectory.size())).find('/') == std::string::npos) // not in a further subdirectory + { + nldebug("Found changed '%s' in dependency directory '%s'", pathr.c_str(), inputDirectory.c_str()); + return true; + } + } + for (std::set::const_iterator itr = m_ListInputRemoved.begin(), endr = m_ListInputRemoved.end(); itr != endr; ++itr) + { + const std::string &pathr = *itr; + if ((pathr.size() > inputDirectory.size()) + && (pathr.substr(0, inputDirectory.size()) == inputDirectory) // inside the path + && (pathr.substr(inputDirectory.size(), pathr.size() - inputDirectory.size())).find('/') == std::string::npos) // not in a further subdirectory + { + nldebug("Found removed '%s' in dependency directory '%s'", pathr.c_str(), inputDirectory.c_str()); + return true; + } + } + // nldebug("Input directory '%s' not modified", inputDirectory.c_str()); + return false; +} // ok + +bool CPipelineProcessImpl::hasInputFileBeenModified(const std::string &inputFile) +{ + return m_ListInputAdded.find(inputFile) != m_ListInputAdded.end() + || m_ListInputChanged.find(inputFile) != m_ListInputChanged.end() + || m_ListInputRemoved.find(inputFile) != m_ListInputRemoved.end(); +} // ok + +bool CPipelineProcessImpl::needsToBeRebuilt(const std::vector &inputPaths) +{ + if (m_SubTaskResult != FINISH_SUCCESS) + return false; // Cannot continue on previous failure. + + m_SubTaskResult = FINISH_NOT; + + // Sanity check + for (std::vector::const_iterator it = inputPaths.begin(), end = inputPaths.end(); it != end; ++it) + { + const std::string &path = *it; + if (path[path.size() - 1] == '/') // isDirectory + { + m_SubTaskResult = FINISH_ERROR; + m_SubTaskErrorMessage = std::string("Input file '") + path + "' cannot be a directory"; + return false; // Error, cannot rebuild. + } + if (!isFileDependency(path)) + { + m_SubTaskResult = FINISH_ERROR; + m_SubTaskErrorMessage = std::string("File '") + path + "' is not part of the dependencies"; + return false; // Error, cannot rebuild. + } + } + + // Find out if anything happened to the input files. + bool inputFilesDifferent = false; + for (std::vector::const_iterator it = inputPaths.begin(), end = inputPaths.end(); it != end; ++it) + { + const std::string &path = *it; + if (hasInputFileBeenModified(path)) + { + // Found! + nldebug("Found added/changed/removed input file '%s'", path.c_str()); + inputFilesDifferent = true; + break; + } + } + if (!inputFilesDifferent) + { + nldebug("No added/changed/removed input files in this request"); + if (m_ListOutputChanged.size() == 0 && m_ListOutputRemoved.size() == 0) + { + nldebug("No output files were tampered with since last successful build, rebuild not needed"); + m_SubTaskResult = FINISH_SUCCESS; + return false; // No rebuild required. + } + else + { + nldebug("Output files may have changed, find out which output files are part of these input files"); + m_SubTaskResult = FINISH_SUCCESS; + return needsToBeRebuildSubByOutput(inputPaths, false); + } + } + else // input files have changed + { + // Find out if all the input files have a .output file + bool allInputFilesHaveOutputFile = true; + for (std::vector::const_iterator it = inputPaths.begin(), end = inputPaths.end(); it != end; ++it) + { + const std::string &path = *it; + std::string outputMetaPath = CMetadataStorage::getOutputPath(path, m_ActiveProject->getName(), m_ActivePlugin.Handler); + if (!CFile::fileExists(outputMetaPath)) + { + nldebug("Found an input file that has no .output meta file"); + allInputFilesHaveOutputFile = false; + break; + } + } + if (!allInputFilesHaveOutputFile) + { + nldebug("Not all input files have an .output files, rebuild"); + m_SubTaskResult = FINISH_SUCCESS; + return true; // Rebuild + } + else + { + nldebug("Output files may or may not be up to date, find out more, after the break"); + m_SubTaskResult = FINISH_SUCCESS; + return needsToBeRebuildSubByOutput(inputPaths, true); + } + } + // not reachable +} // ok + +bool CPipelineProcessImpl::needsToBeRebuildSubByOutput(const std::vector &inputPaths, bool inputChanged) +{ + if (m_SubTaskResult != FINISH_SUCCESS) + return false; // Cannot continue on previous failure. + + m_SubTaskResult = FINISH_NOT; + + std::vector outputPaths; + + bool needsFurtherInfo = false; + + // For each inputPath + for (std::vector::const_iterator it = inputPaths.begin(), end = inputPaths.end(); it != end; ++it) + { + const std::string &path = *it; + // FIXME: What if an input file does not exist? Deleted long ago? Just print a warning (& notify terminal), keep calm, and carry on. + nlassert(!CFile::isDirectory(path)); // All input are files! Coding error otherwise, because should already be checked. + std::string metaOutputPath = CMetadataStorage::getOutputPath(path, m_ActiveProject->getName(), m_ActivePlugin.Handler); + nlassert(CFile::fileExists(metaOutputPath)); // Coding error otherwise, already must have checked beforehand that they all exist. + + // Read the .output file + CFileOutput metaOutput; + CMetadataStorage::readOutput(metaOutput, metaOutputPath); + std::vector localOutputPaths; + localOutputPaths.reserve(metaOutput.MacroPaths.size()); + for (std::vector::const_iterator it = metaOutput.MacroPaths.begin(), end = metaOutput.MacroPaths.end(); it != end; ++it) + localOutputPaths.push_back(unMacroPath(*it)); + + // If inputChanged & hasInputFileBeenModified(path) + if (inputChanged && hasInputFileBeenModified(path)) + { + // If .output file was empty + if (metaOutput.MacroPaths.empty()) + { + // Require rebuild because we don't know if there's new output + nldebug("Input file '%s' has output file with no output files, state not known, so rebuild", path.c_str()); + m_SubTaskResult = FINISH_SUCCESS; + return true; // Rebuild. + } + } + + // Copy the .output file into the output paths + for (std::vector::const_iterator it = localOutputPaths.begin(), end = localOutputPaths.end(); it != end; ++it) + outputPaths.push_back(*it); + // End + } + + // Output files may have already been built by a failed build, + // or they might have been tampered with after they were built. + nldebug("Need further information"); + m_SubTaskResult = FINISH_SUCCESS; + return needsToBeRebuiltSub(inputPaths, outputPaths, inputChanged); // to skip input changed checks because these are only input files and no input folders +} // ok ? ? + +bool CPipelineProcessImpl::needsToBeRebuilt(const std::vector &inputPaths, const std::vector &outputPaths) +{ + if (m_SubTaskResult != FINISH_SUCCESS) + return false; // Cannot continue on previous failure. + + m_SubTaskResult = FINISH_NOT; + + // Sanity check of all the input paths, + // they must be either files that are dependency files or inside dependency directories + // or directories that are dependency directories. + for (std::vector::const_iterator it = inputPaths.begin(), end = inputPaths.end(); it != end; ++it) + { + const std::string &path = *it; + if (path[path.size() - 1] == '/') // isDirectory + { + // Check if this directory is in the dependencies. + if (!isDirectoryDependency(path)) + { + m_SubTaskResult = FINISH_ERROR; + m_SubTaskErrorMessage = std::string("Directory '") + path + "' is not part of the dependencies"; + return false; // Error, cannot rebuild. + } + } + else // isFile + { + // Check if this file is in the dependencies. + if (!isFileDependency(path)) + { + m_SubTaskResult = FINISH_ERROR; + m_SubTaskErrorMessage = std::string("File '") + path + "' is not part of the dependencies"; + return false; // Error, cannot rebuild. + } + } + } + + // Check if any of the input has changed + bool inputFilesDifferent = false; + for (std::vector::const_iterator it = inputPaths.begin(), end = inputPaths.end(); it != end; ++it) + { + const std::string &path = *it; + if (path[path.size() - 1] == '/') // isDirectory + { + if (hasInputDirectoryBeenModified(path)) + { + nldebug("Found modified input directory '%s'", path.c_str()); + inputFilesDifferent = true; + break; + } + } + else + { + if (hasInputFileBeenModified(path)) + { + nldebug("Found modified input file '%s'", path.c_str()); + inputFilesDifferent = true; + break; + } + } + } + + if (inputFilesDifferent) + { + nldebug("Input files were modified, check if output files were already built"); + m_SubTaskResult = FINISH_SUCCESS; + return needsToBeRebuiltSub(inputPaths, outputPaths, true); + } + else + { + nldebug("Input files were not modified"); + if (m_ListOutputChanged.size() == 0 && m_ListOutputRemoved.size() == 0) + { + nldebug("No output files were tampered with since last successful build, rebuild not needed"); + m_SubTaskResult = FINISH_SUCCESS; + return false; // No rebuild required. + } + else + { + nldebug("Output files may have changed, find out more"); + m_SubTaskResult = FINISH_SUCCESS; + return needsToBeRebuiltSub(inputPaths, outputPaths, false); + } + } +} + +bool CPipelineProcessImpl::needsToBeRebuiltSub(const std::vector &inputPaths, const std::vector &outputPaths, bool inputModified) +{ + if (m_SubTaskResult != FINISH_SUCCESS) + return false; // Cannot continue on previous failure. + + m_SubTaskResult = FINISH_NOT; + + // Sanity check of all the output paths + // These must all be files, no directories allowed + for (std::vector::const_iterator it = outputPaths.begin(), end = outputPaths.end(); it != end; ++it) + { + const std::string &path = *it; + if (path[path.size() - 1] == '/') // isDirectory + { + m_SubTaskResult = FINISH_ERROR; + m_SubTaskErrorMessage = std::string("Output file '") + path + "' cannot be a directory"; + return false; // Error, cannot rebuild. + } + } + + // Check if any of the output paths are part of the removed output files + for (std::vector::const_iterator it = outputPaths.begin(), end = outputPaths.end(); it != end; ++it) + { + const std::string &path = *it; + if (m_ListOutputRemoved.find(*it) != m_ListOutputRemoved.end()) + { + // If so, rebuild + nldebug("Output file '%s' has been removed, rebuild", path.c_str()); + m_SubTaskResult = FINISH_SUCCESS; + return true; // Rebuild. + } + } + + // Check if any of the output files don't exist + for (std::vector::const_iterator it = outputPaths.begin(), end = outputPaths.end(); it != end; ++it) + { + // If so, rebuild + const std::string &path = *it; + if (!CFile::isExists(path)) + { + // If so, rebuild + nldebug("Output file '%s' does not exist, rebuild", path.c_str()); + m_SubTaskResult = FINISH_SUCCESS; + return true; // Rebuild. + } + } + + // Check if any of the output paths are part of the changed output files + bool outputChanged = false; + for (std::vector::const_iterator it = outputPaths.begin(), end = outputPaths.end(); it != end; ++it) + { + const std::string &path = *it; + if (m_ListOutputChanged.find(*it) != m_ListOutputChanged.end()) + { + nldebug("Output file '%s' has been changed", path.c_str()); + outputChanged = true; + break; + } + } + + if (!outputChanged && !inputModified) + { + nlerror("Should never reach this, this must have been cought earlier normally"); + } + + // Check the .depend files of all the output files // also check that they exist :) + for (std::vector::const_iterator it = outputPaths.begin(), end = outputPaths.end(); it != end; ++it) + { + const std::string &path = *it; + std::string metaDependPath = CMetadataStorage::getDependPath(path); + CFileDepend metaDepend; + if (!CMetadataStorage::readDepend(metaDepend, metaDependPath)) + { + nlwarning("Depend file for existing output '%s' does not exist, this should not happen, rebuild", path.c_str()); + m_SubTaskResult = FINISH_SUCCESS; + return true; // Rebuild. + } + else + { + if (outputChanged) + { + // Compare the output checksum with the status output checksum + CFileStatus metaStatus; + if (!g_DatabaseStatus->updateFileStatus(metaStatus, path)) + { + // FIXME: Is it really necessary to recalculate the crc32 if the status file is broken for an output file? Useful though for some rare cases. + m_SubTaskResult = FINISH_ERROR; + m_SubTaskErrorMessage = std::string("Could not get status for output file '") + path + "', this should never happen at all, coding error"; + return false; // Error, cannot rebuild. + } + else + { + if (metaStatus.CRC32 != metaDepend.CRC32) + { + nlwarning("Status checksum for '%s' does match depend checksum, output file was modified, this should not happen, rebuild", path.c_str()); + m_SubTaskResult = FINISH_SUCCESS; + return true; // Rebuild. + } + } + } + if (inputModified) + { + // Compare the input checksums with the cached input checksums + for (std::vector::const_iterator itd = metaDepend.Dependencies.begin(), endd = metaDepend.Dependencies.end(); itd != endd; ++itd) + { + const CFileDepend::CDependency &dependency = *itd; + std::string dependencyFile = unMacroPath(dependency.MacroPath); + CFileStatus metaStatus; + if (!getDependencyFileStatusCached(metaStatus, dependencyFile)) + { + nlwarning("Output file '%s' depends on unknown file '%s', rebuild", path.c_str(), dependencyFile.c_str()); + m_SubTaskResult = FINISH_SUCCESS; + return true; // Rebuild. + } + else + { + if (metaStatus.CRC32 != metaDepend.CRC32) + { + nldebug("Status checksum for '%s' does match depend checksum, input file was modified, rebuild", path.c_str()); + m_SubTaskResult = FINISH_SUCCESS; + return true; // Rebuild. + } + } + } + } + } + } + + // (if any checksum was different, require rebuild, if no checksums were different, no rebuild is needed) + nldebug("No differences found, no rebuild needed"); + m_SubTaskResult = FINISH_SUCCESS; + return false; // Rebuild not necessary. +} + +/// Set the exit message, exit the plugin immediately afterwards. +void CPipelineProcessImpl::setExit(const TProcessResult exitLevel, const std::string &exitMessage) +{ + m_SubTaskResult = exitLevel; + m_SubTaskErrorMessage = exitMessage; +} + +/// Must verify this regularly to exit the plugin in case something went wrong. +bool CPipelineProcessImpl::needsExit() +{ + // TODO: Bypass error feature. + if (m_SubTaskResult != FINISH_SUCCESS) + return true; + return false; +} + +} /* namespace PIPELINE */ + +/* end of file */ diff --git a/code/nel/tools/pipeline/service/pipeline_process_impl_toollog.cpp b/code/nel/tools/pipeline/service/pipeline_process_impl_toollog.cpp new file mode 100644 index 000000000..009ccf9ce --- /dev/null +++ b/code/nel/tools/pipeline/service/pipeline_process_impl_toollog.cpp @@ -0,0 +1,55 @@ +/** + * \file pipeline_process_impl.cpp + * \brief CPipelineProcessImpl + * \date 2012-08-03 18:22GMT + * \author Jan Boon (Kaetemi) + * CPipelineProcessImpl + */ + +/* + * Copyright (C) 2012 by authors + * + * This file is part of RYZOM CORE PIPELINE. + * RYZOM CORE PIPELINE is free software: you can redistribute it + * and/or modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, either version 2 of + * the License, or (at your option) any later version. + * + * RYZOM CORE PIPELINE is distributed in the hope that it will be + * useful, but WITHOUT ANY WARRANTY; without even the implied warranty + * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with RYZOM CORE PIPELINE; see the file COPYING. If not, see + * . + */ + +#include +#include "pipeline_process_impl.h" + +// STL includes +#include + +// NeL includes +#include +#include +#include + +// Project includes +#include "pipeline_service.h" +#include "pipeline_project.h" + +using namespace std; +// using namespace NLMISC; + +namespace PIPELINE { + +void CPipelineProcessImpl::parseToolLog(const std::string &dependLogFile, const std::string &errorLogFile, bool writeOutputMeta) +{ + +} + +} /* namespace PIPELINE */ + +/* end of file */