From 5f650cff5ad6a10ea34b8e50f77458693e90a1b0 Mon Sep 17 00:00:00 2001 From: kaetemi Date: Sun, 5 Aug 2012 16:30:46 +0200 Subject: [PATCH] Added: #1440 Support for processes where the input files include other input files that are not known in advance --HG-- branch : build_pipeline_v3 --- .../plugin_library/pipeline_process.h | 12 ++-- .../pipeline/plugin_nel/process_interface.cpp | 2 +- .../plugin_nel/process_texture_dds.cpp | 3 +- .../pipeline/service/pipeline_process_impl.h | 8 +-- .../pipeline_process_impl_buildinfo.cpp | 62 +++++++++++++------ 5 files changed, 55 insertions(+), 32 deletions(-) diff --git a/code/nel/tools/pipeline/plugin_library/pipeline_process.h b/code/nel/tools/pipeline/plugin_library/pipeline_process.h index cc2181994..ba4b5d7ed 100644 --- a/code/nel/tools/pipeline/plugin_library/pipeline_process.h +++ b/code/nel/tools/pipeline/plugin_library/pipeline_process.h @@ -97,13 +97,13 @@ public: virtual bool getValueNb(uint &result, const std::string &name) = 0; /// Find out if the plugin needs to rebuild. Input can be files or directories, output can only be files - virtual bool needsToBeRebuilt(const std::vector &inputPaths, const std::vector &outputPaths) = 0; - bool needsToBeRebuilt(const std::vector &inputPaths, const std::string &outputPath) { std::vector outputPaths; outputPaths.push_back(outputPath); return needsToBeRebuilt(inputPaths, outputPaths); } - bool needsToBeRebuilt(const std::string &inputPath, const std::vector &outputPaths) { std::vector inputPaths; inputPaths.push_back(inputPath); return needsToBeRebuilt(inputPaths, outputPaths); } - bool needsToBeRebuilt(const std::string &inputPath, const std::string &outputPath) { std::vector inputPaths; inputPaths.push_back(inputPath); std::vector outputPaths; outputPaths.push_back(outputPath); return needsToBeRebuilt(inputPaths, outputPaths); } + virtual bool needsToBeRebuilt(const std::vector &inputPaths, const std::vector &outputPaths, bool inputDepends) = 0; + bool needsToBeRebuilt(const std::vector &inputPaths, const std::string &outputPath, bool inputDepends) { std::vector outputPaths; outputPaths.push_back(outputPath); return needsToBeRebuilt(inputPaths, outputPaths, inputDepends); } + bool needsToBeRebuilt(const std::string &inputPath, const std::vector &outputPaths, bool inputDepends) { std::vector inputPaths; inputPaths.push_back(inputPath); return needsToBeRebuilt(inputPaths, outputPaths, inputDepends); } + bool needsToBeRebuilt(const std::string &inputPath, const std::string &outputPath, bool inputDepends) { std::vector inputPaths; inputPaths.push_back(inputPath); std::vector outputPaths; outputPaths.push_back(outputPath); return needsToBeRebuilt(inputPaths, outputPaths, inputDepends); } /// Find out if the plugin needs to rebuild. Input can only be files. Must request the service to write an .output metafile during depend log parsing. - virtual bool needsToBeRebuilt(const std::vector &inputPaths) = 0; - bool needsToBeRebuilt(const std::string &inputPath) { std::vector inputPaths; inputPaths.push_back(inputPath); return needsToBeRebuilt(inputPaths); } + virtual bool needsToBeRebuilt(const std::vector &inputPaths, bool inputDepends) = 0; + bool needsToBeRebuilt(const std::string &inputPath, bool inputDepends) { std::vector inputPaths; inputPaths.push_back(inputPath); return needsToBeRebuilt(inputPaths, inputDepends); } /// Create directories for paths. Call for output paths virtual void makePaths(const std::vector &outputPaths) = 0; diff --git a/code/nel/tools/pipeline/plugin_nel/process_interface.cpp b/code/nel/tools/pipeline/plugin_nel/process_interface.cpp index ed417a967..e58aa2b2f 100644 --- a/code/nel/tools/pipeline/plugin_nel/process_interface.cpp +++ b/code/nel/tools/pipeline/plugin_nel/process_interface.cpp @@ -82,7 +82,7 @@ void CProcessInterface::build() std::vector dstFiles; dstFiles.push_back(dstFile); // .tga dstFiles.push_back(dstFile.substr(0, dstFile.size() - 3) + "txt"); // .txt - if (m_PipelineProcess->needsToBeRebuilt(srcDirectories, dstFiles)) + if (m_PipelineProcess->needsToBeRebuilt(srcDirectories, dstFiles, false)) { m_PipelineProcess->makePaths(dstFiles); buildAtlas(dependLog, errorLog, srcDirectories, dstFile); diff --git a/code/nel/tools/pipeline/plugin_nel/process_texture_dds.cpp b/code/nel/tools/pipeline/plugin_nel/process_texture_dds.cpp index 2aea236b0..8f4f76428 100644 --- a/code/nel/tools/pipeline/plugin_nel/process_texture_dds.cpp +++ b/code/nel/tools/pipeline/plugin_nel/process_texture_dds.cpp @@ -112,8 +112,7 @@ void CProcessTextureDDS::build() { const std::string &srcFile = *itf; std::string dstFile = dstDirectory + NLMISC::CFile::getFilenameWithoutExtension(NLMISC::CFile::getFilename(srcFile)) + ".dds"; - // TODO: PARAMETER inputDepends = checkUserColor (because there can be additional inputs depending on the input & depend file of the existing output) (expected input that is waited for must be in the depend log even if it doesn't exist yet!) - if (m_PipelineProcess->needsToBeRebuilt(srcFile, dstFile)) + if (m_PipelineProcess->needsToBeRebuilt(srcFile, dstFile, checkUserColor)) // may have additional inputs if usercolor exists // todo: just always pass _usercolor in the srcFile here if checkUserColor, it's the clean way, but for now this is for debugging purposes done this way { m_PipelineProcess->makePaths(dstFile); buildDDS(dependLog, errorLog, srcFile, dstFile, algorithm, createMipMap, reduceFactor, checkUserColor); diff --git a/code/nel/tools/pipeline/service/pipeline_process_impl.h b/code/nel/tools/pipeline/service/pipeline_process_impl.h index 31c7671fe..5bfc2975b 100644 --- a/code/nel/tools/pipeline/service/pipeline_process_impl.h +++ b/code/nel/tools/pipeline/service/pipeline_process_impl.h @@ -68,8 +68,8 @@ public: virtual bool getValues(std::vector &resultAppend, const std::string &name); virtual bool getValueNb(uint &result, const std::string &name); - virtual bool needsToBeRebuilt(const std::vector &inputPaths, const std::vector &outputPaths); - virtual bool needsToBeRebuilt(const std::vector &inputPaths); + virtual bool needsToBeRebuilt(const std::vector &inputPaths, const std::vector &outputPaths, bool inputDepends); + virtual bool needsToBeRebuilt(const std::vector &inputPaths, bool inputDepends); virtual void makePaths(const std::vector &outputPaths); @@ -118,8 +118,8 @@ private: bool hasInputFileBeenModified(const std::string &inputFile); bool haveFilesBeenAddedInDirectorySince(const std::string &inputDirectory, const std::set &excludeFiles, uint32 since); bool hasFileBeenAddedSince(const std::string &inputFile, uint32 since); - bool needsToBeRebuildSubByOutput(const std::vector &inputPaths, bool inputChanged); - bool needsToBeRebuiltSub(const std::vector &inputPaths, const std::vector &outputPaths, bool inputModified); + bool needsToBeRebuildSubByOutput(const std::vector &inputPaths, bool inputChanged, bool inputDepends); + bool needsToBeRebuiltSub(const std::vector &inputPaths, const std::vector &outputPaths, bool inputModified, bool inputDepends); }; /* class CPipelineProcessImpl */ diff --git a/code/nel/tools/pipeline/service/pipeline_process_impl_buildinfo.cpp b/code/nel/tools/pipeline/service/pipeline_process_impl_buildinfo.cpp index bc09e2cb4..9f212a572 100644 --- a/code/nel/tools/pipeline/service/pipeline_process_impl_buildinfo.cpp +++ b/code/nel/tools/pipeline/service/pipeline_process_impl_buildinfo.cpp @@ -180,7 +180,7 @@ bool CPipelineProcessImpl::hasFileBeenAddedSince(const std::string &inputFile, u return false; } -bool CPipelineProcessImpl::needsToBeRebuilt(const std::vector &inputPaths) +bool CPipelineProcessImpl::needsToBeRebuilt(const std::vector &inputPaths, bool inputDepends) { if (m_SubTaskResult != FINISH_SUCCESS) return false; // Cannot continue on previous failure. @@ -223,15 +223,24 @@ bool CPipelineProcessImpl::needsToBeRebuilt(const std::vector &inpu nldebug("No added/changed/removed input files in this request"); if (m_ListOutputChanged.size() == 0 && m_ListOutputRemoved.size() == 0) { - nldebug("No output files were tampered with since last successful build, rebuild not needed"); - m_SubTaskResult = FINISH_SUCCESS; - return false; // No rebuild required. + if (inputDepends) + { + nldebug("Ouput files did not change, but not all input files are known, need further information"); + m_SubTaskResult = FINISH_SUCCESS; + return needsToBeRebuildSubByOutput(inputPaths, false, inputDepends); + } + else + { + nldebug("No output files were tampered with since last successful build, rebuild not needed"); + m_SubTaskResult = FINISH_SUCCESS; + return false; // No rebuild required. + } } else { nldebug("Output files may have changed, find out which output files are part of these input files"); m_SubTaskResult = FINISH_SUCCESS; - return needsToBeRebuildSubByOutput(inputPaths, false); + return needsToBeRebuildSubByOutput(inputPaths, false, inputDepends); } } else // input files have changed @@ -259,13 +268,13 @@ bool CPipelineProcessImpl::needsToBeRebuilt(const std::vector &inpu { nldebug("Output files may or may not be up to date, find out more, after the break"); m_SubTaskResult = FINISH_SUCCESS; - return needsToBeRebuildSubByOutput(inputPaths, true); + return needsToBeRebuildSubByOutput(inputPaths, true, inputDepends); } } // not reachable } // ok -bool CPipelineProcessImpl::needsToBeRebuildSubByOutput(const std::vector &inputPaths, bool inputChanged) +bool CPipelineProcessImpl::needsToBeRebuildSubByOutput(const std::vector &inputPaths, bool inputChanged, bool inputDepends) { if (m_SubTaskResult != FINISH_SUCCESS) return false; // Cannot continue on previous failure. @@ -316,10 +325,10 @@ bool CPipelineProcessImpl::needsToBeRebuildSubByOutput(const std::vector &inputPaths, const std::vector &outputPaths) +bool CPipelineProcessImpl::needsToBeRebuilt(const std::vector &inputPaths, const std::vector &outputPaths, bool inputDepends) { if (m_SubTaskResult != FINISH_SUCCESS) return false; // Cannot continue on previous failure. @@ -378,17 +387,23 @@ bool CPipelineProcessImpl::needsToBeRebuilt(const std::vector &inpu } } } - + if (inputFilesDifferent) { nldebug("Input files were modified, check if output files were already built"); m_SubTaskResult = FINISH_SUCCESS; - return needsToBeRebuiltSub(inputPaths, outputPaths, true); + return needsToBeRebuiltSub(inputPaths, outputPaths, true, inputDepends); } else { nldebug("Input files were not modified"); - if (m_ListOutputChanged.size() == 0 && m_ListOutputRemoved.size() == 0) + if (inputDepends) + { + nldebug("Not all input files are known, find out more"); + m_SubTaskResult = FINISH_SUCCESS; + return needsToBeRebuiltSub(inputPaths, outputPaths, true, inputDepends); + } + else if (m_ListOutputChanged.size() == 0 && m_ListOutputRemoved.size() == 0) { nldebug("No output files were tampered with since last successful build, rebuild not needed"); m_SubTaskResult = FINISH_SUCCESS; @@ -398,12 +413,12 @@ bool CPipelineProcessImpl::needsToBeRebuilt(const std::vector &inpu { nldebug("Output files may have changed, find out more"); m_SubTaskResult = FINISH_SUCCESS; - return needsToBeRebuiltSub(inputPaths, outputPaths, false); + return needsToBeRebuiltSub(inputPaths, outputPaths, false, inputDepends); } } } -bool CPipelineProcessImpl::needsToBeRebuiltSub(const std::vector &inputPaths, const std::vector &outputPaths, bool inputModified) +bool CPipelineProcessImpl::needsToBeRebuiltSub(const std::vector &inputPaths, const std::vector &outputPaths, bool inputModified, bool inputDepends) { if (m_SubTaskResult != FINISH_SUCCESS) return false; // Cannot continue on previous failure. @@ -463,7 +478,7 @@ bool CPipelineProcessImpl::needsToBeRebuiltSub(const std::vector &i } } - if (!outputChanged && !inputModified) + if (!outputChanged && !inputModified && !inputDepends) { nlerror("Should never reach this, this must have been cought earlier normally"); } @@ -471,6 +486,8 @@ bool CPipelineProcessImpl::needsToBeRebuiltSub(const std::vector &i std::set inputsChecked; uint32 earliestBuildStart = 0xFFFFFFFF; + std::set dependDirectories; + // Check the .depend files of all the output files // also check that they exist :) for (std::vector::const_iterator it = outputPaths.begin(), end = outputPaths.end(); it != end; ++it) { @@ -508,7 +525,7 @@ bool CPipelineProcessImpl::needsToBeRebuiltSub(const std::vector &i } } } - if (inputModified) + if (inputModified || inputDepends) { // Compare the input checksums with the cached input checksums for (std::vector::const_iterator itd = metaDepend.Dependencies.begin(), endd = metaDepend.Dependencies.end(); itd != endd; ++itd) @@ -535,13 +552,20 @@ bool CPipelineProcessImpl::needsToBeRebuiltSub(const std::vector &i } } } + for (std::vector::iterator it_dir = metaDepend.DirectoryDependencies.begin(), end_dir = metaDepend.DirectoryDependencies.end(); it_dir != end_dir; ++it_dir) + { + std::string dir = unMacroPath(*it_dir); + if (dependDirectories.find(dir) == dependDirectories.end()) + dependDirectories.insert(dir); + } } // Find out if any files were added in dependency directories since last build start - if (inputModified) + if (inputModified || inputDepends) { - // TODO: ONLY CHECK INPUT PATHS GIVEN BY DEPEND META FILES - for (std::vector::const_iterator it = inputPaths.begin(), end = inputPaths.end(); it != end; ++it) + // for (std::vector::const_iterator it = inputPaths.begin(), end = inputPaths.end(); it != end; ++it) + // Only check directories given by depend meta files + for (std::set::const_iterator it = dependDirectories.begin(), end = dependDirectories.end(); it != end; ++it) { const std::string &path = *it; if (path[path.size() - 1] == '/') // isDirectory