Added: #1440 Support for processes where the input files include other input files that are not known in advance

--HG--
branch : build_pipeline_v3
hg/feature/build_pipeline_v3
kaetemi 12 years ago
parent 40f10b6765
commit 5f650cff5a

@ -97,13 +97,13 @@ public:
virtual bool getValueNb(uint &result, const std::string &name) = 0; virtual bool getValueNb(uint &result, const std::string &name) = 0;
/// Find out if the plugin needs to rebuild. Input can be files or directories, output can only be files /// Find out if the plugin needs to rebuild. Input can be files or directories, output can only be files
virtual bool needsToBeRebuilt(const std::vector<std::string> &inputPaths, const std::vector<std::string> &outputPaths) = 0; virtual bool needsToBeRebuilt(const std::vector<std::string> &inputPaths, const std::vector<std::string> &outputPaths, bool inputDepends) = 0;
bool needsToBeRebuilt(const std::vector<std::string> &inputPaths, const std::string &outputPath) { std::vector<std::string> outputPaths; outputPaths.push_back(outputPath); return needsToBeRebuilt(inputPaths, outputPaths); } bool needsToBeRebuilt(const std::vector<std::string> &inputPaths, const std::string &outputPath, bool inputDepends) { std::vector<std::string> outputPaths; outputPaths.push_back(outputPath); return needsToBeRebuilt(inputPaths, outputPaths, inputDepends); }
bool needsToBeRebuilt(const std::string &inputPath, const std::vector<std::string> &outputPaths) { std::vector<std::string> inputPaths; inputPaths.push_back(inputPath); return needsToBeRebuilt(inputPaths, outputPaths); } bool needsToBeRebuilt(const std::string &inputPath, const std::vector<std::string> &outputPaths, bool inputDepends) { std::vector<std::string> inputPaths; inputPaths.push_back(inputPath); return needsToBeRebuilt(inputPaths, outputPaths, inputDepends); }
bool needsToBeRebuilt(const std::string &inputPath, const std::string &outputPath) { std::vector<std::string> inputPaths; inputPaths.push_back(inputPath); std::vector<std::string> outputPaths; outputPaths.push_back(outputPath); return needsToBeRebuilt(inputPaths, outputPaths); } bool needsToBeRebuilt(const std::string &inputPath, const std::string &outputPath, bool inputDepends) { std::vector<std::string> inputPaths; inputPaths.push_back(inputPath); std::vector<std::string> outputPaths; outputPaths.push_back(outputPath); return needsToBeRebuilt(inputPaths, outputPaths, inputDepends); }
/// Find out if the plugin needs to rebuild. Input can only be files. Must request the service to write an .output metafile during depend log parsing. /// Find out if the plugin needs to rebuild. Input can only be files. Must request the service to write an .output metafile during depend log parsing.
virtual bool needsToBeRebuilt(const std::vector<std::string> &inputPaths) = 0; virtual bool needsToBeRebuilt(const std::vector<std::string> &inputPaths, bool inputDepends) = 0;
bool needsToBeRebuilt(const std::string &inputPath) { std::vector<std::string> inputPaths; inputPaths.push_back(inputPath); return needsToBeRebuilt(inputPaths); } bool needsToBeRebuilt(const std::string &inputPath, bool inputDepends) { std::vector<std::string> inputPaths; inputPaths.push_back(inputPath); return needsToBeRebuilt(inputPaths, inputDepends); }
/// Create directories for paths. Call for output paths /// Create directories for paths. Call for output paths
virtual void makePaths(const std::vector<std::string> &outputPaths) = 0; virtual void makePaths(const std::vector<std::string> &outputPaths) = 0;

@ -82,7 +82,7 @@ void CProcessInterface::build()
std::vector<std::string> dstFiles; std::vector<std::string> dstFiles;
dstFiles.push_back(dstFile); // .tga dstFiles.push_back(dstFile); // .tga
dstFiles.push_back(dstFile.substr(0, dstFile.size() - 3) + "txt"); // .txt dstFiles.push_back(dstFile.substr(0, dstFile.size() - 3) + "txt"); // .txt
if (m_PipelineProcess->needsToBeRebuilt(srcDirectories, dstFiles)) if (m_PipelineProcess->needsToBeRebuilt(srcDirectories, dstFiles, false))
{ {
m_PipelineProcess->makePaths(dstFiles); m_PipelineProcess->makePaths(dstFiles);
buildAtlas(dependLog, errorLog, srcDirectories, dstFile); buildAtlas(dependLog, errorLog, srcDirectories, dstFile);

@ -112,8 +112,7 @@ void CProcessTextureDDS::build()
{ {
const std::string &srcFile = *itf; const std::string &srcFile = *itf;
std::string dstFile = dstDirectory + NLMISC::CFile::getFilenameWithoutExtension(NLMISC::CFile::getFilename(srcFile)) + ".dds"; std::string dstFile = dstDirectory + NLMISC::CFile::getFilenameWithoutExtension(NLMISC::CFile::getFilename(srcFile)) + ".dds";
// TODO: PARAMETER inputDepends = checkUserColor (because there can be additional inputs depending on the input & depend file of the existing output) (expected input that is waited for must be in the depend log even if it doesn't exist yet!) if (m_PipelineProcess->needsToBeRebuilt(srcFile, dstFile, checkUserColor)) // may have additional inputs if usercolor exists // todo: just always pass _usercolor in the srcFile here if checkUserColor, it's the clean way, but for now this is for debugging purposes done this way
if (m_PipelineProcess->needsToBeRebuilt(srcFile, dstFile))
{ {
m_PipelineProcess->makePaths(dstFile); m_PipelineProcess->makePaths(dstFile);
buildDDS(dependLog, errorLog, srcFile, dstFile, algorithm, createMipMap, reduceFactor, checkUserColor); buildDDS(dependLog, errorLog, srcFile, dstFile, algorithm, createMipMap, reduceFactor, checkUserColor);

@ -68,8 +68,8 @@ public:
virtual bool getValues(std::vector<std::string> &resultAppend, const std::string &name); virtual bool getValues(std::vector<std::string> &resultAppend, const std::string &name);
virtual bool getValueNb(uint &result, const std::string &name); virtual bool getValueNb(uint &result, const std::string &name);
virtual bool needsToBeRebuilt(const std::vector<std::string> &inputPaths, const std::vector<std::string> &outputPaths); virtual bool needsToBeRebuilt(const std::vector<std::string> &inputPaths, const std::vector<std::string> &outputPaths, bool inputDepends);
virtual bool needsToBeRebuilt(const std::vector<std::string> &inputPaths); virtual bool needsToBeRebuilt(const std::vector<std::string> &inputPaths, bool inputDepends);
virtual void makePaths(const std::vector<std::string> &outputPaths); virtual void makePaths(const std::vector<std::string> &outputPaths);
@ -118,8 +118,8 @@ private:
bool hasInputFileBeenModified(const std::string &inputFile); bool hasInputFileBeenModified(const std::string &inputFile);
bool haveFilesBeenAddedInDirectorySince(const std::string &inputDirectory, const std::set<std::string> &excludeFiles, uint32 since); bool haveFilesBeenAddedInDirectorySince(const std::string &inputDirectory, const std::set<std::string> &excludeFiles, uint32 since);
bool hasFileBeenAddedSince(const std::string &inputFile, uint32 since); bool hasFileBeenAddedSince(const std::string &inputFile, uint32 since);
bool needsToBeRebuildSubByOutput(const std::vector<std::string> &inputPaths, bool inputChanged); bool needsToBeRebuildSubByOutput(const std::vector<std::string> &inputPaths, bool inputChanged, bool inputDepends);
bool needsToBeRebuiltSub(const std::vector<std::string> &inputPaths, const std::vector<std::string> &outputPaths, bool inputModified); bool needsToBeRebuiltSub(const std::vector<std::string> &inputPaths, const std::vector<std::string> &outputPaths, bool inputModified, bool inputDepends);
}; /* class CPipelineProcessImpl */ }; /* class CPipelineProcessImpl */

@ -180,7 +180,7 @@ bool CPipelineProcessImpl::hasFileBeenAddedSince(const std::string &inputFile, u
return false; return false;
} }
bool CPipelineProcessImpl::needsToBeRebuilt(const std::vector<std::string> &inputPaths) bool CPipelineProcessImpl::needsToBeRebuilt(const std::vector<std::string> &inputPaths, bool inputDepends)
{ {
if (m_SubTaskResult != FINISH_SUCCESS) if (m_SubTaskResult != FINISH_SUCCESS)
return false; // Cannot continue on previous failure. return false; // Cannot continue on previous failure.
@ -222,16 +222,25 @@ bool CPipelineProcessImpl::needsToBeRebuilt(const std::vector<std::string> &inpu
{ {
nldebug("No added/changed/removed input files in this request"); nldebug("No added/changed/removed input files in this request");
if (m_ListOutputChanged.size() == 0 && m_ListOutputRemoved.size() == 0) if (m_ListOutputChanged.size() == 0 && m_ListOutputRemoved.size() == 0)
{
if (inputDepends)
{
nldebug("Ouput files did not change, but not all input files are known, need further information");
m_SubTaskResult = FINISH_SUCCESS;
return needsToBeRebuildSubByOutput(inputPaths, false, inputDepends);
}
else
{ {
nldebug("No output files were tampered with since last successful build, rebuild not needed"); nldebug("No output files were tampered with since last successful build, rebuild not needed");
m_SubTaskResult = FINISH_SUCCESS; m_SubTaskResult = FINISH_SUCCESS;
return false; // No rebuild required. return false; // No rebuild required.
} }
}
else else
{ {
nldebug("Output files may have changed, find out which output files are part of these input files"); nldebug("Output files may have changed, find out which output files are part of these input files");
m_SubTaskResult = FINISH_SUCCESS; m_SubTaskResult = FINISH_SUCCESS;
return needsToBeRebuildSubByOutput(inputPaths, false); return needsToBeRebuildSubByOutput(inputPaths, false, inputDepends);
} }
} }
else // input files have changed else // input files have changed
@ -259,13 +268,13 @@ bool CPipelineProcessImpl::needsToBeRebuilt(const std::vector<std::string> &inpu
{ {
nldebug("Output files may or may not be up to date, find out more, after the break"); nldebug("Output files may or may not be up to date, find out more, after the break");
m_SubTaskResult = FINISH_SUCCESS; m_SubTaskResult = FINISH_SUCCESS;
return needsToBeRebuildSubByOutput(inputPaths, true); return needsToBeRebuildSubByOutput(inputPaths, true, inputDepends);
} }
} }
// not reachable // not reachable
} // ok } // ok
bool CPipelineProcessImpl::needsToBeRebuildSubByOutput(const std::vector<std::string> &inputPaths, bool inputChanged) bool CPipelineProcessImpl::needsToBeRebuildSubByOutput(const std::vector<std::string> &inputPaths, bool inputChanged, bool inputDepends)
{ {
if (m_SubTaskResult != FINISH_SUCCESS) if (m_SubTaskResult != FINISH_SUCCESS)
return false; // Cannot continue on previous failure. return false; // Cannot continue on previous failure.
@ -316,10 +325,10 @@ bool CPipelineProcessImpl::needsToBeRebuildSubByOutput(const std::vector<std::st
// or they might have been tampered with after they were built. // or they might have been tampered with after they were built.
nldebug("Need further information"); nldebug("Need further information");
m_SubTaskResult = FINISH_SUCCESS; m_SubTaskResult = FINISH_SUCCESS;
return needsToBeRebuiltSub(inputPaths, outputPaths, inputChanged); // to skip input changed checks because these are only input files and no input folders return needsToBeRebuiltSub(inputPaths, outputPaths, inputChanged, inputDepends); // to skip input changed checks because these are only input files and no input folders
} // ok ? ? } // ok ? ?
bool CPipelineProcessImpl::needsToBeRebuilt(const std::vector<std::string> &inputPaths, const std::vector<std::string> &outputPaths) bool CPipelineProcessImpl::needsToBeRebuilt(const std::vector<std::string> &inputPaths, const std::vector<std::string> &outputPaths, bool inputDepends)
{ {
if (m_SubTaskResult != FINISH_SUCCESS) if (m_SubTaskResult != FINISH_SUCCESS)
return false; // Cannot continue on previous failure. return false; // Cannot continue on previous failure.
@ -383,12 +392,18 @@ bool CPipelineProcessImpl::needsToBeRebuilt(const std::vector<std::string> &inpu
{ {
nldebug("Input files were modified, check if output files were already built"); nldebug("Input files were modified, check if output files were already built");
m_SubTaskResult = FINISH_SUCCESS; m_SubTaskResult = FINISH_SUCCESS;
return needsToBeRebuiltSub(inputPaths, outputPaths, true); return needsToBeRebuiltSub(inputPaths, outputPaths, true, inputDepends);
} }
else else
{ {
nldebug("Input files were not modified"); nldebug("Input files were not modified");
if (m_ListOutputChanged.size() == 0 && m_ListOutputRemoved.size() == 0) if (inputDepends)
{
nldebug("Not all input files are known, find out more");
m_SubTaskResult = FINISH_SUCCESS;
return needsToBeRebuiltSub(inputPaths, outputPaths, true, inputDepends);
}
else if (m_ListOutputChanged.size() == 0 && m_ListOutputRemoved.size() == 0)
{ {
nldebug("No output files were tampered with since last successful build, rebuild not needed"); nldebug("No output files were tampered with since last successful build, rebuild not needed");
m_SubTaskResult = FINISH_SUCCESS; m_SubTaskResult = FINISH_SUCCESS;
@ -398,12 +413,12 @@ bool CPipelineProcessImpl::needsToBeRebuilt(const std::vector<std::string> &inpu
{ {
nldebug("Output files may have changed, find out more"); nldebug("Output files may have changed, find out more");
m_SubTaskResult = FINISH_SUCCESS; m_SubTaskResult = FINISH_SUCCESS;
return needsToBeRebuiltSub(inputPaths, outputPaths, false); return needsToBeRebuiltSub(inputPaths, outputPaths, false, inputDepends);
} }
} }
} }
bool CPipelineProcessImpl::needsToBeRebuiltSub(const std::vector<std::string> &inputPaths, const std::vector<std::string> &outputPaths, bool inputModified) bool CPipelineProcessImpl::needsToBeRebuiltSub(const std::vector<std::string> &inputPaths, const std::vector<std::string> &outputPaths, bool inputModified, bool inputDepends)
{ {
if (m_SubTaskResult != FINISH_SUCCESS) if (m_SubTaskResult != FINISH_SUCCESS)
return false; // Cannot continue on previous failure. return false; // Cannot continue on previous failure.
@ -463,7 +478,7 @@ bool CPipelineProcessImpl::needsToBeRebuiltSub(const std::vector<std::string> &i
} }
} }
if (!outputChanged && !inputModified) if (!outputChanged && !inputModified && !inputDepends)
{ {
nlerror("Should never reach this, this must have been cought earlier normally"); nlerror("Should never reach this, this must have been cought earlier normally");
} }
@ -471,6 +486,8 @@ bool CPipelineProcessImpl::needsToBeRebuiltSub(const std::vector<std::string> &i
std::set<std::string> inputsChecked; std::set<std::string> inputsChecked;
uint32 earliestBuildStart = 0xFFFFFFFF; uint32 earliestBuildStart = 0xFFFFFFFF;
std::set<std::string> dependDirectories;
// Check the .depend files of all the output files // also check that they exist :) // Check the .depend files of all the output files // also check that they exist :)
for (std::vector<std::string>::const_iterator it = outputPaths.begin(), end = outputPaths.end(); it != end; ++it) for (std::vector<std::string>::const_iterator it = outputPaths.begin(), end = outputPaths.end(); it != end; ++it)
{ {
@ -508,7 +525,7 @@ bool CPipelineProcessImpl::needsToBeRebuiltSub(const std::vector<std::string> &i
} }
} }
} }
if (inputModified) if (inputModified || inputDepends)
{ {
// Compare the input checksums with the cached input checksums // Compare the input checksums with the cached input checksums
for (std::vector<CFileDepend::CDependency>::const_iterator itd = metaDepend.Dependencies.begin(), endd = metaDepend.Dependencies.end(); itd != endd; ++itd) for (std::vector<CFileDepend::CDependency>::const_iterator itd = metaDepend.Dependencies.begin(), endd = metaDepend.Dependencies.end(); itd != endd; ++itd)
@ -535,13 +552,20 @@ bool CPipelineProcessImpl::needsToBeRebuiltSub(const std::vector<std::string> &i
} }
} }
} }
for (std::vector<std::string>::iterator it_dir = metaDepend.DirectoryDependencies.begin(), end_dir = metaDepend.DirectoryDependencies.end(); it_dir != end_dir; ++it_dir)
{
std::string dir = unMacroPath(*it_dir);
if (dependDirectories.find(dir) == dependDirectories.end())
dependDirectories.insert(dir);
}
} }
// Find out if any files were added in dependency directories since last build start // Find out if any files were added in dependency directories since last build start
if (inputModified) if (inputModified || inputDepends)
{ {
// TODO: ONLY CHECK INPUT PATHS GIVEN BY DEPEND META FILES // for (std::vector<std::string>::const_iterator it = inputPaths.begin(), end = inputPaths.end(); it != end; ++it)
for (std::vector<std::string>::const_iterator it = inputPaths.begin(), end = inputPaths.end(); it != end; ++it) // Only check directories given by depend meta files
for (std::set<std::string>::const_iterator it = dependDirectories.begin(), end = dependDirectories.end(); it != end; ++it)
{ {
const std::string &path = *it; const std::string &path = *it;
if (path[path.size() - 1] == '/') // isDirectory if (path[path.size() - 1] == '/') // isDirectory

Loading…
Cancel
Save