From 7fb8ccca881d3c49833f3321407d6fc443041623 Mon Sep 17 00:00:00 2001 From: kaetemi Date: Sat, 4 Aug 2012 13:07:06 +0200 Subject: [PATCH] Added: #1440 Detect added input files for output files that were built in a non-successful build --HG-- branch : build_pipeline_v3 --- .../pipeline/service/metadata_storage.cpp | 3 +- .../tools/pipeline/service/metadata_storage.h | 1 + .../pipeline/service/pipeline_process_impl.h | 2 + .../pipeline_process_impl_buildinfo.cpp | 77 +++++++++++++++++++ 4 files changed, 82 insertions(+), 1 deletion(-) diff --git a/code/nel/tools/pipeline/service/metadata_storage.cpp b/code/nel/tools/pipeline/service/metadata_storage.cpp index 31260a509..e8c7e0aae 100644 --- a/code/nel/tools/pipeline/service/metadata_storage.cpp +++ b/code/nel/tools/pipeline/service/metadata_storage.cpp @@ -80,7 +80,8 @@ void CFileDepend::CDependency::serial(NLMISC::IStream &stream) throw (NLMISC::ES void CFileDepend::serial(NLMISC::IStream &stream) throw (NLMISC::EStream) { - uint version = stream.serialVersion(1); + uint version = stream.serialVersion(2); + if (version >= 2) stream.serial(BuildStart); else BuildStart = 0; stream.serial(CRC32); stream.serialCont(Dependencies); stream.serialCont(RuntimeDependencies); diff --git a/code/nel/tools/pipeline/service/metadata_storage.h b/code/nel/tools/pipeline/service/metadata_storage.h index 1f243e0fc..8fdbc089f 100644 --- a/code/nel/tools/pipeline/service/metadata_storage.h +++ b/code/nel/tools/pipeline/service/metadata_storage.h @@ -104,6 +104,7 @@ public: struct CFileDepend { public: + uint32 BuildStart; // The process build start when this file was built uint32 CRC32; // Checksum of the current file struct CDependency { diff --git a/code/nel/tools/pipeline/service/pipeline_process_impl.h b/code/nel/tools/pipeline/service/pipeline_process_impl.h index 23adf8054..58aea6c5f 100644 --- a/code/nel/tools/pipeline/service/pipeline_process_impl.h +++ b/code/nel/tools/pipeline/service/pipeline_process_impl.h @@ -113,6 +113,8 @@ private: bool isDirectoryDependency(const std::string &path); bool hasInputDirectoryBeenModified(const std::string &inputDirectory); bool hasInputFileBeenModified(const std::string &inputFile); + bool haveFilesBeenAddedInDirectorySince(const std::string &inputDirectory, const std::set &excludeFiles, uint32 since); + bool hasFileBeenAddedSince(const std::string &inputFile, uint32 since); bool needsToBeRebuildSubByOutput(const std::vector &inputPaths, bool inputChanged); bool needsToBeRebuiltSub(const std::vector &inputPaths, const std::vector &outputPaths, bool inputModified); diff --git a/code/nel/tools/pipeline/service/pipeline_process_impl_buildinfo.cpp b/code/nel/tools/pipeline/service/pipeline_process_impl_buildinfo.cpp index 9623612b8..db1959690 100644 --- a/code/nel/tools/pipeline/service/pipeline_process_impl_buildinfo.cpp +++ b/code/nel/tools/pipeline/service/pipeline_process_impl_buildinfo.cpp @@ -139,6 +139,47 @@ bool CPipelineProcessImpl::hasInputFileBeenModified(const std::string &inputFile || m_ListInputRemoved.find(inputFile) != m_ListInputRemoved.end(); } // ok +bool CPipelineProcessImpl::haveFilesBeenAddedInDirectorySince(const std::string &inputDirectory, const std::set &excludeFiles, uint32 since) +{ + for (std::set::const_iterator itr = m_ListInputAdded.begin(), endr = m_ListInputAdded.end(); itr != endr; ++itr) + { + const std::string &pathr = *itr; + if (excludeFiles.find(pathr) == excludeFiles.end()) + { + if ((pathr.size() > inputDirectory.size()) + && (pathr.substr(0, inputDirectory.size()) == inputDirectory) // inside the path + && (pathr.substr(inputDirectory.size(), pathr.size() - inputDirectory.size())).find('/') == std::string::npos) // not in a further subdirectory + { + CFileStatus status; + if (!getDependencyFileStatusCached(status, pathr)) + nlerror("Cannot get cached status of known file"); + if (status.FirstSeen >= since) // or > ? + { + nldebug("Found newly added '%s' in dependency directory '%s'", pathr.c_str(), inputDirectory.c_str()); + return true; + } + } + } + } + return false; +} + +bool CPipelineProcessImpl::hasFileBeenAddedSince(const std::string &inputFile, uint32 since) +{ + if (m_ListInputAdded.find(inputFile) != m_ListInputAdded.end()) + { + CFileStatus status; + if (!getDependencyFileStatusCached(status, inputFile)) + nlerror("Cannot get cached status of known file"); + if (status.FirstSeen >= since) // or > ? + { + nldebug("Found newly added '%s' in dependency files", inputFile.c_str()); + return true; + } + } + return false; +} + bool CPipelineProcessImpl::needsToBeRebuilt(const std::vector &inputPaths) { if (m_SubTaskResult != FINISH_SUCCESS) @@ -426,6 +467,9 @@ bool CPipelineProcessImpl::needsToBeRebuiltSub(const std::vector &i { nlerror("Should never reach this, this must have been cought earlier normally"); } + + std::set inputsChecked; + uint32 earliestBuildStart = 0xFFFFFFFF; // Check the .depend files of all the output files // also check that they exist :) for (std::vector::const_iterator it = outputPaths.begin(), end = outputPaths.end(); it != end; ++it) @@ -441,6 +485,8 @@ bool CPipelineProcessImpl::needsToBeRebuiltSub(const std::vector &i } else { + if (metaDepend.BuildStart < earliestBuildStart) + earliestBuildStart = metaDepend.BuildStart; if (outputChanged) { // Compare the output checksum with the status output checksum @@ -485,6 +531,37 @@ bool CPipelineProcessImpl::needsToBeRebuiltSub(const std::vector &i return true; // Rebuild. } } + inputsChecked.insert(dependencyFile); + } + } + } + } + + // Find out if any files were added in dependency directories since last build start + if (inputModified) + { + for (std::vector::const_iterator it = inputPaths.begin(), end = inputPaths.end(); it != end; ++it) + { + const std::string &path = *it; + if (path[path.size() - 1] == '/') // isDirectory + { + if (haveFilesBeenAddedInDirectorySince(path, inputsChecked, earliestBuildStart)) + { + nldebug("Found a file added after last build start in a dependency directory that is not known by the depend files, rebuild"); + m_SubTaskResult = FINISH_SUCCESS; + return true; // Rebuild. + } + } + else // isFile + { + if (inputsChecked.find(path) == inputsChecked.end()) + { + if (hasFileBeenAddedSince(path, earliestBuildStart)) + { + nldebug("Found a dependency file added after last build start that is not known by the depend files, rebuild"); + m_SubTaskResult = FINISH_SUCCESS; + return true; // Rebuild. + } } } }