Added: #1440 Detect added input files for output files that were built in a non-successful build

--HG--
branch : build_pipeline_v3

code/nel/tools/pipeline/service/metadata_storage.cpp

@@ -80,7 +80,8 @@ void CFileDepend::CDependency::serial(NLMISC::IStream &stream) throw (NLMISC::ES
 
 void CFileDepend::serial(NLMISC::IStream &stream) throw (NLMISC::EStream)
 {
-	uint version = stream.serialVersion(1);
+	uint version = stream.serialVersion(2);
+	if (version >= 2) stream.serial(BuildStart); else BuildStart = 0;
 	stream.serial(CRC32);
 	stream.serialCont(Dependencies);
 	stream.serialCont(RuntimeDependencies);

code/nel/tools/pipeline/service/metadata_storage.h

@@ -104,6 +104,7 @@ public:
 struct CFileDepend
 {
 public:
+	uint32 BuildStart; // The process build start when this file was built
 	uint32 CRC32; // Checksum of the current file
 	struct CDependency
 	{

code/nel/tools/pipeline/service/pipeline_process_impl.h

@@ -113,6 +113,8 @@ private:
 	bool isDirectoryDependency(const std::string &path);
 	bool hasInputDirectoryBeenModified(const std::string &inputDirectory);
 	bool hasInputFileBeenModified(const std::string &inputFile);
+	bool haveFilesBeenAddedInDirectorySince(const std::string &inputDirectory, const std::set<std::string> &excludeFiles, uint32 since);
+	bool hasFileBeenAddedSince(const std::string &inputFile, uint32 since);
 	bool needsToBeRebuildSubByOutput(const std::vector<std::string> &inputPaths, bool inputChanged);
 	bool needsToBeRebuiltSub(const std::vector<std::string> &inputPaths, const std::vector<std::string> &outputPaths, bool inputModified);
 

code/nel/tools/pipeline/service/pipeline_process_impl_buildinfo.cpp

@@ -139,6 +139,47 @@ bool CPipelineProcessImpl::hasInputFileBeenModified(const std::string &inputFile
 		|| m_ListInputRemoved.find(inputFile) != m_ListInputRemoved.end();
 } // ok
 
+bool CPipelineProcessImpl::haveFilesBeenAddedInDirectorySince(const std::string &inputDirectory, const std::set<std::string> &excludeFiles, uint32 since)
+{
+	for (std::set<std::string>::const_iterator itr = m_ListInputAdded.begin(), endr = m_ListInputAdded.end(); itr != endr; ++itr)
+	{
+		const std::string &pathr = *itr;
+		if (excludeFiles.find(pathr) == excludeFiles.end())
+		{
+			if ((pathr.size() > inputDirectory.size())
+				&& (pathr.substr(0, inputDirectory.size()) == inputDirectory) // inside the path
+				&& (pathr.substr(inputDirectory.size(), pathr.size() - inputDirectory.size())).find('/') == std::string::npos) // not in a further subdirectory 
+			{
+				CFileStatus status;
+				if (!getDependencyFileStatusCached(status, pathr))
+					nlerror("Cannot get cached status of known file");
+				if (status.FirstSeen >= since) // or > ?
+				{
+					nldebug("Found newly added '%s' in dependency directory '%s'", pathr.c_str(), inputDirectory.c_str());
+					return true;
+				}
+			}
+		}
+	}
+	return false;
+}
+
+bool CPipelineProcessImpl::hasFileBeenAddedSince(const std::string &inputFile, uint32 since)
+{
+	if (m_ListInputAdded.find(inputFile) != m_ListInputAdded.end())
+	{
+		CFileStatus status;
+		if (!getDependencyFileStatusCached(status, inputFile))
+			nlerror("Cannot get cached status of known file");
+		if (status.FirstSeen >= since) // or > ?
+		{
+			nldebug("Found newly added '%s' in dependency files", inputFile.c_str());
+			return true;
+		}
+	}
+	return false;
+}
+
 bool CPipelineProcessImpl::needsToBeRebuilt(const std::vector<std::string> &inputPaths)
 {
 	if (m_SubTaskResult != FINISH_SUCCESS)
@@ -426,6 +467,9 @@ bool CPipelineProcessImpl::needsToBeRebuiltSub(const std::vector<std::string> &i
 	{
 		nlerror("Should never reach this, this must have been cought earlier normally");
 	}
+
+	std::set<std::string> inputsChecked;
+	uint32 earliestBuildStart = 0xFFFFFFFF;
 	
 	// Check the .depend files of all the output files // also check that they exist :)
 	for (std::vector<std::string>::const_iterator it = outputPaths.begin(), end = outputPaths.end(); it != end; ++it)
@@ -441,6 +485,8 @@ bool CPipelineProcessImpl::needsToBeRebuiltSub(const std::vector<std::string> &i
 		}
 		else
 		{
+			if (metaDepend.BuildStart < earliestBuildStart)
+				earliestBuildStart = metaDepend.BuildStart;
 			if (outputChanged)
 			{
 				// Compare the output checksum with the status output checksum
@@ -485,6 +531,37 @@ bool CPipelineProcessImpl::needsToBeRebuiltSub(const std::vector<std::string> &i
 							return true; // Rebuild.
 						}
 					}
+					inputsChecked.insert(dependencyFile);
+				}
+			}
+		}
+	}
+
+	// Find out if any files were added in dependency directories since last build start
+	if (inputModified)
+	{
+		for (std::vector<std::string>::const_iterator it = inputPaths.begin(), end = inputPaths.end(); it != end; ++it)
+		{
+			const std::string &path = *it;
+			if (path[path.size() - 1] == '/') // isDirectory
+			{
+				if (haveFilesBeenAddedInDirectorySince(path, inputsChecked, earliestBuildStart))
+				{
+					nldebug("Found a file added after last build start in a dependency directory that is not known by the depend files, rebuild");
+					m_SubTaskResult = FINISH_SUCCESS;
+					return true; // Rebuild.
+				}
+			}
+			else // isFile
+			{
+				if (inputsChecked.find(path) == inputsChecked.end())
+				{
+					if (hasFileBeenAddedSince(path, earliestBuildStart))
+					{
+						nldebug("Found a dependency file added after last build start that is not known by the depend files, rebuild");
+						m_SubTaskResult = FINISH_SUCCESS;
+						return true; // Rebuild.
+					}
 				}
 			}
 		}