Added: #1440 More of the needsToBeRebuilt function

--HG--
branch : build_pipeline_v3

code/nel/tools/pipeline/service/database_status.cpp

@@ -74,7 +74,7 @@ bool CDatabaseStatus::getFileStatus(CFileStatus &fileStatus, const std::string &
 	m_StatusMutex.unlock_shared();
 	return seemsValid;
 }
-
+/*
 bool CDatabaseStatus::getFileStatus(std::map<std::string, CFileStatus> &fileStatusMap, std::map<std::string, CFileRemove> &fileRemoveMap, const std::vector<std::string> &paths)
 {
 	nlassert(false); // not used
@@ -193,7 +193,7 @@ bool CDatabaseStatus::getFileStatus(std::map<std::string, CFileStatus> &fileStat
 	// no issues occured apparently
 	return true;
 }
-
+*/
 bool CDatabaseStatus::getRemoved(std::map<std::string, CFileRemove> &fileRemoveMap, const std::vector<std::string> &paths)
 {
 	for (std::vector<std::string>::const_iterator it = paths.begin(), end = paths.end(); it != end; ++it)
@@ -402,8 +402,33 @@ public:
 	}
 };
 
+struct CRememberTheFileStatus
+{
+	CFileStatus FileStatus;
+	bool Success;
+	void cb(const std::string &/*filePath*/, const CFileStatus &fileStatus, bool success)
+	{
+		FileStatus = fileStatus;
+		Success = success;
+	}
+};
+
 } /* anonymous namespace */
 
+/// Updates a file status synchronously. Returns the result in metaStatus.
+bool CDatabaseStatus::updateFileStatus(CFileStatus &metaStatus, const std::string &filePath)
+{
+	CRememberTheFileStatus result;
+	CUpdateFileStatus *ufs = new CUpdateFileStatus();
+	ufs->StatusMutex = &m_StatusMutex;
+	ufs->FilePath = filePath;
+	ufs->Callback = TFileStatusCallback(&result, &CRememberTheFileStatus::cb);
+	ufs->run();
+	ufs = NULL; // deleted by run
+	metaStatus = result.FileStatus;
+	return result.Success;
+}
+
 IRunnable *CDatabaseStatus::updateFileStatus(const TFileStatusCallback &callback, const std::string &filePath)
 {
 	/*if (!g_IsMaster)
@@ -620,10 +645,10 @@ void updateDirectoryStatus(CDatabaseStatus* ds, CDatabaseStatusUpdater &updater,
 	// <-- END REMOVE
 }
 
-} /* anonymous namespace */
-
 void dummyFileStatusCallback(const std::string &/*filePath*/, const CFileStatus &/*fileStatus*/, bool /*success*/) { }
 
+} /* anonymous namespace */
+
 void CDatabaseStatus::updateDatabaseStatus(const CCallback<void> &callback)
 {
 	/*if (!g_IsMaster)

code/nel/tools/pipeline/service/database_status.h

@@ -69,14 +69,16 @@ public:
 	
 	/// Tries to read the last file status. Return false if the status is invalid. Call updateFileStatus if the result is false to update asynchronously.
 	bool getFileStatus(CFileStatus &fileStatus, const std::string &filePath) const;
-	/// Updates the file status asynchronously. The new file status is broadcast to clients and slaves afterwards. Warning: If g_IsExiting during callback then update likely did not happen.
+	/// Updates a file status synchronously. Returns the result in metaStatus. filepPath is NOT a macro path.
+	bool updateFileStatus(CFileStatus &metaStatus, const std::string &filePath);
+	/// Updates the file status asynchronously. Warning: If g_IsExiting during callback then update likely did not happen.
 	NLMISC::IRunnable *updateFileStatus(const TFileStatusCallback &callback, const std::string &filePath);
 	/// Runs an update of the complete {{DatabaseDirectory}} status asynchronously. Warning: If g_IsExiting during callback then update is incomplete. Callback is always called when done (or failed).
 	void updateDatabaseStatus(const CCallback<void> &callback);
 	/// Runs an update of the file status of given paths asynchronously. Warning: If g_IsExiting during callback then update is incomplete. Callback is always called when done (or failed). Do NOT use the wait parameter. Do NOT use recurse, please. Recurse directories beforehand. Paths may contain db and pl macros.
 	void updateDatabaseStatus(const CCallback<void> &callback, const TFileStatusCallback &fileStatusCallback, const std::vector<std::string> &paths, bool wait = false, bool recurse = false);
 	/// Gets the last file statuses of given paths in a map. Directories are scanned for files, non recursively. Returns false if one of the statuses is bad (not updated; file changed inbetween). Considered as build error.
-	bool getFileStatus(std::map<std::string, CFileStatus> &fileStatusMap, std::map<std::string, CFileRemove> &fileRemoveMap, const std::vector<std::string> &paths);
+	// bool getFileStatus(std::map<std::string, CFileStatus> &fileStatusMap, std::map<std::string, CFileRemove> &fileRemoveMap, const std::vector<std::string> &paths);
 	/// Gets all removed files
 	bool getRemoved(std::map<std::string, CFileRemove> &fileRemoveMap, const std::vector<std::string> &paths);
 

code/nel/tools/pipeline/service/metadata_storage.h

@@ -178,8 +178,8 @@ public:
 	// static void eraseRemove(const std::string &path);
 
 	static std::string getDependPath(const std::string &file);
-	static bool readDepend(CFileDepend &status, const std::string &metaPath);
+	static bool readDepend(CFileDepend &depend, const std::string &metaPath);
-	static void writeDepend(const CFileDepend &status, const std::string &metaPath);
+	static void writeDepend(const CFileDepend &depend, const std::string &metaPath);
 
 	// Pathname for result metadata is like .../project.projectname.meta/pluginname.result
 	static std::string getResultPath(const std::string &projectName, const std::string &pluginName);

code/nel/tools/pipeline/service/module_pipeline_slave.cpp

@@ -713,6 +713,7 @@ public:
 		for (std::vector<std::string>::const_iterator it = inputPaths.begin(), end = inputPaths.end(); it != end; ++it)
 		{
 			const std::string &path = *it;
+			// FIXME: What if an input file does not exist? Deleted long ago? Just print a warning (& notify terminal), keep calm, and carry on.
 			nlassert(!CFile::isDirectory(path)); // All input are files! Coding error otherwise, because should already be checked.
 			std::string metaOutputPath = CMetadataStorage::getOutputPath(path, m_ActiveProject->getName(), m_ActivePlugin.Handler);
 			nlassert(CFile::fileExists(metaOutputPath)); // Coding error otherwise, already must have checked beforehand that they all exist.
@@ -844,134 +845,108 @@ public:
 
 		// Sanity check of all the output paths
 		// These must all be files, no directories allowed
+		for (std::vector<std::string>::const_iterator it = outputPaths.begin(), end = outputPaths.end(); it != end; ++it)
+		{
+			const std::string &path = *it;
+			if (path[path.size() - 1] == '/') // isDirectory
+			{
+				m_SubTaskResult = FINISH_ERROR;
+				m_SubTaskErrorMessage = std::string("Output file '") + path + "' cannot be a directory";
+				return false; // Error, cannot rebuild.
+			}
+		}
 
 		// Check if any of the output paths are part of the removed output files
-			// If so, rebuild
+		for (std::vector<std::string>::const_iterator it = outputPaths.begin(), end = outputPaths.end(); it != end; ++it)
-		// Check if any of the output files don't exist
-			// If so, rebuild
-		
-		// bool outputChanged = Check if any of the output paths are part of the changed output files
-		
-		// Check the .depend files of all the output files // also check that they exist :)
-			// If outputChanged
-				// Compare the output checksum with the cached output checksum
-			// If inputModified
-				// Compare the input checksums with the cached input checksums
-		// (if any checksum was different, require rebuild, if no checksums were different, no rebuild is needed)
-	}
-
-	/// Returns false if the file does not need to be built, or if an error occured.
-	/// Must verify needsExit() afterwards.
-	/// Input paths may be files or directories.
-	/// Output paths can ONLY be files!
-	bool needsToBeRebuiltOld(const std::vector<std::string> &inputPaths, const std::vector<std::string> &outputPaths)
-	{
-		// TODO: REWRITE THIS
-
-		if (m_SubTaskResult != FINISH_SUCCESS)
-			return false; // Cannot continue on previous failure.
-
-		m_SubTaskResult = FINISH_NOT;
-		for (std::vector<std::string>::const_iterator it = inputPaths.begin(), end = inputPaths.end(); it != end; ++it)
 		{
 			const std::string &path = *it;
-			if (path[path.size() - 1] == '/') // isDirectory
+			if (m_ListOutputRemoved.find(*it) != m_ListOutputRemoved.end())
 			{
-				// Check if this directory is in the dependencies.
+				// If so, rebuild
-				if (!isDirectoryDependency(path))
+				nldebug("Output file '%s' has been removed, rebuild", path.c_str());
-				{
+				m_SubTaskResult = FINISH_SUCCESS;
-					m_SubTaskResult = FINISH_ERROR;
+				return true; // Rebuild.
-					m_SubTaskErrorMessage = std::string("Directory '") + path + "' is not part of the dependencies";
-					return false; // Error, cannot rebuild.
-				}
-				// Check if any files added/changed/removed are part of this directory (slow).
-				for (std::set<std::string>::const_iterator itr = m_ListInputAdded.begin(), endr = m_ListInputAdded.end(); itr != endr; ++itr)
-				{
-					const std::string &pathr = *it;
-					if ((pathr.size() > path.size())
-						&& (pathr.substr(0, path.size()) == path) // inside the path
-						&& (pathr.substr(path.size(), pathr.size() - path.size())).find('/') == std::string::npos) // not in a further subdirectory 
-					{
-						nldebug("Found added '%s' in dependency directory '%s', rebuild", pathr.c_str(), path.c_str());
-						m_SubTaskResult = FINISH_SUCCESS;
-						return true; // Rebuild.
-					}
-				}
-				for (std::set<std::string>::const_iterator itr = m_ListInputChanged.begin(), endr = m_ListInputChanged.end(); itr != endr; ++itr)
-				{
-					const std::string &pathr = *it;
-					if ((pathr.size() > path.size())
-						&& (pathr.substr(0, path.size()) == path) // inside the path
-						&& (pathr.substr(path.size(), pathr.size() - path.size())).find('/') == std::string::npos) // not in a further subdirectory 
-					{
-						nldebug("Found changed '%s' in dependency directory '%s', rebuild", pathr.c_str(), path.c_str());
-						m_SubTaskResult = FINISH_SUCCESS;
-						return true; // Rebuild.
-					}
-				}
-				for (std::set<std::string>::const_iterator itr = m_ListInputRemoved.begin(), endr = m_ListInputRemoved.end(); itr != endr; ++itr)
-				{
-					const std::string &pathr = *it;
-					if ((pathr.size() > path.size())
-						&& (pathr.substr(0, path.size()) == path) // inside the path
-						&& (pathr.substr(path.size(), pathr.size() - path.size())).find('/') == std::string::npos) // not in a further subdirectory 
-					{
-						nldebug("Found removed '%s' in dependency directory '%s', rebuild", pathr.c_str(), path.c_str());
-						m_SubTaskResult = FINISH_SUCCESS;
-						return true; // Rebuild.
-					}
-				}
 			}
-			else // isFile
+		}
+
+		// Check if any of the output files don't exist
+		for (std::vector<std::string>::const_iterator it = outputPaths.begin(), end = outputPaths.end(); it != end; ++it)
+		{
+			// If so, rebuild
+			const std::string &path = *it;
+			if (!CFile::isExists(path))
 			{
-				// Check if this file is in the dependencies.
+				// If so, rebuild
-				if (!isFileDependency(path))
+				nldebug("Output file '%s' does not exist, rebuild", path.c_str());
-				{
+				m_SubTaskResult = FINISH_SUCCESS;
-					m_SubTaskResult = FINISH_ERROR;
+				return true; // Rebuild.
-					m_SubTaskErrorMessage = std::string("File '") + path + "' is not part of the dependencies";
-					return false; // Error, cannot rebuild.
-				}
-				// Check if this file is in added/changed/removed.
-				if (m_ListInputAdded.find(path) != m_ListInputAdded.end()
-					|| m_ListInputChanged.find(path) != m_ListInputChanged.end()
-					|| m_ListInputRemoved.find(path) != m_ListInputRemoved.end())
-				{
-					// Found!
-					nldebug("Found added/changed/removed input file '%s', rebuild", path.c_str());
-					m_SubTaskResult = FINISH_SUCCESS;
-					return true; // Rebuild.
-				}
 			}
 		}
+		
+		// Check if any of the output paths are part of the changed output files
+		bool outputChanged = false; 
 		for (std::vector<std::string>::const_iterator it = outputPaths.begin(), end = outputPaths.end(); it != end; ++it)
 		{
 			const std::string &path = *it;
-			if (path[path.size() - 1] == '/') // isDirectory
+			if (m_ListOutputChanged.find(*it) != m_ListOutputChanged.end())
 			{
-				m_SubTaskResult = FINISH_ERROR;
+				nldebug("Output file '%s' has been changed", path.c_str());
-				m_SubTaskErrorMessage = std::string("Output file '") + path + "' cannot be a directory";
+				outputChanged = true;
-				return false; // Error, cannot rebuild.
+				break;
 			}
-			if (m_ListOutputRemoved.find(path) != m_ListOutputRemoved.end())
+		}
+
+		if (!outputChanged && !inputModified)
+		{
+			nlerror("Should never reach this, this must have been cought earlier normally");
+		}
+		
+		// Check the .depend files of all the output files // also check that they exist :)
+		for (std::vector<std::string>::const_iterator it = outputPaths.begin(), end = outputPaths.end(); it != end; ++it)
+		{
+			const std::string &path = *it;
+			std::string metaDependPath = CMetadataStorage::getDependPath(path);
+			CFileDepend metaDepend;
+			if (!CMetadataStorage::readDepend(metaDepend, metaDependPath))
 			{
-				nldebug("Found removed output file '%s', rebuild", path.c_str());
+				nlwarning("Depend file for existing output '%s' does not exist, this should not happen, rebuild", path.c_str());
 				m_SubTaskResult = FINISH_SUCCESS;
-				return true;
+				return true; // Rebuild.
 			}
-			if (m_ListOutputChanged.find(path) != m_ListOutputChanged.end())
+			else
 			{
-				nlwarning("Changed output files not implemented yet. Previous build was likely incomplete. Please wipe the output directory. Rebuilding anyways");
+				if (outputChanged)
-				// For now always rebuild and don't check if the output file is up-to-date from a previous incomplete build.
+				{
-				nldebug("Found changed output file '%s', rebuild", path.c_str());
+					// Compare the output checksum with the status output checksum
-				m_SubTaskResult = FINISH_SUCCESS;
+					CFileStatus metaStatus;
-				return true;
+					if (!CDatabaseStatus::updateFileStatus(metaStatus, path))
-				//m_SubTaskResult = FINISH_ERROR;
+					{
-				//m_SubTaskErrorMessage = std::string("Changed output files not implemented yet. Previous build was likely incomplete. Please wipe the output directory");
+						m_SubTaskResult = FINISH_ERROR;
-				//return false; // Error, cannot rebuild.
+						m_SubTaskErrorMessage = std::string("Could not get status for output file '") + path + "'";
+						return false; // Error, cannot rebuild.
+					}
+					else
+					{
+						if (metaStatus.CRC32 != metaDepend.CRC32)
+						{
+							nlwarning("Status checksum for '%s' does match depend checksum, output file was modified, this should not happen, rebuild", path.c_str());
+							m_SubTaskResult = FINISH_SUCCESS;
+							return true; // Rebuild.
+						}
+					}
+				}
+				if (inputModified)
+				{
+					// Compare the input checksums with the cached input checksums
+
+				}
 			}
 		}
+
+		// (if any checksum was different, require rebuild, if no checksums were different, no rebuild is needed)
+		nldebug("No differences found, no rebuild needed");
 		m_SubTaskResult = FINISH_SUCCESS;
-		return false; // Does not need rebuild.
+		return false; // Rebuild not necessary.
 	}
 
 	/// Set the exit message, exit the plugin immediately afterwards.