Added: #1440 Parse depend log and write depend meta files

--HG--
branch : build_pipeline_v3

code/nel/tools/pipeline/plugin_library/pipeline_process.h

@@ -103,7 +103,7 @@ public:
 	virtual void makePaths(const std::vector<std::string> &outputPaths) = 0;
 	void makePaths(const std::string &outputPath) { std::vector<std::string> outputPaths; outputPaths.push_back(outputPath); makePaths(outputPaths); }
 
-	/// Parse the depend and error logs. Only set writeOutputMeta true if the output is not known in advance by the plugin, see needsToBeRebuilt
+	/// Parse the depend and error logs. Only set writeOutputMeta true if the output is not known in advance by the plugin, see needsToBeRebuilt. The log files are deleted for you afterwards
 	virtual void parseToolLog(const std::string &dependLogFile, const std::string &errorLogFile, bool writeOutputMeta) = 0;
 
 	/// Check if the plugin needs to exit, exit plugin immediately if true.

code/nel/tools/pipeline/plugin_nel/process_interface.cpp

@@ -97,7 +97,7 @@ void CProcessInterface::build()
 				{
 					m_PipelineProcess->makePaths(dstFiles);
 					buildAtlas(dependLog, errorLog, srcDirectories, dstFile);
-					m_PipelineProcess->parseToolLog("", "", false);
+					m_PipelineProcess->parseToolLog(dependLog, errorLog, false);
 				}
 				if (m_PipelineProcess->needsExit()) return;
 			}

code/nel/tools/pipeline/service/pipeline_process_impl.cpp

@@ -102,16 +102,16 @@ std::string CPipelineProcessImpl::getTempDirectory()
 
 void CPipelineProcessImpl::deleteDirectoryIfEmpty(const std::string &path)
 {
-	std::vector<std::string> dummy;
+	/*std::vector<std::string> dummy;
 	NLMISC::CPath::getPathContent(path, false, true, true, dummy);
 	if (dummy.size())
 	{
 		nlwarning("Directory '%s' not empty", path.c_str());
 	}
 	else
-	{
+	{*/
 		NLMISC::CFile::deleteDirectory(path);
-	}
+	/*}*/
 }
 
 bool CPipelineProcessImpl::getValue(std::string &result, const std::string &name)

code/nel/tools/pipeline/service/pipeline_process_impl_buildinfo.cpp

@@ -456,7 +456,7 @@ bool CPipelineProcessImpl::needsToBeRebuiltSub(const std::vector<std::string> &i
 				{
 					if (metaStatus.CRC32 != metaDepend.CRC32)
 					{
-						nlwarning("Status checksum for '%s' does match depend checksum, output file was modified, this should not happen, rebuild", path.c_str());
+						nlwarning("Status checksum %i for output file '%s' does match depend checksum %i, output file was modified, this should not happen, rebuild", metaStatus.CRC32, path.c_str(), metaDepend.CRC32);
 						m_SubTaskResult = FINISH_SUCCESS;
 						return true; // Rebuild.
 					}
@@ -478,9 +478,9 @@ bool CPipelineProcessImpl::needsToBeRebuiltSub(const std::vector<std::string> &i
 					}
 					else
 					{
-						if (metaStatus.CRC32 != metaDepend.CRC32)
+						if (metaStatus.CRC32 != dependency.CRC32)
 						{
-							nldebug("Status checksum for '%s' does match depend checksum, input file was modified, rebuild", path.c_str());
+							nldebug("Status checksum %i for input file '%s' does match depend checksum %i, input file was modified, rebuild", metaStatus.CRC32, dependencyFile.c_str(), dependency.CRC32);
 							m_SubTaskResult = FINISH_SUCCESS;
 							return true; // Rebuild.
 						}
@@ -506,8 +506,11 @@ void CPipelineProcessImpl::setExit(const TProcessResult exitLevel, const std::st
 /// Must verify this regularly to exit the plugin in case something went wrong.
 bool CPipelineProcessImpl::needsExit()
 {
+	if (g_IsExiting)
+		return true;
+	
 	// TODO: Bypass error feature.
-	if (m_SubTaskResult != FINISH_SUCCESS)
+	if (m_SubTaskResult != FINISH_SUCCESS) // m_SubTaskResult can only be FINISH_SUCCESS or FINISH_ERROR (or FINISH_NOT in case of incomplete implementation)
 		return true;
 	return false;
 }

code/nel/tools/pipeline/service/pipeline_process_impl_toollog.cpp

@@ -35,20 +35,200 @@
 #include <nel/misc/time_nl.h>
 #include <nel/misc/app_context.h>
 #include <nel/misc/debug.h>
+#include <nel/misc/file.h>
+#include <nel/misc/path.h>
+#include <nel/misc/tool_logger.h>
 
 // Project includes
 #include "pipeline_service.h"
 #include "pipeline_project.h"
+#include "metadata_storage.h"
+#include "database_status.h"
 
 using namespace std;
 // using namespace NLMISC;
 
 namespace PIPELINE {
 
+namespace {
+	
+void readTabbedLine(std::vector<std::string> &tabbedLine, NLMISC::CIFile &file)
+{
+	char r;
+	file.serial(r);
+	char buffer[4096];
+	ptrdiff_t idx = 0;
+	while (r != '\n')
+	{
+		switch (r)
+		{
+		case '\r':
+			break;
+		case '\t':
+			buffer[idx] = '\0';
+			tabbedLine.push_back(std::string(buffer, idx));
+			idx = 0;
+			break;
+		default:
+			buffer[idx] = r;
+			++idx;
+			if (idx >= 4096)
+				return; // bad
+			break;
+		}
+		if (file.eof())
+			break;
+		file.serial(r);
+	}
+	buffer[idx] = '\0';
+	tabbedLine.push_back(std::string(buffer, idx));
+}
+
+} /* anonymous namespace */
+
 void CPipelineProcessImpl::parseToolLog(const std::string &dependLogFile, const std::string &errorLogFile, bool writeOutputMeta)
 {
-	m_SubTaskErrorMessage = "Log parsing not implemented, goodbye";
+	m_SubTaskResult = FINISH_NOT;
+
+	// Parse error log
+	{
+		// ...
+	}
+
+	// Parse depend log
+	{
+		std::map<std::string, CFileDepend> metaDepends;
+		std::map<std::string, CFileOutput> metaOutputs;
+		std::map<std::string, CFileStatus> statusCache;
+		
+		NLMISC::CIFile file;
+		file.open(dependLogFile, true);
+
+		std::vector<std::string> tabbedLine;
+		readTabbedLine(tabbedLine, file);
+		uint line = 0;
+
+		if (tabbedLine.size() != 3 || tabbedLine[0] != "type" || tabbedLine[1] != "output_file" || tabbedLine[2] != "input_file")
+		{
+			m_SubTaskErrorMessage = "Bad depend file format";
+			m_SubTaskResult = FINISH_ERROR;
+			file.close();
+			return;
+		}
+		
+		for (; ; )
+		{
+			if (file.eof())
+				break;
+			tabbedLine.clear();
+			readTabbedLine(tabbedLine, file);
+			++line;
+			if (tabbedLine.size() == 0)
+				continue;
+			if (tabbedLine.size() != 3)
+			{
+				std::stringstream ss;
+				ss << "Bad line " << line << " in depend file";
+				m_SubTaskErrorMessage = ss.str();
+				m_SubTaskResult = FINISH_ERROR;
+				file.close();
+				return;
+			}
+			// Read line
+			{
+				TDepend type;
+				if (tabbedLine[0] == "BUILD")
+					type = BUILD;
+				else if (tabbedLine[0] == "RUNTIME")
+					type = RUNTIME;
+				else
+				{
+					std::stringstream ss;
+					ss << "Invalid type at line " << line << " in depend file";
+					m_SubTaskErrorMessage = ss.str();
+					m_SubTaskResult = FINISH_ERROR;
+					file.close();
+					return;
+				}
+				std::string outputFile = standardizePath(tabbedLine[1], false);
+				std::string outputFileMacro = macroPath(outputFile);
+				std::string inputFile = standardizePath(tabbedLine[2], false);
+				std::string inputFileMacro = macroPath(inputFile);
+				std::map<std::string, CFileDepend>::iterator metaDependIt = metaDepends.find(outputFile);
+				if (metaDependIt == metaDepends.end())
+				{
+					metaDepends[outputFile] = CFileDepend();
+					metaDependIt = metaDepends.find(outputFile);
+					CFileStatus status;
+					nldebug("Update status for output file '%s', calculate checksum", outputFile.c_str());
+					g_DatabaseStatus->updateFileStatus(status, outputFile); // calculate the checksum of the output, this takes a while
+					metaDependIt->second.CRC32 = status.CRC32;
+					// nldebug("Checksum %i; %i", metaDependIt->second.CRC32, status.CRC32);
+
+					m_ResultCurrent.MacroPaths.push_back(outputFileMacro);
+					CProcessResult::CFileResult prfr;
+					prfr.Level = STATE_SUCCESS; // dunno if still needed?
+					prfr.CRC32 = status.CRC32;
+					m_ResultCurrent.FileResults.push_back(prfr);
+				}
+				CFileDepend::CDependency dependency;
+				dependency.MacroPath = inputFileMacro;
+				std::map<std::string, CFileStatus>::iterator statusIt = statusCache.find(inputFile);
+				if (statusIt == statusCache.end())
+				{
+					if (!isFileDependency(inputFile))
+					{
+						m_SubTaskErrorMessage = std::string("Invalid dependency '") + inputFile + "'";
+						m_SubTaskResult = FINISH_ERROR;
+						file.close();
+						return;
+					}
+					CFileStatus statusOriginal;
+					if (!getDependencyFileStatusCached(statusOriginal, inputFile))
+					{
+						m_SubTaskErrorMessage = std::string("Cached status for '") + inputFile + "' does not exist, this may be a programming error";
+						m_SubTaskResult = FINISH_ERROR;
+						file.close();
+						return;
+					}
+					CFileStatus status;
+					if (!getDependencyFileStatusLatest(status, inputFile))
+					{
+						m_SubTaskErrorMessage = std::string("Invalid status for '") + inputFile + "', file may have changed during build";
 						m_SubTaskResult = FINISH_ERROR;
+						file.close();
+						return;
+					}
+					if (statusOriginal.CRC32 != status.CRC32)
+					{
+						m_SubTaskErrorMessage = std::string("Status checksums changed for '") + inputFile + "', file has changed during build";
+						m_SubTaskResult = FINISH_ERROR;
+						file.close();
+						return;
+					}
+					statusCache[inputFile] = status;
+					statusIt = statusCache.find(inputFile);
+				}
+				dependency.CRC32 = statusIt->second.CRC32;
+				metaDependIt->second.Dependencies.push_back(dependency);
+			}
+		}
+
+		file.close();
+
+		// Write depend meta files
+		for (std::map<std::string, CFileDepend>::iterator it = metaDepends.begin(), end = metaDepends.end(); it != end; ++it)
+		{
+			const CFileDepend &depend = it->second;
+			CMetadataStorage::writeDepend(depend, CMetadataStorage::getDependPath(it->first));
+		}
+	}
+
+	NLMISC::CFile::deleteFile(dependLogFile);
+	
+	// m_SubTaskErrorMessage = "Log parsing not implemented, goodbye";
+	// m_SubTaskResult = FINISH_ERROR;
+	m_SubTaskResult = FINISH_SUCCESS;
 }
 
 } /* namespace PIPELINE */