From a28341d04c2661676f59c78746bfd4b80b51982a Mon Sep 17 00:00:00 2001 From: kaetemi Date: Sat, 4 Aug 2012 12:12:26 +0200 Subject: [PATCH] Added: #1440 Parse depend log and write depend meta files --HG-- branch : build_pipeline_v3 --- .../plugin_library/pipeline_process.h | 2 +- .../pipeline/plugin_nel/process_interface.cpp | 2 +- .../service/pipeline_process_impl.cpp | 6 +- .../pipeline_process_impl_buildinfo.cpp | 11 +- .../service/pipeline_process_impl_toollog.cpp | 184 +++++++++++++++++- 5 files changed, 194 insertions(+), 11 deletions(-) diff --git a/code/nel/tools/pipeline/plugin_library/pipeline_process.h b/code/nel/tools/pipeline/plugin_library/pipeline_process.h index b546de19f..5d87f7ff3 100644 --- a/code/nel/tools/pipeline/plugin_library/pipeline_process.h +++ b/code/nel/tools/pipeline/plugin_library/pipeline_process.h @@ -103,7 +103,7 @@ public: virtual void makePaths(const std::vector &outputPaths) = 0; void makePaths(const std::string &outputPath) { std::vector outputPaths; outputPaths.push_back(outputPath); makePaths(outputPaths); } - /// Parse the depend and error logs. Only set writeOutputMeta true if the output is not known in advance by the plugin, see needsToBeRebuilt + /// Parse the depend and error logs. Only set writeOutputMeta true if the output is not known in advance by the plugin, see needsToBeRebuilt. The log files are deleted for you afterwards virtual void parseToolLog(const std::string &dependLogFile, const std::string &errorLogFile, bool writeOutputMeta) = 0; /// Check if the plugin needs to exit, exit plugin immediately if true. diff --git a/code/nel/tools/pipeline/plugin_nel/process_interface.cpp b/code/nel/tools/pipeline/plugin_nel/process_interface.cpp index cc76bbebc..cfd434cdf 100644 --- a/code/nel/tools/pipeline/plugin_nel/process_interface.cpp +++ b/code/nel/tools/pipeline/plugin_nel/process_interface.cpp @@ -97,7 +97,7 @@ void CProcessInterface::build() { m_PipelineProcess->makePaths(dstFiles); buildAtlas(dependLog, errorLog, srcDirectories, dstFile); - m_PipelineProcess->parseToolLog("", "", false); + m_PipelineProcess->parseToolLog(dependLog, errorLog, false); } if (m_PipelineProcess->needsExit()) return; } diff --git a/code/nel/tools/pipeline/service/pipeline_process_impl.cpp b/code/nel/tools/pipeline/service/pipeline_process_impl.cpp index 49ef18f6c..e76cbb04a 100644 --- a/code/nel/tools/pipeline/service/pipeline_process_impl.cpp +++ b/code/nel/tools/pipeline/service/pipeline_process_impl.cpp @@ -102,16 +102,16 @@ std::string CPipelineProcessImpl::getTempDirectory() void CPipelineProcessImpl::deleteDirectoryIfEmpty(const std::string &path) { - std::vector dummy; + /*std::vector dummy; NLMISC::CPath::getPathContent(path, false, true, true, dummy); if (dummy.size()) { nlwarning("Directory '%s' not empty", path.c_str()); } else - { + {*/ NLMISC::CFile::deleteDirectory(path); - } + /*}*/ } bool CPipelineProcessImpl::getValue(std::string &result, const std::string &name) diff --git a/code/nel/tools/pipeline/service/pipeline_process_impl_buildinfo.cpp b/code/nel/tools/pipeline/service/pipeline_process_impl_buildinfo.cpp index 6765379a3..e7f80ff18 100644 --- a/code/nel/tools/pipeline/service/pipeline_process_impl_buildinfo.cpp +++ b/code/nel/tools/pipeline/service/pipeline_process_impl_buildinfo.cpp @@ -456,7 +456,7 @@ bool CPipelineProcessImpl::needsToBeRebuiltSub(const std::vector &i { if (metaStatus.CRC32 != metaDepend.CRC32) { - nlwarning("Status checksum for '%s' does match depend checksum, output file was modified, this should not happen, rebuild", path.c_str()); + nlwarning("Status checksum %i for output file '%s' does match depend checksum %i, output file was modified, this should not happen, rebuild", metaStatus.CRC32, path.c_str(), metaDepend.CRC32); m_SubTaskResult = FINISH_SUCCESS; return true; // Rebuild. } @@ -478,9 +478,9 @@ bool CPipelineProcessImpl::needsToBeRebuiltSub(const std::vector &i } else { - if (metaStatus.CRC32 != metaDepend.CRC32) + if (metaStatus.CRC32 != dependency.CRC32) { - nldebug("Status checksum for '%s' does match depend checksum, input file was modified, rebuild", path.c_str()); + nldebug("Status checksum %i for input file '%s' does match depend checksum %i, input file was modified, rebuild", metaStatus.CRC32, dependencyFile.c_str(), dependency.CRC32); m_SubTaskResult = FINISH_SUCCESS; return true; // Rebuild. } @@ -506,8 +506,11 @@ void CPipelineProcessImpl::setExit(const TProcessResult exitLevel, const std::st /// Must verify this regularly to exit the plugin in case something went wrong. bool CPipelineProcessImpl::needsExit() { + if (g_IsExiting) + return true; + // TODO: Bypass error feature. - if (m_SubTaskResult != FINISH_SUCCESS) + if (m_SubTaskResult != FINISH_SUCCESS) // m_SubTaskResult can only be FINISH_SUCCESS or FINISH_ERROR (or FINISH_NOT in case of incomplete implementation) return true; return false; } diff --git a/code/nel/tools/pipeline/service/pipeline_process_impl_toollog.cpp b/code/nel/tools/pipeline/service/pipeline_process_impl_toollog.cpp index 4324d488e..b6f9cb156 100644 --- a/code/nel/tools/pipeline/service/pipeline_process_impl_toollog.cpp +++ b/code/nel/tools/pipeline/service/pipeline_process_impl_toollog.cpp @@ -35,20 +35,200 @@ #include #include #include +#include +#include +#include // Project includes #include "pipeline_service.h" #include "pipeline_project.h" +#include "metadata_storage.h" +#include "database_status.h" using namespace std; // using namespace NLMISC; namespace PIPELINE { +namespace { + +void readTabbedLine(std::vector &tabbedLine, NLMISC::CIFile &file) +{ + char r; + file.serial(r); + char buffer[4096]; + ptrdiff_t idx = 0; + while (r != '\n') + { + switch (r) + { + case '\r': + break; + case '\t': + buffer[idx] = '\0'; + tabbedLine.push_back(std::string(buffer, idx)); + idx = 0; + break; + default: + buffer[idx] = r; + ++idx; + if (idx >= 4096) + return; // bad + break; + } + if (file.eof()) + break; + file.serial(r); + } + buffer[idx] = '\0'; + tabbedLine.push_back(std::string(buffer, idx)); +} + +} /* anonymous namespace */ + void CPipelineProcessImpl::parseToolLog(const std::string &dependLogFile, const std::string &errorLogFile, bool writeOutputMeta) { - m_SubTaskErrorMessage = "Log parsing not implemented, goodbye"; - m_SubTaskResult = FINISH_ERROR; + m_SubTaskResult = FINISH_NOT; + + // Parse error log + { + // ... + } + + // Parse depend log + { + std::map metaDepends; + std::map metaOutputs; + std::map statusCache; + + NLMISC::CIFile file; + file.open(dependLogFile, true); + + std::vector tabbedLine; + readTabbedLine(tabbedLine, file); + uint line = 0; + + if (tabbedLine.size() != 3 || tabbedLine[0] != "type" || tabbedLine[1] != "output_file" || tabbedLine[2] != "input_file") + { + m_SubTaskErrorMessage = "Bad depend file format"; + m_SubTaskResult = FINISH_ERROR; + file.close(); + return; + } + + for (; ; ) + { + if (file.eof()) + break; + tabbedLine.clear(); + readTabbedLine(tabbedLine, file); + ++line; + if (tabbedLine.size() == 0) + continue; + if (tabbedLine.size() != 3) + { + std::stringstream ss; + ss << "Bad line " << line << " in depend file"; + m_SubTaskErrorMessage = ss.str(); + m_SubTaskResult = FINISH_ERROR; + file.close(); + return; + } + // Read line + { + TDepend type; + if (tabbedLine[0] == "BUILD") + type = BUILD; + else if (tabbedLine[0] == "RUNTIME") + type = RUNTIME; + else + { + std::stringstream ss; + ss << "Invalid type at line " << line << " in depend file"; + m_SubTaskErrorMessage = ss.str(); + m_SubTaskResult = FINISH_ERROR; + file.close(); + return; + } + std::string outputFile = standardizePath(tabbedLine[1], false); + std::string outputFileMacro = macroPath(outputFile); + std::string inputFile = standardizePath(tabbedLine[2], false); + std::string inputFileMacro = macroPath(inputFile); + std::map::iterator metaDependIt = metaDepends.find(outputFile); + if (metaDependIt == metaDepends.end()) + { + metaDepends[outputFile] = CFileDepend(); + metaDependIt = metaDepends.find(outputFile); + CFileStatus status; + nldebug("Update status for output file '%s', calculate checksum", outputFile.c_str()); + g_DatabaseStatus->updateFileStatus(status, outputFile); // calculate the checksum of the output, this takes a while + metaDependIt->second.CRC32 = status.CRC32; + // nldebug("Checksum %i; %i", metaDependIt->second.CRC32, status.CRC32); + + m_ResultCurrent.MacroPaths.push_back(outputFileMacro); + CProcessResult::CFileResult prfr; + prfr.Level = STATE_SUCCESS; // dunno if still needed? + prfr.CRC32 = status.CRC32; + m_ResultCurrent.FileResults.push_back(prfr); + } + CFileDepend::CDependency dependency; + dependency.MacroPath = inputFileMacro; + std::map::iterator statusIt = statusCache.find(inputFile); + if (statusIt == statusCache.end()) + { + if (!isFileDependency(inputFile)) + { + m_SubTaskErrorMessage = std::string("Invalid dependency '") + inputFile + "'"; + m_SubTaskResult = FINISH_ERROR; + file.close(); + return; + } + CFileStatus statusOriginal; + if (!getDependencyFileStatusCached(statusOriginal, inputFile)) + { + m_SubTaskErrorMessage = std::string("Cached status for '") + inputFile + "' does not exist, this may be a programming error"; + m_SubTaskResult = FINISH_ERROR; + file.close(); + return; + } + CFileStatus status; + if (!getDependencyFileStatusLatest(status, inputFile)) + { + m_SubTaskErrorMessage = std::string("Invalid status for '") + inputFile + "', file may have changed during build"; + m_SubTaskResult = FINISH_ERROR; + file.close(); + return; + } + if (statusOriginal.CRC32 != status.CRC32) + { + m_SubTaskErrorMessage = std::string("Status checksums changed for '") + inputFile + "', file has changed during build"; + m_SubTaskResult = FINISH_ERROR; + file.close(); + return; + } + statusCache[inputFile] = status; + statusIt = statusCache.find(inputFile); + } + dependency.CRC32 = statusIt->second.CRC32; + metaDependIt->second.Dependencies.push_back(dependency); + } + } + + file.close(); + + // Write depend meta files + for (std::map::iterator it = metaDepends.begin(), end = metaDepends.end(); it != end; ++it) + { + const CFileDepend &depend = it->second; + CMetadataStorage::writeDepend(depend, CMetadataStorage::getDependPath(it->first)); + } + } + + NLMISC::CFile::deleteFile(dependLogFile); + + // m_SubTaskErrorMessage = "Log parsing not implemented, goodbye"; + // m_SubTaskResult = FINISH_ERROR; + m_SubTaskResult = FINISH_SUCCESS; } } /* namespace PIPELINE */