Added: #1440 More of the needsToBeRebuilt function

--HG--
branch : build_pipeline_v3
hg/feature/build_pipeline_v3
kaetemi 12 years ago
parent fc6e3c0cab
commit 48e6bd4290

@ -74,7 +74,7 @@ bool CDatabaseStatus::getFileStatus(CFileStatus &fileStatus, const std::string &
m_StatusMutex.unlock_shared(); m_StatusMutex.unlock_shared();
return seemsValid; return seemsValid;
} }
/*
bool CDatabaseStatus::getFileStatus(std::map<std::string, CFileStatus> &fileStatusMap, std::map<std::string, CFileRemove> &fileRemoveMap, const std::vector<std::string> &paths) bool CDatabaseStatus::getFileStatus(std::map<std::string, CFileStatus> &fileStatusMap, std::map<std::string, CFileRemove> &fileRemoveMap, const std::vector<std::string> &paths)
{ {
nlassert(false); // not used nlassert(false); // not used
@ -193,7 +193,7 @@ bool CDatabaseStatus::getFileStatus(std::map<std::string, CFileStatus> &fileStat
// no issues occured apparently // no issues occured apparently
return true; return true;
} }
*/
bool CDatabaseStatus::getRemoved(std::map<std::string, CFileRemove> &fileRemoveMap, const std::vector<std::string> &paths) bool CDatabaseStatus::getRemoved(std::map<std::string, CFileRemove> &fileRemoveMap, const std::vector<std::string> &paths)
{ {
for (std::vector<std::string>::const_iterator it = paths.begin(), end = paths.end(); it != end; ++it) for (std::vector<std::string>::const_iterator it = paths.begin(), end = paths.end(); it != end; ++it)
@ -402,8 +402,33 @@ public:
} }
}; };
struct CRememberTheFileStatus
{
CFileStatus FileStatus;
bool Success;
void cb(const std::string &/*filePath*/, const CFileStatus &fileStatus, bool success)
{
FileStatus = fileStatus;
Success = success;
}
};
} /* anonymous namespace */ } /* anonymous namespace */
/// Updates a file status synchronously. Returns the result in metaStatus.
bool CDatabaseStatus::updateFileStatus(CFileStatus &metaStatus, const std::string &filePath)
{
CRememberTheFileStatus result;
CUpdateFileStatus *ufs = new CUpdateFileStatus();
ufs->StatusMutex = &m_StatusMutex;
ufs->FilePath = filePath;
ufs->Callback = TFileStatusCallback(&result, &CRememberTheFileStatus::cb);
ufs->run();
ufs = NULL; // deleted by run
metaStatus = result.FileStatus;
return result.Success;
}
IRunnable *CDatabaseStatus::updateFileStatus(const TFileStatusCallback &callback, const std::string &filePath) IRunnable *CDatabaseStatus::updateFileStatus(const TFileStatusCallback &callback, const std::string &filePath)
{ {
/*if (!g_IsMaster) /*if (!g_IsMaster)
@ -620,10 +645,10 @@ void updateDirectoryStatus(CDatabaseStatus* ds, CDatabaseStatusUpdater &updater,
// <-- END REMOVE // <-- END REMOVE
} }
} /* anonymous namespace */
void dummyFileStatusCallback(const std::string &/*filePath*/, const CFileStatus &/*fileStatus*/, bool /*success*/) { } void dummyFileStatusCallback(const std::string &/*filePath*/, const CFileStatus &/*fileStatus*/, bool /*success*/) { }
} /* anonymous namespace */
void CDatabaseStatus::updateDatabaseStatus(const CCallback<void> &callback) void CDatabaseStatus::updateDatabaseStatus(const CCallback<void> &callback)
{ {
/*if (!g_IsMaster) /*if (!g_IsMaster)

@ -69,14 +69,16 @@ public:
/// Tries to read the last file status. Return false if the status is invalid. Call updateFileStatus if the result is false to update asynchronously. /// Tries to read the last file status. Return false if the status is invalid. Call updateFileStatus if the result is false to update asynchronously.
bool getFileStatus(CFileStatus &fileStatus, const std::string &filePath) const; bool getFileStatus(CFileStatus &fileStatus, const std::string &filePath) const;
/// Updates the file status asynchronously. The new file status is broadcast to clients and slaves afterwards. Warning: If g_IsExiting during callback then update likely did not happen. /// Updates a file status synchronously. Returns the result in metaStatus. filepPath is NOT a macro path.
bool updateFileStatus(CFileStatus &metaStatus, const std::string &filePath);
/// Updates the file status asynchronously. Warning: If g_IsExiting during callback then update likely did not happen.
NLMISC::IRunnable *updateFileStatus(const TFileStatusCallback &callback, const std::string &filePath); NLMISC::IRunnable *updateFileStatus(const TFileStatusCallback &callback, const std::string &filePath);
/// Runs an update of the complete {{DatabaseDirectory}} status asynchronously. Warning: If g_IsExiting during callback then update is incomplete. Callback is always called when done (or failed). /// Runs an update of the complete {{DatabaseDirectory}} status asynchronously. Warning: If g_IsExiting during callback then update is incomplete. Callback is always called when done (or failed).
void updateDatabaseStatus(const CCallback<void> &callback); void updateDatabaseStatus(const CCallback<void> &callback);
/// Runs an update of the file status of given paths asynchronously. Warning: If g_IsExiting during callback then update is incomplete. Callback is always called when done (or failed). Do NOT use the wait parameter. Do NOT use recurse, please. Recurse directories beforehand. Paths may contain db and pl macros. /// Runs an update of the file status of given paths asynchronously. Warning: If g_IsExiting during callback then update is incomplete. Callback is always called when done (or failed). Do NOT use the wait parameter. Do NOT use recurse, please. Recurse directories beforehand. Paths may contain db and pl macros.
void updateDatabaseStatus(const CCallback<void> &callback, const TFileStatusCallback &fileStatusCallback, const std::vector<std::string> &paths, bool wait = false, bool recurse = false); void updateDatabaseStatus(const CCallback<void> &callback, const TFileStatusCallback &fileStatusCallback, const std::vector<std::string> &paths, bool wait = false, bool recurse = false);
/// Gets the last file statuses of given paths in a map. Directories are scanned for files, non recursively. Returns false if one of the statuses is bad (not updated; file changed inbetween). Considered as build error. /// Gets the last file statuses of given paths in a map. Directories are scanned for files, non recursively. Returns false if one of the statuses is bad (not updated; file changed inbetween). Considered as build error.
bool getFileStatus(std::map<std::string, CFileStatus> &fileStatusMap, std::map<std::string, CFileRemove> &fileRemoveMap, const std::vector<std::string> &paths); // bool getFileStatus(std::map<std::string, CFileStatus> &fileStatusMap, std::map<std::string, CFileRemove> &fileRemoveMap, const std::vector<std::string> &paths);
/// Gets all removed files /// Gets all removed files
bool getRemoved(std::map<std::string, CFileRemove> &fileRemoveMap, const std::vector<std::string> &paths); bool getRemoved(std::map<std::string, CFileRemove> &fileRemoveMap, const std::vector<std::string> &paths);

@ -178,8 +178,8 @@ public:
// static void eraseRemove(const std::string &path); // static void eraseRemove(const std::string &path);
static std::string getDependPath(const std::string &file); static std::string getDependPath(const std::string &file);
static bool readDepend(CFileDepend &status, const std::string &metaPath); static bool readDepend(CFileDepend &depend, const std::string &metaPath);
static void writeDepend(const CFileDepend &status, const std::string &metaPath); static void writeDepend(const CFileDepend &depend, const std::string &metaPath);
// Pathname for result metadata is like .../project.projectname.meta/pluginname.result // Pathname for result metadata is like .../project.projectname.meta/pluginname.result
static std::string getResultPath(const std::string &projectName, const std::string &pluginName); static std::string getResultPath(const std::string &projectName, const std::string &pluginName);

@ -713,6 +713,7 @@ public:
for (std::vector<std::string>::const_iterator it = inputPaths.begin(), end = inputPaths.end(); it != end; ++it) for (std::vector<std::string>::const_iterator it = inputPaths.begin(), end = inputPaths.end(); it != end; ++it)
{ {
const std::string &path = *it; const std::string &path = *it;
// FIXME: What if an input file does not exist? Deleted long ago? Just print a warning (& notify terminal), keep calm, and carry on.
nlassert(!CFile::isDirectory(path)); // All input are files! Coding error otherwise, because should already be checked. nlassert(!CFile::isDirectory(path)); // All input are files! Coding error otherwise, because should already be checked.
std::string metaOutputPath = CMetadataStorage::getOutputPath(path, m_ActiveProject->getName(), m_ActivePlugin.Handler); std::string metaOutputPath = CMetadataStorage::getOutputPath(path, m_ActiveProject->getName(), m_ActivePlugin.Handler);
nlassert(CFile::fileExists(metaOutputPath)); // Coding error otherwise, already must have checked beforehand that they all exist. nlassert(CFile::fileExists(metaOutputPath)); // Coding error otherwise, already must have checked beforehand that they all exist.
@ -844,134 +845,108 @@ public:
// Sanity check of all the output paths // Sanity check of all the output paths
// These must all be files, no directories allowed // These must all be files, no directories allowed
for (std::vector<std::string>::const_iterator it = outputPaths.begin(), end = outputPaths.end(); it != end; ++it)
// Check if any of the output paths are part of the removed output files
// If so, rebuild
// Check if any of the output files don't exist
// If so, rebuild
// bool outputChanged = Check if any of the output paths are part of the changed output files
// Check the .depend files of all the output files // also check that they exist :)
// If outputChanged
// Compare the output checksum with the cached output checksum
// If inputModified
// Compare the input checksums with the cached input checksums
// (if any checksum was different, require rebuild, if no checksums were different, no rebuild is needed)
}
/// Returns false if the file does not need to be built, or if an error occured.
/// Must verify needsExit() afterwards.
/// Input paths may be files or directories.
/// Output paths can ONLY be files!
bool needsToBeRebuiltOld(const std::vector<std::string> &inputPaths, const std::vector<std::string> &outputPaths)
{
// TODO: REWRITE THIS
if (m_SubTaskResult != FINISH_SUCCESS)
return false; // Cannot continue on previous failure.
m_SubTaskResult = FINISH_NOT;
for (std::vector<std::string>::const_iterator it = inputPaths.begin(), end = inputPaths.end(); it != end; ++it)
{ {
const std::string &path = *it; const std::string &path = *it;
if (path[path.size() - 1] == '/') // isDirectory if (path[path.size() - 1] == '/') // isDirectory
{
// Check if this directory is in the dependencies.
if (!isDirectoryDependency(path))
{ {
m_SubTaskResult = FINISH_ERROR; m_SubTaskResult = FINISH_ERROR;
m_SubTaskErrorMessage = std::string("Directory '") + path + "' is not part of the dependencies"; m_SubTaskErrorMessage = std::string("Output file '") + path + "' cannot be a directory";
return false; // Error, cannot rebuild. return false; // Error, cannot rebuild.
} }
// Check if any files added/changed/removed are part of this directory (slow). }
for (std::set<std::string>::const_iterator itr = m_ListInputAdded.begin(), endr = m_ListInputAdded.end(); itr != endr; ++itr)
// Check if any of the output paths are part of the removed output files
for (std::vector<std::string>::const_iterator it = outputPaths.begin(), end = outputPaths.end(); it != end; ++it)
{ {
const std::string &pathr = *it; const std::string &path = *it;
if ((pathr.size() > path.size()) if (m_ListOutputRemoved.find(*it) != m_ListOutputRemoved.end())
&& (pathr.substr(0, path.size()) == path) // inside the path
&& (pathr.substr(path.size(), pathr.size() - path.size())).find('/') == std::string::npos) // not in a further subdirectory
{ {
nldebug("Found added '%s' in dependency directory '%s', rebuild", pathr.c_str(), path.c_str()); // If so, rebuild
nldebug("Output file '%s' has been removed, rebuild", path.c_str());
m_SubTaskResult = FINISH_SUCCESS; m_SubTaskResult = FINISH_SUCCESS;
return true; // Rebuild. return true; // Rebuild.
} }
} }
for (std::set<std::string>::const_iterator itr = m_ListInputChanged.begin(), endr = m_ListInputChanged.end(); itr != endr; ++itr)
// Check if any of the output files don't exist
for (std::vector<std::string>::const_iterator it = outputPaths.begin(), end = outputPaths.end(); it != end; ++it)
{ {
const std::string &pathr = *it; // If so, rebuild
if ((pathr.size() > path.size()) const std::string &path = *it;
&& (pathr.substr(0, path.size()) == path) // inside the path if (!CFile::isExists(path))
&& (pathr.substr(path.size(), pathr.size() - path.size())).find('/') == std::string::npos) // not in a further subdirectory
{ {
nldebug("Found changed '%s' in dependency directory '%s', rebuild", pathr.c_str(), path.c_str()); // If so, rebuild
nldebug("Output file '%s' does not exist, rebuild", path.c_str());
m_SubTaskResult = FINISH_SUCCESS; m_SubTaskResult = FINISH_SUCCESS;
return true; // Rebuild. return true; // Rebuild.
} }
} }
for (std::set<std::string>::const_iterator itr = m_ListInputRemoved.begin(), endr = m_ListInputRemoved.end(); itr != endr; ++itr)
// Check if any of the output paths are part of the changed output files
bool outputChanged = false;
for (std::vector<std::string>::const_iterator it = outputPaths.begin(), end = outputPaths.end(); it != end; ++it)
{ {
const std::string &pathr = *it; const std::string &path = *it;
if ((pathr.size() > path.size()) if (m_ListOutputChanged.find(*it) != m_ListOutputChanged.end())
&& (pathr.substr(0, path.size()) == path) // inside the path
&& (pathr.substr(path.size(), pathr.size() - path.size())).find('/') == std::string::npos) // not in a further subdirectory
{ {
nldebug("Found removed '%s' in dependency directory '%s', rebuild", pathr.c_str(), path.c_str()); nldebug("Output file '%s' has been changed", path.c_str());
m_SubTaskResult = FINISH_SUCCESS; outputChanged = true;
return true; // Rebuild. break;
}
} }
} }
else // isFile
{ if (!outputChanged && !inputModified)
// Check if this file is in the dependencies.
if (!isFileDependency(path))
{ {
m_SubTaskResult = FINISH_ERROR; nlerror("Should never reach this, this must have been cought earlier normally");
m_SubTaskErrorMessage = std::string("File '") + path + "' is not part of the dependencies";
return false; // Error, cannot rebuild.
} }
// Check if this file is in added/changed/removed.
if (m_ListInputAdded.find(path) != m_ListInputAdded.end() // Check the .depend files of all the output files // also check that they exist :)
|| m_ListInputChanged.find(path) != m_ListInputChanged.end() for (std::vector<std::string>::const_iterator it = outputPaths.begin(), end = outputPaths.end(); it != end; ++it)
|| m_ListInputRemoved.find(path) != m_ListInputRemoved.end())
{ {
// Found! const std::string &path = *it;
nldebug("Found added/changed/removed input file '%s', rebuild", path.c_str()); std::string metaDependPath = CMetadataStorage::getDependPath(path);
CFileDepend metaDepend;
if (!CMetadataStorage::readDepend(metaDepend, metaDependPath))
{
nlwarning("Depend file for existing output '%s' does not exist, this should not happen, rebuild", path.c_str());
m_SubTaskResult = FINISH_SUCCESS; m_SubTaskResult = FINISH_SUCCESS;
return true; // Rebuild. return true; // Rebuild.
} }
} else
}
for (std::vector<std::string>::const_iterator it = outputPaths.begin(), end = outputPaths.end(); it != end; ++it)
{ {
const std::string &path = *it; if (outputChanged)
if (path[path.size() - 1] == '/') // isDirectory {
// Compare the output checksum with the status output checksum
CFileStatus metaStatus;
if (!CDatabaseStatus::updateFileStatus(metaStatus, path))
{ {
m_SubTaskResult = FINISH_ERROR; m_SubTaskResult = FINISH_ERROR;
m_SubTaskErrorMessage = std::string("Output file '") + path + "' cannot be a directory"; m_SubTaskErrorMessage = std::string("Could not get status for output file '") + path + "'";
return false; // Error, cannot rebuild. return false; // Error, cannot rebuild.
} }
if (m_ListOutputRemoved.find(path) != m_ListOutputRemoved.end()) else
{ {
nldebug("Found removed output file '%s', rebuild", path.c_str()); if (metaStatus.CRC32 != metaDepend.CRC32)
{
nlwarning("Status checksum for '%s' does match depend checksum, output file was modified, this should not happen, rebuild", path.c_str());
m_SubTaskResult = FINISH_SUCCESS; m_SubTaskResult = FINISH_SUCCESS;
return true; return true; // Rebuild.
}
}
} }
if (m_ListOutputChanged.find(path) != m_ListOutputChanged.end()) if (inputModified)
{ {
nlwarning("Changed output files not implemented yet. Previous build was likely incomplete. Please wipe the output directory. Rebuilding anyways"); // Compare the input checksums with the cached input checksums
// For now always rebuild and don't check if the output file is up-to-date from a previous incomplete build.
nldebug("Found changed output file '%s', rebuild", path.c_str()); }
m_SubTaskResult = FINISH_SUCCESS;
return true;
//m_SubTaskResult = FINISH_ERROR;
//m_SubTaskErrorMessage = std::string("Changed output files not implemented yet. Previous build was likely incomplete. Please wipe the output directory");
//return false; // Error, cannot rebuild.
} }
} }
// (if any checksum was different, require rebuild, if no checksums were different, no rebuild is needed)
nldebug("No differences found, no rebuild needed");
m_SubTaskResult = FINISH_SUCCESS; m_SubTaskResult = FINISH_SUCCESS;
return false; // Does not need rebuild. return false; // Rebuild not necessary.
} }
/// Set the exit message, exit the plugin immediately afterwards. /// Set the exit message, exit the plugin immediately afterwards.

Loading…
Cancel
Save