// NeL - MMORPG Framework // Copyright (C) 2010 Winch Gate Property Limited // // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU Affero General Public License as // published by the Free Software Foundation, either version 3 of the // License, or (at your option) any later version. // // This program is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU Affero General Public License for more details. // // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . #include "stdmisc.h" #include "nel/misc/diff_tool.h" #include "nel/misc/path.h" using namespace NLMISC; using namespace std; namespace STRING_MANAGER { uint64 makePhraseHash(const TPhrase &phrase) { ucstring text; text = phrase.Parameters; for (uint i=0; i &stringInfos, bool forceRehash, ucchar openMark, ucchar closeMark, bool specialCase) { /* uint8 *buffer = 0; uint size; try { CIFile fp(filename); size = fp.getFileSize(); buffer = new uint8[size]; fp.serialBuffer(buffer, size); } catch(Exception &e) { nlinfo("Can't open file [%s] (%s)\n", filename.c_str(), e.what()); return true; } */ /* FILE *fp = fopen(filename.c_str(), "rb"); if (fp == NULL) { nlinfo("Can't open file [%s]\n", filename.c_str()); if (buffer != 0) delete [] buffer; return true; } // move to end of file fseek(fp, 0, SEEK_END); fpos_t pos; fgetpos(fp, &pos); uint8 *buffer = new uint8[uint(pos)]; rewind(fp); uint size = fread(buffer, 1, uint(pos), fp); fclose (fp); */ ucstring text; CI18N::readTextFile(filename, text, false, false, true, CI18N::LINE_FMT_CRLF); // CI18N::readTextBuffer(buffer, size, text); // delete [] buffer; // ok, parse the file now. ucstring::const_iterator first(text.begin()), last(text.end()); std::string lastLabel("nothing"); while (first != last) { TStringInfo si; CI18N::skipWhiteSpace(first, last, &si.Comments); if (first == last) { // check if there is only swap command remaining in comment if (si.Comments.find(ucstring("// DIFF SWAP ")) != ucstring::npos) { stringInfos.push_back(si); } break; } // try to read a #fileline preprocessor command if (CI18N::matchToken("#fileline", first, last)) { // for now, just skip uint32 lineCounter =0; // we count line another way CI18N::skipLine(first, last, lineCounter); // begin parse of next line continue; } if (!CI18N::parseLabel(first, last, si.Identifier)) { uint32 line = countLine(text, first); nlwarning("DT: Fatal : In '%s', line %u: Invalid label after '%s'\n", filename.c_str(), line, lastLabel.c_str()); return false; } lastLabel = si.Identifier; CI18N::skipWhiteSpace(first, last, &si.Comments); if (!CI18N::parseMarkedString(openMark, closeMark, first, last, si.Text)) { uint32 line = countLine(text, first); nlwarning("DT: Fatal : In '%s', line %u: Invalid text value for label %s\n", filename.c_str(), line, lastLabel.c_str()); return false; } if (specialCase) { CI18N::skipWhiteSpace(first, last, &si.Comments); if (!CI18N::parseMarkedString(openMark, closeMark, first, last, si.Text2)) { uint32 line = countLine(text, first); nlwarning("DT: Fatal: In '%s' line %u: Invalid text2 value label %s\n", filename.c_str(), line, lastLabel.c_str()); return false; } } if (forceRehash || !parseHashFromComment(si.Comments, si.HashValue)) { // compute the hash value from text. si.HashValue = CI18N::makeHash(si.Text); // nldebug("Generating hash for %s as %s", si.Identifier.c_str(), CI18N::hashToString(si.HashValue).c_str()); } else { // nldebug("Comment = [%s]", si.Comments.toString().c_str()); // nldebug("Retrieving hash for %s as %s", si.Identifier.c_str(), CI18N::hashToString(si.HashValue).c_str()); } stringInfos.push_back(si); } // check identifier uniqueness { bool error = false; set unik; set::iterator it; for (uint i=0; i &strings, bool removeDiffComments, bool noDiffInfo) { ucstring diff; vector::const_iterator first(strings.begin()), last(strings.end()); for (; first != last; ++first) { ucstring str; const TStringInfo &si = *first; string comment = si.Comments.toString(); vector lines; explode(comment, string("\n"), lines, true); uint i; for (i=0; i &phrases, bool forceRehash) { ucstring doc; CI18N::readTextFile(filename, doc, false, false, true, CI18N::LINE_FMT_CRLF); return readPhraseFileFromString(doc, filename, phrases, forceRehash); } bool readPhraseFileFromString(ucstring const& doc, const std::string &filename, vector &phrases, bool forceRehash) { std::string lastRead("nothing"); ucstring::const_iterator first(doc.begin()), last(doc.end()); while (first != last) { TPhrase phrase; // parse the phrase CI18N::skipWhiteSpace(first, last, &phrase.Comments); if (first == last) { if (!phrase.Comments.empty()) { // push the resulting comment phrases.push_back(phrase); } break; } // try to read a #fileline preprocessor command if (CI18N::matchToken("#fileline", first, last)) { // for now, just skip uint32 lineCounter =0; // we count line another way CI18N::skipLine(first, last, lineCounter); // begin parse of next line continue; } if (!CI18N::parseLabel(first, last, phrase.Identifier)) { uint32 line = countLine(doc, first); nlwarning("DT: In '%s' line %u: Error parsing phrase identifier after %s\n", filename.c_str(), line, lastRead.c_str()); return false; } // nldebug("DT: parsing phrase '%s'", phrase.Identifier.c_str()); lastRead = phrase.Identifier; CI18N::skipWhiteSpace(first, last, &phrase.Comments); if (!CI18N::parseMarkedString('(', ')', first, last, phrase.Parameters)) { uint32 line = countLine(doc, first); nlwarning("DT: in '%s', line %u: Error parsing parameter list for phrase %s\n", filename.c_str(), line, phrase.Identifier.c_str()); return false; } CI18N::skipWhiteSpace(first, last, &phrase.Comments); if (first == last || *first != '{') { uint32 line = countLine(doc, first); nlwarning("DT: In '%s', line %u: Error parsing block opening '{' in phase %s\n", filename.c_str(), line, phrase.Identifier.c_str()); return false; } ++first; ucstring temp; while (first != last && *first != '}') { TClause clause; // append the comment preread at previous pass clause.Comments = temp; temp.erase(); // parse the clauses CI18N::skipWhiteSpace(first, last, &clause.Comments); if (first == last) { nlwarning("DT: Found end of file in non closed block for phrase %s\n", phrase.Identifier.c_str()); return false; } if (*first == '}') break; // skip the conditional expression ucstring cond; while (first != last && *first == '(') { if (!CI18N::parseMarkedString('(', ')', first, last, cond)) { uint32 line = countLine(doc, first); nlwarning("DT: In '%s' line %u: Error parsing conditional expression in phrase %s, clause %u\n", filename.c_str(), line, phrase.Identifier.c_str(), phrase.Clauses.size()+1); return false; } clause.Conditions += "(" + cond + ") "; CI18N::skipWhiteSpace(first, last, &clause.Comments); } if (first == last) { nlwarning("DT: in '%s': Found end of file in non closed block for phrase %s\n", filename.c_str(), phrase.Identifier.c_str()); return false; } // read the idnetifier (if any) CI18N::parseLabel(first, last, clause.Identifier); CI18N::skipWhiteSpace(first, last, &temp); // read the text if (CI18N::parseMarkedString('[', ']', first, last, clause.Text)) { // the last read comment is for this clause. clause.Comments += temp; temp.erase(); } else { uint32 line = countLine(doc, first); nlwarning("DT: in '%s' line %u: Error reading text for clause %u (%s) in phrase %s\n", filename.c_str(), line, phrase.Clauses.size()+1, clause.Identifier.c_str(), phrase.Identifier.c_str()); return false; } phrase.Clauses.push_back(clause); } CI18N::skipWhiteSpace(first, last); if (first == last || *first != '}') { uint32 line = countLine(doc, first); nlwarning("DT: in '%s' line %u: Missing block closing tag '}' in phrase %s\n", filename.c_str(), line, phrase.Identifier.c_str()); return false; } ++first; // handle hash value. if (forceRehash || !parseHashFromComment(phrase.Comments, phrase.HashValue)) { // the hash is not in the comment, compute it. phrase.HashValue = makePhraseHash(phrase); if (forceRehash) { // the has is perhaps in the comment ucstring::size_type pos = phrase.Comments.find(ucstring("// HASH_VALUE")); if (pos != ucstring::npos) { phrase.Comments = phrase.Comments.substr(0, pos); } } } // nldebug("DT : storing phrase '%s'", phrase.Identifier.c_str()); phrases.push_back(phrase); } // check identifier uniqueness { bool error = false; set unik; set::iterator it; for (uint i=0; i &phrases, bool removeDiffComments) { ucstring ret; vector::const_iterator first(phrases.begin()), last(phrases.end()); for (; first != last; ++first) { const TPhrase &p = *first; if (removeDiffComments) { string comment = p.Comments.toString(); vector lines; explode(comment, string("\n"), lines, true); uint i; for (i=0; i Avoid allocation / free vector strArray; // append a '\0' strArray.resize(str.size()+1); strArray[strArray.size()-1]= 0; memcpy(&strArray[0], &str[0], str.size()*sizeof(ucchar)); // **** Build array of lines. just point to strArray, and fill 0 where appropriated vector lines; lines.reserve(500); ucstring::size_type pos = 0; ucstring::size_type lastPos = 0; while ((pos = str.find(nl, lastPos)) != ucstring::npos) { if (pos>lastPos) { strArray[pos]= 0; // nldebug("Found line : [%s]", ucstring(&strArray[lastPos]).toString().c_str()); lines.push_back(&strArray[lastPos]); } lastPos = pos + 2; } // Must add last line if no \r\n ending if (lastPos < str.size()) { pos= str.size(); strArray[pos]= 0; // nldebug("Found line : [%s]", ucstring(&strArray[lastPos]).toString().c_str()); lines.push_back(&strArray[lastPos]); } // nldebug("Found %u lines", lines.size()); // **** Do 2 pass.1st count the cell number, then fill. => avoid reallocation uint newColCount= 0; uint i; for (i=0; i 0) { // look for the first non '* tagged' or 'DIFF_CMD' column uint nameCol = 0; while (nameCol < worksheet.ColCount && (*worksheet.getData(0, nameCol).begin() == uint16('*') || worksheet.getData(0, nameCol) == ucstring("DIFF_CMD"))) ++nameCol; if (nameCol < worksheet.ColCount ) { // ok we can check unikness bool error = false; set unik; set::iterator it; for (uint j=0; j columnOk; columnOk.resize(sheet.ColCount, false); for (uint k=1; k hashValue; hashValue.resize(worksheet.Data[0].size()); for (uint j=0; j 0 && hashValue[j] && (!worksheet.Data[i][j].empty() && worksheet.Data[i][j][0] != '_')) text += "_"; text += worksheet.Data[i][j]; if (j != worksheet.Data[i].size()-1) text += '\t'; } text += nl; } return text; } } // namespace STRING_MANAGER