Merge branch 'develop' into ryzomclassic-develop

ryzomclassic-develop
kaetemi 4 years ago
commit b89441d673

@ -249,11 +249,14 @@ void appendToLower(std::string &res, const char *str, ptrdiff_t &i);
void appendToLower(std::string &res, const std::string &str, ptrdiff_t &i); void appendToLower(std::string &res, const std::string &str, ptrdiff_t &i);
void appendToUpper(std::string &res, const char *str, ptrdiff_t &i); void appendToUpper(std::string &res, const char *str, ptrdiff_t &i);
void appendToUpper(std::string &res, const std::string &str, ptrdiff_t &i); void appendToUpper(std::string &res, const std::string &str, ptrdiff_t &i);
void appendToTitle(std::string &res, const char *str, ptrdiff_t &i);
void appendToTitle(std::string &res, const std::string &str, ptrdiff_t &i);
/** UTF-8 case insensitive compare */ /** UTF-8 case insensitive compare */
int compareCaseInsensitive(const char *a, const char *b); int compareCaseInsensitive(const char *a, const char *b);
int compareCaseInsensitive(const char *a, size_t lenA, const char *b, size_t lenB); int compareCaseInsensitive(const char *a, size_t lenA, const char *b, size_t lenB);
inline int compareCaseInsensitive(const std::string &a, const std::string &b) { return compareCaseInsensitive(&a[0], a.size(), &b[0], b.size()); } inline int compareCaseInsensitive(const std::string &a, const std::string &b) { return compareCaseInsensitive(&a[0], a.size(), &b[0], b.size()); }
inline bool ltCaseInsensitive(const std::string &a, const std::string &b) { return compareCaseInsensitive(&a[0], a.size(), &b[0], b.size()) < 0; }
/** ASCII to lowercase. Useful for internal identifiers. /** ASCII to lowercase. Useful for internal identifiers.
* Characters outside of the 7-bit ASCII space, and control characters, are replaced. * Characters outside of the 7-bit ASCII space, and control characters, are replaced.

@ -229,7 +229,7 @@ uint32 CTextContextUser::textPush(const char *format, ...)
char *str; char *str;
NLMISC_CONVERT_VARGS (str, format, NLMISC::MaxCStringSize); NLMISC_CONVERT_VARGS (str, format, NLMISC::MaxCStringSize);
return _TextContext.textPush(ucstring(str)) ; return _TextContext.textPush(str) ;
} }
uint32 CTextContextUser::textPush(NLMISC::CUtfStringView sv) uint32 CTextContextUser::textPush(NLMISC::CUtfStringView sv)
{ {
@ -340,7 +340,7 @@ void CTextContextUser::printfAt(float x, float y, const char * format, ...)
char *str; char *str;
NLMISC_CONVERT_VARGS (str, format, NLMISC::MaxCStringSize); NLMISC_CONVERT_VARGS (str, format, NLMISC::MaxCStringSize);
_TextContext.printAt(x, y, ucstring(str)) ; _TextContext.printAt(x, y, str) ;
_DriverUser->restoreMatrixContext(); _DriverUser->restoreMatrixContext();
} }

@ -55,21 +55,25 @@ namespace NLGUI
CGroupEditBox::IComboKeyHandler* CGroupEditBox::comboKeyHandler = NULL; CGroupEditBox::IComboKeyHandler* CGroupEditBox::comboKeyHandler = NULL;
// For now, just trim unsupported codepoints to make emoji fallback to text form // For now, just trim unsupported codepoints to make emoji fallback to text form
static bool supportedCodepoint(u32char c) static u32char supportedCodepoint(u32char c)
{ {
if (c >= 0xFE00 && c < 0xFE10) if (c >= 0xFE00 && c < 0xFE10)
return false; // Variation Selectors return 0; // Variation Selectors, unsupported
else if (c >= 0xE0100 && c < 0xE01F0) else if (c >= 0xE0100 && c < 0xE01F0)
return false; // Variation Selectors Supplement return 0; // Variation Selectors Supplement, unsupported
else if (c >= 0x200B && c < 0x2010) else if (c >= 0x200B && c < 0x2010)
return false; // ZERO WIDTH JOINER, etcetera return 0; // ZERO WIDTH JOINER, etcetera, unsupported
else if (c >= 0x2028 && c < 0x202F) else if (c >= 0x2028 && c < 0x202F)
return false; // PARAGRAPH SEPARATOR, etcetera return 0; // PARAGRAPH SEPARATOR, etcetera, unsupported
else if (c >= 0x2060 && c < 0x2070) else if (c >= 0x2060 && c < 0x2070)
return false; // WORD JOINER, etcetera return 0; // WORD JOINER, etcetera, unsupported
else if (c == 0xFEFF) else if (c == 0xFEFF)
return false; // BOM return 0; // BOM, unsupported
return true; else if ((c & 0xFC00) == 0xD800)
return 0xFFFD; // UTF-16 surrogate, unmatched pair, invalid, set to replacement character
else if ((c & 0xFC00) == 0xDC00)
return 0xFFFD; // UTF-16 surrogate, unmatched pair, invalid, set to replacement character
return c;
} }
// For now, just trim unsupported codepoints to make emoji fallback to text form // For now, just trim unsupported codepoints to make emoji fallback to text form
@ -79,8 +83,9 @@ namespace NLGUI
res.reserve(str.size()); res.reserve(str.size());
for (::u32string::const_iterator it(str.begin()), end(str.end()); it != end; ++it) for (::u32string::const_iterator it(str.begin()), end(str.end()); it != end; ++it)
{ {
if (supportedCodepoint(*it)) u32char c = supportedCodepoint(*it);
res.push_back(*it); if (c) // This also trims NUL
res.push_back(c);
} }
return res; return res;
} }
@ -1142,7 +1147,8 @@ namespace NLGUI
u32char c = isKeyRETURN ? '\n' : rEDK.getChar(); u32char c = isKeyRETURN ? '\n' : rEDK.getChar();
if (isFiltered(c)) return; if (isFiltered(c)) return;
if (!supportedCodepoint(c)) return; // For now, just trim unsupported codepoints to make emoji fallback to text form c = supportedCodepoint(c);
if (!c) return; // For now, just trim unsupported codepoints to make emoji fallback to text form
switch(_EntryType) switch(_EntryType)
{ {
case Integer: case Integer:

@ -68,7 +68,7 @@ namespace NLGUI
if (!isSeparator(c)) if (!isSeparator(c))
{ {
if (newString) if (newString)
NLMISC::appendToUpper(res, str, i); NLMISC::appendToTitle(res, str, i);
else else
NLMISC::appendToLower(res, str, i); NLMISC::appendToLower(res, str, i);
newString = false; newString = false;
@ -99,7 +99,7 @@ namespace NLGUI
else else
{ {
if (newSentence) if (newSentence)
NLMISC::appendToUpper(res, str, i); NLMISC::appendToTitle(res, str, i);
else else
NLMISC::appendToLower(res, str, i); NLMISC::appendToLower(res, str, i);
@ -128,7 +128,7 @@ namespace NLGUI
else else
{ {
if (newWord) if (newWord)
NLMISC::appendToUpper(res, str, i); NLMISC::appendToTitle(res, str, i);
else else
NLMISC::appendToLower(res, str, i); NLMISC::appendToLower(res, str, i);

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

@ -12,6 +12,7 @@ IF(WITH_NEL_TOOLS)
lock lock
make_sheet_id make_sheet_id
xml_packer xml_packer
utf_generator
) )
IF(WITH_QT OR WITH_QT5) IF(WITH_QT OR WITH_QT5)

@ -0,0 +1,11 @@
FILE(GLOB SRC *.cpp *.h *.rc *.rc2)
SOURCE_GROUP("" FILES ${SRC})
ADD_EXECUTABLE(nl_utf_generator ${SRC})
TARGET_LINK_LIBRARIES(nl_utf_generator nelmisc)
NL_DEFAULT_PROPS(nl_utf_generator "NeL, Tools, Misc: UTF Generator")
NL_ADD_RUNTIME_FLAGS(nl_utf_generator)
INSTALL(TARGETS nl_utf_generator RUNTIME DESTINATION ${NL_BIN_PREFIX} COMPONENT toolsmisc)

Binary file not shown.

After

Width:  |  Height:  |  Size: 3.6 KiB

@ -0,0 +1,425 @@
// NeL - MMORPG Framework <https://wiki.ryzom.dev/>
// Copyright (C) 2020 Jan BOON (Kaetemi) <jan.boon@kaetemi.be>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
#include "nel/misc/types_nl.h"
#include "nel/misc/debug.h"
#include "nel/misc/common.h"
#include "nel/misc/string_common.h"
#include "nel/misc/string_view.h"
#include "nel/misc/utf_string_view.h"
#include <fstream>
#include <sstream>
#include <string>
#include <iostream>
#include <iomanip>
void printStringMap(const std::string &name, std::map<char, std::string> &m, bool trim)
{
std::cout << "static const char " << name << "[" << std::dec << (trim ? "64" : "256") << " * 4] = {\n";
bool zero = false;
for (int i = 0; i < (trim ? 64 : 256); ++i)
{
int x = trim ? i + 0x80 : i;
if (m.find(x) == m.end())
{
if (x % 8 == 7)
{
zero = false;
std::cout << "0, 0, 0, 0,\n";
}
else
{
zero = true;
std::cout << "0, 0, 0, 0, ";
}
}
else
{
if (zero) std::cout << "\n";
std::stringstream ss;
ss << "'\\x" << std::hex << std::setw(2) << std::setfill('0') << std::uppercase << (m[x].length() > 0 ? (unsigned char)m[x][0] : 0)
<< "', '\\x" << std::hex << std::setw(2) << std::setfill('0') << std::uppercase << (m[x].length() > 1 ? (unsigned char)m[x][1] : 0)
<< "', '\\x" << std::hex << std::setw(2) << std::setfill('0') << std::uppercase << (m[x].length() > 2 ? (unsigned char)m[x][2] : 0)
<< "', 0,\n";
std::cout << ss.str();
zero = false;
}
}
if (zero) std::cout << "\n";
std::cout << "};\n\n";
}
void printMapMap(const std::string &name, const std::string &strName, std::map<char, std::map<char, std::string>> &m, int base, int size)
{
std::cout << "static const char *" << name << "[" << size << "] = {\n";
bool zero = false;
for (int i = base; i < (base + size); ++i)
{
int x = i;
if (m.find(x) == m.end())
{
if (x % 32 == 1315)
{
zero = false;
std::cout << "0, \n";
}
else
{
zero = true;
std::cout << "0, ";
}
}
else
{
if (zero) std::cout << "\n";
std::stringstream n;
n << strName;
n << std::hex << std::setw(2) << std::setfill('0') << std::uppercase << (unsigned int)x;
std::cout << n.str() << ",\n";
zero = false;
}
}
if (zero) std::cout << "\n";
std::cout << std::dec << "};\n\n";
}
void printMapMapMap(const std::string &name, const std::string &mapName, std::map<char, std::map<char, std::map<char, std::string>>> &m, int base, int size)
{
std::cout << "static const char **" << name << "[" << size << "] = {\n";
bool zero = false;
for (int i = base; i < (base + size); ++i)
{
int x = i;
if (m.find(x) == m.end())
{
if (x % 32 == 1315)
{
zero = false;
std::cout << "0, \n";
}
else
{
zero = true;
std::cout << "0, ";
}
}
else
{
if (zero) std::cout << "\n";
std::stringstream n;
n << mapName;
n << std::hex << std::setw(2) << std::setfill('0') << std::uppercase << (unsigned int)x;
std::cout << n.str() << ",\n";
zero = false;
}
}
if (zero) std::cout << "\n";
std::cout << "};\n\n";
}
void printMapMapMapMap(const std::string &name, const std::string &mapName, std::map<char, std::map<char, std::map<char, std::map<char, std::string>>>> &m, int base, int size)
{
std::cout << "static const char ***" << name << "[" << size << "] = {\n";
bool zero = false;
for (int i = base; i < (base + size); ++i)
{
int x = i;
if (m.find(x) == m.end())
{
if (x % 32 == 1315)
{
zero = false;
std::cout << "0, \n";
}
else
{
zero = true;
std::cout << "0, ";
}
}
else
{
if (zero) std::cout << "\n";
std::stringstream n;
n << mapName;
n << std::hex << std::setw(2) << std::setfill('0') << std::uppercase << (unsigned int)x;
std::cout << n.str() << ",\n";
zero = false;
}
}
if (zero) std::cout << "\n";
std::cout << "};\n\n";
}
void generateMap(const std::string &file, const std::string &name, const std::vector<u32char> &map)
{
std::map<char, std::string> m1;
std::map<char, std::map<char, std::string>> m2;
std::map<char, std::map<char, std::map<char, std::string>>> m3;
std::map<char, std::map<char, std::map<char, std::map<char, std::string>>>> m4;
for (u32char i = 0; i < 0x110000; ++i)
{
if (map[i] != i)
{
std::string from;
NLMISC::CUtfStringView::append(from, i);
std::string to;
NLMISC::CUtfStringView::append(to, map[i]);
// assert(from.size() == to.size());
if (from.length() == 1)
{
m1[from[0]] = to;
}
else if (from.length() == 2)
{
if (m2.find(from[0]) == m2.end())
m2[from[0]] = std::map<char, std::string>();
m2[from[0]][from[1]] = to;
}
else if (from.length() == 3)
{
if (m3.find(from[0]) == m3.end())
m3[from[0]] = std::map<char, std::map<char, std::string>>();
if (m3[from[0]].find(from[1]) == m3[from[0]].end())
m3[from[0]][from[1]] = std::map<char, std::string>();
m3[from[0]][from[1]][from[2]] = to;
}
else if (from.length() == 4)
{
if (m4.find(from[0]) == m4.end())
m4[from[0]] = std::map<char, std::map<char, std::map<char, std::string>>>();
if (m4[from[0]].find(from[1]) == m4[from[0]].end())
m4[from[0]][from[1]] = std::map<char, std::map<char, std::string>>();
if (m4[from[0]][from[1]].find(from[2]) == m4[from[0]][from[1]].end())
m4[from[0]][from[1]][from[2]] = std::map<char, std::string>();
m4[from[0]][from[1]][from[2]][from[3]] = to;
}
}
}
printStringMap("s_" + name, m1, false);
for (int i = 0; i < 256; ++i)
{
std::stringstream n;
n << "s_" << name;
if (m2.find(i) != m2.end())
{
n << std::hex << std::setw(2) << std::setfill('0') << std::uppercase << (unsigned int)i;
printStringMap(n.str(), m2[i], true);
}
else if (m3.find(i) != m3.end())
{
n << std::hex << std::setw(2) << std::setfill('0') << std::uppercase << (unsigned int)i;
for (int j = 0; j < 256; ++j)
{
if (m3[i].find(j) != m3[i].end())
{
std::stringstream nn;
nn << n.str();
nn << std::hex << std::setw(2) << std::setfill('0') << std::uppercase << (unsigned int)j;
printStringMap(nn.str(), m3[i][j], true);
}
}
}
else if (m4.find(i) != m4.end())
{
n << std::hex << std::setw(2) << std::setfill('0') << std::uppercase << (unsigned int)i;
for (int j = 0; j < 256; ++j)
{
if (m4[i].find(j) != m4[i].end())
{
std::stringstream nn;
nn << n.str();
nn << std::hex << std::setw(2) << std::setfill('0') << std::uppercase << (unsigned int)j;
for (int k = 0; k < 256; ++k)
{
if (m4[i][j].find(k) != m4[i][j].end())
{
std::stringstream nnn;
nnn << nn.str();
nnn << std::hex << std::setw(2) << std::setfill('0') << std::uppercase << (unsigned int)k;
printStringMap(nnn.str(), m4[i][j][k], true);
}
}
}
}
}
}
printMapMap("s_" + name + "Map", "s_" + name, m2, 0xC0, 32);
for (int i = 0; i < 256; ++i)
{
std::stringstream n;
n << "s_" << name << "Map";
std::stringstream nn;
nn << "s_" << name;
if (m3.find(i) != m3.end())
{
n << std::hex << std::setw(2) << std::setfill('0') << std::uppercase << (unsigned int)i;
nn << std::hex << std::setw(2) << std::setfill('0') << std::uppercase << (unsigned int)i;
printMapMap(n.str(), nn.str(), m3[i], 0x80, 64);
}
}
printMapMapMap("s_" + name + "MapMap", "s_" + name + "Map", m3, 0xE0, 16);
for (int i = 0; i < 256; ++i)
{
std::stringstream n;
n << "s_" << name << "Map";
std::stringstream nn;
nn << "s_" << name;
std::stringstream nnn;
nnn << "s_" << name << "MapMap";
if (m4.find(i) != m4.end())
{
n << std::hex << std::setw(2) << std::setfill('0') << std::uppercase << (unsigned int)i;
nn << std::hex << std::setw(2) << std::setfill('0') << std::uppercase << (unsigned int)i;
nnn << std::hex << std::setw(2) << std::setfill('0') << std::uppercase << (unsigned int)i;
for (int j = 0; j < 256; ++j)
{
if (m4[i].find(j) != m4[i].end())
{
std::stringstream n2, nn2;
n2 << n.str() << std::hex << std::setw(2) << std::setfill('0') << std::uppercase << (unsigned int)j;
nn2 << nn.str() << std::hex << std::setw(2) << std::setfill('0') << std::uppercase << (unsigned int)j;
printMapMap(n2.str(), nn2.str(), m4[i][j], 0x80, 64);
}
}
printMapMapMap(nnn.str(), n.str(), m4[i], 0x80, 64);
}
}
printMapMapMapMap("s_" + name + "MapMapMap", "s_" + name + "MapMap", m4, 0xF0, 8);
}
int main (int argc, char **argv)
{
std::ifstream fi("UnicodeData.txt");
std::vector<u32char> upper;
std::vector<u32char> lower;
std::vector<u32char> title;
std::vector<u32char> ci;
upper.resize(0x110000);
lower.resize(0x110000);
title.resize(0x110000);
ci.resize(0x110000);
for (u32char i = 0; i < 0x110000; ++i)
{
upper[i] = i;
lower[i] = i;
title[i] = i;
ci[i] = i;
}
std::string line;
while (std::getline(fi, line))
{
std::vector<std::string> cols;
NLMISC::explode(line, nlstr(";"), cols, false);
nlassert(cols.size() == 15);
u32char c = NLMISC::atoiInt64(cols[0].c_str(), 16);
u32char up = NLMISC::atoiInt64(cols[12].c_str(), 16);
u32char low = NLMISC::atoiInt64(cols[13].c_str(), 16);
u32char tit = NLMISC::atoiInt64(cols[14].c_str(), 16);
if (up) upper[c] = up;
if (low) lower[c] = low;
if (tit) title[c] = tit;
}
std::vector<u32char> ref;
int rounds = 0;
for (;;)
{
ref = ci;
for (u32char i = 0; i < 0x110000; ++i)
{
ci[i] = title[ci[i]];
}
for (u32char i = 0; i < 0x110000; ++i)
{
ci[i] = upper[ci[i]];
}
for (u32char i = 0; i < 0x110000; ++i)
{
ci[i] = lower[ci[i]];
}
bool equal = true;
for (u32char i = 0; i < 0x110000; ++i)
{
if (ci[i] != ref[i])
equal = false;
}
++rounds;
std::cout << rounds << std::endl;
if (equal)
break;
}
for (u32char i = 0; i < 0x110000; ++i)
{
if (ci[i] != lower[i])
std::cout << i << std::endl;
}
generateMap("string_to_upper", "StringToUpper", upper);
//generateMap("string_to_lower", "StringToLower", lower);
//generateMap("string_to_title", "StringToTitle", title);
//generateMap("string_to_ci", "StringToCaseInsensitive", ci);
std::string test = nlstr("Οὐχὶ ταὐτὰ παρίσταταί μοι γιγνώσκειν, ὦ ἄνδρες ᾿Αθηναῖοι,");
std::string testUpper = NLMISC::toUpper(test);
std::string testLower = NLMISC::toLower(test);
std::string testUpper2 = NLMISC::toUpper(testLower);
std::string testLower2 = NLMISC::toLower(testUpper);
std::cout << test << std::endl;
std::cout << testUpper << std::endl;
std::cout << testLower << std::endl;
std::cout << testUpper2 << std::endl;
std::cout << testLower2 << std::endl;
int cci1 = NLMISC::compareCaseInsensitive("bAAAAfdsklj", "Cldsfjslkf");
int cci2 = NLMISC::compareCaseInsensitive("Cldsfjslkf", "bAAAAfdsklj");
int strc1 = strcmp(NLMISC::toLower("bAAAAfdsklj").c_str(), NLMISC::toLower("Cldsfjslkf").c_str());
int strc2 = strcmp(NLMISC::toLower("Cldsfjslkf").c_str(), NLMISC::toLower("bAAAAfdsklj").c_str());
int bcci1 = NLMISC::compareCaseInsensitive("bAAAAfdsklj", "AnlsqFDS");
int bcci2 = NLMISC::compareCaseInsensitive("AnlsqFDS", "bAAAAfdsklj");
int bstrc1 = strcmp(NLMISC::toLower("bAAAAfdsklj").c_str(), NLMISC::toLower("AnlsqFDS").c_str());
int bstrc2 = strcmp(NLMISC::toLower("AnlsqFDS").c_str(), NLMISC::toLower("bAAAAfdsklj").c_str());
std::vector<std::string> arr;
arr.push_back("AnlsqFDS");
arr.push_back("yozeRNZE");
arr.push_back("yOzeihfn");
arr.push_back("bAAAAfdsklj");
arr.push_back("Cldsfjslkf");
std::sort(arr.begin(), arr.end(), NLMISC::ltCaseInsensitive);
for (int i = 0; i < arr.size(); ++i)
std::cout << arr[i] << std::endl;
return EXIT_SUCCESS;
}
/* end of file */

@ -0,0 +1,42 @@
#include <windows.h>
#include "config.h"
IDI_MAIN_ICON ICON DISCARDABLE "gold_pill.ico"
#ifdef _DEBUG
#define NL_FILEEXT "_d"
#else
#define NL_FILEEXT ""
#endif
VS_VERSION_INFO VERSIONINFO
FILEVERSION NL_VERSION_RC
PRODUCTVERSION NL_VERSION_RC
FILEFLAGSMASK VS_FFI_FILEFLAGSMASK
#ifdef _DEBUG
FILEFLAGS VS_FF_DEBUG
#else
FILEFLAGS 0x0L
#endif
FILEOS VOS_NT_WINDOWS32
FILETYPE VFT_APP
FILESUBTYPE 0x0L
BEGIN
BLOCK "StringFileInfo"
BEGIN
BLOCK "040904b0"
BEGIN
VALUE "CompanyName", AUTHOR
VALUE "FileDescription", "NeL UTF Generator"
VALUE "FileVersion", NL_VERSION
VALUE "LegalCopyright", COPYRIGHT
VALUE "OriginalFilename", "nl_utf_generator" NL_FILEEXT ".exe"
VALUE "ProductName", "NeL Tools"
VALUE "ProductVersion", NL_PRODUCT_VERSION
END
END
BLOCK "VarFileInfo"
BEGIN
VALUE "Translation", 0x9, 1200
END
END

@ -135,13 +135,16 @@ namespace STRING_MANAGER
nlinfo("SM : Try to open the string cache : %s", _CacheFilename.c_str()); nlinfo("SM : Try to open the string cache : %s", _CacheFilename.c_str());
if (CFile::fileExists(_CacheFilename)) if (CFile::fileExists(_CacheFilename) && CFile::getFileSize(_CacheFilename))
{ {
// there is a cache file, check date reset it if needed // there is a cache file, check date reset it if needed
{ {
NLMISC::CIFile file(_CacheFilename); NLMISC::CIFile file(_CacheFilename);
file.setVersionException(false, false); file.setVersionException(false, false);
file.serialVersion(currentVersion); file.serialVersion(currentVersion);
if (file.getPos() + sizeof(_Timestamp) > file.getFileSize())
_Timestamp = ~timestamp;
else
file.serial(_Timestamp); file.serial(_Timestamp);
} }

Loading…
Cancel
Save