(svn r26770) -Fix [Squirrel]: in case SQChar is a char (previously everything non-Windows or non-Unicode, now everything), the lexer throws away the higher bytes of characters

2 files changed, 18 insertions, 5 deletions

diff --git a/src/3rdparty/squirrel/squirrel/sqlexer.cpp b/src/3rdparty/squirrel/squirrel/sqlexer.cpp
index c03048e9a..2f6e84cdc 100644
--- a/src/3rdparty/squirrel/squirrel/sqlexer.cpp
+++ b/src/3rdparty/squirrel/squirrel/sqlexer.cpp
@@ -9,13 +9,13 @@
 #include "sqcompiler.h"
 #include "sqlexer.h"
 
+#include "../../../stdafx.h"
+#include "../../../string_func.h"
+
 #define CUR_CHAR (_currdata)
 #define RETURN_TOKEN(t) { _prevtoken = _curtoken; _curtoken = t; return t;}
 #define IS_EOB() (CUR_CHAR <= SQUIRREL_EOB)
 #define NEXT() {Next();_currentcolumn++;}
-#define INIT_TEMP_STRING() { _longstr.resize(0);}
-#define APPEND_CHAR(c) { _longstr.push_back(c);}
-#define TERMINATE_BUFFER() {_longstr.push_back(_SC('\0'));}
 #define ADD_KEYWORD(key,id) _keywords->NewSlot( SQString::Create(ss, _SC(#key)) ,SQInteger(id))
 
 SQLexer::SQLexer(){}
@@ -24,6 +24,15 @@ SQLexer::~SQLexer()
 	_keywords->Release();
 }
 
+void SQLexer::APPEND_CHAR(LexChar c)
+{
+	char buf[4];
+	int chars = Utf8Encode(buf, c);
+	for (int i = 0; i < chars; i++) {
+		_longstr.push_back(buf[i]);
+	}
+}
+
 void SQLexer::Init(SQSharedState *ss, SQLEXREADFUNC rg, SQUserPointer up,CompilerErrorFunc efunc,void *ed)
 {
 	_errfunc = efunc;
@@ -273,7 +282,7 @@ SQInteger SQLexer::GetIDType(SQChar *s)
 }
 
 
-SQInteger SQLexer::ReadString(SQChar ndelim,bool verbatim)
+SQInteger SQLexer::ReadString(LexChar ndelim,bool verbatim)
 {
 	INIT_TEMP_STRING();
 	NEXT();
diff --git a/src/3rdparty/squirrel/squirrel/sqlexer.h b/src/3rdparty/squirrel/squirrel/sqlexer.h
index 1d81ab3c8..232c29cc0 100644
--- a/src/3rdparty/squirrel/squirrel/sqlexer.h
+++ b/src/3rdparty/squirrel/squirrel/sqlexer.h
@@ -14,13 +14,17 @@ struct SQLexer
 	const SQChar *Tok2Str(SQInteger tok);
 private:
 	SQInteger GetIDType(SQChar *s);
-	SQInteger ReadString(SQChar ndelim,bool verbatim);
+	SQInteger ReadString(LexChar ndelim,bool verbatim);
 	SQInteger ReadNumber();
 	void LexBlockComment();
 	SQInteger ReadID();
 	void Next();
 	SQInteger _curtoken;
 	SQTable *_keywords;
+	void INIT_TEMP_STRING() { _longstr.resize(0); }
+	void APPEND_CHAR(LexChar c);
+	void TERMINATE_BUFFER() { _longstr.push_back(_SC('\0')); }
+
 public:
 	SQInteger _prevtoken;
 	SQInteger _currentline;