nitro/G-Earth
1
0
Fork 0
mirror of https://github.com/sirjonasxx/G-Earth.git synced 2024-11-23 08:50:52 +01:00
Code Issues Packages Projects Releases Wiki Activity

utf-8 string prediction

Browse Source
This commit is contained in:
sirjonasxx 2021-01-21 04:32:12 +01:00
parent fbef4ad4ab
commit e4c5a941ee
1 changed files with 22 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
G-Earth/src/main/java/gearth/misc/packetrepresentation/prediction/checkers/StringChecker.java
View File

@ -64,13 +64,29 @@ public class StringChecker extends TypeChecker<String> {
}; };
for (int i = 0; i < s.length(); i++) { for (int i = 0; i < s.length(); i++) {
score *= penalties[isCommon(
asChars[i],
asBytes[i]
)];
if (score < 0.001) { // detect UTF8 extended chars
return 0; if ((asBytes[i] & 0b11100000) == 0b11000000 && i < s.length() - 1 && (asBytes[i+1] & 0b11000000) == 0b10000000) {
i += 1;
score *= penalties[2]*penalties[2];
}
else if ((asBytes[i] & 0b11110000) == 0b11100000 && i < s.length() - 2 && (asBytes[i+1] & 0b11000000) == 0b10000000 && (asBytes[i+2] & 0b11000000) == 0b10000000) {
i += 2;
score *= penalties[2]*penalties[2]*penalties[2];
}
else if ((asBytes[i] & 0b11111000) == 0b11110000 && i < s.length() - 3 && (asBytes[i+1] & 0b11000000) == 0b10000000 && (asBytes[i+2] & 0b11000000) == 0b10000000 && (asBytes[i+3] & 0b11000000) == 0b10000000) {
i += 3;
score *= penalties[2]*penalties[2]*penalties[2]*penalties[2];
}
else {
score *= penalties[isCommon(
asChars[i],
asBytes[i]
)];
if (score < 0.001) {
return 0;
}
} }
} }