ember/fluffychat
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

4181 japanese words grouped when should be separate (#4184)

Browse source 
* fix: only continue pickup up adjacent tokens while they are punctuation

* uncomment commented out code
This commit is contained in:
ggurdin 2025-09-29 16:00:16 -04:00 • committed by GitHub
parent b159bfe1c9
commit d1f840eb60
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 11 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
lib/pangea/message_token_text/tokens_util.dart
View file

@ -35,6 +35,12 @@ class TokenPosition {
required this.startIndex,
required this.endIndex,
});
Map<String, dynamic> toJson() => {
'token': token?.toJson(),
'startIndex': startIndex,
'endIndex': endIndex,
};
}
class TokensUtil {
@ -157,10 +163,14 @@ class TokensUtil {
final int startIndex = i;
if (isPunct || nextIsPunct) {
while (nextToken != null && currentToken?.end == nextToken.start) {
bool punctPickup = true;
while (nextToken != null &&
currentToken?.end == nextToken.start &&
punctPickup) {
i++;
currentToken = nextToken;
nextToken = i < tokens.length - 1 ? tokens[i + 1] : null;
punctPickup = nextToken?.pos == 'PUNCT';
}
}