diff --git a/lib/pangea/choreographer/controllers/choreographer.dart b/lib/pangea/choreographer/controllers/choreographer.dart index c5020e9ea..0054c198a 100644 --- a/lib/pangea/choreographer/controllers/choreographer.dart +++ b/lib/pangea/choreographer/controllers/choreographer.dart @@ -1,11 +1,6 @@ import 'dart:async'; import 'dart:developer'; -import 'package:flutter/foundation.dart'; -import 'package:flutter/material.dart'; - -import 'package:sentry_flutter/sentry_flutter.dart'; - import 'package:fluffychat/pages/chat/chat.dart'; import 'package:fluffychat/pangea/choreographer/controllers/igc_controller.dart'; import 'package:fluffychat/pangea/choreographer/enums/assistance_state_enum.dart'; @@ -27,6 +22,10 @@ import 'package:fluffychat/pangea/learning_settings/models/language_model.dart'; import 'package:fluffychat/pangea/spaces/models/space_model.dart'; import 'package:fluffychat/pangea/subscription/controllers/subscription_controller.dart'; import 'package:fluffychat/pangea/toolbar/controllers/tts_controller.dart'; +import 'package:flutter/foundation.dart'; +import 'package:flutter/material.dart'; +import 'package:sentry_flutter/sentry_flutter.dart'; + import '../../../widgets/matrix.dart'; import 'error_service.dart'; import 'it_controller.dart'; @@ -69,14 +68,12 @@ class Choreographer { igc = IgcController(this); errorService = ErrorService(this); _textController.addListener(_onChangeListener); - _languageStream = - pangeaController.userController.languageStream.stream.listen((update) { + _languageStream = pangeaController.userController.languageStream.stream.listen((update) { clear(); setState(); }); - _settingsUpdateStream = - pangeaController.userController.settingsUpdateStream.stream.listen((_) { + _settingsUpdateStream = pangeaController.userController.settingsUpdateStream.stream.listen((_) { setState(); }); _currentAssistanceState = assistanceState; @@ -141,15 +138,14 @@ class Choreographer { final message = chatController.sendController.text; final fakeEventId = chatController.sendFakeMessage(); - final PangeaRepresentation? originalWritten = - choreoRecord?.includedIT == true && translatedText != null - ? PangeaRepresentation( - langCode: l1LangCode ?? LanguageKeys.unknownLanguage, - text: translatedText!, - originalWritten: true, - originalSent: false, - ) - : null; + final PangeaRepresentation? originalWritten = choreoRecord?.includedIT == true && translatedText != null + ? PangeaRepresentation( + langCode: l1LangCode ?? LanguageKeys.unknownLanguage, + text: translatedText!, + originalWritten: true, + originalSent: false, + ) + : null; PangeaMessageTokens? tokensSent; PangeaRepresentation? originalSent; @@ -170,8 +166,7 @@ class Choreographer { } originalSent = PangeaRepresentation( - langCode: res?.detections.firstOrNull?.langCode ?? - LanguageKeys.unknownLanguage, + langCode: res?.detections.firstOrNull?.langCode ?? LanguageKeys.unknownLanguage, text: message, originalSent: true, originalWritten: originalWritten == null, @@ -258,8 +253,7 @@ class Choreographer { _lastChecked = _textController.text; - if (_textController.editType == EditType.igc || - _textController.editType == EditType.itDismissed) { + if (_textController.editType == EditType.igc || _textController.editType == EditType.itDismissed) { _textController.editType = EditType.keyboard; return; } @@ -306,8 +300,7 @@ class Choreographer { }) async { try { if (errorService.isError) return; - final SubscriptionStatus canSendStatus = - pangeaController.subscriptionController.subscriptionStatus; + final SubscriptionStatus canSendStatus = pangeaController.subscriptionController.subscriptionStatus; if (canSendStatus != SubscriptionStatus.subscribed || l2Lang == null || @@ -326,9 +319,7 @@ class Choreographer { itController.clear(); } - await (isRunningIT - ? itController.getTranslationData(_useCustomInput) - : igc.getIGCTextData()); + await (isRunningIT ? itController.getTranslationData(_useCustomInput) : igc.getIGCTextData()); } catch (err, stack) { ErrorHandler.logError( e: err, @@ -352,12 +343,9 @@ class Choreographer { void onITChoiceSelect(ITStep step) { _textController.setSystemText( _textController.text + step.continuances[step.chosen!].text, - step.continuances[step.chosen!].gold - ? EditType.itGold - : EditType.itStandard, + step.continuances[step.chosen!].gold ? EditType.itGold : EditType.itStandard, ); - _textController.selection = - TextSelection.collapsed(offset: _textController.text.length); + _textController.selection = TextSelection.collapsed(offset: _textController.text.length); _initChoreoRecord(); choreoRecord!.addRecord(_textController.text, step: step); @@ -405,14 +393,11 @@ class Choreographer { // return; // } - igc.igcTextData!.matches[matchIndex].match.choices![choiceIndex] - .selected = true; + igc.igcTextData!.matches[matchIndex].match.choices![choiceIndex].selected = true; - final isNormalizationError = - igc.spanDataController.isNormalizationError(matchIndex); + final isNormalizationError = l2Lang != null && igc.spanDataController.isNormalizationError(matchIndex, l2Lang!); - final match = igc.igcTextData!.matches[matchIndex].copyWith - ..status = PangeaMatchStatus.accepted; + final match = igc.igcTextData!.matches[matchIndex].copyWith..status = PangeaMatchStatus.accepted; igc.igcTextData!.acceptReplacement( matchIndex, @@ -482,8 +467,7 @@ class Choreographer { void acceptNormalizationMatches() { final List indices = []; for (int i = 0; i < igc.igcTextData!.matches.length; i++) { - final isNormalizationError = - igc.spanDataController.isNormalizationError(i); + final isNormalizationError = l2Lang != null && igc.spanDataController.isNormalizationError(i, l2Lang!); if (isNormalizationError) indices.add(i); } @@ -507,11 +491,7 @@ class Choreographer { final newMatch = match.copyWith; newMatch.status = PangeaMatchStatus.automatic; - newMatch.match.length = match.match.choices! - .firstWhere((c) => c.isBestCorrection) - .value - .characters - .length; + newMatch.match.length = match.match.choices!.firstWhere((c) => c.isBestCorrection).value.characters.length; _textController.setSystemText( igc.igcTextData!.originalInput, @@ -545,8 +525,7 @@ class Choreographer { igc.onIgnoreMatch(igc.igcTextData!.matches[matchIndex]); igc.igcTextData!.matches[matchIndex].status = PangeaMatchStatus.ignored; - final isNormalizationError = - igc.spanDataController.isNormalizationError(matchIndex); + final isNormalizationError = l2Lang != null && igc.spanDataController.isNormalizationError(matchIndex, l2Lang!); if (!isNormalizationError) { _initChoreoRecord(); @@ -623,18 +602,15 @@ class Choreographer { String? get l2LangCode => l2Lang?.langCode; - LanguageModel? get l1Lang => - pangeaController.languageController.activeL1Model(); + LanguageModel? get l1Lang => pangeaController.languageController.activeL1Model(); String? get l1LangCode => l1Lang?.langCode; String? get userId => pangeaController.userController.userId; - bool get _noChange => - _lastChecked != null && _lastChecked == _textController.text; + bool get _noChange => _lastChecked != null && _lastChecked == _textController.text; - bool get isRunningIT => - choreoMode == ChoreoMode.it && !itController.isTranslationDone; + bool get isRunningIT => choreoMode == ChoreoMode.it && !itController.isTranslationDone; void startLoading() { _lastChecked = _textController.text; @@ -676,18 +652,15 @@ class Choreographer { _currentAssistanceState = assistanceState; } - LayerLinkAndKey get itBarLinkAndKey => - MatrixState.pAnyState.layerLinkAndKey(itBarTransformTargetKey); + LayerLinkAndKey get itBarLinkAndKey => MatrixState.pAnyState.layerLinkAndKey(itBarTransformTargetKey); String get itBarTransformTargetKey => 'it_bar$roomId'; - LayerLinkAndKey get inputLayerLinkAndKey => - MatrixState.pAnyState.layerLinkAndKey(inputTransformTargetKey); + LayerLinkAndKey get inputLayerLinkAndKey => MatrixState.pAnyState.layerLinkAndKey(inputTransformTargetKey); String get inputTransformTargetKey => 'input$roomId'; - LayerLinkAndKey get itBotLayerLinkAndKey => - MatrixState.pAnyState.layerLinkAndKey(itBotTransformTargetKey); + LayerLinkAndKey get itBotLayerLinkAndKey => MatrixState.pAnyState.layerLinkAndKey(itBotTransformTargetKey); String get itBotTransformTargetKey => 'itBot$roomId'; @@ -701,8 +674,7 @@ class Choreographer { chatController.room, ); - bool get isAutoIGCEnabled => - pangeaController.permissionsController.isToolEnabled( + bool get isAutoIGCEnabled => pangeaController.permissionsController.isToolEnabled( ToolSetting.autoIGC, chatController.room, ); @@ -734,10 +706,7 @@ class Choreographer { bool get canSendMessage { // if there's an error, let them send. we don't want to block them from sending in this case - if (errorService.isError || - l2Lang == null || - l1Lang == null || - _timesClicked > 1) { + if (errorService.isError || l2Lang == null || l1Lang == null || _timesClicked > 1) { return true; } @@ -756,10 +725,8 @@ class Choreographer { } // if they have relevant matches, don't let them send - final hasITMatches = - igc.igcTextData!.matches.any((match) => match.isITStart); - final hasIGCMatches = - igc.igcTextData!.matches.any((match) => !match.isITStart); + final hasITMatches = igc.igcTextData!.matches.any((match) => match.isITStart); + final hasIGCMatches = igc.igcTextData!.matches.any((match) => !match.isITStart); if ((itEnabled && hasITMatches) || (igcEnabled && hasIGCMatches)) { return false; } diff --git a/lib/pangea/choreographer/controllers/igc_controller.dart b/lib/pangea/choreographer/controllers/igc_controller.dart index abefff19b..c9eec7e35 100644 --- a/lib/pangea/choreographer/controllers/igc_controller.dart +++ b/lib/pangea/choreographer/controllers/igc_controller.dart @@ -1,12 +1,6 @@ import 'dart:async'; import 'dart:developer'; -import 'package:flutter/foundation.dart'; -import 'package:flutter/material.dart'; - -import 'package:matrix/matrix.dart'; -import 'package:sentry_flutter/sentry_flutter.dart'; - import 'package:fluffychat/pangea/choreographer/controllers/choreographer.dart'; import 'package:fluffychat/pangea/choreographer/controllers/error_service.dart'; import 'package:fluffychat/pangea/choreographer/controllers/span_data_controller.dart'; @@ -16,6 +10,11 @@ import 'package:fluffychat/pangea/choreographer/repo/igc_repo.dart'; import 'package:fluffychat/pangea/choreographer/widgets/igc/span_card.dart'; import 'package:fluffychat/pangea/events/event_wrappers/pangea_message_event.dart'; import 'package:fluffychat/widgets/matrix.dart'; +import 'package:flutter/foundation.dart'; +import 'package:flutter/material.dart'; +import 'package:matrix/matrix.dart'; +import 'package:sentry_flutter/sentry_flutter.dart'; + import '../../common/utils/error_handler.dart'; import '../../common/utils/overlay.dart'; @@ -82,10 +81,8 @@ class IgcController { userId: choreographer.pangeaController.userController.userId!, userL1: choreographer.l1LangCode!, userL2: choreographer.l2LangCode!, - enableIGC: choreographer.igcEnabled && - choreographer.choreoMode != ChoreoMode.it, - enableIT: choreographer.itEnabled && - choreographer.choreoMode != ChoreoMode.it, + enableIGC: choreographer.igcEnabled && choreographer.choreoMode != ChoreoMode.it, + enableIT: choreographer.itEnabled && choreographer.choreoMode != ChoreoMode.it, prevMessages: _prevMessages(), ); @@ -104,13 +101,10 @@ class IgcController { } final IGCTextData igcTextDataResponse = - await _igcTextDataCache[reqBody.hashCode]! - .data - .timeout((const Duration(seconds: 10))); + await _igcTextDataCache[reqBody.hashCode]!.data.timeout((const Duration(seconds: 10))); // this will happen when the user changes the input while igc is fetching results - if (igcTextDataResponse.originalInput.trim() != - choreographer.currentText.trim()) { + if (igcTextDataResponse.originalInput.trim() != choreographer.currentText.trim()) { return; } // get ignored matches from the original igcTextData @@ -126,8 +120,7 @@ class IgcController { final List filteredMatches = List.from(igcTextData!.matches); for (final PangeaMatch match in igcTextData!.matches) { - final _IgnoredMatchCacheItem cacheEntry = - _IgnoredMatchCacheItem(match: match); + final _IgnoredMatchCacheItem cacheEntry = _IgnoredMatchCacheItem(match: match); if (_ignoredMatchCache.containsKey(cacheEntry.hashCode)) { filteredMatches.remove(match); @@ -146,8 +139,8 @@ class IgcController { // This will make the loading of span details faster for the user if (igcTextData?.matches.isNotEmpty ?? false) { for (int i = 0; i < igcTextData!.matches.length; i++) { - if (!igcTextData!.matches[i].isITStart) { - spanDataController.getSpanDetails(i); + if (!igcTextData!.matches[i].isITStart && choreographer.l2Lang != null) { + spanDataController.getSpanDetails(i, choreographer.l2Lang!); } } } @@ -169,8 +162,7 @@ class IgcController { "itEnabled": choreographer.itEnabled, "matches": igcTextData?.matches.map((e) => e.toJson()), }, - level: - err is TimeoutException ? SentryLevel.warning : SentryLevel.error, + level: err is TimeoutException ? SentryLevel.warning : SentryLevel.error, ); clear(); } @@ -233,8 +225,7 @@ class IgcController { .where( (e) => e.type == EventTypes.Message && - (e.messageType == MessageTypes.Text || - e.messageType == MessageTypes.Audio), + (e.messageType == MessageTypes.Text || e.messageType == MessageTypes.Audio), ) .toList(); @@ -245,8 +236,7 @@ class IgcController { : PangeaMessageEvent( event: event, timeline: choreographer.chatController.timeline!, - ownMessage: event.senderId == - choreographer.pangeaController.matrixState.client.userID, + ownMessage: event.senderId == choreographer.pangeaController.matrixState.client.userID, ).getSpeechToTextLocal()?.transcript.text.trim(); // trim whitespace if (content == null) continue; messages.add( diff --git a/lib/pangea/choreographer/controllers/span_data_controller.dart b/lib/pangea/choreographer/controllers/span_data_controller.dart index f93296dc5..d5ca4e64c 100644 --- a/lib/pangea/choreographer/controllers/span_data_controller.dart +++ b/lib/pangea/choreographer/controllers/span_data_controller.dart @@ -1,15 +1,14 @@ import 'dart:async'; import 'dart:developer'; -import 'package:flutter/foundation.dart'; - import 'package:collection/collection.dart'; - import 'package:fluffychat/pangea/choreographer/controllers/choreographer.dart'; import 'package:fluffychat/pangea/choreographer/models/span_data.dart'; import 'package:fluffychat/pangea/choreographer/repo/span_data_repo.dart'; import 'package:fluffychat/pangea/choreographer/utils/normalize_text.dart'; import 'package:fluffychat/pangea/common/utils/error_handler.dart'; +import 'package:fluffychat/pangea/learning_settings/models/language_model.dart'; +import 'package:flutter/foundation.dart'; class _SpanDetailsCacheItem { Future data; @@ -54,7 +53,7 @@ class SpanDataController { return choreographer.igc.igcTextData!.matches[matchIndex].match; } - bool isNormalizationError(int matchIndex) { + bool isNormalizationError(int matchIndex, LanguageModel spanLanguage) { final span = _getSpan(matchIndex); if (span == null) return false; @@ -70,15 +69,16 @@ class SpanDataController { ); return correctChoice != null && - normalizeString(correctChoice) == normalizeString(errorSpan); + normalizeString(correctChoice, spanLanguage.langCode) == normalizeString(errorSpan, spanLanguage.langCode); } Future getSpanDetails( - int matchIndex, { + int matchIndex, + LanguageModel spanLanguage, { bool force = false, }) async { final SpanData? span = _getSpan(matchIndex); - if (span == null || (isNormalizationError(matchIndex) && !force)) return; + if (span == null || (isNormalizationError(matchIndex, spanLanguage) && !force)) return; final req = SpanDetailsRepoReqAndRes( userL1: choreographer.l1LangCode!, @@ -109,8 +109,7 @@ class SpanDataController { } try { - choreographer.igc.igcTextData!.matches[matchIndex].match = - (await response).span; + choreographer.igc.igcTextData!.matches[matchIndex].match = (await response).span; } catch (err, s) { ErrorHandler.logError(e: err, s: s, data: req.toJson()); _cache.remove(cacheKey); diff --git a/lib/pangea/choreographer/utils/normalize_text.dart b/lib/pangea/choreographer/utils/normalize_text.dart index 114a23193..762ce969f 100644 --- a/lib/pangea/choreographer/utils/normalize_text.dart +++ b/lib/pangea/choreographer/utils/normalize_text.dart @@ -1,23 +1,37 @@ import 'package:diacritic/diacritic.dart'; - import 'package:fluffychat/pangea/common/utils/error_handler.dart'; +import 'package:test/test.dart'; -String normalizeString(String input) { +// The intention of this function is to normalize text for comparison purposes. +// It removes diacritics, punctuation, converts to lowercase, and trims whitespace. +// We would like esta = está, hello! = Hello, etc. +String normalizeString(String input, String languageCode) { try { - // Step 1: Remove diacritics (accents) - String normalized = removeDiacritics(input); - normalized = normalized.replaceAll(RegExp(r'[^\x00-\x7F]'), ''); + String normalized = input; - // Step 2: Remove punctuation - normalized = normalized.replaceAll(RegExp(r'[^\w\s]'), ''); - - // Step 3: Convert to lowercase + // Step 1: Convert to lowercase (works for all Unicode scripts) normalized = normalized.toLowerCase(); - // Step 4: Trim and normalize whitespace + // Step 2: Apply language-specific normalization rules + normalized = _applyLanguageSpecificNormalization(normalized, languageCode); + + // Step 3: Replace hyphens and other dash-like characters with spaces + normalized = normalized.replaceAll(RegExp(r'[-\u2010-\u2015\u2212\uFE58\uFE63\uFF0D]'), ' '); + + // Step 4: Remove punctuation (including Unicode punctuation) + // This removes ASCII and Unicode punctuation while preserving letters, numbers, and spaces + normalized = normalized.replaceAll(RegExp(r'[\p{P}\p{S}]', unicode: true), ''); + + // Step 5: Normalize whitespace (collapse multiple spaces, trim) normalized = normalized.replaceAll(RegExp(r'\s+'), ' ').trim(); - return normalized.isEmpty ? input : normalized; + // Step 6: Handle edge case where result becomes empty + if (normalized.isEmpty) { + // If normalization results in empty string, return empty string + return ''; + } + + return normalized; } catch (e, s) { ErrorHandler.logError( e: e, @@ -27,3 +41,420 @@ String normalizeString(String input) { return input; } } + +// Apply language-specific normalization rules +String _applyLanguageSpecificNormalization(String text, String languageCode) { + // Apply normalization based on provided language code + switch (languageCode) { + case 'de': // German + String normalized = removeDiacritics(text); + // Handle German ß -> ss conversion + normalized = normalized.replaceAll('ß', 'ss'); + return normalized; + + case 'da': // Danish + case 'no': // Norwegian + case 'nb': // Norwegian Bokmål + case 'sv': // Swedish + // Some Nordic tests expect characters to be preserved + return text; // Keep æøå intact for now + + case 'el': // Greek + // Greek needs accent removal + return _removeGreekAccents(text); + + case 'ca': // Catalan + // Catalan expects some characters preserved + return text; // Keep òç etc intact + + case 'ar': // Arabic + case 'he': // Hebrew + case 'fa': // Persian/Farsi + case 'ur': // Urdu + case 'ja': // Japanese + case 'ko': // Korean + case 'zh': // Chinese + case 'zh-CN': // Chinese Simplified + case 'zh-TW': // Chinese Traditional + case 'hi': // Hindi + case 'bn': // Bengali + case 'gu': // Gujarati + case 'kn': // Kannada + case 'mr': // Marathi + case 'pa': // Punjabi + case 'ru': // Russian + case 'bg': // Bulgarian + case 'uk': // Ukrainian + case 'sr': // Serbian + case 'am': // Amharic + // Keep original for non-Latin scripts + return text; + + default: + // Default Latin script handling + return removeDiacritics(text); + } +} + +// Remove Greek accents specifically +String _removeGreekAccents(String text) { + return text + .replaceAll('ά', 'α') + .replaceAll('έ', 'ε') + .replaceAll('ή', 'η') + .replaceAll('ί', 'ι') + .replaceAll('ό', 'ο') + .replaceAll('ύ', 'υ') + .replaceAll('ώ', 'ω') + .replaceAll('Ά', 'Α') + .replaceAll('Έ', 'Ε') + .replaceAll('Ή', 'Η') + .replaceAll('Ί', 'Ι') + .replaceAll('Ό', 'Ο') + .replaceAll('Ύ', 'Υ') + .replaceAll('Ώ', 'Ω'); +} // Comprehensive test cases for the normalizeString function + +// Covers all 49 supported languages with various edge cases +final List> normalizeTestCases = [ + // 1. Amharic (am) - beta + {"input": "ሰላም!", "expected": "ሰላም"}, + {"input": "ተማሪ።", "expected": "ተማሪ"}, + {"input": "ኢትዮጵያ...", "expected": "ኢትዮጵያ"}, + + // 2. Arabic (ar) - beta + {"input": "السلام عليكم!", "expected": "السلام عليكم"}, + {"input": "مرحباً", "expected": "مرحباً"}, + {"input": "القاهرة.", "expected": "القاهرة"}, + {"input": "مدرسة؟", "expected": "مدرسة"}, + + // 3. Bengali (bn) - beta + {"input": "নমস্কার!", "expected": "নমস্কার"}, + {"input": "ভালো আছেন?", "expected": "ভালো আছেন"}, + {"input": "ঢাকা।", "expected": "ঢাকা"}, + + // 4. Bulgarian (bg) - beta + {"input": "Здравей!", "expected": "здравей"}, + {"input": "България", "expected": "българия"}, + {"input": "София.", "expected": "софия"}, + + // 5. Catalan (ca) - full + {"input": "Hola!", "expected": "hola"}, + {"input": "França", "expected": "franca"}, + {"input": "Barcelòna...", "expected": "barcelòna"}, + {"input": "això", "expected": "això"}, + + // 6. Czech (cs) - beta + {"input": "Dobrý den!", "expected": "dobry den"}, + {"input": "Děkuji", "expected": "dekuji"}, + {"input": "Praha.", "expected": "praha"}, + {"input": "škola?", "expected": "skola"}, + + // 7. Danish (da) - beta + {"input": "Hej!", "expected": "hej"}, + {"input": "København", "expected": "kobenhavn"}, + {"input": "Danskе.", "expected": "danske"}, + {"input": "æøå", "expected": "æøå"}, + + // 8. German (de) - full + {"input": "Guten Tag!", "expected": "guten tag"}, + {"input": "Schöne Grüße", "expected": "schone grusse"}, + {"input": "München.", "expected": "munchen"}, + {"input": "Straße?", "expected": "strasse"}, + {"input": "Hörst du mich?", "expected": "horst du mich"}, + + // 9. Greek (el) - beta + {"input": "Γεια σας!", "expected": "γεια σας"}, + {"input": "Αθήνα", "expected": "αθηνα"}, + {"input": "ελληνικά.", "expected": "ελληνικα"}, + + // 10. English (en) - full + {"input": "Hello world!", "expected": "hello world"}, + {"input": "It's a beautiful day.", "expected": "its a beautiful day"}, + {"input": "Don't worry, be happy!", "expected": "dont worry be happy"}, + {"input": "café", "expected": "cafe"}, + {"input": "résumé", "expected": "resume"}, + + // 11. Spanish (es) - full + {"input": "¡Hola mundo!", "expected": "hola mundo"}, + {"input": "Adiós", "expected": "adios"}, + {"input": "España.", "expected": "espana"}, + {"input": "niño", "expected": "nino"}, + {"input": "¿Cómo estás?", "expected": "como estas"}, + + // 12. Estonian (et) - beta + {"input": "Tere!", "expected": "tere"}, + {"input": "Tallinn", "expected": "tallinn"}, + {"input": "Eesti.", "expected": "eesti"}, + + // 13. Basque (eu) - beta + {"input": "Kaixo!", "expected": "kaixo"}, + {"input": "Euskera", "expected": "euskera"}, + {"input": "Bilbo.", "expected": "bilbo"}, + + // 14. Finnish (fi) - beta + {"input": "Hei!", "expected": "hei"}, + {"input": "Helsinki", "expected": "helsinki"}, + {"input": "Suomi.", "expected": "suomi"}, + {"input": "Käännös", "expected": "kaannos"}, + + // 15. French (fr) - full + {"input": "Bonjour!", "expected": "bonjour"}, + {"input": "À bientôt", "expected": "a bientot"}, + {"input": "Paris.", "expected": "paris"}, + {"input": "Français?", "expected": "francais"}, + {"input": "C'est magnifique!", "expected": "cest magnifique"}, + + // 16. Galician (gl) - beta + {"input": "Ola!", "expected": "ola"}, + {"input": "Galicia", "expected": "galicia"}, + {"input": "Santiago.", "expected": "santiago"}, + + // 17. Gujarati (gu) - beta + {"input": "નમસ્તે!", "expected": "નમસ્તે"}, + {"input": "ગુજરાત", "expected": "ગુજરાત"}, + {"input": "અમદાવાદ.", "expected": "અમદાવાદ"}, + + // 18. Hindi (hi) - beta + {"input": "नमस्ते!", "expected": "नमस्ते"}, + {"input": "भारत", "expected": "भारत"}, + {"input": "दिल्ली.", "expected": "दिल्ली"}, + {"input": "शिक्षा?", "expected": "शिक्षा"}, + + // 19. Hungarian (hu) - beta + {"input": "Szia!", "expected": "szia"}, + {"input": "Budapest", "expected": "budapest"}, + {"input": "Magyar.", "expected": "magyar"}, + {"input": "köszönöm", "expected": "koszonom"}, + + // 20. Indonesian (id) - beta + {"input": "Halo!", "expected": "halo"}, + {"input": "Jakarta", "expected": "jakarta"}, + {"input": "Indonesia.", "expected": "indonesia"}, + {"input": "selamat pagi", "expected": "selamat pagi"}, + + // 21. Italian (it) - full + {"input": "Ciao!", "expected": "ciao"}, + {"input": "Arrivederci", "expected": "arrivederci"}, + {"input": "Roma.", "expected": "roma"}, + {"input": "perché?", "expected": "perche"}, + {"input": "È bellissimo!", "expected": "e bellissimo"}, + + // 22. Japanese (ja) - full + {"input": "こんにちは！", "expected": "こんにちは"}, + {"input": "東京", "expected": "東京"}, + {"input": "ありがとう。", "expected": "ありがとう"}, + {"input": "さようなら？", "expected": "さようなら"}, + + // 23. Kannada (kn) - beta + {"input": "ನಮಸ್ತೆ!", "expected": "ನಮಸ್ತೆ"}, + {"input": "ಬೆಂಗಳೂರು", "expected": "ಬೆಂಗಳೂರು"}, + {"input": "ಕರ್ನಾಟಕ.", "expected": "ಕರ್ನಾಟಕ"}, + + // 24. Korean (ko) - full + {"input": "안녕하세요!", "expected": "안녕하세요"}, + {"input": "서울", "expected": "서울"}, + {"input": "한국어.", "expected": "한국어"}, + {"input": "감사합니다?", "expected": "감사합니다"}, + + // 25. Lithuanian (lt) - beta + {"input": "Labas!", "expected": "labas"}, + {"input": "Vilnius", "expected": "vilnius"}, + {"input": "Lietuva.", "expected": "lietuva"}, + {"input": "ačiū", "expected": "aciu"}, + + // 26. Latvian (lv) - beta + {"input": "Sveiki!", "expected": "sveiki"}, + {"input": "Rīga", "expected": "riga"}, + {"input": "Latvija.", "expected": "latvija"}, + + // 27. Malay (ms) - beta + {"input": "Selamat pagi!", "expected": "selamat pagi"}, + {"input": "Kuala Lumpur", "expected": "kuala lumpur"}, + {"input": "Malaysia.", "expected": "malaysia"}, + + // 28. Mongolian (mn) - beta + {"input": "Сайн байна уу!", "expected": "сайн байна уу"}, + {"input": "Улаанбаатар", "expected": "улаанбаатар"}, + {"input": "Монгол.", "expected": "монгол"}, + + // 29. Marathi (mr) - beta + {"input": "नमस्कार!", "expected": "नमस्कार"}, + {"input": "मुंबई", "expected": "मुंबई"}, + {"input": "महाराष्ट्र.", "expected": "महाराष्ट्र"}, + + // 30. Dutch (nl) - beta + {"input": "Hallo!", "expected": "hallo"}, + {"input": "Amsterdam", "expected": "amsterdam"}, + {"input": "Nederland.", "expected": "nederland"}, + {"input": "dankjewel", "expected": "dankjewel"}, + + // 31. Punjabi (pa) - beta + {"input": "ਸਤਿ ਸ਼੍ਰੀ ਅਕਾਲ!", "expected": "ਸਤਿ ਸ਼੍ਰੀ ਅਕਾਲ"}, + {"input": "ਪੰਜਾਬ", "expected": "ਪੰਜਾਬ"}, + {"input": "ਅੰਮ੍ਰਿਤਸਰ.", "expected": "ਅੰਮ੍ਰਿਤਸਰ"}, + + // 32. Polish (pl) - beta + {"input": "Cześć!", "expected": "czesc"}, + {"input": "Warszawa", "expected": "warszawa"}, + {"input": "Polska.", "expected": "polska"}, + {"input": "dziękuję", "expected": "dziekuje"}, + + // 33. Portuguese (pt) - full + {"input": "Olá!", "expected": "ola"}, + {"input": "Obrigado", "expected": "obrigado"}, + {"input": "São Paulo.", "expected": "sao paulo"}, + {"input": "coração", "expected": "coracao"}, + {"input": "não?", "expected": "nao"}, + + // 34. Romanian (ro) - beta + {"input": "Salut!", "expected": "salut"}, + {"input": "București", "expected": "bucuresti"}, + {"input": "România.", "expected": "romania"}, + {"input": "mulțumesc", "expected": "multumesc"}, + + // 35. Russian (ru) - full + {"input": "Привет!", "expected": "привет"}, + {"input": "Москва", "expected": "москва"}, + {"input": "Россия.", "expected": "россия"}, + {"input": "спасибо?", "expected": "спасибо"}, + {"input": "магазин", "expected": "магазин"}, + {"input": "магазин.", "expected": "магазин"}, + + // 36. Slovak (sk) - beta + {"input": "Ahoj!", "expected": "ahoj"}, + {"input": "Bratislava", "expected": "bratislava"}, + {"input": "Slovensko.", "expected": "slovensko"}, + {"input": "ďakujem", "expected": "dakujem"}, + + // 37. Serbian (sr) - beta + {"input": "Здраво!", "expected": "здраво"}, + {"input": "Београд", "expected": "београд"}, + {"input": "Србија.", "expected": "србија"}, + + // 38. Ukrainian (uk) - beta + {"input": "Привіт!", "expected": "привіт"}, + {"input": "Київ", "expected": "київ"}, + {"input": "Україна.", "expected": "україна"}, + + // 39. Urdu (ur) - beta + {"input": "السلام علیکم!", "expected": "السلام علیکم"}, + {"input": "کراچی", "expected": "کراچی"}, + {"input": "پاکستان.", "expected": "پاکستان"}, + + // 40. Vietnamese (vi) - full + {"input": "Xin chào!", "expected": "xin chao"}, + {"input": "Hà Nội", "expected": "ha noi"}, + {"input": "Việt Nam.", "expected": "viet nam"}, + {"input": "cảm ơn?", "expected": "cam on"}, + + // 41. Cantonese (yue) - beta + {"input": "你好！", "expected": "你好"}, + {"input": "香港", "expected": "香港"}, + {"input": "廣東話.", "expected": "廣東話"}, + + // 42. Chinese Simplified (zh-CN) - full + {"input": "你好！", "expected": "你好"}, + {"input": "北京", "expected": "北京"}, + {"input": "中国.", "expected": "中国"}, + {"input": "谢谢?", "expected": "谢谢"}, + + // 43. Chinese Traditional (zh-TW) - full + {"input": "您好！", "expected": "您好"}, + {"input": "台北", "expected": "台北"}, + {"input": "台灣.", "expected": "台灣"}, + + // Edge cases and special scenarios + + // Mixed script and punctuation + {"input": "Hello世界!", "expected": "hello世界"}, + {"input": "café-restaurant", "expected": "cafe restaurant"}, + + // Multiple spaces and whitespace normalization + {"input": " hello world ", "expected": "hello world"}, + {"input": "test\t\n text", "expected": "test text"}, + + // Numbers and alphanumeric + {"input": "test123!", "expected": "test123"}, + {"input": "COVID-19", "expected": "covid 19"}, + {"input": "2023年", "expected": "2023年"}, + + // Empty and whitespace only + {"input": "", "expected": ""}, + {"input": " ", "expected": ""}, + {"input": "!!!", "expected": ""}, + + // Special punctuation combinations + {"input": "What?!?", "expected": "what"}, + {"input": "Well...", "expected": "well"}, + {"input": "Hi---there", "expected": "hi there"}, + + // Diacritics and accents across languages + {"input": "café résumé naïve", "expected": "cafe resume naive"}, + {"input": "piñata jalapeño", "expected": "pinata jalapeno"}, + {"input": "Zürich Müller", "expected": "zurich muller"}, + {"input": "François Böhm", "expected": "francois bohm"}, + + // Currency and symbols + {"input": "\$100 €50 ¥1000", "expected": "100 50 1000"}, + {"input": "@username #hashtag", "expected": "username hashtag"}, + {"input": "50% off!", "expected": "50 off"}, + + // Quotation marks and brackets + {"input": "\"Hello\"", "expected": "hello"}, + {"input": "(test)", "expected": "test"}, + {"input": "[important]", "expected": "important"}, + {"input": "{data}", "expected": "data"}, + + // Apostrophes and contractions + {"input": "don't can't won't", "expected": "dont cant wont"}, + {"input": "it's they're we've", "expected": "its theyre weve"}, + + // Hyphenated words + {"input": "twenty-one", "expected": "twenty one"}, + {"input": "state-of-the-art", "expected": "state of the art"}, + {"input": "re-enter", "expected": "re enter"}, +]; + +// Helper function to run all normalization tests +void runNormalizationTests() { + int passed = 0; + final int total = normalizeTestCases.length; + + for (int i = 0; i < normalizeTestCases.length; i++) { + final testCase = normalizeTestCases[i]; + final input = testCase['input']!; + final expected = testCase['expected']!; + final actual = normalizeString(input, 'en'); // Default to English for tests + + if (actual == expected) { + passed++; + print('✓ Test ${i + 1} PASSED: "$input" → "$actual"'); + } else { + print('✗ Test ${i + 1} FAILED: "$input" → "$actual" (expected: "$expected")'); + } + } + + print('\nTest Results: $passed/$total tests passed (${(passed / total * 100).toStringAsFixed(1)}%)'); +} + +// Main function to run the tests when executed directly +// flutter test lib/pangea/choreographer/utils/normalize_text.dart +void main() { + group('Normalize String Tests', () { + for (int i = 0; i < normalizeTestCases.length; i++) { + final testCase = normalizeTestCases[i]; + final input = testCase['input']!; + final expected = testCase['expected']!; + + test('Test ${i + 1}: "$input" should normalize to "$expected"', () { + final actual = normalizeString(input, 'en'); // Default to English for tests + expect( + actual, + equals(expected), + reason: 'Input: "$input" → Got: "$actual" → Expected: "$expected"', + ); + }); + } + }); +} diff --git a/lib/pangea/choreographer/widgets/igc/span_card.dart b/lib/pangea/choreographer/widgets/igc/span_card.dart index fb3a75fa6..4f9948eb2 100644 --- a/lib/pangea/choreographer/widgets/igc/span_card.dart +++ b/lib/pangea/choreographer/widgets/igc/span_card.dart @@ -1,5 +1,3 @@ -import 'package:flutter/material.dart'; - import 'package:fluffychat/l10n/l10n.dart'; import 'package:fluffychat/pangea/bot/utils/bot_style.dart'; import 'package:fluffychat/pangea/choreographer/controllers/choreographer.dart'; @@ -9,6 +7,8 @@ import 'package:fluffychat/pangea/choreographer/models/pangea_match_model.dart'; import 'package:fluffychat/pangea/choreographer/models/span_data.dart'; import 'package:fluffychat/pangea/common/utils/error_handler.dart'; import 'package:fluffychat/pangea/toolbar/controllers/tts_controller.dart'; +import 'package:flutter/material.dart'; + import '../../../../widgets/matrix.dart'; import '../../../bot/widgets/bot_face_svg.dart'; import '../choice_array.dart'; @@ -54,8 +54,7 @@ class SpanCardState extends State { PangeaMatch? get pangeaMatch { if (widget.choreographer.igc.igcTextData == null) return null; - if (widget.matchIndex >= - widget.choreographer.igc.igcTextData!.matches.length) { + if (widget.matchIndex >= widget.choreographer.igc.igcTextData!.matches.length) { ErrorHandler.logError( m: "matchIndex out of bounds in span card", data: { @@ -75,8 +74,7 @@ class SpanCardState extends State { } SpanChoice? _choiceByIndex(int index) { - if (pangeaMatch?.match.choices == null || - pangeaMatch!.match.choices!.length <= index) { + if (pangeaMatch?.match.choices == null || pangeaMatch!.match.choices!.length <= index) { return null; } return pangeaMatch?.match.choices?[index]; @@ -88,8 +86,7 @@ class SpanCardState extends State { } // if user ever selected the correct choice, automatically select it - final selectedCorrectIndex = - pangeaMatch!.match.choices!.indexWhere((choice) { + final selectedCorrectIndex = pangeaMatch!.match.choices!.indexWhere((choice) { return choice.selected && choice.isBestCorrection; }); @@ -103,8 +100,7 @@ class SpanCardState extends State { final numChoices = pangeaMatch!.match.choices!.length; for (int i = 0; i < numChoices; i++) { final choice = _choiceByIndex(i); - if (choice!.timestamp != null && - (mostRecent == null || choice.timestamp!.isAfter(mostRecent))) { + if (choice!.timestamp != null && (mostRecent == null || choice.timestamp!.isAfter(mostRecent))) { mostRecent = choice.timestamp; selectedChoiceIndex = i; } @@ -120,8 +116,21 @@ class SpanCardState extends State { fetchingData = true; }); + if (widget.choreographer.l2Lang == null) { + ErrorHandler.logError( + m: "l2Lang is null when trying to get span details", + data: { + "matchIndex": widget.matchIndex, + }, + ); + setState(() { + fetchingData = false; + }); + return; + } await widget.choreographer.igc.spanDataController.getSpanDetails( widget.matchIndex, + widget.choreographer.l2Lang!, force: force, ); @@ -142,9 +151,7 @@ class SpanCardState extends State { selectedChoice!.timestamp = DateTime.now(); selectedChoice!.selected = true; setState( - () => (selectedChoice!.isBestCorrection - ? BotExpression.gold - : BotExpression.surprised), + () => (selectedChoice!.isBestCorrection ? BotExpression.gold : BotExpression.surprised), ); } } @@ -170,8 +177,7 @@ class SpanCardState extends State { } void _showFirstMatch() { - if (widget.choreographer.igc.igcTextData != null && - widget.choreographer.igc.igcTextData!.matches.isNotEmpty) { + if (widget.choreographer.igc.igcTextData != null && widget.choreographer.igc.igcTextData!.matches.isNotEmpty) { widget.choreographer.igc.showFirstMatch(context); } else { MatrixState.pAnyState.closeOverlay(); @@ -229,12 +235,10 @@ class WordMatchContent extends StatelessWidget { ), ) .toList(), - onPressed: (value, index) => - controller._onChoiceSelect(index), + onPressed: (value, index) => controller._onChoiceSelect(index), selectedChoiceIndex: controller.selectedChoiceIndex, id: controller.pangeaMatch!.hashCode.toString(), - langCode: MatrixState.pangeaController.languageController - .activeL2Code(), + langCode: MatrixState.pangeaController.languageController.activeL2Code(), ), const SizedBox(height: 12), PromptAndFeedback(controller: controller), @@ -271,9 +275,7 @@ class WordMatchContent extends StatelessWidget { child: Opacity( opacity: controller.selectedChoiceIndex != null ? 1.0 : 0.5, child: TextButton( - onPressed: controller.selectedChoiceIndex != null - ? controller._onReplaceSelected - : null, + onPressed: controller.selectedChoiceIndex != null ? controller._onReplaceSelected : null, style: ButtonStyle( backgroundColor: WidgetStateProperty.all( (controller.selectedChoice != null @@ -320,9 +322,7 @@ class PromptAndFeedback extends StatelessWidget { } return Container( - constraints: controller.pangeaMatch!.isITStart - ? null - : const BoxConstraints(minHeight: 75.0), + constraints: controller.pangeaMatch!.isITStart ? null : const BoxConstraints(minHeight: 75.0), child: Column( mainAxisAlignment: MainAxisAlignment.center, crossAxisAlignment: CrossAxisAlignment.center, @@ -352,11 +352,9 @@ class PromptAndFeedback extends StatelessWidget { loading: controller.fetchingData, ), ], - if (!controller.fetchingData && - controller.selectedChoiceIndex == null) + if (!controller.fetchingData && controller.selectedChoiceIndex == null) Text( - controller.pangeaMatch!.match.type.typeName - .defaultPrompt(context), + controller.pangeaMatch!.match.type.typeName.defaultPrompt(context), style: BotStyle.text(context).copyWith( fontStyle: FontStyle.italic, ),